diff options
81 files changed, 17101 insertions, 1168 deletions
@@ -337,7 +337,7 @@ examples: #### Pattern -A pattern is a string enclosed with `"` and represents a regular expression. A pattern that appears in production rules is used in lexical analysis. For more information on the syntax of regular expressions, please see [maleeni's documents](https://github.com/nihei9/maleeni/blob/main/README.md). vartan uses [maleeni](https://github.com/nihei9/maleeni) as a lexer. +A pattern is a string enclosed with `"` and represents a regular expression. A pattern that appears in production rules is used in lexical analysis. For more information on the syntax of regular expressions, please see [Regular Expression](#regular-expression). examples: @@ -731,3 +731,127 @@ eq_exprs 1:2: unexpected token: ';' (semi_colon): expected: eq 1:7: unexpected token: ';' (semi_colon): expected: int ``` + +### Regular Expression + +⚠️ vartan doesn't allow you to use some code points. See [Unavailable Code Points](#unavailable-code-points). + +#### Composites + +Concatenation and alternation allow you to combine multiple characters or multiple patterns into one pattern. + +| Pattern | Matches | +|------------|----------------| +| `abc` | `abc` | +| `abc\|def` | `abc` or `def` | + +#### Single Characters + +In addition to using ordinary characters, there are other ways to represent a single character: + +* dot expression +* bracket expressions +* code point expressions +* character property expressions +* escape sequences + +##### Dot Expression + +The dot expression matches any one chracter. + +| Pattern | Matches | +|---------|-------------------| +| `.` | any one character | + +##### Bracket Expressions + +The bracket expressions are represented by enclosing characters in `[ ]` or `[^ ]`. `[^ ]` is negation of `[ ]`. For instance, `[ab]` matches one of `a` or `b`, and `[^ab]` matches any one character except `a` and `b`. + +| Pattern | Matches | +|----------|--------------------------------------------------| +| `[abc]` | `a`, `b`, or `c` | +| `[^abc]` | any one character except `a`, `b`, and `c` | +| `[a-z]` | one in the range of `a` to `z` | +| `[a-]` | `a` or `-` | +| `[-z]` | `-` or `z` | +| `[-]` | `-` | +| `[^a-z]` | any one character except the range of `a` to `z` | +| `[a^]` | `a` or `^` | + +##### Code Point Expressions + +The code point expressions match a character that has a specified code point. The code points consists of a four or six digits hex string. + +| Pattern | Matches | +|--------------|-----------------------------| +| `\u{000A}` | U+000A (LF) | +| `\u{3042}` | U+3042 (hiragana `あ`) | +| `\u{01F63A}` | U+1F63A (grinning cat `😺`) | + +##### Character Property Expressions + +The character property expressions match a character that has a specified character property of the Unicode. Currently, vartan supports `General_Category`, `Script`, `Alphabetic`, `Lowercase`, `Uppercase`, and `White_Space`. When you omitted the equal symbol and a right-side value, vartan interprets a symbol in `\p{...}` as the `General_Category` value. + +| Pattern | Matches | +|-------------------------------|--------------------------------------------------------| +| `\p{General_Category=Letter}` | any one character whose `General_Category` is `Letter` | +| `\p{gc=Letter}` | the same as `\p{General_Category=Letter}` | +| `\p{Letter}` | the same as `\p{General_Category=Letter}` | +| `\p{l}` | the same as `\p{General_Category=Letter}` | +| `\p{Script=Latin}` | any one character whose `Script` is `Latin` | +| `\p{Alphabetic=yes}` | any one character whose `Alphabetic` is `yes` | +| `\p{Lowercase=yes}` | any one character whose `Lowercase` is `yes` | +| `\p{Uppercase=yes}` | any one character whose `Uppercase` is `yes` | +| `\p{White_Space=yes}` | any one character whose `White_Space` is `yes` | + +##### Escape Sequences + +As you escape the special character with `\`, you can write a rule that matches the special character itself. +The following escape sequences are available outside of bracket expressions. + +| Pattern | Matches | +|---------|---------| +| `\.` | `.` | +| `\?` | `?` | +| `\*` | `*` | +| `\+` | `+` | +| `\(` | `(` | +| `\)` | `)` | +| `\[` | `[` | +| `\\|` | `\|` | +| `\\` | `\` | + +The following escape sequences are available inside bracket expressions. + +| Pattern | Matches | +|---------|---------| +| `\^` | `^` | +| `\-` | `-` | +| `\]` | `]` | + +#### Repetitions + +The repetitions match a string that repeats the previous single character or group. + +| Pattern | Matches | +|---------|------------------| +| `a*` | zero or more `a` | +| `a+` | one or more `a` | +| `a?` | zero or one `a` | + +#### Grouping + +`(` and `)` groups any patterns. + +| Pattern | Matches | +|-------------|-------------------------------------------------| +| `a(bc)*d` | `ad`, `abcd`, `abcbcd`, and so on | +| `(ab\|cd)+` | `ab`, `cd`, `abcd`, `cdab`, `abcdab`, and so on | + +#### Unavailable Code Points + +Lexical specifications and source files to be analyzed cannot contain the following code points. + +When you write a pattern that implicitly contains the unavailable code points, vartan will automatically generate a pattern that doesn't contain the unavailable code points and replaces the original pattern. However, when you explicitly use the unavailable code points (like `\u{U+D800}` or `\p{General_Category=Cs}`), vartan will occur an error. + +* surrogate code points: U+D800..U+DFFF diff --git a/cmd/ucdgen/main.go b/cmd/ucdgen/main.go new file mode 100644 index 0000000..6d7d33e --- /dev/null +++ b/cmd/ucdgen/main.go @@ -0,0 +1,98 @@ +package main + +import ( + "fmt" + "net/http" + "os" + "strings" + "text/template" + + "github.com/nihei9/vartan/ucd" +) + +func main() { + err := gen() + if err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } +} + +func gen() error { + var propValAliases *ucd.PropertyValueAliases + { + resp, err := http.Get("https://www.unicode.org/Public/13.0.0/ucd/PropertyValueAliases.txt") + if err != nil { + return err + } + defer resp.Body.Close() + propValAliases, err = ucd.ParsePropertyValueAliases(resp.Body) + if err != nil { + return err + } + } + var unicodeData *ucd.UnicodeData + { + resp, err := http.Get("https://www.unicode.org/Public/13.0.0/ucd/UnicodeData.txt") + if err != nil { + return err + } + defer resp.Body.Close() + unicodeData, err = ucd.ParseUnicodeData(resp.Body, propValAliases) + if err != nil { + return err + } + } + var scripts *ucd.Scripts + { + resp, err := http.Get("https://www.unicode.org/Public/13.0.0/ucd/Scripts.txt") + if err != nil { + return err + } + defer resp.Body.Close() + scripts, err = ucd.ParseScripts(resp.Body, propValAliases) + if err != nil { + return err + } + } + var propList *ucd.PropList + { + resp, err := http.Get("https://www.unicode.org/Public/13.0.0/ucd/PropList.txt") + if err != nil { + return err + } + defer resp.Body.Close() + propList, err = ucd.ParsePropList(resp.Body) + if err != nil { + return err + } + } + tmpl, err := template.ParseFiles("../ucd/codepoint.go.tmpl") + if err != nil { + return err + } + var b strings.Builder + err = tmpl.Execute(&b, struct { + GeneratorName string + UnicodeData *ucd.UnicodeData + Scripts *ucd.Scripts + PropList *ucd.PropList + PropertyValueAliases *ucd.PropertyValueAliases + }{ + GeneratorName: "generator/main.go", + UnicodeData: unicodeData, + Scripts: scripts, + PropList: propList, + PropertyValueAliases: propValAliases, + }) + if err != nil { + return err + } + f, err := os.OpenFile("../ucd/codepoint.go", os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) + if err != nil { + return err + } + defer f.Close() + fmt.Fprint(f, b.String()) + return nil +} diff --git a/cmd/vartan-go/generate.go b/cmd/vartan-go/generate.go index 13ba5c0..7d95ed7 100644 --- a/cmd/vartan-go/generate.go +++ b/cmd/vartan-go/generate.go @@ -6,8 +6,8 @@ import ( "io" "os" - mldriver "github.com/nihei9/maleeni/driver" - "github.com/nihei9/vartan/driver" + "github.com/nihei9/vartan/driver/lexer" + "github.com/nihei9/vartan/driver/parser" spec "github.com/nihei9/vartan/spec/grammar" "github.com/spf13/cobra" ) @@ -42,7 +42,7 @@ func runGenerate(cmd *cobra.Command, args []string) error { } { - b, err := mldriver.GenLexer(cgram.LexicalSpecification.Maleeni.Spec, *generateFlags.pkgName) + b, err := lexer.GenLexer(cgram.Lexical, *generateFlags.pkgName) if err != nil { return fmt.Errorf("Failed to generate a lexer: %w", err) } @@ -62,7 +62,7 @@ func runGenerate(cmd *cobra.Command, args []string) error { } { - b, err := driver.GenParser(cgram, *generateFlags.pkgName) + b, err := parser.GenParser(cgram, *generateFlags.pkgName) if err != nil { return fmt.Errorf("Failed to generate a parser: %w", err) } @@ -82,7 +82,7 @@ func runGenerate(cmd *cobra.Command, args []string) error { } { - b, err := driver.GenSemanticAction(*generateFlags.pkgName) + b, err := parser.GenSemanticAction(*generateFlags.pkgName) if err != nil { return fmt.Errorf("Failed to generate a semantic action set: %w", err) } diff --git a/cmd/vartan/compile.go b/cmd/vartan/compile.go index 49e383d..e645366 100644 --- a/cmd/vartan/compile.go +++ b/cmd/vartan/compile.go @@ -10,6 +10,7 @@ import ( verr "github.com/nihei9/vartan/error" "github.com/nihei9/vartan/grammar" spec "github.com/nihei9/vartan/spec/grammar" + "github.com/nihei9/vartan/spec/grammar/parser" "github.com/spf13/cobra" ) @@ -78,17 +79,12 @@ func runCompile(cmd *cobra.Command, args []string) (retErr error) { } } - gram, err := readGrammar(grmPath) + gram, report, err := readGrammar(grmPath) if err != nil { return err } - cgram, report, err := grammar.Compile(gram, grammar.EnableReporting()) - if err != nil { - return err - } - - err = writeCompiledGrammarAndReport(cgram, report, *compileFlags.output) + err = writeCompiledGrammarAndReport(gram, report, *compileFlags.output) if err != nil { return fmt.Errorf("Cannot write an output files: %w", err) } @@ -113,22 +109,22 @@ func runCompile(cmd *cobra.Command, args []string) (retErr error) { return nil } -func readGrammar(path string) (grm *grammar.Grammar, retErr error) { +func readGrammar(path string) (*spec.CompiledGrammar, *spec.Report, error) { f, err := os.Open(path) if err != nil { - return nil, fmt.Errorf("Cannot open the grammar file %s: %w", path, err) + return nil, nil, fmt.Errorf("Cannot open the grammar file %s: %w", path, err) } defer f.Close() - ast, err := spec.Parse(f) + ast, err := parser.Parse(f) if err != nil { - return nil, err + return nil, nil, err } b := grammar.GrammarBuilder{ AST: ast, } - return b.Build() + return b.Build(grammar.EnableReporting()) } // writeCompiledGrammarAndReport writes a compiled grammar and a report to a files located at a specified path. diff --git a/cmd/vartan/parse.go b/cmd/vartan/parse.go index ed35a60..f21a03b 100644 --- a/cmd/vartan/parse.go +++ b/cmd/vartan/parse.go @@ -7,7 +7,7 @@ import ( "os" "strings" - "github.com/nihei9/vartan/driver" + driver "github.com/nihei9/vartan/driver/parser" spec "github.com/nihei9/vartan/spec/grammar" "github.com/nihei9/vartan/tester" "github.com/spf13/cobra" @@ -172,7 +172,7 @@ func writeSyntaxErrorMessage(b *strings.Builder, cgram *spec.CompiledGrammar, sy case tok.Invalid(): fmt.Fprintf(b, "'%v' (<invalid>)", string(tok.Lexeme())) default: - if kind := cgram.ParsingTable.Terminals[tok.TerminalID()]; kind != "" { + if kind := cgram.Syntactic.Terminals[tok.TerminalID()]; kind != "" { fmt.Fprintf(b, "'%v' (%v)", string(tok.Lexeme()), kind) } else { fmt.Fprintf(b, "'%v'", string(tok.Lexeme())) diff --git a/cmd/vartan/test.go b/cmd/vartan/test.go index 50ba8ca..9bd88ff 100644 --- a/cmd/vartan/test.go +++ b/cmd/vartan/test.go @@ -5,7 +5,6 @@ import ( "fmt" "os" - "github.com/nihei9/vartan/grammar" "github.com/nihei9/vartan/tester" "github.com/spf13/cobra" ) @@ -22,14 +21,10 @@ func init() { } func runTest(cmd *cobra.Command, args []string) error { - g, err := readGrammar(args[0]) + gram, _, err := readGrammar(args[0]) if err != nil { return fmt.Errorf("Cannot read a grammar: %w", err) } - cg, _, err := grammar.Compile(g) - if err != nil { - return fmt.Errorf("Cannot read a compiled grammar: %w", err) - } var cs []*tester.TestCaseWithMetadata { @@ -47,7 +42,7 @@ func runTest(cmd *cobra.Command, args []string) error { } t := &tester.Tester{ - Grammar: cg, + Grammar: gram, Cases: cs, } rs := t.Run() diff --git a/compressor/compressor.go b/compressor/compressor.go new file mode 100644 index 0000000..cdfeacb --- /dev/null +++ b/compressor/compressor.go @@ -0,0 +1,214 @@ +package compressor + +import ( + "encoding/binary" + "fmt" + "sort" +) + +type OriginalTable struct { + entries []int + rowCount int + colCount int +} + +func NewOriginalTable(entries []int, colCount int) (*OriginalTable, error) { + if len(entries) == 0 { + return nil, fmt.Errorf("enries is empty") + } + if colCount <= 0 { + return nil, fmt.Errorf("colCount must be >=1") + } + if len(entries)%colCount != 0 { + return nil, fmt.Errorf("entries length or column count are incorrect; entries length: %v, column count: %v", len(entries), colCount) + } + + return &OriginalTable{ + entries: entries, + rowCount: len(entries) / colCount, + colCount: colCount, + }, nil +} + +type Compressor interface { + Compress(orig *OriginalTable) error + Lookup(row, col int) (int, error) + OriginalTableSize() (int, int) +} + +var ( + _ Compressor = &UniqueEntriesTable{} + _ Compressor = &RowDisplacementTable{} +) + +type UniqueEntriesTable struct { + UniqueEntries []int + RowNums []int + OriginalRowCount int + OriginalColCount int +} + +func NewUniqueEntriesTable() *UniqueEntriesTable { + return &UniqueEntriesTable{} +} + +func (tab *UniqueEntriesTable) Lookup(row, col int) (int, error) { + if row < 0 || row >= tab.OriginalRowCount || col < 0 || col >= tab.OriginalColCount { + return 0, fmt.Errorf("indexes are out of range: [%v, %v]", row, col) + } + return tab.UniqueEntries[tab.RowNums[row]*tab.OriginalColCount+col], nil +} + +func (tab *UniqueEntriesTable) OriginalTableSize() (int, int) { + return tab.OriginalRowCount, tab.OriginalColCount +} + +func (tab *UniqueEntriesTable) Compress(orig *OriginalTable) error { + var uniqueEntries []int + rowNums := make([]int, orig.rowCount) + hash2RowNum := map[string]int{} + nextRowNum := 0 + for row := 0; row < orig.rowCount; row++ { + var rowHash string + { + buf := make([]byte, 0, orig.colCount*8) + for col := 0; col < orig.colCount; col++ { + b := make([]byte, 8) + binary.PutUvarint(b, uint64(orig.entries[row*orig.colCount+col])) + buf = append(buf, b...) + } + rowHash = string(buf) + } + rowNum, ok := hash2RowNum[rowHash] + if !ok { + rowNum = nextRowNum + nextRowNum++ + hash2RowNum[rowHash] = rowNum + start := row * orig.colCount + entry := append([]int{}, orig.entries[start:start+orig.colCount]...) + uniqueEntries = append(uniqueEntries, entry...) + } + rowNums[row] = rowNum + } + + tab.UniqueEntries = uniqueEntries + tab.RowNums = rowNums + tab.OriginalRowCount = orig.rowCount + tab.OriginalColCount = orig.colCount + + return nil +} + +const ForbiddenValue = -1 + +type RowDisplacementTable struct { + OriginalRowCount int + OriginalColCount int + EmptyValue int + Entries []int + Bounds []int + RowDisplacement []int +} + +func NewRowDisplacementTable(emptyValue int) *RowDisplacementTable { + return &RowDisplacementTable{ + EmptyValue: emptyValue, + } +} + +func (tab *RowDisplacementTable) Lookup(row int, col int) (int, error) { + if row < 0 || row >= tab.OriginalRowCount || col < 0 || col >= tab.OriginalColCount { + return tab.EmptyValue, fmt.Errorf("indexes are out of range: [%v, %v]", row, col) + } + d := tab.RowDisplacement[row] + if tab.Bounds[d+col] != row { + return tab.EmptyValue, nil + } + return tab.Entries[d+col], nil +} + +func (tab *RowDisplacementTable) OriginalTableSize() (int, int) { + return tab.OriginalRowCount, tab.OriginalColCount +} + +type rowInfo struct { + rowNum int + nonEmptyCount int + nonEmptyCol []int +} + +func (tab *RowDisplacementTable) Compress(orig *OriginalTable) error { + rowInfo := make([]rowInfo, orig.rowCount) + { + row := 0 + col := 0 + rowInfo[0].rowNum = 0 + for _, v := range orig.entries { + if col == orig.colCount { + row++ + col = 0 + rowInfo[row].rowNum = row + } + if v != tab.EmptyValue { + rowInfo[row].nonEmptyCount++ + rowInfo[row].nonEmptyCol = append(rowInfo[row].nonEmptyCol, col) + } + col++ + } + + sort.SliceStable(rowInfo, func(i int, j int) bool { + return rowInfo[i].nonEmptyCount > rowInfo[j].nonEmptyCount + }) + } + + origEntriesLen := len(orig.entries) + entries := make([]int, origEntriesLen) + bounds := make([]int, origEntriesLen) + resultBottom := orig.colCount + rowDisplacement := make([]int, orig.rowCount) + { + for i := 0; i < origEntriesLen; i++ { + entries[i] = tab.EmptyValue + bounds[i] = ForbiddenValue + } + + nextRowDisplacement := 0 + for _, rInfo := range rowInfo { + if rInfo.nonEmptyCount <= 0 { + continue + } + + for { + isOverlapped := false + for _, col := range rInfo.nonEmptyCol { + if entries[nextRowDisplacement+col] == tab.EmptyValue { + continue + } + nextRowDisplacement++ + isOverlapped = true + break + } + if isOverlapped { + continue + } + + rowDisplacement[rInfo.rowNum] = nextRowDisplacement + for _, col := range rInfo.nonEmptyCol { + entries[nextRowDisplacement+col] = orig.entries[(rInfo.rowNum*orig.colCount)+col] + bounds[nextRowDisplacement+col] = rInfo.rowNum + } + resultBottom = nextRowDisplacement + orig.colCount + nextRowDisplacement++ + break + } + } + } + + tab.OriginalRowCount = orig.rowCount + tab.OriginalColCount = orig.colCount + tab.Entries = entries[:resultBottom] + tab.Bounds = bounds[:resultBottom] + tab.RowDisplacement = rowDisplacement + + return nil +} diff --git a/compressor/compressor_test.go b/compressor/compressor_test.go new file mode 100644 index 0000000..621b731 --- /dev/null +++ b/compressor/compressor_test.go @@ -0,0 +1,122 @@ +package compressor + +import ( + "fmt" + "testing" +) + +func TestCompressor_Compress(t *testing.T) { + x := 0 // an empty value + + allCompressors := func() []Compressor { + return []Compressor{ + NewUniqueEntriesTable(), + NewRowDisplacementTable(x), + } + } + + tests := []struct { + original []int + rowCount int + colCount int + compressors []Compressor + }{ + { + original: []int{ + 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + }, + rowCount: 3, + colCount: 5, + compressors: allCompressors(), + }, + { + original: []int{ + x, x, x, x, x, + x, x, x, x, x, + x, x, x, x, x, + }, + rowCount: 3, + colCount: 5, + compressors: allCompressors(), + }, + { + original: []int{ + 1, 1, 1, 1, 1, + x, x, x, x, x, + 1, 1, 1, 1, 1, + }, + rowCount: 3, + colCount: 5, + compressors: allCompressors(), + }, + { + original: []int{ + 1, x, 1, 1, 1, + 1, 1, x, 1, 1, + 1, 1, 1, x, 1, + }, + rowCount: 3, + colCount: 5, + compressors: allCompressors(), + }, + } + for i, tt := range tests { + for _, comp := range tt.compressors { + t.Run(fmt.Sprintf("%T #%v", comp, i), func(t *testing.T) { + dup := make([]int, len(tt.original)) + copy(dup, tt.original) + + orig, err := NewOriginalTable(tt.original, tt.colCount) + if err != nil { + t.Fatal(err) + } + err = comp.Compress(orig) + if err != nil { + t.Fatal(err) + } + rowCount, colCount := comp.OriginalTableSize() + if rowCount != tt.rowCount || colCount != tt.colCount { + t.Fatalf("unexpected table size; want: %vx%v, got: %vx%v", tt.rowCount, tt.colCount, rowCount, colCount) + } + for i := 0; i < tt.rowCount; i++ { + for j := 0; j < tt.colCount; j++ { + v, err := comp.Lookup(i, j) + if err != nil { + t.Fatal(err) + } + expected := tt.original[i*tt.colCount+j] + if v != expected { + t.Fatalf("unexpected entry (%v, %v); want: %v, got: %v", i, j, expected, v) + } + } + } + + // Calling with out-of-range indexes should be an error. + if _, err := comp.Lookup(0, -1); err == nil { + t.Fatalf("expected error didn't occur (0, -1)") + } + if _, err := comp.Lookup(-1, 0); err == nil { + t.Fatalf("expected error didn't occur (-1, 0)") + } + if _, err := comp.Lookup(rowCount-1, colCount); err == nil { + t.Fatalf("expected error didn't occur (%v, %v)", rowCount-1, colCount) + } + if _, err := comp.Lookup(rowCount, colCount-1); err == nil { + t.Fatalf("expected error didn't occur (%v, %v)", rowCount, colCount-1) + } + + // The compressor must not break the original table. + for i := 0; i < tt.rowCount; i++ { + for j := 0; j < tt.colCount; j++ { + idx := i*tt.colCount + j + if tt.original[idx] != dup[idx] { + t.Fatalf("the original table is broken (%v, %v); want: %v, got: %v", i, j, dup[idx], tt.original[idx]) + } + } + } + }) + } + } +} diff --git a/driver/lexer/lexer.go b/driver/lexer/lexer.go new file mode 100644 index 0000000..de7cdbd --- /dev/null +++ b/driver/lexer/lexer.go @@ -0,0 +1,311 @@ +package lexer + +import ( + "fmt" + "io" +) + +type ModeID int + +func (id ModeID) Int() int { + return int(id) +} + +type StateID int + +func (id StateID) Int() int { + return int(id) +} + +type KindID int + +func (id KindID) Int() int { + return int(id) +} + +type ModeKindID int + +func (id ModeKindID) Int() int { + return int(id) +} + +type LexSpec interface { + InitialMode() ModeID + Pop(mode ModeID, modeKind ModeKindID) bool + Push(mode ModeID, modeKind ModeKindID) (ModeID, bool) + ModeName(mode ModeID) string + InitialState(mode ModeID) StateID + NextState(mode ModeID, state StateID, v int) (StateID, bool) + Accept(mode ModeID, state StateID) (ModeKindID, bool) + KindIDAndName(mode ModeID, modeKind ModeKindID) (KindID, string) +} + +// Token representes a token. +type Token struct { + // ModeID is an ID of a lex mode. + ModeID ModeID + + // KindID is an ID of a kind. This is unique among all modes. + KindID KindID + + // ModeKindID is an ID of a lexical kind. This is unique only within a mode. + // Note that you need to use KindID field if you want to identify a kind across all modes. + ModeKindID ModeKindID + + // Row is a row number where a lexeme appears. + Row int + + // Col is a column number where a lexeme appears. + // Note that Col is counted in code points, not bytes. + Col int + + // Lexeme is a byte sequence matched a pattern of a lexical specification. + Lexeme []byte + + // When this field is true, it means the token is the EOF token. + EOF bool + + // When this field is true, it means the token is an error token. + Invalid bool +} + +type LexerOption func(l *Lexer) error + +// DisableModeTransition disables the active mode transition. Thus, even if the lexical specification has the push and pop +// operations, the lexer doesn't perform these operations. When the lexical specification has multiple modes, and this option is +// enabled, you need to call the Lexer.Push and Lexer.Pop methods to perform the mode transition. You can use the Lexer.Mode method +// to know the current lex mode. +func DisableModeTransition() LexerOption { + return func(l *Lexer) error { + l.passiveModeTran = true + return nil + } +} + +type Lexer struct { + spec LexSpec + src []byte + srcPtr int + row int + col int + prevRow int + prevCol int + tokBuf []*Token + modeStack []ModeID + passiveModeTran bool +} + +// NewLexer returns a new lexer. +func NewLexer(spec LexSpec, src io.Reader, opts ...LexerOption) (*Lexer, error) { + b, err := io.ReadAll(src) + if err != nil { + return nil, err + } + l := &Lexer{ + spec: spec, + src: b, + srcPtr: 0, + row: 0, + col: 0, + modeStack: []ModeID{ + spec.InitialMode(), + }, + passiveModeTran: false, + } + for _, opt := range opts { + err := opt(l) + if err != nil { + return nil, err + } + } + + return l, nil +} + +// Next returns a next token. +func (l *Lexer) Next() (*Token, error) { + if len(l.tokBuf) > 0 { + tok := l.tokBuf[0] + l.tokBuf = l.tokBuf[1:] + return tok, nil + } + + tok, err := l.nextAndTransition() + if err != nil { + return nil, err + } + if !tok.Invalid { + return tok, nil + } + errTok := tok + for { + tok, err = l.nextAndTransition() + if err != nil { + return nil, err + } + if !tok.Invalid { + break + } + errTok.Lexeme = append(errTok.Lexeme, tok.Lexeme...) + } + l.tokBuf = append(l.tokBuf, tok) + + return errTok, nil +} + +func (l *Lexer) nextAndTransition() (*Token, error) { + tok, err := l.next() + if err != nil { + return nil, err + } + if tok.EOF || tok.Invalid { + return tok, nil + } + if l.passiveModeTran { + return tok, nil + } + mode := l.Mode() + if l.spec.Pop(mode, tok.ModeKindID) { + err := l.PopMode() + if err != nil { + return nil, err + } + } + if mode, ok := l.spec.Push(mode, tok.ModeKindID); ok { + l.PushMode(mode) + } + // The checking length of the mode stack must be at after pop and push operations because those operations can be performed + // at the same time. When the mode stack has just one element and popped it, the mode stack will be temporarily emptied. + // However, since a push operation may be performed immediately after it, the lexer allows the stack to be temporarily empty. + if len(l.modeStack) == 0 { + return nil, fmt.Errorf("a mode stack must have at least one element") + } + return tok, nil +} + +func (l *Lexer) next() (*Token, error) { + mode := l.Mode() + state := l.spec.InitialState(mode) + buf := []byte{} + unfixedBufLen := 0 + row := l.row + col := l.col + var tok *Token + for { + v, eof := l.read() + if eof { + if tok != nil { + l.unread(unfixedBufLen) + return tok, nil + } + // When `buf` has unaccepted data and reads the EOF, the lexer treats the buffered data as an invalid token. + if len(buf) > 0 { + return &Token{ + ModeID: mode, + ModeKindID: 0, + Lexeme: buf, + Row: row, + Col: col, + Invalid: true, + }, nil + } + return &Token{ + ModeID: mode, + ModeKindID: 0, + Row: 0, + Col: 0, + EOF: true, + }, nil + } + buf = append(buf, v) + unfixedBufLen++ + nextState, ok := l.spec.NextState(mode, state, int(v)) + if !ok { + if tok != nil { + l.unread(unfixedBufLen) + return tok, nil + } + return &Token{ + ModeID: mode, + ModeKindID: 0, + Lexeme: buf, + Row: row, + Col: col, + Invalid: true, + }, nil + } + state = nextState + if modeKindID, ok := l.spec.Accept(mode, state); ok { + kindID, _ := l.spec.KindIDAndName(mode, modeKindID) + tok = &Token{ + ModeID: mode, + KindID: kindID, + ModeKindID: modeKindID, + Lexeme: buf, + Row: row, + Col: col, + } + unfixedBufLen = 0 + } + } +} + +// Mode returns the current lex mode. +func (l *Lexer) Mode() ModeID { + return l.modeStack[len(l.modeStack)-1] +} + +// PushMode adds a lex mode onto the mode stack. +func (l *Lexer) PushMode(mode ModeID) { + l.modeStack = append(l.modeStack, mode) +} + +// PopMode removes a lex mode from the top of the mode stack. +func (l *Lexer) PopMode() error { + sLen := len(l.modeStack) + if sLen == 0 { + return fmt.Errorf("cannot pop a lex mode from a lex mode stack any more") + } + l.modeStack = l.modeStack[:sLen-1] + return nil +} + +func (l *Lexer) read() (byte, bool) { + if l.srcPtr >= len(l.src) { + return 0, true + } + + b := l.src[l.srcPtr] + l.srcPtr++ + + l.prevRow = l.row + l.prevCol = l.col + + // Count the token positions. + // The driver treats LF as the end of lines and counts columns in code points, not bytes. + // To count in code points, we refer to the First Byte column in the Table 3-6. + // + // Reference: + // - [Table 3-6] https://www.unicode.org/versions/Unicode13.0.0/ch03.pdf > Table 3-6. UTF-8 Bit Distribution + if b < 128 { + // 0x0A is LF. + if b == 0x0A { + l.row++ + l.col = 0 + } else { + l.col++ + } + } else if b>>5 == 6 || b>>4 == 14 || b>>3 == 30 { + l.col++ + } + + return b, false +} + +// We must not call this function consecutively to record the token position correctly. +func (l *Lexer) unread(n int) { + l.srcPtr -= n + + l.row = l.prevRow + l.col = l.prevCol +} diff --git a/driver/lexer/lexer_test.go b/driver/lexer/lexer_test.go new file mode 100644 index 0000000..247cc73 --- /dev/null +++ b/driver/lexer/lexer_test.go @@ -0,0 +1,918 @@ +package lexer + +import ( + "bytes" + "fmt" + "strings" + "testing" + + "github.com/nihei9/vartan/grammar/lexical" + spec "github.com/nihei9/vartan/spec/grammar" +) + +func newLexEntry(modes []string, kind string, pattern string, push string, pop bool) *lexical.LexEntry { + ms := []spec.LexModeName{} + for _, m := range modes { + ms = append(ms, spec.LexModeName(m)) + } + return &lexical.LexEntry{ + Kind: spec.LexKindName(kind), + Pattern: pattern, + Modes: ms, + Push: spec.LexModeName(push), + Pop: pop, + } +} + +func newLexEntryDefaultNOP(kind string, pattern string) *lexical.LexEntry { + return &lexical.LexEntry{ + Kind: spec.LexKindName(kind), + Pattern: pattern, + Modes: []spec.LexModeName{ + spec.LexModeNameDefault, + }, + } +} + +func newLexEntryFragment(kind string, pattern string) *lexical.LexEntry { + return &lexical.LexEntry{ + Kind: spec.LexKindName(kind), + Pattern: pattern, + Fragment: true, + } +} + +func newToken(modeID ModeID, kindID KindID, modeKindID ModeKindID, lexeme []byte) *Token { + return &Token{ + ModeID: modeID, + KindID: kindID, + ModeKindID: modeKindID, + Lexeme: lexeme, + } +} + +func newTokenDefault(kindID int, modeKindID int, lexeme []byte) *Token { + return newToken( + ModeID(spec.LexModeIDDefault.Int()), + KindID(spec.LexKindID(kindID).Int()), + ModeKindID(spec.LexModeKindID(modeKindID).Int()), + lexeme, + ) +} + +func newEOFToken(modeID ModeID, modeName string) *Token { + return &Token{ + ModeID: modeID, + ModeKindID: 0, + EOF: true, + } +} + +func newEOFTokenDefault() *Token { + return newEOFToken(ModeID(spec.LexModeIDDefault.Int()), spec.LexModeNameDefault.String()) +} + +func newInvalidTokenDefault(lexeme []byte) *Token { + return &Token{ + ModeID: ModeID(spec.LexModeIDDefault.Int()), + ModeKindID: 0, + Lexeme: lexeme, + Invalid: true, + } +} + +func withPos(tok *Token, row, col int) *Token { + tok.Row = row + tok.Col = col + return tok +} + +func TestLexer_Next(t *testing.T) { + test := []struct { + lspec *lexical.LexSpec + src string + tokens []*Token + passiveModeTran bool + tran func(l *Lexer, tok *Token) error + }{ + { + lspec: &lexical.LexSpec{ + Entries: []*lexical.LexEntry{ + newLexEntryDefaultNOP("t1", "(a|b)*abb"), + newLexEntryDefaultNOP("t2", " +"), + }, + }, + src: "abb aabb aaabb babb bbabb abbbabb", + tokens: []*Token{ + newTokenDefault(1, 1, []byte("abb")), + newTokenDefault(2, 2, []byte(" ")), + newTokenDefault(1, 1, []byte("aabb")), + newTokenDefault(2, 2, []byte(" ")), + newTokenDefault(1, 1, []byte("aaabb")), + newTokenDefault(2, 2, []byte(" ")), + newTokenDefault(1, 1, []byte("babb")), + newTokenDefault(2, 2, []byte(" ")), + newTokenDefault(1, 1, []byte("bbabb")), + newTokenDefault(2, 2, []byte(" ")), + newTokenDefault(1, 1, []byte("abbbabb")), + newEOFTokenDefault(), + }, + }, + { + lspec: &lexical.LexSpec{ + Entries: []*lexical.LexEntry{ + newLexEntryDefaultNOP("t1", "b?a+"), + newLexEntryDefaultNOP("t2", "(ab)?(cd)+"), + newLexEntryDefaultNOP("t3", " +"), + }, + }, + src: "ba baaa a aaa abcd abcdcdcd cd cdcdcd", + tokens: []*Token{ + newTokenDefault(1, 1, []byte("ba")), + newTokenDefault(3, 3, []byte(" ")), + newTokenDefault(1, 1, []byte("baaa")), + newTokenDefault(3, 3, []byte(" ")), + newTokenDefault(1, 1, []byte("a")), + newTokenDefault(3, 3, []byte(" ")), + newTokenDefault(1, 1, []byte("aaa")), + newTokenDefault(3, 3, []byte(" ")), + newTokenDefault(2, 2, []byte("abcd")), + newTokenDefault(3, 3, []byte(" ")), + newTokenDefault(2, 2, []byte("abcdcdcd")), + newTokenDefault(3, 3, []byte(" ")), + newTokenDefault(2, 2, []byte("cd")), + newTokenDefault(3, 3, []byte(" ")), + newTokenDefault(2, 2, []byte("cdcdcd")), + newEOFTokenDefault(), + }, + }, + { + lspec: &lexical.LexSpec{ + Entries: []*lexical.LexEntry{ + newLexEntryDefaultNOP("t1", "."), + }, + }, + src: string([]byte{ + 0x00, + 0x7f, + 0xc2, 0x80, + 0xdf, 0xbf, + 0xe1, 0x80, 0x80, + 0xec, 0xbf, 0xbf, + 0xed, 0x80, 0x80, + 0xed, 0x9f, 0xbf, + 0xee, 0x80, 0x80, + 0xef, 0xbf, 0xbf, + 0xf0, 0x90, 0x80, 0x80, + 0xf0, 0xbf, 0xbf, 0xbf, + 0xf1, 0x80, 0x80, 0x80, + 0xf3, 0xbf, 0xbf, 0xbf, + 0xf4, 0x80, 0x80, 0x80, + 0xf4, 0x8f, 0xbf, 0xbf, + }), + tokens: []*Token{ + newTokenDefault(1, 1, []byte{0x00}), + newTokenDefault(1, 1, []byte{0x7f}), + newTokenDefault(1, 1, []byte{0xc2, 0x80}), + newTokenDefault(1, 1, []byte{0xdf, 0xbf}), + newTokenDefault(1, 1, []byte{0xe1, 0x80, 0x80}), + newTokenDefault(1, 1, []byte{0xec, 0xbf, 0xbf}), + newTokenDefault(1, 1, []byte{0xed, 0x80, 0x80}), + newTokenDefault(1, 1, []byte{0xed, 0x9f, 0xbf}), + newTokenDefault(1, 1, []byte{0xee, 0x80, 0x80}), + newTokenDefault(1, 1, []byte{0xef, 0xbf, 0xbf}), + newTokenDefault(1, 1, []byte{0xf0, 0x90, 0x80, 0x80}), + newTokenDefault(1, 1, []byte{0xf0, 0xbf, 0xbf, 0xbf}), + newTokenDefault(1, 1, []byte{0xf1, 0x80, 0x80, 0x80}), + newTokenDefault(1, 1, []byte{0xf3, 0xbf, 0xbf, 0xbf}), + newTokenDefault(1, 1, []byte{0xf4, 0x80, 0x80, 0x80}), + newTokenDefault(1, 1, []byte{0xf4, 0x8f, 0xbf, 0xbf}), + newEOFTokenDefault(), + }, + }, + { + lspec: &lexical.LexSpec{ + Entries: []*lexical.LexEntry{ + newLexEntryDefaultNOP("t1", "[ab.*+?|()[\\]]"), + }, + }, + src: "ab.*+?|()[]", + tokens: []*Token{ + newTokenDefault(1, 1, []byte("a")), + newTokenDefault(1, 1, []byte("b")), + newTokenDefault(1, 1, []byte(".")), + newTokenDefault(1, 1, []byte("*")), + newTokenDefault(1, 1, []byte("+")), + newTokenDefault(1, 1, []byte("?")), + newTokenDefault(1, 1, []byte("|")), + newTokenDefault(1, 1, []byte("(")), + newTokenDefault(1, 1, []byte(")")), + newTokenDefault(1, 1, []byte("[")), + newTokenDefault(1, 1, []byte("]")), + newEOFTokenDefault(), + }, + }, + { + lspec: &lexical.LexSpec{ + Entries: []*lexical.LexEntry{ + // all 1 byte characters except null character (U+0000) + // + // NOTE: + // vartan cannot handle the null character in patterns because lexical.lexer, + // specifically read() and restore(), recognizes the null characters as that a symbol doesn't exist. + // If a pattern needs a null character, use code point expression \u{0000}. + newLexEntryDefaultNOP("char_1_byte", "[\x01-\x7f]"), + }, + }, + src: string([]byte{ + 0x01, + 0x02, + 0x7e, + 0x7f, + }), + tokens: []*Token{ + newTokenDefault(1, 1, []byte{0x01}), + newTokenDefault(1, 1, []byte{0x02}), + newTokenDefault(1, 1, []byte{0x7e}), + newTokenDefault(1, 1, []byte{0x7f}), + newEOFTokenDefault(), + }, + }, + { + lspec: &lexical.LexSpec{ + Entries: []*lexical.LexEntry{ + // all 2 byte characters + newLexEntryDefaultNOP("char_2_byte", "[\xc2\x80-\xdf\xbf]"), + }, + }, + src: string([]byte{ + 0xc2, 0x80, + 0xc2, 0x81, + 0xdf, 0xbe, + 0xdf, 0xbf, + }), + tokens: []*Token{ + newTokenDefault(1, 1, []byte{0xc2, 0x80}), + newTokenDefault(1, 1, []byte{0xc2, 0x81}), + newTokenDefault(1, 1, []byte{0xdf, 0xbe}), + newTokenDefault(1, 1, []byte{0xdf, 0xbf}), + newEOFTokenDefault(), + }, + }, + { + lspec: &lexical.LexSpec{ + Entries: []*lexical.LexEntry{ + // All bytes are the same. + newLexEntryDefaultNOP("char_3_byte", "[\xe0\xa0\x80-\xe0\xa0\x80]"), + }, + }, + src: string([]byte{ + 0xe0, 0xa0, 0x80, + }), + tokens: []*Token{ + newTokenDefault(1, 1, []byte{0xe0, 0xa0, 0x80}), + newEOFTokenDefault(), + }, + }, + { + lspec: &lexical.LexSpec{ + Entries: []*lexical.LexEntry{ + // The first two bytes are the same. + newLexEntryDefaultNOP("char_3_byte", "[\xe0\xa0\x80-\xe0\xa0\xbf]"), + }, + }, + src: string([]byte{ + 0xe0, 0xa0, 0x80, + 0xe0, 0xa0, 0x81, + 0xe0, 0xa0, 0xbe, + 0xe0, 0xa0, 0xbf, + }), + tokens: []*Token{ + newTokenDefault(1, 1, []byte{0xe0, 0xa0, 0x80}), + newTokenDefault(1, 1, []byte{0xe0, 0xa0, 0x81}), + newTokenDefault(1, 1, []byte{0xe0, 0xa0, 0xbe}), + newTokenDefault(1, 1, []byte{0xe0, 0xa0, 0xbf}), + newEOFTokenDefault(), + }, + }, + { + lspec: &lexical.LexSpec{ + Entries: []*lexical.LexEntry{ + // The first byte are the same. + newLexEntryDefaultNOP("char_3_byte", "[\xe0\xa0\x80-\xe0\xbf\xbf]"), + }, + }, + src: string([]byte{ + 0xe0, 0xa0, 0x80, + 0xe0, 0xa0, 0x81, + 0xe0, 0xbf, 0xbe, + 0xe0, 0xbf, 0xbf, + }), + tokens: []*Token{ + newTokenDefault(1, 1, []byte{0xe0, 0xa0, 0x80}), + newTokenDefault(1, 1, []byte{0xe0, 0xa0, 0x81}), + newTokenDefault(1, 1, []byte{0xe0, 0xbf, 0xbe}), + newTokenDefault(1, 1, []byte{0xe0, 0xbf, 0xbf}), + newEOFTokenDefault(), + }, + }, + { + lspec: &lexical.LexSpec{ + Entries: []*lexical.LexEntry{ + // all 3 byte characters + newLexEntryDefaultNOP("char_3_byte", "[\xe0\xa0\x80-\xef\xbf\xbf]"), + }, + }, + src: string([]byte{ + 0xe0, 0xa0, 0x80, + 0xe0, 0xa0, 0x81, + 0xe0, 0xbf, 0xbe, + 0xe0, 0xbf, 0xbf, + 0xe1, 0x80, 0x80, + 0xe1, 0x80, 0x81, + 0xec, 0xbf, 0xbe, + 0xec, 0xbf, 0xbf, + 0xed, 0x80, 0x80, + 0xed, 0x80, 0x81, + 0xed, 0x9f, 0xbe, + 0xed, 0x9f, 0xbf, + 0xee, 0x80, 0x80, + 0xee, 0x80, 0x81, + 0xef, 0xbf, 0xbe, + 0xef, 0xbf, 0xbf, + }), + tokens: []*Token{ + newTokenDefault(1, 1, []byte{0xe0, 0xa0, 0x80}), + newTokenDefault(1, 1, []byte{0xe0, 0xa0, 0x81}), + newTokenDefault(1, 1, []byte{0xe0, 0xbf, 0xbe}), + newTokenDefault(1, 1, []byte{0xe0, 0xbf, 0xbf}), + newTokenDefault(1, 1, []byte{0xe1, 0x80, 0x80}), + newTokenDefault(1, 1, []byte{0xe1, 0x80, 0x81}), + newTokenDefault(1, 1, []byte{0xec, 0xbf, 0xbe}), + newTokenDefault(1, 1, []byte{0xec, 0xbf, 0xbf}), + newTokenDefault(1, 1, []byte{0xed, 0x80, 0x80}), + newTokenDefault(1, 1, []byte{0xed, 0x80, 0x81}), + newTokenDefault(1, 1, []byte{0xed, 0x9f, 0xbe}), + newTokenDefault(1, 1, []byte{0xed, 0x9f, 0xbf}), + newTokenDefault(1, 1, []byte{0xee, 0x80, 0x80}), + newTokenDefault(1, 1, []byte{0xee, 0x80, 0x81}), + newTokenDefault(1, 1, []byte{0xef, 0xbf, 0xbe}), + newTokenDefault(1, 1, []byte{0xef, 0xbf, 0xbf}), + newEOFTokenDefault(), + }, + }, + { + lspec: &lexical.LexSpec{ + Entries: []*lexical.LexEntry{ + // All bytes are the same. + newLexEntryDefaultNOP("char_4_byte", "[\xf0\x90\x80\x80-\xf0\x90\x80\x80]"), + }, + }, + src: string([]byte{ + 0xf0, 0x90, 0x80, 0x80, + }), + tokens: []*Token{ + newTokenDefault(1, 1, []byte{0xf0, 0x90, 0x80, 0x80}), + newEOFTokenDefault(), + }, + }, + { + lspec: &lexical.LexSpec{ + Entries: []*lexical.LexEntry{ + // The first 3 bytes are the same. + newLexEntryDefaultNOP("char_4_byte", "[\xf0\x90\x80\x80-\xf0\x90\x80\xbf]"), + }, + }, + src: string([]byte{ + 0xf0, 0x90, 0x80, 0x80, + 0xf0, 0x90, 0x80, 0x81, + 0xf0, 0x90, 0x80, 0xbe, + 0xf0, 0x90, 0x80, 0xbf, + }), + tokens: []*Token{ + newTokenDefault(1, 1, []byte{0xf0, 0x90, 0x80, 0x80}), + newTokenDefault(1, 1, []byte{0xf0, 0x90, 0x80, 0x81}), + newTokenDefault(1, 1, []byte{0xf0, 0x90, 0x80, 0xbe}), + newTokenDefault(1, 1, []byte{0xf0, 0x90, 0x80, 0xbf}), + newEOFTokenDefault(), + }, + }, + { + lspec: &lexical.LexSpec{ + Entries: []*lexical.LexEntry{ + // The first 2 bytes are the same. + newLexEntryDefaultNOP("char_4_byte", "[\xf0\x90\x80\x80-\xf0\x90\xbf\xbf]"), + }, + }, + src: string([]byte{ + 0xf0, 0x90, 0x80, 0x80, + 0xf0, 0x90, 0x80, 0x81, + 0xf0, 0x90, 0xbf, 0xbe, + 0xf0, 0x90, 0xbf, 0xbf, + }), + tokens: []*Token{ + newTokenDefault(1, 1, []byte{0xf0, 0x90, 0x80, 0x80}), + newTokenDefault(1, 1, []byte{0xf0, 0x90, 0x80, 0x81}), + newTokenDefault(1, 1, []byte{0xf0, 0x90, 0xbf, 0xbe}), + newTokenDefault(1, 1, []byte{0xf0, 0x90, 0xbf, 0xbf}), + newEOFTokenDefault(), + }, + }, + { + lspec: &lexical.LexSpec{ + Entries: []*lexical.LexEntry{ + // The first byte are the same. + newLexEntryDefaultNOP("char_4_byte", "[\xf0\x90\x80\x80-\xf0\xbf\xbf\xbf]"), + }, + }, + src: string([]byte{ + 0xf0, 0x90, 0x80, 0x80, + 0xf0, 0x90, 0x80, 0x81, + 0xf0, 0xbf, 0xbf, 0xbe, + 0xf0, 0xbf, 0xbf, 0xbf, + }), + tokens: []*Token{ + newTokenDefault(1, 1, []byte{0xf0, 0x90, 0x80, 0x80}), + newTokenDefault(1, 1, []byte{0xf0, 0x90, 0x80, 0x81}), + newTokenDefault(1, 1, []byte{0xf0, 0xbf, 0xbf, 0xbe}), + newTokenDefault(1, 1, []byte{0xf0, 0xbf, 0xbf, 0xbf}), + newEOFTokenDefault(), + }, + }, + { + lspec: &lexical.LexSpec{ + Entries: []*lexical.LexEntry{ + // all 4 byte characters + newLexEntryDefaultNOP("char_4_byte", "[\xf0\x90\x80\x80-\xf4\x8f\xbf\xbf]"), + }, + }, + src: string([]byte{ + 0xf0, 0x90, 0x80, 0x80, + 0xf0, 0x90, 0x80, 0x81, + 0xf0, 0xbf, 0xbf, 0xbe, + 0xf0, 0xbf, 0xbf, 0xbf, + 0xf1, 0x80, 0x80, 0x80, + 0xf1, 0x80, 0x80, 0x81, + 0xf3, 0xbf, 0xbf, 0xbe, + 0xf3, 0xbf, 0xbf, 0xbf, + 0xf4, 0x80, 0x80, 0x80, + 0xf4, 0x80, 0x80, 0x81, + 0xf4, 0x8f, 0xbf, 0xbe, + 0xf4, 0x8f, 0xbf, 0xbf, + }), + tokens: []*Token{ + newTokenDefault(1, 1, []byte{0xf0, 0x90, 0x80, 0x80}), + newTokenDefault(1, 1, []byte{0xf0, 0x90, 0x80, 0x81}), + newTokenDefault(1, 1, []byte{0xf0, 0xbf, 0xbf, 0xbe}), + newTokenDefault(1, 1, []byte{0xf0, 0xbf, 0xbf, 0xbf}), + newTokenDefault(1, 1, []byte{0xf1, 0x80, 0x80, 0x80}), + newTokenDefault(1, 1, []byte{0xf1, 0x80, 0x80, 0x81}), + newTokenDefault(1, 1, []byte{0xf3, 0xbf, 0xbf, 0xbe}), + newTokenDefault(1, 1, []byte{0xf3, 0xbf, 0xbf, 0xbf}), + newTokenDefault(1, 1, []byte{0xf4, 0x80, 0x80, 0x80}), + newTokenDefault(1, 1, []byte{0xf4, 0x80, 0x80, 0x81}), + newTokenDefault(1, 1, []byte{0xf4, 0x8f, 0xbf, 0xbe}), + newTokenDefault(1, 1, []byte{0xf4, 0x8f, 0xbf, 0xbf}), + newEOFTokenDefault(), + }, + }, + { + lspec: &lexical.LexSpec{ + Entries: []*lexical.LexEntry{ + newLexEntryDefaultNOP("non_number", "[^0-9]+[0-9]"), + }, + }, + src: "foo9", + tokens: []*Token{ + newTokenDefault(1, 1, []byte("foo9")), + newEOFTokenDefault(), + }, + }, + { + lspec: &lexical.LexSpec{ + Entries: []*lexical.LexEntry{ + newLexEntryDefaultNOP("char_1_byte", "\\u{006E}"), + newLexEntryDefaultNOP("char_2_byte", "\\u{03BD}"), + newLexEntryDefaultNOP("char_3_byte", "\\u{306B}"), + newLexEntryDefaultNOP("char_4_byte", "\\u{01F638}"), + }, + }, + src: "nνに😸", + tokens: []*Token{ + newTokenDefault(1, 1, []byte{0x6E}), + newTokenDefault(2, 2, []byte{0xCE, 0xBD}), + newTokenDefault(3, 3, []byte{0xE3, 0x81, 0xAB}), + newTokenDefault(4, 4, []byte{0xF0, 0x9F, 0x98, 0xB8}), + newEOFTokenDefault(), + }, + }, + { + lspec: &lexical.LexSpec{ + Entries: []*lexical.LexEntry{ + newLexEntryDefaultNOP("code_points_alt", "[\\u{006E}\\u{03BD}\\u{306B}\\u{01F638}]"), + }, + }, + src: "nνに😸", + tokens: []*Token{ + newTokenDefault(1, 1, []byte{0x6E}), + newTokenDefault(1, 1, []byte{0xCE, 0xBD}), + newTokenDefault(1, 1, []byte{0xE3, 0x81, 0xAB}), + newTokenDefault(1, 1, []byte{0xF0, 0x9F, 0x98, 0xB8}), + newEOFTokenDefault(), + }, + }, + { + lspec: &lexical.LexSpec{ + Entries: []*lexical.LexEntry{ + newLexEntryDefaultNOP("t1", "\\f{a2c}\\f{d2f}+"), + newLexEntryFragment("a2c", "abc"), + newLexEntryFragment("d2f", "def"), + }, + }, + src: "abcdefdefabcdef", + tokens: []*Token{ + newTokenDefault(1, 1, []byte("abcdefdef")), + newTokenDefault(1, 1, []byte("abcdef")), + newEOFTokenDefault(), + }, + }, + { + lspec: &lexical.LexSpec{ + Entries: []*lexical.LexEntry{ + newLexEntryDefaultNOP("t1", "(\\f{a2c}|\\f{d2f})+"), + newLexEntryFragment("a2c", "abc"), + newLexEntryFragment("d2f", "def"), + }, + }, + src: "abcdefdefabc", + tokens: []*Token{ + newTokenDefault(1, 1, []byte("abcdefdefabc")), + newEOFTokenDefault(), + }, + }, + { + lspec: &lexical.LexSpec{ + Entries: []*lexical.LexEntry{ + newLexEntryDefaultNOP("t1", "\\f{a2c_or_d2f}+"), + newLexEntryFragment("a2c_or_d2f", "\\f{a2c}|\\f{d2f}"), + newLexEntryFragment("a2c", "abc"), + newLexEntryFragment("d2f", "def"), + }, + }, + src: "abcdefdefabc", + tokens: []*Token{ + newTokenDefault(1, 1, []byte("abcdefdefabc")), + newEOFTokenDefault(), + }, + }, + { + lspec: &lexical.LexSpec{ + Entries: []*lexical.LexEntry{ + newLexEntryDefaultNOP("white_space", ` *`), + newLexEntry([]string{"default"}, "string_open", `"`, "string", false), + newLexEntry([]string{"string"}, "escape_sequence", `\\[n"\\]`, "", false), + newLexEntry([]string{"string"}, "char_sequence", `[^"\\]*`, "", false), + newLexEntry([]string{"string"}, "string_close", `"`, "", true), + }, + }, + src: `"" "Hello world.\n\"Hello world.\""`, + tokens: []*Token{ + newToken(1, 2, 2, []byte(`"`)), + newToken(2, 5, 3, []byte(`"`)), + newToken(1, 1, 1, []byte(` `)), + newToken(1, 2, 2, []byte(`"`)), + newToken(2, 4, 2, []byte(`Hello world.`)), + newToken(2, 3, 1, []byte(`\n`)), + newToken(2, 3, 1, []byte(`\"`)), + newToken(2, 4, 2, []byte(`Hello world.`)), + newToken(2, 3, 1, []byte(`\"`)), + newToken(2, 5, 3, []byte(`"`)), + newEOFTokenDefault(), + }, + }, + { + lspec: &lexical.LexSpec{ + Entries: []*lexical.LexEntry{ + // `white_space` is enabled in multiple modes. + newLexEntry([]string{"default", "state_a", "state_b"}, "white_space", ` *`, "", false), + newLexEntry([]string{"default"}, "char_a", `a`, "state_a", false), + newLexEntry([]string{"state_a"}, "char_b", `b`, "state_b", false), + newLexEntry([]string{"state_a"}, "back_from_a", `<`, "", true), + newLexEntry([]string{"state_b"}, "back_from_b", `<`, "", true), + }, + }, + src: ` a b < < `, + tokens: []*Token{ + newToken(1, 1, 1, []byte(` `)), + newToken(1, 2, 2, []byte(`a`)), + newToken(2, 1, 1, []byte(` `)), + newToken(2, 3, 2, []byte(`b`)), + newToken(3, 1, 1, []byte(` `)), + newToken(3, 5, 2, []byte(`<`)), + newToken(2, 1, 1, []byte(` `)), + newToken(2, 4, 3, []byte(`<`)), + newToken(1, 1, 1, []byte(` `)), + newEOFTokenDefault(), + }, + }, + { + lspec: &lexical.LexSpec{ + Entries: []*lexical.LexEntry{ + newLexEntry([]string{"default", "mode_1", "mode_2"}, "white_space", ` *`, "", false), + newLexEntry([]string{"default"}, "char", `.`, "", false), + newLexEntry([]string{"default"}, "push_1", `-> 1`, "", false), + newLexEntry([]string{"mode_1"}, "push_2", `-> 2`, "", false), + newLexEntry([]string{"mode_1"}, "pop_1", `<-`, "", false), + newLexEntry([]string{"mode_2"}, "pop_2", `<-`, "", false), + }, + }, + src: `-> 1 -> 2 <- <- a`, + tokens: []*Token{ + newToken(1, 3, 3, []byte(`-> 1`)), + newToken(2, 1, 1, []byte(` `)), + newToken(2, 4, 2, []byte(`-> 2`)), + newToken(3, 1, 1, []byte(` `)), + newToken(3, 6, 2, []byte(`<-`)), + newToken(2, 1, 1, []byte(` `)), + newToken(2, 5, 3, []byte(`<-`)), + newToken(1, 1, 1, []byte(` `)), + newToken(1, 2, 2, []byte(`a`)), + newEOFTokenDefault(), + }, + passiveModeTran: true, + tran: func(l *Lexer, tok *Token) error { + switch l.spec.ModeName(l.Mode()) { + case "default": + switch tok.KindID { + case 3: // push_1 + l.PushMode(2) + } + case "mode_1": + switch tok.KindID { + case 4: // push_2 + l.PushMode(3) + case 5: // pop_1 + return l.PopMode() + } + case "mode_2": + switch tok.KindID { + case 6: // pop_2 + return l.PopMode() + } + } + return nil + }, + }, + { + lspec: &lexical.LexSpec{ + Entries: []*lexical.LexEntry{ + newLexEntry([]string{"default", "mode_1", "mode_2"}, "white_space", ` *`, "", false), + newLexEntry([]string{"default"}, "char", `.`, "", false), + newLexEntry([]string{"default"}, "push_1", `-> 1`, "mode_1", false), + newLexEntry([]string{"mode_1"}, "push_2", `-> 2`, "", false), + newLexEntry([]string{"mode_1"}, "pop_1", `<-`, "", false), + newLexEntry([]string{"mode_2"}, "pop_2", `<-`, "", true), + }, + }, + src: `-> 1 -> 2 <- <- a`, + tokens: []*Token{ + newToken(1, 3, 3, []byte(`-> 1`)), + newToken(2, 1, 1, []byte(` `)), + newToken(2, 4, 2, []byte(`-> 2`)), + newToken(3, 1, 1, []byte(` `)), + newToken(3, 6, 2, []byte(`<-`)), + newToken(2, 1, 1, []byte(` `)), + newToken(2, 5, 3, []byte(`<-`)), + newToken(1, 1, 1, []byte(` `)), + newToken(1, 2, 2, []byte(`a`)), + newEOFTokenDefault(), + }, + // Active mode transition and an external transition function can be used together. + passiveModeTran: false, + tran: func(l *Lexer, tok *Token) error { + switch l.spec.ModeName(l.Mode()) { + case "mode_1": + switch tok.KindID { + case 4: // push_2 + l.PushMode(3) + case 5: // pop_1 + return l.PopMode() + } + } + return nil + }, + }, + { + lspec: &lexical.LexSpec{ + Entries: []*lexical.LexEntry{ + newLexEntryDefaultNOP("dot", spec.EscapePattern(`.`)), + newLexEntryDefaultNOP("star", spec.EscapePattern(`*`)), + newLexEntryDefaultNOP("plus", spec.EscapePattern(`+`)), + newLexEntryDefaultNOP("question", spec.EscapePattern(`?`)), + newLexEntryDefaultNOP("vbar", spec.EscapePattern(`|`)), + newLexEntryDefaultNOP("lparen", spec.EscapePattern(`(`)), + newLexEntryDefaultNOP("rparen", spec.EscapePattern(`)`)), + newLexEntryDefaultNOP("lbrace", spec.EscapePattern(`[`)), + newLexEntryDefaultNOP("backslash", spec.EscapePattern(`\`)), + }, + }, + src: `.*+?|()[\`, + tokens: []*Token{ + newTokenDefault(1, 1, []byte(`.`)), + newTokenDefault(2, 2, []byte(`*`)), + newTokenDefault(3, 3, []byte(`+`)), + newTokenDefault(4, 4, []byte(`?`)), + newTokenDefault(5, 5, []byte(`|`)), + newTokenDefault(6, 6, []byte(`(`)), + newTokenDefault(7, 7, []byte(`)`)), + newTokenDefault(8, 8, []byte(`[`)), + newTokenDefault(9, 9, []byte(`\`)), + newEOFTokenDefault(), + }, + }, + // Character properties are available in a bracket expression. + { + lspec: &lexical.LexSpec{ + Entries: []*lexical.LexEntry{ + newLexEntryDefaultNOP("letter", `[\p{Letter}]+`), + newLexEntryDefaultNOP("non_letter", `[^\p{Letter}]+`), + }, + }, + src: `foo123`, + tokens: []*Token{ + newTokenDefault(1, 1, []byte(`foo`)), + newTokenDefault(2, 2, []byte(`123`)), + newEOFTokenDefault(), + }, + }, + // The driver can continue lexical analysis even after it detects an invalid token. + { + lspec: &lexical.LexSpec{ + Entries: []*lexical.LexEntry{ + newLexEntryDefaultNOP("lower", `[a-z]+`), + }, + }, + src: `foo123bar`, + tokens: []*Token{ + newTokenDefault(1, 1, []byte(`foo`)), + newInvalidTokenDefault([]byte(`123`)), + newTokenDefault(1, 1, []byte(`bar`)), + newEOFTokenDefault(), + }, + }, + } + for i, tt := range test { + for compLv := lexical.CompressionLevelMin; compLv <= lexical.CompressionLevelMax; compLv++ { + t.Run(fmt.Sprintf("#%v-%v", i, compLv), func(t *testing.T) { + clspec, err, cerrs := lexical.Compile(tt.lspec, compLv) + if err != nil { + for _, cerr := range cerrs { + t.Logf("%#v", cerr) + } + t.Fatalf("unexpected error: %v", err) + } + opts := []LexerOption{} + if tt.passiveModeTran { + opts = append(opts, DisableModeTransition()) + } + lexer, err := NewLexer(NewLexSpec(clspec), strings.NewReader(tt.src), opts...) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + for _, eTok := range tt.tokens { + tok, err := lexer.Next() + if err != nil { + t.Log(err) + break + } + testToken(t, eTok, tok, false) + + if tok.EOF { + break + } + + if tt.tran != nil { + err := tt.tran(lexer, tok) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + } + } + }) + } + } +} + +func TestLexer_Next_WithPosition(t *testing.T) { + lspec := &lexical.LexSpec{ + Entries: []*lexical.LexEntry{ + newLexEntryDefaultNOP("newline", `\u{000A}+`), + newLexEntryDefaultNOP("any", `.`), + }, + } + + clspec, err, _ := lexical.Compile(lspec, lexical.CompressionLevelMax) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + src := string([]byte{ + 0x00, + 0x7F, + 0x0A, + + 0xC2, 0x80, + 0xDF, 0xBF, + 0x0A, + + 0xE0, 0xA0, 0x80, + 0xE0, 0xBF, 0xBF, + 0xE1, 0x80, 0x80, + 0xEC, 0xBF, 0xBF, + 0xED, 0x80, 0x80, + 0xED, 0x9F, 0xBF, + 0xEE, 0x80, 0x80, + 0xEF, 0xBF, 0xBF, + 0x0A, + + 0xF0, 0x90, 0x80, 0x80, + 0xF0, 0xBF, 0xBF, 0xBF, + 0xF1, 0x80, 0x80, 0x80, + 0xF3, 0xBF, 0xBF, 0xBF, + 0xF4, 0x80, 0x80, 0x80, + 0xF4, 0x8F, 0xBF, 0xBF, + 0x0A, + 0x0A, + 0x0A, + }) + + expected := []*Token{ + withPos(newTokenDefault(2, 2, []byte{0x00}), 0, 0), + withPos(newTokenDefault(2, 2, []byte{0x7F}), 0, 1), + withPos(newTokenDefault(1, 1, []byte{0x0A}), 0, 2), + + withPos(newTokenDefault(2, 2, []byte{0xC2, 0x80}), 1, 0), + withPos(newTokenDefault(2, 2, []byte{0xDF, 0xBF}), 1, 1), + withPos(newTokenDefault(1, 1, []byte{0x0A}), 1, 2), + + withPos(newTokenDefault(2, 2, []byte{0xE0, 0xA0, 0x80}), 2, 0), + withPos(newTokenDefault(2, 2, []byte{0xE0, 0xBF, 0xBF}), 2, 1), + withPos(newTokenDefault(2, 2, []byte{0xE1, 0x80, 0x80}), 2, 2), + withPos(newTokenDefault(2, 2, []byte{0xEC, 0xBF, 0xBF}), 2, 3), + withPos(newTokenDefault(2, 2, []byte{0xED, 0x80, 0x80}), 2, 4), + withPos(newTokenDefault(2, 2, []byte{0xED, 0x9F, 0xBF}), 2, 5), + withPos(newTokenDefault(2, 2, []byte{0xEE, 0x80, 0x80}), 2, 6), + withPos(newTokenDefault(2, 2, []byte{0xEF, 0xBF, 0xBF}), 2, 7), + withPos(newTokenDefault(1, 1, []byte{0x0A}), 2, 8), + + withPos(newTokenDefault(2, 2, []byte{0xF0, 0x90, 0x80, 0x80}), 3, 0), + withPos(newTokenDefault(2, 2, []byte{0xF0, 0xBF, 0xBF, 0xBF}), 3, 1), + withPos(newTokenDefault(2, 2, []byte{0xF1, 0x80, 0x80, 0x80}), 3, 2), + withPos(newTokenDefault(2, 2, []byte{0xF3, 0xBF, 0xBF, 0xBF}), 3, 3), + withPos(newTokenDefault(2, 2, []byte{0xF4, 0x80, 0x80, 0x80}), 3, 4), + withPos(newTokenDefault(2, 2, []byte{0xF4, 0x8F, 0xBF, 0xBF}), 3, 5), + + // When a token contains multiple line breaks, the driver sets the token position to + // the line number where a lexeme first appears. + withPos(newTokenDefault(1, 1, []byte{0x0A, 0x0A, 0x0A}), 3, 6), + + withPos(newEOFTokenDefault(), 0, 0), + } + + lexer, err := NewLexer(NewLexSpec(clspec), strings.NewReader(src)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + for _, eTok := range expected { + tok, err := lexer.Next() + if err != nil { + t.Fatal(err) + } + + testToken(t, eTok, tok, true) + + if tok.EOF { + break + } + } +} + +func testToken(t *testing.T, expected, actual *Token, checkPosition bool) { + t.Helper() + + if actual.ModeID != expected.ModeID || + actual.KindID != expected.KindID || + actual.ModeKindID != expected.ModeKindID || + !bytes.Equal(actual.Lexeme, expected.Lexeme) || + actual.EOF != expected.EOF || + actual.Invalid != expected.Invalid { + t.Fatalf(`unexpected token; want: %v ("%#v"), got: %v ("%#v")`, expected, string(expected.Lexeme), actual, string(actual.Lexeme)) + } + + if checkPosition { + if actual.Row != expected.Row || actual.Col != expected.Col { + t.Fatalf(`unexpected token; want: %v ("%#v"), got: %v ("%#v")`, expected, string(expected.Lexeme), actual, string(actual.Lexeme)) + } + } +} diff --git a/driver/lexer/spec.go b/driver/lexer/spec.go new file mode 100644 index 0000000..23debbf --- /dev/null +++ b/driver/lexer/spec.go @@ -0,0 +1,71 @@ +package lexer + +import spec "github.com/nihei9/vartan/spec/grammar" + +type lexSpec struct { + spec *spec.LexicalSpec +} + +func NewLexSpec(spec *spec.LexicalSpec) *lexSpec { + return &lexSpec{ + spec: spec, + } +} + +func (s *lexSpec) InitialMode() ModeID { + return ModeID(s.spec.InitialModeID.Int()) +} + +func (s *lexSpec) Pop(mode ModeID, modeKind ModeKindID) bool { + return s.spec.Specs[mode].Pop[modeKind] == 1 +} + +func (s *lexSpec) Push(mode ModeID, modeKind ModeKindID) (ModeID, bool) { + modeID := s.spec.Specs[mode].Push[modeKind] + return ModeID(modeID.Int()), !modeID.IsNil() +} + +func (s *lexSpec) ModeName(mode ModeID) string { + return s.spec.ModeNames[mode].String() +} + +func (s *lexSpec) InitialState(mode ModeID) StateID { + return StateID(s.spec.Specs[mode].DFA.InitialStateID.Int()) +} + +func (s *lexSpec) NextState(mode ModeID, state StateID, v int) (StateID, bool) { + switch s.spec.CompressionLevel { + case 2: + tran := s.spec.Specs[mode].DFA.Transition + rowNum := tran.RowNums[state] + d := tran.UniqueEntries.RowDisplacement[rowNum] + if tran.UniqueEntries.Bounds[d+v] != rowNum { + return StateID(tran.UniqueEntries.EmptyValue.Int()), false + } + return StateID(tran.UniqueEntries.Entries[d+v].Int()), true + case 1: + tran := s.spec.Specs[mode].DFA.Transition + next := tran.UncompressedUniqueEntries[tran.RowNums[state]*tran.OriginalColCount+v] + if next == spec.StateIDNil { + return StateID(spec.StateIDNil.Int()), false + } + return StateID(next.Int()), true + } + + modeSpec := s.spec.Specs[mode] + next := modeSpec.DFA.UncompressedTransition[state.Int()*modeSpec.DFA.ColCount+v] + if next == spec.StateIDNil { + return StateID(spec.StateIDNil), false + } + return StateID(next.Int()), true +} + +func (s *lexSpec) Accept(mode ModeID, state StateID) (ModeKindID, bool) { + modeKindID := s.spec.Specs[mode].DFA.AcceptingStates[state] + return ModeKindID(modeKindID.Int()), modeKindID != spec.LexModeKindIDNil +} + +func (s *lexSpec) KindIDAndName(mode ModeID, modeKind ModeKindID) (KindID, string) { + kindID := s.spec.KindIDs[mode][modeKind] + return KindID(kindID.Int()), s.spec.KindNames[kindID].String() +} diff --git a/driver/lexer/template.go b/driver/lexer/template.go new file mode 100644 index 0000000..52f9ebd --- /dev/null +++ b/driver/lexer/template.go @@ -0,0 +1,760 @@ +package lexer + +import ( + "bytes" + _ "embed" + "fmt" + "go/ast" + "go/format" + "go/parser" + "go/token" + "strings" + "text/template" + + "github.com/nihei9/vartan/grammar/lexical" + spec "github.com/nihei9/vartan/spec/grammar" +) + +//go:embed lexer.go +var lexerCoreSrc string + +func GenLexer(lexSpec *spec.LexicalSpec, pkgName string) ([]byte, error) { + var lexerSrc string + { + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, "lexer.go", lexerCoreSrc, parser.ParseComments) + if err != nil { + return nil, err + } + + var b strings.Builder + err = format.Node(&b, fset, f) + if err != nil { + return nil, err + } + + lexerSrc = b.String() + } + + var modeIDsSrc string + { + var b strings.Builder + fmt.Fprintf(&b, "const (\n") + for i, k := range lexSpec.ModeNames { + if i == spec.LexModeIDNil.Int() { + fmt.Fprintf(&b, " ModeIDNil ModeID = %v\n", i) + continue + } + fmt.Fprintf(&b, " ModeID%v ModeID = %v\n", lexical.SnakeCaseToUpperCamelCase(k.String()), i) + } + fmt.Fprintf(&b, ")") + + modeIDsSrc = b.String() + } + + var modeNamesSrc string + { + var b strings.Builder + fmt.Fprintf(&b, "const (\n") + for i, k := range lexSpec.ModeNames { + if i == spec.LexModeIDNil.Int() { + fmt.Fprintf(&b, " ModeNameNil = %#v\n", "") + continue + } + fmt.Fprintf(&b, " ModeName%v = %#v\n", lexical.SnakeCaseToUpperCamelCase(k.String()), k) + } + fmt.Fprintf(&b, ")") + + modeNamesSrc = b.String() + } + + var modeIDToNameSrc string + { + var b strings.Builder + fmt.Fprintf(&b, ` +// ModeIDToName converts a mode ID to a name. +func ModeIDToName(id ModeID) string { + switch id {`) + for i, k := range lexSpec.ModeNames { + if i == spec.LexModeIDNil.Int() { + fmt.Fprintf(&b, ` + case ModeIDNil: + return ModeNameNil`) + continue + } + name := lexical.SnakeCaseToUpperCamelCase(k.String()) + fmt.Fprintf(&b, ` + case ModeID%v: + return ModeName%v`, name, name) + } + fmt.Fprintf(&b, ` + } + return "" +} +`) + + modeIDToNameSrc = b.String() + } + + var kindIDsSrc string + { + var b strings.Builder + fmt.Fprintf(&b, "const (\n") + for i, k := range lexSpec.KindNames { + if i == spec.LexKindIDNil.Int() { + fmt.Fprintf(&b, " KindIDNil KindID = %v\n", i) + continue + } + fmt.Fprintf(&b, " KindID%v KindID = %v\n", lexical.SnakeCaseToUpperCamelCase(k.String()), i) + } + fmt.Fprintf(&b, ")") + + kindIDsSrc = b.String() + } + + var kindNamesSrc string + { + var b strings.Builder + fmt.Fprintf(&b, "const (\n") + fmt.Fprintf(&b, " KindNameNil = %#v\n", "") + for _, k := range lexSpec.KindNames[1:] { + fmt.Fprintf(&b, " KindName%v = %#v\n", lexical.SnakeCaseToUpperCamelCase(k.String()), k) + } + fmt.Fprintf(&b, ")") + + kindNamesSrc = b.String() + } + + var kindIDToNameSrc string + { + var b strings.Builder + fmt.Fprintf(&b, ` +// KindIDToName converts a kind ID to a name. +func KindIDToName(id KindID) string { + switch id {`) + for i, k := range lexSpec.KindNames { + if i == spec.LexModeIDNil.Int() { + fmt.Fprintf(&b, ` + case KindIDNil: + return KindNameNil`) + continue + } + name := lexical.SnakeCaseToUpperCamelCase(k.String()) + fmt.Fprintf(&b, ` + case KindID%v: + return KindName%v`, name, name) + } + fmt.Fprintf(&b, ` + } + return "" +} +`) + + kindIDToNameSrc = b.String() + } + + var specSrc string + { + t, err := template.New("").Funcs(genTemplateFuncs(lexSpec)).Parse(lexSpecTemplate) + if err != nil { + return nil, err + } + + var b strings.Builder + err = t.Execute(&b, map[string]interface{}{ + "initialModeID": "ModeID" + lexical.SnakeCaseToUpperCamelCase(lexSpec.ModeNames[lexSpec.InitialModeID].String()), + "modeIDNil": "ModeIDNil", + "modeKindIDNil": spec.LexModeKindIDNil, + "stateIDNil": spec.StateIDNil, + "compressionLevel": lexSpec.CompressionLevel, + }) + if err != nil { + return nil, err + } + + specSrc = b.String() + } + + var src string + { + tmpl := `// Code generated by vartan-go. DO NOT EDIT. +{{ .lexerSrc }} + +{{ .modeIDsSrc }} + +{{ .modeNamesSrc }} + +{{ .modeIDToNameSrc }} + +{{ .kindIDsSrc }} + +{{ .kindNamesSrc }} + +{{ .kindIDToNameSrc }} + +{{ .specSrc }} +` + + t, err := template.New("").Parse(tmpl) + if err != nil { + return nil, err + } + + var b strings.Builder + err = t.Execute(&b, map[string]string{ + "lexerSrc": lexerSrc, + "modeIDsSrc": modeIDsSrc, + "modeNamesSrc": modeNamesSrc, + "modeIDToNameSrc": modeIDToNameSrc, + "kindIDsSrc": kindIDsSrc, + "kindNamesSrc": kindNamesSrc, + "kindIDToNameSrc": kindIDToNameSrc, + "specSrc": specSrc, + }) + if err != nil { + return nil, err + } + + src = b.String() + } + + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, "", src, parser.ParseComments) + if err != nil { + return nil, err + } + + f.Name = ast.NewIdent(pkgName) + + var b bytes.Buffer + err = format.Node(&b, fset, f) + if err != nil { + return nil, err + } + + return b.Bytes(), nil +} + +const lexSpecTemplate = ` +type lexSpec struct { + pop [][]bool + push [][]ModeID + modeNames []string + initialStates []StateID + acceptances [][]ModeKindID + kindIDs [][]KindID + kindNames []string + initialModeID ModeID + modeIDNil ModeID + modeKindIDNil ModeKindID + stateIDNil StateID + + rowNums [][]int + rowDisplacements [][]int + bounds [][]int + entries [][]StateID + originalColCounts []int +} + +func NewLexSpec() *lexSpec { + return &lexSpec{ + pop: {{ genPopTable }}, + push: {{ genPushTable }}, + modeNames: {{ genModeNameTable }}, + initialStates: {{ genInitialStateTable }}, + acceptances: {{ genAcceptTable }}, + kindIDs: {{ genKindIDTable }}, + kindNames: {{ genKindNameTable }}, + initialModeID: {{ .initialModeID }}, + modeIDNil: {{ .modeIDNil }}, + modeKindIDNil: {{ .modeKindIDNil }}, + stateIDNil: {{ .stateIDNil }}, + + rowNums: {{ genRowNums }}, + rowDisplacements: {{ genRowDisplacements }}, + bounds: {{ genBounds }}, + entries: {{ genEntries }}, + originalColCounts: {{ genOriginalColCounts }}, + } +} + +func (s *lexSpec) InitialMode() ModeID { + return s.initialModeID +} + +func (s *lexSpec) Pop(mode ModeID, modeKind ModeKindID) bool { + return s.pop[mode][modeKind] +} + +func (s *lexSpec) Push(mode ModeID, modeKind ModeKindID) (ModeID, bool) { + id := s.push[mode][modeKind] + return id, id != s.modeIDNil +} + +func (s *lexSpec) ModeName(mode ModeID) string { + return s.modeNames[mode] +} + +func (s *lexSpec) InitialState(mode ModeID) StateID { + return s.initialStates[mode] +} + +func (s *lexSpec) NextState(mode ModeID, state StateID, v int) (StateID, bool) { +{{ if eq .compressionLevel 2 -}} + rowNum := s.rowNums[mode][state] + d := s.rowDisplacements[mode][rowNum] + if s.bounds[mode][d+v] != rowNum { + return s.stateIDNil, false + } + return s.entries[mode][d+v], true +{{ else if eq .compressionLevel 1 -}} + rowNum := s.rowNums[mode][state] + colCount := s.originalColCounts[mode] + next := s.entries[mode][rowNum*colCount+v] + if next == s.stateIDNil { + return s.stateIDNil, false + } + return next, true +{{ else -}} + colCount := s.originalColCounts[mode] + next := s.entries[mode][int(state)*colCount+v] + if next == s.stateIDNil { + return s.stateIDNil, false + } + return next, true +{{ end -}} +} + +func (s *lexSpec) Accept(mode ModeID, state StateID) (ModeKindID, bool) { + id := s.acceptances[mode][state] + return id, id != s.modeKindIDNil +} + +func (s *lexSpec) KindIDAndName(mode ModeID, modeKind ModeKindID) (KindID, string) { + id := s.kindIDs[mode][modeKind] + return id, s.kindNames[id] +} +` + +func genTemplateFuncs(lexSpec *spec.LexicalSpec) template.FuncMap { + fns := template.FuncMap{ + "genPopTable": func() string { + var b strings.Builder + fmt.Fprintf(&b, "[][]bool{\n") + for i, s := range lexSpec.Specs { + if i == spec.LexModeIDNil.Int() { + fmt.Fprintf(&b, "nil,\n") + continue + } + + c := 1 + fmt.Fprintf(&b, "{\n") + for _, v := range s.Pop { + fmt.Fprintf(&b, "%v, ", v != 0) + + if c == 20 { + fmt.Fprintf(&b, "\n") + c = 1 + } else { + c++ + } + } + if c > 1 { + fmt.Fprintf(&b, "\n") + } + fmt.Fprintf(&b, "},\n") + } + fmt.Fprintf(&b, "}") + return b.String() + }, + "genPushTable": func() string { + var b strings.Builder + fmt.Fprintf(&b, "[][]ModeID{\n") + for i, s := range lexSpec.Specs { + if i == spec.LexModeIDNil.Int() { + fmt.Fprintf(&b, "nil,\n") + continue + } + + c := 1 + fmt.Fprintf(&b, "{\n") + for _, v := range s.Push { + fmt.Fprintf(&b, "%v,", v) + + if c == 20 { + fmt.Fprintf(&b, "\n") + c = 1 + } else { + c++ + } + } + if c > 1 { + fmt.Fprintf(&b, "\n") + } + fmt.Fprintf(&b, "},\n") + } + fmt.Fprintf(&b, "}") + return b.String() + }, + "genModeNameTable": func() string { + var b strings.Builder + fmt.Fprintf(&b, "[]string{\n") + for i, name := range lexSpec.ModeNames { + if i == spec.LexModeIDNil.Int() { + fmt.Fprintf(&b, "ModeNameNil,\n") + continue + } + fmt.Fprintf(&b, "ModeName%v,\n", lexical.SnakeCaseToUpperCamelCase(name.String())) + } + fmt.Fprintf(&b, "}") + return b.String() + }, + "genInitialStateTable": func() string { + var b strings.Builder + fmt.Fprintf(&b, "[]StateID{\n") + for i, s := range lexSpec.Specs { + if i == spec.LexModeIDNil.Int() { + fmt.Fprintf(&b, "%v,\n", spec.StateIDNil) + continue + } + + fmt.Fprintf(&b, "%v,\n", s.DFA.InitialStateID) + } + fmt.Fprintf(&b, "}") + return b.String() + }, + "genAcceptTable": func() string { + var b strings.Builder + fmt.Fprintf(&b, "[][]ModeKindID{\n") + for i, s := range lexSpec.Specs { + if i == spec.LexModeIDNil.Int() { + fmt.Fprintf(&b, "nil,\n") + continue + } + + c := 1 + fmt.Fprintf(&b, "{\n") + for _, v := range s.DFA.AcceptingStates { + fmt.Fprintf(&b, "%v,", v) + + if c == 20 { + fmt.Fprintf(&b, "\n") + c = 1 + } else { + c++ + } + } + if c > 1 { + fmt.Fprintf(&b, "\n") + } + fmt.Fprintf(&b, "},\n") + } + fmt.Fprintf(&b, "}") + return b.String() + }, + "genKindIDTable": func() string { + var b strings.Builder + fmt.Fprintf(&b, "[][]KindID{\n") + for i, ids := range lexSpec.KindIDs { + if i == spec.LexModeIDNil.Int() { + fmt.Fprintf(&b, "nil,\n") + continue + } + + fmt.Fprintf(&b, "{\n") + for j, id := range ids { + if j == spec.LexModeKindIDNil.Int() { + fmt.Fprintf(&b, "KindIDNil,\n") + continue + } + fmt.Fprintf(&b, "KindID%v,\n", lexical.SnakeCaseToUpperCamelCase(string(lexSpec.KindNames[id].String()))) + } + fmt.Fprintf(&b, "},\n") + } + fmt.Fprintf(&b, "}") + return b.String() + }, + "genKindNameTable": func() string { + var b strings.Builder + fmt.Fprintf(&b, "[]string{\n") + for i, name := range lexSpec.KindNames { + if i == spec.LexKindIDNil.Int() { + fmt.Fprintf(&b, "KindNameNil,\n") + continue + } + fmt.Fprintf(&b, "KindName%v,\n", lexical.SnakeCaseToUpperCamelCase(name.String())) + } + fmt.Fprintf(&b, "}") + return b.String() + }, + } + + switch lexSpec.CompressionLevel { + case 2: + fns["genRowNums"] = func() string { + var b strings.Builder + fmt.Fprintf(&b, "[][]int{\n") + for i, s := range lexSpec.Specs { + if i == spec.LexModeIDNil.Int() { + fmt.Fprintf(&b, "nil,\n") + continue + } + + c := 1 + fmt.Fprintf(&b, "{\n") + for _, v := range s.DFA.Transition.RowNums { + fmt.Fprintf(&b, "%v,", v) + + if c == 20 { + fmt.Fprintf(&b, "\n") + c = 1 + } else { + c++ + } + } + if c > 1 { + fmt.Fprintf(&b, "\n") + } + fmt.Fprintf(&b, "},\n") + } + fmt.Fprintf(&b, "}") + return b.String() + } + + fns["genRowDisplacements"] = func() string { + var b strings.Builder + fmt.Fprintf(&b, "[][]int{\n") + for i, s := range lexSpec.Specs { + if i == spec.LexModeIDNil.Int() { + fmt.Fprintf(&b, "nil,\n") + continue + } + + c := 1 + fmt.Fprintf(&b, "{\n") + for _, d := range s.DFA.Transition.UniqueEntries.RowDisplacement { + fmt.Fprintf(&b, "%v,", d) + + if c == 20 { + fmt.Fprintf(&b, "\n") + c = 1 + } else { + c++ + } + } + if c > 1 { + fmt.Fprintf(&b, "\n") + } + fmt.Fprintf(&b, "},\n") + } + fmt.Fprintf(&b, "}") + return b.String() + } + + fns["genBounds"] = func() string { + var b strings.Builder + fmt.Fprintf(&b, "[][]int{\n") + for i, s := range lexSpec.Specs { + if i == spec.LexModeIDNil.Int() { + fmt.Fprintf(&b, "nil,\n") + continue + } + + c := 1 + fmt.Fprintf(&b, "{\n") + for _, v := range s.DFA.Transition.UniqueEntries.Bounds { + fmt.Fprintf(&b, "%v,", v) + + if c == 20 { + fmt.Fprintf(&b, "\n") + c = 1 + } else { + c++ + } + } + if c > 1 { + fmt.Fprintf(&b, "\n") + } + fmt.Fprintf(&b, "},\n") + } + fmt.Fprintf(&b, "}") + return b.String() + } + + fns["genEntries"] = func() string { + var b strings.Builder + fmt.Fprintf(&b, "[][]StateID{\n") + for i, s := range lexSpec.Specs { + if i == spec.LexModeIDNil.Int() { + fmt.Fprintf(&b, "nil,\n") + continue + } + + c := 1 + fmt.Fprintf(&b, "{\n") + for _, v := range s.DFA.Transition.UniqueEntries.Entries { + fmt.Fprintf(&b, "%v,", v) + + if c == 20 { + fmt.Fprintf(&b, "\n") + c = 1 + } else { + c++ + } + } + if c > 1 { + fmt.Fprintf(&b, "\n") + } + fmt.Fprintf(&b, "},\n") + } + fmt.Fprintf(&b, "}") + return b.String() + } + + fns["genOriginalColCounts"] = func() string { + return "nil" + } + case 1: + fns["genRowNums"] = func() string { + var b strings.Builder + fmt.Fprintf(&b, "[][]int{\n") + for i, s := range lexSpec.Specs { + if i == spec.LexModeIDNil.Int() { + fmt.Fprintf(&b, "nil,\n") + continue + } + + c := 1 + fmt.Fprintf(&b, "{\n") + for _, v := range s.DFA.Transition.RowNums { + fmt.Fprintf(&b, "%v,", v) + + if c == 20 { + fmt.Fprintf(&b, "\n") + c = 1 + } else { + c++ + } + } + if c > 1 { + fmt.Fprintf(&b, "\n") + } + fmt.Fprintf(&b, "},\n") + } + fmt.Fprintf(&b, "}") + return b.String() + } + + fns["genRowDisplacements"] = func() string { + return "nil" + } + + fns["genBounds"] = func() string { + return "nil" + } + + fns["genEntries"] = func() string { + var b strings.Builder + fmt.Fprintf(&b, "[][]StateID{\n") + for i, s := range lexSpec.Specs { + if i == spec.LexModeIDNil.Int() { + fmt.Fprintf(&b, "nil,\n") + continue + } + + c := 1 + fmt.Fprintf(&b, "{\n") + for _, v := range s.DFA.Transition.UncompressedUniqueEntries { + fmt.Fprintf(&b, "%v,", v) + + if c == 20 { + fmt.Fprintf(&b, "\n") + c = 1 + } else { + c++ + } + } + if c > 1 { + fmt.Fprintf(&b, "\n") + } + fmt.Fprintf(&b, "},\n") + } + fmt.Fprintf(&b, "}") + return b.String() + } + + fns["genOriginalColCounts"] = func() string { + var b strings.Builder + fmt.Fprintf(&b, "[]int{\n") + for i, s := range lexSpec.Specs { + if i == spec.LexModeIDNil.Int() { + fmt.Fprintf(&b, "0,\n") + continue + } + + fmt.Fprintf(&b, "%v,\n", s.DFA.Transition.OriginalColCount) + } + fmt.Fprintf(&b, "}") + return b.String() + } + default: + fns["genRowNums"] = func() string { + return "nil" + } + + fns["genRowDisplacements"] = func() string { + return "nil" + } + + fns["genBounds"] = func() string { + return "nil" + } + + fns["genEntries"] = func() string { + var b strings.Builder + fmt.Fprintf(&b, "[][]StateID{\n") + for i, s := range lexSpec.Specs { + if i == spec.LexModeIDNil.Int() { + fmt.Fprintf(&b, "nil,\n") + continue + } + + c := 1 + fmt.Fprintf(&b, "{\n") + for _, v := range s.DFA.UncompressedTransition { + fmt.Fprintf(&b, "%v,", v) + + if c == 20 { + fmt.Fprintf(&b, "\n") + c = 1 + } else { + c++ + } + } + if c > 1 { + fmt.Fprintf(&b, "\n") + } + fmt.Fprintf(&b, "},\n") + } + fmt.Fprintf(&b, "}") + return b.String() + } + + fns["genOriginalColCounts"] = func() string { + var b strings.Builder + fmt.Fprintf(&b, "[]int{\n") + for i, s := range lexSpec.Specs { + if i == spec.LexModeIDNil.Int() { + fmt.Fprintf(&b, "0,\n") + continue + } + + fmt.Fprintf(&b, "%v,\n", s.DFA.ColCount) + } + fmt.Fprintf(&b, "}") + return b.String() + } + } + + return fns +} diff --git a/driver/conflict_test.go b/driver/parser/conflict_test.go index 3b0c5fb..21b829a 100644 --- a/driver/conflict_test.go +++ b/driver/parser/conflict_test.go @@ -1,11 +1,11 @@ -package driver +package parser import ( "strings" "testing" "github.com/nihei9/vartan/grammar" - spec "github.com/nihei9/vartan/spec/grammar" + "github.com/nihei9/vartan/spec/grammar/parser" ) func TestParserWithConflicts(t *testing.T) { @@ -486,7 +486,7 @@ assign: '='; for _, tt := range tests { t.Run(tt.caption, func(t *testing.T) { - ast, err := spec.Parse(strings.NewReader(tt.specSrc)) + ast, err := parser.Parse(strings.NewReader(tt.specSrc)) if err != nil { t.Fatal(err) } @@ -494,12 +494,7 @@ assign: '='; b := grammar.GrammarBuilder{ AST: ast, } - g, err := b.Build() - if err != nil { - t.Fatal(err) - } - - cg, _, err := grammar.Compile(g) + cg, _, err := b.Build() if err != nil { t.Fatal(err) } diff --git a/driver/lac_test.go b/driver/parser/lac_test.go index 2274583..14bd2cf 100644 --- a/driver/lac_test.go +++ b/driver/parser/lac_test.go @@ -1,11 +1,11 @@ -package driver +package parser import ( "strings" "testing" "github.com/nihei9/vartan/grammar" - spec "github.com/nihei9/vartan/spec/grammar" + "github.com/nihei9/vartan/spec/grammar/parser" ) func TestParserWithLAC(t *testing.T) { @@ -43,7 +43,7 @@ d: 'd'; "miss", } - ast, err := spec.Parse(strings.NewReader(specSrc)) + ast, err := parser.Parse(strings.NewReader(specSrc)) if err != nil { t.Fatal(err) } @@ -51,12 +51,7 @@ d: 'd'; b := grammar.GrammarBuilder{ AST: ast, } - g, err := b.Build() - if err != nil { - t.Fatal(err) - } - - gram, _, err := grammar.Compile(g) + gram, _, err := b.Build() if err != nil { t.Fatal(err) } diff --git a/driver/parser.go b/driver/parser/parser.go index a152c3d..05f7d38 100644 --- a/driver/parser.go +++ b/driver/parser/parser.go @@ -1,4 +1,4 @@ -package driver +package parser import ( "fmt" diff --git a/driver/parser_test.go b/driver/parser/parser_test.go index da4f714..b1b9e4f 100644 --- a/driver/parser_test.go +++ b/driver/parser/parser_test.go @@ -1,4 +1,4 @@ -package driver +package parser import ( "fmt" @@ -6,7 +6,7 @@ import ( "testing" "github.com/nihei9/vartan/grammar" - spec "github.com/nihei9/vartan/spec/grammar" + "github.com/nihei9/vartan/spec/grammar/parser" ) func termNode(kind string, text string, children ...*Node) *Node { @@ -765,7 +765,7 @@ bar: 'bar'; for i, tt := range tests { t.Run(fmt.Sprintf("#%v", i), func(t *testing.T) { - ast, err := spec.Parse(strings.NewReader(tt.specSrc)) + ast, err := parser.Parse(strings.NewReader(tt.specSrc)) if err != nil { t.Fatal(err) } @@ -773,12 +773,7 @@ bar: 'bar'; b := grammar.GrammarBuilder{ AST: ast, } - g, err := b.Build() - if err != nil { - t.Fatal(err) - } - - cg, _, err := grammar.Compile(g) + cg, _, err := b.Build() if err != nil { t.Fatal(err) } diff --git a/driver/semantic_action.go b/driver/parser/semantic_action.go index 7e5a773..f709d4f 100644 --- a/driver/semantic_action.go +++ b/driver/parser/semantic_action.go @@ -1,4 +1,4 @@ -package driver +package parser import ( "encoding/json" diff --git a/driver/semantic_action_test.go b/driver/parser/semantic_action_test.go index 3d5f711..c98a12f 100644 --- a/driver/semantic_action_test.go +++ b/driver/parser/semantic_action_test.go @@ -1,4 +1,4 @@ -package driver +package parser import ( "fmt" @@ -7,6 +7,7 @@ import ( "github.com/nihei9/vartan/grammar" spec "github.com/nihei9/vartan/spec/grammar" + "github.com/nihei9/vartan/spec/grammar/parser" ) type testSemAct struct { @@ -15,7 +16,7 @@ type testSemAct struct { } func (a *testSemAct) Shift(tok VToken, recovered bool) { - t := a.gram.ParsingTable.Terminals[tok.TerminalID()] + t := a.gram.Syntactic.Terminals[tok.TerminalID()] if recovered { a.actLog = append(a.actLog, fmt.Sprintf("shift/%v/recovered", t)) } else { @@ -24,8 +25,8 @@ func (a *testSemAct) Shift(tok VToken, recovered bool) { } func (a *testSemAct) Reduce(prodNum int, recovered bool) { - lhsSym := a.gram.ParsingTable.LHSSymbols[prodNum] - lhsText := a.gram.ParsingTable.NonTerminals[lhsSym] + lhsSym := a.gram.Syntactic.LHSSymbols[prodNum] + lhsText := a.gram.Syntactic.NonTerminals[lhsSym] if recovered { a.actLog = append(a.actLog, fmt.Sprintf("reduce/%v/recovered", lhsText)) } else { @@ -181,7 +182,7 @@ char } for _, tt := range tests { t.Run(tt.caption, func(t *testing.T) { - ast, err := spec.Parse(strings.NewReader(tt.specSrc)) + ast, err := parser.Parse(strings.NewReader(tt.specSrc)) if err != nil { t.Fatal(err) } @@ -189,12 +190,7 @@ char b := grammar.GrammarBuilder{ AST: ast, } - g, err := b.Build() - if err != nil { - t.Fatal(err) - } - - gram, _, err := grammar.Compile(g) + gram, _, err := b.Build() if err != nil { t.Fatal(err) } diff --git a/driver/spec.go b/driver/parser/spec.go index cf3c7b0..1d57bae 100644 --- a/driver/spec.go +++ b/driver/parser/spec.go @@ -1,4 +1,4 @@ -package driver +package parser import spec "github.com/nihei9/vartan/spec/grammar" @@ -13,59 +13,59 @@ func NewGrammar(g *spec.CompiledGrammar) *grammarImpl { } func (g *grammarImpl) InitialState() int { - return g.g.ParsingTable.InitialState + return g.g.Syntactic.InitialState } func (g *grammarImpl) StartProduction() int { - return g.g.ParsingTable.StartProduction + return g.g.Syntactic.StartProduction } func (g *grammarImpl) RecoverProduction(prod int) bool { - return g.g.ParsingTable.RecoverProductions[prod] != 0 + return g.g.Syntactic.RecoverProductions[prod] != 0 } func (g *grammarImpl) Action(state int, terminal int) int { - return g.g.ParsingTable.Action[state*g.g.ParsingTable.TerminalCount+terminal] + return g.g.Syntactic.Action[state*g.g.Syntactic.TerminalCount+terminal] } func (g *grammarImpl) GoTo(state int, lhs int) int { - return g.g.ParsingTable.GoTo[state*g.g.ParsingTable.NonTerminalCount+lhs] + return g.g.Syntactic.GoTo[state*g.g.Syntactic.NonTerminalCount+lhs] } func (g *grammarImpl) AlternativeSymbolCount(prod int) int { - return g.g.ParsingTable.AlternativeSymbolCounts[prod] + return g.g.Syntactic.AlternativeSymbolCounts[prod] } func (g *grammarImpl) TerminalCount() int { - return g.g.ParsingTable.TerminalCount + return g.g.Syntactic.TerminalCount } func (g *grammarImpl) SkipTerminal(terminal int) bool { - return g.g.ParsingTable.TerminalSkip[terminal] == 1 + return g.g.Syntactic.TerminalSkip[terminal] == 1 } func (g *grammarImpl) ErrorTrapperState(state int) bool { - return g.g.ParsingTable.ErrorTrapperStates[state] != 0 + return g.g.Syntactic.ErrorTrapperStates[state] != 0 } func (g *grammarImpl) NonTerminal(nonTerminal int) string { - return g.g.ParsingTable.NonTerminals[nonTerminal] + return g.g.Syntactic.NonTerminals[nonTerminal] } func (g *grammarImpl) LHS(prod int) int { - return g.g.ParsingTable.LHSSymbols[prod] + return g.g.Syntactic.LHSSymbols[prod] } func (g *grammarImpl) EOF() int { - return g.g.ParsingTable.EOFSymbol + return g.g.Syntactic.EOFSymbol } func (g *grammarImpl) Error() int { - return g.g.ParsingTable.ErrorSymbol + return g.g.Syntactic.ErrorSymbol } func (g *grammarImpl) Terminal(terminal int) string { - return g.g.ParsingTable.Terminals[terminal] + return g.g.Syntactic.Terminals[terminal] } func (g *grammarImpl) ASTAction(prod int) []int { diff --git a/driver/syntax_error_test.go b/driver/parser/syntax_error_test.go index 683e355..71175be 100644 --- a/driver/syntax_error_test.go +++ b/driver/parser/syntax_error_test.go @@ -1,4 +1,4 @@ -package driver +package parser import ( "fmt" @@ -7,7 +7,7 @@ import ( "testing" "github.com/nihei9/vartan/grammar" - spec "github.com/nihei9/vartan/spec/grammar" + "github.com/nihei9/vartan/spec/grammar/parser" ) func TestParserWithSyntaxErrors(t *testing.T) { @@ -123,7 +123,7 @@ c } for i, tt := range tests { t.Run(fmt.Sprintf("#%v", i), func(t *testing.T) { - ast, err := spec.Parse(strings.NewReader(tt.specSrc)) + ast, err := parser.Parse(strings.NewReader(tt.specSrc)) if err != nil { t.Fatal(err) } @@ -131,12 +131,7 @@ c b := grammar.GrammarBuilder{ AST: ast, } - g, err := b.Build() - if err != nil { - t.Fatal(err) - } - - gram, _, err := grammar.Compile(g) + gram, _, err := b.Build() if err != nil { t.Fatal(err) } @@ -253,7 +248,7 @@ foo } for i, tt := range tests { t.Run(fmt.Sprintf("#%v", i), func(t *testing.T) { - ast, err := spec.Parse(strings.NewReader(tt.specSrc)) + ast, err := parser.Parse(strings.NewReader(tt.specSrc)) if err != nil { t.Fatal(err) } @@ -261,12 +256,7 @@ foo b := grammar.GrammarBuilder{ AST: ast, } - g, err := b.Build() - if err != nil { - t.Fatal(err) - } - - gram, _, err := grammar.Compile(g) + gram, _, err := b.Build() if err != nil { t.Fatal(err) } diff --git a/driver/template.go b/driver/parser/template.go index 321b2dd..96eb71f 100644 --- a/driver/template.go +++ b/driver/parser/template.go @@ -1,4 +1,4 @@ -package driver +package parser import ( "bytes" @@ -49,12 +49,12 @@ func GenParser(cgram *spec.CompiledGrammar, pkgName string) ([]byte, error) { var b strings.Builder err = t.Execute(&b, map[string]interface{}{ - "initialState": cgram.ParsingTable.InitialState, - "startProduction": cgram.ParsingTable.StartProduction, - "terminalCount": cgram.ParsingTable.TerminalCount, - "nonTerminalCount": cgram.ParsingTable.NonTerminalCount, - "eofSymbol": cgram.ParsingTable.EOFSymbol, - "errorSymbol": cgram.ParsingTable.ErrorSymbol, + "initialState": cgram.Syntactic.InitialState, + "startProduction": cgram.Syntactic.StartProduction, + "terminalCount": cgram.Syntactic.TerminalCount, + "nonTerminalCount": cgram.Syntactic.NonTerminalCount, + "eofSymbol": cgram.Syntactic.EOFSymbol, + "errorSymbol": cgram.Syntactic.ErrorSymbol, }) if err != nil { return nil, err @@ -233,7 +233,7 @@ func genGrammarTemplateFuncs(cgram *spec.CompiledGrammar) template.FuncMap { var b strings.Builder fmt.Fprintf(&b, "[]int{\n") c := 1 - for _, v := range cgram.ParsingTable.RecoverProductions { + for _, v := range cgram.Syntactic.RecoverProductions { fmt.Fprintf(&b, "%v, ", v) if c == 20 { fmt.Fprintf(&b, "\n") @@ -252,7 +252,7 @@ func genGrammarTemplateFuncs(cgram *spec.CompiledGrammar) template.FuncMap { var b strings.Builder fmt.Fprintf(&b, "[]int{\n") c := 1 - for _, v := range cgram.ParsingTable.Action { + for _, v := range cgram.Syntactic.Action { fmt.Fprintf(&b, "%v, ", v) if c == 20 { fmt.Fprintf(&b, "\n") @@ -271,7 +271,7 @@ func genGrammarTemplateFuncs(cgram *spec.CompiledGrammar) template.FuncMap { var b strings.Builder fmt.Fprintf(&b, "[]int{\n") c := 1 - for _, v := range cgram.ParsingTable.GoTo { + for _, v := range cgram.Syntactic.GoTo { fmt.Fprintf(&b, "%v, ", v) if c == 20 { fmt.Fprintf(&b, "\n") @@ -290,7 +290,7 @@ func genGrammarTemplateFuncs(cgram *spec.CompiledGrammar) template.FuncMap { var b strings.Builder fmt.Fprintf(&b, "[]int{\n") c := 1 - for _, v := range cgram.ParsingTable.AlternativeSymbolCounts { + for _, v := range cgram.Syntactic.AlternativeSymbolCounts { fmt.Fprintf(&b, "%v, ", v) if c == 20 { fmt.Fprintf(&b, "\n") @@ -309,7 +309,7 @@ func genGrammarTemplateFuncs(cgram *spec.CompiledGrammar) template.FuncMap { var b strings.Builder fmt.Fprintf(&b, "[]int{\n") c := 1 - for _, v := range cgram.ParsingTable.ErrorTrapperStates { + for _, v := range cgram.Syntactic.ErrorTrapperStates { fmt.Fprintf(&b, "%v, ", v) if c == 20 { fmt.Fprintf(&b, "\n") @@ -327,7 +327,7 @@ func genGrammarTemplateFuncs(cgram *spec.CompiledGrammar) template.FuncMap { "genNonTerminals": func() string { var b strings.Builder fmt.Fprintf(&b, "[]string{\n") - for _, v := range cgram.ParsingTable.NonTerminals { + for _, v := range cgram.Syntactic.NonTerminals { fmt.Fprintf(&b, "%v,\n", strconv.Quote(v)) } fmt.Fprintf(&b, "}") @@ -337,7 +337,7 @@ func genGrammarTemplateFuncs(cgram *spec.CompiledGrammar) template.FuncMap { var b strings.Builder fmt.Fprintf(&b, "[]int{\n") c := 1 - for _, v := range cgram.ParsingTable.LHSSymbols { + for _, v := range cgram.Syntactic.LHSSymbols { fmt.Fprintf(&b, "%v, ", v) if c == 20 { fmt.Fprintf(&b, "\n") @@ -355,7 +355,7 @@ func genGrammarTemplateFuncs(cgram *spec.CompiledGrammar) template.FuncMap { "genTerminals": func() string { var b strings.Builder fmt.Fprintf(&b, "[]string{\n") - for _, v := range cgram.ParsingTable.Terminals { + for _, v := range cgram.Syntactic.Terminals { fmt.Fprintf(&b, "%v,\n", strconv.Quote(v)) } fmt.Fprintf(&b, "}") @@ -365,7 +365,7 @@ func genGrammarTemplateFuncs(cgram *spec.CompiledGrammar) template.FuncMap { var b strings.Builder fmt.Fprintf(&b, "[]int{\n") c := 1 - for _, v := range cgram.ParsingTable.TerminalSkip { + for _, v := range cgram.Syntactic.TerminalSkip { fmt.Fprintf(&b, "%v, ", v) if c == 20 { fmt.Fprintf(&b, "\n") @@ -473,7 +473,7 @@ func genLexerTemplateFuncs(cgram *spec.CompiledGrammar) template.FuncMap { var b strings.Builder fmt.Fprintf(&b, "[]int{\n") c := 1 - for _, v := range cgram.LexicalSpecification.Maleeni.KindToTerminal { + for _, v := range cgram.Syntactic.KindToTerminal { fmt.Fprintf(&b, "%v, ", v) if c == 20 { fmt.Fprintf(&b, "\n") diff --git a/driver/token_stream.go b/driver/parser/token_stream.go index eaf56c6..0bc9e32 100644 --- a/driver/token_stream.go +++ b/driver/parser/token_stream.go @@ -1,15 +1,15 @@ -package driver +package parser import ( "io" - mldriver "github.com/nihei9/maleeni/driver" + "github.com/nihei9/vartan/driver/lexer" spec "github.com/nihei9/vartan/spec/grammar" ) type vToken struct { terminalID int - tok *mldriver.Token + tok *lexer.Token } func (t *vToken) TerminalID() int { @@ -33,19 +33,19 @@ func (t *vToken) Position() (int, int) { } type tokenStream struct { - lex *mldriver.Lexer + lex *lexer.Lexer kindToTerminal []int } func NewTokenStream(g *spec.CompiledGrammar, src io.Reader) (TokenStream, error) { - lex, err := mldriver.NewLexer(mldriver.NewLexSpec(g.LexicalSpecification.Maleeni.Spec), src) + lex, err := lexer.NewLexer(lexer.NewLexSpec(g.Lexical), src) if err != nil { return nil, err } return &tokenStream{ lex: lex, - kindToTerminal: g.LexicalSpecification.Maleeni.KindToTerminal, + kindToTerminal: g.Syntactic.KindToTerminal, }, nil } @@ -2,10 +2,7 @@ module github.com/nihei9/vartan go 1.19 -require ( - github.com/nihei9/maleeni v0.6.1 - github.com/spf13/cobra v1.4.0 -) +require github.com/spf13/cobra v1.4.0 require ( github.com/inconshreveable/mousetrap v1.0.0 // indirect @@ -1,291 +1,10 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= -github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/nihei9/maleeni v0.6.1 h1:F/FCmzj79JxC2phpTxRme3ghrQILwhmqxHPcYFE4+eY= -github.com/nihei9/maleeni v0.6.1/go.mod h1:d5x5jHHuema6IUi+aDPczMZQ4AlNokcKbEgB5T+70dI= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q= github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/grammar/first.go b/grammar/first.go index 72de282..923f8ed 100644 --- a/grammar/first.go +++ b/grammar/first.go @@ -1,20 +1,24 @@ package grammar -import "fmt" +import ( + "fmt" + + "github.com/nihei9/vartan/grammar/symbol" +) type firstEntry struct { - symbols map[symbol]struct{} + symbols map[symbol.Symbol]struct{} empty bool } func newFirstEntry() *firstEntry { return &firstEntry{ - symbols: map[symbol]struct{}{}, + symbols: map[symbol.Symbol]struct{}{}, empty: false, } } -func (e *firstEntry) add(sym symbol) bool { +func (e *firstEntry) add(sym symbol.Symbol) bool { if _, ok := e.symbols[sym]; ok { return false } @@ -45,12 +49,12 @@ func (e *firstEntry) mergeExceptEmpty(target *firstEntry) bool { } type firstSet struct { - set map[symbol]*firstEntry + set map[symbol.Symbol]*firstEntry } func newFirstSet(prods *productionSet) *firstSet { fst := &firstSet{ - set: map[symbol]*firstEntry{}, + set: map[symbol.Symbol]*firstEntry{}, } for _, prod := range prods.getAllProductions() { if _, ok := fst.set[prod.lhs]; ok { @@ -69,7 +73,7 @@ func (fst *firstSet) find(prod *production, head int) (*firstEntry, error) { return entry, nil } for _, sym := range prod.rhs[head:] { - if sym.isTerminal() { + if sym.IsTerminal() { entry.add(sym) return entry, nil } @@ -89,7 +93,7 @@ func (fst *firstSet) find(prod *production, head int) (*firstEntry, error) { return entry, nil } -func (fst *firstSet) findBySymbol(sym symbol) *firstEntry { +func (fst *firstSet) findBySymbol(sym symbol.Symbol) *firstEntry { return fst.set[sym] } @@ -130,7 +134,7 @@ func genProdFirstEntry(cc *firstComContext, acc *firstEntry, prod *production) ( } for _, sym := range prod.rhs { - if sym.isTerminal() { + if sym.IsTerminal() { return acc.add(sym), nil } diff --git a/grammar/first_test.go b/grammar/first_test.go index 21ee4df..1eff309 100644 --- a/grammar/first_test.go +++ b/grammar/first_test.go @@ -4,7 +4,8 @@ import ( "strings" "testing" - spec "github.com/nihei9/vartan/spec/grammar" + "github.com/nihei9/vartan/grammar/symbol" + "github.com/nihei9/vartan/spec/grammar/parser" ) type first struct { @@ -137,7 +138,7 @@ bar: "bar"; fst, gram := genActualFirst(t, tt.src) for _, ttFirst := range tt.first { - lhsSym, ok := gram.symbolTable.toSymbol(ttFirst.lhs) + lhsSym, ok := gram.symbolTable.ToSymbol(ttFirst.lhs) if !ok { t.Fatalf("a symbol was not found; symbol: %v", ttFirst.lhs) } @@ -161,14 +162,14 @@ bar: "bar"; } func genActualFirst(t *testing.T, src string) (*firstSet, *Grammar) { - ast, err := spec.Parse(strings.NewReader(src)) + ast, err := parser.Parse(strings.NewReader(src)) if err != nil { t.Fatal(err) } b := GrammarBuilder{ AST: ast, } - gram, err := b.Build() + gram, err := b.build() if err != nil { t.Fatal(err) } @@ -183,7 +184,7 @@ func genActualFirst(t *testing.T, src string) (*firstSet, *Grammar) { return fst, gram } -func genExpectedFirstEntry(t *testing.T, symbols []string, empty bool, symTab *symbolTableReader) *firstEntry { +func genExpectedFirstEntry(t *testing.T, symbols []string, empty bool, symTab *symbol.SymbolTableReader) *firstEntry { t.Helper() entry := newFirstEntry() @@ -191,7 +192,7 @@ func genExpectedFirstEntry(t *testing.T, symbols []string, empty bool, symTab *s entry.addEmpty() } for _, sym := range symbols { - symSym, ok := symTab.toSymbol(sym) + symSym, ok := symTab.ToSymbol(sym) if !ok { t.Fatalf("a symbol was not found; symbol: %v", sym) } diff --git a/grammar/grammar.go b/grammar/grammar.go index 50272e0..1e05289 100644 --- a/grammar/grammar.go +++ b/grammar/grammar.go @@ -5,10 +5,11 @@ import ( "io" "strings" - mlcompiler "github.com/nihei9/maleeni/compiler" - mlspec "github.com/nihei9/maleeni/spec" verr "github.com/nihei9/vartan/error" + "github.com/nihei9/vartan/grammar/lexical" + "github.com/nihei9/vartan/grammar/symbol" spec "github.com/nihei9/vartan/spec/grammar" + "github.com/nihei9/vartan/spec/grammar/parser" ) type astActionEntry struct { @@ -33,8 +34,8 @@ const ( // We use the priority of the production to resolve shift/reduce conflicts. type precAndAssoc struct { // termPrec and termAssoc represent the precedence of the terminal symbols. - termPrec map[symbolNum]int - termAssoc map[symbolNum]assocType + termPrec map[symbol.SymbolNum]int + termAssoc map[symbol.SymbolNum]assocType // prodPrec and prodAssoc represent the precedence and the associativities of the production. // These values are inherited from the right-most terminal symbols in the RHS of the productions. @@ -42,7 +43,7 @@ type precAndAssoc struct { prodAssoc map[productionNum]assocType } -func (pa *precAndAssoc) terminalPrecedence(sym symbolNum) int { +func (pa *precAndAssoc) terminalPrecedence(sym symbol.SymbolNum) int { prec, ok := pa.termPrec[sym] if !ok { return precNil @@ -51,7 +52,7 @@ func (pa *precAndAssoc) terminalPrecedence(sym symbolNum) int { return prec } -func (pa *precAndAssoc) terminalAssociativity(sym symbolNum) assocType { +func (pa *precAndAssoc) terminalAssociativity(sym symbol.SymbolNum) assocType { assoc, ok := pa.termAssoc[sym] if !ok { return assocTypeNil @@ -82,12 +83,12 @@ const reservedSymbolNameError = "error" type Grammar struct { name string - lexSpec *mlspec.LexSpec - skipLexKinds []mlspec.LexKindName + lexSpec *lexical.LexSpec + skipSymbols []symbol.Symbol productionSet *productionSet - augmentedStartSymbol symbol - errorSymbol symbol - symbolTable *symbolTableReader + augmentedStartSymbol symbol.Symbol + errorSymbol symbol.Symbol + symbolTable *symbol.SymbolTableReader astActions map[productionID][]*astActionEntry precAndAssoc *precAndAssoc @@ -95,13 +96,34 @@ type Grammar struct { recoverProductions map[productionID]struct{} } +type buildConfig struct { + isReportingEnabled bool +} + +type BuildOption func(config *buildConfig) + +func EnableReporting() BuildOption { + return func(config *buildConfig) { + config.isReportingEnabled = true + } +} + type GrammarBuilder struct { - AST *spec.RootNode + AST *parser.RootNode errs verr.SpecErrors } -func (b *GrammarBuilder) Build() (*Grammar, error) { +func (b *GrammarBuilder) Build(opts ...BuildOption) (*spec.CompiledGrammar, *spec.Report, error) { + gram, err := b.build() + if err != nil { + return nil, nil, err + } + + return compile(gram, opts...) +} + +func (b *GrammarBuilder) build() (*Grammar, error) { var specName string { errOccurred := false @@ -143,12 +165,12 @@ func (b *GrammarBuilder) Build() (*Grammar, error) { return nil, err } - lexSpec, err := b.genLexSpec(b.AST) + lexSpec, skip, err := b.genLexSpecAndSkipSymbols(symTab.Reader(), b.AST) if err != nil { return nil, err } - prodsAndActs, err := b.genProductionsAndActions(b.AST, symTab.reader(), ss.errSym, ss.augStartSym, ss.startSym) + prodsAndActs, err := b.genProductionsAndActions(b.AST, symTab.Reader(), ss.errSym, ss.augStartSym, ss.startSym) if err != nil { return nil, err } @@ -156,7 +178,7 @@ func (b *GrammarBuilder) Build() (*Grammar, error) { return nil, b.errs } - pa, err := b.genPrecAndAssoc(symTab.reader(), ss.errSym, prodsAndActs) + pa, err := b.genPrecAndAssoc(symTab.Reader(), ss.errSym, prodsAndActs) if err != nil { return nil, err } @@ -171,20 +193,23 @@ func (b *GrammarBuilder) Build() (*Grammar, error) { // When a terminal symbol that cannot be reached from the start symbol has the skip directive, // the compiler treats its terminal as a used symbol, not unused. - for _, sym := range lexSpec.skip { - s := sym.String() - if _, ok := syms.unusedTerminals[s]; !ok { - prod := syms.usedTerminals[s] - b.errs = append(b.errs, &verr.SpecError{ - Cause: semErrTermCannotBeSkipped, - Detail: s, - Row: prod.Pos.Row, - Col: prod.Pos.Col, - }) - continue - } + { + r := symTab.Reader() + for _, sym := range skip { + s, _ := r.ToText(sym) + if _, ok := syms.unusedTerminals[s]; !ok { + prod := syms.usedTerminals[s] + b.errs = append(b.errs, &verr.SpecError{ + Cause: semErrTermCannotBeSkipped, + Detail: s, + Row: prod.Pos.Row, + Col: prod.Pos.Col, + }) + continue + } - delete(syms.unusedTerminals, s) + delete(syms.unusedTerminals, s) + } } for sym, prod := range syms.unusedProductions { @@ -209,16 +234,14 @@ func (b *GrammarBuilder) Build() (*Grammar, error) { return nil, b.errs } - lexSpec.lexSpec.Name = specName - return &Grammar{ name: specName, - lexSpec: lexSpec.lexSpec, - skipLexKinds: lexSpec.skip, + lexSpec: lexSpec, + skipSymbols: skip, productionSet: prodsAndActs.prods, augmentedStartSymbol: prodsAndActs.augStartSym, errorSymbol: ss.errSym, - symbolTable: symTab.reader(), + symbolTable: symTab.Reader(), astActions: prodsAndActs.astActs, recoverProductions: prodsAndActs.recoverProds, precAndAssoc: pa, @@ -226,14 +249,14 @@ func (b *GrammarBuilder) Build() (*Grammar, error) { } type usedAndUnusedSymbols struct { - unusedProductions map[string]*spec.ProductionNode - unusedTerminals map[string]*spec.ProductionNode - usedTerminals map[string]*spec.ProductionNode + unusedProductions map[string]*parser.ProductionNode + unusedTerminals map[string]*parser.ProductionNode + usedTerminals map[string]*parser.ProductionNode } -func findUsedAndUnusedSymbols(root *spec.RootNode) *usedAndUnusedSymbols { - prods := map[string]*spec.ProductionNode{} - lexProds := map[string]*spec.ProductionNode{} +func findUsedAndUnusedSymbols(root *parser.RootNode) *usedAndUnusedSymbols { + prods := map[string]*parser.ProductionNode{} + lexProds := map[string]*parser.ProductionNode{} mark := map[string]bool{} { for _, p := range root.Productions { @@ -262,9 +285,9 @@ func findUsedAndUnusedSymbols(root *spec.RootNode) *usedAndUnusedSymbols { delete(mark, reservedSymbolNameError) } - usedTerms := make(map[string]*spec.ProductionNode, len(lexProds)) - unusedProds := map[string]*spec.ProductionNode{} - unusedTerms := map[string]*spec.ProductionNode{} + usedTerms := make(map[string]*parser.ProductionNode, len(lexProds)) + unusedProds := map[string]*parser.ProductionNode{} + unusedTerms := map[string]*parser.ProductionNode{} for sym, used := range mark { if p, ok := prods[sym]; ok { if used { @@ -294,7 +317,7 @@ func findUsedAndUnusedSymbols(root *spec.RootNode) *usedAndUnusedSymbols { } } -func markUsedSymbols(mark map[string]bool, marked map[string]bool, prods map[string]*spec.ProductionNode, prod *spec.ProductionNode) { +func markUsedSymbols(mark map[string]bool, marked map[string]bool, prods map[string]*parser.ProductionNode, prod *parser.ProductionNode) { if marked[prod.LHS] { return } @@ -320,7 +343,7 @@ func markUsedSymbols(mark map[string]bool, marked map[string]bool, prods map[str } } -func (b *GrammarBuilder) checkSpellingInconsistenciesOfUserDefinedIDs(root *spec.RootNode) { +func (b *GrammarBuilder) checkSpellingInconsistenciesOfUserDefinedIDs(root *parser.RootNode) { var ids []string { for _, prod := range root.Productions { @@ -344,7 +367,7 @@ func (b *GrammarBuilder) checkSpellingInconsistenciesOfUserDefinedIDs(root *spec } } - duplicated := mlspec.FindSpellingInconsistencies(ids) + duplicated := lexical.FindSpellingInconsistencies(ids) if len(duplicated) == 0 { return } @@ -367,7 +390,7 @@ func (b *GrammarBuilder) checkSpellingInconsistenciesOfUserDefinedIDs(root *spec } } -func collectUserDefinedIDsFromDirective(dir *spec.DirectiveNode) []string { +func collectUserDefinedIDsFromDirective(dir *parser.DirectiveNode) []string { var ids []string for _, param := range dir.Parameters { if param.Group != nil { @@ -386,20 +409,20 @@ func collectUserDefinedIDsFromDirective(dir *spec.DirectiveNode) []string { } type symbols struct { - errSym symbol - augStartSym symbol - startSym symbol + errSym symbol.Symbol + augStartSym symbol.Symbol + startSym symbol.Symbol } -func (b *GrammarBuilder) genSymbolTable(root *spec.RootNode) (*symbolTable, *symbols, error) { - symTab := newSymbolTable() - w := symTab.writer() - r := symTab.reader() +func (b *GrammarBuilder) genSymbolTable(root *parser.RootNode) (*symbol.SymbolTable, *symbols, error) { + symTab := symbol.NewSymbolTable() + w := symTab.Writer() + r := symTab.Reader() // We need to register the reserved symbol before registering others. - var errSym symbol + var errSym symbol.Symbol { - sym, err := w.registerTerminalSymbol(reservedSymbolNameError) + sym, err := w.RegisterTerminalSymbol(reservedSymbolNameError) if err != nil { return nil, nil, err } @@ -407,7 +430,7 @@ func (b *GrammarBuilder) genSymbolTable(root *spec.RootNode) (*symbolTable, *sym } for _, prod := range root.LexProductions { - if sym, exist := r.toSymbol(prod.LHS); exist { + if sym, exist := r.ToSymbol(prod.LHS); exist { if sym == errSym { b.errs = append(b.errs, &verr.SpecError{ Cause: semErrErrSymIsReserved, @@ -426,7 +449,7 @@ func (b *GrammarBuilder) genSymbolTable(root *spec.RootNode) (*symbolTable, *sym continue } - _, err := w.registerTerminalSymbol(prod.LHS) + _, err := w.RegisterTerminalSymbol(prod.LHS) if err != nil { return nil, nil, err } @@ -435,7 +458,7 @@ func (b *GrammarBuilder) genSymbolTable(root *spec.RootNode) (*symbolTable, *sym startProd := root.Productions[0] augStartText := fmt.Sprintf("%s'", startProd.LHS) var err error - augStartSym, err := w.registerStartSymbol(augStartText) + augStartSym, err := w.RegisterStartSymbol(augStartText) if err != nil { return nil, nil, err } @@ -447,7 +470,7 @@ func (b *GrammarBuilder) genSymbolTable(root *spec.RootNode) (*symbolTable, *sym }) } - startSym, err := w.registerNonTerminalSymbol(startProd.LHS) + startSym, err := w.RegisterNonTerminalSymbol(startProd.LHS) if err != nil { return nil, nil, err } @@ -460,11 +483,11 @@ func (b *GrammarBuilder) genSymbolTable(root *spec.RootNode) (*symbolTable, *sym } for _, prod := range root.Productions { - sym, err := w.registerNonTerminalSymbol(prod.LHS) + sym, err := w.RegisterNonTerminalSymbol(prod.LHS) if err != nil { return nil, nil, err } - if sym.isTerminal() { + if sym.IsTerminal() { b.errs = append(b.errs, &verr.SpecError{ Cause: semErrDuplicateName, Detail: prod.LHS, @@ -488,25 +511,21 @@ func (b *GrammarBuilder) genSymbolTable(root *spec.RootNode) (*symbolTable, *sym }, nil } -type lexSpec struct { - lexSpec *mlspec.LexSpec - skip []mlspec.LexKindName -} - -func (b *GrammarBuilder) genLexSpec(root *spec.RootNode) (*lexSpec, error) { - entries := []*mlspec.LexEntry{} - skipKinds := []mlspec.LexKindName{} +func (b *GrammarBuilder) genLexSpecAndSkipSymbols(symTab *symbol.SymbolTableReader, root *parser.RootNode) (*lexical.LexSpec, []symbol.Symbol, error) { + entries := []*lexical.LexEntry{} + skipSyms := []symbol.Symbol{} for _, prod := range root.LexProductions { entry, skip, specErr, err := genLexEntry(prod) if err != nil { - return nil, err + return nil, nil, err } if specErr != nil { b.errs = append(b.errs, specErr) continue } if skip { - skipKinds = append(skipKinds, mlspec.LexKindName(prod.LHS)) + sym, _ := symTab.ToSymbol(prod.LHS) + skipSyms = append(skipSyms, sym) } entries = append(entries, entry) } @@ -524,35 +543,32 @@ func (b *GrammarBuilder) genLexSpec(root *spec.RootNode) (*lexSpec, error) { } checkedFragments[fragment.LHS] = struct{}{} - entries = append(entries, &mlspec.LexEntry{ + entries = append(entries, &lexical.LexEntry{ Fragment: true, - Kind: mlspec.LexKindName(fragment.LHS), - Pattern: mlspec.LexPattern(fragment.RHS), + Kind: spec.LexKindName(fragment.LHS), + Pattern: fragment.RHS, }) } - return &lexSpec{ - lexSpec: &mlspec.LexSpec{ - Entries: entries, - }, - skip: skipKinds, - }, nil + return &lexical.LexSpec{ + Entries: entries, + }, skipSyms, nil } -func genLexEntry(prod *spec.ProductionNode) (*mlspec.LexEntry, bool, *verr.SpecError, error) { +func genLexEntry(prod *parser.ProductionNode) (*lexical.LexEntry, bool, *verr.SpecError, error) { alt := prod.RHS[0] elem := alt.Elements[0] var pattern string if elem.Literally { - pattern = mlspec.EscapePattern(elem.Pattern) + pattern = spec.EscapePattern(elem.Pattern) } else { pattern = elem.Pattern } - var modes []mlspec.LexModeName + var modes []spec.LexModeName var skip bool - var push mlspec.LexModeName + var push spec.LexModeName var pop bool dirConsumed := map[string]struct{}{} for _, dir := range prod.Directives { @@ -585,7 +601,7 @@ func genLexEntry(prod *spec.ProductionNode) (*mlspec.LexEntry, bool, *verr.SpecE Col: param.Pos.Col, }, nil } - modes = append(modes, mlspec.LexModeName(param.ID)) + modes = append(modes, spec.LexModeName(param.ID)) } case "skip": if len(dir.Parameters) > 0 { @@ -606,7 +622,7 @@ func genLexEntry(prod *spec.ProductionNode) (*mlspec.LexEntry, bool, *verr.SpecE Col: dir.Pos.Col, }, nil } - push = mlspec.LexModeName(dir.Parameters[0].ID) + push = spec.LexModeName(dir.Parameters[0].ID) case "pop": if len(dir.Parameters) > 0 { return nil, false, &verr.SpecError{ @@ -636,10 +652,10 @@ func genLexEntry(prod *spec.ProductionNode) (*mlspec.LexEntry, bool, *verr.SpecE }, nil } - return &mlspec.LexEntry{ + return &lexical.LexEntry{ Modes: modes, - Kind: mlspec.LexKindName(prod.LHS), - Pattern: mlspec.LexPattern(pattern), + Kind: spec.LexKindName(prod.LHS), + Pattern: pattern, Push: push, Pop: pop, }, skip, nil, nil @@ -647,15 +663,15 @@ func genLexEntry(prod *spec.ProductionNode) (*mlspec.LexEntry, bool, *verr.SpecE type productionsAndActions struct { prods *productionSet - augStartSym symbol + augStartSym symbol.Symbol astActs map[productionID][]*astActionEntry - prodPrecsTerm map[productionID]symbol + prodPrecsTerm map[productionID]symbol.Symbol prodPrecsOrdSym map[productionID]string - prodPrecPoss map[productionID]*spec.Position + prodPrecPoss map[productionID]*parser.Position recoverProds map[productionID]struct{} } -func (b *GrammarBuilder) genProductionsAndActions(root *spec.RootNode, symTab *symbolTableReader, errSym symbol, augStartSym symbol, startSym symbol) (*productionsAndActions, error) { +func (b *GrammarBuilder) genProductionsAndActions(root *parser.RootNode, symTab *symbol.SymbolTableReader, errSym symbol.Symbol, augStartSym symbol.Symbol, startSym symbol.Symbol) (*productionsAndActions, error) { if len(root.Productions) == 0 { b.errs = append(b.errs, &verr.SpecError{ Cause: semErrNoProduction, @@ -665,12 +681,12 @@ func (b *GrammarBuilder) genProductionsAndActions(root *spec.RootNode, symTab *s prods := newProductionSet() astActs := map[productionID][]*astActionEntry{} - prodPrecsTerm := map[productionID]symbol{} + prodPrecsTerm := map[productionID]symbol.Symbol{} prodPrecsOrdSym := map[productionID]string{} - prodPrecPoss := map[productionID]*spec.Position{} + prodPrecPoss := map[productionID]*parser.Position{} recoverProds := map[productionID]struct{}{} - p, err := newProduction(augStartSym, []symbol{ + p, err := newProduction(augStartSym, []symbol.Symbol{ startSym, }) if err != nil { @@ -680,7 +696,7 @@ func (b *GrammarBuilder) genProductionsAndActions(root *spec.RootNode, symTab *s prods.append(p) for _, prod := range root.Productions { - lhsSym, ok := symTab.toSymbol(prod.LHS) + lhsSym, ok := symTab.ToSymbol(prod.LHS) if !ok { // All symbols are assumed to be pre-detected, so it's a bug if we cannot find them here. return nil, fmt.Errorf("symbol '%v' is undefined", prod.LHS) @@ -698,11 +714,11 @@ func (b *GrammarBuilder) genProductionsAndActions(root *spec.RootNode, symTab *s LOOP_RHS: for _, alt := range prod.RHS { - altSyms := make([]symbol, len(alt.Elements)) + altSyms := make([]symbol.Symbol, len(alt.Elements)) offsets := map[string]int{} ambiguousIDOffsets := map[string]struct{}{} for i, elem := range alt.Elements { - sym, ok := symTab.toSymbol(elem.ID) + sym, ok := symTab.ToSymbol(elem.ID) if !ok { b.errs = append(b.errs, &verr.SpecError{ Cause: semErrUndefinedSym, @@ -724,7 +740,7 @@ func (b *GrammarBuilder) genProductionsAndActions(root *spec.RootNode, symTab *s }) continue LOOP_RHS } - if _, found := symTab.toSymbol(elem.Label.Name); found { + if _, found := symTab.ToSymbol(elem.Label.Name); found { b.errs = append(b.errs, &verr.SpecError{ Cause: semErrInvalidLabel, Detail: elem.Label.Name, @@ -877,12 +893,12 @@ func (b *GrammarBuilder) genProductionsAndActions(root *spec.RootNode, symTab *s }) continue LOOP_RHS } - elemSym, ok := symTab.toSymbol(elem.ID) + elemSym, ok := symTab.ToSymbol(elem.ID) if !ok { // If the symbol was not found, it's a bug. return nil, fmt.Errorf("a symbol corresponding to an ID (%v) was not found", elem.ID) } - if elemSym.isTerminal() { + if elemSym.IsTerminal() { b.errs = append(b.errs, &verr.SpecError{ Cause: semErrDirInvalidParam, Detail: fmt.Sprintf("the expansion symbol cannot be applied to a terminal symbol (%v: %v)", param.ID, elem.ID), @@ -912,7 +928,7 @@ func (b *GrammarBuilder) genProductionsAndActions(root *spec.RootNode, symTab *s param := dir.Parameters[0] switch { case param.ID != "": - sym, ok := symTab.toSymbol(param.ID) + sym, ok := symTab.ToSymbol(param.ID) if !ok { b.errs = append(b.errs, &verr.SpecError{ Cause: semErrDirInvalidParam, @@ -930,7 +946,7 @@ func (b *GrammarBuilder) genProductionsAndActions(root *spec.RootNode, symTab *s Col: param.Pos.Col, }) } - if !sym.isTerminal() { + if !sym.IsTerminal() { b.errs = append(b.errs, &verr.SpecError{ Cause: semErrDirInvalidParam, Detail: fmt.Sprintf("the symbol must be a terminal: %v", param.ID), @@ -980,12 +996,12 @@ func (b *GrammarBuilder) genProductionsAndActions(root *spec.RootNode, symTab *s }, nil } -func (b *GrammarBuilder) genPrecAndAssoc(symTab *symbolTableReader, errSym symbol, prodsAndActs *productionsAndActions) (*precAndAssoc, error) { - termPrec := map[symbolNum]int{} - termAssoc := map[symbolNum]assocType{} +func (b *GrammarBuilder) genPrecAndAssoc(symTab *symbol.SymbolTableReader, errSym symbol.Symbol, prodsAndActs *productionsAndActions) (*precAndAssoc, error) { + termPrec := map[symbol.SymbolNum]int{} + termAssoc := map[symbol.SymbolNum]assocType{} ordSymPrec := map[string]int{} { - var precGroup []*spec.DirectiveNode + var precGroup []*parser.DirectiveNode for _, dir := range b.AST.Directives { if dir.Name == "prec" { if dir.Parameters == nil || len(dir.Parameters) != 1 || dir.Parameters[0].Group == nil { @@ -1045,7 +1061,7 @@ func (b *GrammarBuilder) genPrecAndAssoc(symTab *symbolTableReader, errSym symbo for _, p := range dir.Parameters { switch { case p.ID != "": - sym, ok := symTab.toSymbol(p.ID) + sym, ok := symTab.ToSymbol(p.ID) if !ok { b.errs = append(b.errs, &verr.SpecError{ Cause: semErrDirInvalidParam, @@ -1064,7 +1080,7 @@ func (b *GrammarBuilder) genPrecAndAssoc(symTab *symbolTableReader, errSym symbo }) return nil, nil } - if !sym.isTerminal() { + if !sym.IsTerminal() { b.errs = append(b.errs, &verr.SpecError{ Cause: semErrDirInvalidParam, Detail: fmt.Sprintf("associativity can take only terminal symbol ('%v' is a non-terminal)", p.ID), @@ -1073,7 +1089,7 @@ func (b *GrammarBuilder) genPrecAndAssoc(symTab *symbolTableReader, errSym symbo }) return nil, nil } - if prec, alreadySet := termPrec[sym.num()]; alreadySet { + if prec, alreadySet := termPrec[sym.Num()]; alreadySet { if prec == precN { b.errs = append(b.errs, &verr.SpecError{ Cause: semErrDuplicateAssoc, @@ -1081,7 +1097,7 @@ func (b *GrammarBuilder) genPrecAndAssoc(symTab *symbolTableReader, errSym symbo Row: p.Pos.Row, Col: p.Pos.Col, }) - } else if assoc := termAssoc[sym.num()]; assoc == assocTy { + } else if assoc := termAssoc[sym.Num()]; assoc == assocTy { b.errs = append(b.errs, &verr.SpecError{ Cause: semErrDuplicateAssoc, Detail: fmt.Sprintf("'%v' already has different precedence", p.ID), @@ -1099,8 +1115,8 @@ func (b *GrammarBuilder) genPrecAndAssoc(symTab *symbolTableReader, errSym symbo break ASSOC_PARAM_LOOP } - termPrec[sym.num()] = precN - termAssoc[sym.num()] = assocTy + termPrec[sym.Num()] = precN + termAssoc[sym.Num()] = assocTy case p.OrderedSymbol != "": if prec, alreadySet := ordSymPrec[p.OrderedSymbol]; alreadySet { if prec == precN { @@ -1145,11 +1161,11 @@ func (b *GrammarBuilder) genPrecAndAssoc(symTab *symbolTableReader, errSym symbo for _, prod := range prodsAndActs.prods.getAllProductions() { // A #prec directive changes only precedence, not associativity. if term, ok := prodsAndActs.prodPrecsTerm[prod.id]; ok { - if prec, ok := termPrec[term.num()]; ok { + if prec, ok := termPrec[term.Num()]; ok { prodPrec[prod.num] = prec prodAssoc[prod.num] = assocTypeNil } else { - text, _ := symTab.toText(term) + text, _ := symTab.ToText(term) b.errs = append(b.errs, &verr.SpecError{ Cause: semErrUndefinedPrec, Detail: text, @@ -1171,16 +1187,16 @@ func (b *GrammarBuilder) genPrecAndAssoc(symTab *symbolTableReader, errSym symbo } } else { // A production inherits precedence and associativity from the right-most terminal symbol. - mostrightTerm := symbolNil + mostrightTerm := symbol.SymbolNil for _, sym := range prod.rhs { - if !sym.isTerminal() { + if !sym.IsTerminal() { continue } mostrightTerm = sym } - if !mostrightTerm.isNil() { - prodPrec[prod.num] = termPrec[mostrightTerm.num()] - prodAssoc[prod.num] = termAssoc[mostrightTerm.num()] + if !mostrightTerm.IsNil() { + prodPrec[prod.num] = termPrec[mostrightTerm.Num()] + prodAssoc[prod.num] = termAssoc[mostrightTerm.Num()] } } } @@ -1196,25 +1212,13 @@ func (b *GrammarBuilder) genPrecAndAssoc(symTab *symbolTableReader, errSym symbo }, nil } -type compileConfig struct { - isReportingEnabled bool -} - -type CompileOption func(config *compileConfig) - -func EnableReporting() CompileOption { - return func(config *compileConfig) { - config.isReportingEnabled = true - } -} - -func Compile(gram *Grammar, opts ...CompileOption) (*spec.CompiledGrammar, *spec.Report, error) { - config := &compileConfig{} +func compile(gram *Grammar, opts ...BuildOption) (*spec.CompiledGrammar, *spec.Report, error) { + config := &buildConfig{} for _, opt := range opts { opt(config) } - lexSpec, err, cErrs := mlcompiler.Compile(gram.lexSpec, mlcompiler.CompressionLevel(mlcompiler.CompressionLevelMax)) + lexSpec, err, cErrs := lexical.Compile(gram.lexSpec, lexical.CompressionLevelMax) if err != nil { if len(cErrs) > 0 { var b strings.Builder @@ -1230,35 +1234,44 @@ func Compile(gram *Grammar, opts ...CompileOption) (*spec.CompiledGrammar, *spec kind2Term := make([]int, len(lexSpec.KindNames)) for i, k := range lexSpec.KindNames { - if k == mlspec.LexKindNameNil { - kind2Term[mlspec.LexKindIDNil] = symbolNil.num().Int() + if k == spec.LexKindNameNil { + kind2Term[spec.LexKindIDNil] = symbol.SymbolNil.Num().Int() continue } - sym, ok := gram.symbolTable.toSymbol(k.String()) + sym, ok := gram.symbolTable.ToSymbol(k.String()) if !ok { return nil, nil, fmt.Errorf("terminal symbol '%v' was not found in a symbol table", k) } - kind2Term[i] = sym.num().Int() + kind2Term[i] = sym.Num().Int() } - termTexts, err := gram.symbolTable.terminalTexts() + termTexts, err := gram.symbolTable.TerminalTexts() if err != nil { return nil, nil, err } - termSkip := make([]int, len(termTexts)) - for i, k := range lexSpec.KindNames { - for _, sk := range gram.skipLexKinds { - if k != sk { - continue + var termSkip []int + { + r := gram.symbolTable.Reader() + // I want to use gram.symbolTable.terminalSymbols() here instead of gram.symbolTable.terminalTexts(), + // but gram.symbolTable.terminalSymbols() is different in length from terminalTexts + // because it does not contain a predefined symbol, like EOF. + // Therefore, we use terminalTexts, although it takes more time to lookup for symbols. + termSkip = make([]int, len(termTexts)) + for _, t := range termTexts { + s, _ := r.ToSymbol(t) + for _, sk := range gram.skipSymbols { + if s != sk { + continue + } + termSkip[s.Num()] = 1 + break } - termSkip[kind2Term[i]] = 1 - break } } - nonTerms, err := gram.symbolTable.nonTerminalTexts() + nonTerms, err := gram.symbolTable.NonTerminalTexts() if err != nil { return nil, nil, err } @@ -1316,7 +1329,7 @@ func Compile(gram *Grammar, opts ...CompileOption) (*spec.CompiledGrammar, *spec recoverProds := make([]int, len(gram.productionSet.getAllProductions())+1) astActEnties := make([][]int, len(gram.productionSet.getAllProductions())+1) for _, p := range gram.productionSet.getAllProductions() { - lhsSyms[p.num] = p.lhs.num().Int() + lhsSyms[p.num] = p.lhs.Num().Int() altSymCounts[p.num] = p.rhsLen if _, ok := gram.recoverProductions[p.id]; ok { @@ -1339,15 +1352,9 @@ func Compile(gram *Grammar, opts ...CompileOption) (*spec.CompiledGrammar, *spec } return &spec.CompiledGrammar{ - Name: gram.name, - LexicalSpecification: &spec.LexicalSpecification{ - Lexer: "maleeni", - Maleeni: &spec.Maleeni{ - Spec: lexSpec, - KindToTerminal: kind2Term, - }, - }, - ParsingTable: &spec.ParsingTable{ + Name: gram.name, + Lexical: lexSpec, + Syntactic: &spec.SyntacticSpec{ Action: action, GoTo: goTo, StateCount: tab.stateCount, @@ -1358,10 +1365,11 @@ func Compile(gram *Grammar, opts ...CompileOption) (*spec.CompiledGrammar, *spec Terminals: termTexts, TerminalCount: tab.terminalCount, TerminalSkip: termSkip, + KindToTerminal: kind2Term, NonTerminals: nonTerms, NonTerminalCount: tab.nonTerminalCount, - EOFSymbol: symbolEOF.num().Int(), - ErrorSymbol: gram.errorSymbol.num().Int(), + EOFSymbol: symbol.SymbolEOF.Num().Int(), + ErrorSymbol: gram.errorSymbol.Num().Int(), ErrorTrapperStates: tab.errorTrapperStates, RecoverProductions: recoverProds, }, @@ -1371,7 +1379,7 @@ func Compile(gram *Grammar, opts ...CompileOption) (*spec.CompiledGrammar, *spec }, report, nil } -func writeCompileError(w io.Writer, cErr *mlcompiler.CompileError) { +func writeCompileError(w io.Writer, cErr *lexical.CompileError) { if cErr.Fragment { fmt.Fprintf(w, "fragment ") } diff --git a/grammar/grammar_test.go b/grammar/grammar_test.go index f6cb681..e3cf668 100644 --- a/grammar/grammar_test.go +++ b/grammar/grammar_test.go @@ -5,7 +5,7 @@ import ( "testing" verr "github.com/nihei9/vartan/error" - spec "github.com/nihei9/vartan/spec/grammar" + "github.com/nihei9/vartan/spec/grammar/parser" ) func TestGrammarBuilderOK(t *testing.T) { @@ -243,9 +243,9 @@ baz var fooPrec int var fooAssoc assocType { - s, _ := g.symbolTable.toSymbol("foo") - fooPrec = g.precAndAssoc.terminalPrecedence(s.num()) - fooAssoc = g.precAndAssoc.terminalAssociativity(s.num()) + s, _ := g.symbolTable.ToSymbol("foo") + fooPrec = g.precAndAssoc.terminalPrecedence(s.Num()) + fooAssoc = g.precAndAssoc.terminalAssociativity(s.Num()) } if fooPrec != 1 || fooAssoc != assocTypeLeft { t.Fatalf("unexpected terminal precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", 1, assocTypeLeft, fooPrec, fooAssoc) @@ -253,9 +253,9 @@ baz var barPrec int var barAssoc assocType { - s, _ := g.symbolTable.toSymbol("bar") - barPrec = g.precAndAssoc.terminalPrecedence(s.num()) - barAssoc = g.precAndAssoc.terminalAssociativity(s.num()) + s, _ := g.symbolTable.ToSymbol("bar") + barPrec = g.precAndAssoc.terminalPrecedence(s.Num()) + barAssoc = g.precAndAssoc.terminalAssociativity(s.Num()) } if barPrec != 1 || barAssoc != assocTypeLeft { t.Fatalf("unexpected terminal precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", 1, assocTypeLeft, barPrec, barAssoc) @@ -263,9 +263,9 @@ baz var bazPrec int var bazAssoc assocType { - s, _ := g.symbolTable.toSymbol("baz") - bazPrec = g.precAndAssoc.terminalPrecedence(s.num()) - bazAssoc = g.precAndAssoc.terminalAssociativity(s.num()) + s, _ := g.symbolTable.ToSymbol("baz") + bazPrec = g.precAndAssoc.terminalPrecedence(s.Num()) + bazAssoc = g.precAndAssoc.terminalAssociativity(s.Num()) } if bazPrec != precNil || bazAssoc != assocTypeNil { t.Fatalf("unexpected terminal precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", precNil, assocTypeNil, bazPrec, bazAssoc) @@ -296,9 +296,9 @@ baz var fooPrec int var fooAssoc assocType { - s, _ := g.symbolTable.toSymbol("foo") - fooPrec = g.precAndAssoc.terminalPrecedence(s.num()) - fooAssoc = g.precAndAssoc.terminalAssociativity(s.num()) + s, _ := g.symbolTable.ToSymbol("foo") + fooPrec = g.precAndAssoc.terminalPrecedence(s.Num()) + fooAssoc = g.precAndAssoc.terminalAssociativity(s.Num()) } if fooPrec != 1 || fooAssoc != assocTypeRight { t.Fatalf("unexpected terminal precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", 1, assocTypeRight, fooPrec, fooAssoc) @@ -306,9 +306,9 @@ baz var barPrec int var barAssoc assocType { - s, _ := g.symbolTable.toSymbol("bar") - barPrec = g.precAndAssoc.terminalPrecedence(s.num()) - barAssoc = g.precAndAssoc.terminalAssociativity(s.num()) + s, _ := g.symbolTable.ToSymbol("bar") + barPrec = g.precAndAssoc.terminalPrecedence(s.Num()) + barAssoc = g.precAndAssoc.terminalAssociativity(s.Num()) } if barPrec != 1 || barAssoc != assocTypeRight { t.Fatalf("unexpected terminal precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", 1, assocTypeRight, barPrec, barAssoc) @@ -316,9 +316,9 @@ baz var bazPrec int var bazAssoc assocType { - s, _ := g.symbolTable.toSymbol("baz") - bazPrec = g.precAndAssoc.terminalPrecedence(s.num()) - bazAssoc = g.precAndAssoc.terminalAssociativity(s.num()) + s, _ := g.symbolTable.ToSymbol("baz") + bazPrec = g.precAndAssoc.terminalPrecedence(s.Num()) + bazAssoc = g.precAndAssoc.terminalAssociativity(s.Num()) } if bazPrec != precNil || bazAssoc != assocTypeNil { t.Fatalf("unexpected terminal precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", precNil, assocTypeNil, bazPrec, bazAssoc) @@ -349,9 +349,9 @@ baz var fooPrec int var fooAssoc assocType { - s, _ := g.symbolTable.toSymbol("foo") - fooPrec = g.precAndAssoc.terminalPrecedence(s.num()) - fooAssoc = g.precAndAssoc.terminalAssociativity(s.num()) + s, _ := g.symbolTable.ToSymbol("foo") + fooPrec = g.precAndAssoc.terminalPrecedence(s.Num()) + fooAssoc = g.precAndAssoc.terminalAssociativity(s.Num()) } if fooPrec != 1 || fooAssoc != assocTypeNil { t.Fatalf("unexpected terminal precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", 1, assocTypeNil, fooPrec, fooAssoc) @@ -359,9 +359,9 @@ baz var barPrec int var barAssoc assocType { - s, _ := g.symbolTable.toSymbol("bar") - barPrec = g.precAndAssoc.terminalPrecedence(s.num()) - barAssoc = g.precAndAssoc.terminalAssociativity(s.num()) + s, _ := g.symbolTable.ToSymbol("bar") + barPrec = g.precAndAssoc.terminalPrecedence(s.Num()) + barAssoc = g.precAndAssoc.terminalAssociativity(s.Num()) } if barPrec != 1 || barAssoc != assocTypeNil { t.Fatalf("unexpected terminal precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", 1, assocTypeNil, barPrec, barAssoc) @@ -369,9 +369,9 @@ baz var bazPrec int var bazAssoc assocType { - s, _ := g.symbolTable.toSymbol("baz") - bazPrec = g.precAndAssoc.terminalPrecedence(s.num()) - bazAssoc = g.precAndAssoc.terminalAssociativity(s.num()) + s, _ := g.symbolTable.ToSymbol("baz") + bazPrec = g.precAndAssoc.terminalPrecedence(s.Num()) + bazAssoc = g.precAndAssoc.terminalAssociativity(s.Num()) } if bazPrec != precNil || bazAssoc != assocTypeNil { t.Fatalf("unexpected terminal precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", precNil, assocTypeNil, bazPrec, bazAssoc) @@ -400,14 +400,14 @@ bar var barPrec int var barAssoc assocType { - s, _ := g.symbolTable.toSymbol("bar") - barPrec = g.precAndAssoc.terminalPrecedence(s.num()) - barAssoc = g.precAndAssoc.terminalAssociativity(s.num()) + s, _ := g.symbolTable.ToSymbol("bar") + barPrec = g.precAndAssoc.terminalPrecedence(s.Num()) + barAssoc = g.precAndAssoc.terminalAssociativity(s.Num()) } var sPrec int var sAssoc assocType { - s, _ := g.symbolTable.toSymbol("s") + s, _ := g.symbolTable.ToSymbol("s") ps, _ := g.productionSet.findByLHS(s) sPrec = g.precAndAssoc.productionPredence(ps[0].num) sAssoc = g.precAndAssoc.productionAssociativity(ps[0].num) @@ -443,14 +443,14 @@ bar var barPrec int var barAssoc assocType { - s, _ := g.symbolTable.toSymbol("bar") - barPrec = g.precAndAssoc.terminalPrecedence(s.num()) - barAssoc = g.precAndAssoc.terminalAssociativity(s.num()) + s, _ := g.symbolTable.ToSymbol("bar") + barPrec = g.precAndAssoc.terminalPrecedence(s.Num()) + barAssoc = g.precAndAssoc.terminalAssociativity(s.Num()) } var sPrec int var sAssoc assocType { - s, _ := g.symbolTable.toSymbol("s") + s, _ := g.symbolTable.ToSymbol("s") ps, _ := g.productionSet.findByLHS(s) sPrec = g.precAndAssoc.productionPredence(ps[0].num) sAssoc = g.precAndAssoc.productionAssociativity(ps[0].num) @@ -489,21 +489,21 @@ bar var fooPrec int var fooAssoc assocType { - s, _ := g.symbolTable.toSymbol("foo") - fooPrec = g.precAndAssoc.terminalPrecedence(s.num()) - fooAssoc = g.precAndAssoc.terminalAssociativity(s.num()) + s, _ := g.symbolTable.ToSymbol("foo") + fooPrec = g.precAndAssoc.terminalPrecedence(s.Num()) + fooAssoc = g.precAndAssoc.terminalAssociativity(s.Num()) } var barPrec int var barAssoc assocType { - s, _ := g.symbolTable.toSymbol("bar") - barPrec = g.precAndAssoc.terminalPrecedence(s.num()) - barAssoc = g.precAndAssoc.terminalAssociativity(s.num()) + s, _ := g.symbolTable.ToSymbol("bar") + barPrec = g.precAndAssoc.terminalPrecedence(s.Num()) + barAssoc = g.precAndAssoc.terminalAssociativity(s.Num()) } var aPrec int var aAssoc assocType { - s, _ := g.symbolTable.toSymbol("a") + s, _ := g.symbolTable.ToSymbol("a") ps, _ := g.productionSet.findByLHS(s) aPrec = g.precAndAssoc.productionPredence(ps[0].num) aAssoc = g.precAndAssoc.productionAssociativity(ps[0].num) @@ -511,7 +511,7 @@ bar var sPrec int var sAssoc assocType { - s, _ := g.symbolTable.toSymbol("s") + s, _ := g.symbolTable.ToSymbol("s") ps, _ := g.productionSet.findByLHS(s) sPrec = g.precAndAssoc.productionPredence(ps[0].num) sAssoc = g.precAndAssoc.productionAssociativity(ps[0].num) @@ -567,7 +567,7 @@ bra var alt4Prec int var alt4Assoc assocType { - s, _ := g.symbolTable.toSymbol("s") + s, _ := g.symbolTable.ToSymbol("s") ps, _ := g.productionSet.findByLHS(s) alt1Prec = g.precAndAssoc.productionPredence(ps[0].num) alt1Assoc = g.precAndAssoc.productionAssociativity(ps[0].num) @@ -615,14 +615,14 @@ foo var fooPrec int var fooAssoc assocType { - s, _ := g.symbolTable.toSymbol("foo") - fooPrec = g.precAndAssoc.terminalPrecedence(s.num()) - fooAssoc = g.precAndAssoc.terminalAssociativity(s.num()) + s, _ := g.symbolTable.ToSymbol("foo") + fooPrec = g.precAndAssoc.terminalPrecedence(s.Num()) + fooAssoc = g.precAndAssoc.terminalAssociativity(s.Num()) } var aPrec int var aAssoc assocType { - s, _ := g.symbolTable.toSymbol("a") + s, _ := g.symbolTable.ToSymbol("a") ps, _ := g.productionSet.findByLHS(s) aPrec = g.precAndAssoc.productionPredence(ps[0].num) aAssoc = g.precAndAssoc.productionAssociativity(ps[0].num) @@ -630,7 +630,7 @@ foo var sPrec int var sAssoc assocType { - s, _ := g.symbolTable.toSymbol("s") + s, _ := g.symbolTable.ToSymbol("s") ps, _ := g.productionSet.findByLHS(s) sPrec = g.precAndAssoc.productionPredence(ps[0].num) sAssoc = g.precAndAssoc.productionAssociativity(ps[0].num) @@ -668,14 +668,14 @@ bar var fooPrec int var fooAssoc assocType { - s, _ := g.symbolTable.toSymbol("foo") - fooPrec = g.precAndAssoc.terminalPrecedence(s.num()) - fooAssoc = g.precAndAssoc.terminalAssociativity(s.num()) + s, _ := g.symbolTable.ToSymbol("foo") + fooPrec = g.precAndAssoc.terminalPrecedence(s.Num()) + fooAssoc = g.precAndAssoc.terminalAssociativity(s.Num()) } var sPrec int var sAssoc assocType { - s, _ := g.symbolTable.toSymbol("s") + s, _ := g.symbolTable.ToSymbol("s") ps, _ := g.productionSet.findByLHS(s) sPrec = g.precAndAssoc.productionPredence(ps[0].num) sAssoc = g.precAndAssoc.productionAssociativity(ps[0].num) @@ -711,21 +711,21 @@ bar var fooPrec int var fooAssoc assocType { - s, _ := g.symbolTable.toSymbol("foo") - fooPrec = g.precAndAssoc.terminalPrecedence(s.num()) - fooAssoc = g.precAndAssoc.terminalAssociativity(s.num()) + s, _ := g.symbolTable.ToSymbol("foo") + fooPrec = g.precAndAssoc.terminalPrecedence(s.Num()) + fooAssoc = g.precAndAssoc.terminalAssociativity(s.Num()) } var barPrec int var barAssoc assocType { - s, _ := g.symbolTable.toSymbol("bar") - barPrec = g.precAndAssoc.terminalPrecedence(s.num()) - barAssoc = g.precAndAssoc.terminalAssociativity(s.num()) + s, _ := g.symbolTable.ToSymbol("bar") + barPrec = g.precAndAssoc.terminalPrecedence(s.Num()) + barAssoc = g.precAndAssoc.terminalAssociativity(s.Num()) } var sPrec int var sAssoc assocType { - s, _ := g.symbolTable.toSymbol("s") + s, _ := g.symbolTable.ToSymbol("s") ps, _ := g.productionSet.findByLHS(s) sPrec = g.precAndAssoc.productionPredence(ps[0].num) sAssoc = g.precAndAssoc.productionAssociativity(ps[0].num) @@ -766,9 +766,9 @@ bar var fooPrec int var fooAssoc assocType { - s, _ := g.symbolTable.toSymbol("foo") - fooPrec = g.precAndAssoc.terminalPrecedence(s.num()) - fooAssoc = g.precAndAssoc.terminalAssociativity(s.num()) + s, _ := g.symbolTable.ToSymbol("foo") + fooPrec = g.precAndAssoc.terminalPrecedence(s.Num()) + fooAssoc = g.precAndAssoc.terminalAssociativity(s.Num()) } if fooPrec != 2 || fooAssoc != assocTypeRight { t.Fatalf("unexpected terminal precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", 2, assocTypeRight, fooPrec, fooAssoc) @@ -776,9 +776,9 @@ bar var barPrec int var barAssoc assocType { - s, _ := g.symbolTable.toSymbol("bar") - barPrec = g.precAndAssoc.terminalPrecedence(s.num()) - barAssoc = g.precAndAssoc.terminalAssociativity(s.num()) + s, _ := g.symbolTable.ToSymbol("bar") + barPrec = g.precAndAssoc.terminalPrecedence(s.Num()) + barAssoc = g.precAndAssoc.terminalAssociativity(s.Num()) } if barPrec != 2 || barAssoc != assocTypeRight { t.Fatalf("unexpected terminal precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", 2, assocTypeRight, barPrec, barAssoc) @@ -788,7 +788,7 @@ bar var alt2Prec int var alt2Assoc assocType { - s, _ := g.symbolTable.toSymbol("s") + s, _ := g.symbolTable.ToSymbol("s") ps, _ := g.productionSet.findByLHS(s) alt1Prec = g.precAndAssoc.productionPredence(ps[0].num) alt1Assoc = g.precAndAssoc.productionAssociativity(ps[0].num) @@ -828,9 +828,9 @@ bar var fooPrec int var fooAssoc assocType { - s, _ := g.symbolTable.toSymbol("foo") - fooPrec = g.precAndAssoc.terminalPrecedence(s.num()) - fooAssoc = g.precAndAssoc.terminalAssociativity(s.num()) + s, _ := g.symbolTable.ToSymbol("foo") + fooPrec = g.precAndAssoc.terminalPrecedence(s.Num()) + fooAssoc = g.precAndAssoc.terminalAssociativity(s.Num()) } if fooPrec != 2 || fooAssoc != assocTypeLeft { t.Fatalf("unexpected terminal precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", 2, assocTypeLeft, fooPrec, fooAssoc) @@ -838,9 +838,9 @@ bar var barPrec int var barAssoc assocType { - s, _ := g.symbolTable.toSymbol("bar") - barPrec = g.precAndAssoc.terminalPrecedence(s.num()) - barAssoc = g.precAndAssoc.terminalAssociativity(s.num()) + s, _ := g.symbolTable.ToSymbol("bar") + barPrec = g.precAndAssoc.terminalPrecedence(s.Num()) + barAssoc = g.precAndAssoc.terminalAssociativity(s.Num()) } if barPrec != 2 || barAssoc != assocTypeLeft { t.Fatalf("unexpected terminal precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", 2, assocTypeLeft, barPrec, barAssoc) @@ -850,7 +850,7 @@ bar var alt2Prec int var alt2Assoc assocType { - s, _ := g.symbolTable.toSymbol("s") + s, _ := g.symbolTable.ToSymbol("s") ps, _ := g.productionSet.findByLHS(s) alt1Prec = g.precAndAssoc.productionPredence(ps[0].num) alt1Assoc = g.precAndAssoc.productionAssociativity(ps[0].num) @@ -891,9 +891,9 @@ bar var fooPrec int var fooAssoc assocType { - s, _ := g.symbolTable.toSymbol("foo") - fooPrec = g.precAndAssoc.terminalPrecedence(s.num()) - fooAssoc = g.precAndAssoc.terminalAssociativity(s.num()) + s, _ := g.symbolTable.ToSymbol("foo") + fooPrec = g.precAndAssoc.terminalPrecedence(s.Num()) + fooAssoc = g.precAndAssoc.terminalAssociativity(s.Num()) } if fooPrec != 2 || fooAssoc != assocTypeLeft { t.Fatalf("unexpected terminal precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", 2, assocTypeLeft, fooPrec, fooAssoc) @@ -901,9 +901,9 @@ bar var barPrec int var barAssoc assocType { - s, _ := g.symbolTable.toSymbol("bar") - barPrec = g.precAndAssoc.terminalPrecedence(s.num()) - barAssoc = g.precAndAssoc.terminalAssociativity(s.num()) + s, _ := g.symbolTable.ToSymbol("bar") + barPrec = g.precAndAssoc.terminalPrecedence(s.Num()) + barAssoc = g.precAndAssoc.terminalAssociativity(s.Num()) } if barPrec != 3 || barAssoc != assocTypeRight { t.Fatalf("unexpected terminal precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", 3, assocTypeRight, barPrec, barAssoc) @@ -913,7 +913,7 @@ bar var alt2Prec int var alt2Assoc assocType { - s, _ := g.symbolTable.toSymbol("s") + s, _ := g.symbolTable.ToSymbol("s") ps, _ := g.productionSet.findByLHS(s) alt1Prec = g.precAndAssoc.productionPredence(ps[0].num) alt1Assoc = g.precAndAssoc.productionAssociativity(ps[0].num) @@ -952,16 +952,16 @@ bar var fooPrec int var fooAssoc assocType { - s, _ := g.symbolTable.toSymbol("foo") - fooPrec = g.precAndAssoc.terminalPrecedence(s.num()) - fooAssoc = g.precAndAssoc.terminalAssociativity(s.num()) + s, _ := g.symbolTable.ToSymbol("foo") + fooPrec = g.precAndAssoc.terminalPrecedence(s.Num()) + fooAssoc = g.precAndAssoc.terminalAssociativity(s.Num()) } var barPrec int var barAssoc assocType { - s, _ := g.symbolTable.toSymbol("bar") - barPrec = g.precAndAssoc.terminalPrecedence(s.num()) - barAssoc = g.precAndAssoc.terminalAssociativity(s.num()) + s, _ := g.symbolTable.ToSymbol("bar") + barPrec = g.precAndAssoc.terminalPrecedence(s.Num()) + barAssoc = g.precAndAssoc.terminalAssociativity(s.Num()) } if fooPrec != 1 || fooAssoc != assocTypeLeft { t.Fatalf("unexpected terminal precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", 1, assocTypeLeft, fooPrec, fooAssoc) @@ -974,7 +974,7 @@ bar var alt2Prec int var alt2Assoc assocType { - s, _ := g.symbolTable.toSymbol("s") + s, _ := g.symbolTable.ToSymbol("s") ps, _ := g.productionSet.findByLHS(s) alt1Prec = g.precAndAssoc.productionPredence(ps[0].num) alt1Assoc = g.precAndAssoc.productionAssociativity(ps[0].num) @@ -1016,9 +1016,9 @@ bar var barPrec int var barAssoc assocType { - s, _ := g.symbolTable.toSymbol("bar") - barPrec = g.precAndAssoc.terminalPrecedence(s.num()) - barAssoc = g.precAndAssoc.terminalAssociativity(s.num()) + s, _ := g.symbolTable.ToSymbol("bar") + barPrec = g.precAndAssoc.terminalPrecedence(s.Num()) + barAssoc = g.precAndAssoc.terminalAssociativity(s.Num()) } if barPrec != 1 || barAssoc != assocTypeLeft { t.Fatalf("unexpected terminal precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", 1, assocTypeLeft, barPrec, barAssoc) @@ -1028,7 +1028,7 @@ bar var alt2Prec int var alt2Assoc assocType { - s, _ := g.symbolTable.toSymbol("s") + s, _ := g.symbolTable.ToSymbol("s") ps, _ := g.productionSet.findByLHS(s) alt1Prec = g.precAndAssoc.productionPredence(ps[0].num) alt1Assoc = g.precAndAssoc.productionAssociativity(ps[0].num) @@ -1052,7 +1052,7 @@ bar for _, test := range tests { t.Run(test.caption, func(t *testing.T) { - ast, err := spec.Parse(strings.NewReader(test.specSrc)) + ast, err := parser.Parse(strings.NewReader(test.specSrc)) if err != nil { t.Fatal(err) } @@ -1060,7 +1060,7 @@ bar b := GrammarBuilder{ AST: ast, } - g, err := b.Build() + g, err := b.build() if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -1075,7 +1075,7 @@ func TestGrammarBuilderSpecError(t *testing.T) { type specErrTest struct { caption string specSrc string - errs []*SemanticError + errs []error } spellingInconsistenciesTests := []*specErrTest{ @@ -1094,7 +1094,7 @@ a_1 foo : 'foo'; `, - errs: []*SemanticError{semErrSpellingInconsistency}, + errs: []error{semErrSpellingInconsistency}, }, { caption: "a spelling inconsistency appears among terminal symbols", @@ -1110,7 +1110,7 @@ foo1 foo_1 : 'foo_1'; `, - errs: []*SemanticError{semErrSpellingInconsistency}, + errs: []error{semErrSpellingInconsistency}, }, { caption: "a spelling inconsistency appears among non-terminal and terminal symbols", @@ -1124,7 +1124,7 @@ a1 a_1 : 'a_1'; `, - errs: []*SemanticError{semErrSpellingInconsistency}, + errs: []error{semErrSpellingInconsistency}, }, { caption: "a spelling inconsistency appears among ordered symbols whose precedence is the same", @@ -1145,7 +1145,7 @@ foo bar : 'bar'; `, - errs: []*SemanticError{semErrSpellingInconsistency}, + errs: []error{semErrSpellingInconsistency}, }, { caption: "a spelling inconsistency appears among ordered symbols whose precedence is not the same", @@ -1167,7 +1167,7 @@ foo bar : 'bar'; `, - errs: []*SemanticError{semErrSpellingInconsistency}, + errs: []error{semErrSpellingInconsistency}, }, { caption: "a spelling inconsistency appears among labels the same alternative contains", @@ -1181,7 +1181,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrSpellingInconsistency}, + errs: []error{semErrSpellingInconsistency}, }, { caption: "a spelling inconsistency appears among labels the same production contains", @@ -1198,7 +1198,7 @@ foo bar : 'bar'; `, - errs: []*SemanticError{semErrSpellingInconsistency}, + errs: []error{semErrSpellingInconsistency}, }, { caption: "a spelling inconsistency appears among labels different productions contain", @@ -1217,7 +1217,7 @@ foo bar : 'bar'; `, - errs: []*SemanticError{semErrSpellingInconsistency}, + errs: []error{semErrSpellingInconsistency}, }, } @@ -1237,7 +1237,7 @@ b foo : "foo"; `, - errs: []*SemanticError{semErrUnusedProduction}, + errs: []error{semErrUnusedProduction}, }, { caption: "a terminal symbol `bar` is unused", @@ -1253,7 +1253,7 @@ foo bar : "bar"; `, - errs: []*SemanticError{semErrUnusedTerminal}, + errs: []error{semErrUnusedTerminal}, }, { caption: "a production `b` and terminal symbol `bar` is unused", @@ -1272,7 +1272,7 @@ foo bar : "bar"; `, - errs: []*SemanticError{ + errs: []error{ semErrUnusedProduction, semErrUnusedTerminal, }, @@ -1289,7 +1289,7 @@ s #prec foo foo : 'foo'; `, - errs: []*SemanticError{semErrInvalidProdDir}, + errs: []error{semErrInvalidProdDir}, }, { caption: "a lexical production cannot have alternative directives", @@ -1303,7 +1303,7 @@ s foo : 'foo' #skip; `, - errs: []*SemanticError{semErrInvalidAltDir}, + errs: []error{semErrInvalidAltDir}, }, { caption: "a production directive must not be duplicated", @@ -1317,7 +1317,7 @@ s foo #skip #skip : 'foo'; `, - errs: []*SemanticError{semErrDuplicateDir}, + errs: []error{semErrDuplicateDir}, }, { caption: "an alternative directive must not be duplicated", @@ -1333,7 +1333,7 @@ foo bar : 'bar'; `, - errs: []*SemanticError{semErrDuplicateDir}, + errs: []error{semErrDuplicateDir}, }, { caption: "a production must not have a duplicate alternative (non-empty alternatives)", @@ -1348,7 +1348,7 @@ s foo : "foo"; `, - errs: []*SemanticError{semErrDuplicateProduction}, + errs: []error{semErrDuplicateProduction}, }, { caption: "a production must not have a duplicate alternative (non-empty and split alternatives)", @@ -1371,7 +1371,7 @@ foo bar : "bar"; `, - errs: []*SemanticError{semErrDuplicateProduction}, + errs: []error{semErrDuplicateProduction}, }, { caption: "a production must not have a duplicate alternative (empty alternatives)", @@ -1390,7 +1390,7 @@ a foo : "foo"; `, - errs: []*SemanticError{semErrDuplicateProduction}, + errs: []error{semErrDuplicateProduction}, }, { caption: "a production must not have a duplicate alternative (empty and split alternatives)", @@ -1412,7 +1412,7 @@ a foo : "foo"; `, - errs: []*SemanticError{semErrDuplicateProduction}, + errs: []error{semErrDuplicateProduction}, }, { caption: "a terminal symbol and a non-terminal symbol (start symbol) are duplicates", @@ -1428,7 +1428,7 @@ foo s : "a"; `, - errs: []*SemanticError{semErrDuplicateName}, + errs: []error{semErrDuplicateName}, }, { caption: "a terminal symbol and a non-terminal symbol (not start symbol) are duplicates", @@ -1450,7 +1450,7 @@ bar a : "a"; `, - errs: []*SemanticError{semErrDuplicateName}, + errs: []error{semErrDuplicateName}, }, { caption: "an invalid top-level directive", @@ -1466,7 +1466,7 @@ s a : 'a'; `, - errs: []*SemanticError{semErrDirInvalidName}, + errs: []error{semErrDirInvalidName}, }, { caption: "a label must be unique in an alternative", @@ -1482,7 +1482,7 @@ foo bar : 'bar'; `, - errs: []*SemanticError{semErrDuplicateLabel}, + errs: []error{semErrDuplicateLabel}, }, { caption: "a label cannot be the same name as terminal symbols", @@ -1498,7 +1498,7 @@ foo bar : 'bar'; `, - errs: []*SemanticError{semErrDuplicateLabel}, + errs: []error{semErrDuplicateLabel}, }, { caption: "a label cannot be the same name as non-terminal symbols", @@ -1518,7 +1518,7 @@ foo bar : 'bar'; `, - errs: []*SemanticError{ + errs: []error{ semErrInvalidLabel, }, }, @@ -1535,7 +1535,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrNoGrammarName}, + errs: []error{semErrNoGrammarName}, }, { caption: "the `#name` directive needs an ID parameter", @@ -1549,7 +1549,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#name` directive cannot take a pattern parameter", @@ -1563,7 +1563,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#name` directive cannot take a string parameter", @@ -1577,7 +1577,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#name` directive takes just one parameter", @@ -1591,7 +1591,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, } @@ -1610,7 +1610,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#prec` directive cannot take an ID parameter", @@ -1626,7 +1626,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#prec` directive cannot take an ordered symbol parameter", @@ -1642,7 +1642,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#prec` directive cannot take a pattern parameter", @@ -1658,7 +1658,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#prec` directive cannot take a string parameter", @@ -1674,7 +1674,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#prec` directive takes just one directive group parameter", @@ -1690,7 +1690,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, } @@ -1711,7 +1711,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#left` directive cannot be applied to an error symbol", @@ -1732,7 +1732,7 @@ foo semi_colon : ';'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#left` directive cannot take an undefined symbol", @@ -1750,7 +1750,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#left` directive cannot take a non-terminal symbol", @@ -1768,7 +1768,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#left` directive cannot take a pattern parameter", @@ -1786,7 +1786,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#left` directive cannot take a string parameter", @@ -1804,7 +1804,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#left` directive cannot take a directive parameter", @@ -1822,7 +1822,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#left` dirctive cannot be specified multiple times for a terminal symbol", @@ -1840,7 +1840,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDuplicateAssoc}, + errs: []error{semErrDuplicateAssoc}, }, { caption: "the `#left` dirctive cannot be specified multiple times for an ordered symbol", @@ -1858,7 +1858,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDuplicateAssoc}, + errs: []error{semErrDuplicateAssoc}, }, { caption: "a terminal symbol cannot have different precedence", @@ -1877,7 +1877,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDuplicateAssoc}, + errs: []error{semErrDuplicateAssoc}, }, { caption: "an ordered symbol cannot have different precedence", @@ -1896,7 +1896,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDuplicateAssoc}, + errs: []error{semErrDuplicateAssoc}, }, { caption: "a terminal symbol cannot have different associativity", @@ -1915,7 +1915,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDuplicateAssoc}, + errs: []error{semErrDuplicateAssoc}, }, { caption: "an ordered symbol cannot have different associativity", @@ -1934,7 +1934,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDuplicateAssoc}, + errs: []error{semErrDuplicateAssoc}, }, } @@ -1955,7 +1955,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#right` directive cannot be applied to an error symbol", @@ -1976,7 +1976,7 @@ foo semi_colon : ';'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#right` directive cannot take an undefined symbol", @@ -1994,7 +1994,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#right` directive cannot take a non-terminal symbol", @@ -2012,7 +2012,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#right` directive cannot take a pattern parameter", @@ -2030,7 +2030,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#right` directive cannot take a string parameter", @@ -2048,7 +2048,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#right` directive cannot take a directive group parameter", @@ -2066,7 +2066,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#right` directive cannot be specified multiple times for a terminal symbol", @@ -2084,7 +2084,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDuplicateAssoc}, + errs: []error{semErrDuplicateAssoc}, }, { caption: "the `#right` directive cannot be specified multiple times for an ordered symbol", @@ -2102,7 +2102,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDuplicateAssoc}, + errs: []error{semErrDuplicateAssoc}, }, { caption: "a terminal symbol cannot have different precedence", @@ -2121,7 +2121,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDuplicateAssoc}, + errs: []error{semErrDuplicateAssoc}, }, { caption: "an ordered symbol cannot have different precedence", @@ -2140,7 +2140,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDuplicateAssoc}, + errs: []error{semErrDuplicateAssoc}, }, { caption: "a terminal symbol cannot have different associativity", @@ -2159,7 +2159,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDuplicateAssoc}, + errs: []error{semErrDuplicateAssoc}, }, { caption: "an ordered symbol cannot have different associativity", @@ -2178,7 +2178,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDuplicateAssoc}, + errs: []error{semErrDuplicateAssoc}, }, } @@ -2199,7 +2199,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#assign` directive cannot be applied to an error symbol", @@ -2220,7 +2220,7 @@ foo semi_colon : ';'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#assign` directive cannot take an undefined symbol", @@ -2238,7 +2238,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#assign` directive cannot take a non-terminal symbol", @@ -2256,7 +2256,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#assign` directive cannot take a pattern parameter", @@ -2274,7 +2274,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#assign` directive cannot take a string parameter", @@ -2292,7 +2292,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#assign` directive cannot take a directive parameter", @@ -2310,7 +2310,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#assign` dirctive cannot be specified multiple times for a terminal symbol", @@ -2328,7 +2328,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDuplicateAssoc}, + errs: []error{semErrDuplicateAssoc}, }, { caption: "the `#assign` dirctive cannot be specified multiple times for an ordered symbol", @@ -2346,7 +2346,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDuplicateAssoc}, + errs: []error{semErrDuplicateAssoc}, }, { caption: "a terminal symbol cannot have different precedence", @@ -2365,7 +2365,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDuplicateAssoc}, + errs: []error{semErrDuplicateAssoc}, }, { caption: "an ordered symbol cannot have different precedence", @@ -2384,7 +2384,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDuplicateAssoc}, + errs: []error{semErrDuplicateAssoc}, }, { caption: "a terminal symbol cannot have different associativity", @@ -2403,7 +2403,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDuplicateAssoc}, + errs: []error{semErrDuplicateAssoc}, }, { caption: "an ordered symbol cannot have different associativity", @@ -2422,7 +2422,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDuplicateAssoc}, + errs: []error{semErrDuplicateAssoc}, }, } @@ -2441,7 +2441,7 @@ error foo: 'foo'; `, - errs: []*SemanticError{ + errs: []error{ semErrErrSymIsReserved, semErrDuplicateName, }, @@ -2457,7 +2457,7 @@ s error: 'error'; `, - errs: []*SemanticError{semErrErrSymIsReserved}, + errs: []error{semErrErrSymIsReserved}, }, { caption: "cannot use the error symbol as a terminal symbol, even if given the skip directive", @@ -2473,7 +2473,7 @@ foo error #skip : 'error'; `, - errs: []*SemanticError{semErrErrSymIsReserved}, + errs: []error{semErrErrSymIsReserved}, }, } @@ -2490,7 +2490,7 @@ s foo : "foo"; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#ast` directive cannot take an ordered symbol parameter", @@ -2508,7 +2508,7 @@ s foo : "foo"; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#ast` directive cannot take a pattern parameter", @@ -2522,7 +2522,7 @@ s foo : "foo"; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#ast` directive cannot take a string parameter", @@ -2536,7 +2536,7 @@ s foo : "foo"; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#ast` directive cannot take a directive group parameter", @@ -2550,7 +2550,7 @@ s foo : "foo"; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "a parameter of the `#ast` directive must be either a symbol or a label in an alternative", @@ -2566,7 +2566,7 @@ foo bar : "bar"; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "a symbol in a different alternative cannot be a parameter of the `#ast` directive", @@ -2583,7 +2583,7 @@ foo bar : "bar"; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "a label in a different alternative cannot be a parameter of the `#ast` directive", @@ -2600,7 +2600,7 @@ foo bar : "bar"; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "a symbol can appear in the `#ast` directive only once", @@ -2614,7 +2614,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDuplicateElem}, + errs: []error{semErrDuplicateElem}, }, { caption: "a label can appear in the `#ast` directive only once", @@ -2628,7 +2628,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDuplicateElem}, + errs: []error{semErrDuplicateElem}, }, { caption: "a symbol can appear in the `#ast` directive only once, even if the symbol has a label", @@ -2642,7 +2642,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDuplicateElem}, + errs: []error{semErrDuplicateElem}, }, { caption: "symbol `foo` is ambiguous because it appears in an alternative twice", @@ -2656,7 +2656,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrAmbiguousElem}, + errs: []error{semErrAmbiguousElem}, }, { caption: "symbol `foo` is ambiguous because it appears in an alternative twice, even if one of them has a label", @@ -2670,7 +2670,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrAmbiguousElem}, + errs: []error{semErrAmbiguousElem}, }, { caption: "the expansion operator cannot be applied to a terminal symbol", @@ -2684,7 +2684,7 @@ s foo : "foo"; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, } @@ -2701,7 +2701,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#prec` directive cannot be applied to an error symbol", @@ -2715,7 +2715,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#prec` directive cannot take an undefined symbol", @@ -2729,7 +2729,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#prec` directive cannot take a non-terminal symbol", @@ -2752,7 +2752,7 @@ foo bar : 'bar'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#prec` directive cannot take an undefined ordered symbol parameter", @@ -2766,7 +2766,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrUndefinedOrdSym}, + errs: []error{semErrUndefinedOrdSym}, }, { caption: "the `#prec` directive cannot take a pattern parameter", @@ -2780,7 +2780,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#prec` directive cannot take a string parameter", @@ -2794,7 +2794,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#prec` directive cannot take a directive parameter", @@ -2808,7 +2808,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "a symbol the `#prec` directive takes must be given precedence explicitly", @@ -2824,7 +2824,7 @@ foo bar : 'bar'; `, - errs: []*SemanticError{semErrUndefinedPrec}, + errs: []error{semErrUndefinedPrec}, }, } @@ -2841,7 +2841,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#recover` directive cannot take an ordered symbol parameter", @@ -2859,7 +2859,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#recover` directive cannot take a pattern parameter", @@ -2873,7 +2873,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#recover` directive cannot take a string parameter", @@ -2887,7 +2887,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#recover` directive cannot take a directive group parameter", @@ -2901,7 +2901,7 @@ s foo : 'foo'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, } @@ -2918,7 +2918,7 @@ s fragment f : 'fragment'; `, - errs: []*SemanticError{semErrUndefinedSym}, + errs: []error{semErrUndefinedSym}, }, { caption: "fragments cannot be duplicated", @@ -2936,7 +2936,7 @@ fragment f fragment f : 'fragment 2'; `, - errs: []*SemanticError{semErrDuplicateFragment}, + errs: []error{semErrDuplicateFragment}, }, } @@ -2955,7 +2955,7 @@ foo #push mode_1 bar #mode : 'bar'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#mode` directive cannot take an ordered symbol parameter", @@ -2975,7 +2975,7 @@ foo bar #mode $x : 'bar'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#mode` directive cannot take a pattern parameter", @@ -2991,7 +2991,7 @@ foo #push mode_1 bar #mode "mode_1" : 'bar'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#mode` directive cannot take a string parameter", @@ -3007,7 +3007,7 @@ foo #push mode_1 bar #mode 'mode_1' : 'bar'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#mode` directive cannot take a directive group parameter", @@ -3023,7 +3023,7 @@ foo #push mode_1 bar #mode () : 'bar'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, } @@ -3042,7 +3042,7 @@ foo #push bar #mode mode_1 : 'bar'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#push` directive takes just one ID parameter", @@ -3058,7 +3058,7 @@ foo #push mode_1 mode_2 bar #mode mode_1 : 'bar'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#push` directive cannot take an ordered symbol parameter", @@ -3078,7 +3078,7 @@ foo #push $x bar : 'bar'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#push` directive cannot take a pattern parameter", @@ -3094,7 +3094,7 @@ foo #push "mode_1" bar #mode mode_1 : 'bar'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#push` directive cannot take a string parameter", @@ -3110,7 +3110,7 @@ foo #push 'mode_1' bar #mode mode_1 : 'bar'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#push` directive cannot take a directive group parameter", @@ -3126,7 +3126,7 @@ foo #push () bar #mode mode_1 : 'bar'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, } @@ -3147,7 +3147,7 @@ bar #mode mode_1 baz #pop mode_1 : 'baz'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#pop` directive cannot take an ordered symbol parameter", @@ -3169,7 +3169,7 @@ bar #mode mode_1 baz #pop $x : 'baz'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#pop` directive cannot take a pattern parameter", @@ -3187,7 +3187,7 @@ bar #mode mode_1 baz #pop "mode_1" : 'baz'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#pop` directive cannot take a string parameter", @@ -3205,7 +3205,7 @@ bar #mode mode_1 baz #pop 'mode_1' : 'baz'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#pop` directive cannot take a directive parameter", @@ -3223,7 +3223,7 @@ bar #mode mode_1 baz #pop () : 'baz'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, } @@ -3242,7 +3242,7 @@ foo #skip bar bar : 'bar'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#skip` directive cannot take an ordered symbol parameter", @@ -3262,7 +3262,7 @@ foo #skip $x bar : 'bar'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#skip` directive cannot take a pattern parameter", @@ -3278,7 +3278,7 @@ foo #skip "bar" bar : 'bar'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#skip` directive cannot take a string parameter", @@ -3294,7 +3294,7 @@ foo #skip 'bar' bar : 'bar'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "the `#skip` directive cannot take a directive group parameter", @@ -3310,7 +3310,7 @@ foo #skip () bar : 'bar'; `, - errs: []*SemanticError{semErrDirInvalidParam}, + errs: []error{semErrDirInvalidParam}, }, { caption: "a terminal symbol used in productions cannot have the skip directive", @@ -3326,7 +3326,7 @@ foo #skip bar : 'bar'; `, - errs: []*SemanticError{semErrTermCannotBeSkipped}, + errs: []error{semErrTermCannotBeSkipped}, }, } @@ -3349,7 +3349,7 @@ bar tests = append(tests, skipDirTests...) for _, test := range tests { t.Run(test.caption, func(t *testing.T) { - ast, err := spec.Parse(strings.NewReader(test.specSrc)) + ast, err := parser.Parse(strings.NewReader(test.specSrc)) if err != nil { t.Fatal(err) } @@ -3357,7 +3357,7 @@ bar b := GrammarBuilder{ AST: ast, } - _, err = b.Build() + _, err = b.build() if err == nil { t.Fatal("an expected error didn't occur") } diff --git a/grammar/item.go b/grammar/item.go index 100d920..84c4911 100644 --- a/grammar/item.go +++ b/grammar/item.go @@ -6,6 +6,8 @@ import ( "fmt" "sort" "strconv" + + "github.com/nihei9/vartan/grammar/symbol" ) type lrItemID [32]byte @@ -19,7 +21,7 @@ func (id lrItemID) num() uint32 { } type lookAhead struct { - symbols map[symbol]struct{} + symbols map[symbol.Symbol]struct{} // When propagation is true, an item propagates look-ahead symbols to other items. propagation bool @@ -38,7 +40,7 @@ type lrItem struct { // 2 | T | E → E +・T // 3 | Nil | E → E + T・ dot int - dottedSymbol symbol + dottedSymbol symbol.Symbol // When initial is true, the LHS of the production is the augmented start symbol and dot is 0. // It looks like S' →・S. @@ -74,13 +76,13 @@ func newLR0Item(prod *production, dot int) (*lrItem, error) { id = sha256.Sum256(b) } - dottedSymbol := symbolNil + dottedSymbol := symbol.SymbolNil if dot < prod.rhsLen { dottedSymbol = prod.rhs[dot] } initial := false - if prod.lhs.isStart() && dot == 0 { + if prod.lhs.IsStart() && dot == 0 { initial = true } @@ -176,7 +178,7 @@ func (n stateNum) next() stateNum { type lrState struct { *kernel num stateNum - next map[symbol]kernelID + next map[symbol.Symbol]kernelID reducible map[productionID]struct{} // emptyProdItems stores items that have an empty production like `p → ε` and is reducible. diff --git a/grammar/lalr1.go b/grammar/lalr1.go index f1b8149..1667d84 100644 --- a/grammar/lalr1.go +++ b/grammar/lalr1.go @@ -1,6 +1,10 @@ package grammar -import "fmt" +import ( + "fmt" + + "github.com/nihei9/vartan/grammar/symbol" +) type stateAndLRItem struct { kernelID kernelID @@ -19,8 +23,8 @@ type lalr1Automaton struct { func genLALR1Automaton(lr0 *lr0Automaton, prods *productionSet, first *firstSet) (*lalr1Automaton, error) { // Set the look-ahead symbol <EOF> to the initial item: [S' → ・S, $] iniState := lr0.states[lr0.initialState] - iniState.items[0].lookAhead.symbols = map[symbol]struct{}{ - symbolEOF: {}, + iniState.items[0].lookAhead.symbols = map[symbol.Symbol]struct{}{ + symbol.SymbolEOF: {}, } var props []*propagation @@ -55,7 +59,7 @@ func genLALR1Automaton(lr0 *lr0Automaton, prods *productionSet, first *firstSet) return nil, fmt.Errorf("reducible item not found: %v", item.id) } if reducibleItem.lookAhead.symbols == nil { - reducibleItem.lookAhead.symbols = map[symbol]struct{}{} + reducibleItem.lookAhead.symbols = map[symbol.Symbol]struct{}{} } for a := range item.lookAhead.symbols { reducibleItem.lookAhead.symbols[a] = struct{}{} @@ -104,7 +108,7 @@ func genLALR1Automaton(lr0 *lr0Automaton, prods *productionSet, first *firstSet) } if nextItem.lookAhead.symbols == nil { - nextItem.lookAhead.symbols = map[symbol]struct{}{} + nextItem.lookAhead.symbols = map[symbol.Symbol]struct{}{} } for a := range item.lookAhead.symbols { @@ -138,7 +142,7 @@ func genLALR1Automaton(lr0 *lr0Automaton, prods *productionSet, first *firstSet) func genLALR1Closure(srcItem *lrItem, prods *productionSet, first *firstSet) ([]*lrItem, error) { items := []*lrItem{} - knownItems := map[lrItemID]map[symbol]struct{}{} + knownItems := map[lrItemID]map[symbol.Symbol]struct{}{} knownItemsProp := map[lrItemID]struct{}{} uncheckedItems := []*lrItem{} items = append(items, srcItem) @@ -146,7 +150,7 @@ func genLALR1Closure(srcItem *lrItem, prods *productionSet, first *firstSet) ([] for len(uncheckedItems) > 0 { nextUncheckedItems := []*lrItem{} for _, item := range uncheckedItems { - if item.dottedSymbol.isTerminal() { + if item.dottedSymbol.IsTerminal() { continue } @@ -155,7 +159,7 @@ func genLALR1Closure(srcItem *lrItem, prods *productionSet, first *firstSet) ([] return nil, fmt.Errorf("production not found: %v", item.prod) } - var fstSyms []symbol + var fstSyms []symbol.Symbol var isFstNullable bool { fst, err := first.find(p, item.dot+1) @@ -163,7 +167,7 @@ func genLALR1Closure(srcItem *lrItem, prods *productionSet, first *firstSet) ([] return nil, err } - fstSyms = make([]symbol, len(fst.symbols)) + fstSyms = make([]symbol.Symbol, len(fst.symbols)) i := 0 for s := range fst.symbols { fstSyms[i] = s @@ -176,7 +180,7 @@ func genLALR1Closure(srcItem *lrItem, prods *productionSet, first *firstSet) ([] ps, _ := prods.findByLHS(item.dottedSymbol) for _, prod := range ps { - var lookAhead []symbol + var lookAhead []symbol.Symbol { var lookAheadCount int if isFstNullable { @@ -185,7 +189,7 @@ func genLALR1Closure(srcItem *lrItem, prods *productionSet, first *firstSet) ([] lookAheadCount = len(fstSyms) } - lookAhead = make([]symbol, lookAheadCount) + lookAhead = make([]symbol.Symbol, lookAheadCount) i := 0 for _, s := range fstSyms { lookAhead[i] = s @@ -210,13 +214,13 @@ func genLALR1Closure(srcItem *lrItem, prods *productionSet, first *firstSet) ([] } } - newItem.lookAhead.symbols = map[symbol]struct{}{ + newItem.lookAhead.symbols = map[symbol.Symbol]struct{}{ a: {}, } items = append(items, newItem) if knownItems[newItem.id] == nil { - knownItems[newItem.id] = map[symbol]struct{}{} + knownItems[newItem.id] = map[symbol.Symbol]struct{}{} } knownItems[newItem.id][a] = struct{}{} nextUncheckedItems = append(nextUncheckedItems, newItem) @@ -297,7 +301,7 @@ func propagateLookAhead(lr0 *lr0Automaton, props []*propagation) error { } if destItem.lookAhead.symbols == nil { - destItem.lookAhead.symbols = map[symbol]struct{}{} + destItem.lookAhead.symbols = map[symbol.Symbol]struct{}{} } destItem.lookAhead.symbols[a] = struct{}{} diff --git a/grammar/lalr1_test.go b/grammar/lalr1_test.go index d6d0371..c57dc5c 100644 --- a/grammar/lalr1_test.go +++ b/grammar/lalr1_test.go @@ -4,7 +4,8 @@ import ( "strings" "testing" - spec "github.com/nihei9/vartan/spec/grammar" + "github.com/nihei9/vartan/grammar/symbol" + "github.com/nihei9/vartan/spec/grammar/parser" ) func TestGenLALR1Automaton(t *testing.T) { @@ -23,15 +24,14 @@ id: "[A-Za-z0-9_]+"; var gram *Grammar var automaton *lalr1Automaton { - ast, err := spec.Parse(strings.NewReader(src)) + ast, err := parser.Parse(strings.NewReader(src)) if err != nil { t.Fatal(err) } b := GrammarBuilder{ AST: ast, } - - gram, err = b.Build() + gram, err = b.build() if err != nil { t.Fatal(err) } @@ -66,42 +66,42 @@ id: "[A-Za-z0-9_]+"; expectedKernels := map[int][]*lrItem{ 0: { - withLookAhead(genLR0Item("s'", 0, "s"), symbolEOF), + withLookAhead(genLR0Item("s'", 0, "s"), symbol.SymbolEOF), }, 1: { - withLookAhead(genLR0Item("s'", 1, "s"), symbolEOF), + withLookAhead(genLR0Item("s'", 1, "s"), symbol.SymbolEOF), }, 2: { - withLookAhead(genLR0Item("s", 1, "l", "eq", "r"), symbolEOF), - withLookAhead(genLR0Item("r", 1, "l"), symbolEOF), + withLookAhead(genLR0Item("s", 1, "l", "eq", "r"), symbol.SymbolEOF), + withLookAhead(genLR0Item("r", 1, "l"), symbol.SymbolEOF), }, 3: { - withLookAhead(genLR0Item("s", 1, "r"), symbolEOF), + withLookAhead(genLR0Item("s", 1, "r"), symbol.SymbolEOF), }, 4: { - withLookAhead(genLR0Item("l", 1, "ref", "r"), genSym("eq"), symbolEOF), + withLookAhead(genLR0Item("l", 1, "ref", "r"), genSym("eq"), symbol.SymbolEOF), }, 5: { - withLookAhead(genLR0Item("l", 1, "id"), genSym("eq"), symbolEOF), + withLookAhead(genLR0Item("l", 1, "id"), genSym("eq"), symbol.SymbolEOF), }, 6: { - withLookAhead(genLR0Item("s", 2, "l", "eq", "r"), symbolEOF), + withLookAhead(genLR0Item("s", 2, "l", "eq", "r"), symbol.SymbolEOF), }, 7: { - withLookAhead(genLR0Item("l", 2, "ref", "r"), genSym("eq"), symbolEOF), + withLookAhead(genLR0Item("l", 2, "ref", "r"), genSym("eq"), symbol.SymbolEOF), }, 8: { - withLookAhead(genLR0Item("r", 1, "l"), genSym("eq"), symbolEOF), + withLookAhead(genLR0Item("r", 1, "l"), genSym("eq"), symbol.SymbolEOF), }, 9: { - withLookAhead(genLR0Item("s", 3, "l", "eq", "r"), symbolEOF), + withLookAhead(genLR0Item("s", 3, "l", "eq", "r"), symbol.SymbolEOF), }, } expectedStates := []*expectedLRState{ { kernelItems: expectedKernels[0], - nextStates: map[symbol][]*lrItem{ + nextStates: map[symbol.Symbol][]*lrItem{ genSym("s"): expectedKernels[1], genSym("l"): expectedKernels[2], genSym("r"): expectedKernels[3], @@ -112,14 +112,14 @@ id: "[A-Za-z0-9_]+"; }, { kernelItems: expectedKernels[1], - nextStates: map[symbol][]*lrItem{}, + nextStates: map[symbol.Symbol][]*lrItem{}, reducibleProds: []*production{ genProd("s'", "s"), }, }, { kernelItems: expectedKernels[2], - nextStates: map[symbol][]*lrItem{ + nextStates: map[symbol.Symbol][]*lrItem{ genSym("eq"): expectedKernels[6], }, reducibleProds: []*production{ @@ -128,14 +128,14 @@ id: "[A-Za-z0-9_]+"; }, { kernelItems: expectedKernels[3], - nextStates: map[symbol][]*lrItem{}, + nextStates: map[symbol.Symbol][]*lrItem{}, reducibleProds: []*production{ genProd("s", "r"), }, }, { kernelItems: expectedKernels[4], - nextStates: map[symbol][]*lrItem{ + nextStates: map[symbol.Symbol][]*lrItem{ genSym("r"): expectedKernels[7], genSym("l"): expectedKernels[8], genSym("ref"): expectedKernels[4], @@ -145,14 +145,14 @@ id: "[A-Za-z0-9_]+"; }, { kernelItems: expectedKernels[5], - nextStates: map[symbol][]*lrItem{}, + nextStates: map[symbol.Symbol][]*lrItem{}, reducibleProds: []*production{ genProd("l", "id"), }, }, { kernelItems: expectedKernels[6], - nextStates: map[symbol][]*lrItem{ + nextStates: map[symbol.Symbol][]*lrItem{ genSym("r"): expectedKernels[9], genSym("l"): expectedKernels[8], genSym("ref"): expectedKernels[4], @@ -162,21 +162,21 @@ id: "[A-Za-z0-9_]+"; }, { kernelItems: expectedKernels[7], - nextStates: map[symbol][]*lrItem{}, + nextStates: map[symbol.Symbol][]*lrItem{}, reducibleProds: []*production{ genProd("l", "ref", "r"), }, }, { kernelItems: expectedKernels[8], - nextStates: map[symbol][]*lrItem{}, + nextStates: map[symbol.Symbol][]*lrItem{}, reducibleProds: []*production{ genProd("r", "l"), }, }, { kernelItems: expectedKernels[9], - nextStates: map[symbol][]*lrItem{}, + nextStates: map[symbol.Symbol][]*lrItem{}, reducibleProds: []*production{ genProd("s", "l", "eq", "r"), }, diff --git a/grammar/lexical/compiler.go b/grammar/lexical/compiler.go new file mode 100644 index 0000000..61aa3f2 --- /dev/null +++ b/grammar/lexical/compiler.go @@ -0,0 +1,413 @@ +package lexical + +import ( + "bytes" + "fmt" + + "github.com/nihei9/vartan/compressor" + "github.com/nihei9/vartan/grammar/lexical/dfa" + psr "github.com/nihei9/vartan/grammar/lexical/parser" + spec "github.com/nihei9/vartan/spec/grammar" +) + +type CompileError struct { + Kind spec.LexKindName + Fragment bool + Cause error + Detail string +} + +func Compile(lexspec *LexSpec, compLv int) (*spec.LexicalSpec, error, []*CompileError) { + err := lexspec.Validate() + if err != nil { + return nil, fmt.Errorf("invalid lexical specification:\n%w", err), nil + } + + modeEntries, modeNames, modeName2ID, fragmetns := groupEntriesByLexMode(lexspec.Entries) + + modeSpecs := []*spec.CompiledLexModeSpec{ + nil, + } + for i, es := range modeEntries[1:] { + modeName := modeNames[i+1] + modeSpec, err, cerrs := compile(es, modeName2ID, fragmetns, compLv) + if err != nil { + return nil, fmt.Errorf("failed to compile in %v mode: %w", modeName, err), cerrs + } + modeSpecs = append(modeSpecs, modeSpec) + } + + var kindNames []spec.LexKindName + var name2ID map[spec.LexKindName]spec.LexKindID + { + name2ID = map[spec.LexKindName]spec.LexKindID{} + id := spec.LexKindIDMin + for _, modeSpec := range modeSpecs[1:] { + for _, name := range modeSpec.KindNames[1:] { + if _, ok := name2ID[name]; ok { + continue + } + name2ID[name] = id + id++ + } + } + + kindNames = make([]spec.LexKindName, len(name2ID)+1) + for name, id := range name2ID { + kindNames[id] = name + } + } + + var kindIDs [][]spec.LexKindID + { + kindIDs = make([][]spec.LexKindID, len(modeSpecs)) + for i, modeSpec := range modeSpecs[1:] { + ids := make([]spec.LexKindID, len(modeSpec.KindNames)) + for modeID, name := range modeSpec.KindNames { + if modeID == 0 { + continue + } + ids[modeID] = name2ID[name] + } + kindIDs[i+1] = ids + } + } + + return &spec.LexicalSpec{ + InitialModeID: spec.LexModeIDDefault, + ModeNames: modeNames, + KindNames: kindNames, + KindIDs: kindIDs, + CompressionLevel: compLv, + Specs: modeSpecs, + }, nil, nil +} + +func groupEntriesByLexMode(entries []*LexEntry) ([][]*LexEntry, []spec.LexModeName, map[spec.LexModeName]spec.LexModeID, map[spec.LexKindName]*LexEntry) { + modeNames := []spec.LexModeName{ + spec.LexModeNameNil, + spec.LexModeNameDefault, + } + modeName2ID := map[spec.LexModeName]spec.LexModeID{ + spec.LexModeNameNil: spec.LexModeIDNil, + spec.LexModeNameDefault: spec.LexModeIDDefault, + } + lastModeID := spec.LexModeIDDefault + modeEntries := [][]*LexEntry{ + nil, + {}, + } + fragments := map[spec.LexKindName]*LexEntry{} + for _, e := range entries { + if e.Fragment { + fragments[e.Kind] = e + continue + } + ms := e.Modes + if len(ms) == 0 { + ms = []spec.LexModeName{ + spec.LexModeNameDefault, + } + } + for _, modeName := range ms { + modeID, ok := modeName2ID[modeName] + if !ok { + modeID = lastModeID + 1 + lastModeID = modeID + modeName2ID[modeName] = modeID + modeNames = append(modeNames, modeName) + modeEntries = append(modeEntries, []*LexEntry{}) + } + modeEntries[modeID] = append(modeEntries[modeID], e) + } + } + return modeEntries, modeNames, modeName2ID, fragments +} + +func compile( + entries []*LexEntry, + modeName2ID map[spec.LexModeName]spec.LexModeID, + fragments map[spec.LexKindName]*LexEntry, + compLv int, +) (*spec.CompiledLexModeSpec, error, []*CompileError) { + var kindNames []spec.LexKindName + kindIDToName := map[spec.LexModeKindID]spec.LexKindName{} + var patterns map[spec.LexModeKindID][]byte + { + kindNames = append(kindNames, spec.LexKindNameNil) + patterns = map[spec.LexModeKindID][]byte{} + for i, e := range entries { + kindID := spec.LexModeKindID(i + 1) + + kindNames = append(kindNames, e.Kind) + kindIDToName[kindID] = e.Kind + patterns[kindID] = []byte(e.Pattern) + } + } + + push := []spec.LexModeID{ + spec.LexModeIDNil, + } + pop := []int{ + 0, + } + for _, e := range entries { + pushV := spec.LexModeIDNil + if e.Push != "" { + pushV = modeName2ID[e.Push] + } + push = append(push, pushV) + popV := 0 + if e.Pop { + popV = 1 + } + pop = append(pop, popV) + } + + fragmentPatterns := map[spec.LexKindName][]byte{} + for k, e := range fragments { + fragmentPatterns[k] = []byte(e.Pattern) + } + + fragmentCPTrees := make(map[spec.LexKindName]psr.CPTree, len(fragmentPatterns)) + { + var cerrs []*CompileError + for kind, pat := range fragmentPatterns { + p := psr.NewParser(kind, bytes.NewReader(pat)) + t, err := p.Parse() + if err != nil { + if err == psr.ParseErr { + detail, cause := p.Error() + cerrs = append(cerrs, &CompileError{ + Kind: kind, + Fragment: true, + Cause: cause, + Detail: detail, + }) + } else { + cerrs = append(cerrs, &CompileError{ + Kind: kind, + Fragment: true, + Cause: err, + }) + } + continue + } + fragmentCPTrees[kind] = t + } + if len(cerrs) > 0 { + return nil, fmt.Errorf("compile error"), cerrs + } + + err := psr.CompleteFragments(fragmentCPTrees) + if err != nil { + if err == psr.ParseErr { + for _, frag := range fragmentCPTrees { + kind, frags, err := frag.Describe() + if err != nil { + return nil, err, nil + } + + cerrs = append(cerrs, &CompileError{ + Kind: kind, + Fragment: true, + Cause: fmt.Errorf("fragment contains undefined fragments or cycles"), + Detail: fmt.Sprintf("%v", frags), + }) + } + + return nil, fmt.Errorf("compile error"), cerrs + } + + return nil, err, nil + } + } + + cpTrees := map[spec.LexModeKindID]psr.CPTree{} + { + pats := make([]*psr.PatternEntry, len(patterns)+1) + pats[spec.LexModeKindIDNil] = &psr.PatternEntry{ + ID: spec.LexModeKindIDNil, + } + for id, pattern := range patterns { + pats[id] = &psr.PatternEntry{ + ID: id, + Pattern: pattern, + } + } + + var cerrs []*CompileError + for _, pat := range pats { + if pat.ID == spec.LexModeKindIDNil { + continue + } + + p := psr.NewParser(kindIDToName[pat.ID], bytes.NewReader(pat.Pattern)) + t, err := p.Parse() + if err != nil { + if err == psr.ParseErr { + detail, cause := p.Error() + cerrs = append(cerrs, &CompileError{ + Kind: kindIDToName[pat.ID], + Fragment: false, + Cause: cause, + Detail: detail, + }) + } else { + cerrs = append(cerrs, &CompileError{ + Kind: kindIDToName[pat.ID], + Fragment: false, + Cause: err, + }) + } + continue + } + + complete, err := psr.ApplyFragments(t, fragmentCPTrees) + if err != nil { + return nil, err, nil + } + if !complete { + _, frags, err := t.Describe() + if err != nil { + return nil, err, nil + } + + cerrs = append(cerrs, &CompileError{ + Kind: kindIDToName[pat.ID], + Fragment: false, + Cause: fmt.Errorf("pattern contains undefined fragments"), + Detail: fmt.Sprintf("%v", frags), + }) + continue + } + + cpTrees[pat.ID] = t + } + if len(cerrs) > 0 { + return nil, fmt.Errorf("compile error"), cerrs + } + } + + var tranTab *spec.TransitionTable + { + root, symTab, err := dfa.ConvertCPTreeToByteTree(cpTrees) + if err != nil { + return nil, err, nil + } + d := dfa.GenDFA(root, symTab) + tranTab, err = dfa.GenTransitionTable(d) + if err != nil { + return nil, err, nil + } + } + + var err error + switch compLv { + case 2: + tranTab, err = compressTransitionTableLv2(tranTab) + if err != nil { + return nil, err, nil + } + case 1: + tranTab, err = compressTransitionTableLv1(tranTab) + if err != nil { + return nil, err, nil + } + } + + return &spec.CompiledLexModeSpec{ + KindNames: kindNames, + Push: push, + Pop: pop, + DFA: tranTab, + }, nil, nil +} + +const ( + CompressionLevelMin = 0 + CompressionLevelMax = 2 +) + +func compressTransitionTableLv2(tranTab *spec.TransitionTable) (*spec.TransitionTable, error) { + ueTab := compressor.NewUniqueEntriesTable() + { + orig, err := compressor.NewOriginalTable(convertStateIDSliceToIntSlice(tranTab.UncompressedTransition), tranTab.ColCount) + if err != nil { + return nil, err + } + err = ueTab.Compress(orig) + if err != nil { + return nil, err + } + } + + rdTab := compressor.NewRowDisplacementTable(0) + { + orig, err := compressor.NewOriginalTable(ueTab.UniqueEntries, ueTab.OriginalColCount) + if err != nil { + return nil, err + } + err = rdTab.Compress(orig) + if err != nil { + return nil, err + } + } + + tranTab.Transition = &spec.UniqueEntriesTable{ + UniqueEntries: &spec.RowDisplacementTable{ + OriginalRowCount: rdTab.OriginalRowCount, + OriginalColCount: rdTab.OriginalColCount, + EmptyValue: spec.StateIDNil, + Entries: convertIntSliceToStateIDSlice(rdTab.Entries), + Bounds: rdTab.Bounds, + RowDisplacement: rdTab.RowDisplacement, + }, + RowNums: ueTab.RowNums, + OriginalRowCount: ueTab.OriginalRowCount, + OriginalColCount: ueTab.OriginalColCount, + } + tranTab.UncompressedTransition = nil + + return tranTab, nil +} + +func compressTransitionTableLv1(tranTab *spec.TransitionTable) (*spec.TransitionTable, error) { + ueTab := compressor.NewUniqueEntriesTable() + { + orig, err := compressor.NewOriginalTable(convertStateIDSliceToIntSlice(tranTab.UncompressedTransition), tranTab.ColCount) + if err != nil { + return nil, err + } + err = ueTab.Compress(orig) + if err != nil { + return nil, err + } + } + + tranTab.Transition = &spec.UniqueEntriesTable{ + UncompressedUniqueEntries: convertIntSliceToStateIDSlice(ueTab.UniqueEntries), + RowNums: ueTab.RowNums, + OriginalRowCount: ueTab.OriginalRowCount, + OriginalColCount: ueTab.OriginalColCount, + } + tranTab.UncompressedTransition = nil + + return tranTab, nil +} + +func convertStateIDSliceToIntSlice(s []spec.StateID) []int { + is := make([]int, len(s)) + for i, v := range s { + is[i] = v.Int() + } + return is +} + +func convertIntSliceToStateIDSlice(s []int) []spec.StateID { + ss := make([]spec.StateID, len(s)) + for i, v := range s { + ss[i] = spec.StateID(v) + } + return ss +} diff --git a/grammar/lexical/compiler_test.go b/grammar/lexical/compiler_test.go new file mode 100644 index 0000000..f2ef0f2 --- /dev/null +++ b/grammar/lexical/compiler_test.go @@ -0,0 +1,338 @@ +package lexical + +import ( + "encoding/json" + "fmt" + "testing" + + spec "github.com/nihei9/vartan/spec/grammar" +) + +func TestLexSpec_Validate(t *testing.T) { + // We expect that the spelling inconsistency error will occur. + spec := &LexSpec{ + Entries: []*LexEntry{ + { + Modes: []spec.LexModeName{ + // 'Default' is the spelling inconsistency because 'default' is predefined. + "Default", + }, + Kind: "foo", + Pattern: "foo", + }, + }, + } + err := spec.Validate() + if err == nil { + t.Fatalf("expected error didn't occur") + } +} + +func TestSnakeCaseToUpperCamelCase(t *testing.T) { + tests := []struct { + snake string + camel string + }{ + { + snake: "foo", + camel: "Foo", + }, + { + snake: "foo_bar", + camel: "FooBar", + }, + { + snake: "foo_bar_baz", + camel: "FooBarBaz", + }, + { + snake: "Foo", + camel: "Foo", + }, + { + snake: "fooBar", + camel: "FooBar", + }, + { + snake: "FOO", + camel: "FOO", + }, + { + snake: "FOO_BAR", + camel: "FOOBAR", + }, + { + snake: "_foo_bar_", + camel: "FooBar", + }, + { + snake: "___foo___bar___", + camel: "FooBar", + }, + } + for _, tt := range tests { + c := SnakeCaseToUpperCamelCase(tt.snake) + if c != tt.camel { + t.Errorf("unexpected string; want: %v, got: %v", tt.camel, c) + } + } +} + +func TestFindSpellingInconsistencies(t *testing.T) { + tests := []struct { + ids []string + duplicated [][]string + }{ + { + ids: []string{"foo", "foo"}, + duplicated: nil, + }, + { + ids: []string{"foo", "Foo"}, + duplicated: [][]string{{"Foo", "foo"}}, + }, + { + ids: []string{"foo", "foo", "Foo"}, + duplicated: [][]string{{"Foo", "foo"}}, + }, + { + ids: []string{"foo_bar_baz", "FooBarBaz"}, + duplicated: [][]string{{"FooBarBaz", "foo_bar_baz"}}, + }, + { + ids: []string{"foo", "Foo", "bar", "Bar"}, + duplicated: [][]string{{"Bar", "bar"}, {"Foo", "foo"}}, + }, + { + ids: []string{"foo", "Foo", "bar", "Bar", "baz", "bra"}, + duplicated: [][]string{{"Bar", "bar"}, {"Foo", "foo"}}, + }, + } + for i, tt := range tests { + t.Run(fmt.Sprintf("#%v", i), func(t *testing.T) { + duplicated := FindSpellingInconsistencies(tt.ids) + if len(duplicated) != len(tt.duplicated) { + t.Fatalf("unexpected IDs; want: %#v, got: %#v", tt.duplicated, duplicated) + } + for i, dupIDs := range duplicated { + if len(dupIDs) != len(tt.duplicated[i]) { + t.Fatalf("unexpected IDs; want: %#v, got: %#v", tt.duplicated[i], dupIDs) + } + for j, id := range dupIDs { + if id != tt.duplicated[i][j] { + t.Fatalf("unexpected IDs; want: %#v, got: %#v", tt.duplicated[i], dupIDs) + } + } + } + }) + } +} + +func TestCompile(t *testing.T) { + tests := []struct { + Caption string + Spec string + Err bool + }{ + { + Caption: "allow duplicates names between fragments and non-fragments", + Spec: ` +{ + "name": "test", + "entries": [ + { + "kind": "a2z", + "pattern": "\\f{a2z}" + }, + { + "fragment": true, + "kind": "a2z", + "pattern": "[a-z]" + } + ] +} +`, + }, + { + Caption: "don't allow duplicates names in non-fragments", + Spec: ` +{ + "name": "test", + "entries": [ + { + "kind": "a2z", + "pattern": "a|b|c|d|e|f|g|h|i|j|k|l|m|n|o|p|q|r|s|t|u|v|w|x|y|z" + }, + { + "kind": "a2z", + "pattern": "[a-z]" + } + ] +} +`, + Err: true, + }, + { + Caption: "don't allow duplicates names in fragments", + Spec: ` +{ + "name": "test", + "entries": [ + { + "kind": "a2z", + "pattern": "\\f{a2z}" + }, + { + "fragments": true, + "kind": "a2z", + "pattern": "a|b|c|d|e|f|g|h|i|j|k|l|m|n|o|p|q|r|s|t|u|v|w|x|y|z" + }, + { + "fragments": true, + "kind": "a2z", + "pattern": "[a-z]" + } + ] +} +`, + Err: true, + }, + { + Caption: "don't allow kind names in the same mode to contain spelling inconsistencies", + Spec: ` +{ + "name": "test", + "entries": [ + { + "kind": "foo_1", + "pattern": "foo_1" + }, + { + "kind": "foo1", + "pattern": "foo1" + } + ] +} +`, + Err: true, + }, + { + Caption: "don't allow kind names across modes to contain spelling inconsistencies", + Spec: ` +{ + "name": "test", + "entries": [ + { + "modes": ["default"], + "kind": "foo_1", + "pattern": "foo_1" + }, + { + "modes": ["other_mode"], + "kind": "foo1", + "pattern": "foo1" + } + ] +} +`, + Err: true, + }, + { + Caption: "don't allow mode names to contain spelling inconsistencies", + Spec: ` +{ + "name": "test", + "entries": [ + { + "modes": ["foo_1"], + "kind": "a", + "pattern": "a" + }, + { + "modes": ["foo1"], + "kind": "b", + "pattern": "b" + } + ] +} +`, + Err: true, + }, + { + Caption: "allow fragment names in the same mode to contain spelling inconsistencies because fragments will not appear in output files", + Spec: ` +{ + "name": "test", + "entries": [ + { + "kind": "a", + "pattern": "a" + }, + { + "fragment": true, + "kind": "foo_1", + "pattern": "foo_1" + }, + { + "fragment": true, + "kind": "foo1", + "pattern": "foo1" + } + ] +} +`, + }, + { + Caption: "allow fragment names across modes to contain spelling inconsistencies because fragments will not appear in output files", + Spec: ` +{ + "name": "test", + "entries": [ + { + "modes": ["default"], + "kind": "a", + "pattern": "a" + }, + { + "modes": ["default"], + "fragment": true, + "kind": "foo_1", + "pattern": "foo_1" + }, + { + "modes": ["other_mode"], + "fragment": true, + "kind": "foo1", + "pattern": "foo1" + } + ] +} +`, + }, + } + for i, tt := range tests { + t.Run(fmt.Sprintf("#%v %s", i, tt.Caption), func(t *testing.T) { + lspec := &LexSpec{} + err := json.Unmarshal([]byte(tt.Spec), lspec) + if err != nil { + t.Fatalf("%v", err) + } + clspec, err, _ := Compile(lspec, CompressionLevelMin) + if tt.Err { + if err == nil { + t.Fatalf("expected an error") + } + if clspec != nil { + t.Fatalf("Compile function mustn't return a compiled specification") + } + } else { + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if clspec == nil { + t.Fatalf("Compile function must return a compiled specification") + } + } + }) + } +} diff --git a/grammar/lexical/dfa/dfa.go b/grammar/lexical/dfa/dfa.go new file mode 100644 index 0000000..850264a --- /dev/null +++ b/grammar/lexical/dfa/dfa.go @@ -0,0 +1,173 @@ +package dfa + +import ( + "sort" + + spec "github.com/nihei9/vartan/spec/grammar" +) + +type symbolTable struct { + symPos2Byte map[symbolPosition]byteRange + endPos2ID map[symbolPosition]spec.LexModeKindID +} + +func genSymbolTable(root byteTree) *symbolTable { + symTab := &symbolTable{ + symPos2Byte: map[symbolPosition]byteRange{}, + endPos2ID: map[symbolPosition]spec.LexModeKindID{}, + } + return genSymTab(symTab, root) +} + +func genSymTab(symTab *symbolTable, node byteTree) *symbolTable { + if node == nil { + return symTab + } + + switch n := node.(type) { + case *symbolNode: + symTab.symPos2Byte[n.pos] = byteRange{ + from: n.from, + to: n.to, + } + case *endMarkerNode: + symTab.endPos2ID[n.pos] = n.id + default: + left, right := node.children() + genSymTab(symTab, left) + genSymTab(symTab, right) + } + return symTab +} + +type DFA struct { + States []string + InitialState string + AcceptingStatesTable map[string]spec.LexModeKindID + TransitionTable map[string][256]string +} + +func GenDFA(root byteTree, symTab *symbolTable) *DFA { + initialState := root.first() + initialStateHash := initialState.hash() + stateMap := map[string]*symbolPositionSet{ + initialStateHash: initialState, + } + tranTab := map[string][256]string{} + { + follow := genFollowTable(root) + unmarkedStates := map[string]*symbolPositionSet{ + initialStateHash: initialState, + } + for len(unmarkedStates) > 0 { + nextUnmarkedStates := map[string]*symbolPositionSet{} + for hash, state := range unmarkedStates { + tranTabOfState := [256]*symbolPositionSet{} + for _, pos := range state.set() { + if pos.isEndMark() { + continue + } + valRange := symTab.symPos2Byte[pos] + for symVal := valRange.from; symVal <= valRange.to; symVal++ { + if tranTabOfState[symVal] == nil { + tranTabOfState[symVal] = newSymbolPositionSet() + } + tranTabOfState[symVal].merge(follow[pos]) + } + } + for _, t := range tranTabOfState { + if t == nil { + continue + } + h := t.hash() + if _, ok := stateMap[h]; ok { + continue + } + stateMap[h] = t + nextUnmarkedStates[h] = t + } + tabOfState := [256]string{} + for v, t := range tranTabOfState { + if t == nil { + continue + } + tabOfState[v] = t.hash() + } + tranTab[hash] = tabOfState + } + unmarkedStates = nextUnmarkedStates + } + } + + accTab := map[string]spec.LexModeKindID{} + { + for h, s := range stateMap { + for _, pos := range s.set() { + if !pos.isEndMark() { + continue + } + priorID, ok := accTab[h] + if !ok { + accTab[h] = symTab.endPos2ID[pos] + } else { + id := symTab.endPos2ID[pos] + if id < priorID { + accTab[h] = id + } + } + } + } + } + + var states []string + { + for s := range stateMap { + states = append(states, s) + } + sort.Slice(states, func(i, j int) bool { + return states[i] < states[j] + }) + } + + return &DFA{ + States: states, + InitialState: initialStateHash, + AcceptingStatesTable: accTab, + TransitionTable: tranTab, + } +} + +func GenTransitionTable(dfa *DFA) (*spec.TransitionTable, error) { + stateHash2ID := map[string]spec.StateID{} + for i, s := range dfa.States { + // Since 0 represents an invalid value in a transition table, + // assign a number greater than or equal to 1 to states. + stateHash2ID[s] = spec.StateID(i + spec.StateIDMin.Int()) + } + + acc := make([]spec.LexModeKindID, len(dfa.States)+1) + for _, s := range dfa.States { + id, ok := dfa.AcceptingStatesTable[s] + if !ok { + continue + } + acc[stateHash2ID[s]] = id + } + + rowCount := len(dfa.States) + 1 + colCount := 256 + tran := make([]spec.StateID, rowCount*colCount) + for s, tab := range dfa.TransitionTable { + for v, to := range tab { + tran[stateHash2ID[s].Int()*256+v] = stateHash2ID[to] + } + } + + return &spec.TransitionTable{ + InitialStateID: stateHash2ID[dfa.InitialState], + AcceptingStates: acc, + UncompressedTransition: tran, + RowCount: rowCount, + ColCount: colCount, + }, nil +} diff --git a/grammar/lexical/dfa/dfa_test.go b/grammar/lexical/dfa/dfa_test.go new file mode 100644 index 0000000..ae71875 --- /dev/null +++ b/grammar/lexical/dfa/dfa_test.go @@ -0,0 +1,121 @@ +package dfa + +import ( + "strings" + "testing" + + "github.com/nihei9/vartan/grammar/lexical/parser" + spec "github.com/nihei9/vartan/spec/grammar" +) + +func TestGenDFA(t *testing.T) { + p := parser.NewParser(spec.LexKindName("test"), strings.NewReader("(a|b)*abb")) + cpt, err := p.Parse() + if err != nil { + t.Fatal(err) + } + bt, symTab, err := ConvertCPTreeToByteTree(map[spec.LexModeKindID]parser.CPTree{ + spec.LexModeKindIDMin: cpt, + }) + if err != nil { + t.Fatal(err) + } + dfa := GenDFA(bt, symTab) + if dfa == nil { + t.Fatalf("DFA is nil") + } + + symPos := func(n uint16) symbolPosition { + pos, err := newSymbolPosition(n, false) + if err != nil { + panic(err) + } + return pos + } + + endPos := func(n uint16) symbolPosition { + pos, err := newSymbolPosition(n, true) + if err != nil { + panic(err) + } + return pos + } + + s0 := newSymbolPositionSet().add(symPos(1)).add(symPos(2)).add(symPos(3)) + s1 := newSymbolPositionSet().add(symPos(1)).add(symPos(2)).add(symPos(3)).add(symPos(4)) + s2 := newSymbolPositionSet().add(symPos(1)).add(symPos(2)).add(symPos(3)).add(symPos(5)) + s3 := newSymbolPositionSet().add(symPos(1)).add(symPos(2)).add(symPos(3)).add(endPos(6)) + + rune2Int := func(char rune, index int) uint8 { + return uint8([]byte(string(char))[index]) + } + + tranS0 := [256]string{} + tranS0[rune2Int('a', 0)] = s1.hash() + tranS0[rune2Int('b', 0)] = s0.hash() + + tranS1 := [256]string{} + tranS1[rune2Int('a', 0)] = s1.hash() + tranS1[rune2Int('b', 0)] = s2.hash() + + tranS2 := [256]string{} + tranS2[rune2Int('a', 0)] = s1.hash() + tranS2[rune2Int('b', 0)] = s3.hash() + + tranS3 := [256]string{} + tranS3[rune2Int('a', 0)] = s1.hash() + tranS3[rune2Int('b', 0)] = s0.hash() + + expectedTranTab := map[string][256]string{ + s0.hash(): tranS0, + s1.hash(): tranS1, + s2.hash(): tranS2, + s3.hash(): tranS3, + } + if len(dfa.TransitionTable) != len(expectedTranTab) { + t.Errorf("transition table is mismatched: want: %v entries, got: %v entries", len(expectedTranTab), len(dfa.TransitionTable)) + } + for h, eTranTab := range expectedTranTab { + tranTab, ok := dfa.TransitionTable[h] + if !ok { + t.Errorf("no entry; hash: %v", h) + continue + } + if len(tranTab) != len(eTranTab) { + t.Errorf("transition table is mismatched: hash: %v, want: %v entries, got: %v entries", h, len(eTranTab), len(tranTab)) + } + for c, eNext := range eTranTab { + if eNext == "" { + continue + } + + next := tranTab[c] + if next == "" { + t.Errorf("no enatry: hash: %v, char: %v", h, c) + } + if next != eNext { + t.Errorf("next state is mismatched: want: %v, got: %v", eNext, next) + } + } + } + + if dfa.InitialState != s0.hash() { + t.Errorf("initial state is mismatched: want: %v, got: %v", s0.hash(), dfa.InitialState) + } + + accTab := map[string]spec.LexModeKindID{ + s3.hash(): 1, + } + if len(dfa.AcceptingStatesTable) != len(accTab) { + t.Errorf("accepting states are mismatched: want: %v entries, got: %v entries", len(accTab), len(dfa.AcceptingStatesTable)) + } + for eState, eID := range accTab { + id, ok := dfa.AcceptingStatesTable[eState] + if !ok { + t.Errorf("accepting state is not found: state: %v", eState) + } + if id != eID { + t.Errorf("ID is mismatched: state: %v, want: %v, got: %v", eState, eID, id) + } + } +} diff --git a/grammar/lexical/dfa/symbol_position.go b/grammar/lexical/dfa/symbol_position.go new file mode 100644 index 0000000..f154251 --- /dev/null +++ b/grammar/lexical/dfa/symbol_position.go @@ -0,0 +1,182 @@ +package dfa + +import ( + "encoding/binary" + "fmt" + "strings" +) + +type symbolPosition uint16 + +const ( + symbolPositionNil symbolPosition = 0x0000 + + symbolPositionMin uint16 = 0x0001 + symbolPositionMax uint16 = 0x7fff + + symbolPositionMaskSymbol uint16 = 0x0000 + symbolPositionMaskEndMark uint16 = 0x8000 + + symbolPositionMaskValue uint16 = 0x7fff +) + +func newSymbolPosition(n uint16, endMark bool) (symbolPosition, error) { + if n < symbolPositionMin || n > symbolPositionMax { + return symbolPositionNil, fmt.Errorf("symbol position must be within %v to %v: n: %v, endMark: %v", symbolPositionMin, symbolPositionMax, n, endMark) + } + if endMark { + return symbolPosition(n | symbolPositionMaskEndMark), nil + } + return symbolPosition(n | symbolPositionMaskSymbol), nil +} + +func (p symbolPosition) String() string { + if p.isEndMark() { + return fmt.Sprintf("end#%v", uint16(p)&symbolPositionMaskValue) + } + return fmt.Sprintf("sym#%v", uint16(p)&symbolPositionMaskValue) +} + +func (p symbolPosition) isEndMark() bool { + return uint16(p)&symbolPositionMaskEndMark > 1 +} + +func (p symbolPosition) describe() (uint16, bool) { + v := uint16(p) & symbolPositionMaskValue + if p.isEndMark() { + return v, true + } + return v, false +} + +type symbolPositionSet struct { + // `s` represents a set of symbol positions. + // However, immediately after adding a symbol position, the elements may be duplicated. + // When you need an aligned set with no duplicates, you can get such value via the set function. + s []symbolPosition + sorted bool +} + +func newSymbolPositionSet() *symbolPositionSet { + return &symbolPositionSet{ + s: []symbolPosition{}, + sorted: false, + } +} + +func (s *symbolPositionSet) String() string { + if len(s.s) <= 0 { + return "{}" + } + ps := s.sortAndRemoveDuplicates() + var b strings.Builder + fmt.Fprintf(&b, "{") + for i, p := range ps { + if i <= 0 { + fmt.Fprintf(&b, "%v", p) + continue + } + fmt.Fprintf(&b, ", %v", p) + } + fmt.Fprintf(&b, "}") + return b.String() +} + +func (s *symbolPositionSet) set() []symbolPosition { + s.sortAndRemoveDuplicates() + return s.s +} + +func (s *symbolPositionSet) add(pos symbolPosition) *symbolPositionSet { + s.s = append(s.s, pos) + s.sorted = false + return s +} + +func (s *symbolPositionSet) merge(t *symbolPositionSet) *symbolPositionSet { + s.s = append(s.s, t.s...) + s.sorted = false + return s +} + +func (s *symbolPositionSet) hash() string { + if len(s.s) <= 0 { + return "" + } + sorted := s.sortAndRemoveDuplicates() + var buf []byte + for _, p := range sorted { + b := make([]byte, 8) + binary.PutUvarint(b, uint64(p)) + buf = append(buf, b...) + } + // Convert to a string to be able to use it as a key of a map. + // But note this byte sequence is made from values of symbol positions, + // so this is not a well-formed UTF-8 sequence. + return string(buf) +} + +func (s *symbolPositionSet) sortAndRemoveDuplicates() []symbolPosition { + if s.sorted { + return s.s + } + + sortSymbolPositions(s.s, 0, len(s.s)-1) + + // Remove duplicates. + lastV := s.s[0] + nextIdx := 1 + for _, v := range s.s[1:] { + if v == lastV { + continue + } + s.s[nextIdx] = v + nextIdx++ + lastV = v + } + s.s = s.s[:nextIdx] + s.sorted = true + + return s.s +} + +// sortSymbolPositions sorts a slice of symbol positions as it uses quick sort. +func sortSymbolPositions(ps []symbolPosition, left, right int) { + if left >= right { + return + } + var pivot symbolPosition + { + // Use a median as a pivot. + p1 := ps[left] + p2 := ps[(left+right)/2] + p3 := ps[right] + if p1 > p2 { + p1, p2 = p2, p1 + } + if p2 > p3 { + p2 = p3 + if p1 > p2 { + p2 = p1 + } + } + pivot = p2 + } + i := left + j := right + for i <= j { + for ps[i] < pivot { + i++ + } + for ps[j] > pivot { + j-- + } + if i <= j { + ps[i], ps[j] = ps[j], ps[i] + i++ + j-- + } + } + sortSymbolPositions(ps, left, j) + sortSymbolPositions(ps, i, right) +} diff --git a/grammar/lexical/dfa/symbol_position_test.go b/grammar/lexical/dfa/symbol_position_test.go new file mode 100644 index 0000000..c867f64 --- /dev/null +++ b/grammar/lexical/dfa/symbol_position_test.go @@ -0,0 +1,79 @@ +package dfa + +import ( + "fmt" + "testing" +) + +func TestNewSymbolPosition(t *testing.T) { + tests := []struct { + n uint16 + endMark bool + err bool + }{ + { + n: 0, + endMark: false, + err: true, + }, + { + n: 0, + endMark: true, + err: true, + }, + { + n: symbolPositionMin - 1, + endMark: false, + err: true, + }, + { + n: symbolPositionMin - 1, + endMark: true, + err: true, + }, + { + n: symbolPositionMin, + endMark: false, + }, + { + n: symbolPositionMin, + endMark: true, + }, + { + n: symbolPositionMax, + endMark: false, + }, + { + n: symbolPositionMax, + endMark: true, + }, + { + n: symbolPositionMax + 1, + endMark: false, + err: true, + }, + { + n: symbolPositionMax + 1, + endMark: true, + err: true, + }, + } + for i, tt := range tests { + t.Run(fmt.Sprintf("#%v n: %v, endMark: %v", i, tt.n, tt.endMark), func(t *testing.T) { + pos, err := newSymbolPosition(tt.n, tt.endMark) + if tt.err { + if err == nil { + t.Fatal("err is nil") + } + return + } + if err != nil { + t.Fatal(err) + } + n, endMark := pos.describe() + if n != tt.n || endMark != tt.endMark { + t.Errorf("unexpected symbol position: want: n: %v, endMark: %v, got: n: %v, endMark: %v", tt.n, tt.endMark, n, endMark) + } + }) + } +} diff --git a/grammar/lexical/dfa/tree.go b/grammar/lexical/dfa/tree.go new file mode 100644 index 0000000..cd6081e --- /dev/null +++ b/grammar/lexical/dfa/tree.go @@ -0,0 +1,567 @@ +package dfa + +import ( + "fmt" + "io" + "sort" + + "github.com/nihei9/vartan/grammar/lexical/parser" + spec "github.com/nihei9/vartan/spec/grammar" + "github.com/nihei9/vartan/utf8" +) + +type byteTree interface { + fmt.Stringer + children() (byteTree, byteTree) + nullable() bool + first() *symbolPositionSet + last() *symbolPositionSet + clone() byteTree +} + +var ( + _ byteTree = &symbolNode{} + _ byteTree = &endMarkerNode{} + _ byteTree = &concatNode{} + _ byteTree = &altNode{} + _ byteTree = &repeatNode{} + _ byteTree = &optionNode{} +) + +type byteRange struct { + from byte + to byte +} + +type symbolNode struct { + byteRange + pos symbolPosition + firstMemo *symbolPositionSet + lastMemo *symbolPositionSet +} + +func newSymbolNode(value byte) *symbolNode { + return &symbolNode{ + byteRange: byteRange{ + from: value, + to: value, + }, + pos: symbolPositionNil, + } +} + +func newRangeSymbolNode(from, to byte) *symbolNode { + return &symbolNode{ + byteRange: byteRange{ + from: from, + to: to, + }, + pos: symbolPositionNil, + } +} + +func (n *symbolNode) String() string { + return fmt.Sprintf("symbol: value: %v-%v, pos: %v", n.from, n.to, n.pos) +} + +func (n *symbolNode) children() (byteTree, byteTree) { + return nil, nil +} + +func (n *symbolNode) nullable() bool { + return false +} + +func (n *symbolNode) first() *symbolPositionSet { + if n.firstMemo == nil { + n.firstMemo = newSymbolPositionSet() + n.firstMemo.add(n.pos) + } + return n.firstMemo +} + +func (n *symbolNode) last() *symbolPositionSet { + if n.lastMemo == nil { + n.lastMemo = newSymbolPositionSet() + n.lastMemo.add(n.pos) + } + return n.lastMemo +} + +func (n *symbolNode) clone() byteTree { + return newRangeSymbolNode(n.from, n.to) +} + +type endMarkerNode struct { + id spec.LexModeKindID + pos symbolPosition + firstMemo *symbolPositionSet + lastMemo *symbolPositionSet +} + +func newEndMarkerNode(id spec.LexModeKindID) *endMarkerNode { + return &endMarkerNode{ + id: id, + pos: symbolPositionNil, + } +} + +func (n *endMarkerNode) String() string { + return fmt.Sprintf("end: pos: %v", n.pos) +} + +func (n *endMarkerNode) children() (byteTree, byteTree) { + return nil, nil +} + +func (n *endMarkerNode) nullable() bool { + return false +} + +func (n *endMarkerNode) first() *symbolPositionSet { + if n.firstMemo == nil { + n.firstMemo = newSymbolPositionSet() + n.firstMemo.add(n.pos) + } + return n.firstMemo +} + +func (n *endMarkerNode) last() *symbolPositionSet { + if n.lastMemo == nil { + n.lastMemo = newSymbolPositionSet() + n.lastMemo.add(n.pos) + } + return n.lastMemo +} + +func (n *endMarkerNode) clone() byteTree { + return newEndMarkerNode(n.id) +} + +type concatNode struct { + left byteTree + right byteTree + firstMemo *symbolPositionSet + lastMemo *symbolPositionSet +} + +func newConcatNode(left, right byteTree) *concatNode { + return &concatNode{ + left: left, + right: right, + } +} + +func (n *concatNode) String() string { + return "concat" +} + +func (n *concatNode) children() (byteTree, byteTree) { + return n.left, n.right +} + +func (n *concatNode) nullable() bool { + return n.left.nullable() && n.right.nullable() +} + +func (n *concatNode) first() *symbolPositionSet { + if n.firstMemo == nil { + n.firstMemo = newSymbolPositionSet() + n.firstMemo.merge(n.left.first()) + if n.left.nullable() { + n.firstMemo.merge(n.right.first()) + } + n.firstMemo.sortAndRemoveDuplicates() + } + return n.firstMemo +} + +func (n *concatNode) last() *symbolPositionSet { + if n.lastMemo == nil { + n.lastMemo = newSymbolPositionSet() + n.lastMemo.merge(n.right.last()) + if n.right.nullable() { + n.lastMemo.merge(n.left.last()) + } + n.lastMemo.sortAndRemoveDuplicates() + } + return n.lastMemo +} + +func (n *concatNode) clone() byteTree { + return newConcatNode(n.left.clone(), n.right.clone()) +} + +type altNode struct { + left byteTree + right byteTree + firstMemo *symbolPositionSet + lastMemo *symbolPositionSet +} + +func newAltNode(left, right byteTree) *altNode { + return &altNode{ + left: left, + right: right, + } +} + +func (n *altNode) String() string { + return "alt" +} + +func (n *altNode) children() (byteTree, byteTree) { + return n.left, n.right +} + +func (n *altNode) nullable() bool { + return n.left.nullable() || n.right.nullable() +} + +func (n *altNode) first() *symbolPositionSet { + if n.firstMemo == nil { + n.firstMemo = newSymbolPositionSet() + n.firstMemo.merge(n.left.first()) + n.firstMemo.merge(n.right.first()) + n.firstMemo.sortAndRemoveDuplicates() + } + return n.firstMemo +} + +func (n *altNode) last() *symbolPositionSet { + if n.lastMemo == nil { + n.lastMemo = newSymbolPositionSet() + n.lastMemo.merge(n.left.last()) + n.lastMemo.merge(n.right.last()) + n.lastMemo.sortAndRemoveDuplicates() + } + return n.lastMemo +} + +func (n *altNode) clone() byteTree { + return newAltNode(n.left.clone(), n.right.clone()) +} + +type repeatNode struct { + left byteTree + firstMemo *symbolPositionSet + lastMemo *symbolPositionSet +} + +func newRepeatNode(left byteTree) *repeatNode { + return &repeatNode{ + left: left, + } +} + +func (n *repeatNode) String() string { + return "repeat" +} + +func (n *repeatNode) children() (byteTree, byteTree) { + return n.left, nil +} + +func (n *repeatNode) nullable() bool { + return true +} + +func (n *repeatNode) first() *symbolPositionSet { + if n.firstMemo == nil { + n.firstMemo = newSymbolPositionSet() + n.firstMemo.merge(n.left.first()) + n.firstMemo.sortAndRemoveDuplicates() + } + return n.firstMemo +} + +func (n *repeatNode) last() *symbolPositionSet { + if n.lastMemo == nil { + n.lastMemo = newSymbolPositionSet() + n.lastMemo.merge(n.left.last()) + n.lastMemo.sortAndRemoveDuplicates() + } + return n.lastMemo +} + +func (n *repeatNode) clone() byteTree { + return newRepeatNode(n.left.clone()) +} + +type optionNode struct { + left byteTree + firstMemo *symbolPositionSet + lastMemo *symbolPositionSet +} + +func newOptionNode(left byteTree) *optionNode { + return &optionNode{ + left: left, + } +} + +func (n *optionNode) String() string { + return "option" +} + +func (n *optionNode) children() (byteTree, byteTree) { + return n.left, nil +} + +func (n *optionNode) nullable() bool { + return true +} + +func (n *optionNode) first() *symbolPositionSet { + if n.firstMemo == nil { + n.firstMemo = newSymbolPositionSet() + n.firstMemo.merge(n.left.first()) + n.firstMemo.sortAndRemoveDuplicates() + } + return n.firstMemo +} + +func (n *optionNode) last() *symbolPositionSet { + if n.lastMemo == nil { + n.lastMemo = newSymbolPositionSet() + n.lastMemo.merge(n.left.last()) + n.lastMemo.sortAndRemoveDuplicates() + } + return n.lastMemo +} + +func (n *optionNode) clone() byteTree { + return newOptionNode(n.left.clone()) +} + +type followTable map[symbolPosition]*symbolPositionSet + +func genFollowTable(root byteTree) followTable { + follow := followTable{} + calcFollow(follow, root) + return follow +} + +func calcFollow(follow followTable, ast byteTree) { + if ast == nil { + return + } + left, right := ast.children() + calcFollow(follow, left) + calcFollow(follow, right) + switch n := ast.(type) { + case *concatNode: + l, r := n.children() + for _, p := range l.last().set() { + if _, ok := follow[p]; !ok { + follow[p] = newSymbolPositionSet() + } + follow[p].merge(r.first()) + } + case *repeatNode: + for _, p := range n.last().set() { + if _, ok := follow[p]; !ok { + follow[p] = newSymbolPositionSet() + } + follow[p].merge(n.first()) + } + } +} + +func positionSymbols(node byteTree, n uint16) (uint16, error) { + if node == nil { + return n, nil + } + + l, r := node.children() + p := n + p, err := positionSymbols(l, p) + if err != nil { + return p, err + } + p, err = positionSymbols(r, p) + if err != nil { + return p, err + } + switch n := node.(type) { + case *symbolNode: + n.pos, err = newSymbolPosition(p, false) + if err != nil { + return p, err + } + p++ + case *endMarkerNode: + n.pos, err = newSymbolPosition(p, true) + if err != nil { + return p, err + } + p++ + } + node.first() + node.last() + return p, nil +} + +func concat(ts ...byteTree) byteTree { + nonNilNodes := []byteTree{} + for _, t := range ts { + if t == nil { + continue + } + nonNilNodes = append(nonNilNodes, t) + } + if len(nonNilNodes) <= 0 { + return nil + } + if len(nonNilNodes) == 1 { + return nonNilNodes[0] + } + concat := newConcatNode(nonNilNodes[0], nonNilNodes[1]) + for _, t := range nonNilNodes[2:] { + concat = newConcatNode(concat, t) + } + return concat +} + +func oneOf(ts ...byteTree) byteTree { + nonNilNodes := []byteTree{} + for _, t := range ts { + if t == nil { + continue + } + nonNilNodes = append(nonNilNodes, t) + } + if len(nonNilNodes) <= 0 { + return nil + } + if len(nonNilNodes) == 1 { + return nonNilNodes[0] + } + alt := newAltNode(nonNilNodes[0], nonNilNodes[1]) + for _, t := range nonNilNodes[2:] { + alt = newAltNode(alt, t) + } + return alt +} + +//nolint:unused +func printByteTree(w io.Writer, t byteTree, ruledLine string, childRuledLinePrefix string, withAttrs bool) { + if t == nil { + return + } + fmt.Fprintf(w, "%v%v", ruledLine, t) + if withAttrs { + fmt.Fprintf(w, ", nullable: %v, first: %v, last: %v", t.nullable(), t.first(), t.last()) + } + fmt.Fprintf(w, "\n") + left, right := t.children() + children := []byteTree{} + if left != nil { + children = append(children, left) + } + if right != nil { + children = append(children, right) + } + num := len(children) + for i, child := range children { + line := "└─ " + if num > 1 { + if i == 0 { + line = "├─ " + } else if i < num-1 { + line = "│ " + } + } + prefix := "│ " + if i >= num-1 { + prefix = " " + } + printByteTree(w, child, childRuledLinePrefix+line, childRuledLinePrefix+prefix, withAttrs) + } +} + +func ConvertCPTreeToByteTree(cpTrees map[spec.LexModeKindID]parser.CPTree) (byteTree, *symbolTable, error) { + var ids []spec.LexModeKindID + for id := range cpTrees { + ids = append(ids, id) + } + sort.Slice(ids, func(i, j int) bool { + return ids[i] < ids[j] + }) + + var bt byteTree + for _, id := range ids { + cpTree := cpTrees[id] + t, err := convCPTreeToByteTree(cpTree) + if err != nil { + return nil, nil, err + } + bt = oneOf(bt, concat(t, newEndMarkerNode(id))) + } + _, err := positionSymbols(bt, symbolPositionMin) + if err != nil { + return nil, nil, err + } + + return bt, genSymbolTable(bt), nil +} + +func convCPTreeToByteTree(cpTree parser.CPTree) (byteTree, error) { + if from, to, ok := cpTree.Range(); ok { + bs, err := utf8.GenCharBlocks(from, to) + if err != nil { + return nil, err + } + var a byteTree + for _, b := range bs { + var c byteTree + for i := 0; i < len(b.From); i++ { + c = concat(c, newRangeSymbolNode(b.From[i], b.To[i])) + } + a = oneOf(a, c) + } + return a, nil + } + + if tree, ok := cpTree.Repeatable(); ok { + t, err := convCPTreeToByteTree(tree) + if err != nil { + return nil, err + } + return newRepeatNode(t), nil + } + + if tree, ok := cpTree.Optional(); ok { + t, err := convCPTreeToByteTree(tree) + if err != nil { + return nil, err + } + return newOptionNode(t), nil + } + + if left, right, ok := cpTree.Concatenation(); ok { + l, err := convCPTreeToByteTree(left) + if err != nil { + return nil, err + } + r, err := convCPTreeToByteTree(right) + if err != nil { + return nil, err + } + return newConcatNode(l, r), nil + } + + if left, right, ok := cpTree.Alternatives(); ok { + l, err := convCPTreeToByteTree(left) + if err != nil { + return nil, err + } + r, err := convCPTreeToByteTree(right) + if err != nil { + return nil, err + } + return newAltNode(l, r), nil + } + + return nil, fmt.Errorf("invalid tree type: %T", cpTree) +} diff --git a/grammar/lexical/dfa/tree_test.go b/grammar/lexical/dfa/tree_test.go new file mode 100644 index 0000000..e0abe64 --- /dev/null +++ b/grammar/lexical/dfa/tree_test.go @@ -0,0 +1,257 @@ +package dfa + +import ( + "fmt" + "strings" + "testing" + + "github.com/nihei9/vartan/grammar/lexical/parser" + spec "github.com/nihei9/vartan/spec/grammar" +) + +func TestByteTree(t *testing.T) { + tests := []struct { + root byteTree + nullable bool + first *symbolPositionSet + last *symbolPositionSet + }{ + { + root: newSymbolNodeWithPos(0, 1), + nullable: false, + first: newSymbolPositionSet().add(1), + last: newSymbolPositionSet().add(1), + }, + { + root: newEndMarkerNodeWithPos(1, 1), + nullable: false, + first: newSymbolPositionSet().add(1), + last: newSymbolPositionSet().add(1), + }, + { + root: newConcatNode( + newSymbolNodeWithPos(0, 1), + newSymbolNodeWithPos(0, 2), + ), + nullable: false, + first: newSymbolPositionSet().add(1), + last: newSymbolPositionSet().add(2), + }, + { + root: newConcatNode( + newRepeatNode(newSymbolNodeWithPos(0, 1)), + newSymbolNodeWithPos(0, 2), + ), + nullable: false, + first: newSymbolPositionSet().add(1).add(2), + last: newSymbolPositionSet().add(2), + }, + { + root: newConcatNode( + newSymbolNodeWithPos(0, 1), + newRepeatNode(newSymbolNodeWithPos(0, 2)), + ), + nullable: false, + first: newSymbolPositionSet().add(1), + last: newSymbolPositionSet().add(1).add(2), + }, + { + root: newConcatNode( + newRepeatNode(newSymbolNodeWithPos(0, 1)), + newRepeatNode(newSymbolNodeWithPos(0, 2)), + ), + nullable: true, + first: newSymbolPositionSet().add(1).add(2), + last: newSymbolPositionSet().add(1).add(2), + }, + { + root: newAltNode( + newSymbolNodeWithPos(0, 1), + newSymbolNodeWithPos(0, 2), + ), + nullable: false, + first: newSymbolPositionSet().add(1).add(2), + last: newSymbolPositionSet().add(1).add(2), + }, + { + root: newAltNode( + newRepeatNode(newSymbolNodeWithPos(0, 1)), + newSymbolNodeWithPos(0, 2), + ), + nullable: true, + first: newSymbolPositionSet().add(1).add(2), + last: newSymbolPositionSet().add(1).add(2), + }, + { + root: newAltNode( + newSymbolNodeWithPos(0, 1), + newRepeatNode(newSymbolNodeWithPos(0, 2)), + ), + nullable: true, + first: newSymbolPositionSet().add(1).add(2), + last: newSymbolPositionSet().add(1).add(2), + }, + { + root: newAltNode( + newRepeatNode(newSymbolNodeWithPos(0, 1)), + newRepeatNode(newSymbolNodeWithPos(0, 2)), + ), + nullable: true, + first: newSymbolPositionSet().add(1).add(2), + last: newSymbolPositionSet().add(1).add(2), + }, + { + root: newRepeatNode(newSymbolNodeWithPos(0, 1)), + nullable: true, + first: newSymbolPositionSet().add(1), + last: newSymbolPositionSet().add(1), + }, + { + root: newOptionNode(newSymbolNodeWithPos(0, 1)), + nullable: true, + first: newSymbolPositionSet().add(1), + last: newSymbolPositionSet().add(1), + }, + } + for i, tt := range tests { + t.Run(fmt.Sprintf("#%v", i), func(t *testing.T) { + if tt.root.nullable() != tt.nullable { + t.Errorf("unexpected nullable attribute; want: %v, got: %v", tt.nullable, tt.root.nullable()) + } + if tt.first.hash() != tt.root.first().hash() { + t.Errorf("unexpected first positions attribute; want: %v, got: %v", tt.first, tt.root.first()) + } + if tt.last.hash() != tt.root.last().hash() { + t.Errorf("unexpected last positions attribute; want: %v, got: %v", tt.last, tt.root.last()) + } + }) + } +} + +func newSymbolNodeWithPos(v byte, pos symbolPosition) *symbolNode { + n := newSymbolNode(v) + n.pos = pos + return n +} + +func newEndMarkerNodeWithPos(id int, pos symbolPosition) *endMarkerNode { + n := newEndMarkerNode(spec.LexModeKindID(id)) + n.pos = pos + return n +} + +func TestFollowAndSymbolTable(t *testing.T) { + symPos := func(n uint16) symbolPosition { + pos, err := newSymbolPosition(n, false) + if err != nil { + panic(err) + } + return pos + } + + endPos := func(n uint16) symbolPosition { + pos, err := newSymbolPosition(n, true) + if err != nil { + panic(err) + } + return pos + } + + p := parser.NewParser(spec.LexKindName("test"), strings.NewReader("(a|b)*abb")) + cpt, err := p.Parse() + if err != nil { + t.Fatal(err) + } + + bt, symTab, err := ConvertCPTreeToByteTree(map[spec.LexModeKindID]parser.CPTree{ + spec.LexModeKindIDMin: cpt, + }) + if err != nil { + t.Fatal(err) + } + + { + followTab := genFollowTable(bt) + if followTab == nil { + t.Fatal("follow table is nil") + } + expectedFollowTab := followTable{ + 1: newSymbolPositionSet().add(symPos(1)).add(symPos(2)).add(symPos(3)), + 2: newSymbolPositionSet().add(symPos(1)).add(symPos(2)).add(symPos(3)), + 3: newSymbolPositionSet().add(symPos(4)), + 4: newSymbolPositionSet().add(symPos(5)), + 5: newSymbolPositionSet().add(endPos(6)), + } + testFollowTable(t, expectedFollowTab, followTab) + } + + { + entry := func(v byte) byteRange { + return byteRange{ + from: v, + to: v, + } + } + + expectedSymTab := &symbolTable{ + symPos2Byte: map[symbolPosition]byteRange{ + symPos(1): entry(byte('a')), + symPos(2): entry(byte('b')), + symPos(3): entry(byte('a')), + symPos(4): entry(byte('b')), + symPos(5): entry(byte('b')), + }, + endPos2ID: map[symbolPosition]spec.LexModeKindID{ + endPos(6): 1, + }, + } + testSymbolTable(t, expectedSymTab, symTab) + } +} + +func testFollowTable(t *testing.T, expected, actual followTable) { + if len(actual) != len(expected) { + t.Errorf("unexpected number of the follow table entries; want: %v, got: %v", len(expected), len(actual)) + } + for ePos, eSet := range expected { + aSet, ok := actual[ePos] + if !ok { + t.Fatalf("follow entry is not found: position: %v, follow: %v", ePos, eSet) + } + if aSet.hash() != eSet.hash() { + t.Fatalf("follow entry of position %v is mismatched: want: %v, got: %v", ePos, aSet, eSet) + } + } +} + +func testSymbolTable(t *testing.T, expected, actual *symbolTable) { + t.Helper() + + if len(actual.symPos2Byte) != len(expected.symPos2Byte) { + t.Errorf("unexpected symPos2Byte entries: want: %v entries, got: %v entries", len(expected.symPos2Byte), len(actual.symPos2Byte)) + } + for ePos, eByte := range expected.symPos2Byte { + byte, ok := actual.symPos2Byte[ePos] + if !ok { + t.Errorf("a symbol position entry is not found: %v -> %v", ePos, eByte) + continue + } + if byte.from != eByte.from || byte.to != eByte.to { + t.Errorf("unexpected symbol position entry: want: %v -> %v, got: %v -> %v", ePos, eByte, ePos, byte) + } + } + + if len(actual.endPos2ID) != len(expected.endPos2ID) { + t.Errorf("unexpected endPos2ID entries: want: %v entries, got: %v entries", len(expected.endPos2ID), len(actual.endPos2ID)) + } + for ePos, eID := range expected.endPos2ID { + id, ok := actual.endPos2ID[ePos] + if !ok { + t.Errorf("an end position entry is not found: %v -> %v", ePos, eID) + continue + } + if id != eID { + t.Errorf("unexpected end position entry: want: %v -> %v, got: %v -> %v", ePos, eID, ePos, id) + } + } +} diff --git a/grammar/lexical/entry.go b/grammar/lexical/entry.go new file mode 100644 index 0000000..c9f8691 --- /dev/null +++ b/grammar/lexical/entry.go @@ -0,0 +1,171 @@ +package lexical + +import ( + "fmt" + "sort" + "strings" + + spec "github.com/nihei9/vartan/spec/grammar" +) + +type LexEntry struct { + Kind spec.LexKindName + Pattern string + Modes []spec.LexModeName + Push spec.LexModeName + Pop bool + Fragment bool +} + +type LexSpec struct { + Entries []*LexEntry +} + +func (s *LexSpec) Validate() error { + if len(s.Entries) <= 0 { + return fmt.Errorf("the lexical specification must have at least one entry") + } + { + ks := map[string]struct{}{} + fks := map[string]struct{}{} + for _, e := range s.Entries { + // Allow duplicate names between fragments and non-fragments. + if e.Fragment { + if _, exist := fks[e.Kind.String()]; exist { + return fmt.Errorf("kinds `%v` are duplicates", e.Kind) + } + fks[e.Kind.String()] = struct{}{} + } else { + if _, exist := ks[e.Kind.String()]; exist { + return fmt.Errorf("kinds `%v` are duplicates", e.Kind) + } + ks[e.Kind.String()] = struct{}{} + } + } + } + { + kinds := []string{} + modes := []string{ + spec.LexModeNameDefault.String(), // This is a predefined mode. + } + for _, e := range s.Entries { + if e.Fragment { + continue + } + + kinds = append(kinds, e.Kind.String()) + + for _, m := range e.Modes { + modes = append(modes, m.String()) + } + } + + kindErrs := findSpellingInconsistenciesErrors(kinds, nil) + modeErrs := findSpellingInconsistenciesErrors(modes, func(ids []string) error { + if SnakeCaseToUpperCamelCase(ids[0]) == SnakeCaseToUpperCamelCase(spec.LexModeNameDefault.String()) { + var b strings.Builder + fmt.Fprintf(&b, "%+v", ids[0]) + for _, id := range ids[1:] { + fmt.Fprintf(&b, ", %+v", id) + } + return fmt.Errorf("these identifiers are treated as the same. please use the same spelling as predefined '%v': %v", spec.LexModeNameDefault, b.String()) + } + return nil + }) + errs := append(kindErrs, modeErrs...) + if len(errs) > 0 { + var b strings.Builder + fmt.Fprintf(&b, "%v", errs[0]) + for _, err := range errs[1:] { + fmt.Fprintf(&b, "\n%v", err) + } + return fmt.Errorf(b.String()) + } + } + + return nil +} + +func findSpellingInconsistenciesErrors(ids []string, hook func(ids []string) error) []error { + duplicated := FindSpellingInconsistencies(ids) + if len(duplicated) == 0 { + return nil + } + + var errs []error + for _, dup := range duplicated { + if hook != nil { + err := hook(dup) + if err != nil { + errs = append(errs, err) + continue + } + } + + var b strings.Builder + fmt.Fprintf(&b, "%+v", dup[0]) + for _, id := range dup[1:] { + fmt.Fprintf(&b, ", %+v", id) + } + err := fmt.Errorf("these identifiers are treated as the same. please use the same spelling: %v", b.String()) + errs = append(errs, err) + } + + return errs +} + +// FindSpellingInconsistencies finds spelling inconsistencies in identifiers. The identifiers are considered to be the same +// if they are spelled the same when expressed in UpperCamelCase. For example, `left_paren` and `LeftParen` are spelled the same +// in UpperCamelCase. Thus they are considere to be spelling inconsistency. +func FindSpellingInconsistencies(ids []string) [][]string { + m := map[string][]string{} + for _, id := range removeDuplicates(ids) { + c := SnakeCaseToUpperCamelCase(id) + m[c] = append(m[c], id) + } + + var duplicated [][]string + for _, camels := range m { + if len(camels) == 1 { + continue + } + duplicated = append(duplicated, camels) + } + + for _, dup := range duplicated { + sort.Slice(dup, func(i, j int) bool { + return dup[i] < dup[j] + }) + } + sort.Slice(duplicated, func(i, j int) bool { + return duplicated[i][0] < duplicated[j][0] + }) + + return duplicated +} + +func removeDuplicates(s []string) []string { + m := map[string]struct{}{} + for _, v := range s { + m[v] = struct{}{} + } + + var unique []string + for v := range m { + unique = append(unique, v) + } + + return unique +} + +func SnakeCaseToUpperCamelCase(snake string) string { + elems := strings.Split(snake, "_") + for i, e := range elems { + if len(e) == 0 { + continue + } + elems[i] = strings.ToUpper(string(e[0])) + e[1:] + } + + return strings.Join(elems, "") +} diff --git a/grammar/lexical/parser/error.go b/grammar/lexical/parser/error.go new file mode 100644 index 0000000..be81da4 --- /dev/null +++ b/grammar/lexical/parser/error.go @@ -0,0 +1,36 @@ +package parser + +import "fmt" + +var ( + ParseErr = fmt.Errorf("parse error") + + // lexical errors + synErrIncompletedEscSeq = fmt.Errorf("incompleted escape sequence; unexpected EOF following \\") + synErrInvalidEscSeq = fmt.Errorf("invalid escape sequence") + synErrInvalidCodePoint = fmt.Errorf("code points must consist of just 4 or 6 hex digits") + synErrCharPropInvalidSymbol = fmt.Errorf("invalid character property symbol") + SynErrFragmentInvalidSymbol = fmt.Errorf("invalid fragment symbol") + + // syntax errors + synErrUnexpectedToken = fmt.Errorf("unexpected token") + synErrNullPattern = fmt.Errorf("a pattern must be a non-empty byte sequence") + synErrUnmatchablePattern = fmt.Errorf("a pattern cannot match any characters") + synErrAltLackOfOperand = fmt.Errorf("an alternation expression must have operands") + synErrRepNoTarget = fmt.Errorf("a repeat expression must have an operand") + synErrGroupNoElem = fmt.Errorf("a grouping expression must include at least one character") + synErrGroupUnclosed = fmt.Errorf("unclosed grouping expression") + synErrGroupNoInitiator = fmt.Errorf(") needs preceding (") + synErrGroupInvalidForm = fmt.Errorf("invalid grouping expression") + synErrBExpNoElem = fmt.Errorf("a bracket expression must include at least one character") + synErrBExpUnclosed = fmt.Errorf("unclosed bracket expression") + synErrBExpInvalidForm = fmt.Errorf("invalid bracket expression") + synErrRangeInvalidOrder = fmt.Errorf("a range expression with invalid order") + synErrRangePropIsUnavailable = fmt.Errorf("a property expression is unavailable in a range expression") + synErrRangeInvalidForm = fmt.Errorf("invalid range expression") + synErrCPExpInvalidForm = fmt.Errorf("invalid code point expression") + synErrCPExpOutOfRange = fmt.Errorf("a code point must be between U+0000 to U+10FFFF") + synErrCharPropExpInvalidForm = fmt.Errorf("invalid character property expression") + synErrCharPropUnsupported = fmt.Errorf("unsupported character property") + synErrFragmentExpInvalidForm = fmt.Errorf("invalid fragment expression") +) diff --git a/grammar/lexical/parser/fragment.go b/grammar/lexical/parser/fragment.go new file mode 100644 index 0000000..fc6f16b --- /dev/null +++ b/grammar/lexical/parser/fragment.go @@ -0,0 +1,72 @@ +package parser + +import ( + "fmt" + + spec "github.com/nihei9/vartan/spec/grammar" +) + +type incompleteFragment struct { + kind spec.LexKindName + root *rootNode +} + +func CompleteFragments(fragments map[spec.LexKindName]CPTree) error { + if len(fragments) == 0 { + return nil + } + + completeFragments := map[spec.LexKindName]CPTree{} + incompleteFragments := []*incompleteFragment{} + for kind, tree := range fragments { + root, ok := tree.(*rootNode) + if !ok { + return fmt.Errorf("CompleteFragments can take only *rootNode: %T", tree) + } + if root.incomplete() { + incompleteFragments = append(incompleteFragments, &incompleteFragment{ + kind: kind, + root: root, + }) + } else { + completeFragments[kind] = root + } + } + for len(incompleteFragments) > 0 { + lastIncompCount := len(incompleteFragments) + remainingFragments := []*incompleteFragment{} + for _, e := range incompleteFragments { + complete, err := ApplyFragments(e.root, completeFragments) + if err != nil { + return err + } + if !complete { + remainingFragments = append(remainingFragments, e) + } else { + completeFragments[e.kind] = e.root + } + } + incompleteFragments = remainingFragments + if len(incompleteFragments) == lastIncompCount { + return ParseErr + } + } + + return nil +} + +func ApplyFragments(t CPTree, fragments map[spec.LexKindName]CPTree) (bool, error) { + root, ok := t.(*rootNode) + if !ok { + return false, fmt.Errorf("ApplyFragments can take only *rootNode type: %T", t) + } + + for name, frag := range fragments { + err := root.applyFragment(name, frag) + if err != nil { + return false, err + } + } + + return !root.incomplete(), nil +} diff --git a/grammar/lexical/parser/lexer.go b/grammar/lexical/parser/lexer.go new file mode 100644 index 0000000..3861825 --- /dev/null +++ b/grammar/lexical/parser/lexer.go @@ -0,0 +1,594 @@ +package parser + +import ( + "bufio" + "fmt" + "io" + "strings" +) + +type tokenKind string + +const ( + tokenKindChar tokenKind = "char" + tokenKindAnyChar tokenKind = "." + tokenKindRepeat tokenKind = "*" + tokenKindRepeatOneOrMore tokenKind = "+" + tokenKindOption tokenKind = "?" + tokenKindAlt tokenKind = "|" + tokenKindGroupOpen tokenKind = "(" + tokenKindGroupClose tokenKind = ")" + tokenKindBExpOpen tokenKind = "[" + tokenKindInverseBExpOpen tokenKind = "[^" + tokenKindBExpClose tokenKind = "]" + tokenKindCharRange tokenKind = "-" + tokenKindCodePointLeader tokenKind = "\\u" + tokenKindCharPropLeader tokenKind = "\\p" + tokenKindFragmentLeader tokenKind = "\\f" + tokenKindLBrace tokenKind = "{" + tokenKindRBrace tokenKind = "}" + tokenKindEqual tokenKind = "=" + tokenKindCodePoint tokenKind = "code point" + tokenKindCharPropSymbol tokenKind = "character property symbol" + tokenKindFragmentSymbol tokenKind = "fragment symbol" + tokenKindEOF tokenKind = "eof" +) + +type token struct { + kind tokenKind + char rune + propSymbol string + codePoint string + fragmentSymbol string +} + +const nullChar = '\u0000' + +func newToken(kind tokenKind, char rune) *token { + return &token{ + kind: kind, + char: char, + } +} + +func newCodePointToken(codePoint string) *token { + return &token{ + kind: tokenKindCodePoint, + codePoint: codePoint, + } +} + +func newCharPropSymbolToken(propSymbol string) *token { + return &token{ + kind: tokenKindCharPropSymbol, + propSymbol: propSymbol, + } +} + +func newFragmentSymbolToken(fragmentSymbol string) *token { + return &token{ + kind: tokenKindFragmentSymbol, + fragmentSymbol: fragmentSymbol, + } +} + +type lexerMode string + +const ( + lexerModeDefault lexerMode = "default" + lexerModeBExp lexerMode = "bracket expression" + lexerModeCPExp lexerMode = "code point expression" + lexerModeCharPropExp lexerMode = "character property expression" + lexerModeFragmentExp lexerMode = "fragment expression" +) + +type lexerModeStack struct { + stack []lexerMode +} + +func newLexerModeStack() *lexerModeStack { + return &lexerModeStack{ + stack: []lexerMode{ + lexerModeDefault, + }, + } +} + +func (s *lexerModeStack) top() lexerMode { + return s.stack[len(s.stack)-1] +} + +func (s *lexerModeStack) push(m lexerMode) { + s.stack = append(s.stack, m) +} + +func (s *lexerModeStack) pop() { + s.stack = s.stack[:len(s.stack)-1] +} + +type rangeState string + +// [a-z] +// ^^^^ +// |||`-- ready +// ||`-- expect range terminator +// |`-- read range initiator +// `-- ready +const ( + rangeStateReady rangeState = "ready" + rangeStateReadRangeInitiator rangeState = "read range initiator" + rangeStateExpectRangeTerminator rangeState = "expect range terminator" +) + +type lexer struct { + src *bufio.Reader + peekChar2 rune + peekEOF2 bool + peekChar1 rune + peekEOF1 bool + lastChar rune + reachedEOF bool + prevChar1 rune + prevEOF1 bool + prevChar2 rune + pervEOF2 bool + modeStack *lexerModeStack + rangeState rangeState + + errCause error + errDetail string +} + +func newLexer(src io.Reader) *lexer { + return &lexer{ + src: bufio.NewReader(src), + peekChar2: nullChar, + peekEOF2: false, + peekChar1: nullChar, + peekEOF1: false, + lastChar: nullChar, + reachedEOF: false, + prevChar1: nullChar, + prevEOF1: false, + prevChar2: nullChar, + pervEOF2: false, + modeStack: newLexerModeStack(), + rangeState: rangeStateReady, + } +} + +func (l *lexer) error() (string, error) { + return l.errDetail, l.errCause +} + +func (l *lexer) next() (*token, error) { + c, eof, err := l.read() + if err != nil { + return nil, err + } + if eof { + return newToken(tokenKindEOF, nullChar), nil + } + + switch l.modeStack.top() { + case lexerModeBExp: + tok, err := l.nextInBExp(c) + if err != nil { + return nil, err + } + if tok.kind == tokenKindChar || tok.kind == tokenKindCodePointLeader || tok.kind == tokenKindCharPropLeader { + switch l.rangeState { + case rangeStateReady: + l.rangeState = rangeStateReadRangeInitiator + case rangeStateExpectRangeTerminator: + l.rangeState = rangeStateReady + } + } + switch tok.kind { + case tokenKindBExpClose: + l.modeStack.pop() + case tokenKindCharRange: + l.rangeState = rangeStateExpectRangeTerminator + case tokenKindCodePointLeader: + l.modeStack.push(lexerModeCPExp) + case tokenKindCharPropLeader: + l.modeStack.push(lexerModeCharPropExp) + } + return tok, nil + case lexerModeCPExp: + tok, err := l.nextInCodePoint(c) + if err != nil { + return nil, err + } + switch tok.kind { + case tokenKindRBrace: + l.modeStack.pop() + } + return tok, nil + case lexerModeCharPropExp: + tok, err := l.nextInCharProp(c) + if err != nil { + return nil, err + } + switch tok.kind { + case tokenKindRBrace: + l.modeStack.pop() + } + return tok, nil + case lexerModeFragmentExp: + tok, err := l.nextInFragment(c) + if err != nil { + return nil, err + } + switch tok.kind { + case tokenKindRBrace: + l.modeStack.pop() + } + return tok, nil + default: + tok, err := l.nextInDefault(c) + if err != nil { + return nil, err + } + switch tok.kind { + case tokenKindBExpOpen: + l.modeStack.push(lexerModeBExp) + l.rangeState = rangeStateReady + case tokenKindInverseBExpOpen: + l.modeStack.push(lexerModeBExp) + l.rangeState = rangeStateReady + case tokenKindCodePointLeader: + l.modeStack.push(lexerModeCPExp) + case tokenKindCharPropLeader: + l.modeStack.push(lexerModeCharPropExp) + case tokenKindFragmentLeader: + l.modeStack.push(lexerModeFragmentExp) + } + return tok, nil + } +} + +func (l *lexer) nextInDefault(c rune) (*token, error) { + switch c { + case '*': + return newToken(tokenKindRepeat, nullChar), nil + case '+': + return newToken(tokenKindRepeatOneOrMore, nullChar), nil + case '?': + return newToken(tokenKindOption, nullChar), nil + case '.': + return newToken(tokenKindAnyChar, nullChar), nil + case '|': + return newToken(tokenKindAlt, nullChar), nil + case '(': + return newToken(tokenKindGroupOpen, nullChar), nil + case ')': + return newToken(tokenKindGroupClose, nullChar), nil + case '[': + c1, eof, err := l.read() + if err != nil { + return nil, err + } + if eof { + err := l.restore() + if err != nil { + return nil, err + } + return newToken(tokenKindBExpOpen, nullChar), nil + } + if c1 != '^' { + err := l.restore() + if err != nil { + return nil, err + } + return newToken(tokenKindBExpOpen, nullChar), nil + } + c2, eof, err := l.read() + if err != nil { + return nil, err + } + if eof { + err := l.restore() + if err != nil { + return nil, err + } + return newToken(tokenKindInverseBExpOpen, nullChar), nil + } + if c2 != ']' { + err := l.restore() + if err != nil { + return nil, err + } + return newToken(tokenKindInverseBExpOpen, nullChar), nil + } + err = l.restore() + if err != nil { + return nil, err + } + err = l.restore() + if err != nil { + return nil, err + } + return newToken(tokenKindBExpOpen, nullChar), nil + case '\\': + c, eof, err := l.read() + if err != nil { + return nil, err + } + if eof { + l.errCause = synErrIncompletedEscSeq + return nil, ParseErr + } + if c == 'u' { + return newToken(tokenKindCodePointLeader, nullChar), nil + } + if c == 'p' { + return newToken(tokenKindCharPropLeader, nullChar), nil + } + if c == 'f' { + return newToken(tokenKindFragmentLeader, nullChar), nil + } + if c == '\\' || c == '.' || c == '*' || c == '+' || c == '?' || c == '|' || c == '(' || c == ')' || c == '[' || c == ']' { + return newToken(tokenKindChar, c), nil + } + l.errCause = synErrInvalidEscSeq + l.errDetail = fmt.Sprintf("\\%v is not supported", string(c)) + return nil, ParseErr + default: + return newToken(tokenKindChar, c), nil + } +} + +func (l *lexer) nextInBExp(c rune) (*token, error) { + switch c { + case '-': + if l.rangeState != rangeStateReadRangeInitiator { + return newToken(tokenKindChar, c), nil + } + c1, eof, err := l.read() + if err != nil { + return nil, err + } + if eof { + err := l.restore() + if err != nil { + return nil, err + } + return newToken(tokenKindChar, c), nil + } + if c1 != ']' { + err := l.restore() + if err != nil { + return nil, err + } + return newToken(tokenKindCharRange, nullChar), nil + } + err = l.restore() + if err != nil { + return nil, err + } + return newToken(tokenKindChar, c), nil + case ']': + return newToken(tokenKindBExpClose, nullChar), nil + case '\\': + c, eof, err := l.read() + if err != nil { + return nil, err + } + if eof { + l.errCause = synErrIncompletedEscSeq + return nil, ParseErr + } + if c == 'u' { + return newToken(tokenKindCodePointLeader, nullChar), nil + } + if c == 'p' { + return newToken(tokenKindCharPropLeader, nullChar), nil + } + if c == '\\' || c == '^' || c == '-' || c == ']' { + return newToken(tokenKindChar, c), nil + } + l.errCause = synErrInvalidEscSeq + l.errDetail = fmt.Sprintf("\\%v is not supported in a bracket expression", string(c)) + return nil, ParseErr + default: + return newToken(tokenKindChar, c), nil + } +} + +func (l *lexer) nextInCodePoint(c rune) (*token, error) { + switch c { + case '{': + return newToken(tokenKindLBrace, nullChar), nil + case '}': + return newToken(tokenKindRBrace, nullChar), nil + default: + if !isHexDigit(c) { + l.errCause = synErrInvalidCodePoint + return nil, ParseErr + } + var b strings.Builder + fmt.Fprint(&b, string(c)) + n := 1 + for { + c, eof, err := l.read() + if err != nil { + return nil, err + } + if eof { + err := l.restore() + if err != nil { + return nil, err + } + break + } + if c == '}' { + err := l.restore() + if err != nil { + return nil, err + } + break + } + if !isHexDigit(c) || n >= 6 { + l.errCause = synErrInvalidCodePoint + return nil, ParseErr + } + fmt.Fprint(&b, string(c)) + n++ + } + cp := b.String() + cpLen := len(cp) + if !(cpLen == 4 || cpLen == 6) { + l.errCause = synErrInvalidCodePoint + return nil, ParseErr + } + return newCodePointToken(b.String()), nil + } +} + +func isHexDigit(c rune) bool { + if c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c >= 'a' && c <= 'z' { + return true + } + return false +} + +func (l *lexer) nextInCharProp(c rune) (*token, error) { + switch c { + case '{': + return newToken(tokenKindLBrace, nullChar), nil + case '}': + return newToken(tokenKindRBrace, nullChar), nil + case '=': + return newToken(tokenKindEqual, nullChar), nil + default: + var b strings.Builder + fmt.Fprint(&b, string(c)) + n := 1 + for { + c, eof, err := l.read() + if err != nil { + return nil, err + } + if eof { + err := l.restore() + if err != nil { + return nil, err + } + break + } + if c == '}' || c == '=' { + err := l.restore() + if err != nil { + return nil, err + } + break + } + fmt.Fprint(&b, string(c)) + n++ + } + sym := strings.TrimSpace(b.String()) + if len(sym) == 0 { + l.errCause = synErrCharPropInvalidSymbol + return nil, ParseErr + } + return newCharPropSymbolToken(sym), nil + } +} + +func (l *lexer) nextInFragment(c rune) (*token, error) { + switch c { + case '{': + return newToken(tokenKindLBrace, nullChar), nil + case '}': + return newToken(tokenKindRBrace, nullChar), nil + default: + var b strings.Builder + fmt.Fprint(&b, string(c)) + n := 1 + for { + c, eof, err := l.read() + if err != nil { + return nil, err + } + if eof { + err := l.restore() + if err != nil { + return nil, err + } + break + } + if c == '}' { + err := l.restore() + if err != nil { + return nil, err + } + break + } + fmt.Fprint(&b, string(c)) + n++ + } + sym := strings.TrimSpace(b.String()) + if len(sym) == 0 { + l.errCause = SynErrFragmentInvalidSymbol + return nil, ParseErr + } + return newFragmentSymbolToken(sym), nil + } +} + +func (l *lexer) read() (rune, bool, error) { + if l.reachedEOF { + return l.lastChar, l.reachedEOF, nil + } + if l.peekChar1 != nullChar || l.peekEOF1 { + l.prevChar2 = l.prevChar1 + l.pervEOF2 = l.prevEOF1 + l.prevChar1 = l.lastChar + l.prevEOF1 = l.reachedEOF + l.lastChar = l.peekChar1 + l.reachedEOF = l.peekEOF1 + l.peekChar1 = l.peekChar2 + l.peekEOF1 = l.peekEOF2 + l.peekChar2 = nullChar + l.peekEOF2 = false + return l.lastChar, l.reachedEOF, nil + } + c, _, err := l.src.ReadRune() + if err != nil { + if err == io.EOF { + l.prevChar2 = l.prevChar1 + l.pervEOF2 = l.prevEOF1 + l.prevChar1 = l.lastChar + l.prevEOF1 = l.reachedEOF + l.lastChar = nullChar + l.reachedEOF = true + return l.lastChar, l.reachedEOF, nil + } + return nullChar, false, err + } + l.prevChar2 = l.prevChar1 + l.pervEOF2 = l.prevEOF1 + l.prevChar1 = l.lastChar + l.prevEOF1 = l.reachedEOF + l.lastChar = c + l.reachedEOF = false + return l.lastChar, l.reachedEOF, nil +} + +func (l *lexer) restore() error { + if l.lastChar == nullChar && !l.reachedEOF { + return fmt.Errorf("failed to call restore() because the last character is null") + } + l.peekChar2 = l.peekChar1 + l.peekEOF2 = l.peekEOF1 + l.peekChar1 = l.lastChar + l.peekEOF1 = l.reachedEOF + l.lastChar = l.prevChar1 + l.reachedEOF = l.prevEOF1 + l.prevChar1 = l.prevChar2 + l.prevEOF1 = l.pervEOF2 + l.prevChar2 = nullChar + l.pervEOF2 = false + return nil +} diff --git a/grammar/lexical/parser/lexer_test.go b/grammar/lexical/parser/lexer_test.go new file mode 100644 index 0000000..055466e --- /dev/null +++ b/grammar/lexical/parser/lexer_test.go @@ -0,0 +1,524 @@ +package parser + +import ( + "strings" + "testing" +) + +func TestLexer(t *testing.T) { + tests := []struct { + caption string + src string + tokens []*token + err error + }{ + { + caption: "lexer can recognize ordinaly characters", + src: "123abcいろは", + tokens: []*token{ + newToken(tokenKindChar, '1'), + newToken(tokenKindChar, '2'), + newToken(tokenKindChar, '3'), + newToken(tokenKindChar, 'a'), + newToken(tokenKindChar, 'b'), + newToken(tokenKindChar, 'c'), + newToken(tokenKindChar, 'い'), + newToken(tokenKindChar, 'ろ'), + newToken(tokenKindChar, 'は'), + newToken(tokenKindEOF, nullChar), + }, + }, + { + caption: "lexer can recognize the special characters in default mode", + src: ".*+?|()[\\u", + tokens: []*token{ + newToken(tokenKindAnyChar, nullChar), + newToken(tokenKindRepeat, nullChar), + newToken(tokenKindRepeatOneOrMore, nullChar), + newToken(tokenKindOption, nullChar), + newToken(tokenKindAlt, nullChar), + newToken(tokenKindGroupOpen, nullChar), + newToken(tokenKindGroupClose, nullChar), + newToken(tokenKindBExpOpen, nullChar), + newToken(tokenKindCodePointLeader, nullChar), + newToken(tokenKindEOF, nullChar), + }, + }, + { + caption: "lexer can recognize the escape sequences in default mode", + src: "\\\\\\.\\*\\+\\?\\|\\(\\)\\[", + tokens: []*token{ + newToken(tokenKindChar, '\\'), + newToken(tokenKindChar, '.'), + newToken(tokenKindChar, '*'), + newToken(tokenKindChar, '+'), + newToken(tokenKindChar, '?'), + newToken(tokenKindChar, '|'), + newToken(tokenKindChar, '('), + newToken(tokenKindChar, ')'), + newToken(tokenKindChar, '['), + newToken(tokenKindEOF, nullChar), + }, + }, + { + caption: "], {, and } are treated as an ordinary character in default mode", + src: "]{}", + tokens: []*token{ + newToken(tokenKindChar, ']'), + newToken(tokenKindChar, '{'), + newToken(tokenKindChar, '}'), + newToken(tokenKindEOF, nullChar), + }, + }, + { + caption: "lexer can recognize the special characters in bracket expression mode", + src: "[a-z\\u{09AF}][^a-z\\u{09abcf}]", + tokens: []*token{ + newToken(tokenKindBExpOpen, nullChar), + newToken(tokenKindChar, 'a'), + newToken(tokenKindCharRange, nullChar), + newToken(tokenKindChar, 'z'), + newToken(tokenKindCodePointLeader, nullChar), + newToken(tokenKindLBrace, nullChar), + newCodePointToken("09AF"), + newToken(tokenKindRBrace, nullChar), + newToken(tokenKindBExpClose, nullChar), + newToken(tokenKindInverseBExpOpen, nullChar), + newToken(tokenKindChar, 'a'), + newToken(tokenKindCharRange, nullChar), + newToken(tokenKindChar, 'z'), + newToken(tokenKindCodePointLeader, nullChar), + newToken(tokenKindLBrace, nullChar), + newCodePointToken("09abcf"), + newToken(tokenKindRBrace, nullChar), + newToken(tokenKindBExpClose, nullChar), + newToken(tokenKindEOF, nullChar), + }, + }, + { + caption: "lexer can recognize the escape sequences in bracket expression mode", + src: "[\\^a\\-z]", + tokens: []*token{ + newToken(tokenKindBExpOpen, nullChar), + newToken(tokenKindChar, '^'), + newToken(tokenKindChar, 'a'), + newToken(tokenKindChar, '-'), + newToken(tokenKindChar, 'z'), + newToken(tokenKindBExpClose, nullChar), + newToken(tokenKindEOF, nullChar), + }, + }, + { + caption: "in a bracket expression, the special characters are also handled as normal characters", + src: "[\\\\.*+?|()[", + tokens: []*token{ + newToken(tokenKindBExpOpen, nullChar), + newToken(tokenKindChar, '\\'), + newToken(tokenKindChar, '.'), + newToken(tokenKindChar, '*'), + newToken(tokenKindChar, '+'), + newToken(tokenKindChar, '?'), + newToken(tokenKindChar, '|'), + newToken(tokenKindChar, '('), + newToken(tokenKindChar, ')'), + newToken(tokenKindChar, '['), + newToken(tokenKindEOF, nullChar), + }, + }, + { + caption: "hyphen symbols that appear in bracket expressions are handled as the character range symbol or ordinary characters", + // [...-...][...-][-...][-] + // ~~~~~~~ ~ ~ ~ + // ^ ^ ^ ^ + // | | | `-- Ordinary Character (b) + // | | `-- Ordinary Character (b) + // | `-- Ordinary Character (b) + // `-- Character Range (a) + // + // a. *-* is handled as a character-range expression. + // b. *-, -*, or - are handled as ordinary characters. + src: "[a-z][a-][-z][-][--][---][^a-z][^a-][^-z][^-][^--][^---]", + tokens: []*token{ + newToken(tokenKindBExpOpen, nullChar), + newToken(tokenKindChar, 'a'), + newToken(tokenKindCharRange, nullChar), + newToken(tokenKindChar, 'z'), + newToken(tokenKindBExpClose, nullChar), + newToken(tokenKindBExpOpen, nullChar), + newToken(tokenKindChar, 'a'), + newToken(tokenKindChar, '-'), + newToken(tokenKindBExpClose, nullChar), + newToken(tokenKindBExpOpen, nullChar), + newToken(tokenKindChar, '-'), + newToken(tokenKindChar, 'z'), + newToken(tokenKindBExpClose, nullChar), + newToken(tokenKindBExpOpen, nullChar), + newToken(tokenKindChar, '-'), + newToken(tokenKindBExpClose, nullChar), + newToken(tokenKindBExpOpen, nullChar), + newToken(tokenKindChar, '-'), + newToken(tokenKindChar, '-'), + newToken(tokenKindBExpClose, nullChar), + newToken(tokenKindBExpOpen, nullChar), + newToken(tokenKindChar, '-'), + newToken(tokenKindCharRange, nullChar), + newToken(tokenKindChar, '-'), + newToken(tokenKindBExpClose, nullChar), + + newToken(tokenKindInverseBExpOpen, nullChar), + newToken(tokenKindChar, 'a'), + newToken(tokenKindCharRange, nullChar), + newToken(tokenKindChar, 'z'), + newToken(tokenKindBExpClose, nullChar), + newToken(tokenKindInverseBExpOpen, nullChar), + newToken(tokenKindChar, 'a'), + newToken(tokenKindChar, '-'), + newToken(tokenKindBExpClose, nullChar), + newToken(tokenKindInverseBExpOpen, nullChar), + newToken(tokenKindChar, '-'), + newToken(tokenKindChar, 'z'), + newToken(tokenKindBExpClose, nullChar), + newToken(tokenKindInverseBExpOpen, nullChar), + newToken(tokenKindChar, '-'), + newToken(tokenKindBExpClose, nullChar), + newToken(tokenKindInverseBExpOpen, nullChar), + newToken(tokenKindChar, '-'), + newToken(tokenKindChar, '-'), + newToken(tokenKindBExpClose, nullChar), + newToken(tokenKindInverseBExpOpen, nullChar), + newToken(tokenKindChar, '-'), + newToken(tokenKindCharRange, nullChar), + newToken(tokenKindChar, '-'), + newToken(tokenKindBExpClose, nullChar), + + newToken(tokenKindEOF, nullChar), + }, + }, + { + caption: "caret symbols that appear in bracket expressions are handled as the logical inverse symbol or ordinary characters", + // [^...^...][^] + // ~~ ~ ~~ + // ^ ^ ^^ + // | | |`-- Ordinary Character (c) + // | | `-- Bracket Expression + // | `-- Ordinary Character (b) + // `-- Inverse Bracket Expression (a) + // + // a. Bracket expressions that have a caret symbol at the beginning are handled as logical inverse expressions. + // b. caret symbols that appear as the second and the subsequent symbols are handled as ordinary symbols. + // c. When a bracket expression has just one symbol, a caret symbol at the beginning is handled as an ordinary character. + src: "[^^][^]", + tokens: []*token{ + newToken(tokenKindInverseBExpOpen, nullChar), + newToken(tokenKindChar, '^'), + newToken(tokenKindBExpClose, nullChar), + newToken(tokenKindBExpOpen, nullChar), + newToken(tokenKindChar, '^'), + newToken(tokenKindBExpClose, nullChar), + newToken(tokenKindEOF, nullChar), + }, + }, + { + caption: "lexer raises an error when an invalid escape sequence appears", + src: "\\@", + err: synErrInvalidEscSeq, + }, + { + caption: "lexer raises an error when the incomplete escape sequence (EOF following \\) appears", + src: "\\", + err: synErrIncompletedEscSeq, + }, + { + caption: "lexer raises an error when an invalid escape sequence appears", + src: "[\\@", + tokens: []*token{ + newToken(tokenKindBExpOpen, nullChar), + }, + err: synErrInvalidEscSeq, + }, + { + caption: "lexer raises an error when the incomplete escape sequence (EOF following \\) appears", + src: "[\\", + tokens: []*token{ + newToken(tokenKindBExpOpen, nullChar), + }, + err: synErrIncompletedEscSeq, + }, + { + caption: "lexer can recognize the special characters and code points in code point expression mode", + src: "\\u{0123}\\u{4567}\\u{89abcd}\\u{efAB}\\u{CDEF01}[\\u{0123}\\u{4567}\\u{89abcd}\\u{efAB}\\u{CDEF01}][^\\u{0123}\\u{4567}\\u{89abcd}\\u{efAB}\\u{CDEF01}]", + tokens: []*token{ + newToken(tokenKindCodePointLeader, nullChar), + newToken(tokenKindLBrace, nullChar), + newCodePointToken("0123"), + newToken(tokenKindRBrace, nullChar), + newToken(tokenKindCodePointLeader, nullChar), + newToken(tokenKindLBrace, nullChar), + newCodePointToken("4567"), + newToken(tokenKindRBrace, nullChar), + newToken(tokenKindCodePointLeader, nullChar), + newToken(tokenKindLBrace, nullChar), + newCodePointToken("89abcd"), + newToken(tokenKindRBrace, nullChar), + newToken(tokenKindCodePointLeader, nullChar), + newToken(tokenKindLBrace, nullChar), + newCodePointToken("efAB"), + newToken(tokenKindRBrace, nullChar), + newToken(tokenKindCodePointLeader, nullChar), + newToken(tokenKindLBrace, nullChar), + newCodePointToken("CDEF01"), + newToken(tokenKindRBrace, nullChar), + + newToken(tokenKindBExpOpen, nullChar), + newToken(tokenKindCodePointLeader, nullChar), + newToken(tokenKindLBrace, nullChar), + newCodePointToken("0123"), + newToken(tokenKindRBrace, nullChar), + newToken(tokenKindCodePointLeader, nullChar), + newToken(tokenKindLBrace, nullChar), + newCodePointToken("4567"), + newToken(tokenKindRBrace, nullChar), + newToken(tokenKindCodePointLeader, nullChar), + newToken(tokenKindLBrace, nullChar), + newCodePointToken("89abcd"), + newToken(tokenKindRBrace, nullChar), + newToken(tokenKindCodePointLeader, nullChar), + newToken(tokenKindLBrace, nullChar), + newCodePointToken("efAB"), + newToken(tokenKindRBrace, nullChar), + newToken(tokenKindCodePointLeader, nullChar), + newToken(tokenKindLBrace, nullChar), + newCodePointToken("CDEF01"), + newToken(tokenKindRBrace, nullChar), + newToken(tokenKindBExpClose, nullChar), + + newToken(tokenKindInverseBExpOpen, nullChar), + newToken(tokenKindCodePointLeader, nullChar), + newToken(tokenKindLBrace, nullChar), + newCodePointToken("0123"), + newToken(tokenKindRBrace, nullChar), + newToken(tokenKindCodePointLeader, nullChar), + newToken(tokenKindLBrace, nullChar), + newCodePointToken("4567"), + newToken(tokenKindRBrace, nullChar), + newToken(tokenKindCodePointLeader, nullChar), + newToken(tokenKindLBrace, nullChar), + newCodePointToken("89abcd"), + newToken(tokenKindRBrace, nullChar), + newToken(tokenKindCodePointLeader, nullChar), + newToken(tokenKindLBrace, nullChar), + newCodePointToken("efAB"), + newToken(tokenKindRBrace, nullChar), + newToken(tokenKindCodePointLeader, nullChar), + newToken(tokenKindLBrace, nullChar), + newCodePointToken("CDEF01"), + newToken(tokenKindRBrace, nullChar), + newToken(tokenKindBExpClose, nullChar), + + newToken(tokenKindEOF, nullChar), + }, + }, + { + caption: "a one digit hex string isn't a valid code point", + src: "\\u{0", + tokens: []*token{ + newToken(tokenKindCodePointLeader, nullChar), + newToken(tokenKindLBrace, nullChar), + }, + err: synErrInvalidCodePoint, + }, + { + caption: "a two digits hex string isn't a valid code point", + src: "\\u{01", + tokens: []*token{ + newToken(tokenKindCodePointLeader, nullChar), + newToken(tokenKindLBrace, nullChar), + }, + err: synErrInvalidCodePoint, + }, + { + caption: "a three digits hex string isn't a valid code point", + src: "\\u{012", + tokens: []*token{ + newToken(tokenKindCodePointLeader, nullChar), + newToken(tokenKindLBrace, nullChar), + }, + err: synErrInvalidCodePoint, + }, + { + caption: "a four digits hex string is a valid code point", + src: "\\u{0123}", + tokens: []*token{ + newToken(tokenKindCodePointLeader, nullChar), + newToken(tokenKindLBrace, nullChar), + newCodePointToken("0123"), + newToken(tokenKindRBrace, nullChar), + }, + }, + { + caption: "a five digits hex string isn't a valid code point", + src: "\\u{01234", + tokens: []*token{ + newToken(tokenKindCodePointLeader, nullChar), + newToken(tokenKindLBrace, nullChar), + }, + err: synErrInvalidCodePoint, + }, + { + caption: "a six digits hex string is a valid code point", + src: "\\u{012345}", + tokens: []*token{ + newToken(tokenKindCodePointLeader, nullChar), + newToken(tokenKindLBrace, nullChar), + newCodePointToken("012345"), + newToken(tokenKindRBrace, nullChar), + }, + }, + { + caption: "a seven digits hex string isn't a valid code point", + src: "\\u{0123456", + tokens: []*token{ + newToken(tokenKindCodePointLeader, nullChar), + newToken(tokenKindLBrace, nullChar), + }, + err: synErrInvalidCodePoint, + }, + { + caption: "a code point must be hex digits", + src: "\\u{g", + tokens: []*token{ + newToken(tokenKindCodePointLeader, nullChar), + newToken(tokenKindLBrace, nullChar), + }, + err: synErrInvalidCodePoint, + }, + { + caption: "a code point must be hex digits", + src: "\\u{G", + tokens: []*token{ + newToken(tokenKindCodePointLeader, nullChar), + newToken(tokenKindLBrace, nullChar), + }, + err: synErrInvalidCodePoint, + }, + { + caption: "lexer can recognize the special characters and symbols in character property expression mode", + src: "\\p{Letter}\\p{General_Category=Letter}[\\p{Letter}\\p{General_Category=Letter}][^\\p{Letter}\\p{General_Category=Letter}]", + tokens: []*token{ + newToken(tokenKindCharPropLeader, nullChar), + newToken(tokenKindLBrace, nullChar), + newCharPropSymbolToken("Letter"), + newToken(tokenKindRBrace, nullChar), + newToken(tokenKindCharPropLeader, nullChar), + newToken(tokenKindLBrace, nullChar), + newCharPropSymbolToken("General_Category"), + newToken(tokenKindEqual, nullChar), + newCharPropSymbolToken("Letter"), + newToken(tokenKindRBrace, nullChar), + + newToken(tokenKindBExpOpen, nullChar), + newToken(tokenKindCharPropLeader, nullChar), + newToken(tokenKindLBrace, nullChar), + newCharPropSymbolToken("Letter"), + newToken(tokenKindRBrace, nullChar), + newToken(tokenKindCharPropLeader, nullChar), + newToken(tokenKindLBrace, nullChar), + newCharPropSymbolToken("General_Category"), + newToken(tokenKindEqual, nullChar), + newCharPropSymbolToken("Letter"), + newToken(tokenKindRBrace, nullChar), + newToken(tokenKindBExpClose, nullChar), + + newToken(tokenKindInverseBExpOpen, nullChar), + newToken(tokenKindCharPropLeader, nullChar), + newToken(tokenKindLBrace, nullChar), + newCharPropSymbolToken("Letter"), + newToken(tokenKindRBrace, nullChar), + newToken(tokenKindCharPropLeader, nullChar), + newToken(tokenKindLBrace, nullChar), + newCharPropSymbolToken("General_Category"), + newToken(tokenKindEqual, nullChar), + newCharPropSymbolToken("Letter"), + newToken(tokenKindRBrace, nullChar), + newToken(tokenKindBExpClose, nullChar), + + newToken(tokenKindEOF, nullChar), + }, + }, + { + caption: "lexer can recognize the special characters and symbols in fragment expression mode", + src: "\\f{integer}", + tokens: []*token{ + newToken(tokenKindFragmentLeader, nullChar), + newToken(tokenKindLBrace, nullChar), + newFragmentSymbolToken("integer"), + newToken(tokenKindRBrace, nullChar), + + newToken(tokenKindEOF, nullChar), + }, + }, + { + caption: "a fragment expression is not supported in a bracket expression", + src: "[\\f", + tokens: []*token{ + newToken(tokenKindBExpOpen, nullChar), + }, + err: synErrInvalidEscSeq, + }, + { + caption: "a fragment expression is not supported in an inverse bracket expression", + src: "[^\\f", + tokens: []*token{ + newToken(tokenKindInverseBExpOpen, nullChar), + }, + err: synErrInvalidEscSeq, + }, + } + for _, tt := range tests { + t.Run(tt.caption, func(t *testing.T) { + lex := newLexer(strings.NewReader(tt.src)) + var err error + var tok *token + i := 0 + for { + tok, err = lex.next() + if err != nil { + break + } + if i >= len(tt.tokens) { + break + } + eTok := tt.tokens[i] + i++ + testToken(t, tok, eTok) + + if tok.kind == tokenKindEOF { + break + } + } + if tt.err != nil { + if err != ParseErr { + t.Fatalf("unexpected error: want: %v, got: %v", ParseErr, err) + } + detail, cause := lex.error() + if cause != tt.err { + t.Fatalf("unexpected error: want: %v, got: %v (%v)", tt.err, cause, detail) + } + } else { + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + } + if i < len(tt.tokens) { + t.Fatalf("expecte more tokens") + } + }) + } +} + +func testToken(t *testing.T, a, e *token) { + t.Helper() + if e.kind != a.kind || e.char != a.char || e.codePoint != a.codePoint { + t.Fatalf("unexpected token: want: %+v, got: %+v", e, a) + } +} diff --git a/grammar/lexical/parser/parser.go b/grammar/lexical/parser/parser.go new file mode 100644 index 0000000..89362b8 --- /dev/null +++ b/grammar/lexical/parser/parser.go @@ -0,0 +1,531 @@ +package parser + +import ( + "bytes" + "fmt" + "io" + "strconv" + + spec "github.com/nihei9/vartan/spec/grammar" + "github.com/nihei9/vartan/ucd" +) + +type PatternEntry struct { + ID spec.LexModeKindID + Pattern []byte +} + +type parser struct { + kind spec.LexKindName + lex *lexer + peekedTok *token + lastTok *token + + // If and only if isContributoryPropertyExposed is true, the parser interprets contributory properties that + // appear in property expressions. + // + // The contributory properties are not exposed, and users cannot use those properties because the parser + // follows [UAX #44 5.13 Property APIs]. For instance, \p{Other_Alphabetic} is invalid. + // + // isContributoryPropertyExposed is set to true when the parser is generated recursively. The parser needs to + // interpret derived properties internally because the derived properties consist of other properties that + // may contain the contributory properties. + // + // [UAX #44 5.13 Property APIs] says: + // > The following subtypes of Unicode character properties should generally not be exposed in APIs, + // > except in limited circumstances. They may not be useful, particularly in public API collections, + // > and may instead prove misleading to the users of such API collections. + // > * Contributory properties are not recommended for public APIs. + // > ... + // https://unicode.org/reports/tr44/#Property_APIs + isContributoryPropertyExposed bool + + errCause error + errDetail string +} + +func NewParser(kind spec.LexKindName, src io.Reader) *parser { + return &parser{ + kind: kind, + lex: newLexer(src), + isContributoryPropertyExposed: false, + } +} + +func (p *parser) exposeContributoryProperty() { + p.isContributoryPropertyExposed = true +} + +func (p *parser) Error() (string, error) { + return p.errDetail, p.errCause +} + +func (p *parser) Parse() (root CPTree, retErr error) { + defer func() { + err := recover() + if err != nil { + var ok bool + retErr, ok = err.(error) + if !ok { + panic(err) + } + return + } + }() + + return newRootNode(p.kind, p.parseRegexp()), nil +} + +func (p *parser) parseRegexp() CPTree { + alt := p.parseAlt() + if alt == nil { + if p.consume(tokenKindGroupClose) { + p.raiseParseError(synErrGroupNoInitiator, "") + } + p.raiseParseError(synErrNullPattern, "") + } + if p.consume(tokenKindGroupClose) { + p.raiseParseError(synErrGroupNoInitiator, "") + } + p.expect(tokenKindEOF) + return alt +} + +func (p *parser) parseAlt() CPTree { + left := p.parseConcat() + if left == nil { + if p.consume(tokenKindAlt) { + p.raiseParseError(synErrAltLackOfOperand, "") + } + return nil + } + for { + if !p.consume(tokenKindAlt) { + break + } + right := p.parseConcat() + if right == nil { + p.raiseParseError(synErrAltLackOfOperand, "") + } + left = newAltNode(left, right) + } + return left +} + +func (p *parser) parseConcat() CPTree { + left := p.parseRepeat() + for { + right := p.parseRepeat() + if right == nil { + break + } + left = newConcatNode(left, right) + } + return left +} + +func (p *parser) parseRepeat() CPTree { + group := p.parseGroup() + if group == nil { + if p.consume(tokenKindRepeat) { + p.raiseParseError(synErrRepNoTarget, "* needs an operand") + } + if p.consume(tokenKindRepeatOneOrMore) { + p.raiseParseError(synErrRepNoTarget, "+ needs an operand") + } + if p.consume(tokenKindOption) { + p.raiseParseError(synErrRepNoTarget, "? needs an operand") + } + return nil + } + if p.consume(tokenKindRepeat) { + return newRepeatNode(group) + } + if p.consume(tokenKindRepeatOneOrMore) { + return newRepeatOneOrMoreNode(group) + } + if p.consume(tokenKindOption) { + return newOptionNode(group) + } + return group +} + +func (p *parser) parseGroup() CPTree { + if p.consume(tokenKindGroupOpen) { + alt := p.parseAlt() + if alt == nil { + if p.consume(tokenKindEOF) { + p.raiseParseError(synErrGroupUnclosed, "") + } + p.raiseParseError(synErrGroupNoElem, "") + } + if p.consume(tokenKindEOF) { + p.raiseParseError(synErrGroupUnclosed, "") + } + if !p.consume(tokenKindGroupClose) { + p.raiseParseError(synErrGroupInvalidForm, "") + } + return alt + } + return p.parseSingleChar() +} + +func (p *parser) parseSingleChar() CPTree { + if p.consume(tokenKindAnyChar) { + return genAnyCharAST() + } + if p.consume(tokenKindBExpOpen) { + left := p.parseBExpElem() + if left == nil { + if p.consume(tokenKindEOF) { + p.raiseParseError(synErrBExpUnclosed, "") + } + p.raiseParseError(synErrBExpNoElem, "") + } + for { + right := p.parseBExpElem() + if right == nil { + break + } + left = newAltNode(left, right) + } + if p.consume(tokenKindEOF) { + p.raiseParseError(synErrBExpUnclosed, "") + } + p.expect(tokenKindBExpClose) + return left + } + if p.consume(tokenKindInverseBExpOpen) { + elem := p.parseBExpElem() + if elem == nil { + if p.consume(tokenKindEOF) { + p.raiseParseError(synErrBExpUnclosed, "") + } + p.raiseParseError(synErrBExpNoElem, "") + } + inverse := exclude(elem, genAnyCharAST()) + if inverse == nil { + p.raiseParseError(synErrUnmatchablePattern, "") + } + for { + elem := p.parseBExpElem() + if elem == nil { + break + } + inverse = exclude(elem, inverse) + if inverse == nil { + p.raiseParseError(synErrUnmatchablePattern, "") + } + } + if p.consume(tokenKindEOF) { + p.raiseParseError(synErrBExpUnclosed, "") + } + p.expect(tokenKindBExpClose) + return inverse + } + if p.consume(tokenKindCodePointLeader) { + return p.parseCodePoint() + } + if p.consume(tokenKindCharPropLeader) { + return p.parseCharProp() + } + if p.consume(tokenKindFragmentLeader) { + return p.parseFragment() + } + c := p.parseNormalChar() + if c == nil { + if p.consume(tokenKindBExpClose) { + p.raiseParseError(synErrBExpInvalidForm, "") + } + return nil + } + return c +} + +func (p *parser) parseBExpElem() CPTree { + var left CPTree + switch { + case p.consume(tokenKindCodePointLeader): + left = p.parseCodePoint() + case p.consume(tokenKindCharPropLeader): + left = p.parseCharProp() + if p.consume(tokenKindCharRange) { + p.raiseParseError(synErrRangePropIsUnavailable, "") + } + default: + left = p.parseNormalChar() + } + if left == nil { + return nil + } + if !p.consume(tokenKindCharRange) { + return left + } + var right CPTree + switch { + case p.consume(tokenKindCodePointLeader): + right = p.parseCodePoint() + case p.consume(tokenKindCharPropLeader): + p.raiseParseError(synErrRangePropIsUnavailable, "") + default: + right = p.parseNormalChar() + } + if right == nil { + p.raiseParseError(synErrRangeInvalidForm, "") + } + from, _, _ := left.Range() + _, to, _ := right.Range() + if !isValidOrder(from, to) { + p.raiseParseError(synErrRangeInvalidOrder, fmt.Sprintf("%X..%X", from, to)) + } + return newRangeSymbolNode(from, to) +} + +func (p *parser) parseCodePoint() CPTree { + if !p.consume(tokenKindLBrace) { + p.raiseParseError(synErrCPExpInvalidForm, "") + } + if !p.consume(tokenKindCodePoint) { + p.raiseParseError(synErrCPExpInvalidForm, "") + } + + n, err := strconv.ParseInt(p.lastTok.codePoint, 16, 64) + if err != nil { + panic(fmt.Errorf("failed to decode a code point (%v) into a int: %v", p.lastTok.codePoint, err)) + } + if n < 0x0000 || n > 0x10FFFF { + p.raiseParseError(synErrCPExpOutOfRange, "") + } + + sym := newSymbolNode(rune(n)) + + if !p.consume(tokenKindRBrace) { + p.raiseParseError(synErrCPExpInvalidForm, "") + } + + return sym +} + +func (p *parser) parseCharProp() CPTree { + if !p.consume(tokenKindLBrace) { + p.raiseParseError(synErrCharPropExpInvalidForm, "") + } + var sym1, sym2 string + if !p.consume(tokenKindCharPropSymbol) { + p.raiseParseError(synErrCharPropExpInvalidForm, "") + } + sym1 = p.lastTok.propSymbol + if p.consume(tokenKindEqual) { + if !p.consume(tokenKindCharPropSymbol) { + p.raiseParseError(synErrCharPropExpInvalidForm, "") + } + sym2 = p.lastTok.propSymbol + } + + var alt CPTree + var propName, propVal string + if sym2 != "" { + propName = sym1 + propVal = sym2 + } else { + propName = "" + propVal = sym1 + } + if !p.isContributoryPropertyExposed && ucd.IsContributoryProperty(propName) { + p.raiseParseError(synErrCharPropUnsupported, propName) + } + pat, err := ucd.NormalizeCharacterProperty(propName, propVal) + if err != nil { + p.raiseParseError(synErrCharPropUnsupported, err.Error()) + } + if pat != "" { + p := NewParser(p.kind, bytes.NewReader([]byte(pat))) + p.exposeContributoryProperty() + ast, err := p.Parse() + if err != nil { + panic(err) + } + alt = ast + } else { + cpRanges, inverse, err := ucd.FindCodePointRanges(propName, propVal) + if err != nil { + p.raiseParseError(synErrCharPropUnsupported, err.Error()) + } + if inverse { + r := cpRanges[0] + alt = exclude(newRangeSymbolNode(r.From, r.To), genAnyCharAST()) + if alt == nil { + p.raiseParseError(synErrUnmatchablePattern, "") + } + for _, r := range cpRanges[1:] { + alt = exclude(newRangeSymbolNode(r.From, r.To), alt) + if alt == nil { + p.raiseParseError(synErrUnmatchablePattern, "") + } + } + } else { + for _, r := range cpRanges { + alt = genAltNode( + alt, + newRangeSymbolNode(r.From, r.To), + ) + } + } + } + + if !p.consume(tokenKindRBrace) { + p.raiseParseError(synErrCharPropExpInvalidForm, "") + } + + return alt +} + +func (p *parser) parseFragment() CPTree { + if !p.consume(tokenKindLBrace) { + p.raiseParseError(synErrFragmentExpInvalidForm, "") + } + if !p.consume(tokenKindFragmentSymbol) { + p.raiseParseError(synErrFragmentExpInvalidForm, "") + } + sym := p.lastTok.fragmentSymbol + + if !p.consume(tokenKindRBrace) { + p.raiseParseError(synErrFragmentExpInvalidForm, "") + } + + return newFragmentNode(spec.LexKindName(sym), nil) +} + +func (p *parser) parseNormalChar() CPTree { + if !p.consume(tokenKindChar) { + return nil + } + return newSymbolNode(p.lastTok.char) +} + +func exclude(symbol, base CPTree) CPTree { + if left, right, ok := symbol.Alternatives(); ok { + return exclude(right, exclude(left, base)) + } + + if left, right, ok := base.Alternatives(); ok { + return genAltNode( + exclude(symbol, left), + exclude(symbol, right), + ) + } + + if bFrom, bTo, ok := base.Range(); ok { + sFrom, sTo, ok := symbol.Range() + if !ok { + panic(fmt.Errorf("invalid symbol tree: %T", symbol)) + } + + switch { + case sFrom > bFrom && sTo < bTo: + return genAltNode( + newRangeSymbolNode(bFrom, sFrom-1), + newRangeSymbolNode(sTo+1, bTo), + ) + case sFrom <= bFrom && sTo >= bFrom && sTo < bTo: + return newRangeSymbolNode(sTo+1, bTo) + case sFrom > bFrom && sFrom <= bTo && sTo >= bTo: + return newRangeSymbolNode(bFrom, sFrom-1) + case sFrom <= bFrom && sTo >= bTo: + return nil + default: + return base + } + } + + panic(fmt.Errorf("invalid base tree: %T", base)) +} + +func genAnyCharAST() CPTree { + return newRangeSymbolNode(0x0, 0x10FFFF) +} + +func isValidOrder(from, to rune) bool { + return from <= to +} + +func genConcatNode(cs ...CPTree) CPTree { + nonNilNodes := []CPTree{} + for _, c := range cs { + if c == nil { + continue + } + nonNilNodes = append(nonNilNodes, c) + } + if len(nonNilNodes) <= 0 { + return nil + } + if len(nonNilNodes) == 1 { + return nonNilNodes[0] + } + concat := newConcatNode(nonNilNodes[0], nonNilNodes[1]) + for _, c := range nonNilNodes[2:] { + concat = newConcatNode(concat, c) + } + return concat +} + +func genAltNode(cs ...CPTree) CPTree { + nonNilNodes := []CPTree{} + for _, c := range cs { + if c == nil { + continue + } + nonNilNodes = append(nonNilNodes, c) + } + if len(nonNilNodes) <= 0 { + return nil + } + if len(nonNilNodes) == 1 { + return nonNilNodes[0] + } + alt := newAltNode(nonNilNodes[0], nonNilNodes[1]) + for _, c := range nonNilNodes[2:] { + alt = newAltNode(alt, c) + } + return alt +} + +func (p *parser) expect(expected tokenKind) { + if !p.consume(expected) { + tok := p.peekedTok + p.raiseParseError(synErrUnexpectedToken, fmt.Sprintf("expected: %v, actual: %v", expected, tok.kind)) + } +} + +func (p *parser) consume(expected tokenKind) bool { + var tok *token + var err error + if p.peekedTok != nil { + tok = p.peekedTok + p.peekedTok = nil + } else { + tok, err = p.lex.next() + if err != nil { + if err == ParseErr { + detail, cause := p.lex.error() + p.raiseParseError(cause, detail) + } + panic(err) + } + } + p.lastTok = tok + if tok.kind == expected { + return true + } + p.peekedTok = tok + p.lastTok = nil + + return false +} + +func (p *parser) raiseParseError(err error, detail string) { + p.errCause = err + p.errDetail = detail + panic(ParseErr) +} diff --git a/grammar/lexical/parser/parser_test.go b/grammar/lexical/parser/parser_test.go new file mode 100644 index 0000000..d6cc4a8 --- /dev/null +++ b/grammar/lexical/parser/parser_test.go @@ -0,0 +1,1389 @@ +package parser + +import ( + "fmt" + "reflect" + "strings" + "testing" + + spec "github.com/nihei9/vartan/spec/grammar" + "github.com/nihei9/vartan/ucd" +) + +func TestParse(t *testing.T) { + tests := []struct { + pattern string + fragments map[spec.LexKindName]string + ast CPTree + syntaxError error + + // When an AST is large, as patterns containing a character property expression, this test only checks + // that the pattern is parsable. The check of the validity of such AST is performed by checking that it + // can be matched correctly using the driver. + skipTestAST bool + }{ + { + pattern: "a", + ast: newSymbolNode('a'), + }, + { + pattern: "abc", + ast: genConcatNode( + newSymbolNode('a'), + newSymbolNode('b'), + newSymbolNode('c'), + ), + }, + { + pattern: "a?", + ast: newOptionNode( + newSymbolNode('a'), + ), + }, + { + pattern: "[abc]?", + ast: newOptionNode( + genAltNode( + newSymbolNode('a'), + newSymbolNode('b'), + newSymbolNode('c'), + ), + ), + }, + { + pattern: "\\u{3042}?", + ast: newOptionNode( + newSymbolNode('\u3042'), + ), + }, + { + pattern: "\\p{Letter}?", + skipTestAST: true, + }, + { + pattern: "\\f{a2c}?", + fragments: map[spec.LexKindName]string{ + "a2c": "abc", + }, + ast: newOptionNode( + newFragmentNode("a2c", + genConcatNode( + newSymbolNode('a'), + newSymbolNode('b'), + newSymbolNode('c'), + ), + ), + ), + }, + { + pattern: "(a)?", + ast: newOptionNode( + newSymbolNode('a'), + ), + }, + { + pattern: "((a?)?)?", + ast: newOptionNode( + newOptionNode( + newOptionNode( + newSymbolNode('a'), + ), + ), + ), + }, + { + pattern: "(abc)?", + ast: newOptionNode( + genConcatNode( + newSymbolNode('a'), + newSymbolNode('b'), + newSymbolNode('c'), + ), + ), + }, + { + pattern: "(a|b)?", + ast: newOptionNode( + genAltNode( + newSymbolNode('a'), + newSymbolNode('b'), + ), + ), + }, + { + pattern: "?", + syntaxError: synErrRepNoTarget, + }, + { + pattern: "(?)", + syntaxError: synErrRepNoTarget, + }, + { + pattern: "a|?", + syntaxError: synErrRepNoTarget, + }, + { + pattern: "?|b", + syntaxError: synErrRepNoTarget, + }, + { + pattern: "a??", + syntaxError: synErrRepNoTarget, + }, + { + pattern: "a*", + ast: newRepeatNode( + newSymbolNode('a'), + ), + }, + { + pattern: "[abc]*", + ast: newRepeatNode( + genAltNode( + newSymbolNode('a'), + newSymbolNode('b'), + newSymbolNode('c'), + ), + ), + }, + { + pattern: "\\u{3042}*", + ast: newRepeatNode( + newSymbolNode('\u3042'), + ), + }, + { + pattern: "\\p{Letter}*", + skipTestAST: true, + }, + { + pattern: "\\f{a2c}*", + fragments: map[spec.LexKindName]string{ + "a2c": "abc", + }, + ast: newRepeatNode( + newFragmentNode("a2c", + genConcatNode( + newSymbolNode('a'), + newSymbolNode('b'), + newSymbolNode('c'), + ), + ), + ), + }, + { + pattern: "((a*)*)*", + ast: newRepeatNode( + newRepeatNode( + newRepeatNode( + newSymbolNode('a'), + ), + ), + ), + }, + { + pattern: "(abc)*", + ast: newRepeatNode( + genConcatNode( + newSymbolNode('a'), + newSymbolNode('b'), + newSymbolNode('c'), + ), + ), + }, + { + pattern: "(a|b)*", + ast: newRepeatNode( + genAltNode( + newSymbolNode('a'), + newSymbolNode('b'), + ), + ), + }, + { + pattern: "*", + syntaxError: synErrRepNoTarget, + }, + { + pattern: "(*)", + syntaxError: synErrRepNoTarget, + }, + { + pattern: "a|*", + syntaxError: synErrRepNoTarget, + }, + { + pattern: "*|b", + syntaxError: synErrRepNoTarget, + }, + { + pattern: "a**", + syntaxError: synErrRepNoTarget, + }, + { + pattern: "a+", + ast: genConcatNode( + newSymbolNode('a'), + newRepeatNode( + newSymbolNode('a'), + ), + ), + }, + { + pattern: "[abc]+", + ast: genConcatNode( + genAltNode( + newSymbolNode('a'), + newSymbolNode('b'), + newSymbolNode('c'), + ), + newRepeatNode( + genAltNode( + newSymbolNode('a'), + newSymbolNode('b'), + newSymbolNode('c'), + ), + ), + ), + }, + { + pattern: "\\u{3042}+", + ast: genConcatNode( + newSymbolNode('\u3042'), + newRepeatNode( + newSymbolNode('\u3042'), + ), + ), + }, + { + pattern: "\\p{Letter}+", + skipTestAST: true, + }, + { + pattern: "\\f{a2c}+", + fragments: map[spec.LexKindName]string{ + "a2c": "abc", + }, + ast: genConcatNode( + newFragmentNode("a2c", + genConcatNode( + newSymbolNode('a'), + newSymbolNode('b'), + newSymbolNode('c'), + ), + ), + newRepeatNode( + newFragmentNode("a2c", + genConcatNode( + newSymbolNode('a'), + newSymbolNode('b'), + newSymbolNode('c'), + ), + ), + ), + ), + }, + { + pattern: "((a+)+)+", + ast: genConcatNode( + genConcatNode( + genConcatNode( + genConcatNode( + newSymbolNode('a'), + newRepeatNode( + newSymbolNode('a'), + ), + ), + newRepeatNode( + genConcatNode( + newSymbolNode('a'), + newRepeatNode( + newSymbolNode('a'), + ), + ), + ), + ), + newRepeatNode( + genConcatNode( + genConcatNode( + newSymbolNode('a'), + newRepeatNode( + newSymbolNode('a'), + ), + ), + newRepeatNode( + genConcatNode( + newSymbolNode('a'), + newRepeatNode( + newSymbolNode('a'), + ), + ), + ), + ), + ), + ), + ), + }, + { + pattern: "(abc)+", + ast: genConcatNode( + genConcatNode( + newSymbolNode('a'), + newSymbolNode('b'), + newSymbolNode('c'), + ), + newRepeatNode( + genConcatNode( + newSymbolNode('a'), + newSymbolNode('b'), + newSymbolNode('c'), + ), + ), + ), + }, + { + pattern: "(a|b)+", + ast: genConcatNode( + genAltNode( + newSymbolNode('a'), + newSymbolNode('b'), + ), + newRepeatNode( + genAltNode( + newSymbolNode('a'), + newSymbolNode('b'), + ), + ), + ), + }, + { + pattern: "+", + syntaxError: synErrRepNoTarget, + }, + { + pattern: "(+)", + syntaxError: synErrRepNoTarget, + }, + { + pattern: "a|+", + syntaxError: synErrRepNoTarget, + }, + { + pattern: "+|b", + syntaxError: synErrRepNoTarget, + }, + { + pattern: "a++", + syntaxError: synErrRepNoTarget, + }, + { + pattern: ".", + ast: newRangeSymbolNode(0x00, 0x10FFFF), + }, + { + pattern: "[a]", + ast: newSymbolNode('a'), + }, + { + pattern: "[abc]", + ast: genAltNode( + newSymbolNode('a'), + newSymbolNode('b'), + newSymbolNode('c'), + ), + }, + { + pattern: "[a-z]", + ast: newRangeSymbolNode('a', 'z'), + }, + { + pattern: "[A-Za-z]", + ast: genAltNode( + newRangeSymbolNode('A', 'Z'), + newRangeSymbolNode('a', 'z'), + ), + }, + { + pattern: "[\\u{004E}]", + ast: newSymbolNode('N'), + }, + { + pattern: "[\\u{0061}-\\u{007A}]", + ast: newRangeSymbolNode('a', 'z'), + }, + { + pattern: "[\\p{Lu}]", + skipTestAST: true, + }, + { + pattern: "[a-\\p{Lu}]", + syntaxError: synErrRangePropIsUnavailable, + }, + { + pattern: "[\\p{Lu}-z]", + syntaxError: synErrRangePropIsUnavailable, + }, + { + pattern: "[\\p{Lu}-\\p{Ll}]", + syntaxError: synErrRangePropIsUnavailable, + }, + { + pattern: "[z-a]", + syntaxError: synErrRangeInvalidOrder, + }, + { + pattern: "a[]", + syntaxError: synErrBExpNoElem, + }, + { + pattern: "[]a", + syntaxError: synErrBExpNoElem, + }, + { + pattern: "[]", + syntaxError: synErrBExpNoElem, + }, + { + pattern: "[^\\u{004E}]", + ast: genAltNode( + newRangeSymbolNode(0x00, '\u004E'-1), + newRangeSymbolNode('\u004E'+1, 0x10FFFF), + ), + }, + { + pattern: "[^\\u{0061}-\\u{007A}]", + ast: genAltNode( + newRangeSymbolNode(0x00, '\u0061'-1), + newRangeSymbolNode('\u007A'+1, 0x10FFFF), + ), + }, + { + pattern: "[^\\p{Lu}]", + skipTestAST: true, + }, + { + pattern: "[^a-\\p{Lu}]", + syntaxError: synErrRangePropIsUnavailable, + }, + { + pattern: "[^\\p{Lu}-z]", + syntaxError: synErrRangePropIsUnavailable, + }, + { + pattern: "[^\\p{Lu}-\\p{Ll}]", + syntaxError: synErrRangePropIsUnavailable, + }, + { + pattern: "[^\\u{0000}-\\u{10FFFF}]", + syntaxError: synErrUnmatchablePattern, + }, + { + pattern: "[^\\u{0000}-\\u{FFFF}\\u{010000}-\\u{10FFFF}]", + syntaxError: synErrUnmatchablePattern, + }, + { + pattern: "[^]", + ast: newSymbolNode('^'), + }, + { + pattern: "[", + syntaxError: synErrBExpUnclosed, + }, + { + pattern: "([", + syntaxError: synErrBExpUnclosed, + }, + { + pattern: "[a", + syntaxError: synErrBExpUnclosed, + }, + { + pattern: "([a", + syntaxError: synErrBExpUnclosed, + }, + { + pattern: "[a-", + syntaxError: synErrBExpUnclosed, + }, + { + pattern: "([a-", + syntaxError: synErrBExpUnclosed, + }, + { + pattern: "[^", + syntaxError: synErrBExpUnclosed, + }, + { + pattern: "([^", + syntaxError: synErrBExpUnclosed, + }, + { + pattern: "[^a", + syntaxError: synErrBExpUnclosed, + }, + { + pattern: "([^a", + syntaxError: synErrBExpUnclosed, + }, + { + pattern: "[^a-", + syntaxError: synErrBExpUnclosed, + }, + { + pattern: "([^a-", + syntaxError: synErrBExpUnclosed, + }, + { + pattern: "]", + ast: newSymbolNode(']'), + }, + { + pattern: "(]", + syntaxError: synErrGroupUnclosed, + }, + { + pattern: "a]", + ast: genConcatNode( + newSymbolNode('a'), + newSymbolNode(']'), + ), + }, + { + pattern: "(a]", + syntaxError: synErrGroupUnclosed, + }, + { + pattern: "([)", + syntaxError: synErrBExpUnclosed, + }, + { + pattern: "([a)", + syntaxError: synErrBExpUnclosed, + }, + { + pattern: "[a-]", + ast: genAltNode( + newSymbolNode('a'), + newSymbolNode('-'), + ), + }, + { + pattern: "[^a-]", + ast: genAltNode( + newRangeSymbolNode(0x00, 0x2C), + newRangeSymbolNode(0x2E, 0x60), + newRangeSymbolNode(0x62, 0x10FFFF), + ), + }, + { + pattern: "[-z]", + ast: genAltNode( + newSymbolNode('-'), + newSymbolNode('z'), + ), + }, + { + pattern: "[^-z]", + ast: newAltNode( + newRangeSymbolNode(0x00, 0x2C), + newAltNode( + newRangeSymbolNode(0x2E, 0x79), + newRangeSymbolNode(0x7B, 0x10FFFF), + ), + ), + }, + { + pattern: "[-]", + ast: newSymbolNode('-'), + }, + { + pattern: "[^-]", + ast: genAltNode( + newRangeSymbolNode(0x00, 0x2C), + newRangeSymbolNode(0x2E, 0x10FFFF), + ), + }, + { + pattern: "[^01]", + ast: genAltNode( + newRangeSymbolNode(0x00, '0'-1), + newRangeSymbolNode('1'+1, 0x10FFFF), + ), + }, + { + pattern: "[^10]", + ast: genAltNode( + newRangeSymbolNode(0x00, '0'-1), + newRangeSymbolNode('1'+1, 0x10FFFF), + ), + }, + { + pattern: "[^a-z]", + ast: genAltNode( + newRangeSymbolNode(0x00, 'a'-1), + newRangeSymbolNode('z'+1, 0x10FFFF), + ), + }, + { + pattern: "[^az]", + ast: genAltNode( + newRangeSymbolNode(0x00, 'a'-1), + genAltNode( + newRangeSymbolNode('a'+1, 'z'-1), + newRangeSymbolNode('z'+1, 0x10FFFF), + ), + ), + }, + { + pattern: "\\u{006E}", + ast: newSymbolNode('\u006E'), + }, + { + pattern: "\\u{03BD}", + ast: newSymbolNode('\u03BD'), + }, + { + pattern: "\\u{306B}", + ast: newSymbolNode('\u306B'), + }, + { + pattern: "\\u{01F638}", + ast: newSymbolNode('\U0001F638'), + }, + { + pattern: "\\u{0000}", + ast: newSymbolNode('\u0000'), + }, + { + pattern: "\\u{10FFFF}", + ast: newSymbolNode('\U0010FFFF'), + }, + { + pattern: "\\u{110000}", + syntaxError: synErrCPExpOutOfRange, + }, + { + pattern: "\\u", + syntaxError: synErrCPExpInvalidForm, + }, + { + pattern: "\\u{", + syntaxError: synErrCPExpInvalidForm, + }, + { + pattern: "\\u{03BD", + syntaxError: synErrCPExpInvalidForm, + }, + { + pattern: "\\u{}", + syntaxError: synErrCPExpInvalidForm, + }, + { + pattern: "\\p{Letter}", + skipTestAST: true, + }, + { + pattern: "\\p{General_Category=Letter}", + skipTestAST: true, + }, + { + pattern: "\\p{ Letter }", + skipTestAST: true, + }, + { + pattern: "\\p{ General_Category = Letter }", + skipTestAST: true, + }, + { + pattern: "\\p", + syntaxError: synErrCharPropExpInvalidForm, + }, + { + pattern: "\\p{", + syntaxError: synErrCharPropExpInvalidForm, + }, + { + pattern: "\\p{Letter", + syntaxError: synErrCharPropExpInvalidForm, + }, + { + pattern: "\\p{General_Category=}", + syntaxError: synErrCharPropExpInvalidForm, + }, + { + pattern: "\\p{General_Category= }", + syntaxError: synErrCharPropInvalidSymbol, + }, + { + pattern: "\\p{=Letter}", + syntaxError: synErrCharPropExpInvalidForm, + }, + { + pattern: "\\p{ =Letter}", + syntaxError: synErrCharPropInvalidSymbol, + }, + { + pattern: "\\p{=}", + syntaxError: synErrCharPropExpInvalidForm, + }, + { + pattern: "\\p{}", + syntaxError: synErrCharPropExpInvalidForm, + }, + { + pattern: "\\f{a2c}", + fragments: map[spec.LexKindName]string{ + "a2c": "abc", + }, + ast: newFragmentNode("a2c", + genConcatNode( + newSymbolNode('a'), + newSymbolNode('b'), + newSymbolNode('c'), + ), + ), + }, + { + pattern: "\\f{ a2c }", + fragments: map[spec.LexKindName]string{ + "a2c": "abc", + }, + ast: newFragmentNode("a2c", + genConcatNode( + newSymbolNode('a'), + newSymbolNode('b'), + newSymbolNode('c'), + ), + ), + }, + { + pattern: "\\f", + syntaxError: synErrFragmentExpInvalidForm, + }, + { + pattern: "\\f{", + syntaxError: synErrFragmentExpInvalidForm, + }, + { + pattern: "\\f{a2c", + fragments: map[spec.LexKindName]string{ + "a2c": "abc", + }, + syntaxError: synErrFragmentExpInvalidForm, + }, + { + pattern: "(a)", + ast: newSymbolNode('a'), + }, + { + pattern: "(((a)))", + ast: newSymbolNode('a'), + }, + { + pattern: "a()", + syntaxError: synErrGroupNoElem, + }, + { + pattern: "()a", + syntaxError: synErrGroupNoElem, + }, + { + pattern: "()", + syntaxError: synErrGroupNoElem, + }, + { + pattern: "(", + syntaxError: synErrGroupUnclosed, + }, + { + pattern: "a(", + syntaxError: synErrGroupUnclosed, + }, + { + pattern: "(a", + syntaxError: synErrGroupUnclosed, + }, + { + pattern: "((", + syntaxError: synErrGroupUnclosed, + }, + { + pattern: "((a)", + syntaxError: synErrGroupUnclosed, + }, + { + pattern: ")", + syntaxError: synErrGroupNoInitiator, + }, + { + pattern: "a)", + syntaxError: synErrGroupNoInitiator, + }, + { + pattern: ")a", + syntaxError: synErrGroupNoInitiator, + }, + { + pattern: "))", + syntaxError: synErrGroupNoInitiator, + }, + { + pattern: "(a))", + syntaxError: synErrGroupNoInitiator, + }, + { + pattern: "Mulder|Scully", + ast: genAltNode( + genConcatNode( + newSymbolNode('M'), + newSymbolNode('u'), + newSymbolNode('l'), + newSymbolNode('d'), + newSymbolNode('e'), + newSymbolNode('r'), + ), + genConcatNode( + newSymbolNode('S'), + newSymbolNode('c'), + newSymbolNode('u'), + newSymbolNode('l'), + newSymbolNode('l'), + newSymbolNode('y'), + ), + ), + }, + { + pattern: "Langly|Frohike|Byers", + ast: genAltNode( + genConcatNode( + newSymbolNode('L'), + newSymbolNode('a'), + newSymbolNode('n'), + newSymbolNode('g'), + newSymbolNode('l'), + newSymbolNode('y'), + ), + genConcatNode( + newSymbolNode('F'), + newSymbolNode('r'), + newSymbolNode('o'), + newSymbolNode('h'), + newSymbolNode('i'), + newSymbolNode('k'), + newSymbolNode('e'), + ), + genConcatNode( + newSymbolNode('B'), + newSymbolNode('y'), + newSymbolNode('e'), + newSymbolNode('r'), + newSymbolNode('s'), + ), + ), + }, + { + pattern: "|", + syntaxError: synErrAltLackOfOperand, + }, + { + pattern: "||", + syntaxError: synErrAltLackOfOperand, + }, + { + pattern: "Mulder|", + syntaxError: synErrAltLackOfOperand, + }, + { + pattern: "|Scully", + syntaxError: synErrAltLackOfOperand, + }, + { + pattern: "Langly|Frohike|", + syntaxError: synErrAltLackOfOperand, + }, + { + pattern: "Langly||Byers", + syntaxError: synErrAltLackOfOperand, + }, + { + pattern: "|Frohike|Byers", + syntaxError: synErrAltLackOfOperand, + }, + { + pattern: "|Frohike|", + syntaxError: synErrAltLackOfOperand, + }, + { + pattern: "Fox(|)Mulder", + syntaxError: synErrAltLackOfOperand, + }, + { + pattern: "(Fox|)Mulder", + syntaxError: synErrAltLackOfOperand, + }, + { + pattern: "Fox(|Mulder)", + syntaxError: synErrAltLackOfOperand, + }, + } + for i, tt := range tests { + t.Run(fmt.Sprintf("#%v %v", i, tt.pattern), func(t *testing.T) { + fragmentTrees := map[spec.LexKindName]CPTree{} + for kind, pattern := range tt.fragments { + p := NewParser(kind, strings.NewReader(pattern)) + root, err := p.Parse() + if err != nil { + t.Fatal(err) + } + + fragmentTrees[kind] = root + } + err := CompleteFragments(fragmentTrees) + if err != nil { + t.Fatal(err) + } + + p := NewParser(spec.LexKindName("test"), strings.NewReader(tt.pattern)) + root, err := p.Parse() + if tt.syntaxError != nil { + // printCPTree(os.Stdout, root, "", "") + if err != ParseErr { + t.Fatalf("unexpected error: want: %v, got: %v", ParseErr, err) + } + _, synErr := p.Error() + if synErr != tt.syntaxError { + t.Fatalf("unexpected syntax error: want: %v, got: %v", tt.syntaxError, synErr) + } + if root != nil { + t.Fatalf("tree must be nil") + } + } else { + if err != nil { + detail, cause := p.Error() + t.Fatalf("%v: %v: %v", err, cause, detail) + } + if root == nil { + t.Fatal("tree must be non-nil") + } + + complete, err := ApplyFragments(root, fragmentTrees) + if err != nil { + t.Fatal(err) + } + if !complete { + t.Fatalf("incomplete fragments") + } + + // printCPTree(os.Stdout, root, "", "") + if !tt.skipTestAST { + r := root.(*rootNode) + testAST(t, tt.ast, r.tree) + } + } + }) + } +} + +func TestParse_ContributoryPropertyIsNotExposed(t *testing.T) { + for _, cProp := range ucd.ContributoryProperties() { + t.Run(fmt.Sprintf("%v", cProp), func(t *testing.T) { + p := NewParser(spec.LexKindName("test"), strings.NewReader(fmt.Sprintf(`\p{%v=yes}`, cProp))) + root, err := p.Parse() + if err == nil { + t.Fatalf("expected syntax error: got: nil") + } + _, synErr := p.Error() + if synErr != synErrCharPropUnsupported { + t.Fatalf("unexpected syntax error: want: %v, got: %v", synErrCharPropUnsupported, synErr) + } + if root != nil { + t.Fatalf("tree is not nil") + } + }) + } +} + +func TestExclude(t *testing.T) { + for _, test := range []struct { + caption string + target CPTree + base CPTree + result CPTree + }{ + // t.From > b.From && t.To < b.To + + // |t.From - b.From| = 1 + // |b.To - t.To| = 1 + // + // Target (t): +--+ + // Base (b): +--+--+--+ + // Result (b - t): +--+ +--+ + { + caption: "|t.From - b.From| = 1 && |b.To - t.To| = 1", + target: newSymbolNode('1'), + base: newRangeSymbolNode('0', '2'), + result: newAltNode( + newSymbolNode('0'), + newSymbolNode('2'), + ), + }, + // |t.From - b.From| > 1 + // |b.To - t.To| > 1 + // + // Target (t): +--+ + // Base (b): +--+--+--+--+--+ + // Result (b - t): +--+--+ +--+--+ + { + caption: "|t.From - b.From| > 1 && |b.To - t.To| > 1", + target: newSymbolNode('2'), + base: newRangeSymbolNode('0', '4'), + result: newAltNode( + newRangeSymbolNode('0', '1'), + newRangeSymbolNode('3', '4'), + ), + }, + + // t.From <= b.From && t.To >= b.From && t.To < b.To + + // |b.From - t.From| = 0 + // |t.To - b.From| = 0 + // |b.To - t.To| = 1 + // + // Target (t): +--+ + // Base (b): +--+--+ + // Result (b - t): +--+ + { + caption: "|b.From - t.From| = 0 && |t.To - b.From| = 0 && |b.To - t.To| = 1", + target: newSymbolNode('0'), + base: newRangeSymbolNode('0', '1'), + result: newSymbolNode('1'), + }, + // |b.From - t.From| = 0 + // |t.To - b.From| = 0 + // |b.To - t.To| > 1 + // + // Target (t): +--+ + // Base (b): +--+--+--+ + // Result (b - t): +--+--+ + { + caption: "|b.From - t.From| = 0 && |t.To - b.From| = 0 && |b.To - t.To| > 1", + target: newSymbolNode('0'), + base: newRangeSymbolNode('0', '2'), + result: newRangeSymbolNode('1', '2'), + }, + // |b.From - t.From| = 0 + // |t.To - b.From| > 0 + // |b.To - t.To| = 1 + // + // Target (t): +--+--+ + // Base (b): +--+--+--+ + // Result (b - t): +--+ + { + caption: "|b.From - t.From| = 0 && |t.To - b.From| > 0 && |b.To - t.To| = 1", + target: newRangeSymbolNode('0', '1'), + base: newRangeSymbolNode('0', '2'), + result: newSymbolNode('2'), + }, + // |b.From - t.From| = 0 + // |t.To - b.From| > 0 + // |b.To - t.To| > 1 + // + // Target (t): +--+--+ + // Base (b): +--+--+--+--+ + // Result (b - t): +--+--+ + { + caption: "|b.From - t.From| = 0 && |t.To - b.From| > 0 && |b.To - t.To| > 1", + target: newRangeSymbolNode('0', '1'), + base: newRangeSymbolNode('0', '3'), + result: newRangeSymbolNode('2', '3'), + }, + // |b.From - t.From| > 0 + // |t.To - b.From| = 0 + // |b.To - t.To| = 1 + // + // Target (t): +--+--+ + // Base (b): +--+--+ + // Result (b - t): +--+ + { + caption: "|b.From - t.From| > 0 && |t.To - b.From| = 0 && |b.To - t.To| = 1", + target: newRangeSymbolNode('0', '1'), + base: newRangeSymbolNode('1', '2'), + result: newSymbolNode('2'), + }, + // |b.From - t.From| > 0 + // |t.To - b.From| = 0 + // |b.To - t.To| > 1 + // + // Target (t): +--+--+ + // Base (b): +--+--+--+ + // Result (b - t): +--+--+ + { + caption: "|b.From - t.From| > 0 && |t.To - b.From| = 0 && |b.To - t.To| > 1", + target: newRangeSymbolNode('0', '1'), + base: newRangeSymbolNode('1', '3'), + result: newRangeSymbolNode('2', '3'), + }, + // |b.From - t.From| > 0 + // |t.To - b.From| > 0 + // |b.To - t.To| = 1 + // + // Target (t): +--+--+--+ + // Base (b): +--+--+--+ + // Result (b - t): +--+ + { + caption: "|b.From - t.From| > 0 && |t.To - b.From| > 0 && |b.To - t.To| = 1", + target: newRangeSymbolNode('0', '2'), + base: newRangeSymbolNode('1', '3'), + result: newSymbolNode('3'), + }, + // |b.From - t.From| > 0 + // |t.To - b.From| > 0 + // |b.To - t.To| > 1 + // + // Target (t): +--+--+--+ + // Base (b): +--+--+--+--+ + // Result (b - t): +--+--+ + { + caption: "|b.From - t.From| > 0 && |t.To - b.From| > 0 && |b.To - t.To| > 1", + target: newRangeSymbolNode('0', '2'), + base: newRangeSymbolNode('1', '4'), + result: newRangeSymbolNode('3', '4'), + }, + + // t.From > b.From && t.From <= b.To && t.To >= b.To + + // |t.From - b.From| = 1 + // |b.To - t.From| = 0 + // |t.To - b.To| = 0 + // + // Target (t): +--+ + // Base (b): +--+--+ + // Result (b - t): +--+ + { + caption: "|t.From - b.From| = 1 && |b.To - t.From| = 0 && |t.To - b.To| = 0", + target: newSymbolNode('1'), + base: newRangeSymbolNode('0', '1'), + result: newSymbolNode('0'), + }, + // |t.From - b.From| = 1 + // |b.To - t.From| = 0 + // |t.To - b.To| > 0 + // + // Target (t): +--+--+ + // Base (b): +--+--+ + // Result (b - t): +--+ + { + caption: "|t.From - b.From| = 1 && |b.To - t.From| = 0 && |t.To - b.To| > 0", + target: newRangeSymbolNode('1', '2'), + base: newRangeSymbolNode('0', '1'), + result: newSymbolNode('0'), + }, + // |t.From - b.From| = 1 + // |b.To - t.From| > 0 + // |t.To - b.To| = 0 + // + // Target (t): +--+--+ + // Base (b): +--+--+--+ + // Result (b - t): +--+ + { + caption: "|t.From - b.From| = 1 && |b.To - t.From| > 0 && |t.To - b.To| = 0", + target: newRangeSymbolNode('1', '2'), + base: newRangeSymbolNode('0', '2'), + result: newSymbolNode('0'), + }, + // |t.From - b.From| = 1 + // |b.To - t.From| > 0 + // |t.To - b.To| > 0 + // + // Target (t): +--+--+--+ + // Base (b): +--+--+--+ + // Result (b - t): +--+ + { + caption: "|t.From - b.From| = 1 && |b.To - t.From| > 0 && |t.To - b.To| > 0", + target: newRangeSymbolNode('1', '3'), + base: newRangeSymbolNode('0', '2'), + result: newSymbolNode('0'), + }, + // |t.From - b.From| > 1 + // |b.To - t.From| = 0 + // |t.To - b.To| = 0 + // + // Target (t): +--+ + // Base (b): +--+--+--+ + // Result (b - t): +--+--+ + { + caption: "|t.From - b.From| > 1 && |b.To - t.From| = 0 && |t.To - b.To| = 0", + target: newSymbolNode('2'), + base: newRangeSymbolNode('0', '2'), + result: newRangeSymbolNode('0', '1'), + }, + // |t.From - b.From| > 1 + // |b.To - t.From| = 0 + // |t.To - b.To| > 0 + // + // Target (t): +--+--+ + // Base (b): +--+--+--+ + // Result (b - t): +--+--+ + { + caption: "|t.From - b.From| > 1 && |b.To - t.From| = 0 && |t.To - b.To| > 0", + target: newRangeSymbolNode('2', '3'), + base: newRangeSymbolNode('0', '2'), + result: newRangeSymbolNode('0', '1'), + }, + // |t.From - b.From| > 1 + // |b.To - t.From| > 0 + // |t.To - b.To| = 0 + // + // Target (t): +--+--+ + // Base (b): +--+--+--+--+ + // Result (b - t): +--+--+ + { + caption: "|t.From - b.From| > 1 && |b.To - t.From| > 0 && |t.To - b.To| = 0", + target: newRangeSymbolNode('2', '3'), + base: newRangeSymbolNode('0', '3'), + result: newRangeSymbolNode('0', '1'), + }, + // |t.From - b.From| > 1 + // |b.To - t.From| > 0 + // |t.To - b.To| > 0 + // + // Target (t): +--+--+--+ + // Base (b): +--+--+--+--+ + // Result (b - t): +--+--+ + { + caption: "|t.From - b.From| > 1 && |b.To - t.From| > 0 && |t.To - b.To| > 0", + target: newRangeSymbolNode('2', '4'), + base: newRangeSymbolNode('0', '3'), + result: newRangeSymbolNode('0', '1'), + }, + + // t.From <= b.From && t.To >= b.To + + // |b.From - t.From| = 0 + // |t.To - b.To| = 0 + // + // Target (t): +--+ + // Base (b): +--+ + // Result (b - t): N/A + { + caption: "|b.From - t.From| = 0 && |t.To - b.To| = 0", + target: newSymbolNode('0'), + base: newSymbolNode('0'), + result: nil, + }, + // |b.From - t.From| = 0 + // |t.To - b.To| > 0 + // + // Target (t): +--+--+ + // Base (b): +--+ + // Result (b - t): N/A + { + caption: "|b.From - t.From| = 0 && |t.To - b.To| > 0", + target: newRangeSymbolNode('0', '1'), + base: newSymbolNode('0'), + result: nil, + }, + // |b.From - t.From| > 0 + // |t.To - b.To| = 0 + // + // Target (t): +--+--+ + // Base (b): +--+ + // Result (b - t): N/A + { + caption: "|b.From - t.From| > 0 && |t.To - b.To| = 0", + target: newRangeSymbolNode('0', '1'), + base: newSymbolNode('1'), + result: nil, + }, + // |b.From - t.From| > 0 + // |t.To - b.To| > 0 + // + // Target (t): +--+--+--+ + // Base (b): +--+ + // Result (b - t): N/A + { + caption: "|b.From - t.From| > 0 && |t.To - b.To| > 0", + target: newRangeSymbolNode('0', '2'), + base: newSymbolNode('1'), + result: nil, + }, + + // Others + + // |b.From - t.From| = 1 + // + // Target (t): +--+ + // Base (b): +--+ + // Result (b - t): +--+ + { + caption: "|b.From - t.From| = 1", + target: newSymbolNode('0'), + base: newSymbolNode('1'), + result: newSymbolNode('1'), + }, + // |b.From - t.From| > 1 + // + // Target (t): +--+ + // Base (b): +--+ + // Result (b - t): +--+ + { + caption: "|b.From - t.From| > 1", + target: newSymbolNode('0'), + base: newSymbolNode('2'), + result: newSymbolNode('2'), + }, + // |t.To - b.To| = 1 + // + // Target (t): +--+ + // Base (b): +--+ + // Result (b - t): +--+ + { + caption: "|t.To - b.To| = 1", + target: newSymbolNode('1'), + base: newSymbolNode('0'), + result: newSymbolNode('0'), + }, + // |t.To - b.To| > 1 + // + // Target (t): +--+ + // Base (b): +--+ + // Result (b - t): +--+ + { + caption: "|t.To - b.To| > 1", + target: newSymbolNode('2'), + base: newSymbolNode('0'), + result: newSymbolNode('0'), + }, + } { + t.Run(test.caption, func(t *testing.T) { + r := exclude(test.target, test.base) + testAST(t, test.result, r) + }) + } +} + +func testAST(t *testing.T, expected, actual CPTree) { + t.Helper() + + aTy := reflect.TypeOf(actual) + eTy := reflect.TypeOf(expected) + if eTy != aTy { + t.Fatalf("unexpected node: want: %+v, got: %+v", eTy, aTy) + } + + if actual == nil { + return + } + + switch e := expected.(type) { + case *symbolNode: + a := actual.(*symbolNode) + if a.From != e.From || a.To != e.To { + t.Fatalf("unexpected node: want: %+v, got: %+v", e, a) + } + } + eLeft, eRight := expected.children() + aLeft, aRight := actual.children() + testAST(t, eLeft, aLeft) + testAST(t, eRight, aRight) +} diff --git a/grammar/lexical/parser/tree.go b/grammar/lexical/parser/tree.go new file mode 100644 index 0000000..3d9d197 --- /dev/null +++ b/grammar/lexical/parser/tree.go @@ -0,0 +1,459 @@ +package parser + +import ( + "fmt" + "io" + "sort" + + spec "github.com/nihei9/vartan/spec/grammar" +) + +type CPRange struct { + From rune + To rune +} + +type CPTree interface { + fmt.Stringer + Range() (rune, rune, bool) + Optional() (CPTree, bool) + Repeatable() (CPTree, bool) + Concatenation() (CPTree, CPTree, bool) + Alternatives() (CPTree, CPTree, bool) + Describe() (spec.LexKindName, []spec.LexKindName, error) + + children() (CPTree, CPTree) + clone() CPTree +} + +var ( + _ CPTree = &rootNode{} + _ CPTree = &symbolNode{} + _ CPTree = &concatNode{} + _ CPTree = &altNode{} + _ CPTree = &quantifierNode{} + _ CPTree = &fragmentNode{} +) + +type rootNode struct { + kind spec.LexKindName + tree CPTree + fragments map[spec.LexKindName][]*fragmentNode +} + +func newRootNode(kind spec.LexKindName, t CPTree) *rootNode { + fragments := map[spec.LexKindName][]*fragmentNode{} + collectFragments(t, fragments) + + return &rootNode{ + kind: kind, + tree: t, + fragments: fragments, + } +} + +func collectFragments(n CPTree, fragments map[spec.LexKindName][]*fragmentNode) { + if n == nil { + return + } + + if f, ok := n.(*fragmentNode); ok { + fragments[f.kind] = append(fragments[f.kind], f) + return + } + + l, r := n.children() + collectFragments(l, fragments) + collectFragments(r, fragments) +} + +func (n *rootNode) String() string { + return fmt.Sprintf("root: %v: %v fragments", n.kind, len(n.fragments)) +} + +func (n *rootNode) Range() (rune, rune, bool) { + return n.tree.Range() +} + +func (n *rootNode) Optional() (CPTree, bool) { + return n.tree.Optional() +} + +func (n *rootNode) Repeatable() (CPTree, bool) { + return n.tree.Repeatable() +} + +func (n *rootNode) Concatenation() (CPTree, CPTree, bool) { + return n.tree.Concatenation() +} + +func (n *rootNode) Alternatives() (CPTree, CPTree, bool) { + return n.tree.Alternatives() +} + +func (n *rootNode) Describe() (spec.LexKindName, []spec.LexKindName, error) { + var frags []spec.LexKindName + for f := range n.fragments { + frags = append(frags, spec.LexKindName(f)) + } + sort.Slice(frags, func(i, j int) bool { + return frags[i] < frags[j] + }) + + return n.kind, frags, nil +} + +func (n *rootNode) children() (CPTree, CPTree) { + return n.tree.children() +} + +func (n *rootNode) clone() CPTree { + return n.tree.clone() +} + +func (n *rootNode) incomplete() bool { + return len(n.fragments) > 0 +} + +func (n *rootNode) applyFragment(kind spec.LexKindName, fragment CPTree) error { + root, ok := fragment.(*rootNode) + if !ok { + return fmt.Errorf("applyFragment can take only *rootNode: %T", fragment) + } + if root.incomplete() { + return fmt.Errorf("fragment is incomplete") + } + + fs, ok := n.fragments[kind] + if !ok { + return nil + } + for _, f := range fs { + f.tree = root.clone() + } + delete(n.fragments, kind) + + return nil +} + +type symbolNode struct { + CPRange +} + +func newSymbolNode(cp rune) *symbolNode { + return &symbolNode{ + CPRange: CPRange{ + From: cp, + To: cp, + }, + } +} + +func newRangeSymbolNode(from, to rune) *symbolNode { + return &symbolNode{ + CPRange: CPRange{ + From: from, + To: to, + }, + } +} + +func (n *symbolNode) String() string { + return fmt.Sprintf("symbol: %X..%X", n.From, n.To) +} + +func (n *symbolNode) Range() (rune, rune, bool) { + return n.From, n.To, true +} + +func (n *symbolNode) Optional() (CPTree, bool) { + return nil, false +} + +func (n *symbolNode) Repeatable() (CPTree, bool) { + return nil, false +} + +func (n *symbolNode) Concatenation() (CPTree, CPTree, bool) { + return nil, nil, false +} + +func (n *symbolNode) Alternatives() (CPTree, CPTree, bool) { + return nil, nil, false +} + +func (n *symbolNode) Describe() (spec.LexKindName, []spec.LexKindName, error) { + return spec.LexKindNameNil, nil, fmt.Errorf("%T cannot describe", n) +} + +func (n *symbolNode) children() (CPTree, CPTree) { + return nil, nil +} + +func (n *symbolNode) clone() CPTree { + return newRangeSymbolNode(n.From, n.To) +} + +type concatNode struct { + left CPTree + right CPTree +} + +func newConcatNode(left, right CPTree) *concatNode { + return &concatNode{ + left: left, + right: right, + } +} + +func (n *concatNode) String() string { + return "concat" +} + +func (n *concatNode) Range() (rune, rune, bool) { + return 0, 0, false +} + +func (n *concatNode) Optional() (CPTree, bool) { + return nil, false +} + +func (n *concatNode) Repeatable() (CPTree, bool) { + return nil, false +} + +func (n *concatNode) Concatenation() (CPTree, CPTree, bool) { + return n.left, n.right, true +} + +func (n *concatNode) Alternatives() (CPTree, CPTree, bool) { + return nil, nil, false +} + +func (n *concatNode) Describe() (spec.LexKindName, []spec.LexKindName, error) { + return spec.LexKindNameNil, nil, fmt.Errorf("%T cannot describe", n) +} + +func (n *concatNode) children() (CPTree, CPTree) { + return n.left, n.right +} + +func (n *concatNode) clone() CPTree { + if n == nil { + return nil + } + return newConcatNode(n.left.clone(), n.right.clone()) +} + +type altNode struct { + left CPTree + right CPTree +} + +func newAltNode(left, right CPTree) *altNode { + return &altNode{ + left: left, + right: right, + } +} + +func (n *altNode) String() string { + return "alt" +} + +func (n *altNode) Range() (rune, rune, bool) { + return 0, 0, false +} + +func (n *altNode) Optional() (CPTree, bool) { + return nil, false +} + +func (n *altNode) Repeatable() (CPTree, bool) { + return nil, false +} + +func (n *altNode) Concatenation() (CPTree, CPTree, bool) { + return nil, nil, false +} + +func (n *altNode) Alternatives() (CPTree, CPTree, bool) { + return n.left, n.right, true +} + +func (n *altNode) Describe() (spec.LexKindName, []spec.LexKindName, error) { + return spec.LexKindNameNil, nil, fmt.Errorf("%T cannot describe", n) +} + +func (n *altNode) children() (CPTree, CPTree) { + return n.left, n.right +} + +func (n *altNode) clone() CPTree { + return newAltNode(n.left.clone(), n.right.clone()) +} + +type quantifierNode struct { + optional bool + repeatable bool + tree CPTree +} + +func (n *quantifierNode) String() string { + switch { + case n.repeatable: + return "repeatable (>= 0 times)" + case n.optional: + return "optional (0 or 1 times)" + default: + return "invalid quantifier" + } +} + +func newRepeatNode(t CPTree) *quantifierNode { + return &quantifierNode{ + repeatable: true, + tree: t, + } +} + +func newRepeatOneOrMoreNode(t CPTree) *concatNode { + return newConcatNode( + t, + &quantifierNode{ + repeatable: true, + tree: t.clone(), + }) +} + +func newOptionNode(t CPTree) *quantifierNode { + return &quantifierNode{ + optional: true, + tree: t, + } +} + +func (n *quantifierNode) Range() (rune, rune, bool) { + return 0, 0, false +} + +func (n *quantifierNode) Optional() (CPTree, bool) { + return n.tree, n.optional +} + +func (n *quantifierNode) Repeatable() (CPTree, bool) { + return n.tree, n.repeatable +} + +func (n *quantifierNode) Concatenation() (CPTree, CPTree, bool) { + return nil, nil, false +} + +func (n *quantifierNode) Alternatives() (CPTree, CPTree, bool) { + return nil, nil, false +} + +func (n *quantifierNode) Describe() (spec.LexKindName, []spec.LexKindName, error) { + return spec.LexKindNameNil, nil, fmt.Errorf("%T cannot describe", n) +} + +func (n *quantifierNode) children() (CPTree, CPTree) { + return n.tree, nil +} + +func (n *quantifierNode) clone() CPTree { + if n.repeatable { + return newRepeatNode(n.tree.clone()) + } + return newOptionNode(n.tree.clone()) +} + +type fragmentNode struct { + kind spec.LexKindName + tree CPTree +} + +func newFragmentNode(kind spec.LexKindName, t CPTree) *fragmentNode { + return &fragmentNode{ + kind: kind, + tree: t, + } +} + +func (n *fragmentNode) String() string { + return fmt.Sprintf("fragment: %v", n.kind) +} + +func (n *fragmentNode) Range() (rune, rune, bool) { + return n.tree.Range() +} + +func (n *fragmentNode) Optional() (CPTree, bool) { + return n.tree.Optional() +} + +func (n *fragmentNode) Repeatable() (CPTree, bool) { + return n.tree.Repeatable() +} + +func (n *fragmentNode) Concatenation() (CPTree, CPTree, bool) { + return n.tree.Concatenation() +} + +func (n *fragmentNode) Alternatives() (CPTree, CPTree, bool) { + return n.tree.Alternatives() +} + +func (n *fragmentNode) Describe() (spec.LexKindName, []spec.LexKindName, error) { + return spec.LexKindNameNil, nil, fmt.Errorf("%T cannot describe", n) +} + +func (n *fragmentNode) children() (CPTree, CPTree) { + return n.tree.children() +} + +func (n *fragmentNode) clone() CPTree { + if n.tree == nil { + return newFragmentNode(n.kind, nil) + } + return newFragmentNode(n.kind, n.tree.clone()) +} + +//nolint:unused +func printCPTree(w io.Writer, t CPTree, ruledLine string, childRuledLinePrefix string) { + if t == nil { + return + } + fmt.Fprintf(w, "%v%v\n", ruledLine, t) + children := []CPTree{} + switch n := t.(type) { + case *rootNode: + children = append(children, n.tree) + case *fragmentNode: + children = append(children, n.tree) + default: + left, right := t.children() + if left != nil { + children = append(children, left) + } + if right != nil { + children = append(children, right) + } + } + num := len(children) + for i, child := range children { + line := "└─ " + if num > 1 { + if i == 0 { + line = "├─ " + } else if i < num-1 { + line = "│ " + } + } + prefix := "│ " + if i >= num-1 { + prefix = " " + } + printCPTree(w, child, childRuledLinePrefix+line, childRuledLinePrefix+prefix) + } +} diff --git a/grammar/lr0.go b/grammar/lr0.go index dea5254..77ad2e0 100644 --- a/grammar/lr0.go +++ b/grammar/lr0.go @@ -3,6 +3,8 @@ package grammar import ( "fmt" "sort" + + "github.com/nihei9/vartan/grammar/symbol" ) type lr0Automaton struct { @@ -10,8 +12,8 @@ type lr0Automaton struct { states map[kernelID]*lrState } -func genLR0Automaton(prods *productionSet, startSym symbol, errSym symbol) (*lr0Automaton, error) { - if !startSym.isStart() { +func genLR0Automaton(prods *productionSet, startSym symbol.Symbol, errSym symbol.Symbol) (*lr0Automaton, error) { + if !startSym.IsStart() { return nil, fmt.Errorf("passed symbold is not a start symbol") } @@ -67,7 +69,7 @@ func genLR0Automaton(prods *productionSet, startSym symbol, errSym symbol) (*lr0 return automaton, nil } -func genStateAndNeighbourKernels(k *kernel, prods *productionSet, errSym symbol) (*lrState, []*kernel, error) { +func genStateAndNeighbourKernels(k *kernel, prods *productionSet, errSym symbol.Symbol) (*lrState, []*kernel, error) { items, err := genLR0Closure(k, prods) if err != nil { return nil, nil, err @@ -77,7 +79,7 @@ func genStateAndNeighbourKernels(k *kernel, prods *productionSet, errSym symbol) return nil, nil, err } - next := map[symbol]kernelID{} + next := map[symbol.Symbol]kernelID{} kernels := []*kernel{} for _, n := range neighbours { next[n.symbol] = n.kernel.id @@ -125,7 +127,7 @@ func genLR0Closure(k *kernel, prods *productionSet) ([]*lrItem, error) { for len(uncheckedItems) > 0 { nextUncheckedItems := []*lrItem{} for _, item := range uncheckedItems { - if item.dottedSymbol.isTerminal() { + if item.dottedSymbol.IsTerminal() { continue } @@ -150,14 +152,14 @@ func genLR0Closure(k *kernel, prods *productionSet) ([]*lrItem, error) { } type neighbourKernel struct { - symbol symbol + symbol symbol.Symbol kernel *kernel } func genNeighbourKernels(items []*lrItem, prods *productionSet) ([]*neighbourKernel, error) { - kItemMap := map[symbol][]*lrItem{} + kItemMap := map[symbol.Symbol][]*lrItem{} for _, item := range items { - if item.dottedSymbol.isNil() { + if item.dottedSymbol.IsNil() { continue } prod, ok := prods.findByID(item.prod) @@ -171,7 +173,7 @@ func genNeighbourKernels(items []*lrItem, prods *productionSet) ([]*neighbourKer kItemMap[item.dottedSymbol] = append(kItemMap[item.dottedSymbol], kItem) } - nextSyms := []symbol{} + nextSyms := []symbol.Symbol{} for sym := range kItemMap { nextSyms = append(nextSyms, sym) } diff --git a/grammar/lr0_test.go b/grammar/lr0_test.go index 0d0b134..99d4e5b 100644 --- a/grammar/lr0_test.go +++ b/grammar/lr0_test.go @@ -5,12 +5,13 @@ import ( "strings" "testing" - spec "github.com/nihei9/vartan/spec/grammar" + "github.com/nihei9/vartan/grammar/symbol" + "github.com/nihei9/vartan/spec/grammar/parser" ) type expectedLRState struct { kernelItems []*lrItem - nextStates map[symbol][]*lrItem + nextStates map[symbol.Symbol][]*lrItem reducibleProds []*production emptyProdItems []*lrItem } @@ -41,15 +42,14 @@ id: "[A-Za-z_][0-9A-Za-z_]*"; var gram *Grammar var automaton *lr0Automaton { - ast, err := spec.Parse(strings.NewReader(src)) + ast, err := parser.Parse(strings.NewReader(src)) if err != nil { t.Fatal(err) } b := GrammarBuilder{ AST: ast, } - - gram, err = b.Build() + gram, err = b.build() if err != nil { t.Fatal(err) } @@ -118,7 +118,7 @@ id: "[A-Za-z_][0-9A-Za-z_]*"; expectedStates := []*expectedLRState{ { kernelItems: expectedKernels[0], - nextStates: map[symbol][]*lrItem{ + nextStates: map[symbol.Symbol][]*lrItem{ genSym("expr"): expectedKernels[1], genSym("term"): expectedKernels[2], genSym("factor"): expectedKernels[3], @@ -129,7 +129,7 @@ id: "[A-Za-z_][0-9A-Za-z_]*"; }, { kernelItems: expectedKernels[1], - nextStates: map[symbol][]*lrItem{ + nextStates: map[symbol.Symbol][]*lrItem{ genSym("add"): expectedKernels[6], }, reducibleProds: []*production{ @@ -138,7 +138,7 @@ id: "[A-Za-z_][0-9A-Za-z_]*"; }, { kernelItems: expectedKernels[2], - nextStates: map[symbol][]*lrItem{ + nextStates: map[symbol.Symbol][]*lrItem{ genSym("mul"): expectedKernels[7], }, reducibleProds: []*production{ @@ -147,14 +147,14 @@ id: "[A-Za-z_][0-9A-Za-z_]*"; }, { kernelItems: expectedKernels[3], - nextStates: map[symbol][]*lrItem{}, + nextStates: map[symbol.Symbol][]*lrItem{}, reducibleProds: []*production{ genProd("term", "factor"), }, }, { kernelItems: expectedKernels[4], - nextStates: map[symbol][]*lrItem{ + nextStates: map[symbol.Symbol][]*lrItem{ genSym("expr"): expectedKernels[8], genSym("term"): expectedKernels[2], genSym("factor"): expectedKernels[3], @@ -165,14 +165,14 @@ id: "[A-Za-z_][0-9A-Za-z_]*"; }, { kernelItems: expectedKernels[5], - nextStates: map[symbol][]*lrItem{}, + nextStates: map[symbol.Symbol][]*lrItem{}, reducibleProds: []*production{ genProd("factor", "id"), }, }, { kernelItems: expectedKernels[6], - nextStates: map[symbol][]*lrItem{ + nextStates: map[symbol.Symbol][]*lrItem{ genSym("term"): expectedKernels[9], genSym("factor"): expectedKernels[3], genSym("l_paren"): expectedKernels[4], @@ -182,7 +182,7 @@ id: "[A-Za-z_][0-9A-Za-z_]*"; }, { kernelItems: expectedKernels[7], - nextStates: map[symbol][]*lrItem{ + nextStates: map[symbol.Symbol][]*lrItem{ genSym("factor"): expectedKernels[10], genSym("l_paren"): expectedKernels[4], genSym("id"): expectedKernels[5], @@ -191,7 +191,7 @@ id: "[A-Za-z_][0-9A-Za-z_]*"; }, { kernelItems: expectedKernels[8], - nextStates: map[symbol][]*lrItem{ + nextStates: map[symbol.Symbol][]*lrItem{ genSym("add"): expectedKernels[6], genSym("r_paren"): expectedKernels[11], }, @@ -199,7 +199,7 @@ id: "[A-Za-z_][0-9A-Za-z_]*"; }, { kernelItems: expectedKernels[9], - nextStates: map[symbol][]*lrItem{ + nextStates: map[symbol.Symbol][]*lrItem{ genSym("mul"): expectedKernels[7], }, reducibleProds: []*production{ @@ -208,14 +208,14 @@ id: "[A-Za-z_][0-9A-Za-z_]*"; }, { kernelItems: expectedKernels[10], - nextStates: map[symbol][]*lrItem{}, + nextStates: map[symbol.Symbol][]*lrItem{}, reducibleProds: []*production{ genProd("term", "term", "mul", "factor"), }, }, { kernelItems: expectedKernels[11], - nextStates: map[symbol][]*lrItem{}, + nextStates: map[symbol.Symbol][]*lrItem{}, reducibleProds: []*production{ genProd("factor", "l_paren", "expr", "r_paren"), }, @@ -246,7 +246,7 @@ b: "bar"; var gram *Grammar var automaton *lr0Automaton { - ast, err := spec.Parse(strings.NewReader(src)) + ast, err := parser.Parse(strings.NewReader(src)) if err != nil { t.Fatal(err) } @@ -254,7 +254,7 @@ b: "bar"; b := GrammarBuilder{ AST: ast, } - gram, err = b.Build() + gram, err = b.build() if err != nil { t.Fatal(err) } @@ -298,7 +298,7 @@ b: "bar"; expectedStates := []*expectedLRState{ { kernelItems: expectedKernels[0], - nextStates: map[symbol][]*lrItem{ + nextStates: map[symbol.Symbol][]*lrItem{ genSym("s"): expectedKernels[1], genSym("foo"): expectedKernels[2], }, @@ -311,14 +311,14 @@ b: "bar"; }, { kernelItems: expectedKernels[1], - nextStates: map[symbol][]*lrItem{}, + nextStates: map[symbol.Symbol][]*lrItem{}, reducibleProds: []*production{ genProd("s'", "s"), }, }, { kernelItems: expectedKernels[2], - nextStates: map[symbol][]*lrItem{ + nextStates: map[symbol.Symbol][]*lrItem{ genSym("bar"): expectedKernels[3], genSym("b"): expectedKernels[4], }, @@ -331,14 +331,14 @@ b: "bar"; }, { kernelItems: expectedKernels[3], - nextStates: map[symbol][]*lrItem{}, + nextStates: map[symbol.Symbol][]*lrItem{}, reducibleProds: []*production{ genProd("s", "foo", "bar"), }, }, { kernelItems: expectedKernels[4], - nextStates: map[symbol][]*lrItem{}, + nextStates: map[symbol.Symbol][]*lrItem{}, reducibleProds: []*production{ genProd("bar", "b"), }, diff --git a/grammar/parsing_table.go b/grammar/parsing_table.go index 93033a3..53f692e 100644 --- a/grammar/parsing_table.go +++ b/grammar/parsing_table.go @@ -4,6 +4,7 @@ import ( "fmt" "sort" + "github.com/nihei9/vartan/grammar/symbol" spec "github.com/nihei9/vartan/spec/grammar" ) @@ -82,7 +83,7 @@ type conflict interface { type shiftReduceConflict struct { state stateNum - sym symbol + sym symbol.Symbol nextState stateNum prodNum productionNum resolvedBy conflictResolutionMethod @@ -93,7 +94,7 @@ func (c *shiftReduceConflict) conflict() { type reduceReduceConflict struct { state stateNum - sym symbol + sym symbol.Symbol prodNum1 productionNum prodNum2 productionNum resolvedBy conflictResolutionMethod @@ -123,12 +124,12 @@ type ParsingTable struct { InitialState stateNum } -func (t *ParsingTable) getAction(state stateNum, sym symbolNum) (ActionType, stateNum, productionNum) { +func (t *ParsingTable) getAction(state stateNum, sym symbol.SymbolNum) (ActionType, stateNum, productionNum) { pos := state.Int()*t.terminalCount + sym.Int() return t.actionTable[pos].describe() } -func (t *ParsingTable) getGoTo(state stateNum, sym symbolNum) (GoToType, stateNum) { +func (t *ParsingTable) getGoTo(state stateNum, sym symbol.SymbolNum) (GoToType, stateNum) { pos := state.Int()*t.nonTerminalCount + sym.Int() return t.goToTable[pos].describe() } @@ -141,8 +142,8 @@ func (t *ParsingTable) writeAction(row int, col int, act actionEntry) { t.actionTable[row*t.terminalCount+col] = act } -func (t *ParsingTable) writeGoTo(state stateNum, sym symbol, nextState stateNum) { - pos := state.Int()*t.nonTerminalCount + sym.num().Int() +func (t *ParsingTable) writeGoTo(state stateNum, sym symbol.Symbol, nextState stateNum) { + pos := state.Int()*t.nonTerminalCount + sym.Num().Int() t.goToTable[pos] = newGoToEntry(nextState) } @@ -151,7 +152,7 @@ type lrTableBuilder struct { prods *productionSet termCount int nonTermCount int - symTab *symbolTableReader + symTab *symbol.SymbolTableReader precAndAssoc *precAndAssoc conflicts []conflict @@ -179,7 +180,7 @@ func (b *lrTableBuilder) build() (*ParsingTable, error) { for sym, kID := range state.next { nextState := b.automaton.states[kID] - if sym.isTerminal() { + if sym.IsTerminal() { b.writeShiftAction(ptab, state.num, sym, nextState.num) } else { ptab.writeGoTo(state.num, sym, nextState.num) @@ -226,12 +227,12 @@ func (b *lrTableBuilder) build() (*ParsingTable, error) { // writeShiftAction writes a shift action to the parsing table. When a shift/reduce conflict occurred, // we prioritize the shift action. -func (b *lrTableBuilder) writeShiftAction(tab *ParsingTable, state stateNum, sym symbol, nextState stateNum) { - act := tab.readAction(state.Int(), sym.num().Int()) +func (b *lrTableBuilder) writeShiftAction(tab *ParsingTable, state stateNum, sym symbol.Symbol, nextState stateNum) { + act := tab.readAction(state.Int(), sym.Num().Int()) if !act.isEmpty() { ty, _, p := act.describe() if ty == ActionTypeReduce { - act, method := b.resolveSRConflict(sym.num(), p) + act, method := b.resolveSRConflict(sym.Num(), p) b.conflicts = append(b.conflicts, &shiftReduceConflict{ state: state, sym: sym, @@ -240,19 +241,19 @@ func (b *lrTableBuilder) writeShiftAction(tab *ParsingTable, state stateNum, sym resolvedBy: method, }) if act == ActionTypeShift { - tab.writeAction(state.Int(), sym.num().Int(), newShiftActionEntry(nextState)) + tab.writeAction(state.Int(), sym.Num().Int(), newShiftActionEntry(nextState)) } return } } - tab.writeAction(state.Int(), sym.num().Int(), newShiftActionEntry(nextState)) + tab.writeAction(state.Int(), sym.Num().Int(), newShiftActionEntry(nextState)) } // writeReduceAction writes a reduce action to the parsing table. When a shift/reduce conflict occurred, // we prioritize the shift action, and when a reduce/reduce conflict we prioritize the action that reduces // the production with higher priority. Productions defined earlier in the grammar file have a higher priority. -func (b *lrTableBuilder) writeReduceAction(tab *ParsingTable, state stateNum, sym symbol, prod productionNum) { - act := tab.readAction(state.Int(), sym.num().Int()) +func (b *lrTableBuilder) writeReduceAction(tab *ParsingTable, state stateNum, sym symbol.Symbol, prod productionNum) { + act := tab.readAction(state.Int(), sym.Num().Int()) if !act.isEmpty() { ty, s, p := act.describe() switch ty { @@ -269,12 +270,12 @@ func (b *lrTableBuilder) writeReduceAction(tab *ParsingTable, state stateNum, sy resolvedBy: ResolvedByProdOrder, }) if p < prod { - tab.writeAction(state.Int(), sym.num().Int(), newReduceActionEntry(p)) + tab.writeAction(state.Int(), sym.Num().Int(), newReduceActionEntry(p)) } else { - tab.writeAction(state.Int(), sym.num().Int(), newReduceActionEntry(prod)) + tab.writeAction(state.Int(), sym.Num().Int(), newReduceActionEntry(prod)) } case ActionTypeShift: - act, method := b.resolveSRConflict(sym.num(), prod) + act, method := b.resolveSRConflict(sym.Num(), prod) b.conflicts = append(b.conflicts, &shiftReduceConflict{ state: state, sym: sym, @@ -283,15 +284,15 @@ func (b *lrTableBuilder) writeReduceAction(tab *ParsingTable, state stateNum, sy resolvedBy: method, }) if act == ActionTypeReduce { - tab.writeAction(state.Int(), sym.num().Int(), newReduceActionEntry(prod)) + tab.writeAction(state.Int(), sym.Num().Int(), newReduceActionEntry(prod)) } } return } - tab.writeAction(state.Int(), sym.num().Int(), newReduceActionEntry(prod)) + tab.writeAction(state.Int(), sym.Num().Int(), newReduceActionEntry(prod)) } -func (b *lrTableBuilder) resolveSRConflict(sym symbolNum, prod productionNum) (ActionType, conflictResolutionMethod) { +func (b *lrTableBuilder) resolveSRConflict(sym symbol.SymbolNum, prod productionNum) (ActionType, conflictResolutionMethod) { symPrec := b.precAndAssoc.terminalPrecedence(sym) prodPrec := b.precAndAssoc.productionPredence(prod) if symPrec == 0 || prodPrec == 0 { @@ -313,26 +314,26 @@ func (b *lrTableBuilder) resolveSRConflict(sym symbolNum, prod productionNum) (A func (b *lrTableBuilder) genReport(tab *ParsingTable, gram *Grammar) (*spec.Report, error) { var terms []*spec.Terminal { - termSyms := b.symTab.terminalSymbols() + termSyms := b.symTab.TerminalSymbols() terms = make([]*spec.Terminal, len(termSyms)+1) for _, sym := range termSyms { - name, ok := b.symTab.toText(sym) + name, ok := b.symTab.ToText(sym) if !ok { return nil, fmt.Errorf("failed to generate terminals: symbol not found: %v", sym) } term := &spec.Terminal{ - Number: sym.num().Int(), + Number: sym.Num().Int(), Name: name, } - prec := b.precAndAssoc.terminalPrecedence(sym.num()) + prec := b.precAndAssoc.terminalPrecedence(sym.Num()) if prec != precNil { term.Precedence = prec } - assoc := b.precAndAssoc.terminalAssociativity(sym.num()) + assoc := b.precAndAssoc.terminalAssociativity(sym.Num()) switch assoc { case assocTypeLeft: term.Associativity = "l" @@ -340,22 +341,22 @@ func (b *lrTableBuilder) genReport(tab *ParsingTable, gram *Grammar) (*spec.Repo term.Associativity = "r" } - terms[sym.num()] = term + terms[sym.Num()] = term } } var nonTerms []*spec.NonTerminal { - nonTermSyms := b.symTab.nonTerminalSymbols() + nonTermSyms := b.symTab.NonTerminalSymbols() nonTerms = make([]*spec.NonTerminal, len(nonTermSyms)+1) for _, sym := range nonTermSyms { - name, ok := b.symTab.toText(sym) + name, ok := b.symTab.ToText(sym) if !ok { return nil, fmt.Errorf("failed to generate non-terminals: symbol not found: %v", sym) } - nonTerms[sym.num()] = &spec.NonTerminal{ - Number: sym.num().Int(), + nonTerms[sym.Num()] = &spec.NonTerminal{ + Number: sym.Num().Int(), Name: name, } } @@ -368,16 +369,16 @@ func (b *lrTableBuilder) genReport(tab *ParsingTable, gram *Grammar) (*spec.Repo for _, p := range ps { rhs := make([]int, len(p.rhs)) for i, e := range p.rhs { - if e.isTerminal() { - rhs[i] = e.num().Int() + if e.IsTerminal() { + rhs[i] = e.Num().Int() } else { - rhs[i] = e.num().Int() * -1 + rhs[i] = e.Num().Int() * -1 } } prod := &spec.Production{ Number: p.num.Int(), - LHS: p.lhs.num().Int(), + LHS: p.lhs.Num().Int(), RHS: rhs, } @@ -441,33 +442,33 @@ func (b *lrTableBuilder) genReport(tab *ParsingTable, gram *Grammar) (*spec.Repo var goTo []*spec.Transition { TERMINALS_LOOP: - for _, t := range b.symTab.terminalSymbols() { - act, next, prod := tab.getAction(s.num, t.num()) + for _, t := range b.symTab.TerminalSymbols() { + act, next, prod := tab.getAction(s.num, t.Num()) switch act { case ActionTypeShift: shift = append(shift, &spec.Transition{ - Symbol: t.num().Int(), + Symbol: t.Num().Int(), State: next.Int(), }) case ActionTypeReduce: for _, r := range reduce { if r.Production == prod.Int() { - r.LookAhead = append(r.LookAhead, t.num().Int()) + r.LookAhead = append(r.LookAhead, t.Num().Int()) continue TERMINALS_LOOP } } reduce = append(reduce, &spec.Reduce{ - LookAhead: []int{t.num().Int()}, + LookAhead: []int{t.Num().Int()}, Production: prod.Int(), }) } } - for _, n := range b.symTab.nonTerminalSymbols() { - ty, next := tab.getGoTo(s.num, n.num()) + for _, n := range b.symTab.NonTerminalSymbols() { + ty, next := tab.getGoTo(s.num, n.Num()) if ty == GoToTypeRegistered { goTo = append(goTo, &spec.Transition{ - Symbol: n.num().Int(), + Symbol: n.Num().Int(), State: next.Int(), }) } @@ -489,13 +490,13 @@ func (b *lrTableBuilder) genReport(tab *ParsingTable, gram *Grammar) (*spec.Repo { for _, c := range srConflicts[s.num] { conflict := &spec.SRConflict{ - Symbol: c.sym.num().Int(), + Symbol: c.sym.Num().Int(), State: c.nextState.Int(), Production: c.prodNum.Int(), ResolvedBy: c.resolvedBy.Int(), } - ty, s, p := tab.getAction(s.num, c.sym.num()) + ty, s, p := tab.getAction(s.num, c.sym.Num()) switch ty { case ActionTypeShift: n := s.Int() @@ -514,13 +515,13 @@ func (b *lrTableBuilder) genReport(tab *ParsingTable, gram *Grammar) (*spec.Repo for _, c := range rrConflicts[s.num] { conflict := &spec.RRConflict{ - Symbol: c.sym.num().Int(), + Symbol: c.sym.Num().Int(), Production1: c.prodNum1.Int(), Production2: c.prodNum2.Int(), ResolvedBy: c.resolvedBy.Int(), } - _, _, p := tab.getAction(s.num, c.sym.num()) + _, _, p := tab.getAction(s.num, c.sym.Num()) conflict.AdoptedProduction = p.Int() rr = append(rr, conflict) diff --git a/grammar/parsing_table_test.go b/grammar/parsing_table_test.go index fe56722..ae829e6 100644 --- a/grammar/parsing_table_test.go +++ b/grammar/parsing_table_test.go @@ -5,13 +5,14 @@ import ( "strings" "testing" - spec "github.com/nihei9/vartan/spec/grammar" + "github.com/nihei9/vartan/grammar/symbol" + "github.com/nihei9/vartan/spec/grammar/parser" ) type expectedState struct { kernelItems []*lrItem - acts map[symbol]testActionEntry - goTos map[symbol][]*lrItem + acts map[symbol.Symbol]testActionEntry + goTos map[symbol.Symbol][]*lrItem } func TestGenLALRParsingTable(t *testing.T) { @@ -32,14 +33,14 @@ id: "[A-Za-z0-9_]+"; var nonTermCount int var termCount int { - ast, err := spec.Parse(strings.NewReader(src)) + ast, err := parser.Parse(strings.NewReader(src)) if err != nil { t.Fatal(err) } b := GrammarBuilder{ AST: ast, } - gram, err = b.Build() + gram, err = b.build() if err != nil { t.Fatal(err) } @@ -56,11 +57,11 @@ id: "[A-Za-z0-9_]+"; t.Fatal(err) } - nonTermTexts, err := gram.symbolTable.nonTerminalTexts() + nonTermTexts, err := gram.symbolTable.NonTerminalTexts() if err != nil { t.Fatal(err) } - termTexts, err := gram.symbolTable.terminalTexts() + termTexts, err := gram.symbolTable.TerminalTexts() if err != nil { t.Fatal(err) } @@ -89,42 +90,42 @@ id: "[A-Za-z0-9_]+"; expectedKernels := map[int][]*lrItem{ 0: { - withLookAhead(genLR0Item("s'", 0, "s"), symbolEOF), + withLookAhead(genLR0Item("s'", 0, "s"), symbol.SymbolEOF), }, 1: { - withLookAhead(genLR0Item("s'", 1, "s"), symbolEOF), + withLookAhead(genLR0Item("s'", 1, "s"), symbol.SymbolEOF), }, 2: { - withLookAhead(genLR0Item("s", 1, "l", "eq", "r"), symbolEOF), - withLookAhead(genLR0Item("r", 1, "l"), symbolEOF), + withLookAhead(genLR0Item("s", 1, "l", "eq", "r"), symbol.SymbolEOF), + withLookAhead(genLR0Item("r", 1, "l"), symbol.SymbolEOF), }, 3: { - withLookAhead(genLR0Item("s", 1, "r"), symbolEOF), + withLookAhead(genLR0Item("s", 1, "r"), symbol.SymbolEOF), }, 4: { - withLookAhead(genLR0Item("l", 1, "ref", "r"), genSym("eq"), symbolEOF), + withLookAhead(genLR0Item("l", 1, "ref", "r"), genSym("eq"), symbol.SymbolEOF), }, 5: { - withLookAhead(genLR0Item("l", 1, "id"), genSym("eq"), symbolEOF), + withLookAhead(genLR0Item("l", 1, "id"), genSym("eq"), symbol.SymbolEOF), }, 6: { - withLookAhead(genLR0Item("s", 2, "l", "eq", "r"), symbolEOF), + withLookAhead(genLR0Item("s", 2, "l", "eq", "r"), symbol.SymbolEOF), }, 7: { - withLookAhead(genLR0Item("l", 2, "ref", "r"), genSym("eq"), symbolEOF), + withLookAhead(genLR0Item("l", 2, "ref", "r"), genSym("eq"), symbol.SymbolEOF), }, 8: { - withLookAhead(genLR0Item("r", 1, "l"), genSym("eq"), symbolEOF), + withLookAhead(genLR0Item("r", 1, "l"), genSym("eq"), symbol.SymbolEOF), }, 9: { - withLookAhead(genLR0Item("s", 3, "l", "eq", "r"), symbolEOF), + withLookAhead(genLR0Item("s", 3, "l", "eq", "r"), symbol.SymbolEOF), }, } expectedStates := []expectedState{ { kernelItems: expectedKernels[0], - acts: map[symbol]testActionEntry{ + acts: map[symbol.Symbol]testActionEntry{ genSym("ref"): { ty: ActionTypeShift, nextState: expectedKernels[4], @@ -134,7 +135,7 @@ id: "[A-Za-z0-9_]+"; nextState: expectedKernels[5], }, }, - goTos: map[symbol][]*lrItem{ + goTos: map[symbol.Symbol][]*lrItem{ genSym("s"): expectedKernels[1], genSym("l"): expectedKernels[2], genSym("r"): expectedKernels[3], @@ -142,8 +143,8 @@ id: "[A-Za-z0-9_]+"; }, { kernelItems: expectedKernels[1], - acts: map[symbol]testActionEntry{ - symbolEOF: { + acts: map[symbol.Symbol]testActionEntry{ + symbol.SymbolEOF: { ty: ActionTypeReduce, production: genProd("s'", "s"), }, @@ -151,12 +152,12 @@ id: "[A-Za-z0-9_]+"; }, { kernelItems: expectedKernels[2], - acts: map[symbol]testActionEntry{ + acts: map[symbol.Symbol]testActionEntry{ genSym("eq"): { ty: ActionTypeShift, nextState: expectedKernels[6], }, - symbolEOF: { + symbol.SymbolEOF: { ty: ActionTypeReduce, production: genProd("r", "l"), }, @@ -164,8 +165,8 @@ id: "[A-Za-z0-9_]+"; }, { kernelItems: expectedKernels[3], - acts: map[symbol]testActionEntry{ - symbolEOF: { + acts: map[symbol.Symbol]testActionEntry{ + symbol.SymbolEOF: { ty: ActionTypeReduce, production: genProd("s", "r"), }, @@ -173,7 +174,7 @@ id: "[A-Za-z0-9_]+"; }, { kernelItems: expectedKernels[4], - acts: map[symbol]testActionEntry{ + acts: map[symbol.Symbol]testActionEntry{ genSym("ref"): { ty: ActionTypeShift, nextState: expectedKernels[4], @@ -183,19 +184,19 @@ id: "[A-Za-z0-9_]+"; nextState: expectedKernels[5], }, }, - goTos: map[symbol][]*lrItem{ + goTos: map[symbol.Symbol][]*lrItem{ genSym("r"): expectedKernels[7], genSym("l"): expectedKernels[8], }, }, { kernelItems: expectedKernels[5], - acts: map[symbol]testActionEntry{ + acts: map[symbol.Symbol]testActionEntry{ genSym("eq"): { ty: ActionTypeReduce, production: genProd("l", "id"), }, - symbolEOF: { + symbol.SymbolEOF: { ty: ActionTypeReduce, production: genProd("l", "id"), }, @@ -203,7 +204,7 @@ id: "[A-Za-z0-9_]+"; }, { kernelItems: expectedKernels[6], - acts: map[symbol]testActionEntry{ + acts: map[symbol.Symbol]testActionEntry{ genSym("ref"): { ty: ActionTypeShift, nextState: expectedKernels[4], @@ -213,19 +214,19 @@ id: "[A-Za-z0-9_]+"; nextState: expectedKernels[5], }, }, - goTos: map[symbol][]*lrItem{ + goTos: map[symbol.Symbol][]*lrItem{ genSym("l"): expectedKernels[8], genSym("r"): expectedKernels[9], }, }, { kernelItems: expectedKernels[7], - acts: map[symbol]testActionEntry{ + acts: map[symbol.Symbol]testActionEntry{ genSym("eq"): { ty: ActionTypeReduce, production: genProd("l", "ref", "r"), }, - symbolEOF: { + symbol.SymbolEOF: { ty: ActionTypeReduce, production: genProd("l", "ref", "r"), }, @@ -233,12 +234,12 @@ id: "[A-Za-z0-9_]+"; }, { kernelItems: expectedKernels[8], - acts: map[symbol]testActionEntry{ + acts: map[symbol.Symbol]testActionEntry{ genSym("eq"): { ty: ActionTypeReduce, production: genProd("r", "l"), }, - symbolEOF: { + symbol.SymbolEOF: { ty: ActionTypeReduce, production: genProd("r", "l"), }, @@ -246,8 +247,8 @@ id: "[A-Za-z0-9_]+"; }, { kernelItems: expectedKernels[9], - acts: map[symbol]testActionEntry{ - symbolEOF: { + acts: map[symbol.Symbol]testActionEntry{ + symbol.SymbolEOF: { ty: ActionTypeReduce, production: genProd("s", "l", "eq", "r"), }, @@ -287,11 +288,11 @@ id: "[A-Za-z0-9_]+"; } func testAction(t *testing.T, expectedState *expectedState, state *lrState, ptab *ParsingTable, automaton *lr0Automaton, gram *Grammar, termCount int) { - nonEmptyEntries := map[symbolNum]struct{}{} + nonEmptyEntries := map[symbol.SymbolNum]struct{}{} for eSym, eAct := range expectedState.acts { - nonEmptyEntries[eSym.num()] = struct{}{} + nonEmptyEntries[eSym.Num()] = struct{}{} - ty, stateNum, prodNum := ptab.getAction(state.num, eSym.num()) + ty, stateNum, prodNum := ptab.getAction(state.num, eSym.Num()) if ty != eAct.ty { t.Fatalf("action type is mismatched; want: %v, got: %v", eAct.ty, ty) } @@ -319,10 +320,10 @@ func testAction(t *testing.T, expectedState *expectedState, state *lrState, ptab } } for symNum := 0; symNum < termCount; symNum++ { - if _, checked := nonEmptyEntries[symbolNum(symNum)]; checked { + if _, checked := nonEmptyEntries[symbol.SymbolNum(symNum)]; checked { continue } - ty, stateNum, prodNum := ptab.getAction(state.num, symbolNum(symNum)) + ty, stateNum, prodNum := ptab.getAction(state.num, symbol.SymbolNum(symNum)) if ty != ActionTypeError { t.Errorf("unexpected ACTION entry; state: #%v, symbol: #%v, action type: %v, next state: #%v, prodction: #%v", state.num, symNum, ty, stateNum, prodNum) } @@ -330,15 +331,15 @@ func testAction(t *testing.T, expectedState *expectedState, state *lrState, ptab } func testGoTo(t *testing.T, expectedState *expectedState, state *lrState, ptab *ParsingTable, automaton *lr0Automaton, nonTermCount int) { - nonEmptyEntries := map[symbolNum]struct{}{} + nonEmptyEntries := map[symbol.SymbolNum]struct{}{} for eSym, eGoTo := range expectedState.goTos { - nonEmptyEntries[eSym.num()] = struct{}{} + nonEmptyEntries[eSym.Num()] = struct{}{} eNextState, err := newKernel(eGoTo) if err != nil { t.Fatal(err) } - ty, stateNum := ptab.getGoTo(state.num, eSym.num()) + ty, stateNum := ptab.getGoTo(state.num, eSym.Num()) if ty != GoToTypeRegistered { t.Fatalf("GOTO entry was not found; state: #%v, symbol: #%v", state.num, eSym) } @@ -351,10 +352,10 @@ func testGoTo(t *testing.T, expectedState *expectedState, state *lrState, ptab * } } for symNum := 0; symNum < nonTermCount; symNum++ { - if _, checked := nonEmptyEntries[symbolNum(symNum)]; checked { + if _, checked := nonEmptyEntries[symbol.SymbolNum(symNum)]; checked { continue } - ty, _ := ptab.getGoTo(state.num, symbolNum(symNum)) + ty, _ := ptab.getGoTo(state.num, symbol.SymbolNum(symNum)) if ty != GoToTypeError { t.Errorf("unexpected GOTO entry; state: #%v, symbol: #%v", state.num, symNum) } diff --git a/grammar/production.go b/grammar/production.go index 87b392f..1978039 100644 --- a/grammar/production.go +++ b/grammar/production.go @@ -4,6 +4,8 @@ import ( "crypto/sha256" "encoding/hex" "fmt" + + "github.com/nihei9/vartan/grammar/symbol" ) type productionID [32]byte @@ -12,10 +14,10 @@ func (id productionID) String() string { return hex.EncodeToString(id[:]) } -func genProductionID(lhs symbol, rhs []symbol) productionID { - seq := lhs.byte() +func genProductionID(lhs symbol.Symbol, rhs []symbol.Symbol) productionID { + seq := lhs.Byte() for _, sym := range rhs { - seq = append(seq, sym.byte()...) + seq = append(seq, sym.Byte()...) } return productionID(sha256.Sum256(seq)) } @@ -35,17 +37,17 @@ func (n productionNum) Int() int { type production struct { id productionID num productionNum - lhs symbol - rhs []symbol + lhs symbol.Symbol + rhs []symbol.Symbol rhsLen int } -func newProduction(lhs symbol, rhs []symbol) (*production, error) { - if lhs.isNil() { +func newProduction(lhs symbol.Symbol, rhs []symbol.Symbol) (*production, error) { + if lhs.IsNil() { return nil, fmt.Errorf("LHS must be a non-nil symbol; LHS: %v, RHS: %v", lhs, rhs) } for _, sym := range rhs { - if sym.isNil() { + if sym.IsNil() { return nil, fmt.Errorf("a symbol of RHS must be a non-nil symbol; LHS: %v, RHS: %v", lhs, rhs) } } @@ -63,14 +65,14 @@ func (p *production) isEmpty() bool { } type productionSet struct { - lhs2Prods map[symbol][]*production + lhs2Prods map[symbol.Symbol][]*production id2Prod map[productionID]*production num productionNum } func newProductionSet() *productionSet { return &productionSet{ - lhs2Prods: map[symbol][]*production{}, + lhs2Prods: map[symbol.Symbol][]*production{}, id2Prod: map[productionID]*production{}, num: productionNumMin, } @@ -81,7 +83,7 @@ func (ps *productionSet) append(prod *production) { return } - if prod.lhs.isStart() { + if prod.lhs.IsStart() { prod.num = productionNumStart } else { prod.num = ps.num @@ -101,8 +103,8 @@ func (ps *productionSet) findByID(id productionID) (*production, bool) { return prod, ok } -func (ps *productionSet) findByLHS(lhs symbol) ([]*production, bool) { - if lhs.isNil() { +func (ps *productionSet) findByLHS(lhs symbol.Symbol) ([]*production, bool) { + if lhs.IsNil() { return nil, false } diff --git a/grammar/semantic_error.go b/grammar/semantic_error.go index 589e324..88a6b17 100644 --- a/grammar/semantic_error.go +++ b/grammar/semantic_error.go @@ -1,42 +1,30 @@ package grammar -type SemanticError struct { - message string -} - -func newSemanticError(message string) *SemanticError { - return &SemanticError{ - message: message, - } -} - -func (e *SemanticError) Error() string { - return e.message -} +import "errors" var ( - semErrNoGrammarName = newSemanticError("name is missing") - semErrSpellingInconsistency = newSemanticError("the identifiers are treated as the same. please use the same spelling") - semErrDuplicateAssoc = newSemanticError("associativity and precedence cannot be specified multiple times for a symbol") - semErrUndefinedPrec = newSemanticError("symbol must has precedence") - semErrUndefinedOrdSym = newSemanticError("undefined ordered symbol") - semErrUnusedProduction = newSemanticError("unused production") - semErrUnusedTerminal = newSemanticError("unused terminal") - semErrTermCannotBeSkipped = newSemanticError("a terminal used in productions cannot be skipped") - semErrNoProduction = newSemanticError("a grammar needs at least one production") - semErrUndefinedSym = newSemanticError("undefined symbol") - semErrDuplicateProduction = newSemanticError("duplicate production") - semErrDuplicateTerminal = newSemanticError("duplicate terminal") - semErrDuplicateFragment = newSemanticError("duplicate fragment") - semErrDuplicateName = newSemanticError("duplicate names are not allowed between terminals and non-terminals") - semErrErrSymIsReserved = newSemanticError("symbol 'error' is reserved as a terminal symbol") - semErrDuplicateLabel = newSemanticError("a label must be unique in an alternative") - semErrInvalidLabel = newSemanticError("a label must differ from terminal symbols or non-terminal symbols") - semErrDirInvalidName = newSemanticError("invalid directive name") - semErrDirInvalidParam = newSemanticError("invalid parameter") - semErrDuplicateDir = newSemanticError("a directive must not be duplicated") - semErrDuplicateElem = newSemanticError("duplicate element") - semErrAmbiguousElem = newSemanticError("ambiguous element") - semErrInvalidProdDir = newSemanticError("invalid production directive") - semErrInvalidAltDir = newSemanticError("invalid alternative directive") + semErrNoGrammarName = errors.New("name is missing") + semErrSpellingInconsistency = errors.New("the identifiers are treated as the same. please use the same spelling") + semErrDuplicateAssoc = errors.New("associativity and precedence cannot be specified multiple times for a symbol") + semErrUndefinedPrec = errors.New("symbol must has precedence") + semErrUndefinedOrdSym = errors.New("undefined ordered symbol") + semErrUnusedProduction = errors.New("unused production") + semErrUnusedTerminal = errors.New("unused terminal") + semErrTermCannotBeSkipped = errors.New("a terminal used in productions cannot be skipped") + semErrNoProduction = errors.New("a grammar needs at least one production") + semErrUndefinedSym = errors.New("undefined symbol") + semErrDuplicateProduction = errors.New("duplicate production") + semErrDuplicateTerminal = errors.New("duplicate terminal") + semErrDuplicateFragment = errors.New("duplicate fragment") + semErrDuplicateName = errors.New("duplicate names are not allowed between terminals and non-terminals") + semErrErrSymIsReserved = errors.New("symbol 'error' is reserved as a terminal symbol") + semErrDuplicateLabel = errors.New("a label must be unique in an alternative") + semErrInvalidLabel = errors.New("a label must differ from terminal symbols or non-terminal symbols") + semErrDirInvalidName = errors.New("invalid directive name") + semErrDirInvalidParam = errors.New("invalid parameter") + semErrDuplicateDir = errors.New("a directive must not be duplicated") + semErrDuplicateElem = errors.New("duplicate element") + semErrAmbiguousElem = errors.New("ambiguous element") + semErrInvalidProdDir = errors.New("invalid production directive") + semErrInvalidAltDir = errors.New("invalid alternative directive") ) diff --git a/grammar/symbol.go b/grammar/symbol/symbol.go index 9eba032..f9e6a93 100644 --- a/grammar/symbol.go +++ b/grammar/symbol/symbol.go @@ -1,4 +1,4 @@ -package grammar +package symbol import ( "fmt" @@ -16,15 +16,15 @@ func (t symbolKind) String() string { return string(t) } -type symbolNum uint16 +type SymbolNum uint16 -func (n symbolNum) Int() int { +func (n SymbolNum) Int() int { return int(n) } -type symbol uint16 +type Symbol uint16 -func (s symbol) String() string { +func (s Symbol) String() string { kind, isStart, isEOF, num := s.describe() var prefix string switch { @@ -56,24 +56,24 @@ const ( symbolNumStart = uint16(0x0001) // 0000 0000 0000 0001 symbolNumEOF = uint16(0x0001) // 0000 0000 0000 0001 - symbolNil = symbol(0) // 0000 0000 0000 0000 - symbolStart = symbol(maskNonTerminal | maskStartOrEOF | symbolNumStart) // 0100 0000 0000 0001 - symbolEOF = symbol(maskTerminal | maskStartOrEOF | symbolNumEOF) // 1100 0000 0000 0001: The EOF symbol is treated as a terminal symbol. + SymbolNil = Symbol(0) // 0000 0000 0000 0000 + symbolStart = Symbol(maskNonTerminal | maskStartOrEOF | symbolNumStart) // 0100 0000 0000 0001 + SymbolEOF = Symbol(maskTerminal | maskStartOrEOF | symbolNumEOF) // 1100 0000 0000 0001: The EOF symbol is treated as a terminal symbol. // The symbol name contains `<` and `>` to avoid conflicting with user-defined symbols. symbolNameEOF = "<eof>" - nonTerminalNumMin = symbolNum(2) // The number 1 is used by a start symbol. - terminalNumMin = symbolNum(2) // The number 1 is used by the EOF symbol. - symbolNumMax = symbolNum(0xffff) >> 2 // 0011 1111 1111 1111 + nonTerminalNumMin = SymbolNum(2) // The number 1 is used by a start symbol. + terminalNumMin = SymbolNum(2) // The number 1 is used by the EOF symbol. + symbolNumMax = SymbolNum(0xffff) >> 2 // 0011 1111 1111 1111 ) -func newSymbol(kind symbolKind, isStart bool, num symbolNum) (symbol, error) { +func newSymbol(kind symbolKind, isStart bool, num SymbolNum) (Symbol, error) { if num > symbolNumMax { - return symbolNil, fmt.Errorf("a symbol number exceeds the limit; limit: %v, passed: %v", symbolNumMax, num) + return SymbolNil, fmt.Errorf("a symbol number exceeds the limit; limit: %v, passed: %v", symbolNumMax, num) } if kind == symbolKindTerminal && isStart { - return symbolNil, fmt.Errorf("a start symbol must be a non-terminal symbol") + return SymbolNil, fmt.Errorf("a start symbol must be a non-terminal symbol") } kindMask := maskNonTerminal @@ -84,58 +84,58 @@ func newSymbol(kind symbolKind, isStart bool, num symbolNum) (symbol, error) { if isStart { startMask = maskStartOrEOF } - return symbol(kindMask | startMask | uint16(num)), nil + return Symbol(kindMask | startMask | uint16(num)), nil } -func (s symbol) num() symbolNum { +func (s Symbol) Num() SymbolNum { _, _, _, num := s.describe() return num } -func (s symbol) byte() []byte { - if s.isNil() { +func (s Symbol) Byte() []byte { + if s.IsNil() { return []byte{0, 0} } return []byte{byte(uint16(s) >> 8), byte(uint16(s) & 0x00ff)} } -func (s symbol) isNil() bool { +func (s Symbol) IsNil() bool { _, _, _, num := s.describe() return num == 0 } -func (s symbol) isStart() bool { - if s.isNil() { +func (s Symbol) IsStart() bool { + if s.IsNil() { return false } _, isStart, _, _ := s.describe() return isStart } -func (s symbol) isEOF() bool { - if s.isNil() { +func (s Symbol) isEOF() bool { + if s.IsNil() { return false } _, _, isEOF, _ := s.describe() return isEOF } -func (s symbol) isNonTerminal() bool { - if s.isNil() { +func (s Symbol) isNonTerminal() bool { + if s.IsNil() { return false } kind, _, _, _ := s.describe() return kind == symbolKindNonTerminal } -func (s symbol) isTerminal() bool { - if s.isNil() { +func (s Symbol) IsTerminal() bool { + if s.IsNil() { return false } return !s.isNonTerminal() } -func (s symbol) describe() (symbolKind, bool, bool, symbolNum) { +func (s Symbol) describe() (symbolKind, bool, bool, SymbolNum) { kind := symbolKindNonTerminal if uint16(s)&maskKindPart > 0 { kind = symbolKindTerminal @@ -149,34 +149,34 @@ func (s symbol) describe() (symbolKind, bool, bool, symbolNum) { isEOF = true } } - num := symbolNum(uint16(s) & maskNumberPart) + num := SymbolNum(uint16(s) & maskNumberPart) return kind, isStart, isEOF, num } -type symbolTable struct { - text2Sym map[string]symbol - sym2Text map[symbol]string +type SymbolTable struct { + text2Sym map[string]Symbol + sym2Text map[Symbol]string nonTermTexts []string termTexts []string - nonTermNum symbolNum - termNum symbolNum + nonTermNum SymbolNum + termNum SymbolNum } -type symbolTableWriter struct { - *symbolTable +type SymbolTableWriter struct { + *SymbolTable } -type symbolTableReader struct { - *symbolTable +type SymbolTableReader struct { + *SymbolTable } -func newSymbolTable() *symbolTable { - return &symbolTable{ - text2Sym: map[string]symbol{ - symbolNameEOF: symbolEOF, +func NewSymbolTable() *SymbolTable { + return &SymbolTable{ + text2Sym: map[string]Symbol{ + symbolNameEOF: SymbolEOF, }, - sym2Text: map[symbol]string{ - symbolEOF: symbolNameEOF, + sym2Text: map[Symbol]string{ + SymbolEOF: symbolNameEOF, }, termTexts: []string{ "", // Nil @@ -191,32 +191,32 @@ func newSymbolTable() *symbolTable { } } -func (t *symbolTable) writer() *symbolTableWriter { - return &symbolTableWriter{ - symbolTable: t, +func (t *SymbolTable) Writer() *SymbolTableWriter { + return &SymbolTableWriter{ + SymbolTable: t, } } -func (t *symbolTable) reader() *symbolTableReader { - return &symbolTableReader{ - symbolTable: t, +func (t *SymbolTable) Reader() *SymbolTableReader { + return &SymbolTableReader{ + SymbolTable: t, } } -func (w *symbolTableWriter) registerStartSymbol(text string) (symbol, error) { +func (w *SymbolTableWriter) RegisterStartSymbol(text string) (Symbol, error) { w.text2Sym[text] = symbolStart w.sym2Text[symbolStart] = text - w.nonTermTexts[symbolStart.num().Int()] = text + w.nonTermTexts[symbolStart.Num().Int()] = text return symbolStart, nil } -func (w *symbolTableWriter) registerNonTerminalSymbol(text string) (symbol, error) { +func (w *SymbolTableWriter) RegisterNonTerminalSymbol(text string) (Symbol, error) { if sym, ok := w.text2Sym[text]; ok { return sym, nil } sym, err := newSymbol(symbolKindNonTerminal, false, w.nonTermNum) if err != nil { - return symbolNil, err + return SymbolNil, err } w.nonTermNum++ w.text2Sym[text] = sym @@ -225,13 +225,13 @@ func (w *symbolTableWriter) registerNonTerminalSymbol(text string) (symbol, erro return sym, nil } -func (w *symbolTableWriter) registerTerminalSymbol(text string) (symbol, error) { +func (w *SymbolTableWriter) RegisterTerminalSymbol(text string) (Symbol, error) { if sym, ok := w.text2Sym[text]; ok { return sym, nil } sym, err := newSymbol(symbolKindTerminal, false, w.termNum) if err != nil { - return symbolNil, err + return SymbolNil, err } w.termNum++ w.text2Sym[text] = sym @@ -240,22 +240,22 @@ func (w *symbolTableWriter) registerTerminalSymbol(text string) (symbol, error) return sym, nil } -func (r *symbolTableReader) toSymbol(text string) (symbol, bool) { +func (r *SymbolTableReader) ToSymbol(text string) (Symbol, bool) { if sym, ok := r.text2Sym[text]; ok { return sym, true } - return symbolNil, false + return SymbolNil, false } -func (r *symbolTableReader) toText(sym symbol) (string, bool) { +func (r *SymbolTableReader) ToText(sym Symbol) (string, bool) { text, ok := r.sym2Text[sym] return text, ok } -func (r *symbolTableReader) terminalSymbols() []symbol { - syms := make([]symbol, 0, r.termNum.Int()-terminalNumMin.Int()) +func (r *SymbolTableReader) TerminalSymbols() []Symbol { + syms := make([]Symbol, 0, r.termNum.Int()-terminalNumMin.Int()) for sym := range r.sym2Text { - if !sym.isTerminal() || sym.isNil() { + if !sym.IsTerminal() || sym.IsNil() { continue } syms = append(syms, sym) @@ -266,17 +266,17 @@ func (r *symbolTableReader) terminalSymbols() []symbol { return syms } -func (r *symbolTableReader) terminalTexts() ([]string, error) { +func (r *SymbolTableReader) TerminalTexts() ([]string, error) { if r.termNum == terminalNumMin { return nil, fmt.Errorf("symbol table has no terminals") } return r.termTexts, nil } -func (r *symbolTableReader) nonTerminalSymbols() []symbol { - syms := make([]symbol, 0, r.nonTermNum.Int()-nonTerminalNumMin.Int()) +func (r *SymbolTableReader) NonTerminalSymbols() []Symbol { + syms := make([]Symbol, 0, r.nonTermNum.Int()-nonTerminalNumMin.Int()) for sym := range r.sym2Text { - if !sym.isNonTerminal() || sym.isNil() { + if !sym.isNonTerminal() || sym.IsNil() { continue } syms = append(syms, sym) @@ -287,8 +287,8 @@ func (r *symbolTableReader) nonTerminalSymbols() []symbol { return syms } -func (r *symbolTableReader) nonTerminalTexts() ([]string, error) { - if r.nonTermNum == nonTerminalNumMin || r.nonTermTexts[symbolStart.num().Int()] == "" { +func (r *SymbolTableReader) NonTerminalTexts() ([]string, error) { + if r.nonTermNum == nonTerminalNumMin || r.nonTermTexts[symbolStart.Num().Int()] == "" { return nil, fmt.Errorf("symbol table has no terminals or no start symbol") } return r.nonTermTexts, nil diff --git a/grammar/symbol_test.go b/grammar/symbol/symbol_test.go index b9bcbdf..31c3edd 100644 --- a/grammar/symbol_test.go +++ b/grammar/symbol/symbol_test.go @@ -1,19 +1,19 @@ -package grammar +package symbol import "testing" func TestSymbol(t *testing.T) { - tab := newSymbolTable() - w := tab.writer() - _, _ = w.registerStartSymbol("expr'") - _, _ = w.registerNonTerminalSymbol("expr") - _, _ = w.registerNonTerminalSymbol("term") - _, _ = w.registerNonTerminalSymbol("factor") - _, _ = w.registerTerminalSymbol("id") - _, _ = w.registerTerminalSymbol("add") - _, _ = w.registerTerminalSymbol("mul") - _, _ = w.registerTerminalSymbol("l_paren") - _, _ = w.registerTerminalSymbol("r_paren") + tab := NewSymbolTable() + w := tab.Writer() + _, _ = w.RegisterStartSymbol("expr'") + _, _ = w.RegisterNonTerminalSymbol("expr") + _, _ = w.RegisterNonTerminalSymbol("term") + _, _ = w.RegisterNonTerminalSymbol("factor") + _, _ = w.RegisterTerminalSymbol("id") + _, _ = w.RegisterTerminalSymbol("add") + _, _ = w.RegisterTerminalSymbol("mul") + _, _ = w.RegisterTerminalSymbol("l_paren") + _, _ = w.RegisterTerminalSymbol("r_paren") nonTermTexts := []string{ "", // Nil @@ -81,13 +81,13 @@ func TestSymbol(t *testing.T) { } for _, tt := range tests { t.Run(tt.text, func(t *testing.T) { - r := tab.reader() - sym, ok := r.toSymbol(tt.text) + r := tab.Reader() + sym, ok := r.ToSymbol(tt.text) if !ok { t.Fatalf("symbol was not found") } testSymbolProperty(t, sym, tt.isNil, tt.isStart, tt.isEOF, tt.isNonTerminal, tt.isTerminal) - text, ok := r.toText(sym) + text, ok := r.ToText(sym) if !ok { t.Fatalf("text was not found") } @@ -98,16 +98,16 @@ func TestSymbol(t *testing.T) { } t.Run("EOF", func(t *testing.T) { - testSymbolProperty(t, symbolEOF, false, false, true, false, true) + testSymbolProperty(t, SymbolEOF, false, false, true, false, true) }) t.Run("Nil", func(t *testing.T) { - testSymbolProperty(t, symbolNil, true, false, false, false, false) + testSymbolProperty(t, SymbolNil, true, false, false, false, false) }) t.Run("texts of non-terminals", func(t *testing.T) { - r := tab.reader() - ts, err := r.nonTerminalTexts() + r := tab.Reader() + ts, err := r.NonTerminalTexts() if err != nil { t.Fatal(err) } @@ -122,8 +122,8 @@ func TestSymbol(t *testing.T) { }) t.Run("texts of terminals", func(t *testing.T) { - r := tab.reader() - ts, err := r.terminalTexts() + r := tab.Reader() + ts, err := r.TerminalTexts() if err != nil { t.Fatal(err) } @@ -138,13 +138,13 @@ func TestSymbol(t *testing.T) { }) } -func testSymbolProperty(t *testing.T, sym symbol, isNil, isStart, isEOF, isNonTerminal, isTerminal bool) { +func testSymbolProperty(t *testing.T, sym Symbol, isNil, isStart, isEOF, isNonTerminal, isTerminal bool) { t.Helper() - if v := sym.isNil(); v != isNil { + if v := sym.IsNil(); v != isNil { t.Fatalf("isNil property is mismatched; want: %v, got: %v", isNil, v) } - if v := sym.isStart(); v != isStart { + if v := sym.IsStart(); v != isStart { t.Fatalf("isStart property is mismatched; want: %v, got: %v", isStart, v) } if v := sym.isEOF(); v != isEOF { @@ -153,7 +153,7 @@ func testSymbolProperty(t *testing.T, sym symbol, isNil, isStart, isEOF, isNonTe if v := sym.isNonTerminal(); v != isNonTerminal { t.Fatalf("isNonTerminal property is mismatched; want: %v, got: %v", isNonTerminal, v) } - if v := sym.isTerminal(); v != isTerminal { + if v := sym.IsTerminal(); v != isTerminal { t.Fatalf("isTerminal property is mismatched; want: %v, got: %v", isTerminal, v) } } diff --git a/grammar/test_helper_test.go b/grammar/test_helper_test.go index 1dcdede..297a9a3 100644 --- a/grammar/test_helper_test.go +++ b/grammar/test_helper_test.go @@ -1,14 +1,18 @@ package grammar -import "testing" +import ( + "testing" -type testSymbolGenerator func(text string) symbol + "github.com/nihei9/vartan/grammar/symbol" +) -func newTestSymbolGenerator(t *testing.T, symTab *symbolTableReader) testSymbolGenerator { - return func(text string) symbol { +type testSymbolGenerator func(text string) symbol.Symbol + +func newTestSymbolGenerator(t *testing.T, symTab *symbol.SymbolTableReader) testSymbolGenerator { + return func(text string) symbol.Symbol { t.Helper() - sym, ok := symTab.toSymbol(text) + sym, ok := symTab.ToSymbol(text) if !ok { t.Fatalf("symbol was not found: %v", text) } @@ -22,7 +26,7 @@ func newTestProductionGenerator(t *testing.T, genSym testSymbolGenerator) testPr return func(lhs string, rhs ...string) *production { t.Helper() - rhsSym := []symbol{} + rhsSym := []symbol.Symbol{} for _, text := range rhs { rhsSym = append(rhsSym, genSym(text)) } @@ -51,9 +55,9 @@ func newTestLR0ItemGenerator(t *testing.T, genProd testProductionGenerator) test } } -func withLookAhead(item *lrItem, lookAhead ...symbol) *lrItem { +func withLookAhead(item *lrItem, lookAhead ...symbol.Symbol) *lrItem { if item.lookAhead.symbols == nil { - item.lookAhead.symbols = map[symbol]struct{}{} + item.lookAhead.symbols = map[symbol.Symbol]struct{}{} } for _, a := range lookAhead { diff --git a/spec/grammar/grammar.go b/spec/grammar/grammar.go index 995d6aa..bf1ea89 100644 --- a/spec/grammar/grammar.go +++ b/spec/grammar/grammar.go @@ -1,25 +1,141 @@ package grammar -import mlspec "github.com/nihei9/maleeni/spec" +import "strconv" type CompiledGrammar struct { - Name string `json:"name"` - LexicalSpecification *LexicalSpecification `json:"lexical_specification"` - ParsingTable *ParsingTable `json:"parsing_table"` - ASTAction *ASTAction `json:"ast_action"` + Name string `json:"name"` + Lexical *LexicalSpec `json:"lexical"` + Syntactic *SyntacticSpec `json:"syntactic"` + ASTAction *ASTAction `json:"ast_action"` } -type LexicalSpecification struct { - Lexer string `json:"lexer"` - Maleeni *Maleeni `json:"maleeni"` +// StateID represents an ID of a state of a transition table. +type StateID int + +const ( + // StateIDNil represents an empty entry of a transition table. + // When the driver reads this value, it raises an error meaning lexical analysis failed. + StateIDNil = StateID(0) + + // StateIDMin is the minimum value of the state ID. All valid state IDs are represented as + // sequential numbers starting from this value. + StateIDMin = StateID(1) +) + +func (id StateID) Int() int { + return int(id) +} + +// LexModeID represents an ID of a lex mode. +type LexModeID int + +const ( + LexModeIDNil = LexModeID(0) + LexModeIDDefault = LexModeID(1) +) + +func (n LexModeID) String() string { + return strconv.Itoa(int(n)) +} + +func (n LexModeID) Int() int { + return int(n) +} + +func (n LexModeID) IsNil() bool { + return n == LexModeIDNil +} + +// LexModeName represents a name of a lex mode. +type LexModeName string + +const ( + LexModeNameNil = LexModeName("") + LexModeNameDefault = LexModeName("default") +) + +func (m LexModeName) String() string { + return string(m) +} + +// LexKindID represents an ID of a lexical kind and is unique across all modes. +type LexKindID int + +const ( + LexKindIDNil = LexKindID(0) + LexKindIDMin = LexKindID(1) +) + +func (id LexKindID) Int() int { + return int(id) +} + +// LexModeKindID represents an ID of a lexical kind and is unique within a mode. +// Use LexKindID to identify a kind across all modes uniquely. +type LexModeKindID int + +const ( + LexModeKindIDNil = LexModeKindID(0) + LexModeKindIDMin = LexModeKindID(1) +) + +func (id LexModeKindID) Int() int { + return int(id) +} + +// LexKindName represents a name of a lexical kind. +type LexKindName string + +const LexKindNameNil = LexKindName("") + +func (k LexKindName) String() string { + return string(k) +} + +type RowDisplacementTable struct { + OriginalRowCount int `json:"original_row_count"` + OriginalColCount int `json:"original_col_count"` + EmptyValue StateID `json:"empty_value"` + Entries []StateID `json:"entries"` + Bounds []int `json:"bounds"` + RowDisplacement []int `json:"row_displacement"` +} + +type UniqueEntriesTable struct { + UniqueEntries *RowDisplacementTable `json:"unique_entries,omitempty"` + UncompressedUniqueEntries []StateID `json:"uncompressed_unique_entries,omitempty"` + RowNums []int `json:"row_nums"` + OriginalRowCount int `json:"original_row_count"` + OriginalColCount int `json:"original_col_count"` + EmptyValue int `json:"empty_value"` +} + +type TransitionTable struct { + InitialStateID StateID `json:"initial_state_id"` + AcceptingStates []LexModeKindID `json:"accepting_states"` + RowCount int `json:"row_count"` + ColCount int `json:"col_count"` + Transition *UniqueEntriesTable `json:"transition,omitempty"` + UncompressedTransition []StateID `json:"uncompressed_transition,omitempty"` +} + +type CompiledLexModeSpec struct { + KindNames []LexKindName `json:"kind_names"` + Push []LexModeID `json:"push"` + Pop []int `json:"pop"` + DFA *TransitionTable `json:"dfa"` } -type Maleeni struct { - Spec *mlspec.CompiledLexSpec `json:"spec"` - KindToTerminal []int `json:"kind_to_terminal"` +type LexicalSpec struct { + InitialModeID LexModeID `json:"initial_mode_id"` + ModeNames []LexModeName `json:"mode_names"` + KindNames []LexKindName `json:"kind_names"` + KindIDs [][]LexKindID `json:"kind_ids"` + CompressionLevel int `json:"compression_level"` + Specs []*CompiledLexModeSpec `json:"specs"` } -type ParsingTable struct { +type SyntacticSpec struct { Action []int `json:"action"` GoTo []int `json:"goto"` StateCount int `json:"state_count"` @@ -30,6 +146,7 @@ type ParsingTable struct { Terminals []string `json:"terminals"` TerminalCount int `json:"terminal_count"` TerminalSkip []int `json:"terminal_skip"` + KindToTerminal []int `json:"kind_to_terminal"` NonTerminals []string `json:"non_terminals"` NonTerminalCount int `json:"non_terminal_count"` EOFSymbol int `json:"eof_symbol"` diff --git a/spec/grammar/parser/clexspec.json b/spec/grammar/parser/clexspec.json new file mode 100644 index 0000000..d0ed3d3 --- /dev/null +++ b/spec/grammar/parser/clexspec.json @@ -0,0 +1 @@ +{"name":"vartan","initial_mode_id":1,"mode_names":["","default","terminal","string_literal"],"kind_names":["","white_space","newline","line_comment","kw_fragment","identifier","terminal_open","string_literal_open","colon","or","semicolon","label_marker","expansion","directive_marker","ordered_symbol_marker","l_paren","r_paren","pattern","escape_symbol","terminal_close","char_seq","string_literal_close"],"kind_ids":[null,[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16],[0,17,18,19],[0,20,21]],"compression_level":2,"specs":[null,{"kind_names":["","white_space","newline","line_comment","kw_fragment","identifier","terminal_open","string_literal_open","colon","or","semicolon","label_marker","expansion","directive_marker","ordered_symbol_marker","l_paren","r_paren"],"push":[0,0,0,0,0,0,2,3,0,0,0,0,0,0,0,0,0],"pop":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"dfa":{"initial_state_id":1,"accepting_states":[0,0,1,2,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5,5,5,5,5,5,5,4,5,0,0,2,6,7,8,9,10,11,12,13,14,15,16],"row_count":47,"col_count":256,"transition":{"unique_entries":{"original_row_count":27,"original_col_count":256,"empty_value":0,"entries":[5,5,5,5,5,5,5,5,5,5,0,5,5,0,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,7,9,9,9,9,9,9,9,9,9,9,9,9,11,13,13,15,18,18,18,21,2,35,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,36,43,44,0,0,37,45,46,0,0,0,0,33,4,32,32,32,32,32,32,32,32,32,32,38,40,0,0,0,0,41,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,0,0,0,0,32,0,32,32,32,32,32,24,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,0,39,0,0,0,0,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,32,32,32,32,32,32,32,32,32,32,0,0,0,0,0,0,0,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,0,0,0,0,32,0,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,25,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,0,0,0,0,0,0,0,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,0,0,0,0,32,0,26,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,0,0,0,0,0,0,0,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,0,0,0,0,32,0,32,32,32,32,32,32,27,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,0,0,0,0,0,0,0,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,0,0,0,0,32,0,32,32,32,32,32,32,32,32,32,32,32,32,28,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,0,0,0,0,0,0,0,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,0,0,0,0,32,0,32,32,32,32,29,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,0,0,0,0,0,0,0,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,0,0,0,0,32,0,32,32,32,32,32,32,32,32,32,32,32,32,32,30,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,0,0,0,0,0,0,0,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,0,0,0,0,32,0,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,31,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,0,0,0,0,0,0,0,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,0,0,0,0,32,0,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,2,0,35,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5,34,42,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"bounds":[5,5,5,5,5,5,5,5,5,5,-1,5,5,-1,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,1,1,-1,-1,1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,-1,1,1,1,-1,-1,1,1,1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,-1,-1,-1,-1,1,-1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,-1,1,-1,-1,-1,-1,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,17,17,17,17,17,17,17,17,17,17,-1,-1,-1,-1,-1,-1,-1,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,-1,-1,-1,-1,17,-1,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,18,18,18,18,18,18,18,18,18,18,-1,-1,-1,-1,-1,-1,-1,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,-1,-1,-1,-1,18,-1,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,19,19,19,19,19,19,19,19,19,19,-1,-1,-1,-1,-1,-1,-1,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,-1,-1,-1,-1,19,-1,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,20,20,20,20,20,20,20,20,20,20,-1,-1,-1,-1,-1,-1,-1,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,-1,-1,-1,-1,20,-1,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,21,21,21,21,21,21,21,21,21,21,-1,-1,-1,-1,-1,-1,-1,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,-1,-1,-1,-1,21,-1,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,22,22,22,22,22,22,22,22,22,22,-1,-1,-1,-1,-1,-1,-1,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,-1,-1,-1,-1,22,-1,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,23,23,23,23,23,23,23,23,23,23,-1,-1,-1,-1,-1,-1,-1,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,-1,-1,-1,-1,23,-1,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,24,24,24,24,24,24,24,24,24,24,-1,-1,-1,-1,-1,-1,-1,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,-1,-1,-1,-1,24,-1,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,2,-1,3,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,2,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,4,25,26,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1],"row_displacement":[0,236,1554,1555,1556,0,237,1323,301,1387,365,1291,429,493,557,1419,621,765,840,915,990,1065,1140,1215,1290,1558,1559]},"row_nums":[0,1,2,3,4,5,6,7,6,8,6,9,6,10,6,11,12,6,13,14,6,15,16,6,17,18,19,20,21,22,23,24,24,25,26,0,0,0,0,0,0,0,0,0,0,0,0],"original_row_count":47,"original_col_count":256,"empty_value":0}}},{"kind_names":["","pattern","escape_symbol","terminal_close"],"push":[0,0,0,0],"pop":[0,0,0,1],"dfa":{"initial_state_id":1,"accepting_states":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3],"row_count":78,"col_count":256,"transition":{"unique_entries":{"original_row_count":46,"original_col_count":256,"empty_value":0,"entries":[39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,77,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,20,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,3,5,5,5,5,5,5,5,5,5,5,5,5,7,9,9,11,14,14,14,17,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,22,24,24,24,24,24,24,24,24,24,24,24,24,26,28,28,30,33,33,33,36,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,60,62,62,62,62,62,62,62,62,62,62,62,62,64,66,66,68,71,71,71,74,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,0,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,58,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,0,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,0,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,41,43,43,43,43,43,43,43,43,43,43,43,43,45,47,47,49,52,52,52,55,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,54,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,76,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,0,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,0,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,0,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,56,56,56,56,56,56,56,56,56,56,56,56,56,56,56,56,75,75,75,75,75,75,75,75,75,75,75,75,75,75,75,75,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"bounds":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,-1,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,-1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,-1,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,41,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,43,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,-1,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,-1,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,-1,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,33,33,33,33,33,33,33,33,33,33,33,33,33,33,33,33,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1],"row_displacement":[0,0,736,2548,852,2612,916,2372,980,1044,1108,2839,1172,245,2613,1236,2677,1300,2420,1364,1428,1492,2855,1556,735,2678,1620,2742,1684,2468,1748,1812,1876,2871,1940,490,2743,2004,2807,2068,2516,2132,2196,2260,2887,2324]},"row_nums":[0,1,2,3,2,4,2,5,2,6,2,7,8,2,9,10,2,11,12,2,13,2,14,2,15,2,16,2,17,2,18,19,2,20,21,2,22,23,2,24,2,25,2,26,2,27,2,28,2,29,30,2,31,32,2,33,34,2,35,2,36,2,37,2,38,2,39,2,40,41,2,42,43,2,44,45,2,0],"original_row_count":78,"original_col_count":256,"empty_value":0}}},{"kind_names":["","char_seq","string_literal_close"],"push":[0,0,0],"pop":[0,0,1],"dfa":{"initial_state_id":1,"accepting_states":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2],"row_count":40,"col_count":256,"transition":{"unique_entries":{"original_row_count":24,"original_col_count":256,"empty_value":0,"entries":[20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,39,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,3,5,5,5,5,5,5,5,5,5,5,5,5,7,9,9,11,14,14,14,17,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,0,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,0,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,0,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,22,24,24,24,24,24,24,24,24,24,24,24,24,26,28,28,30,33,33,33,36,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,0,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"bounds":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,-1,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,-1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,-1,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,-1,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1],"row_displacement":[0,0,246,1194,362,1258,426,1114,490,554,618,1355,682,245,1259,746,1323,810,1162,874,938,1002,1371,1066]},"row_nums":[0,1,2,3,2,4,2,5,2,6,2,7,8,2,9,10,2,11,12,2,13,2,14,2,15,2,16,2,17,2,18,19,2,20,21,2,22,23,2,0],"original_row_count":40,"original_col_count":256,"empty_value":0}}}]} diff --git a/spec/grammar/lexer.go b/spec/grammar/parser/lexer.go index 4bb5353..1096ae9 100644 --- a/spec/grammar/lexer.go +++ b/spec/grammar/parser/lexer.go @@ -1,7 +1,7 @@ //go:generate maleeni compile lexspec.json -o clexspec.json -//go:generate maleeni-go clexspec.json --package grammar +//go:generate maleeni-go clexspec.json --package parser -package grammar +package parser import ( _ "embed" diff --git a/spec/grammar/lexer_test.go b/spec/grammar/parser/lexer_test.go index 20dfccc..64c2aa2 100644 --- a/spec/grammar/lexer_test.go +++ b/spec/grammar/parser/lexer_test.go @@ -1,4 +1,4 @@ -package grammar +package parser import ( "strings" diff --git a/spec/grammar/lexspec.json b/spec/grammar/parser/lexspec.json index caf1f0e..caf1f0e 100644 --- a/spec/grammar/lexspec.json +++ b/spec/grammar/parser/lexspec.json diff --git a/spec/grammar/parser.go b/spec/grammar/parser/parser.go index 27a7c7d..c0179a6 100644 --- a/spec/grammar/parser.go +++ b/spec/grammar/parser/parser.go @@ -1,11 +1,11 @@ -package grammar +package parser import ( "fmt" "io" - mlspec "github.com/nihei9/maleeni/spec" verr "github.com/nihei9/vartan/error" + spec "github.com/nihei9/vartan/spec/grammar" ) type RootNode struct { @@ -247,7 +247,7 @@ func (p *parser) parseFragment() *FragmentNode { case p.consume(tokenKindTerminalPattern): rhs = p.lastTok.text case p.consume(tokenKindStringLiteral): - rhs = mlspec.EscapePattern(p.lastTok.text) + rhs = spec.EscapePattern(p.lastTok.text) default: raiseSyntaxError(p.pos.Row, synErrFragmentNoPattern) } diff --git a/spec/grammar/parser_test.go b/spec/grammar/parser/parser_test.go index 5a2e07f..498e7c6 100644 --- a/spec/grammar/parser_test.go +++ b/spec/grammar/parser/parser_test.go @@ -1,4 +1,4 @@ -package grammar +package parser import ( "strings" diff --git a/spec/grammar/syntax_error.go b/spec/grammar/parser/syntax_error.go index 1f9664b..719fb94 100644 --- a/spec/grammar/syntax_error.go +++ b/spec/grammar/parser/syntax_error.go @@ -1,4 +1,4 @@ -package grammar +package parser type SyntaxError struct { message string diff --git a/spec/grammar/vartan_lexer.go b/spec/grammar/parser/vartan_lexer.go index cc80554..76ddfde 100644 --- a/spec/grammar/vartan_lexer.go +++ b/spec/grammar/parser/vartan_lexer.go @@ -1,5 +1,5 @@ // Code generated by maleeni-go. DO NOT EDIT. -package grammar +package parser import ( "fmt" diff --git a/spec/grammar/util.go b/spec/grammar/util.go new file mode 100644 index 0000000..bf3f233 --- /dev/null +++ b/spec/grammar/util.go @@ -0,0 +1,21 @@ +package grammar + +import "strings" + +var rep = strings.NewReplacer( + `.`, `\.`, + `*`, `\*`, + `+`, `\+`, + `?`, `\?`, + `|`, `\|`, + `(`, `\(`, + `)`, `\)`, + `[`, `\[`, + `\`, `\\`, +) + +// EscapePattern escapes the special characters. +// For example, EscapePattern(`+`) returns `\+`. +func EscapePattern(s string) string { + return rep.Replace(s) +} diff --git a/spec/test/tree.json b/spec/test/tree.json index 35c939a..f05c2f2 100644 --- a/spec/test/tree.json +++ b/spec/test/tree.json @@ -1 +1 @@ -{"name":"tree","lexical_specification":{"lexer":"maleeni","maleeni":{"spec":{"name":"tree","initial_mode_id":1,"mode_names":["","default","raw_string","interpreted_string"],"kind_names":["","ws","l_paren","r_paren","identifier","raw_string_open","interpreted_string_open","raw_string_body","raw_string_close","interpreted_seq","codepoint_prefix","l_brace","r_brace","hex_digits","escaped_seq","escape_char","interpreted_string_close"],"kind_ids":[null,[0,1,2,3,4,5,6],[0,7,8],[0,9,10,11,12,13,14,15,16]],"compression_level":2,"specs":[null,{"kind_names":["","ws","l_paren","r_paren","identifier","raw_string_open","interpreted_string_open"],"push":[0,0,0,0,0,2,3],"pop":[0,0,0,0,0,0,0],"dfa":{"initial_state_id":1,"accepting_states":[0,0,1,4,2,3,5,6],"row_count":8,"col_count":256,"transition":{"unique_entries":{"original_row_count":4,"original_col_count":256,"empty_value":0,"entries":[0,0,0,0,0,0,0,0,0,2,2,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,7,0,0,0,0,6,4,5,0,0,0,0,0,0,3,3,3,3,3,3,3,3,3,3,0,0,0,0,0,0,0,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,0,0,0,0,3,0,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,0,0,0,0,0,0,0,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,0,0,0,0,3,0,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,2,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"bounds":[-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,-1,-1,1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,-1,1,-1,-1,-1,-1,1,1,1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,-1,-1,-1,-1,1,-1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,3,3,3,3,3,3,3,3,3,3,-1,-1,-1,-1,-1,-1,-1,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,-1,-1,-1,-1,3,-1,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,2,-1,-1,2,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,2,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1],"row_displacement":[0,0,189,75]},"row_nums":[0,1,2,3,0,0,0,0],"original_row_count":8,"original_col_count":256,"empty_value":0}}},{"kind_names":["","raw_string_body","raw_string_close"],"push":[0,0,0],"pop":[0,0,1],"dfa":{"initial_state_id":1,"accepting_states":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2],"row_count":40,"col_count":256,"transition":{"unique_entries":{"original_row_count":24,"original_col_count":256,"empty_value":0,"entries":[20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,39,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,3,5,5,5,5,5,5,5,5,5,5,5,5,7,9,9,11,14,14,14,17,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,0,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,0,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,0,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,22,24,24,24,24,24,24,24,24,24,24,24,24,26,28,28,30,33,33,33,36,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,0,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"bounds":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,-1,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,-1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,-1,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,-1,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1],"row_displacement":[0,0,246,1194,362,1258,426,1114,490,554,618,1355,682,245,1259,746,1323,810,1162,874,938,1002,1371,1066]},"row_nums":[0,1,2,3,2,4,2,5,2,6,2,7,8,2,9,10,2,11,12,2,13,2,14,2,15,2,16,2,17,2,18,19,2,20,21,2,22,23,2,0],"original_row_count":40,"original_col_count":256,"empty_value":0}}},{"kind_names":["","interpreted_seq","codepoint_prefix","l_brace","r_brace","hex_digits","escaped_seq","escape_char","interpreted_string_close"],"push":[0,0,0,0,0,0,0,0,0],"pop":[0,0,0,0,0,0,0,0,1],"dfa":{"initial_state_id":1,"accepting_states":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,7,5,2,3,4,6,8],"row_count":46,"col_count":256,"transition":{"unique_entries":{"original_row_count":26,"original_col_count":256,"empty_value":0,"entries":[20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,45,20,20,20,20,20,20,20,20,20,20,20,20,20,40,40,40,40,40,40,40,40,40,40,20,20,20,20,20,20,20,40,40,40,40,40,40,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,39,20,20,20,20,40,40,40,40,40,40,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,42,20,43,20,20,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,3,5,5,5,5,5,5,5,5,5,5,5,5,7,9,9,11,14,14,14,17,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,0,20,20,20,20,20,20,20,20,20,20,20,20,20,0,0,0,0,0,0,0,0,0,0,20,20,20,20,20,20,20,0,0,0,0,0,0,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,0,20,20,20,20,0,0,0,0,0,0,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,0,20,0,20,20,0,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,0,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,22,24,24,24,24,24,24,24,24,24,24,24,24,26,28,28,30,33,33,33,36,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,0,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,40,40,40,40,40,40,40,40,40,40,0,0,0,0,0,0,0,40,40,40,40,40,40,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,40,40,40,40,40,40,44,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,44,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,41,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"bounds":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,-1,13,13,13,13,13,13,13,13,13,13,13,13,13,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,13,13,13,13,13,13,13,-1,-1,-1,-1,-1,-1,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,-1,13,13,13,13,-1,-1,-1,-1,-1,-1,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,-1,13,-1,13,13,-1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,-1,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,-1,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,25,25,25,25,25,25,25,25,25,25,-1,-1,-1,-1,-1,-1,-1,25,25,25,25,25,25,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,25,25,25,25,25,25,24,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,24,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,24,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1],"row_displacement":[0,0,246,1194,362,1258,426,1114,490,554,618,1436,682,245,1259,746,1323,810,1162,874,938,1002,1452,1066,1504,1435]},"row_nums":[0,1,2,3,2,4,2,5,2,6,2,7,8,2,9,10,2,11,12,2,13,2,14,2,15,2,16,2,17,2,18,19,2,20,21,2,22,23,2,24,25,0,0,0,0,0],"original_row_count":46,"original_col_count":256,"empty_value":0}}}]},"kind_to_terminal":[0,3,4,5,6,7,10,8,9,11,12,13,14,15,16,17,18]}},"parsing_table":{"action":[0,0,0,0,-2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-3,0,0,0,-4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,7,0,0,-2,7,0,-11,0,0,-12,0,0,0,0,0,0,0,0,0,4,0,0,4,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,6,6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-2,-14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,8,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,9,0,0,0,0,0,0,0,0,0,0,0,0,0,0,12,0,0,12,12,0,0,-17,12,0,0,0,0,0,0,0,0,0,0,16,-22,0,16,16,0,0,0,0,0,-23,-24,-25,-26,-27,-28,-29,16,0,0,0,0,5,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,0,0,3,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-30,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,11,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-31,0,0,0,0,0,0,0,0,0,0,0,-23,-24,-25,-26,-27,-28,-29,15,0,18,0,0,18,18,0,0,0,0,0,18,18,18,18,18,18,18,18,0,25,0,0,25,25,0,0,0,0,0,25,25,25,25,25,25,25,25,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-33,0,19,0,0,19,19,0,0,0,0,0,19,19,19,19,19,19,19,19,0,0,0,0,0,0,0,0,0,0,0,0,0,-34,0,0,0,0,0,0,20,0,0,20,20,0,0,0,0,0,20,20,20,20,20,20,20,20,0,21,0,0,21,21,0,0,0,0,0,21,21,21,21,21,21,21,21,0,22,0,0,22,22,0,0,0,0,0,22,22,22,22,22,22,22,22,0,23,0,0,23,23,0,0,0,0,0,23,23,23,23,23,23,23,23,0,24,0,0,24,24,0,0,0,0,0,24,24,24,24,24,24,24,24,0,10,0,0,10,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0,13,0,0,13,13,0,0,0,0,0,0,0,0,0,0,0,0,0,0,17,0,0,17,17,0,0,0,0,0,17,17,17,17,17,17,17,17,0,14,0,0,14,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-35,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-36,0,0,0,0,0,26,0,0,26,26,0,0,0,0,0,26,26,26,26,26,26,26,26],"goto":[0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,6,7,8,9,0,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,13,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,16,0,0,0,0,0,0,0,0,0,0,0,0,0,18,19,20,21,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,32,21,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"state_count":37,"initial_state":0,"start_production":1,"lhs_symbols":[0,1,2,2,2,3,3,3,4,4,5,6,6,7,7,8,8,9,9,10,10,10,10,10,10,10,11],"alternative_symbol_counts":[0,1,4,4,3,2,1,0,1,1,3,1,0,3,3,1,0,2,1,1,1,1,1,1,1,1,4],"terminals":["","\u003ceof\u003e","error","ws","l_paren","r_paren","identifier","raw_string_open","raw_string_body","raw_string_close","interpreted_string_open","interpreted_seq","codepoint_prefix","l_brace","r_brace","hex_digits","escaped_seq","escape_char","interpreted_string_close"],"terminal_count":19,"terminal_skip":[0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"non_terminals":["","tree'","tree","tree_list","string","raw_string","opt_raw_string_body","interpreted_string","opt_interpreted_string_body","interpreted_string_body","interpreted_string_elem","codepoint_expr"],"non_terminal_count":12,"eof_symbol":1,"error_symbol":2,"error_trapper_states":[0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"recover_productions":[0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0]},"ast_action":{"entries":[null,null,[2,-3],[2,3],[2],[-1,2],null,null,null,null,[-2],null,null,[-2],[2],[-1],null,[-1,-2],[-1],null,null,null,null,null,null,null,[3]]}} +{"name":"tree","lexical":{"initial_mode_id":1,"mode_names":["","default","raw_string","interpreted_string"],"kind_names":["","ws","l_paren","r_paren","identifier","raw_string_open","interpreted_string_open","raw_string_body","raw_string_close","interpreted_seq","codepoint_prefix","l_brace","r_brace","hex_digits","escaped_seq","escape_char","interpreted_string_close"],"kind_ids":[null,[0,1,2,3,4,5,6],[0,7,8],[0,9,10,11,12,13,14,15,16]],"compression_level":2,"specs":[null,{"kind_names":["","ws","l_paren","r_paren","identifier","raw_string_open","interpreted_string_open"],"push":[0,0,0,0,0,2,3],"pop":[0,0,0,0,0,0,0],"dfa":{"initial_state_id":1,"accepting_states":[0,0,1,4,2,3,5,6],"row_count":8,"col_count":256,"transition":{"unique_entries":{"original_row_count":4,"original_col_count":256,"empty_value":0,"entries":[0,0,0,0,0,0,0,0,0,2,2,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,7,0,0,0,0,6,4,5,0,0,0,0,0,0,3,3,3,3,3,3,3,3,3,3,0,0,0,0,0,0,0,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,0,0,0,0,3,0,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,0,0,0,0,0,0,0,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,0,0,0,0,3,0,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,2,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"bounds":[-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,-1,-1,1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,-1,1,-1,-1,-1,-1,1,1,1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,-1,-1,-1,-1,1,-1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,3,3,3,3,3,3,3,3,3,3,-1,-1,-1,-1,-1,-1,-1,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,-1,-1,-1,-1,3,-1,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,2,-1,-1,2,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,2,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1],"row_displacement":[0,0,189,75]},"row_nums":[0,1,2,3,0,0,0,0],"original_row_count":8,"original_col_count":256,"empty_value":0}}},{"kind_names":["","raw_string_body","raw_string_close"],"push":[0,0,0],"pop":[0,0,1],"dfa":{"initial_state_id":1,"accepting_states":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2],"row_count":40,"col_count":256,"transition":{"unique_entries":{"original_row_count":24,"original_col_count":256,"empty_value":0,"entries":[20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,39,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,3,5,5,5,5,5,5,5,5,5,5,5,5,7,9,9,11,14,14,14,17,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,0,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,0,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,0,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,22,24,24,24,24,24,24,24,24,24,24,24,24,26,28,28,30,33,33,33,36,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,0,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"bounds":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,-1,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,-1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,-1,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,-1,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1],"row_displacement":[0,0,246,1194,362,1258,426,1114,490,554,618,1355,682,245,1259,746,1323,810,1162,874,938,1002,1371,1066]},"row_nums":[0,1,2,3,2,4,2,5,2,6,2,7,8,2,9,10,2,11,12,2,13,2,14,2,15,2,16,2,17,2,18,19,2,20,21,2,22,23,2,0],"original_row_count":40,"original_col_count":256,"empty_value":0}}},{"kind_names":["","interpreted_seq","codepoint_prefix","l_brace","r_brace","hex_digits","escaped_seq","escape_char","interpreted_string_close"],"push":[0,0,0,0,0,0,0,0,0],"pop":[0,0,0,0,0,0,0,0,1],"dfa":{"initial_state_id":1,"accepting_states":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,7,5,2,3,4,6,8],"row_count":46,"col_count":256,"transition":{"unique_entries":{"original_row_count":26,"original_col_count":256,"empty_value":0,"entries":[20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,45,20,20,20,20,20,20,20,20,20,20,20,20,20,40,40,40,40,40,40,40,40,40,40,20,20,20,20,20,20,20,40,40,40,40,40,40,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,39,20,20,20,20,40,40,40,40,40,40,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,42,20,43,20,20,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,3,5,5,5,5,5,5,5,5,5,5,5,5,7,9,9,11,14,14,14,17,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,0,20,20,20,20,20,20,20,20,20,20,20,20,20,0,0,0,0,0,0,0,0,0,0,20,20,20,20,20,20,20,0,0,0,0,0,0,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,0,20,20,20,20,0,0,0,0,0,0,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,0,20,0,20,20,0,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,0,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,22,24,24,24,24,24,24,24,24,24,24,24,24,26,28,28,30,33,33,33,36,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,0,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,40,40,40,40,40,40,40,40,40,40,0,0,0,0,0,0,0,40,40,40,40,40,40,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,40,40,40,40,40,40,44,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,44,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,41,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"bounds":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,-1,13,13,13,13,13,13,13,13,13,13,13,13,13,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,13,13,13,13,13,13,13,-1,-1,-1,-1,-1,-1,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,-1,13,13,13,13,-1,-1,-1,-1,-1,-1,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,-1,13,-1,13,13,-1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,-1,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,-1,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,25,25,25,25,25,25,25,25,25,25,-1,-1,-1,-1,-1,-1,-1,25,25,25,25,25,25,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,25,25,25,25,25,25,24,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,24,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,24,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1],"row_displacement":[0,0,246,1194,362,1258,426,1114,490,554,618,1436,682,245,1259,746,1323,810,1162,874,938,1002,1452,1066,1504,1435]},"row_nums":[0,1,2,3,2,4,2,5,2,6,2,7,8,2,9,10,2,11,12,2,13,2,14,2,15,2,16,2,17,2,18,19,2,20,21,2,22,23,2,24,25,0,0,0,0,0],"original_row_count":46,"original_col_count":256,"empty_value":0}}}]},"syntactic":{"action":[0,0,0,0,-2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-3,0,0,0,-4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,7,0,0,-2,7,0,-11,0,0,-12,0,0,0,0,0,0,0,0,0,4,0,0,4,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,6,6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-2,-14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,8,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,9,0,0,0,0,0,0,0,0,0,0,0,0,0,0,12,0,0,12,12,0,0,-17,12,0,0,0,0,0,0,0,0,0,0,16,-22,0,16,16,0,0,0,0,0,-23,-24,-25,-26,-27,-28,-29,16,0,0,0,0,5,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,0,0,3,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-30,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,11,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-31,0,0,0,0,0,0,0,0,0,0,0,-23,-24,-25,-26,-27,-28,-29,15,0,18,0,0,18,18,0,0,0,0,0,18,18,18,18,18,18,18,18,0,25,0,0,25,25,0,0,0,0,0,25,25,25,25,25,25,25,25,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-33,0,19,0,0,19,19,0,0,0,0,0,19,19,19,19,19,19,19,19,0,0,0,0,0,0,0,0,0,0,0,0,0,-34,0,0,0,0,0,0,20,0,0,20,20,0,0,0,0,0,20,20,20,20,20,20,20,20,0,21,0,0,21,21,0,0,0,0,0,21,21,21,21,21,21,21,21,0,22,0,0,22,22,0,0,0,0,0,22,22,22,22,22,22,22,22,0,23,0,0,23,23,0,0,0,0,0,23,23,23,23,23,23,23,23,0,24,0,0,24,24,0,0,0,0,0,24,24,24,24,24,24,24,24,0,10,0,0,10,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0,13,0,0,13,13,0,0,0,0,0,0,0,0,0,0,0,0,0,0,17,0,0,17,17,0,0,0,0,0,17,17,17,17,17,17,17,17,0,14,0,0,14,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-35,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-36,0,0,0,0,0,26,0,0,26,26,0,0,0,0,0,26,26,26,26,26,26,26,26],"goto":[0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,6,7,8,9,0,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,13,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,16,0,0,0,0,0,0,0,0,0,0,0,0,0,18,19,20,21,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,32,21,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"state_count":37,"initial_state":0,"start_production":1,"lhs_symbols":[0,1,2,2,2,3,3,3,4,4,5,6,6,7,7,8,8,9,9,10,10,10,10,10,10,10,11],"alternative_symbol_counts":[0,1,4,4,3,2,1,0,1,1,3,1,0,3,3,1,0,2,1,1,1,1,1,1,1,1,4],"terminals":["","\u003ceof\u003e","error","ws","l_paren","r_paren","identifier","raw_string_open","raw_string_body","raw_string_close","interpreted_string_open","interpreted_seq","codepoint_prefix","l_brace","r_brace","hex_digits","escaped_seq","escape_char","interpreted_string_close"],"terminal_count":19,"terminal_skip":[0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"kind_to_terminal":[0,3,4,5,6,7,10,8,9,11,12,13,14,15,16,17,18],"non_terminals":["","tree'","tree","tree_list","string","raw_string","opt_raw_string_body","interpreted_string","opt_interpreted_string_body","interpreted_string_body","interpreted_string_elem","codepoint_expr"],"non_terminal_count":12,"eof_symbol":1,"error_symbol":2,"error_trapper_states":[0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"recover_productions":[0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0]},"ast_action":{"entries":[null,null,[2,-3],[2,3],[2],[-1,2],null,null,null,null,[-2],null,null,[-2],[2],[-1],null,[-1,-2],[-1],null,null,null,null,null,null,null,[3]]}} diff --git a/spec/test/tree_lexer.go b/spec/test/tree_lexer.go index 18032f7..8bb1c87 100644 --- a/spec/test/tree_lexer.go +++ b/spec/test/tree_lexer.go @@ -1,10 +1,9 @@ -// Code generated by maleeni-go. DO NOT EDIT. +// Code generated by vartan-go. DO NOT EDIT. package test import ( "fmt" "io" - "io/ioutil" ) type ModeID int @@ -99,7 +98,7 @@ type Lexer struct { // NewLexer returns a new lexer. func NewLexer(spec LexSpec, src io.Reader, opts ...LexerOption) (*Lexer, error) { - b, err := ioutil.ReadAll(src) + b, err := io.ReadAll(src) if err != nil { return nil, err } diff --git a/tester/tester.go b/tester/tester.go index 187e36c..3e6941f 100644 --- a/tester/tester.go +++ b/tester/tester.go @@ -8,7 +8,7 @@ import ( "runtime/debug" "strings" - "github.com/nihei9/vartan/driver" + driver "github.com/nihei9/vartan/driver/parser" gspec "github.com/nihei9/vartan/spec/grammar" tspec "github.com/nihei9/vartan/spec/test" ) diff --git a/tester/tester_test.go b/tester/tester_test.go index fb007ca..1ea9e31 100644 --- a/tester/tester_test.go +++ b/tester/tester_test.go @@ -6,7 +6,7 @@ import ( "testing" "github.com/nihei9/vartan/grammar" - gspec "github.com/nihei9/vartan/spec/grammar" + "github.com/nihei9/vartan/spec/grammar/parser" tspec "github.com/nihei9/vartan/spec/test" ) @@ -123,18 +123,14 @@ foo foo foo } for i, tt := range tests { t.Run(fmt.Sprintf("#%v", i), func(t *testing.T) { - ast, err := gspec.Parse(strings.NewReader(tt.grammarSrc)) + ast, err := parser.Parse(strings.NewReader(tt.grammarSrc)) if err != nil { t.Fatal(err) } b := grammar.GrammarBuilder{ AST: ast, } - g, err := b.Build() - if err != nil { - t.Fatal(err) - } - cg, _, err := grammar.Compile(g) + cg, _, err := b.Build() if err != nil { t.Fatal(err) } diff --git a/ucd/api.go b/ucd/api.go new file mode 100644 index 0000000..8265d54 --- /dev/null +++ b/ucd/api.go @@ -0,0 +1,180 @@ +//go:generate go run ../cmd/ucdgen/main.go +//go:generate go fmt codepoint.go + +package ucd + +import ( + "fmt" + "strings" +) + +const ( + // https://www.unicode.org/versions/Unicode13.0.0/ch03.pdf + // 3.4 Characters and Encoding + // > D9 Unicode codespace: A range of integers from 0 to 10FFFF16. + codePointMin = 0x0 + codePointMax = 0x10FFFF +) + +func NormalizeCharacterProperty(propName, propVal string) (string, error) { + if propName == "" { + propName = "gc" + } + + name, ok := propertyNameAbbs[normalizeSymbolicValue(propName)] + if !ok { + return "", fmt.Errorf("unsupported character property name: %v", propName) + } + props, ok := derivedCoreProperties[name] + if !ok { + return "", nil + } + var b strings.Builder + yes, ok := binaryValues[normalizeSymbolicValue(propVal)] + if !ok { + return "", fmt.Errorf("unsupported character property value: %v", propVal) + } + if yes { + fmt.Fprint(&b, "[") + } else { + fmt.Fprint(&b, "[^") + } + for _, prop := range props { + fmt.Fprint(&b, prop) + } + fmt.Fprint(&b, "]") + + return b.String(), nil +} + +func IsContributoryProperty(propName string) bool { + if propName == "" { + return false + } + + for _, p := range contributoryProperties { + if propName == p { + return true + } + } + return false +} + +func FindCodePointRanges(propName, propVal string) ([]*CodePointRange, bool, error) { + if propName == "" { + propName = "gc" + } + + name, ok := propertyNameAbbs[normalizeSymbolicValue(propName)] + if !ok { + return nil, false, fmt.Errorf("unsupported character property name: %v", propName) + } + switch name { + case "gc": + val, ok := generalCategoryValueAbbs[normalizeSymbolicValue(propVal)] + if !ok { + return nil, false, fmt.Errorf("unsupported character property value: %v", propVal) + } + if val == generalCategoryValueAbbs[normalizeSymbolicValue(generalCategoryDefaultValue)] { + var allCPs []*CodePointRange + if generalCategoryDefaultRange.From > codePointMin { + allCPs = append(allCPs, &CodePointRange{ + From: codePointMin, + To: generalCategoryDefaultRange.From - 1, + }) + } + if generalCategoryDefaultRange.To < codePointMax { + allCPs = append(allCPs, &CodePointRange{ + From: generalCategoryDefaultRange.To + 1, + To: codePointMax, + }) + } + for _, cp := range generalCategoryCodePoints { + allCPs = append(allCPs, cp...) + } + return allCPs, true, nil + } + vals, ok := compositGeneralCategories[val] + if !ok { + vals = []string{val} + } + var ranges []*CodePointRange + for _, v := range vals { + rs, ok := generalCategoryCodePoints[v] + if !ok { + return nil, false, fmt.Errorf("invalid value of the General_Category property: %v", v) + } + ranges = append(ranges, rs...) + } + return ranges, false, nil + case "sc": + val, ok := scriptValueAbbs[normalizeSymbolicValue(propVal)] + if !ok { + return nil, false, fmt.Errorf("unsupported character property value: %v", propVal) + } + if val == scriptValueAbbs[normalizeSymbolicValue(scriptDefaultValue)] { + var allCPs []*CodePointRange + if scriptDefaultRange.From > codePointMin { + allCPs = append(allCPs, &CodePointRange{ + From: codePointMin, + To: scriptDefaultRange.From - 1, + }) + } + if scriptDefaultRange.To < codePointMax { + allCPs = append(allCPs, &CodePointRange{ + From: scriptDefaultRange.To + 1, + To: codePointMax, + }) + } + for _, cp := range scriptCodepoints { + allCPs = append(allCPs, cp...) + } + return allCPs, true, nil + } + return scriptCodepoints[val], false, nil + case "oalpha": + yes, ok := binaryValues[normalizeSymbolicValue(propVal)] + if !ok { + return nil, false, fmt.Errorf("unsupported character property value: %v", propVal) + } + if yes { + return otherAlphabeticCodePoints, false, nil + } else { + return otherAlphabeticCodePoints, true, nil + } + case "olower": + yes, ok := binaryValues[normalizeSymbolicValue(propVal)] + if !ok { + return nil, false, fmt.Errorf("unsupported character property value: %v", propVal) + } + if yes { + return otherLowercaseCodePoints, false, nil + } else { + return otherLowercaseCodePoints, true, nil + } + case "oupper": + yes, ok := binaryValues[normalizeSymbolicValue(propVal)] + if !ok { + return nil, false, fmt.Errorf("unsupported character property value: %v", propVal) + } + if yes { + return otherUppercaseCodePoints, false, nil + } else { + return otherUppercaseCodePoints, true, nil + } + case "wspace": + yes, ok := binaryValues[normalizeSymbolicValue(propVal)] + if !ok { + return nil, false, fmt.Errorf("unsupported character property value: %v", propVal) + } + if yes { + return whiteSpaceCodePoints, false, nil + } else { + return whiteSpaceCodePoints, true, nil + } + } + + // If the process reaches this code, it's a bug. We must handle all of the properties registered with + // the `propertyNameAbbs`. + return nil, false, fmt.Errorf("character property '%v' is unavailable", propName) +} diff --git a/ucd/codepoint.go b/ucd/codepoint.go new file mode 100644 index 0000000..e9b411e --- /dev/null +++ b/ucd/codepoint.go @@ -0,0 +1,6552 @@ +// Code generated by generator/main.go; DO NOT EDIT. + +package ucd + +// https://www.unicode.org/Public/13.0.0/ucd/PropertyValueAliases.txt +var generalCategoryValueAbbs = map[string]string{ + "c": "c", + "casedletter": "lc", + "cc": "cc", + "cf": "cf", + "closepunctuation": "pe", + "cn": "cn", + "cntrl": "cc", + "co": "co", + "combiningmark": "m", + "connectorpunctuation": "pc", + "control": "cc", + "cs": "cs", + "currencysymbol": "sc", + "dashpunctuation": "pd", + "decimalnumber": "nd", + "digit": "nd", + "enclosingmark": "me", + "finalpunctuation": "pf", + "format": "cf", + "initialpunctuation": "pi", + "l": "l", + "lc": "lc", + "letter": "l", + "letternumber": "nl", + "lineseparator": "zl", + "ll": "ll", + "lm": "lm", + "lo": "lo", + "lowercaseletter": "ll", + "lt": "lt", + "lu": "lu", + "m": "m", + "mark": "m", + "mathsymbol": "sm", + "mc": "mc", + "me": "me", + "mn": "mn", + "modifierletter": "lm", + "modifiersymbol": "sk", + "n": "n", + "nd": "nd", + "nl": "nl", + "no": "no", + "nonspacingmark": "mn", + "number": "n", + "openpunctuation": "ps", + "other": "c", + "otherletter": "lo", + "othernumber": "no", + "otherpunctuation": "po", + "othersymbol": "so", + "p": "p", + "paragraphseparator": "zp", + "pc": "pc", + "pd": "pd", + "pe": "pe", + "pf": "pf", + "pi": "pi", + "po": "po", + "privateuse": "co", + "ps": "ps", + "punct": "p", + "punctuation": "p", + "s": "s", + "sc": "sc", + "separator": "z", + "sk": "sk", + "sm": "sm", + "so": "so", + "spaceseparator": "zs", + "spacingmark": "mc", + "surrogate": "cs", + "symbol": "s", + "titlecaseletter": "lt", + "unassigned": "cn", + "uppercaseletter": "lu", + "z": "z", + "zl": "zl", + "zp": "zp", + "zs": "zs", +} + +// https://www.unicode.org/Public/13.0.0/ucd/PropertyValueAliases.txt +var scriptValueAbbs = map[string]string{ + "adlam": "adlm", + "adlm": "adlm", + "aghb": "aghb", + "ahom": "ahom", + "anatolianhieroglyphs": "hluw", + "arab": "arab", + "arabic": "arab", + "armenian": "armn", + "armi": "armi", + "armn": "armn", + "avestan": "avst", + "avst": "avst", + "bali": "bali", + "balinese": "bali", + "bamu": "bamu", + "bamum": "bamu", + "bass": "bass", + "bassavah": "bass", + "batak": "batk", + "batk": "batk", + "beng": "beng", + "bengali": "beng", + "bhaiksuki": "bhks", + "bhks": "bhks", + "bopo": "bopo", + "bopomofo": "bopo", + "brah": "brah", + "brahmi": "brah", + "brai": "brai", + "braille": "brai", + "bugi": "bugi", + "buginese": "bugi", + "buhd": "buhd", + "buhid": "buhd", + "cakm": "cakm", + "canadianaboriginal": "cans", + "cans": "cans", + "cari": "cari", + "carian": "cari", + "caucasianalbanian": "aghb", + "chakma": "cakm", + "cham": "cham", + "cher": "cher", + "cherokee": "cher", + "chorasmian": "chrs", + "chrs": "chrs", + "common": "zyyy", + "copt": "copt", + "coptic": "copt", + "cprt": "cprt", + "cuneiform": "xsux", + "cypriot": "cprt", + "cyrillic": "cyrl", + "cyrl": "cyrl", + "deseret": "dsrt", + "deva": "deva", + "devanagari": "deva", + "diak": "diak", + "divesakuru": "diak", + "dogr": "dogr", + "dogra": "dogr", + "dsrt": "dsrt", + "dupl": "dupl", + "duployan": "dupl", + "egyp": "egyp", + "egyptianhieroglyphs": "egyp", + "elba": "elba", + "elbasan": "elba", + "elym": "elym", + "elymaic": "elym", + "ethi": "ethi", + "ethiopic": "ethi", + "geor": "geor", + "georgian": "geor", + "glag": "glag", + "glagolitic": "glag", + "gong": "gong", + "gonm": "gonm", + "goth": "goth", + "gothic": "goth", + "gran": "gran", + "grantha": "gran", + "greek": "grek", + "grek": "grek", + "gujarati": "gujr", + "gujr": "gujr", + "gunjalagondi": "gong", + "gurmukhi": "guru", + "guru": "guru", + "han": "hani", + "hang": "hang", + "hangul": "hang", + "hani": "hani", + "hanifirohingya": "rohg", + "hano": "hano", + "hanunoo": "hano", + "hatr": "hatr", + "hatran": "hatr", + "hebr": "hebr", + "hebrew": "hebr", + "hira": "hira", + "hiragana": "hira", + "hluw": "hluw", + "hmng": "hmng", + "hmnp": "hmnp", + "hrkt": "hrkt", + "hung": "hung", + "imperialaramaic": "armi", + "inherited": "zinh", + "inscriptionalpahlavi": "phli", + "inscriptionalparthian": "prti", + "ital": "ital", + "java": "java", + "javanese": "java", + "kaithi": "kthi", + "kali": "kali", + "kana": "kana", + "kannada": "knda", + "katakana": "kana", + "katakanaorhiragana": "hrkt", + "kayahli": "kali", + "khar": "khar", + "kharoshthi": "khar", + "khitansmallscript": "kits", + "khmer": "khmr", + "khmr": "khmr", + "khoj": "khoj", + "khojki": "khoj", + "khudawadi": "sind", + "kits": "kits", + "knda": "knda", + "kthi": "kthi", + "lana": "lana", + "lao": "laoo", + "laoo": "laoo", + "latin": "latn", + "latn": "latn", + "lepc": "lepc", + "lepcha": "lepc", + "limb": "limb", + "limbu": "limb", + "lina": "lina", + "linb": "linb", + "lineara": "lina", + "linearb": "linb", + "lisu": "lisu", + "lyci": "lyci", + "lycian": "lyci", + "lydi": "lydi", + "lydian": "lydi", + "mahajani": "mahj", + "mahj": "mahj", + "maka": "maka", + "makasar": "maka", + "malayalam": "mlym", + "mand": "mand", + "mandaic": "mand", + "mani": "mani", + "manichaean": "mani", + "marc": "marc", + "marchen": "marc", + "masaramgondi": "gonm", + "medefaidrin": "medf", + "medf": "medf", + "meeteimayek": "mtei", + "mend": "mend", + "mendekikakui": "mend", + "merc": "merc", + "mero": "mero", + "meroiticcursive": "merc", + "meroitichieroglyphs": "mero", + "miao": "plrd", + "mlym": "mlym", + "modi": "modi", + "mong": "mong", + "mongolian": "mong", + "mro": "mroo", + "mroo": "mroo", + "mtei": "mtei", + "mult": "mult", + "multani": "mult", + "myanmar": "mymr", + "mymr": "mymr", + "nabataean": "nbat", + "nand": "nand", + "nandinagari": "nand", + "narb": "narb", + "nbat": "nbat", + "newa": "newa", + "newtailue": "talu", + "nko": "nkoo", + "nkoo": "nkoo", + "nshu": "nshu", + "nushu": "nshu", + "nyiakengpuachuehmong": "hmnp", + "ogam": "ogam", + "ogham": "ogam", + "olchiki": "olck", + "olck": "olck", + "oldhungarian": "hung", + "olditalic": "ital", + "oldnortharabian": "narb", + "oldpermic": "perm", + "oldpersian": "xpeo", + "oldsogdian": "sogo", + "oldsoutharabian": "sarb", + "oldturkic": "orkh", + "oriya": "orya", + "orkh": "orkh", + "orya": "orya", + "osage": "osge", + "osge": "osge", + "osma": "osma", + "osmanya": "osma", + "pahawhhmong": "hmng", + "palm": "palm", + "palmyrene": "palm", + "pauc": "pauc", + "paucinhau": "pauc", + "perm": "perm", + "phag": "phag", + "phagspa": "phag", + "phli": "phli", + "phlp": "phlp", + "phnx": "phnx", + "phoenician": "phnx", + "plrd": "plrd", + "prti": "prti", + "psalterpahlavi": "phlp", + "qaac": "copt", + "qaai": "zinh", + "rejang": "rjng", + "rjng": "rjng", + "rohg": "rohg", + "runic": "runr", + "runr": "runr", + "samaritan": "samr", + "samr": "samr", + "sarb": "sarb", + "saur": "saur", + "saurashtra": "saur", + "sgnw": "sgnw", + "sharada": "shrd", + "shavian": "shaw", + "shaw": "shaw", + "shrd": "shrd", + "sidd": "sidd", + "siddham": "sidd", + "signwriting": "sgnw", + "sind": "sind", + "sinh": "sinh", + "sinhala": "sinh", + "sogd": "sogd", + "sogdian": "sogd", + "sogo": "sogo", + "sora": "sora", + "sorasompeng": "sora", + "soyo": "soyo", + "soyombo": "soyo", + "sund": "sund", + "sundanese": "sund", + "sylo": "sylo", + "sylotinagri": "sylo", + "syrc": "syrc", + "syriac": "syrc", + "tagalog": "tglg", + "tagb": "tagb", + "tagbanwa": "tagb", + "taile": "tale", + "taitham": "lana", + "taiviet": "tavt", + "takr": "takr", + "takri": "takr", + "tale": "tale", + "talu": "talu", + "tamil": "taml", + "taml": "taml", + "tang": "tang", + "tangut": "tang", + "tavt": "tavt", + "telu": "telu", + "telugu": "telu", + "tfng": "tfng", + "tglg": "tglg", + "thaa": "thaa", + "thaana": "thaa", + "thai": "thai", + "tibetan": "tibt", + "tibt": "tibt", + "tifinagh": "tfng", + "tirh": "tirh", + "tirhuta": "tirh", + "ugar": "ugar", + "ugaritic": "ugar", + "unknown": "zzzz", + "vai": "vaii", + "vaii": "vaii", + "wancho": "wcho", + "wara": "wara", + "warangciti": "wara", + "wcho": "wcho", + "xpeo": "xpeo", + "xsux": "xsux", + "yezi": "yezi", + "yezidi": "yezi", + "yi": "yiii", + "yiii": "yiii", + "zanabazarsquare": "zanb", + "zanb": "zanb", + "zinh": "zinh", + "zyyy": "zyyy", + "zzzz": "zzzz", +} + +// https://www.unicode.org/Public/13.0.0/ucd/PropertyValueAliases.txt +var ( + generalCategoryDefaultRange = &CodePointRange{ + From: rune(0), + To: rune(1114111), + } + generalCategoryDefaultValue = "unassigned" +) + +// https://www.unicode.org/Public/13.0.0/ucd/UnicodeData.txt +var generalCategoryCodePoints = map[string][]*CodePointRange{ + "cc": { + &CodePointRange{From: rune(0), To: rune(31)}, + &CodePointRange{From: rune(127), To: rune(159)}, + }, + "cf": { + &CodePointRange{From: rune(173), To: rune(173)}, + &CodePointRange{From: rune(1536), To: rune(1541)}, + &CodePointRange{From: rune(1564), To: rune(1564)}, + &CodePointRange{From: rune(1757), To: rune(1757)}, + &CodePointRange{From: rune(1807), To: rune(1807)}, + &CodePointRange{From: rune(2274), To: rune(2274)}, + &CodePointRange{From: rune(6158), To: rune(6158)}, + &CodePointRange{From: rune(8203), To: rune(8207)}, + &CodePointRange{From: rune(8234), To: rune(8238)}, + &CodePointRange{From: rune(8288), To: rune(8292)}, + &CodePointRange{From: rune(8294), To: rune(8303)}, + &CodePointRange{From: rune(65279), To: rune(65279)}, + &CodePointRange{From: rune(65529), To: rune(65531)}, + &CodePointRange{From: rune(69821), To: rune(69821)}, + &CodePointRange{From: rune(69837), To: rune(69837)}, + &CodePointRange{From: rune(78896), To: rune(78904)}, + &CodePointRange{From: rune(113824), To: rune(113827)}, + &CodePointRange{From: rune(119155), To: rune(119162)}, + &CodePointRange{From: rune(917505), To: rune(917505)}, + &CodePointRange{From: rune(917536), To: rune(917631)}, + }, + "co": { + &CodePointRange{From: rune(57344), To: rune(57344)}, + &CodePointRange{From: rune(63743), To: rune(63743)}, + &CodePointRange{From: rune(983040), To: rune(983040)}, + &CodePointRange{From: rune(1048573), To: rune(1048573)}, + &CodePointRange{From: rune(1048576), To: rune(1048576)}, + &CodePointRange{From: rune(1114109), To: rune(1114109)}, + }, + "cs": { + &CodePointRange{From: rune(55296), To: rune(55296)}, + &CodePointRange{From: rune(56191), To: rune(56192)}, + &CodePointRange{From: rune(56319), To: rune(56320)}, + &CodePointRange{From: rune(57343), To: rune(57343)}, + }, + "ll": { + &CodePointRange{From: rune(97), To: rune(122)}, + &CodePointRange{From: rune(181), To: rune(181)}, + &CodePointRange{From: rune(223), To: rune(246)}, + &CodePointRange{From: rune(248), To: rune(255)}, + &CodePointRange{From: rune(257), To: rune(257)}, + &CodePointRange{From: rune(259), To: rune(259)}, + &CodePointRange{From: rune(261), To: rune(261)}, + &CodePointRange{From: rune(263), To: rune(263)}, + &CodePointRange{From: rune(265), To: rune(265)}, + &CodePointRange{From: rune(267), To: rune(267)}, + &CodePointRange{From: rune(269), To: rune(269)}, + &CodePointRange{From: rune(271), To: rune(271)}, + &CodePointRange{From: rune(273), To: rune(273)}, + &CodePointRange{From: rune(275), To: rune(275)}, + &CodePointRange{From: rune(277), To: rune(277)}, + &CodePointRange{From: rune(279), To: rune(279)}, + &CodePointRange{From: rune(281), To: rune(281)}, + &CodePointRange{From: rune(283), To: rune(283)}, + &CodePointRange{From: rune(285), To: rune(285)}, + &CodePointRange{From: rune(287), To: rune(287)}, + &CodePointRange{From: rune(289), To: rune(289)}, + &CodePointRange{From: rune(291), To: rune(291)}, + &CodePointRange{From: rune(293), To: rune(293)}, + &CodePointRange{From: rune(295), To: rune(295)}, + &CodePointRange{From: rune(297), To: rune(297)}, + &CodePointRange{From: rune(299), To: rune(299)}, + &CodePointRange{From: rune(301), To: rune(301)}, + &CodePointRange{From: rune(303), To: rune(303)}, + &CodePointRange{From: rune(305), To: rune(305)}, + &CodePointRange{From: rune(307), To: rune(307)}, + &CodePointRange{From: rune(309), To: rune(309)}, + &CodePointRange{From: rune(311), To: rune(312)}, + &CodePointRange{From: rune(314), To: rune(314)}, + &CodePointRange{From: rune(316), To: rune(316)}, + &CodePointRange{From: rune(318), To: rune(318)}, + &CodePointRange{From: rune(320), To: rune(320)}, + &CodePointRange{From: rune(322), To: rune(322)}, + &CodePointRange{From: rune(324), To: rune(324)}, + &CodePointRange{From: rune(326), To: rune(326)}, + &CodePointRange{From: rune(328), To: rune(329)}, + &CodePointRange{From: rune(331), To: rune(331)}, + &CodePointRange{From: rune(333), To: rune(333)}, + &CodePointRange{From: rune(335), To: rune(335)}, + &CodePointRange{From: rune(337), To: rune(337)}, + &CodePointRange{From: rune(339), To: rune(339)}, + &CodePointRange{From: rune(341), To: rune(341)}, + &CodePointRange{From: rune(343), To: rune(343)}, + &CodePointRange{From: rune(345), To: rune(345)}, + &CodePointRange{From: rune(347), To: rune(347)}, + &CodePointRange{From: rune(349), To: rune(349)}, + &CodePointRange{From: rune(351), To: rune(351)}, + &CodePointRange{From: rune(353), To: rune(353)}, + &CodePointRange{From: rune(355), To: rune(355)}, + &CodePointRange{From: rune(357), To: rune(357)}, + &CodePointRange{From: rune(359), To: rune(359)}, + &CodePointRange{From: rune(361), To: rune(361)}, + &CodePointRange{From: rune(363), To: rune(363)}, + &CodePointRange{From: rune(365), To: rune(365)}, + &CodePointRange{From: rune(367), To: rune(367)}, + &CodePointRange{From: rune(369), To: rune(369)}, + &CodePointRange{From: rune(371), To: rune(371)}, + &CodePointRange{From: rune(373), To: rune(373)}, + &CodePointRange{From: rune(375), To: rune(375)}, + &CodePointRange{From: rune(378), To: rune(378)}, + &CodePointRange{From: rune(380), To: rune(380)}, + &CodePointRange{From: rune(382), To: rune(384)}, + &CodePointRange{From: rune(387), To: rune(387)}, + &CodePointRange{From: rune(389), To: rune(389)}, + &CodePointRange{From: rune(392), To: rune(392)}, + &CodePointRange{From: rune(396), To: rune(397)}, + &CodePointRange{From: rune(402), To: rune(402)}, + &CodePointRange{From: rune(405), To: rune(405)}, + &CodePointRange{From: rune(409), To: rune(411)}, + &CodePointRange{From: rune(414), To: rune(414)}, + &CodePointRange{From: rune(417), To: rune(417)}, + &CodePointRange{From: rune(419), To: rune(419)}, + &CodePointRange{From: rune(421), To: rune(421)}, + &CodePointRange{From: rune(424), To: rune(424)}, + &CodePointRange{From: rune(426), To: rune(427)}, + &CodePointRange{From: rune(429), To: rune(429)}, + &CodePointRange{From: rune(432), To: rune(432)}, + &CodePointRange{From: rune(436), To: rune(436)}, + &CodePointRange{From: rune(438), To: rune(438)}, + &CodePointRange{From: rune(441), To: rune(442)}, + &CodePointRange{From: rune(445), To: rune(447)}, + &CodePointRange{From: rune(454), To: rune(454)}, + &CodePointRange{From: rune(457), To: rune(457)}, + &CodePointRange{From: rune(460), To: rune(460)}, + &CodePointRange{From: rune(462), To: rune(462)}, + &CodePointRange{From: rune(464), To: rune(464)}, + &CodePointRange{From: rune(466), To: rune(466)}, + &CodePointRange{From: rune(468), To: rune(468)}, + &CodePointRange{From: rune(470), To: rune(470)}, + &CodePointRange{From: rune(472), To: rune(472)}, + &CodePointRange{From: rune(474), To: rune(474)}, + &CodePointRange{From: rune(476), To: rune(477)}, + &CodePointRange{From: rune(479), To: rune(479)}, + &CodePointRange{From: rune(481), To: rune(481)}, + &CodePointRange{From: rune(483), To: rune(483)}, + &CodePointRange{From: rune(485), To: rune(485)}, + &CodePointRange{From: rune(487), To: rune(487)}, + &CodePointRange{From: rune(489), To: rune(489)}, + &CodePointRange{From: rune(491), To: rune(491)}, + &CodePointRange{From: rune(493), To: rune(493)}, + &CodePointRange{From: rune(495), To: rune(496)}, + &CodePointRange{From: rune(499), To: rune(499)}, + &CodePointRange{From: rune(501), To: rune(501)}, + &CodePointRange{From: rune(505), To: rune(505)}, + &CodePointRange{From: rune(507), To: rune(507)}, + &CodePointRange{From: rune(509), To: rune(509)}, + &CodePointRange{From: rune(511), To: rune(511)}, + &CodePointRange{From: rune(513), To: rune(513)}, + &CodePointRange{From: rune(515), To: rune(515)}, + &CodePointRange{From: rune(517), To: rune(517)}, + &CodePointRange{From: rune(519), To: rune(519)}, + &CodePointRange{From: rune(521), To: rune(521)}, + &CodePointRange{From: rune(523), To: rune(523)}, + &CodePointRange{From: rune(525), To: rune(525)}, + &CodePointRange{From: rune(527), To: rune(527)}, + &CodePointRange{From: rune(529), To: rune(529)}, + &CodePointRange{From: rune(531), To: rune(531)}, + &CodePointRange{From: rune(533), To: rune(533)}, + &CodePointRange{From: rune(535), To: rune(535)}, + &CodePointRange{From: rune(537), To: rune(537)}, + &CodePointRange{From: rune(539), To: rune(539)}, + &CodePointRange{From: rune(541), To: rune(541)}, + &CodePointRange{From: rune(543), To: rune(543)}, + &CodePointRange{From: rune(545), To: rune(545)}, + &CodePointRange{From: rune(547), To: rune(547)}, + &CodePointRange{From: rune(549), To: rune(549)}, + &CodePointRange{From: rune(551), To: rune(551)}, + &CodePointRange{From: rune(553), To: rune(553)}, + &CodePointRange{From: rune(555), To: rune(555)}, + &CodePointRange{From: rune(557), To: rune(557)}, + &CodePointRange{From: rune(559), To: rune(559)}, + &CodePointRange{From: rune(561), To: rune(561)}, + &CodePointRange{From: rune(563), To: rune(569)}, + &CodePointRange{From: rune(572), To: rune(572)}, + &CodePointRange{From: rune(575), To: rune(576)}, + &CodePointRange{From: rune(578), To: rune(578)}, + &CodePointRange{From: rune(583), To: rune(583)}, + &CodePointRange{From: rune(585), To: rune(585)}, + &CodePointRange{From: rune(587), To: rune(587)}, + &CodePointRange{From: rune(589), To: rune(589)}, + &CodePointRange{From: rune(591), To: rune(659)}, + &CodePointRange{From: rune(661), To: rune(687)}, + &CodePointRange{From: rune(881), To: rune(881)}, + &CodePointRange{From: rune(883), To: rune(883)}, + &CodePointRange{From: rune(887), To: rune(887)}, + &CodePointRange{From: rune(891), To: rune(893)}, + &CodePointRange{From: rune(912), To: rune(912)}, + &CodePointRange{From: rune(940), To: rune(974)}, + &CodePointRange{From: rune(976), To: rune(977)}, + &CodePointRange{From: rune(981), To: rune(983)}, + &CodePointRange{From: rune(985), To: rune(985)}, + &CodePointRange{From: rune(987), To: rune(987)}, + &CodePointRange{From: rune(989), To: rune(989)}, + &CodePointRange{From: rune(991), To: rune(991)}, + &CodePointRange{From: rune(993), To: rune(993)}, + &CodePointRange{From: rune(995), To: rune(995)}, + &CodePointRange{From: rune(997), To: rune(997)}, + &CodePointRange{From: rune(999), To: rune(999)}, + &CodePointRange{From: rune(1001), To: rune(1001)}, + &CodePointRange{From: rune(1003), To: rune(1003)}, + &CodePointRange{From: rune(1005), To: rune(1005)}, + &CodePointRange{From: rune(1007), To: rune(1011)}, + &CodePointRange{From: rune(1013), To: rune(1013)}, + &CodePointRange{From: rune(1016), To: rune(1016)}, + &CodePointRange{From: rune(1019), To: rune(1020)}, + &CodePointRange{From: rune(1072), To: rune(1119)}, + &CodePointRange{From: rune(1121), To: rune(1121)}, + &CodePointRange{From: rune(1123), To: rune(1123)}, + &CodePointRange{From: rune(1125), To: rune(1125)}, + &CodePointRange{From: rune(1127), To: rune(1127)}, + &CodePointRange{From: rune(1129), To: rune(1129)}, + &CodePointRange{From: rune(1131), To: rune(1131)}, + &CodePointRange{From: rune(1133), To: rune(1133)}, + &CodePointRange{From: rune(1135), To: rune(1135)}, + &CodePointRange{From: rune(1137), To: rune(1137)}, + &CodePointRange{From: rune(1139), To: rune(1139)}, + &CodePointRange{From: rune(1141), To: rune(1141)}, + &CodePointRange{From: rune(1143), To: rune(1143)}, + &CodePointRange{From: rune(1145), To: rune(1145)}, + &CodePointRange{From: rune(1147), To: rune(1147)}, + &CodePointRange{From: rune(1149), To: rune(1149)}, + &CodePointRange{From: rune(1151), To: rune(1151)}, + &CodePointRange{From: rune(1153), To: rune(1153)}, + &CodePointRange{From: rune(1163), To: rune(1163)}, + &CodePointRange{From: rune(1165), To: rune(1165)}, + &CodePointRange{From: rune(1167), To: rune(1167)}, + &CodePointRange{From: rune(1169), To: rune(1169)}, + &CodePointRange{From: rune(1171), To: rune(1171)}, + &CodePointRange{From: rune(1173), To: rune(1173)}, + &CodePointRange{From: rune(1175), To: rune(1175)}, + &CodePointRange{From: rune(1177), To: rune(1177)}, + &CodePointRange{From: rune(1179), To: rune(1179)}, + &CodePointRange{From: rune(1181), To: rune(1181)}, + &CodePointRange{From: rune(1183), To: rune(1183)}, + &CodePointRange{From: rune(1185), To: rune(1185)}, + &CodePointRange{From: rune(1187), To: rune(1187)}, + &CodePointRange{From: rune(1189), To: rune(1189)}, + &CodePointRange{From: rune(1191), To: rune(1191)}, + &CodePointRange{From: rune(1193), To: rune(1193)}, + &CodePointRange{From: rune(1195), To: rune(1195)}, + &CodePointRange{From: rune(1197), To: rune(1197)}, + &CodePointRange{From: rune(1199), To: rune(1199)}, + &CodePointRange{From: rune(1201), To: rune(1201)}, + &CodePointRange{From: rune(1203), To: rune(1203)}, + &CodePointRange{From: rune(1205), To: rune(1205)}, + &CodePointRange{From: rune(1207), To: rune(1207)}, + &CodePointRange{From: rune(1209), To: rune(1209)}, + &CodePointRange{From: rune(1211), To: rune(1211)}, + &CodePointRange{From: rune(1213), To: rune(1213)}, + &CodePointRange{From: rune(1215), To: rune(1215)}, + &CodePointRange{From: rune(1218), To: rune(1218)}, + &CodePointRange{From: rune(1220), To: rune(1220)}, + &CodePointRange{From: rune(1222), To: rune(1222)}, + &CodePointRange{From: rune(1224), To: rune(1224)}, + &CodePointRange{From: rune(1226), To: rune(1226)}, + &CodePointRange{From: rune(1228), To: rune(1228)}, + &CodePointRange{From: rune(1230), To: rune(1231)}, + &CodePointRange{From: rune(1233), To: rune(1233)}, + &CodePointRange{From: rune(1235), To: rune(1235)}, + &CodePointRange{From: rune(1237), To: rune(1237)}, + &CodePointRange{From: rune(1239), To: rune(1239)}, + &CodePointRange{From: rune(1241), To: rune(1241)}, + &CodePointRange{From: rune(1243), To: rune(1243)}, + &CodePointRange{From: rune(1245), To: rune(1245)}, + &CodePointRange{From: rune(1247), To: rune(1247)}, + &CodePointRange{From: rune(1249), To: rune(1249)}, + &CodePointRange{From: rune(1251), To: rune(1251)}, + &CodePointRange{From: rune(1253), To: rune(1253)}, + &CodePointRange{From: rune(1255), To: rune(1255)}, + &CodePointRange{From: rune(1257), To: rune(1257)}, + &CodePointRange{From: rune(1259), To: rune(1259)}, + &CodePointRange{From: rune(1261), To: rune(1261)}, + &CodePointRange{From: rune(1263), To: rune(1263)}, + &CodePointRange{From: rune(1265), To: rune(1265)}, + &CodePointRange{From: rune(1267), To: rune(1267)}, + &CodePointRange{From: rune(1269), To: rune(1269)}, + &CodePointRange{From: rune(1271), To: rune(1271)}, + &CodePointRange{From: rune(1273), To: rune(1273)}, + &CodePointRange{From: rune(1275), To: rune(1275)}, + &CodePointRange{From: rune(1277), To: rune(1277)}, + &CodePointRange{From: rune(1279), To: rune(1279)}, + &CodePointRange{From: rune(1281), To: rune(1281)}, + &CodePointRange{From: rune(1283), To: rune(1283)}, + &CodePointRange{From: rune(1285), To: rune(1285)}, + &CodePointRange{From: rune(1287), To: rune(1287)}, + &CodePointRange{From: rune(1289), To: rune(1289)}, + &CodePointRange{From: rune(1291), To: rune(1291)}, + &CodePointRange{From: rune(1293), To: rune(1293)}, + &CodePointRange{From: rune(1295), To: rune(1295)}, + &CodePointRange{From: rune(1297), To: rune(1297)}, + &CodePointRange{From: rune(1299), To: rune(1299)}, + &CodePointRange{From: rune(1301), To: rune(1301)}, + &CodePointRange{From: rune(1303), To: rune(1303)}, + &CodePointRange{From: rune(1305), To: rune(1305)}, + &CodePointRange{From: rune(1307), To: rune(1307)}, + &CodePointRange{From: rune(1309), To: rune(1309)}, + &CodePointRange{From: rune(1311), To: rune(1311)}, + &CodePointRange{From: rune(1313), To: rune(1313)}, + &CodePointRange{From: rune(1315), To: rune(1315)}, + &CodePointRange{From: rune(1317), To: rune(1317)}, + &CodePointRange{From: rune(1319), To: rune(1319)}, + &CodePointRange{From: rune(1321), To: rune(1321)}, + &CodePointRange{From: rune(1323), To: rune(1323)}, + &CodePointRange{From: rune(1325), To: rune(1325)}, + &CodePointRange{From: rune(1327), To: rune(1327)}, + &CodePointRange{From: rune(1376), To: rune(1416)}, + &CodePointRange{From: rune(4304), To: rune(4346)}, + &CodePointRange{From: rune(4349), To: rune(4351)}, + &CodePointRange{From: rune(5112), To: rune(5117)}, + &CodePointRange{From: rune(7296), To: rune(7304)}, + &CodePointRange{From: rune(7424), To: rune(7467)}, + &CodePointRange{From: rune(7531), To: rune(7543)}, + &CodePointRange{From: rune(7545), To: rune(7578)}, + &CodePointRange{From: rune(7681), To: rune(7681)}, + &CodePointRange{From: rune(7683), To: rune(7683)}, + &CodePointRange{From: rune(7685), To: rune(7685)}, + &CodePointRange{From: rune(7687), To: rune(7687)}, + &CodePointRange{From: rune(7689), To: rune(7689)}, + &CodePointRange{From: rune(7691), To: rune(7691)}, + &CodePointRange{From: rune(7693), To: rune(7693)}, + &CodePointRange{From: rune(7695), To: rune(7695)}, + &CodePointRange{From: rune(7697), To: rune(7697)}, + &CodePointRange{From: rune(7699), To: rune(7699)}, + &CodePointRange{From: rune(7701), To: rune(7701)}, + &CodePointRange{From: rune(7703), To: rune(7703)}, + &CodePointRange{From: rune(7705), To: rune(7705)}, + &CodePointRange{From: rune(7707), To: rune(7707)}, + &CodePointRange{From: rune(7709), To: rune(7709)}, + &CodePointRange{From: rune(7711), To: rune(7711)}, + &CodePointRange{From: rune(7713), To: rune(7713)}, + &CodePointRange{From: rune(7715), To: rune(7715)}, + &CodePointRange{From: rune(7717), To: rune(7717)}, + &CodePointRange{From: rune(7719), To: rune(7719)}, + &CodePointRange{From: rune(7721), To: rune(7721)}, + &CodePointRange{From: rune(7723), To: rune(7723)}, + &CodePointRange{From: rune(7725), To: rune(7725)}, + &CodePointRange{From: rune(7727), To: rune(7727)}, + &CodePointRange{From: rune(7729), To: rune(7729)}, + &CodePointRange{From: rune(7731), To: rune(7731)}, + &CodePointRange{From: rune(7733), To: rune(7733)}, + &CodePointRange{From: rune(7735), To: rune(7735)}, + &CodePointRange{From: rune(7737), To: rune(7737)}, + &CodePointRange{From: rune(7739), To: rune(7739)}, + &CodePointRange{From: rune(7741), To: rune(7741)}, + &CodePointRange{From: rune(7743), To: rune(7743)}, + &CodePointRange{From: rune(7745), To: rune(7745)}, + &CodePointRange{From: rune(7747), To: rune(7747)}, + &CodePointRange{From: rune(7749), To: rune(7749)}, + &CodePointRange{From: rune(7751), To: rune(7751)}, + &CodePointRange{From: rune(7753), To: rune(7753)}, + &CodePointRange{From: rune(7755), To: rune(7755)}, + &CodePointRange{From: rune(7757), To: rune(7757)}, + &CodePointRange{From: rune(7759), To: rune(7759)}, + &CodePointRange{From: rune(7761), To: rune(7761)}, + &CodePointRange{From: rune(7763), To: rune(7763)}, + &CodePointRange{From: rune(7765), To: rune(7765)}, + &CodePointRange{From: rune(7767), To: rune(7767)}, + &CodePointRange{From: rune(7769), To: rune(7769)}, + &CodePointRange{From: rune(7771), To: rune(7771)}, + &CodePointRange{From: rune(7773), To: rune(7773)}, + &CodePointRange{From: rune(7775), To: rune(7775)}, + &CodePointRange{From: rune(7777), To: rune(7777)}, + &CodePointRange{From: rune(7779), To: rune(7779)}, + &CodePointRange{From: rune(7781), To: rune(7781)}, + &CodePointRange{From: rune(7783), To: rune(7783)}, + &CodePointRange{From: rune(7785), To: rune(7785)}, + &CodePointRange{From: rune(7787), To: rune(7787)}, + &CodePointRange{From: rune(7789), To: rune(7789)}, + &CodePointRange{From: rune(7791), To: rune(7791)}, + &CodePointRange{From: rune(7793), To: rune(7793)}, + &CodePointRange{From: rune(7795), To: rune(7795)}, + &CodePointRange{From: rune(7797), To: rune(7797)}, + &CodePointRange{From: rune(7799), To: rune(7799)}, + &CodePointRange{From: rune(7801), To: rune(7801)}, + &CodePointRange{From: rune(7803), To: rune(7803)}, + &CodePointRange{From: rune(7805), To: rune(7805)}, + &CodePointRange{From: rune(7807), To: rune(7807)}, + &CodePointRange{From: rune(7809), To: rune(7809)}, + &CodePointRange{From: rune(7811), To: rune(7811)}, + &CodePointRange{From: rune(7813), To: rune(7813)}, + &CodePointRange{From: rune(7815), To: rune(7815)}, + &CodePointRange{From: rune(7817), To: rune(7817)}, + &CodePointRange{From: rune(7819), To: rune(7819)}, + &CodePointRange{From: rune(7821), To: rune(7821)}, + &CodePointRange{From: rune(7823), To: rune(7823)}, + &CodePointRange{From: rune(7825), To: rune(7825)}, + &CodePointRange{From: rune(7827), To: rune(7827)}, + &CodePointRange{From: rune(7829), To: rune(7837)}, + &CodePointRange{From: rune(7839), To: rune(7839)}, + &CodePointRange{From: rune(7841), To: rune(7841)}, + &CodePointRange{From: rune(7843), To: rune(7843)}, + &CodePointRange{From: rune(7845), To: rune(7845)}, + &CodePointRange{From: rune(7847), To: rune(7847)}, + &CodePointRange{From: rune(7849), To: rune(7849)}, + &CodePointRange{From: rune(7851), To: rune(7851)}, + &CodePointRange{From: rune(7853), To: rune(7853)}, + &CodePointRange{From: rune(7855), To: rune(7855)}, + &CodePointRange{From: rune(7857), To: rune(7857)}, + &CodePointRange{From: rune(7859), To: rune(7859)}, + &CodePointRange{From: rune(7861), To: rune(7861)}, + &CodePointRange{From: rune(7863), To: rune(7863)}, + &CodePointRange{From: rune(7865), To: rune(7865)}, + &CodePointRange{From: rune(7867), To: rune(7867)}, + &CodePointRange{From: rune(7869), To: rune(7869)}, + &CodePointRange{From: rune(7871), To: rune(7871)}, + &CodePointRange{From: rune(7873), To: rune(7873)}, + &CodePointRange{From: rune(7875), To: rune(7875)}, + &CodePointRange{From: rune(7877), To: rune(7877)}, + &CodePointRange{From: rune(7879), To: rune(7879)}, + &CodePointRange{From: rune(7881), To: rune(7881)}, + &CodePointRange{From: rune(7883), To: rune(7883)}, + &CodePointRange{From: rune(7885), To: rune(7885)}, + &CodePointRange{From: rune(7887), To: rune(7887)}, + &CodePointRange{From: rune(7889), To: rune(7889)}, + &CodePointRange{From: rune(7891), To: rune(7891)}, + &CodePointRange{From: rune(7893), To: rune(7893)}, + &CodePointRange{From: rune(7895), To: rune(7895)}, + &CodePointRange{From: rune(7897), To: rune(7897)}, + &CodePointRange{From: rune(7899), To: rune(7899)}, + &CodePointRange{From: rune(7901), To: rune(7901)}, + &CodePointRange{From: rune(7903), To: rune(7903)}, + &CodePointRange{From: rune(7905), To: rune(7905)}, + &CodePointRange{From: rune(7907), To: rune(7907)}, + &CodePointRange{From: rune(7909), To: rune(7909)}, + &CodePointRange{From: rune(7911), To: rune(7911)}, + &CodePointRange{From: rune(7913), To: rune(7913)}, + &CodePointRange{From: rune(7915), To: rune(7915)}, + &CodePointRange{From: rune(7917), To: rune(7917)}, + &CodePointRange{From: rune(7919), To: rune(7919)}, + &CodePointRange{From: rune(7921), To: rune(7921)}, + &CodePointRange{From: rune(7923), To: rune(7923)}, + &CodePointRange{From: rune(7925), To: rune(7925)}, + &CodePointRange{From: rune(7927), To: rune(7927)}, + &CodePointRange{From: rune(7929), To: rune(7929)}, + &CodePointRange{From: rune(7931), To: rune(7931)}, + &CodePointRange{From: rune(7933), To: rune(7933)}, + &CodePointRange{From: rune(7935), To: rune(7943)}, + &CodePointRange{From: rune(7952), To: rune(7957)}, + &CodePointRange{From: rune(7968), To: rune(7975)}, + &CodePointRange{From: rune(7984), To: rune(7991)}, + &CodePointRange{From: rune(8000), To: rune(8005)}, + &CodePointRange{From: rune(8016), To: rune(8023)}, + &CodePointRange{From: rune(8032), To: rune(8039)}, + &CodePointRange{From: rune(8048), To: rune(8061)}, + &CodePointRange{From: rune(8064), To: rune(8071)}, + &CodePointRange{From: rune(8080), To: rune(8087)}, + &CodePointRange{From: rune(8096), To: rune(8103)}, + &CodePointRange{From: rune(8112), To: rune(8116)}, + &CodePointRange{From: rune(8118), To: rune(8119)}, + &CodePointRange{From: rune(8126), To: rune(8126)}, + &CodePointRange{From: rune(8130), To: rune(8132)}, + &CodePointRange{From: rune(8134), To: rune(8135)}, + &CodePointRange{From: rune(8144), To: rune(8147)}, + &CodePointRange{From: rune(8150), To: rune(8151)}, + &CodePointRange{From: rune(8160), To: rune(8167)}, + &CodePointRange{From: rune(8178), To: rune(8180)}, + &CodePointRange{From: rune(8182), To: rune(8183)}, + &CodePointRange{From: rune(8458), To: rune(8458)}, + &CodePointRange{From: rune(8462), To: rune(8463)}, + &CodePointRange{From: rune(8467), To: rune(8467)}, + &CodePointRange{From: rune(8495), To: rune(8495)}, + &CodePointRange{From: rune(8500), To: rune(8500)}, + &CodePointRange{From: rune(8505), To: rune(8505)}, + &CodePointRange{From: rune(8508), To: rune(8509)}, + &CodePointRange{From: rune(8518), To: rune(8521)}, + &CodePointRange{From: rune(8526), To: rune(8526)}, + &CodePointRange{From: rune(8580), To: rune(8580)}, + &CodePointRange{From: rune(11312), To: rune(11358)}, + &CodePointRange{From: rune(11361), To: rune(11361)}, + &CodePointRange{From: rune(11365), To: rune(11366)}, + &CodePointRange{From: rune(11368), To: rune(11368)}, + &CodePointRange{From: rune(11370), To: rune(11370)}, + &CodePointRange{From: rune(11372), To: rune(11372)}, + &CodePointRange{From: rune(11377), To: rune(11377)}, + &CodePointRange{From: rune(11379), To: rune(11380)}, + &CodePointRange{From: rune(11382), To: rune(11387)}, + &CodePointRange{From: rune(11393), To: rune(11393)}, + &CodePointRange{From: rune(11395), To: rune(11395)}, + &CodePointRange{From: rune(11397), To: rune(11397)}, + &CodePointRange{From: rune(11399), To: rune(11399)}, + &CodePointRange{From: rune(11401), To: rune(11401)}, + &CodePointRange{From: rune(11403), To: rune(11403)}, + &CodePointRange{From: rune(11405), To: rune(11405)}, + &CodePointRange{From: rune(11407), To: rune(11407)}, + &CodePointRange{From: rune(11409), To: rune(11409)}, + &CodePointRange{From: rune(11411), To: rune(11411)}, + &CodePointRange{From: rune(11413), To: rune(11413)}, + &CodePointRange{From: rune(11415), To: rune(11415)}, + &CodePointRange{From: rune(11417), To: rune(11417)}, + &CodePointRange{From: rune(11419), To: rune(11419)}, + &CodePointRange{From: rune(11421), To: rune(11421)}, + &CodePointRange{From: rune(11423), To: rune(11423)}, + &CodePointRange{From: rune(11425), To: rune(11425)}, + &CodePointRange{From: rune(11427), To: rune(11427)}, + &CodePointRange{From: rune(11429), To: rune(11429)}, + &CodePointRange{From: rune(11431), To: rune(11431)}, + &CodePointRange{From: rune(11433), To: rune(11433)}, + &CodePointRange{From: rune(11435), To: rune(11435)}, + &CodePointRange{From: rune(11437), To: rune(11437)}, + &CodePointRange{From: rune(11439), To: rune(11439)}, + &CodePointRange{From: rune(11441), To: rune(11441)}, + &CodePointRange{From: rune(11443), To: rune(11443)}, + &CodePointRange{From: rune(11445), To: rune(11445)}, + &CodePointRange{From: rune(11447), To: rune(11447)}, + &CodePointRange{From: rune(11449), To: rune(11449)}, + &CodePointRange{From: rune(11451), To: rune(11451)}, + &CodePointRange{From: rune(11453), To: rune(11453)}, + &CodePointRange{From: rune(11455), To: rune(11455)}, + &CodePointRange{From: rune(11457), To: rune(11457)}, + &CodePointRange{From: rune(11459), To: rune(11459)}, + &CodePointRange{From: rune(11461), To: rune(11461)}, + &CodePointRange{From: rune(11463), To: rune(11463)}, + &CodePointRange{From: rune(11465), To: rune(11465)}, + &CodePointRange{From: rune(11467), To: rune(11467)}, + &CodePointRange{From: rune(11469), To: rune(11469)}, + &CodePointRange{From: rune(11471), To: rune(11471)}, + &CodePointRange{From: rune(11473), To: rune(11473)}, + &CodePointRange{From: rune(11475), To: rune(11475)}, + &CodePointRange{From: rune(11477), To: rune(11477)}, + &CodePointRange{From: rune(11479), To: rune(11479)}, + &CodePointRange{From: rune(11481), To: rune(11481)}, + &CodePointRange{From: rune(11483), To: rune(11483)}, + &CodePointRange{From: rune(11485), To: rune(11485)}, + &CodePointRange{From: rune(11487), To: rune(11487)}, + &CodePointRange{From: rune(11489), To: rune(11489)}, + &CodePointRange{From: rune(11491), To: rune(11492)}, + &CodePointRange{From: rune(11500), To: rune(11500)}, + &CodePointRange{From: rune(11502), To: rune(11502)}, + &CodePointRange{From: rune(11507), To: rune(11507)}, + &CodePointRange{From: rune(11520), To: rune(11557)}, + &CodePointRange{From: rune(11559), To: rune(11559)}, + &CodePointRange{From: rune(11565), To: rune(11565)}, + &CodePointRange{From: rune(42561), To: rune(42561)}, + &CodePointRange{From: rune(42563), To: rune(42563)}, + &CodePointRange{From: rune(42565), To: rune(42565)}, + &CodePointRange{From: rune(42567), To: rune(42567)}, + &CodePointRange{From: rune(42569), To: rune(42569)}, + &CodePointRange{From: rune(42571), To: rune(42571)}, + &CodePointRange{From: rune(42573), To: rune(42573)}, + &CodePointRange{From: rune(42575), To: rune(42575)}, + &CodePointRange{From: rune(42577), To: rune(42577)}, + &CodePointRange{From: rune(42579), To: rune(42579)}, + &CodePointRange{From: rune(42581), To: rune(42581)}, + &CodePointRange{From: rune(42583), To: rune(42583)}, + &CodePointRange{From: rune(42585), To: rune(42585)}, + &CodePointRange{From: rune(42587), To: rune(42587)}, + &CodePointRange{From: rune(42589), To: rune(42589)}, + &CodePointRange{From: rune(42591), To: rune(42591)}, + &CodePointRange{From: rune(42593), To: rune(42593)}, + &CodePointRange{From: rune(42595), To: rune(42595)}, + &CodePointRange{From: rune(42597), To: rune(42597)}, + &CodePointRange{From: rune(42599), To: rune(42599)}, + &CodePointRange{From: rune(42601), To: rune(42601)}, + &CodePointRange{From: rune(42603), To: rune(42603)}, + &CodePointRange{From: rune(42605), To: rune(42605)}, + &CodePointRange{From: rune(42625), To: rune(42625)}, + &CodePointRange{From: rune(42627), To: rune(42627)}, + &CodePointRange{From: rune(42629), To: rune(42629)}, + &CodePointRange{From: rune(42631), To: rune(42631)}, + &CodePointRange{From: rune(42633), To: rune(42633)}, + &CodePointRange{From: rune(42635), To: rune(42635)}, + &CodePointRange{From: rune(42637), To: rune(42637)}, + &CodePointRange{From: rune(42639), To: rune(42639)}, + &CodePointRange{From: rune(42641), To: rune(42641)}, + &CodePointRange{From: rune(42643), To: rune(42643)}, + &CodePointRange{From: rune(42645), To: rune(42645)}, + &CodePointRange{From: rune(42647), To: rune(42647)}, + &CodePointRange{From: rune(42649), To: rune(42649)}, + &CodePointRange{From: rune(42651), To: rune(42651)}, + &CodePointRange{From: rune(42787), To: rune(42787)}, + &CodePointRange{From: rune(42789), To: rune(42789)}, + &CodePointRange{From: rune(42791), To: rune(42791)}, + &CodePointRange{From: rune(42793), To: rune(42793)}, + &CodePointRange{From: rune(42795), To: rune(42795)}, + &CodePointRange{From: rune(42797), To: rune(42797)}, + &CodePointRange{From: rune(42799), To: rune(42801)}, + &CodePointRange{From: rune(42803), To: rune(42803)}, + &CodePointRange{From: rune(42805), To: rune(42805)}, + &CodePointRange{From: rune(42807), To: rune(42807)}, + &CodePointRange{From: rune(42809), To: rune(42809)}, + &CodePointRange{From: rune(42811), To: rune(42811)}, + &CodePointRange{From: rune(42813), To: rune(42813)}, + &CodePointRange{From: rune(42815), To: rune(42815)}, + &CodePointRange{From: rune(42817), To: rune(42817)}, + &CodePointRange{From: rune(42819), To: rune(42819)}, + &CodePointRange{From: rune(42821), To: rune(42821)}, + &CodePointRange{From: rune(42823), To: rune(42823)}, + &CodePointRange{From: rune(42825), To: rune(42825)}, + &CodePointRange{From: rune(42827), To: rune(42827)}, + &CodePointRange{From: rune(42829), To: rune(42829)}, + &CodePointRange{From: rune(42831), To: rune(42831)}, + &CodePointRange{From: rune(42833), To: rune(42833)}, + &CodePointRange{From: rune(42835), To: rune(42835)}, + &CodePointRange{From: rune(42837), To: rune(42837)}, + &CodePointRange{From: rune(42839), To: rune(42839)}, + &CodePointRange{From: rune(42841), To: rune(42841)}, + &CodePointRange{From: rune(42843), To: rune(42843)}, + &CodePointRange{From: rune(42845), To: rune(42845)}, + &CodePointRange{From: rune(42847), To: rune(42847)}, + &CodePointRange{From: rune(42849), To: rune(42849)}, + &CodePointRange{From: rune(42851), To: rune(42851)}, + &CodePointRange{From: rune(42853), To: rune(42853)}, + &CodePointRange{From: rune(42855), To: rune(42855)}, + &CodePointRange{From: rune(42857), To: rune(42857)}, + &CodePointRange{From: rune(42859), To: rune(42859)}, + &CodePointRange{From: rune(42861), To: rune(42861)}, + &CodePointRange{From: rune(42863), To: rune(42863)}, + &CodePointRange{From: rune(42865), To: rune(42872)}, + &CodePointRange{From: rune(42874), To: rune(42874)}, + &CodePointRange{From: rune(42876), To: rune(42876)}, + &CodePointRange{From: rune(42879), To: rune(42879)}, + &CodePointRange{From: rune(42881), To: rune(42881)}, + &CodePointRange{From: rune(42883), To: rune(42883)}, + &CodePointRange{From: rune(42885), To: rune(42885)}, + &CodePointRange{From: rune(42887), To: rune(42887)}, + &CodePointRange{From: rune(42892), To: rune(42892)}, + &CodePointRange{From: rune(42894), To: rune(42894)}, + &CodePointRange{From: rune(42897), To: rune(42897)}, + &CodePointRange{From: rune(42899), To: rune(42901)}, + &CodePointRange{From: rune(42903), To: rune(42903)}, + &CodePointRange{From: rune(42905), To: rune(42905)}, + &CodePointRange{From: rune(42907), To: rune(42907)}, + &CodePointRange{From: rune(42909), To: rune(42909)}, + &CodePointRange{From: rune(42911), To: rune(42911)}, + &CodePointRange{From: rune(42913), To: rune(42913)}, + &CodePointRange{From: rune(42915), To: rune(42915)}, + &CodePointRange{From: rune(42917), To: rune(42917)}, + &CodePointRange{From: rune(42919), To: rune(42919)}, + &CodePointRange{From: rune(42921), To: rune(42921)}, + &CodePointRange{From: rune(42927), To: rune(42927)}, + &CodePointRange{From: rune(42933), To: rune(42933)}, + &CodePointRange{From: rune(42935), To: rune(42935)}, + &CodePointRange{From: rune(42937), To: rune(42937)}, + &CodePointRange{From: rune(42939), To: rune(42939)}, + &CodePointRange{From: rune(42941), To: rune(42941)}, + &CodePointRange{From: rune(42943), To: rune(42943)}, + &CodePointRange{From: rune(42947), To: rune(42947)}, + &CodePointRange{From: rune(42952), To: rune(42952)}, + &CodePointRange{From: rune(42954), To: rune(42954)}, + &CodePointRange{From: rune(42998), To: rune(42998)}, + &CodePointRange{From: rune(43002), To: rune(43002)}, + &CodePointRange{From: rune(43824), To: rune(43866)}, + &CodePointRange{From: rune(43872), To: rune(43880)}, + &CodePointRange{From: rune(43888), To: rune(43967)}, + &CodePointRange{From: rune(64256), To: rune(64262)}, + &CodePointRange{From: rune(64275), To: rune(64279)}, + &CodePointRange{From: rune(65345), To: rune(65370)}, + &CodePointRange{From: rune(66600), To: rune(66639)}, + &CodePointRange{From: rune(66776), To: rune(66811)}, + &CodePointRange{From: rune(68800), To: rune(68850)}, + &CodePointRange{From: rune(71872), To: rune(71903)}, + &CodePointRange{From: rune(93792), To: rune(93823)}, + &CodePointRange{From: rune(119834), To: rune(119859)}, + &CodePointRange{From: rune(119886), To: rune(119892)}, + &CodePointRange{From: rune(119894), To: rune(119911)}, + &CodePointRange{From: rune(119938), To: rune(119963)}, + &CodePointRange{From: rune(119990), To: rune(119993)}, + &CodePointRange{From: rune(119995), To: rune(119995)}, + &CodePointRange{From: rune(119997), To: rune(120003)}, + &CodePointRange{From: rune(120005), To: rune(120015)}, + &CodePointRange{From: rune(120042), To: rune(120067)}, + &CodePointRange{From: rune(120094), To: rune(120119)}, + &CodePointRange{From: rune(120146), To: rune(120171)}, + &CodePointRange{From: rune(120198), To: rune(120223)}, + &CodePointRange{From: rune(120250), To: rune(120275)}, + &CodePointRange{From: rune(120302), To: rune(120327)}, + &CodePointRange{From: rune(120354), To: rune(120379)}, + &CodePointRange{From: rune(120406), To: rune(120431)}, + &CodePointRange{From: rune(120458), To: rune(120485)}, + &CodePointRange{From: rune(120514), To: rune(120538)}, + &CodePointRange{From: rune(120540), To: rune(120545)}, + &CodePointRange{From: rune(120572), To: rune(120596)}, + &CodePointRange{From: rune(120598), To: rune(120603)}, + &CodePointRange{From: rune(120630), To: rune(120654)}, + &CodePointRange{From: rune(120656), To: rune(120661)}, + &CodePointRange{From: rune(120688), To: rune(120712)}, + &CodePointRange{From: rune(120714), To: rune(120719)}, + &CodePointRange{From: rune(120746), To: rune(120770)}, + &CodePointRange{From: rune(120772), To: rune(120777)}, + &CodePointRange{From: rune(120779), To: rune(120779)}, + &CodePointRange{From: rune(125218), To: rune(125251)}, + }, + "lm": { + &CodePointRange{From: rune(688), To: rune(705)}, + &CodePointRange{From: rune(710), To: rune(721)}, + &CodePointRange{From: rune(736), To: rune(740)}, + &CodePointRange{From: rune(748), To: rune(748)}, + &CodePointRange{From: rune(750), To: rune(750)}, + &CodePointRange{From: rune(884), To: rune(884)}, + &CodePointRange{From: rune(890), To: rune(890)}, + &CodePointRange{From: rune(1369), To: rune(1369)}, + &CodePointRange{From: rune(1600), To: rune(1600)}, + &CodePointRange{From: rune(1765), To: rune(1766)}, + &CodePointRange{From: rune(2036), To: rune(2037)}, + &CodePointRange{From: rune(2042), To: rune(2042)}, + &CodePointRange{From: rune(2074), To: rune(2074)}, + &CodePointRange{From: rune(2084), To: rune(2084)}, + &CodePointRange{From: rune(2088), To: rune(2088)}, + &CodePointRange{From: rune(2417), To: rune(2417)}, + &CodePointRange{From: rune(3654), To: rune(3654)}, + &CodePointRange{From: rune(3782), To: rune(3782)}, + &CodePointRange{From: rune(4348), To: rune(4348)}, + &CodePointRange{From: rune(6103), To: rune(6103)}, + &CodePointRange{From: rune(6211), To: rune(6211)}, + &CodePointRange{From: rune(6823), To: rune(6823)}, + &CodePointRange{From: rune(7288), To: rune(7293)}, + &CodePointRange{From: rune(7468), To: rune(7530)}, + &CodePointRange{From: rune(7544), To: rune(7544)}, + &CodePointRange{From: rune(7579), To: rune(7615)}, + &CodePointRange{From: rune(8305), To: rune(8305)}, + &CodePointRange{From: rune(8319), To: rune(8319)}, + &CodePointRange{From: rune(8336), To: rune(8348)}, + &CodePointRange{From: rune(11388), To: rune(11389)}, + &CodePointRange{From: rune(11631), To: rune(11631)}, + &CodePointRange{From: rune(11823), To: rune(11823)}, + &CodePointRange{From: rune(12293), To: rune(12293)}, + &CodePointRange{From: rune(12337), To: rune(12341)}, + &CodePointRange{From: rune(12347), To: rune(12347)}, + &CodePointRange{From: rune(12445), To: rune(12446)}, + &CodePointRange{From: rune(12540), To: rune(12542)}, + &CodePointRange{From: rune(40981), To: rune(40981)}, + &CodePointRange{From: rune(42232), To: rune(42237)}, + &CodePointRange{From: rune(42508), To: rune(42508)}, + &CodePointRange{From: rune(42623), To: rune(42623)}, + &CodePointRange{From: rune(42652), To: rune(42653)}, + &CodePointRange{From: rune(42775), To: rune(42783)}, + &CodePointRange{From: rune(42864), To: rune(42864)}, + &CodePointRange{From: rune(42888), To: rune(42888)}, + &CodePointRange{From: rune(43000), To: rune(43001)}, + &CodePointRange{From: rune(43471), To: rune(43471)}, + &CodePointRange{From: rune(43494), To: rune(43494)}, + &CodePointRange{From: rune(43632), To: rune(43632)}, + &CodePointRange{From: rune(43741), To: rune(43741)}, + &CodePointRange{From: rune(43763), To: rune(43764)}, + &CodePointRange{From: rune(43868), To: rune(43871)}, + &CodePointRange{From: rune(43881), To: rune(43881)}, + &CodePointRange{From: rune(65392), To: rune(65392)}, + &CodePointRange{From: rune(65438), To: rune(65439)}, + &CodePointRange{From: rune(92992), To: rune(92995)}, + &CodePointRange{From: rune(94099), To: rune(94111)}, + &CodePointRange{From: rune(94176), To: rune(94177)}, + &CodePointRange{From: rune(94179), To: rune(94179)}, + &CodePointRange{From: rune(123191), To: rune(123197)}, + &CodePointRange{From: rune(125259), To: rune(125259)}, + }, + "lo": { + &CodePointRange{From: rune(170), To: rune(170)}, + &CodePointRange{From: rune(186), To: rune(186)}, + &CodePointRange{From: rune(443), To: rune(443)}, + &CodePointRange{From: rune(448), To: rune(451)}, + &CodePointRange{From: rune(660), To: rune(660)}, + &CodePointRange{From: rune(1488), To: rune(1514)}, + &CodePointRange{From: rune(1519), To: rune(1522)}, + &CodePointRange{From: rune(1568), To: rune(1599)}, + &CodePointRange{From: rune(1601), To: rune(1610)}, + &CodePointRange{From: rune(1646), To: rune(1647)}, + &CodePointRange{From: rune(1649), To: rune(1747)}, + &CodePointRange{From: rune(1749), To: rune(1749)}, + &CodePointRange{From: rune(1774), To: rune(1775)}, + &CodePointRange{From: rune(1786), To: rune(1788)}, + &CodePointRange{From: rune(1791), To: rune(1791)}, + &CodePointRange{From: rune(1808), To: rune(1808)}, + &CodePointRange{From: rune(1810), To: rune(1839)}, + &CodePointRange{From: rune(1869), To: rune(1957)}, + &CodePointRange{From: rune(1969), To: rune(1969)}, + &CodePointRange{From: rune(1994), To: rune(2026)}, + &CodePointRange{From: rune(2048), To: rune(2069)}, + &CodePointRange{From: rune(2112), To: rune(2136)}, + &CodePointRange{From: rune(2144), To: rune(2154)}, + &CodePointRange{From: rune(2208), To: rune(2228)}, + &CodePointRange{From: rune(2230), To: rune(2247)}, + &CodePointRange{From: rune(2308), To: rune(2361)}, + &CodePointRange{From: rune(2365), To: rune(2365)}, + &CodePointRange{From: rune(2384), To: rune(2384)}, + &CodePointRange{From: rune(2392), To: rune(2401)}, + &CodePointRange{From: rune(2418), To: rune(2432)}, + &CodePointRange{From: rune(2437), To: rune(2444)}, + &CodePointRange{From: rune(2447), To: rune(2448)}, + &CodePointRange{From: rune(2451), To: rune(2472)}, + &CodePointRange{From: rune(2474), To: rune(2480)}, + &CodePointRange{From: rune(2482), To: rune(2482)}, + &CodePointRange{From: rune(2486), To: rune(2489)}, + &CodePointRange{From: rune(2493), To: rune(2493)}, + &CodePointRange{From: rune(2510), To: rune(2510)}, + &CodePointRange{From: rune(2524), To: rune(2525)}, + &CodePointRange{From: rune(2527), To: rune(2529)}, + &CodePointRange{From: rune(2544), To: rune(2545)}, + &CodePointRange{From: rune(2556), To: rune(2556)}, + &CodePointRange{From: rune(2565), To: rune(2570)}, + &CodePointRange{From: rune(2575), To: rune(2576)}, + &CodePointRange{From: rune(2579), To: rune(2600)}, + &CodePointRange{From: rune(2602), To: rune(2608)}, + &CodePointRange{From: rune(2610), To: rune(2611)}, + &CodePointRange{From: rune(2613), To: rune(2614)}, + &CodePointRange{From: rune(2616), To: rune(2617)}, + &CodePointRange{From: rune(2649), To: rune(2652)}, + &CodePointRange{From: rune(2654), To: rune(2654)}, + &CodePointRange{From: rune(2674), To: rune(2676)}, + &CodePointRange{From: rune(2693), To: rune(2701)}, + &CodePointRange{From: rune(2703), To: rune(2705)}, + &CodePointRange{From: rune(2707), To: rune(2728)}, + &CodePointRange{From: rune(2730), To: rune(2736)}, + &CodePointRange{From: rune(2738), To: rune(2739)}, + &CodePointRange{From: rune(2741), To: rune(2745)}, + &CodePointRange{From: rune(2749), To: rune(2749)}, + &CodePointRange{From: rune(2768), To: rune(2768)}, + &CodePointRange{From: rune(2784), To: rune(2785)}, + &CodePointRange{From: rune(2809), To: rune(2809)}, + &CodePointRange{From: rune(2821), To: rune(2828)}, + &CodePointRange{From: rune(2831), To: rune(2832)}, + &CodePointRange{From: rune(2835), To: rune(2856)}, + &CodePointRange{From: rune(2858), To: rune(2864)}, + &CodePointRange{From: rune(2866), To: rune(2867)}, + &CodePointRange{From: rune(2869), To: rune(2873)}, + &CodePointRange{From: rune(2877), To: rune(2877)}, + &CodePointRange{From: rune(2908), To: rune(2909)}, + &CodePointRange{From: rune(2911), To: rune(2913)}, + &CodePointRange{From: rune(2929), To: rune(2929)}, + &CodePointRange{From: rune(2947), To: rune(2947)}, + &CodePointRange{From: rune(2949), To: rune(2954)}, + &CodePointRange{From: rune(2958), To: rune(2960)}, + &CodePointRange{From: rune(2962), To: rune(2965)}, + &CodePointRange{From: rune(2969), To: rune(2970)}, + &CodePointRange{From: rune(2972), To: rune(2972)}, + &CodePointRange{From: rune(2974), To: rune(2975)}, + &CodePointRange{From: rune(2979), To: rune(2980)}, + &CodePointRange{From: rune(2984), To: rune(2986)}, + &CodePointRange{From: rune(2990), To: rune(3001)}, + &CodePointRange{From: rune(3024), To: rune(3024)}, + &CodePointRange{From: rune(3077), To: rune(3084)}, + &CodePointRange{From: rune(3086), To: rune(3088)}, + &CodePointRange{From: rune(3090), To: rune(3112)}, + &CodePointRange{From: rune(3114), To: rune(3129)}, + &CodePointRange{From: rune(3133), To: rune(3133)}, + &CodePointRange{From: rune(3160), To: rune(3162)}, + &CodePointRange{From: rune(3168), To: rune(3169)}, + &CodePointRange{From: rune(3200), To: rune(3200)}, + &CodePointRange{From: rune(3205), To: rune(3212)}, + &CodePointRange{From: rune(3214), To: rune(3216)}, + &CodePointRange{From: rune(3218), To: rune(3240)}, + &CodePointRange{From: rune(3242), To: rune(3251)}, + &CodePointRange{From: rune(3253), To: rune(3257)}, + &CodePointRange{From: rune(3261), To: rune(3261)}, + &CodePointRange{From: rune(3294), To: rune(3294)}, + &CodePointRange{From: rune(3296), To: rune(3297)}, + &CodePointRange{From: rune(3313), To: rune(3314)}, + &CodePointRange{From: rune(3332), To: rune(3340)}, + &CodePointRange{From: rune(3342), To: rune(3344)}, + &CodePointRange{From: rune(3346), To: rune(3386)}, + &CodePointRange{From: rune(3389), To: rune(3389)}, + &CodePointRange{From: rune(3406), To: rune(3406)}, + &CodePointRange{From: rune(3412), To: rune(3414)}, + &CodePointRange{From: rune(3423), To: rune(3425)}, + &CodePointRange{From: rune(3450), To: rune(3455)}, + &CodePointRange{From: rune(3461), To: rune(3478)}, + &CodePointRange{From: rune(3482), To: rune(3505)}, + &CodePointRange{From: rune(3507), To: rune(3515)}, + &CodePointRange{From: rune(3517), To: rune(3517)}, + &CodePointRange{From: rune(3520), To: rune(3526)}, + &CodePointRange{From: rune(3585), To: rune(3632)}, + &CodePointRange{From: rune(3634), To: rune(3635)}, + &CodePointRange{From: rune(3648), To: rune(3653)}, + &CodePointRange{From: rune(3713), To: rune(3714)}, + &CodePointRange{From: rune(3716), To: rune(3716)}, + &CodePointRange{From: rune(3718), To: rune(3722)}, + &CodePointRange{From: rune(3724), To: rune(3747)}, + &CodePointRange{From: rune(3749), To: rune(3749)}, + &CodePointRange{From: rune(3751), To: rune(3760)}, + &CodePointRange{From: rune(3762), To: rune(3763)}, + &CodePointRange{From: rune(3773), To: rune(3773)}, + &CodePointRange{From: rune(3776), To: rune(3780)}, + &CodePointRange{From: rune(3804), To: rune(3807)}, + &CodePointRange{From: rune(3840), To: rune(3840)}, + &CodePointRange{From: rune(3904), To: rune(3911)}, + &CodePointRange{From: rune(3913), To: rune(3948)}, + &CodePointRange{From: rune(3976), To: rune(3980)}, + &CodePointRange{From: rune(4096), To: rune(4138)}, + &CodePointRange{From: rune(4159), To: rune(4159)}, + &CodePointRange{From: rune(4176), To: rune(4181)}, + &CodePointRange{From: rune(4186), To: rune(4189)}, + &CodePointRange{From: rune(4193), To: rune(4193)}, + &CodePointRange{From: rune(4197), To: rune(4198)}, + &CodePointRange{From: rune(4206), To: rune(4208)}, + &CodePointRange{From: rune(4213), To: rune(4225)}, + &CodePointRange{From: rune(4238), To: rune(4238)}, + &CodePointRange{From: rune(4352), To: rune(4680)}, + &CodePointRange{From: rune(4682), To: rune(4685)}, + &CodePointRange{From: rune(4688), To: rune(4694)}, + &CodePointRange{From: rune(4696), To: rune(4696)}, + &CodePointRange{From: rune(4698), To: rune(4701)}, + &CodePointRange{From: rune(4704), To: rune(4744)}, + &CodePointRange{From: rune(4746), To: rune(4749)}, + &CodePointRange{From: rune(4752), To: rune(4784)}, + &CodePointRange{From: rune(4786), To: rune(4789)}, + &CodePointRange{From: rune(4792), To: rune(4798)}, + &CodePointRange{From: rune(4800), To: rune(4800)}, + &CodePointRange{From: rune(4802), To: rune(4805)}, + &CodePointRange{From: rune(4808), To: rune(4822)}, + &CodePointRange{From: rune(4824), To: rune(4880)}, + &CodePointRange{From: rune(4882), To: rune(4885)}, + &CodePointRange{From: rune(4888), To: rune(4954)}, + &CodePointRange{From: rune(4992), To: rune(5007)}, + &CodePointRange{From: rune(5121), To: rune(5740)}, + &CodePointRange{From: rune(5743), To: rune(5759)}, + &CodePointRange{From: rune(5761), To: rune(5786)}, + &CodePointRange{From: rune(5792), To: rune(5866)}, + &CodePointRange{From: rune(5873), To: rune(5880)}, + &CodePointRange{From: rune(5888), To: rune(5900)}, + &CodePointRange{From: rune(5902), To: rune(5905)}, + &CodePointRange{From: rune(5920), To: rune(5937)}, + &CodePointRange{From: rune(5952), To: rune(5969)}, + &CodePointRange{From: rune(5984), To: rune(5996)}, + &CodePointRange{From: rune(5998), To: rune(6000)}, + &CodePointRange{From: rune(6016), To: rune(6067)}, + &CodePointRange{From: rune(6108), To: rune(6108)}, + &CodePointRange{From: rune(6176), To: rune(6210)}, + &CodePointRange{From: rune(6212), To: rune(6264)}, + &CodePointRange{From: rune(6272), To: rune(6276)}, + &CodePointRange{From: rune(6279), To: rune(6312)}, + &CodePointRange{From: rune(6314), To: rune(6314)}, + &CodePointRange{From: rune(6320), To: rune(6389)}, + &CodePointRange{From: rune(6400), To: rune(6430)}, + &CodePointRange{From: rune(6480), To: rune(6509)}, + &CodePointRange{From: rune(6512), To: rune(6516)}, + &CodePointRange{From: rune(6528), To: rune(6571)}, + &CodePointRange{From: rune(6576), To: rune(6601)}, + &CodePointRange{From: rune(6656), To: rune(6678)}, + &CodePointRange{From: rune(6688), To: rune(6740)}, + &CodePointRange{From: rune(6917), To: rune(6963)}, + &CodePointRange{From: rune(6981), To: rune(6987)}, + &CodePointRange{From: rune(7043), To: rune(7072)}, + &CodePointRange{From: rune(7086), To: rune(7087)}, + &CodePointRange{From: rune(7098), To: rune(7141)}, + &CodePointRange{From: rune(7168), To: rune(7203)}, + &CodePointRange{From: rune(7245), To: rune(7247)}, + &CodePointRange{From: rune(7258), To: rune(7287)}, + &CodePointRange{From: rune(7401), To: rune(7404)}, + &CodePointRange{From: rune(7406), To: rune(7411)}, + &CodePointRange{From: rune(7413), To: rune(7414)}, + &CodePointRange{From: rune(7418), To: rune(7418)}, + &CodePointRange{From: rune(8501), To: rune(8504)}, + &CodePointRange{From: rune(11568), To: rune(11623)}, + &CodePointRange{From: rune(11648), To: rune(11670)}, + &CodePointRange{From: rune(11680), To: rune(11686)}, + &CodePointRange{From: rune(11688), To: rune(11694)}, + &CodePointRange{From: rune(11696), To: rune(11702)}, + &CodePointRange{From: rune(11704), To: rune(11710)}, + &CodePointRange{From: rune(11712), To: rune(11718)}, + &CodePointRange{From: rune(11720), To: rune(11726)}, + &CodePointRange{From: rune(11728), To: rune(11734)}, + &CodePointRange{From: rune(11736), To: rune(11742)}, + &CodePointRange{From: rune(12294), To: rune(12294)}, + &CodePointRange{From: rune(12348), To: rune(12348)}, + &CodePointRange{From: rune(12353), To: rune(12438)}, + &CodePointRange{From: rune(12447), To: rune(12447)}, + &CodePointRange{From: rune(12449), To: rune(12538)}, + &CodePointRange{From: rune(12543), To: rune(12543)}, + &CodePointRange{From: rune(12549), To: rune(12591)}, + &CodePointRange{From: rune(12593), To: rune(12686)}, + &CodePointRange{From: rune(12704), To: rune(12735)}, + &CodePointRange{From: rune(12784), To: rune(12799)}, + &CodePointRange{From: rune(13312), To: rune(13312)}, + &CodePointRange{From: rune(19903), To: rune(19903)}, + &CodePointRange{From: rune(19968), To: rune(19968)}, + &CodePointRange{From: rune(40956), To: rune(40956)}, + &CodePointRange{From: rune(40960), To: rune(40980)}, + &CodePointRange{From: rune(40982), To: rune(42124)}, + &CodePointRange{From: rune(42192), To: rune(42231)}, + &CodePointRange{From: rune(42240), To: rune(42507)}, + &CodePointRange{From: rune(42512), To: rune(42527)}, + &CodePointRange{From: rune(42538), To: rune(42539)}, + &CodePointRange{From: rune(42606), To: rune(42606)}, + &CodePointRange{From: rune(42656), To: rune(42725)}, + &CodePointRange{From: rune(42895), To: rune(42895)}, + &CodePointRange{From: rune(42999), To: rune(42999)}, + &CodePointRange{From: rune(43003), To: rune(43009)}, + &CodePointRange{From: rune(43011), To: rune(43013)}, + &CodePointRange{From: rune(43015), To: rune(43018)}, + &CodePointRange{From: rune(43020), To: rune(43042)}, + &CodePointRange{From: rune(43072), To: rune(43123)}, + &CodePointRange{From: rune(43138), To: rune(43187)}, + &CodePointRange{From: rune(43250), To: rune(43255)}, + &CodePointRange{From: rune(43259), To: rune(43259)}, + &CodePointRange{From: rune(43261), To: rune(43262)}, + &CodePointRange{From: rune(43274), To: rune(43301)}, + &CodePointRange{From: rune(43312), To: rune(43334)}, + &CodePointRange{From: rune(43360), To: rune(43388)}, + &CodePointRange{From: rune(43396), To: rune(43442)}, + &CodePointRange{From: rune(43488), To: rune(43492)}, + &CodePointRange{From: rune(43495), To: rune(43503)}, + &CodePointRange{From: rune(43514), To: rune(43518)}, + &CodePointRange{From: rune(43520), To: rune(43560)}, + &CodePointRange{From: rune(43584), To: rune(43586)}, + &CodePointRange{From: rune(43588), To: rune(43595)}, + &CodePointRange{From: rune(43616), To: rune(43631)}, + &CodePointRange{From: rune(43633), To: rune(43638)}, + &CodePointRange{From: rune(43642), To: rune(43642)}, + &CodePointRange{From: rune(43646), To: rune(43695)}, + &CodePointRange{From: rune(43697), To: rune(43697)}, + &CodePointRange{From: rune(43701), To: rune(43702)}, + &CodePointRange{From: rune(43705), To: rune(43709)}, + &CodePointRange{From: rune(43712), To: rune(43712)}, + &CodePointRange{From: rune(43714), To: rune(43714)}, + &CodePointRange{From: rune(43739), To: rune(43740)}, + &CodePointRange{From: rune(43744), To: rune(43754)}, + &CodePointRange{From: rune(43762), To: rune(43762)}, + &CodePointRange{From: rune(43777), To: rune(43782)}, + &CodePointRange{From: rune(43785), To: rune(43790)}, + &CodePointRange{From: rune(43793), To: rune(43798)}, + &CodePointRange{From: rune(43808), To: rune(43814)}, + &CodePointRange{From: rune(43816), To: rune(43822)}, + &CodePointRange{From: rune(43968), To: rune(44002)}, + &CodePointRange{From: rune(44032), To: rune(44032)}, + &CodePointRange{From: rune(55203), To: rune(55203)}, + &CodePointRange{From: rune(55216), To: rune(55238)}, + &CodePointRange{From: rune(55243), To: rune(55291)}, + &CodePointRange{From: rune(63744), To: rune(64109)}, + &CodePointRange{From: rune(64112), To: rune(64217)}, + &CodePointRange{From: rune(64285), To: rune(64285)}, + &CodePointRange{From: rune(64287), To: rune(64296)}, + &CodePointRange{From: rune(64298), To: rune(64310)}, + &CodePointRange{From: rune(64312), To: rune(64316)}, + &CodePointRange{From: rune(64318), To: rune(64318)}, + &CodePointRange{From: rune(64320), To: rune(64321)}, + &CodePointRange{From: rune(64323), To: rune(64324)}, + &CodePointRange{From: rune(64326), To: rune(64433)}, + &CodePointRange{From: rune(64467), To: rune(64829)}, + &CodePointRange{From: rune(64848), To: rune(64911)}, + &CodePointRange{From: rune(64914), To: rune(64967)}, + &CodePointRange{From: rune(65008), To: rune(65019)}, + &CodePointRange{From: rune(65136), To: rune(65140)}, + &CodePointRange{From: rune(65142), To: rune(65276)}, + &CodePointRange{From: rune(65382), To: rune(65391)}, + &CodePointRange{From: rune(65393), To: rune(65437)}, + &CodePointRange{From: rune(65440), To: rune(65470)}, + &CodePointRange{From: rune(65474), To: rune(65479)}, + &CodePointRange{From: rune(65482), To: rune(65487)}, + &CodePointRange{From: rune(65490), To: rune(65495)}, + &CodePointRange{From: rune(65498), To: rune(65500)}, + &CodePointRange{From: rune(65536), To: rune(65547)}, + &CodePointRange{From: rune(65549), To: rune(65574)}, + &CodePointRange{From: rune(65576), To: rune(65594)}, + &CodePointRange{From: rune(65596), To: rune(65597)}, + &CodePointRange{From: rune(65599), To: rune(65613)}, + &CodePointRange{From: rune(65616), To: rune(65629)}, + &CodePointRange{From: rune(65664), To: rune(65786)}, + &CodePointRange{From: rune(66176), To: rune(66204)}, + &CodePointRange{From: rune(66208), To: rune(66256)}, + &CodePointRange{From: rune(66304), To: rune(66335)}, + &CodePointRange{From: rune(66349), To: rune(66368)}, + &CodePointRange{From: rune(66370), To: rune(66377)}, + &CodePointRange{From: rune(66384), To: rune(66421)}, + &CodePointRange{From: rune(66432), To: rune(66461)}, + &CodePointRange{From: rune(66464), To: rune(66499)}, + &CodePointRange{From: rune(66504), To: rune(66511)}, + &CodePointRange{From: rune(66640), To: rune(66717)}, + &CodePointRange{From: rune(66816), To: rune(66855)}, + &CodePointRange{From: rune(66864), To: rune(66915)}, + &CodePointRange{From: rune(67072), To: rune(67382)}, + &CodePointRange{From: rune(67392), To: rune(67413)}, + &CodePointRange{From: rune(67424), To: rune(67431)}, + &CodePointRange{From: rune(67584), To: rune(67589)}, + &CodePointRange{From: rune(67592), To: rune(67592)}, + &CodePointRange{From: rune(67594), To: rune(67637)}, + &CodePointRange{From: rune(67639), To: rune(67640)}, + &CodePointRange{From: rune(67644), To: rune(67644)}, + &CodePointRange{From: rune(67647), To: rune(67669)}, + &CodePointRange{From: rune(67680), To: rune(67702)}, + &CodePointRange{From: rune(67712), To: rune(67742)}, + &CodePointRange{From: rune(67808), To: rune(67826)}, + &CodePointRange{From: rune(67828), To: rune(67829)}, + &CodePointRange{From: rune(67840), To: rune(67861)}, + &CodePointRange{From: rune(67872), To: rune(67897)}, + &CodePointRange{From: rune(67968), To: rune(68023)}, + &CodePointRange{From: rune(68030), To: rune(68031)}, + &CodePointRange{From: rune(68096), To: rune(68096)}, + &CodePointRange{From: rune(68112), To: rune(68115)}, + &CodePointRange{From: rune(68117), To: rune(68119)}, + &CodePointRange{From: rune(68121), To: rune(68149)}, + &CodePointRange{From: rune(68192), To: rune(68220)}, + &CodePointRange{From: rune(68224), To: rune(68252)}, + &CodePointRange{From: rune(68288), To: rune(68295)}, + &CodePointRange{From: rune(68297), To: rune(68324)}, + &CodePointRange{From: rune(68352), To: rune(68405)}, + &CodePointRange{From: rune(68416), To: rune(68437)}, + &CodePointRange{From: rune(68448), To: rune(68466)}, + &CodePointRange{From: rune(68480), To: rune(68497)}, + &CodePointRange{From: rune(68608), To: rune(68680)}, + &CodePointRange{From: rune(68864), To: rune(68899)}, + &CodePointRange{From: rune(69248), To: rune(69289)}, + &CodePointRange{From: rune(69296), To: rune(69297)}, + &CodePointRange{From: rune(69376), To: rune(69404)}, + &CodePointRange{From: rune(69415), To: rune(69415)}, + &CodePointRange{From: rune(69424), To: rune(69445)}, + &CodePointRange{From: rune(69552), To: rune(69572)}, + &CodePointRange{From: rune(69600), To: rune(69622)}, + &CodePointRange{From: rune(69635), To: rune(69687)}, + &CodePointRange{From: rune(69763), To: rune(69807)}, + &CodePointRange{From: rune(69840), To: rune(69864)}, + &CodePointRange{From: rune(69891), To: rune(69926)}, + &CodePointRange{From: rune(69956), To: rune(69956)}, + &CodePointRange{From: rune(69959), To: rune(69959)}, + &CodePointRange{From: rune(69968), To: rune(70002)}, + &CodePointRange{From: rune(70006), To: rune(70006)}, + &CodePointRange{From: rune(70019), To: rune(70066)}, + &CodePointRange{From: rune(70081), To: rune(70084)}, + &CodePointRange{From: rune(70106), To: rune(70106)}, + &CodePointRange{From: rune(70108), To: rune(70108)}, + &CodePointRange{From: rune(70144), To: rune(70161)}, + &CodePointRange{From: rune(70163), To: rune(70187)}, + &CodePointRange{From: rune(70272), To: rune(70278)}, + &CodePointRange{From: rune(70280), To: rune(70280)}, + &CodePointRange{From: rune(70282), To: rune(70285)}, + &CodePointRange{From: rune(70287), To: rune(70301)}, + &CodePointRange{From: rune(70303), To: rune(70312)}, + &CodePointRange{From: rune(70320), To: rune(70366)}, + &CodePointRange{From: rune(70405), To: rune(70412)}, + &CodePointRange{From: rune(70415), To: rune(70416)}, + &CodePointRange{From: rune(70419), To: rune(70440)}, + &CodePointRange{From: rune(70442), To: rune(70448)}, + &CodePointRange{From: rune(70450), To: rune(70451)}, + &CodePointRange{From: rune(70453), To: rune(70457)}, + &CodePointRange{From: rune(70461), To: rune(70461)}, + &CodePointRange{From: rune(70480), To: rune(70480)}, + &CodePointRange{From: rune(70493), To: rune(70497)}, + &CodePointRange{From: rune(70656), To: rune(70708)}, + &CodePointRange{From: rune(70727), To: rune(70730)}, + &CodePointRange{From: rune(70751), To: rune(70753)}, + &CodePointRange{From: rune(70784), To: rune(70831)}, + &CodePointRange{From: rune(70852), To: rune(70853)}, + &CodePointRange{From: rune(70855), To: rune(70855)}, + &CodePointRange{From: rune(71040), To: rune(71086)}, + &CodePointRange{From: rune(71128), To: rune(71131)}, + &CodePointRange{From: rune(71168), To: rune(71215)}, + &CodePointRange{From: rune(71236), To: rune(71236)}, + &CodePointRange{From: rune(71296), To: rune(71338)}, + &CodePointRange{From: rune(71352), To: rune(71352)}, + &CodePointRange{From: rune(71424), To: rune(71450)}, + &CodePointRange{From: rune(71680), To: rune(71723)}, + &CodePointRange{From: rune(71935), To: rune(71942)}, + &CodePointRange{From: rune(71945), To: rune(71945)}, + &CodePointRange{From: rune(71948), To: rune(71955)}, + &CodePointRange{From: rune(71957), To: rune(71958)}, + &CodePointRange{From: rune(71960), To: rune(71983)}, + &CodePointRange{From: rune(71999), To: rune(71999)}, + &CodePointRange{From: rune(72001), To: rune(72001)}, + &CodePointRange{From: rune(72096), To: rune(72103)}, + &CodePointRange{From: rune(72106), To: rune(72144)}, + &CodePointRange{From: rune(72161), To: rune(72161)}, + &CodePointRange{From: rune(72163), To: rune(72163)}, + &CodePointRange{From: rune(72192), To: rune(72192)}, + &CodePointRange{From: rune(72203), To: rune(72242)}, + &CodePointRange{From: rune(72250), To: rune(72250)}, + &CodePointRange{From: rune(72272), To: rune(72272)}, + &CodePointRange{From: rune(72284), To: rune(72329)}, + &CodePointRange{From: rune(72349), To: rune(72349)}, + &CodePointRange{From: rune(72384), To: rune(72440)}, + &CodePointRange{From: rune(72704), To: rune(72712)}, + &CodePointRange{From: rune(72714), To: rune(72750)}, + &CodePointRange{From: rune(72768), To: rune(72768)}, + &CodePointRange{From: rune(72818), To: rune(72847)}, + &CodePointRange{From: rune(72960), To: rune(72966)}, + &CodePointRange{From: rune(72968), To: rune(72969)}, + &CodePointRange{From: rune(72971), To: rune(73008)}, + &CodePointRange{From: rune(73030), To: rune(73030)}, + &CodePointRange{From: rune(73056), To: rune(73061)}, + &CodePointRange{From: rune(73063), To: rune(73064)}, + &CodePointRange{From: rune(73066), To: rune(73097)}, + &CodePointRange{From: rune(73112), To: rune(73112)}, + &CodePointRange{From: rune(73440), To: rune(73458)}, + &CodePointRange{From: rune(73648), To: rune(73648)}, + &CodePointRange{From: rune(73728), To: rune(74649)}, + &CodePointRange{From: rune(74880), To: rune(75075)}, + &CodePointRange{From: rune(77824), To: rune(78894)}, + &CodePointRange{From: rune(82944), To: rune(83526)}, + &CodePointRange{From: rune(92160), To: rune(92728)}, + &CodePointRange{From: rune(92736), To: rune(92766)}, + &CodePointRange{From: rune(92880), To: rune(92909)}, + &CodePointRange{From: rune(92928), To: rune(92975)}, + &CodePointRange{From: rune(93027), To: rune(93047)}, + &CodePointRange{From: rune(93053), To: rune(93071)}, + &CodePointRange{From: rune(93952), To: rune(94026)}, + &CodePointRange{From: rune(94032), To: rune(94032)}, + &CodePointRange{From: rune(94208), To: rune(94208)}, + &CodePointRange{From: rune(100343), To: rune(100343)}, + &CodePointRange{From: rune(100352), To: rune(101589)}, + &CodePointRange{From: rune(101632), To: rune(101632)}, + &CodePointRange{From: rune(101640), To: rune(101640)}, + &CodePointRange{From: rune(110592), To: rune(110878)}, + &CodePointRange{From: rune(110928), To: rune(110930)}, + &CodePointRange{From: rune(110948), To: rune(110951)}, + &CodePointRange{From: rune(110960), To: rune(111355)}, + &CodePointRange{From: rune(113664), To: rune(113770)}, + &CodePointRange{From: rune(113776), To: rune(113788)}, + &CodePointRange{From: rune(113792), To: rune(113800)}, + &CodePointRange{From: rune(113808), To: rune(113817)}, + &CodePointRange{From: rune(123136), To: rune(123180)}, + &CodePointRange{From: rune(123214), To: rune(123214)}, + &CodePointRange{From: rune(123584), To: rune(123627)}, + &CodePointRange{From: rune(124928), To: rune(125124)}, + &CodePointRange{From: rune(126464), To: rune(126467)}, + &CodePointRange{From: rune(126469), To: rune(126495)}, + &CodePointRange{From: rune(126497), To: rune(126498)}, + &CodePointRange{From: rune(126500), To: rune(126500)}, + &CodePointRange{From: rune(126503), To: rune(126503)}, + &CodePointRange{From: rune(126505), To: rune(126514)}, + &CodePointRange{From: rune(126516), To: rune(126519)}, + &CodePointRange{From: rune(126521), To: rune(126521)}, + &CodePointRange{From: rune(126523), To: rune(126523)}, + &CodePointRange{From: rune(126530), To: rune(126530)}, + &CodePointRange{From: rune(126535), To: rune(126535)}, + &CodePointRange{From: rune(126537), To: rune(126537)}, + &CodePointRange{From: rune(126539), To: rune(126539)}, + &CodePointRange{From: rune(126541), To: rune(126543)}, + &CodePointRange{From: rune(126545), To: rune(126546)}, + &CodePointRange{From: rune(126548), To: rune(126548)}, + &CodePointRange{From: rune(126551), To: rune(126551)}, + &CodePointRange{From: rune(126553), To: rune(126553)}, + &CodePointRange{From: rune(126555), To: rune(126555)}, + &CodePointRange{From: rune(126557), To: rune(126557)}, + &CodePointRange{From: rune(126559), To: rune(126559)}, + &CodePointRange{From: rune(126561), To: rune(126562)}, + &CodePointRange{From: rune(126564), To: rune(126564)}, + &CodePointRange{From: rune(126567), To: rune(126570)}, + &CodePointRange{From: rune(126572), To: rune(126578)}, + &CodePointRange{From: rune(126580), To: rune(126583)}, + &CodePointRange{From: rune(126585), To: rune(126588)}, + &CodePointRange{From: rune(126590), To: rune(126590)}, + &CodePointRange{From: rune(126592), To: rune(126601)}, + &CodePointRange{From: rune(126603), To: rune(126619)}, + &CodePointRange{From: rune(126625), To: rune(126627)}, + &CodePointRange{From: rune(126629), To: rune(126633)}, + &CodePointRange{From: rune(126635), To: rune(126651)}, + &CodePointRange{From: rune(131072), To: rune(131072)}, + &CodePointRange{From: rune(173789), To: rune(173789)}, + &CodePointRange{From: rune(173824), To: rune(173824)}, + &CodePointRange{From: rune(177972), To: rune(177972)}, + &CodePointRange{From: rune(177984), To: rune(177984)}, + &CodePointRange{From: rune(178205), To: rune(178205)}, + &CodePointRange{From: rune(178208), To: rune(178208)}, + &CodePointRange{From: rune(183969), To: rune(183969)}, + &CodePointRange{From: rune(183984), To: rune(183984)}, + &CodePointRange{From: rune(191456), To: rune(191456)}, + &CodePointRange{From: rune(194560), To: rune(195101)}, + &CodePointRange{From: rune(196608), To: rune(196608)}, + &CodePointRange{From: rune(201546), To: rune(201546)}, + }, + "lt": { + &CodePointRange{From: rune(453), To: rune(453)}, + &CodePointRange{From: rune(456), To: rune(456)}, + &CodePointRange{From: rune(459), To: rune(459)}, + &CodePointRange{From: rune(498), To: rune(498)}, + &CodePointRange{From: rune(8072), To: rune(8079)}, + &CodePointRange{From: rune(8088), To: rune(8095)}, + &CodePointRange{From: rune(8104), To: rune(8111)}, + &CodePointRange{From: rune(8124), To: rune(8124)}, + &CodePointRange{From: rune(8140), To: rune(8140)}, + &CodePointRange{From: rune(8188), To: rune(8188)}, + }, + "lu": { + &CodePointRange{From: rune(65), To: rune(90)}, + &CodePointRange{From: rune(192), To: rune(214)}, + &CodePointRange{From: rune(216), To: rune(222)}, + &CodePointRange{From: rune(256), To: rune(256)}, + &CodePointRange{From: rune(258), To: rune(258)}, + &CodePointRange{From: rune(260), To: rune(260)}, + &CodePointRange{From: rune(262), To: rune(262)}, + &CodePointRange{From: rune(264), To: rune(264)}, + &CodePointRange{From: rune(266), To: rune(266)}, + &CodePointRange{From: rune(268), To: rune(268)}, + &CodePointRange{From: rune(270), To: rune(270)}, + &CodePointRange{From: rune(272), To: rune(272)}, + &CodePointRange{From: rune(274), To: rune(274)}, + &CodePointRange{From: rune(276), To: rune(276)}, + &CodePointRange{From: rune(278), To: rune(278)}, + &CodePointRange{From: rune(280), To: rune(280)}, + &CodePointRange{From: rune(282), To: rune(282)}, + &CodePointRange{From: rune(284), To: rune(284)}, + &CodePointRange{From: rune(286), To: rune(286)}, + &CodePointRange{From: rune(288), To: rune(288)}, + &CodePointRange{From: rune(290), To: rune(290)}, + &CodePointRange{From: rune(292), To: rune(292)}, + &CodePointRange{From: rune(294), To: rune(294)}, + &CodePointRange{From: rune(296), To: rune(296)}, + &CodePointRange{From: rune(298), To: rune(298)}, + &CodePointRange{From: rune(300), To: rune(300)}, + &CodePointRange{From: rune(302), To: rune(302)}, + &CodePointRange{From: rune(304), To: rune(304)}, + &CodePointRange{From: rune(306), To: rune(306)}, + &CodePointRange{From: rune(308), To: rune(308)}, + &CodePointRange{From: rune(310), To: rune(310)}, + &CodePointRange{From: rune(313), To: rune(313)}, + &CodePointRange{From: rune(315), To: rune(315)}, + &CodePointRange{From: rune(317), To: rune(317)}, + &CodePointRange{From: rune(319), To: rune(319)}, + &CodePointRange{From: rune(321), To: rune(321)}, + &CodePointRange{From: rune(323), To: rune(323)}, + &CodePointRange{From: rune(325), To: rune(325)}, + &CodePointRange{From: rune(327), To: rune(327)}, + &CodePointRange{From: rune(330), To: rune(330)}, + &CodePointRange{From: rune(332), To: rune(332)}, + &CodePointRange{From: rune(334), To: rune(334)}, + &CodePointRange{From: rune(336), To: rune(336)}, + &CodePointRange{From: rune(338), To: rune(338)}, + &CodePointRange{From: rune(340), To: rune(340)}, + &CodePointRange{From: rune(342), To: rune(342)}, + &CodePointRange{From: rune(344), To: rune(344)}, + &CodePointRange{From: rune(346), To: rune(346)}, + &CodePointRange{From: rune(348), To: rune(348)}, + &CodePointRange{From: rune(350), To: rune(350)}, + &CodePointRange{From: rune(352), To: rune(352)}, + &CodePointRange{From: rune(354), To: rune(354)}, + &CodePointRange{From: rune(356), To: rune(356)}, + &CodePointRange{From: rune(358), To: rune(358)}, + &CodePointRange{From: rune(360), To: rune(360)}, + &CodePointRange{From: rune(362), To: rune(362)}, + &CodePointRange{From: rune(364), To: rune(364)}, + &CodePointRange{From: rune(366), To: rune(366)}, + &CodePointRange{From: rune(368), To: rune(368)}, + &CodePointRange{From: rune(370), To: rune(370)}, + &CodePointRange{From: rune(372), To: rune(372)}, + &CodePointRange{From: rune(374), To: rune(374)}, + &CodePointRange{From: rune(376), To: rune(377)}, + &CodePointRange{From: rune(379), To: rune(379)}, + &CodePointRange{From: rune(381), To: rune(381)}, + &CodePointRange{From: rune(385), To: rune(386)}, + &CodePointRange{From: rune(388), To: rune(388)}, + &CodePointRange{From: rune(390), To: rune(391)}, + &CodePointRange{From: rune(393), To: rune(395)}, + &CodePointRange{From: rune(398), To: rune(401)}, + &CodePointRange{From: rune(403), To: rune(404)}, + &CodePointRange{From: rune(406), To: rune(408)}, + &CodePointRange{From: rune(412), To: rune(413)}, + &CodePointRange{From: rune(415), To: rune(416)}, + &CodePointRange{From: rune(418), To: rune(418)}, + &CodePointRange{From: rune(420), To: rune(420)}, + &CodePointRange{From: rune(422), To: rune(423)}, + &CodePointRange{From: rune(425), To: rune(425)}, + &CodePointRange{From: rune(428), To: rune(428)}, + &CodePointRange{From: rune(430), To: rune(431)}, + &CodePointRange{From: rune(433), To: rune(435)}, + &CodePointRange{From: rune(437), To: rune(437)}, + &CodePointRange{From: rune(439), To: rune(440)}, + &CodePointRange{From: rune(444), To: rune(444)}, + &CodePointRange{From: rune(452), To: rune(452)}, + &CodePointRange{From: rune(455), To: rune(455)}, + &CodePointRange{From: rune(458), To: rune(458)}, + &CodePointRange{From: rune(461), To: rune(461)}, + &CodePointRange{From: rune(463), To: rune(463)}, + &CodePointRange{From: rune(465), To: rune(465)}, + &CodePointRange{From: rune(467), To: rune(467)}, + &CodePointRange{From: rune(469), To: rune(469)}, + &CodePointRange{From: rune(471), To: rune(471)}, + &CodePointRange{From: rune(473), To: rune(473)}, + &CodePointRange{From: rune(475), To: rune(475)}, + &CodePointRange{From: rune(478), To: rune(478)}, + &CodePointRange{From: rune(480), To: rune(480)}, + &CodePointRange{From: rune(482), To: rune(482)}, + &CodePointRange{From: rune(484), To: rune(484)}, + &CodePointRange{From: rune(486), To: rune(486)}, + &CodePointRange{From: rune(488), To: rune(488)}, + &CodePointRange{From: rune(490), To: rune(490)}, + &CodePointRange{From: rune(492), To: rune(492)}, + &CodePointRange{From: rune(494), To: rune(494)}, + &CodePointRange{From: rune(497), To: rune(497)}, + &CodePointRange{From: rune(500), To: rune(500)}, + &CodePointRange{From: rune(502), To: rune(504)}, + &CodePointRange{From: rune(506), To: rune(506)}, + &CodePointRange{From: rune(508), To: rune(508)}, + &CodePointRange{From: rune(510), To: rune(510)}, + &CodePointRange{From: rune(512), To: rune(512)}, + &CodePointRange{From: rune(514), To: rune(514)}, + &CodePointRange{From: rune(516), To: rune(516)}, + &CodePointRange{From: rune(518), To: rune(518)}, + &CodePointRange{From: rune(520), To: rune(520)}, + &CodePointRange{From: rune(522), To: rune(522)}, + &CodePointRange{From: rune(524), To: rune(524)}, + &CodePointRange{From: rune(526), To: rune(526)}, + &CodePointRange{From: rune(528), To: rune(528)}, + &CodePointRange{From: rune(530), To: rune(530)}, + &CodePointRange{From: rune(532), To: rune(532)}, + &CodePointRange{From: rune(534), To: rune(534)}, + &CodePointRange{From: rune(536), To: rune(536)}, + &CodePointRange{From: rune(538), To: rune(538)}, + &CodePointRange{From: rune(540), To: rune(540)}, + &CodePointRange{From: rune(542), To: rune(542)}, + &CodePointRange{From: rune(544), To: rune(544)}, + &CodePointRange{From: rune(546), To: rune(546)}, + &CodePointRange{From: rune(548), To: rune(548)}, + &CodePointRange{From: rune(550), To: rune(550)}, + &CodePointRange{From: rune(552), To: rune(552)}, + &CodePointRange{From: rune(554), To: rune(554)}, + &CodePointRange{From: rune(556), To: rune(556)}, + &CodePointRange{From: rune(558), To: rune(558)}, + &CodePointRange{From: rune(560), To: rune(560)}, + &CodePointRange{From: rune(562), To: rune(562)}, + &CodePointRange{From: rune(570), To: rune(571)}, + &CodePointRange{From: rune(573), To: rune(574)}, + &CodePointRange{From: rune(577), To: rune(577)}, + &CodePointRange{From: rune(579), To: rune(582)}, + &CodePointRange{From: rune(584), To: rune(584)}, + &CodePointRange{From: rune(586), To: rune(586)}, + &CodePointRange{From: rune(588), To: rune(588)}, + &CodePointRange{From: rune(590), To: rune(590)}, + &CodePointRange{From: rune(880), To: rune(880)}, + &CodePointRange{From: rune(882), To: rune(882)}, + &CodePointRange{From: rune(886), To: rune(886)}, + &CodePointRange{From: rune(895), To: rune(895)}, + &CodePointRange{From: rune(902), To: rune(902)}, + &CodePointRange{From: rune(904), To: rune(906)}, + &CodePointRange{From: rune(908), To: rune(908)}, + &CodePointRange{From: rune(910), To: rune(911)}, + &CodePointRange{From: rune(913), To: rune(929)}, + &CodePointRange{From: rune(931), To: rune(939)}, + &CodePointRange{From: rune(975), To: rune(975)}, + &CodePointRange{From: rune(978), To: rune(980)}, + &CodePointRange{From: rune(984), To: rune(984)}, + &CodePointRange{From: rune(986), To: rune(986)}, + &CodePointRange{From: rune(988), To: rune(988)}, + &CodePointRange{From: rune(990), To: rune(990)}, + &CodePointRange{From: rune(992), To: rune(992)}, + &CodePointRange{From: rune(994), To: rune(994)}, + &CodePointRange{From: rune(996), To: rune(996)}, + &CodePointRange{From: rune(998), To: rune(998)}, + &CodePointRange{From: rune(1000), To: rune(1000)}, + &CodePointRange{From: rune(1002), To: rune(1002)}, + &CodePointRange{From: rune(1004), To: rune(1004)}, + &CodePointRange{From: rune(1006), To: rune(1006)}, + &CodePointRange{From: rune(1012), To: rune(1012)}, + &CodePointRange{From: rune(1015), To: rune(1015)}, + &CodePointRange{From: rune(1017), To: rune(1018)}, + &CodePointRange{From: rune(1021), To: rune(1071)}, + &CodePointRange{From: rune(1120), To: rune(1120)}, + &CodePointRange{From: rune(1122), To: rune(1122)}, + &CodePointRange{From: rune(1124), To: rune(1124)}, + &CodePointRange{From: rune(1126), To: rune(1126)}, + &CodePointRange{From: rune(1128), To: rune(1128)}, + &CodePointRange{From: rune(1130), To: rune(1130)}, + &CodePointRange{From: rune(1132), To: rune(1132)}, + &CodePointRange{From: rune(1134), To: rune(1134)}, + &CodePointRange{From: rune(1136), To: rune(1136)}, + &CodePointRange{From: rune(1138), To: rune(1138)}, + &CodePointRange{From: rune(1140), To: rune(1140)}, + &CodePointRange{From: rune(1142), To: rune(1142)}, + &CodePointRange{From: rune(1144), To: rune(1144)}, + &CodePointRange{From: rune(1146), To: rune(1146)}, + &CodePointRange{From: rune(1148), To: rune(1148)}, + &CodePointRange{From: rune(1150), To: rune(1150)}, + &CodePointRange{From: rune(1152), To: rune(1152)}, + &CodePointRange{From: rune(1162), To: rune(1162)}, + &CodePointRange{From: rune(1164), To: rune(1164)}, + &CodePointRange{From: rune(1166), To: rune(1166)}, + &CodePointRange{From: rune(1168), To: rune(1168)}, + &CodePointRange{From: rune(1170), To: rune(1170)}, + &CodePointRange{From: rune(1172), To: rune(1172)}, + &CodePointRange{From: rune(1174), To: rune(1174)}, + &CodePointRange{From: rune(1176), To: rune(1176)}, + &CodePointRange{From: rune(1178), To: rune(1178)}, + &CodePointRange{From: rune(1180), To: rune(1180)}, + &CodePointRange{From: rune(1182), To: rune(1182)}, + &CodePointRange{From: rune(1184), To: rune(1184)}, + &CodePointRange{From: rune(1186), To: rune(1186)}, + &CodePointRange{From: rune(1188), To: rune(1188)}, + &CodePointRange{From: rune(1190), To: rune(1190)}, + &CodePointRange{From: rune(1192), To: rune(1192)}, + &CodePointRange{From: rune(1194), To: rune(1194)}, + &CodePointRange{From: rune(1196), To: rune(1196)}, + &CodePointRange{From: rune(1198), To: rune(1198)}, + &CodePointRange{From: rune(1200), To: rune(1200)}, + &CodePointRange{From: rune(1202), To: rune(1202)}, + &CodePointRange{From: rune(1204), To: rune(1204)}, + &CodePointRange{From: rune(1206), To: rune(1206)}, + &CodePointRange{From: rune(1208), To: rune(1208)}, + &CodePointRange{From: rune(1210), To: rune(1210)}, + &CodePointRange{From: rune(1212), To: rune(1212)}, + &CodePointRange{From: rune(1214), To: rune(1214)}, + &CodePointRange{From: rune(1216), To: rune(1217)}, + &CodePointRange{From: rune(1219), To: rune(1219)}, + &CodePointRange{From: rune(1221), To: rune(1221)}, + &CodePointRange{From: rune(1223), To: rune(1223)}, + &CodePointRange{From: rune(1225), To: rune(1225)}, + &CodePointRange{From: rune(1227), To: rune(1227)}, + &CodePointRange{From: rune(1229), To: rune(1229)}, + &CodePointRange{From: rune(1232), To: rune(1232)}, + &CodePointRange{From: rune(1234), To: rune(1234)}, + &CodePointRange{From: rune(1236), To: rune(1236)}, + &CodePointRange{From: rune(1238), To: rune(1238)}, + &CodePointRange{From: rune(1240), To: rune(1240)}, + &CodePointRange{From: rune(1242), To: rune(1242)}, + &CodePointRange{From: rune(1244), To: rune(1244)}, + &CodePointRange{From: rune(1246), To: rune(1246)}, + &CodePointRange{From: rune(1248), To: rune(1248)}, + &CodePointRange{From: rune(1250), To: rune(1250)}, + &CodePointRange{From: rune(1252), To: rune(1252)}, + &CodePointRange{From: rune(1254), To: rune(1254)}, + &CodePointRange{From: rune(1256), To: rune(1256)}, + &CodePointRange{From: rune(1258), To: rune(1258)}, + &CodePointRange{From: rune(1260), To: rune(1260)}, + &CodePointRange{From: rune(1262), To: rune(1262)}, + &CodePointRange{From: rune(1264), To: rune(1264)}, + &CodePointRange{From: rune(1266), To: rune(1266)}, + &CodePointRange{From: rune(1268), To: rune(1268)}, + &CodePointRange{From: rune(1270), To: rune(1270)}, + &CodePointRange{From: rune(1272), To: rune(1272)}, + &CodePointRange{From: rune(1274), To: rune(1274)}, + &CodePointRange{From: rune(1276), To: rune(1276)}, + &CodePointRange{From: rune(1278), To: rune(1278)}, + &CodePointRange{From: rune(1280), To: rune(1280)}, + &CodePointRange{From: rune(1282), To: rune(1282)}, + &CodePointRange{From: rune(1284), To: rune(1284)}, + &CodePointRange{From: rune(1286), To: rune(1286)}, + &CodePointRange{From: rune(1288), To: rune(1288)}, + &CodePointRange{From: rune(1290), To: rune(1290)}, + &CodePointRange{From: rune(1292), To: rune(1292)}, + &CodePointRange{From: rune(1294), To: rune(1294)}, + &CodePointRange{From: rune(1296), To: rune(1296)}, + &CodePointRange{From: rune(1298), To: rune(1298)}, + &CodePointRange{From: rune(1300), To: rune(1300)}, + &CodePointRange{From: rune(1302), To: rune(1302)}, + &CodePointRange{From: rune(1304), To: rune(1304)}, + &CodePointRange{From: rune(1306), To: rune(1306)}, + &CodePointRange{From: rune(1308), To: rune(1308)}, + &CodePointRange{From: rune(1310), To: rune(1310)}, + &CodePointRange{From: rune(1312), To: rune(1312)}, + &CodePointRange{From: rune(1314), To: rune(1314)}, + &CodePointRange{From: rune(1316), To: rune(1316)}, + &CodePointRange{From: rune(1318), To: rune(1318)}, + &CodePointRange{From: rune(1320), To: rune(1320)}, + &CodePointRange{From: rune(1322), To: rune(1322)}, + &CodePointRange{From: rune(1324), To: rune(1324)}, + &CodePointRange{From: rune(1326), To: rune(1326)}, + &CodePointRange{From: rune(1329), To: rune(1366)}, + &CodePointRange{From: rune(4256), To: rune(4293)}, + &CodePointRange{From: rune(4295), To: rune(4295)}, + &CodePointRange{From: rune(4301), To: rune(4301)}, + &CodePointRange{From: rune(5024), To: rune(5109)}, + &CodePointRange{From: rune(7312), To: rune(7354)}, + &CodePointRange{From: rune(7357), To: rune(7359)}, + &CodePointRange{From: rune(7680), To: rune(7680)}, + &CodePointRange{From: rune(7682), To: rune(7682)}, + &CodePointRange{From: rune(7684), To: rune(7684)}, + &CodePointRange{From: rune(7686), To: rune(7686)}, + &CodePointRange{From: rune(7688), To: rune(7688)}, + &CodePointRange{From: rune(7690), To: rune(7690)}, + &CodePointRange{From: rune(7692), To: rune(7692)}, + &CodePointRange{From: rune(7694), To: rune(7694)}, + &CodePointRange{From: rune(7696), To: rune(7696)}, + &CodePointRange{From: rune(7698), To: rune(7698)}, + &CodePointRange{From: rune(7700), To: rune(7700)}, + &CodePointRange{From: rune(7702), To: rune(7702)}, + &CodePointRange{From: rune(7704), To: rune(7704)}, + &CodePointRange{From: rune(7706), To: rune(7706)}, + &CodePointRange{From: rune(7708), To: rune(7708)}, + &CodePointRange{From: rune(7710), To: rune(7710)}, + &CodePointRange{From: rune(7712), To: rune(7712)}, + &CodePointRange{From: rune(7714), To: rune(7714)}, + &CodePointRange{From: rune(7716), To: rune(7716)}, + &CodePointRange{From: rune(7718), To: rune(7718)}, + &CodePointRange{From: rune(7720), To: rune(7720)}, + &CodePointRange{From: rune(7722), To: rune(7722)}, + &CodePointRange{From: rune(7724), To: rune(7724)}, + &CodePointRange{From: rune(7726), To: rune(7726)}, + &CodePointRange{From: rune(7728), To: rune(7728)}, + &CodePointRange{From: rune(7730), To: rune(7730)}, + &CodePointRange{From: rune(7732), To: rune(7732)}, + &CodePointRange{From: rune(7734), To: rune(7734)}, + &CodePointRange{From: rune(7736), To: rune(7736)}, + &CodePointRange{From: rune(7738), To: rune(7738)}, + &CodePointRange{From: rune(7740), To: rune(7740)}, + &CodePointRange{From: rune(7742), To: rune(7742)}, + &CodePointRange{From: rune(7744), To: rune(7744)}, + &CodePointRange{From: rune(7746), To: rune(7746)}, + &CodePointRange{From: rune(7748), To: rune(7748)}, + &CodePointRange{From: rune(7750), To: rune(7750)}, + &CodePointRange{From: rune(7752), To: rune(7752)}, + &CodePointRange{From: rune(7754), To: rune(7754)}, + &CodePointRange{From: rune(7756), To: rune(7756)}, + &CodePointRange{From: rune(7758), To: rune(7758)}, + &CodePointRange{From: rune(7760), To: rune(7760)}, + &CodePointRange{From: rune(7762), To: rune(7762)}, + &CodePointRange{From: rune(7764), To: rune(7764)}, + &CodePointRange{From: rune(7766), To: rune(7766)}, + &CodePointRange{From: rune(7768), To: rune(7768)}, + &CodePointRange{From: rune(7770), To: rune(7770)}, + &CodePointRange{From: rune(7772), To: rune(7772)}, + &CodePointRange{From: rune(7774), To: rune(7774)}, + &CodePointRange{From: rune(7776), To: rune(7776)}, + &CodePointRange{From: rune(7778), To: rune(7778)}, + &CodePointRange{From: rune(7780), To: rune(7780)}, + &CodePointRange{From: rune(7782), To: rune(7782)}, + &CodePointRange{From: rune(7784), To: rune(7784)}, + &CodePointRange{From: rune(7786), To: rune(7786)}, + &CodePointRange{From: rune(7788), To: rune(7788)}, + &CodePointRange{From: rune(7790), To: rune(7790)}, + &CodePointRange{From: rune(7792), To: rune(7792)}, + &CodePointRange{From: rune(7794), To: rune(7794)}, + &CodePointRange{From: rune(7796), To: rune(7796)}, + &CodePointRange{From: rune(7798), To: rune(7798)}, + &CodePointRange{From: rune(7800), To: rune(7800)}, + &CodePointRange{From: rune(7802), To: rune(7802)}, + &CodePointRange{From: rune(7804), To: rune(7804)}, + &CodePointRange{From: rune(7806), To: rune(7806)}, + &CodePointRange{From: rune(7808), To: rune(7808)}, + &CodePointRange{From: rune(7810), To: rune(7810)}, + &CodePointRange{From: rune(7812), To: rune(7812)}, + &CodePointRange{From: rune(7814), To: rune(7814)}, + &CodePointRange{From: rune(7816), To: rune(7816)}, + &CodePointRange{From: rune(7818), To: rune(7818)}, + &CodePointRange{From: rune(7820), To: rune(7820)}, + &CodePointRange{From: rune(7822), To: rune(7822)}, + &CodePointRange{From: rune(7824), To: rune(7824)}, + &CodePointRange{From: rune(7826), To: rune(7826)}, + &CodePointRange{From: rune(7828), To: rune(7828)}, + &CodePointRange{From: rune(7838), To: rune(7838)}, + &CodePointRange{From: rune(7840), To: rune(7840)}, + &CodePointRange{From: rune(7842), To: rune(7842)}, + &CodePointRange{From: rune(7844), To: rune(7844)}, + &CodePointRange{From: rune(7846), To: rune(7846)}, + &CodePointRange{From: rune(7848), To: rune(7848)}, + &CodePointRange{From: rune(7850), To: rune(7850)}, + &CodePointRange{From: rune(7852), To: rune(7852)}, + &CodePointRange{From: rune(7854), To: rune(7854)}, + &CodePointRange{From: rune(7856), To: rune(7856)}, + &CodePointRange{From: rune(7858), To: rune(7858)}, + &CodePointRange{From: rune(7860), To: rune(7860)}, + &CodePointRange{From: rune(7862), To: rune(7862)}, + &CodePointRange{From: rune(7864), To: rune(7864)}, + &CodePointRange{From: rune(7866), To: rune(7866)}, + &CodePointRange{From: rune(7868), To: rune(7868)}, + &CodePointRange{From: rune(7870), To: rune(7870)}, + &CodePointRange{From: rune(7872), To: rune(7872)}, + &CodePointRange{From: rune(7874), To: rune(7874)}, + &CodePointRange{From: rune(7876), To: rune(7876)}, + &CodePointRange{From: rune(7878), To: rune(7878)}, + &CodePointRange{From: rune(7880), To: rune(7880)}, + &CodePointRange{From: rune(7882), To: rune(7882)}, + &CodePointRange{From: rune(7884), To: rune(7884)}, + &CodePointRange{From: rune(7886), To: rune(7886)}, + &CodePointRange{From: rune(7888), To: rune(7888)}, + &CodePointRange{From: rune(7890), To: rune(7890)}, + &CodePointRange{From: rune(7892), To: rune(7892)}, + &CodePointRange{From: rune(7894), To: rune(7894)}, + &CodePointRange{From: rune(7896), To: rune(7896)}, + &CodePointRange{From: rune(7898), To: rune(7898)}, + &CodePointRange{From: rune(7900), To: rune(7900)}, + &CodePointRange{From: rune(7902), To: rune(7902)}, + &CodePointRange{From: rune(7904), To: rune(7904)}, + &CodePointRange{From: rune(7906), To: rune(7906)}, + &CodePointRange{From: rune(7908), To: rune(7908)}, + &CodePointRange{From: rune(7910), To: rune(7910)}, + &CodePointRange{From: rune(7912), To: rune(7912)}, + &CodePointRange{From: rune(7914), To: rune(7914)}, + &CodePointRange{From: rune(7916), To: rune(7916)}, + &CodePointRange{From: rune(7918), To: rune(7918)}, + &CodePointRange{From: rune(7920), To: rune(7920)}, + &CodePointRange{From: rune(7922), To: rune(7922)}, + &CodePointRange{From: rune(7924), To: rune(7924)}, + &CodePointRange{From: rune(7926), To: rune(7926)}, + &CodePointRange{From: rune(7928), To: rune(7928)}, + &CodePointRange{From: rune(7930), To: rune(7930)}, + &CodePointRange{From: rune(7932), To: rune(7932)}, + &CodePointRange{From: rune(7934), To: rune(7934)}, + &CodePointRange{From: rune(7944), To: rune(7951)}, + &CodePointRange{From: rune(7960), To: rune(7965)}, + &CodePointRange{From: rune(7976), To: rune(7983)}, + &CodePointRange{From: rune(7992), To: rune(7999)}, + &CodePointRange{From: rune(8008), To: rune(8013)}, + &CodePointRange{From: rune(8025), To: rune(8025)}, + &CodePointRange{From: rune(8027), To: rune(8027)}, + &CodePointRange{From: rune(8029), To: rune(8029)}, + &CodePointRange{From: rune(8031), To: rune(8031)}, + &CodePointRange{From: rune(8040), To: rune(8047)}, + &CodePointRange{From: rune(8120), To: rune(8123)}, + &CodePointRange{From: rune(8136), To: rune(8139)}, + &CodePointRange{From: rune(8152), To: rune(8155)}, + &CodePointRange{From: rune(8168), To: rune(8172)}, + &CodePointRange{From: rune(8184), To: rune(8187)}, + &CodePointRange{From: rune(8450), To: rune(8450)}, + &CodePointRange{From: rune(8455), To: rune(8455)}, + &CodePointRange{From: rune(8459), To: rune(8461)}, + &CodePointRange{From: rune(8464), To: rune(8466)}, + &CodePointRange{From: rune(8469), To: rune(8469)}, + &CodePointRange{From: rune(8473), To: rune(8477)}, + &CodePointRange{From: rune(8484), To: rune(8484)}, + &CodePointRange{From: rune(8486), To: rune(8486)}, + &CodePointRange{From: rune(8488), To: rune(8488)}, + &CodePointRange{From: rune(8490), To: rune(8493)}, + &CodePointRange{From: rune(8496), To: rune(8499)}, + &CodePointRange{From: rune(8510), To: rune(8511)}, + &CodePointRange{From: rune(8517), To: rune(8517)}, + &CodePointRange{From: rune(8579), To: rune(8579)}, + &CodePointRange{From: rune(11264), To: rune(11310)}, + &CodePointRange{From: rune(11360), To: rune(11360)}, + &CodePointRange{From: rune(11362), To: rune(11364)}, + &CodePointRange{From: rune(11367), To: rune(11367)}, + &CodePointRange{From: rune(11369), To: rune(11369)}, + &CodePointRange{From: rune(11371), To: rune(11371)}, + &CodePointRange{From: rune(11373), To: rune(11376)}, + &CodePointRange{From: rune(11378), To: rune(11378)}, + &CodePointRange{From: rune(11381), To: rune(11381)}, + &CodePointRange{From: rune(11390), To: rune(11392)}, + &CodePointRange{From: rune(11394), To: rune(11394)}, + &CodePointRange{From: rune(11396), To: rune(11396)}, + &CodePointRange{From: rune(11398), To: rune(11398)}, + &CodePointRange{From: rune(11400), To: rune(11400)}, + &CodePointRange{From: rune(11402), To: rune(11402)}, + &CodePointRange{From: rune(11404), To: rune(11404)}, + &CodePointRange{From: rune(11406), To: rune(11406)}, + &CodePointRange{From: rune(11408), To: rune(11408)}, + &CodePointRange{From: rune(11410), To: rune(11410)}, + &CodePointRange{From: rune(11412), To: rune(11412)}, + &CodePointRange{From: rune(11414), To: rune(11414)}, + &CodePointRange{From: rune(11416), To: rune(11416)}, + &CodePointRange{From: rune(11418), To: rune(11418)}, + &CodePointRange{From: rune(11420), To: rune(11420)}, + &CodePointRange{From: rune(11422), To: rune(11422)}, + &CodePointRange{From: rune(11424), To: rune(11424)}, + &CodePointRange{From: rune(11426), To: rune(11426)}, + &CodePointRange{From: rune(11428), To: rune(11428)}, + &CodePointRange{From: rune(11430), To: rune(11430)}, + &CodePointRange{From: rune(11432), To: rune(11432)}, + &CodePointRange{From: rune(11434), To: rune(11434)}, + &CodePointRange{From: rune(11436), To: rune(11436)}, + &CodePointRange{From: rune(11438), To: rune(11438)}, + &CodePointRange{From: rune(11440), To: rune(11440)}, + &CodePointRange{From: rune(11442), To: rune(11442)}, + &CodePointRange{From: rune(11444), To: rune(11444)}, + &CodePointRange{From: rune(11446), To: rune(11446)}, + &CodePointRange{From: rune(11448), To: rune(11448)}, + &CodePointRange{From: rune(11450), To: rune(11450)}, + &CodePointRange{From: rune(11452), To: rune(11452)}, + &CodePointRange{From: rune(11454), To: rune(11454)}, + &CodePointRange{From: rune(11456), To: rune(11456)}, + &CodePointRange{From: rune(11458), To: rune(11458)}, + &CodePointRange{From: rune(11460), To: rune(11460)}, + &CodePointRange{From: rune(11462), To: rune(11462)}, + &CodePointRange{From: rune(11464), To: rune(11464)}, + &CodePointRange{From: rune(11466), To: rune(11466)}, + &CodePointRange{From: rune(11468), To: rune(11468)}, + &CodePointRange{From: rune(11470), To: rune(11470)}, + &CodePointRange{From: rune(11472), To: rune(11472)}, + &CodePointRange{From: rune(11474), To: rune(11474)}, + &CodePointRange{From: rune(11476), To: rune(11476)}, + &CodePointRange{From: rune(11478), To: rune(11478)}, + &CodePointRange{From: rune(11480), To: rune(11480)}, + &CodePointRange{From: rune(11482), To: rune(11482)}, + &CodePointRange{From: rune(11484), To: rune(11484)}, + &CodePointRange{From: rune(11486), To: rune(11486)}, + &CodePointRange{From: rune(11488), To: rune(11488)}, + &CodePointRange{From: rune(11490), To: rune(11490)}, + &CodePointRange{From: rune(11499), To: rune(11499)}, + &CodePointRange{From: rune(11501), To: rune(11501)}, + &CodePointRange{From: rune(11506), To: rune(11506)}, + &CodePointRange{From: rune(42560), To: rune(42560)}, + &CodePointRange{From: rune(42562), To: rune(42562)}, + &CodePointRange{From: rune(42564), To: rune(42564)}, + &CodePointRange{From: rune(42566), To: rune(42566)}, + &CodePointRange{From: rune(42568), To: rune(42568)}, + &CodePointRange{From: rune(42570), To: rune(42570)}, + &CodePointRange{From: rune(42572), To: rune(42572)}, + &CodePointRange{From: rune(42574), To: rune(42574)}, + &CodePointRange{From: rune(42576), To: rune(42576)}, + &CodePointRange{From: rune(42578), To: rune(42578)}, + &CodePointRange{From: rune(42580), To: rune(42580)}, + &CodePointRange{From: rune(42582), To: rune(42582)}, + &CodePointRange{From: rune(42584), To: rune(42584)}, + &CodePointRange{From: rune(42586), To: rune(42586)}, + &CodePointRange{From: rune(42588), To: rune(42588)}, + &CodePointRange{From: rune(42590), To: rune(42590)}, + &CodePointRange{From: rune(42592), To: rune(42592)}, + &CodePointRange{From: rune(42594), To: rune(42594)}, + &CodePointRange{From: rune(42596), To: rune(42596)}, + &CodePointRange{From: rune(42598), To: rune(42598)}, + &CodePointRange{From: rune(42600), To: rune(42600)}, + &CodePointRange{From: rune(42602), To: rune(42602)}, + &CodePointRange{From: rune(42604), To: rune(42604)}, + &CodePointRange{From: rune(42624), To: rune(42624)}, + &CodePointRange{From: rune(42626), To: rune(42626)}, + &CodePointRange{From: rune(42628), To: rune(42628)}, + &CodePointRange{From: rune(42630), To: rune(42630)}, + &CodePointRange{From: rune(42632), To: rune(42632)}, + &CodePointRange{From: rune(42634), To: rune(42634)}, + &CodePointRange{From: rune(42636), To: rune(42636)}, + &CodePointRange{From: rune(42638), To: rune(42638)}, + &CodePointRange{From: rune(42640), To: rune(42640)}, + &CodePointRange{From: rune(42642), To: rune(42642)}, + &CodePointRange{From: rune(42644), To: rune(42644)}, + &CodePointRange{From: rune(42646), To: rune(42646)}, + &CodePointRange{From: rune(42648), To: rune(42648)}, + &CodePointRange{From: rune(42650), To: rune(42650)}, + &CodePointRange{From: rune(42786), To: rune(42786)}, + &CodePointRange{From: rune(42788), To: rune(42788)}, + &CodePointRange{From: rune(42790), To: rune(42790)}, + &CodePointRange{From: rune(42792), To: rune(42792)}, + &CodePointRange{From: rune(42794), To: rune(42794)}, + &CodePointRange{From: rune(42796), To: rune(42796)}, + &CodePointRange{From: rune(42798), To: rune(42798)}, + &CodePointRange{From: rune(42802), To: rune(42802)}, + &CodePointRange{From: rune(42804), To: rune(42804)}, + &CodePointRange{From: rune(42806), To: rune(42806)}, + &CodePointRange{From: rune(42808), To: rune(42808)}, + &CodePointRange{From: rune(42810), To: rune(42810)}, + &CodePointRange{From: rune(42812), To: rune(42812)}, + &CodePointRange{From: rune(42814), To: rune(42814)}, + &CodePointRange{From: rune(42816), To: rune(42816)}, + &CodePointRange{From: rune(42818), To: rune(42818)}, + &CodePointRange{From: rune(42820), To: rune(42820)}, + &CodePointRange{From: rune(42822), To: rune(42822)}, + &CodePointRange{From: rune(42824), To: rune(42824)}, + &CodePointRange{From: rune(42826), To: rune(42826)}, + &CodePointRange{From: rune(42828), To: rune(42828)}, + &CodePointRange{From: rune(42830), To: rune(42830)}, + &CodePointRange{From: rune(42832), To: rune(42832)}, + &CodePointRange{From: rune(42834), To: rune(42834)}, + &CodePointRange{From: rune(42836), To: rune(42836)}, + &CodePointRange{From: rune(42838), To: rune(42838)}, + &CodePointRange{From: rune(42840), To: rune(42840)}, + &CodePointRange{From: rune(42842), To: rune(42842)}, + &CodePointRange{From: rune(42844), To: rune(42844)}, + &CodePointRange{From: rune(42846), To: rune(42846)}, + &CodePointRange{From: rune(42848), To: rune(42848)}, + &CodePointRange{From: rune(42850), To: rune(42850)}, + &CodePointRange{From: rune(42852), To: rune(42852)}, + &CodePointRange{From: rune(42854), To: rune(42854)}, + &CodePointRange{From: rune(42856), To: rune(42856)}, + &CodePointRange{From: rune(42858), To: rune(42858)}, + &CodePointRange{From: rune(42860), To: rune(42860)}, + &CodePointRange{From: rune(42862), To: rune(42862)}, + &CodePointRange{From: rune(42873), To: rune(42873)}, + &CodePointRange{From: rune(42875), To: rune(42875)}, + &CodePointRange{From: rune(42877), To: rune(42878)}, + &CodePointRange{From: rune(42880), To: rune(42880)}, + &CodePointRange{From: rune(42882), To: rune(42882)}, + &CodePointRange{From: rune(42884), To: rune(42884)}, + &CodePointRange{From: rune(42886), To: rune(42886)}, + &CodePointRange{From: rune(42891), To: rune(42891)}, + &CodePointRange{From: rune(42893), To: rune(42893)}, + &CodePointRange{From: rune(42896), To: rune(42896)}, + &CodePointRange{From: rune(42898), To: rune(42898)}, + &CodePointRange{From: rune(42902), To: rune(42902)}, + &CodePointRange{From: rune(42904), To: rune(42904)}, + &CodePointRange{From: rune(42906), To: rune(42906)}, + &CodePointRange{From: rune(42908), To: rune(42908)}, + &CodePointRange{From: rune(42910), To: rune(42910)}, + &CodePointRange{From: rune(42912), To: rune(42912)}, + &CodePointRange{From: rune(42914), To: rune(42914)}, + &CodePointRange{From: rune(42916), To: rune(42916)}, + &CodePointRange{From: rune(42918), To: rune(42918)}, + &CodePointRange{From: rune(42920), To: rune(42920)}, + &CodePointRange{From: rune(42922), To: rune(42926)}, + &CodePointRange{From: rune(42928), To: rune(42932)}, + &CodePointRange{From: rune(42934), To: rune(42934)}, + &CodePointRange{From: rune(42936), To: rune(42936)}, + &CodePointRange{From: rune(42938), To: rune(42938)}, + &CodePointRange{From: rune(42940), To: rune(42940)}, + &CodePointRange{From: rune(42942), To: rune(42942)}, + &CodePointRange{From: rune(42946), To: rune(42946)}, + &CodePointRange{From: rune(42948), To: rune(42951)}, + &CodePointRange{From: rune(42953), To: rune(42953)}, + &CodePointRange{From: rune(42997), To: rune(42997)}, + &CodePointRange{From: rune(65313), To: rune(65338)}, + &CodePointRange{From: rune(66560), To: rune(66599)}, + &CodePointRange{From: rune(66736), To: rune(66771)}, + &CodePointRange{From: rune(68736), To: rune(68786)}, + &CodePointRange{From: rune(71840), To: rune(71871)}, + &CodePointRange{From: rune(93760), To: rune(93791)}, + &CodePointRange{From: rune(119808), To: rune(119833)}, + &CodePointRange{From: rune(119860), To: rune(119885)}, + &CodePointRange{From: rune(119912), To: rune(119937)}, + &CodePointRange{From: rune(119964), To: rune(119964)}, + &CodePointRange{From: rune(119966), To: rune(119967)}, + &CodePointRange{From: rune(119970), To: rune(119970)}, + &CodePointRange{From: rune(119973), To: rune(119974)}, + &CodePointRange{From: rune(119977), To: rune(119980)}, + &CodePointRange{From: rune(119982), To: rune(119989)}, + &CodePointRange{From: rune(120016), To: rune(120041)}, + &CodePointRange{From: rune(120068), To: rune(120069)}, + &CodePointRange{From: rune(120071), To: rune(120074)}, + &CodePointRange{From: rune(120077), To: rune(120084)}, + &CodePointRange{From: rune(120086), To: rune(120092)}, + &CodePointRange{From: rune(120120), To: rune(120121)}, + &CodePointRange{From: rune(120123), To: rune(120126)}, + &CodePointRange{From: rune(120128), To: rune(120132)}, + &CodePointRange{From: rune(120134), To: rune(120134)}, + &CodePointRange{From: rune(120138), To: rune(120144)}, + &CodePointRange{From: rune(120172), To: rune(120197)}, + &CodePointRange{From: rune(120224), To: rune(120249)}, + &CodePointRange{From: rune(120276), To: rune(120301)}, + &CodePointRange{From: rune(120328), To: rune(120353)}, + &CodePointRange{From: rune(120380), To: rune(120405)}, + &CodePointRange{From: rune(120432), To: rune(120457)}, + &CodePointRange{From: rune(120488), To: rune(120512)}, + &CodePointRange{From: rune(120546), To: rune(120570)}, + &CodePointRange{From: rune(120604), To: rune(120628)}, + &CodePointRange{From: rune(120662), To: rune(120686)}, + &CodePointRange{From: rune(120720), To: rune(120744)}, + &CodePointRange{From: rune(120778), To: rune(120778)}, + &CodePointRange{From: rune(125184), To: rune(125217)}, + }, + "mc": { + &CodePointRange{From: rune(2307), To: rune(2307)}, + &CodePointRange{From: rune(2363), To: rune(2363)}, + &CodePointRange{From: rune(2366), To: rune(2368)}, + &CodePointRange{From: rune(2377), To: rune(2380)}, + &CodePointRange{From: rune(2382), To: rune(2383)}, + &CodePointRange{From: rune(2434), To: rune(2435)}, + &CodePointRange{From: rune(2494), To: rune(2496)}, + &CodePointRange{From: rune(2503), To: rune(2504)}, + &CodePointRange{From: rune(2507), To: rune(2508)}, + &CodePointRange{From: rune(2519), To: rune(2519)}, + &CodePointRange{From: rune(2563), To: rune(2563)}, + &CodePointRange{From: rune(2622), To: rune(2624)}, + &CodePointRange{From: rune(2691), To: rune(2691)}, + &CodePointRange{From: rune(2750), To: rune(2752)}, + &CodePointRange{From: rune(2761), To: rune(2761)}, + &CodePointRange{From: rune(2763), To: rune(2764)}, + &CodePointRange{From: rune(2818), To: rune(2819)}, + &CodePointRange{From: rune(2878), To: rune(2878)}, + &CodePointRange{From: rune(2880), To: rune(2880)}, + &CodePointRange{From: rune(2887), To: rune(2888)}, + &CodePointRange{From: rune(2891), To: rune(2892)}, + &CodePointRange{From: rune(2903), To: rune(2903)}, + &CodePointRange{From: rune(3006), To: rune(3007)}, + &CodePointRange{From: rune(3009), To: rune(3010)}, + &CodePointRange{From: rune(3014), To: rune(3016)}, + &CodePointRange{From: rune(3018), To: rune(3020)}, + &CodePointRange{From: rune(3031), To: rune(3031)}, + &CodePointRange{From: rune(3073), To: rune(3075)}, + &CodePointRange{From: rune(3137), To: rune(3140)}, + &CodePointRange{From: rune(3202), To: rune(3203)}, + &CodePointRange{From: rune(3262), To: rune(3262)}, + &CodePointRange{From: rune(3264), To: rune(3268)}, + &CodePointRange{From: rune(3271), To: rune(3272)}, + &CodePointRange{From: rune(3274), To: rune(3275)}, + &CodePointRange{From: rune(3285), To: rune(3286)}, + &CodePointRange{From: rune(3330), To: rune(3331)}, + &CodePointRange{From: rune(3390), To: rune(3392)}, + &CodePointRange{From: rune(3398), To: rune(3400)}, + &CodePointRange{From: rune(3402), To: rune(3404)}, + &CodePointRange{From: rune(3415), To: rune(3415)}, + &CodePointRange{From: rune(3458), To: rune(3459)}, + &CodePointRange{From: rune(3535), To: rune(3537)}, + &CodePointRange{From: rune(3544), To: rune(3551)}, + &CodePointRange{From: rune(3570), To: rune(3571)}, + &CodePointRange{From: rune(3902), To: rune(3903)}, + &CodePointRange{From: rune(3967), To: rune(3967)}, + &CodePointRange{From: rune(4139), To: rune(4140)}, + &CodePointRange{From: rune(4145), To: rune(4145)}, + &CodePointRange{From: rune(4152), To: rune(4152)}, + &CodePointRange{From: rune(4155), To: rune(4156)}, + &CodePointRange{From: rune(4182), To: rune(4183)}, + &CodePointRange{From: rune(4194), To: rune(4196)}, + &CodePointRange{From: rune(4199), To: rune(4205)}, + &CodePointRange{From: rune(4227), To: rune(4228)}, + &CodePointRange{From: rune(4231), To: rune(4236)}, + &CodePointRange{From: rune(4239), To: rune(4239)}, + &CodePointRange{From: rune(4250), To: rune(4252)}, + &CodePointRange{From: rune(6070), To: rune(6070)}, + &CodePointRange{From: rune(6078), To: rune(6085)}, + &CodePointRange{From: rune(6087), To: rune(6088)}, + &CodePointRange{From: rune(6435), To: rune(6438)}, + &CodePointRange{From: rune(6441), To: rune(6443)}, + &CodePointRange{From: rune(6448), To: rune(6449)}, + &CodePointRange{From: rune(6451), To: rune(6456)}, + &CodePointRange{From: rune(6681), To: rune(6682)}, + &CodePointRange{From: rune(6741), To: rune(6741)}, + &CodePointRange{From: rune(6743), To: rune(6743)}, + &CodePointRange{From: rune(6753), To: rune(6753)}, + &CodePointRange{From: rune(6755), To: rune(6756)}, + &CodePointRange{From: rune(6765), To: rune(6770)}, + &CodePointRange{From: rune(6916), To: rune(6916)}, + &CodePointRange{From: rune(6965), To: rune(6965)}, + &CodePointRange{From: rune(6971), To: rune(6971)}, + &CodePointRange{From: rune(6973), To: rune(6977)}, + &CodePointRange{From: rune(6979), To: rune(6980)}, + &CodePointRange{From: rune(7042), To: rune(7042)}, + &CodePointRange{From: rune(7073), To: rune(7073)}, + &CodePointRange{From: rune(7078), To: rune(7079)}, + &CodePointRange{From: rune(7082), To: rune(7082)}, + &CodePointRange{From: rune(7143), To: rune(7143)}, + &CodePointRange{From: rune(7146), To: rune(7148)}, + &CodePointRange{From: rune(7150), To: rune(7150)}, + &CodePointRange{From: rune(7154), To: rune(7155)}, + &CodePointRange{From: rune(7204), To: rune(7211)}, + &CodePointRange{From: rune(7220), To: rune(7221)}, + &CodePointRange{From: rune(7393), To: rune(7393)}, + &CodePointRange{From: rune(7415), To: rune(7415)}, + &CodePointRange{From: rune(12334), To: rune(12335)}, + &CodePointRange{From: rune(43043), To: rune(43044)}, + &CodePointRange{From: rune(43047), To: rune(43047)}, + &CodePointRange{From: rune(43136), To: rune(43137)}, + &CodePointRange{From: rune(43188), To: rune(43203)}, + &CodePointRange{From: rune(43346), To: rune(43347)}, + &CodePointRange{From: rune(43395), To: rune(43395)}, + &CodePointRange{From: rune(43444), To: rune(43445)}, + &CodePointRange{From: rune(43450), To: rune(43451)}, + &CodePointRange{From: rune(43454), To: rune(43456)}, + &CodePointRange{From: rune(43567), To: rune(43568)}, + &CodePointRange{From: rune(43571), To: rune(43572)}, + &CodePointRange{From: rune(43597), To: rune(43597)}, + &CodePointRange{From: rune(43643), To: rune(43643)}, + &CodePointRange{From: rune(43645), To: rune(43645)}, + &CodePointRange{From: rune(43755), To: rune(43755)}, + &CodePointRange{From: rune(43758), To: rune(43759)}, + &CodePointRange{From: rune(43765), To: rune(43765)}, + &CodePointRange{From: rune(44003), To: rune(44004)}, + &CodePointRange{From: rune(44006), To: rune(44007)}, + &CodePointRange{From: rune(44009), To: rune(44010)}, + &CodePointRange{From: rune(44012), To: rune(44012)}, + &CodePointRange{From: rune(69632), To: rune(69632)}, + &CodePointRange{From: rune(69634), To: rune(69634)}, + &CodePointRange{From: rune(69762), To: rune(69762)}, + &CodePointRange{From: rune(69808), To: rune(69810)}, + &CodePointRange{From: rune(69815), To: rune(69816)}, + &CodePointRange{From: rune(69932), To: rune(69932)}, + &CodePointRange{From: rune(69957), To: rune(69958)}, + &CodePointRange{From: rune(70018), To: rune(70018)}, + &CodePointRange{From: rune(70067), To: rune(70069)}, + &CodePointRange{From: rune(70079), To: rune(70080)}, + &CodePointRange{From: rune(70094), To: rune(70094)}, + &CodePointRange{From: rune(70188), To: rune(70190)}, + &CodePointRange{From: rune(70194), To: rune(70195)}, + &CodePointRange{From: rune(70197), To: rune(70197)}, + &CodePointRange{From: rune(70368), To: rune(70370)}, + &CodePointRange{From: rune(70402), To: rune(70403)}, + &CodePointRange{From: rune(70462), To: rune(70463)}, + &CodePointRange{From: rune(70465), To: rune(70468)}, + &CodePointRange{From: rune(70471), To: rune(70472)}, + &CodePointRange{From: rune(70475), To: rune(70477)}, + &CodePointRange{From: rune(70487), To: rune(70487)}, + &CodePointRange{From: rune(70498), To: rune(70499)}, + &CodePointRange{From: rune(70709), To: rune(70711)}, + &CodePointRange{From: rune(70720), To: rune(70721)}, + &CodePointRange{From: rune(70725), To: rune(70725)}, + &CodePointRange{From: rune(70832), To: rune(70834)}, + &CodePointRange{From: rune(70841), To: rune(70841)}, + &CodePointRange{From: rune(70843), To: rune(70846)}, + &CodePointRange{From: rune(70849), To: rune(70849)}, + &CodePointRange{From: rune(71087), To: rune(71089)}, + &CodePointRange{From: rune(71096), To: rune(71099)}, + &CodePointRange{From: rune(71102), To: rune(71102)}, + &CodePointRange{From: rune(71216), To: rune(71218)}, + &CodePointRange{From: rune(71227), To: rune(71228)}, + &CodePointRange{From: rune(71230), To: rune(71230)}, + &CodePointRange{From: rune(71340), To: rune(71340)}, + &CodePointRange{From: rune(71342), To: rune(71343)}, + &CodePointRange{From: rune(71350), To: rune(71350)}, + &CodePointRange{From: rune(71456), To: rune(71457)}, + &CodePointRange{From: rune(71462), To: rune(71462)}, + &CodePointRange{From: rune(71724), To: rune(71726)}, + &CodePointRange{From: rune(71736), To: rune(71736)}, + &CodePointRange{From: rune(71984), To: rune(71989)}, + &CodePointRange{From: rune(71991), To: rune(71992)}, + &CodePointRange{From: rune(71997), To: rune(71997)}, + &CodePointRange{From: rune(72000), To: rune(72000)}, + &CodePointRange{From: rune(72002), To: rune(72002)}, + &CodePointRange{From: rune(72145), To: rune(72147)}, + &CodePointRange{From: rune(72156), To: rune(72159)}, + &CodePointRange{From: rune(72164), To: rune(72164)}, + &CodePointRange{From: rune(72249), To: rune(72249)}, + &CodePointRange{From: rune(72279), To: rune(72280)}, + &CodePointRange{From: rune(72343), To: rune(72343)}, + &CodePointRange{From: rune(72751), To: rune(72751)}, + &CodePointRange{From: rune(72766), To: rune(72766)}, + &CodePointRange{From: rune(72873), To: rune(72873)}, + &CodePointRange{From: rune(72881), To: rune(72881)}, + &CodePointRange{From: rune(72884), To: rune(72884)}, + &CodePointRange{From: rune(73098), To: rune(73102)}, + &CodePointRange{From: rune(73107), To: rune(73108)}, + &CodePointRange{From: rune(73110), To: rune(73110)}, + &CodePointRange{From: rune(73461), To: rune(73462)}, + &CodePointRange{From: rune(94033), To: rune(94087)}, + &CodePointRange{From: rune(94192), To: rune(94193)}, + &CodePointRange{From: rune(119141), To: rune(119142)}, + &CodePointRange{From: rune(119149), To: rune(119154)}, + }, + "me": { + &CodePointRange{From: rune(1160), To: rune(1161)}, + &CodePointRange{From: rune(6846), To: rune(6846)}, + &CodePointRange{From: rune(8413), To: rune(8416)}, + &CodePointRange{From: rune(8418), To: rune(8420)}, + &CodePointRange{From: rune(42608), To: rune(42610)}, + }, + "mn": { + &CodePointRange{From: rune(768), To: rune(879)}, + &CodePointRange{From: rune(1155), To: rune(1159)}, + &CodePointRange{From: rune(1425), To: rune(1469)}, + &CodePointRange{From: rune(1471), To: rune(1471)}, + &CodePointRange{From: rune(1473), To: rune(1474)}, + &CodePointRange{From: rune(1476), To: rune(1477)}, + &CodePointRange{From: rune(1479), To: rune(1479)}, + &CodePointRange{From: rune(1552), To: rune(1562)}, + &CodePointRange{From: rune(1611), To: rune(1631)}, + &CodePointRange{From: rune(1648), To: rune(1648)}, + &CodePointRange{From: rune(1750), To: rune(1756)}, + &CodePointRange{From: rune(1759), To: rune(1764)}, + &CodePointRange{From: rune(1767), To: rune(1768)}, + &CodePointRange{From: rune(1770), To: rune(1773)}, + &CodePointRange{From: rune(1809), To: rune(1809)}, + &CodePointRange{From: rune(1840), To: rune(1866)}, + &CodePointRange{From: rune(1958), To: rune(1968)}, + &CodePointRange{From: rune(2027), To: rune(2035)}, + &CodePointRange{From: rune(2045), To: rune(2045)}, + &CodePointRange{From: rune(2070), To: rune(2073)}, + &CodePointRange{From: rune(2075), To: rune(2083)}, + &CodePointRange{From: rune(2085), To: rune(2087)}, + &CodePointRange{From: rune(2089), To: rune(2093)}, + &CodePointRange{From: rune(2137), To: rune(2139)}, + &CodePointRange{From: rune(2259), To: rune(2273)}, + &CodePointRange{From: rune(2275), To: rune(2306)}, + &CodePointRange{From: rune(2362), To: rune(2362)}, + &CodePointRange{From: rune(2364), To: rune(2364)}, + &CodePointRange{From: rune(2369), To: rune(2376)}, + &CodePointRange{From: rune(2381), To: rune(2381)}, + &CodePointRange{From: rune(2385), To: rune(2391)}, + &CodePointRange{From: rune(2402), To: rune(2403)}, + &CodePointRange{From: rune(2433), To: rune(2433)}, + &CodePointRange{From: rune(2492), To: rune(2492)}, + &CodePointRange{From: rune(2497), To: rune(2500)}, + &CodePointRange{From: rune(2509), To: rune(2509)}, + &CodePointRange{From: rune(2530), To: rune(2531)}, + &CodePointRange{From: rune(2558), To: rune(2558)}, + &CodePointRange{From: rune(2561), To: rune(2562)}, + &CodePointRange{From: rune(2620), To: rune(2620)}, + &CodePointRange{From: rune(2625), To: rune(2626)}, + &CodePointRange{From: rune(2631), To: rune(2632)}, + &CodePointRange{From: rune(2635), To: rune(2637)}, + &CodePointRange{From: rune(2641), To: rune(2641)}, + &CodePointRange{From: rune(2672), To: rune(2673)}, + &CodePointRange{From: rune(2677), To: rune(2677)}, + &CodePointRange{From: rune(2689), To: rune(2690)}, + &CodePointRange{From: rune(2748), To: rune(2748)}, + &CodePointRange{From: rune(2753), To: rune(2757)}, + &CodePointRange{From: rune(2759), To: rune(2760)}, + &CodePointRange{From: rune(2765), To: rune(2765)}, + &CodePointRange{From: rune(2786), To: rune(2787)}, + &CodePointRange{From: rune(2810), To: rune(2815)}, + &CodePointRange{From: rune(2817), To: rune(2817)}, + &CodePointRange{From: rune(2876), To: rune(2876)}, + &CodePointRange{From: rune(2879), To: rune(2879)}, + &CodePointRange{From: rune(2881), To: rune(2884)}, + &CodePointRange{From: rune(2893), To: rune(2893)}, + &CodePointRange{From: rune(2901), To: rune(2902)}, + &CodePointRange{From: rune(2914), To: rune(2915)}, + &CodePointRange{From: rune(2946), To: rune(2946)}, + &CodePointRange{From: rune(3008), To: rune(3008)}, + &CodePointRange{From: rune(3021), To: rune(3021)}, + &CodePointRange{From: rune(3072), To: rune(3072)}, + &CodePointRange{From: rune(3076), To: rune(3076)}, + &CodePointRange{From: rune(3134), To: rune(3136)}, + &CodePointRange{From: rune(3142), To: rune(3144)}, + &CodePointRange{From: rune(3146), To: rune(3149)}, + &CodePointRange{From: rune(3157), To: rune(3158)}, + &CodePointRange{From: rune(3170), To: rune(3171)}, + &CodePointRange{From: rune(3201), To: rune(3201)}, + &CodePointRange{From: rune(3260), To: rune(3260)}, + &CodePointRange{From: rune(3263), To: rune(3263)}, + &CodePointRange{From: rune(3270), To: rune(3270)}, + &CodePointRange{From: rune(3276), To: rune(3277)}, + &CodePointRange{From: rune(3298), To: rune(3299)}, + &CodePointRange{From: rune(3328), To: rune(3329)}, + &CodePointRange{From: rune(3387), To: rune(3388)}, + &CodePointRange{From: rune(3393), To: rune(3396)}, + &CodePointRange{From: rune(3405), To: rune(3405)}, + &CodePointRange{From: rune(3426), To: rune(3427)}, + &CodePointRange{From: rune(3457), To: rune(3457)}, + &CodePointRange{From: rune(3530), To: rune(3530)}, + &CodePointRange{From: rune(3538), To: rune(3540)}, + &CodePointRange{From: rune(3542), To: rune(3542)}, + &CodePointRange{From: rune(3633), To: rune(3633)}, + &CodePointRange{From: rune(3636), To: rune(3642)}, + &CodePointRange{From: rune(3655), To: rune(3662)}, + &CodePointRange{From: rune(3761), To: rune(3761)}, + &CodePointRange{From: rune(3764), To: rune(3772)}, + &CodePointRange{From: rune(3784), To: rune(3789)}, + &CodePointRange{From: rune(3864), To: rune(3865)}, + &CodePointRange{From: rune(3893), To: rune(3893)}, + &CodePointRange{From: rune(3895), To: rune(3895)}, + &CodePointRange{From: rune(3897), To: rune(3897)}, + &CodePointRange{From: rune(3953), To: rune(3966)}, + &CodePointRange{From: rune(3968), To: rune(3972)}, + &CodePointRange{From: rune(3974), To: rune(3975)}, + &CodePointRange{From: rune(3981), To: rune(3991)}, + &CodePointRange{From: rune(3993), To: rune(4028)}, + &CodePointRange{From: rune(4038), To: rune(4038)}, + &CodePointRange{From: rune(4141), To: rune(4144)}, + &CodePointRange{From: rune(4146), To: rune(4151)}, + &CodePointRange{From: rune(4153), To: rune(4154)}, + &CodePointRange{From: rune(4157), To: rune(4158)}, + &CodePointRange{From: rune(4184), To: rune(4185)}, + &CodePointRange{From: rune(4190), To: rune(4192)}, + &CodePointRange{From: rune(4209), To: rune(4212)}, + &CodePointRange{From: rune(4226), To: rune(4226)}, + &CodePointRange{From: rune(4229), To: rune(4230)}, + &CodePointRange{From: rune(4237), To: rune(4237)}, + &CodePointRange{From: rune(4253), To: rune(4253)}, + &CodePointRange{From: rune(4957), To: rune(4959)}, + &CodePointRange{From: rune(5906), To: rune(5908)}, + &CodePointRange{From: rune(5938), To: rune(5940)}, + &CodePointRange{From: rune(5970), To: rune(5971)}, + &CodePointRange{From: rune(6002), To: rune(6003)}, + &CodePointRange{From: rune(6068), To: rune(6069)}, + &CodePointRange{From: rune(6071), To: rune(6077)}, + &CodePointRange{From: rune(6086), To: rune(6086)}, + &CodePointRange{From: rune(6089), To: rune(6099)}, + &CodePointRange{From: rune(6109), To: rune(6109)}, + &CodePointRange{From: rune(6155), To: rune(6157)}, + &CodePointRange{From: rune(6277), To: rune(6278)}, + &CodePointRange{From: rune(6313), To: rune(6313)}, + &CodePointRange{From: rune(6432), To: rune(6434)}, + &CodePointRange{From: rune(6439), To: rune(6440)}, + &CodePointRange{From: rune(6450), To: rune(6450)}, + &CodePointRange{From: rune(6457), To: rune(6459)}, + &CodePointRange{From: rune(6679), To: rune(6680)}, + &CodePointRange{From: rune(6683), To: rune(6683)}, + &CodePointRange{From: rune(6742), To: rune(6742)}, + &CodePointRange{From: rune(6744), To: rune(6750)}, + &CodePointRange{From: rune(6752), To: rune(6752)}, + &CodePointRange{From: rune(6754), To: rune(6754)}, + &CodePointRange{From: rune(6757), To: rune(6764)}, + &CodePointRange{From: rune(6771), To: rune(6780)}, + &CodePointRange{From: rune(6783), To: rune(6783)}, + &CodePointRange{From: rune(6832), To: rune(6845)}, + &CodePointRange{From: rune(6847), To: rune(6848)}, + &CodePointRange{From: rune(6912), To: rune(6915)}, + &CodePointRange{From: rune(6964), To: rune(6964)}, + &CodePointRange{From: rune(6966), To: rune(6970)}, + &CodePointRange{From: rune(6972), To: rune(6972)}, + &CodePointRange{From: rune(6978), To: rune(6978)}, + &CodePointRange{From: rune(7019), To: rune(7027)}, + &CodePointRange{From: rune(7040), To: rune(7041)}, + &CodePointRange{From: rune(7074), To: rune(7077)}, + &CodePointRange{From: rune(7080), To: rune(7081)}, + &CodePointRange{From: rune(7083), To: rune(7085)}, + &CodePointRange{From: rune(7142), To: rune(7142)}, + &CodePointRange{From: rune(7144), To: rune(7145)}, + &CodePointRange{From: rune(7149), To: rune(7149)}, + &CodePointRange{From: rune(7151), To: rune(7153)}, + &CodePointRange{From: rune(7212), To: rune(7219)}, + &CodePointRange{From: rune(7222), To: rune(7223)}, + &CodePointRange{From: rune(7376), To: rune(7378)}, + &CodePointRange{From: rune(7380), To: rune(7392)}, + &CodePointRange{From: rune(7394), To: rune(7400)}, + &CodePointRange{From: rune(7405), To: rune(7405)}, + &CodePointRange{From: rune(7412), To: rune(7412)}, + &CodePointRange{From: rune(7416), To: rune(7417)}, + &CodePointRange{From: rune(7616), To: rune(7673)}, + &CodePointRange{From: rune(7675), To: rune(7679)}, + &CodePointRange{From: rune(8400), To: rune(8412)}, + &CodePointRange{From: rune(8417), To: rune(8417)}, + &CodePointRange{From: rune(8421), To: rune(8432)}, + &CodePointRange{From: rune(11503), To: rune(11505)}, + &CodePointRange{From: rune(11647), To: rune(11647)}, + &CodePointRange{From: rune(11744), To: rune(11775)}, + &CodePointRange{From: rune(12330), To: rune(12333)}, + &CodePointRange{From: rune(12441), To: rune(12442)}, + &CodePointRange{From: rune(42607), To: rune(42607)}, + &CodePointRange{From: rune(42612), To: rune(42621)}, + &CodePointRange{From: rune(42654), To: rune(42655)}, + &CodePointRange{From: rune(42736), To: rune(42737)}, + &CodePointRange{From: rune(43010), To: rune(43010)}, + &CodePointRange{From: rune(43014), To: rune(43014)}, + &CodePointRange{From: rune(43019), To: rune(43019)}, + &CodePointRange{From: rune(43045), To: rune(43046)}, + &CodePointRange{From: rune(43052), To: rune(43052)}, + &CodePointRange{From: rune(43204), To: rune(43205)}, + &CodePointRange{From: rune(43232), To: rune(43249)}, + &CodePointRange{From: rune(43263), To: rune(43263)}, + &CodePointRange{From: rune(43302), To: rune(43309)}, + &CodePointRange{From: rune(43335), To: rune(43345)}, + &CodePointRange{From: rune(43392), To: rune(43394)}, + &CodePointRange{From: rune(43443), To: rune(43443)}, + &CodePointRange{From: rune(43446), To: rune(43449)}, + &CodePointRange{From: rune(43452), To: rune(43453)}, + &CodePointRange{From: rune(43493), To: rune(43493)}, + &CodePointRange{From: rune(43561), To: rune(43566)}, + &CodePointRange{From: rune(43569), To: rune(43570)}, + &CodePointRange{From: rune(43573), To: rune(43574)}, + &CodePointRange{From: rune(43587), To: rune(43587)}, + &CodePointRange{From: rune(43596), To: rune(43596)}, + &CodePointRange{From: rune(43644), To: rune(43644)}, + &CodePointRange{From: rune(43696), To: rune(43696)}, + &CodePointRange{From: rune(43698), To: rune(43700)}, + &CodePointRange{From: rune(43703), To: rune(43704)}, + &CodePointRange{From: rune(43710), To: rune(43711)}, + &CodePointRange{From: rune(43713), To: rune(43713)}, + &CodePointRange{From: rune(43756), To: rune(43757)}, + &CodePointRange{From: rune(43766), To: rune(43766)}, + &CodePointRange{From: rune(44005), To: rune(44005)}, + &CodePointRange{From: rune(44008), To: rune(44008)}, + &CodePointRange{From: rune(44013), To: rune(44013)}, + &CodePointRange{From: rune(64286), To: rune(64286)}, + &CodePointRange{From: rune(65024), To: rune(65039)}, + &CodePointRange{From: rune(65056), To: rune(65071)}, + &CodePointRange{From: rune(66045), To: rune(66045)}, + &CodePointRange{From: rune(66272), To: rune(66272)}, + &CodePointRange{From: rune(66422), To: rune(66426)}, + &CodePointRange{From: rune(68097), To: rune(68099)}, + &CodePointRange{From: rune(68101), To: rune(68102)}, + &CodePointRange{From: rune(68108), To: rune(68111)}, + &CodePointRange{From: rune(68152), To: rune(68154)}, + &CodePointRange{From: rune(68159), To: rune(68159)}, + &CodePointRange{From: rune(68325), To: rune(68326)}, + &CodePointRange{From: rune(68900), To: rune(68903)}, + &CodePointRange{From: rune(69291), To: rune(69292)}, + &CodePointRange{From: rune(69446), To: rune(69456)}, + &CodePointRange{From: rune(69633), To: rune(69633)}, + &CodePointRange{From: rune(69688), To: rune(69702)}, + &CodePointRange{From: rune(69759), To: rune(69761)}, + &CodePointRange{From: rune(69811), To: rune(69814)}, + &CodePointRange{From: rune(69817), To: rune(69818)}, + &CodePointRange{From: rune(69888), To: rune(69890)}, + &CodePointRange{From: rune(69927), To: rune(69931)}, + &CodePointRange{From: rune(69933), To: rune(69940)}, + &CodePointRange{From: rune(70003), To: rune(70003)}, + &CodePointRange{From: rune(70016), To: rune(70017)}, + &CodePointRange{From: rune(70070), To: rune(70078)}, + &CodePointRange{From: rune(70089), To: rune(70092)}, + &CodePointRange{From: rune(70095), To: rune(70095)}, + &CodePointRange{From: rune(70191), To: rune(70193)}, + &CodePointRange{From: rune(70196), To: rune(70196)}, + &CodePointRange{From: rune(70198), To: rune(70199)}, + &CodePointRange{From: rune(70206), To: rune(70206)}, + &CodePointRange{From: rune(70367), To: rune(70367)}, + &CodePointRange{From: rune(70371), To: rune(70378)}, + &CodePointRange{From: rune(70400), To: rune(70401)}, + &CodePointRange{From: rune(70459), To: rune(70460)}, + &CodePointRange{From: rune(70464), To: rune(70464)}, + &CodePointRange{From: rune(70502), To: rune(70508)}, + &CodePointRange{From: rune(70512), To: rune(70516)}, + &CodePointRange{From: rune(70712), To: rune(70719)}, + &CodePointRange{From: rune(70722), To: rune(70724)}, + &CodePointRange{From: rune(70726), To: rune(70726)}, + &CodePointRange{From: rune(70750), To: rune(70750)}, + &CodePointRange{From: rune(70835), To: rune(70840)}, + &CodePointRange{From: rune(70842), To: rune(70842)}, + &CodePointRange{From: rune(70847), To: rune(70848)}, + &CodePointRange{From: rune(70850), To: rune(70851)}, + &CodePointRange{From: rune(71090), To: rune(71093)}, + &CodePointRange{From: rune(71100), To: rune(71101)}, + &CodePointRange{From: rune(71103), To: rune(71104)}, + &CodePointRange{From: rune(71132), To: rune(71133)}, + &CodePointRange{From: rune(71219), To: rune(71226)}, + &CodePointRange{From: rune(71229), To: rune(71229)}, + &CodePointRange{From: rune(71231), To: rune(71232)}, + &CodePointRange{From: rune(71339), To: rune(71339)}, + &CodePointRange{From: rune(71341), To: rune(71341)}, + &CodePointRange{From: rune(71344), To: rune(71349)}, + &CodePointRange{From: rune(71351), To: rune(71351)}, + &CodePointRange{From: rune(71453), To: rune(71455)}, + &CodePointRange{From: rune(71458), To: rune(71461)}, + &CodePointRange{From: rune(71463), To: rune(71467)}, + &CodePointRange{From: rune(71727), To: rune(71735)}, + &CodePointRange{From: rune(71737), To: rune(71738)}, + &CodePointRange{From: rune(71995), To: rune(71996)}, + &CodePointRange{From: rune(71998), To: rune(71998)}, + &CodePointRange{From: rune(72003), To: rune(72003)}, + &CodePointRange{From: rune(72148), To: rune(72151)}, + &CodePointRange{From: rune(72154), To: rune(72155)}, + &CodePointRange{From: rune(72160), To: rune(72160)}, + &CodePointRange{From: rune(72193), To: rune(72202)}, + &CodePointRange{From: rune(72243), To: rune(72248)}, + &CodePointRange{From: rune(72251), To: rune(72254)}, + &CodePointRange{From: rune(72263), To: rune(72263)}, + &CodePointRange{From: rune(72273), To: rune(72278)}, + &CodePointRange{From: rune(72281), To: rune(72283)}, + &CodePointRange{From: rune(72330), To: rune(72342)}, + &CodePointRange{From: rune(72344), To: rune(72345)}, + &CodePointRange{From: rune(72752), To: rune(72758)}, + &CodePointRange{From: rune(72760), To: rune(72765)}, + &CodePointRange{From: rune(72767), To: rune(72767)}, + &CodePointRange{From: rune(72850), To: rune(72871)}, + &CodePointRange{From: rune(72874), To: rune(72880)}, + &CodePointRange{From: rune(72882), To: rune(72883)}, + &CodePointRange{From: rune(72885), To: rune(72886)}, + &CodePointRange{From: rune(73009), To: rune(73014)}, + &CodePointRange{From: rune(73018), To: rune(73018)}, + &CodePointRange{From: rune(73020), To: rune(73021)}, + &CodePointRange{From: rune(73023), To: rune(73029)}, + &CodePointRange{From: rune(73031), To: rune(73031)}, + &CodePointRange{From: rune(73104), To: rune(73105)}, + &CodePointRange{From: rune(73109), To: rune(73109)}, + &CodePointRange{From: rune(73111), To: rune(73111)}, + &CodePointRange{From: rune(73459), To: rune(73460)}, + &CodePointRange{From: rune(92912), To: rune(92916)}, + &CodePointRange{From: rune(92976), To: rune(92982)}, + &CodePointRange{From: rune(94031), To: rune(94031)}, + &CodePointRange{From: rune(94095), To: rune(94098)}, + &CodePointRange{From: rune(94180), To: rune(94180)}, + &CodePointRange{From: rune(113821), To: rune(113822)}, + &CodePointRange{From: rune(119143), To: rune(119145)}, + &CodePointRange{From: rune(119163), To: rune(119170)}, + &CodePointRange{From: rune(119173), To: rune(119179)}, + &CodePointRange{From: rune(119210), To: rune(119213)}, + &CodePointRange{From: rune(119362), To: rune(119364)}, + &CodePointRange{From: rune(121344), To: rune(121398)}, + &CodePointRange{From: rune(121403), To: rune(121452)}, + &CodePointRange{From: rune(121461), To: rune(121461)}, + &CodePointRange{From: rune(121476), To: rune(121476)}, + &CodePointRange{From: rune(121499), To: rune(121503)}, + &CodePointRange{From: rune(121505), To: rune(121519)}, + &CodePointRange{From: rune(122880), To: rune(122886)}, + &CodePointRange{From: rune(122888), To: rune(122904)}, + &CodePointRange{From: rune(122907), To: rune(122913)}, + &CodePointRange{From: rune(122915), To: rune(122916)}, + &CodePointRange{From: rune(122918), To: rune(122922)}, + &CodePointRange{From: rune(123184), To: rune(123190)}, + &CodePointRange{From: rune(123628), To: rune(123631)}, + &CodePointRange{From: rune(125136), To: rune(125142)}, + &CodePointRange{From: rune(125252), To: rune(125258)}, + &CodePointRange{From: rune(917760), To: rune(917999)}, + }, + "nd": { + &CodePointRange{From: rune(48), To: rune(57)}, + &CodePointRange{From: rune(1632), To: rune(1641)}, + &CodePointRange{From: rune(1776), To: rune(1785)}, + &CodePointRange{From: rune(1984), To: rune(1993)}, + &CodePointRange{From: rune(2406), To: rune(2415)}, + &CodePointRange{From: rune(2534), To: rune(2543)}, + &CodePointRange{From: rune(2662), To: rune(2671)}, + &CodePointRange{From: rune(2790), To: rune(2799)}, + &CodePointRange{From: rune(2918), To: rune(2927)}, + &CodePointRange{From: rune(3046), To: rune(3055)}, + &CodePointRange{From: rune(3174), To: rune(3183)}, + &CodePointRange{From: rune(3302), To: rune(3311)}, + &CodePointRange{From: rune(3430), To: rune(3439)}, + &CodePointRange{From: rune(3558), To: rune(3567)}, + &CodePointRange{From: rune(3664), To: rune(3673)}, + &CodePointRange{From: rune(3792), To: rune(3801)}, + &CodePointRange{From: rune(3872), To: rune(3881)}, + &CodePointRange{From: rune(4160), To: rune(4169)}, + &CodePointRange{From: rune(4240), To: rune(4249)}, + &CodePointRange{From: rune(6112), To: rune(6121)}, + &CodePointRange{From: rune(6160), To: rune(6169)}, + &CodePointRange{From: rune(6470), To: rune(6479)}, + &CodePointRange{From: rune(6608), To: rune(6617)}, + &CodePointRange{From: rune(6784), To: rune(6793)}, + &CodePointRange{From: rune(6800), To: rune(6809)}, + &CodePointRange{From: rune(6992), To: rune(7001)}, + &CodePointRange{From: rune(7088), To: rune(7097)}, + &CodePointRange{From: rune(7232), To: rune(7241)}, + &CodePointRange{From: rune(7248), To: rune(7257)}, + &CodePointRange{From: rune(42528), To: rune(42537)}, + &CodePointRange{From: rune(43216), To: rune(43225)}, + &CodePointRange{From: rune(43264), To: rune(43273)}, + &CodePointRange{From: rune(43472), To: rune(43481)}, + &CodePointRange{From: rune(43504), To: rune(43513)}, + &CodePointRange{From: rune(43600), To: rune(43609)}, + &CodePointRange{From: rune(44016), To: rune(44025)}, + &CodePointRange{From: rune(65296), To: rune(65305)}, + &CodePointRange{From: rune(66720), To: rune(66729)}, + &CodePointRange{From: rune(68912), To: rune(68921)}, + &CodePointRange{From: rune(69734), To: rune(69743)}, + &CodePointRange{From: rune(69872), To: rune(69881)}, + &CodePointRange{From: rune(69942), To: rune(69951)}, + &CodePointRange{From: rune(70096), To: rune(70105)}, + &CodePointRange{From: rune(70384), To: rune(70393)}, + &CodePointRange{From: rune(70736), To: rune(70745)}, + &CodePointRange{From: rune(70864), To: rune(70873)}, + &CodePointRange{From: rune(71248), To: rune(71257)}, + &CodePointRange{From: rune(71360), To: rune(71369)}, + &CodePointRange{From: rune(71472), To: rune(71481)}, + &CodePointRange{From: rune(71904), To: rune(71913)}, + &CodePointRange{From: rune(72016), To: rune(72025)}, + &CodePointRange{From: rune(72784), To: rune(72793)}, + &CodePointRange{From: rune(73040), To: rune(73049)}, + &CodePointRange{From: rune(73120), To: rune(73129)}, + &CodePointRange{From: rune(92768), To: rune(92777)}, + &CodePointRange{From: rune(93008), To: rune(93017)}, + &CodePointRange{From: rune(120782), To: rune(120831)}, + &CodePointRange{From: rune(123200), To: rune(123209)}, + &CodePointRange{From: rune(123632), To: rune(123641)}, + &CodePointRange{From: rune(125264), To: rune(125273)}, + &CodePointRange{From: rune(130032), To: rune(130041)}, + }, + "nl": { + &CodePointRange{From: rune(5870), To: rune(5872)}, + &CodePointRange{From: rune(8544), To: rune(8578)}, + &CodePointRange{From: rune(8581), To: rune(8584)}, + &CodePointRange{From: rune(12295), To: rune(12295)}, + &CodePointRange{From: rune(12321), To: rune(12329)}, + &CodePointRange{From: rune(12344), To: rune(12346)}, + &CodePointRange{From: rune(42726), To: rune(42735)}, + &CodePointRange{From: rune(65856), To: rune(65908)}, + &CodePointRange{From: rune(66369), To: rune(66369)}, + &CodePointRange{From: rune(66378), To: rune(66378)}, + &CodePointRange{From: rune(66513), To: rune(66517)}, + &CodePointRange{From: rune(74752), To: rune(74862)}, + }, + "no": { + &CodePointRange{From: rune(178), To: rune(179)}, + &CodePointRange{From: rune(185), To: rune(185)}, + &CodePointRange{From: rune(188), To: rune(190)}, + &CodePointRange{From: rune(2548), To: rune(2553)}, + &CodePointRange{From: rune(2930), To: rune(2935)}, + &CodePointRange{From: rune(3056), To: rune(3058)}, + &CodePointRange{From: rune(3192), To: rune(3198)}, + &CodePointRange{From: rune(3416), To: rune(3422)}, + &CodePointRange{From: rune(3440), To: rune(3448)}, + &CodePointRange{From: rune(3882), To: rune(3891)}, + &CodePointRange{From: rune(4969), To: rune(4988)}, + &CodePointRange{From: rune(6128), To: rune(6137)}, + &CodePointRange{From: rune(6618), To: rune(6618)}, + &CodePointRange{From: rune(8304), To: rune(8304)}, + &CodePointRange{From: rune(8308), To: rune(8313)}, + &CodePointRange{From: rune(8320), To: rune(8329)}, + &CodePointRange{From: rune(8528), To: rune(8543)}, + &CodePointRange{From: rune(8585), To: rune(8585)}, + &CodePointRange{From: rune(9312), To: rune(9371)}, + &CodePointRange{From: rune(9450), To: rune(9471)}, + &CodePointRange{From: rune(10102), To: rune(10131)}, + &CodePointRange{From: rune(11517), To: rune(11517)}, + &CodePointRange{From: rune(12690), To: rune(12693)}, + &CodePointRange{From: rune(12832), To: rune(12841)}, + &CodePointRange{From: rune(12872), To: rune(12879)}, + &CodePointRange{From: rune(12881), To: rune(12895)}, + &CodePointRange{From: rune(12928), To: rune(12937)}, + &CodePointRange{From: rune(12977), To: rune(12991)}, + &CodePointRange{From: rune(43056), To: rune(43061)}, + &CodePointRange{From: rune(65799), To: rune(65843)}, + &CodePointRange{From: rune(65909), To: rune(65912)}, + &CodePointRange{From: rune(65930), To: rune(65931)}, + &CodePointRange{From: rune(66273), To: rune(66299)}, + &CodePointRange{From: rune(66336), To: rune(66339)}, + &CodePointRange{From: rune(67672), To: rune(67679)}, + &CodePointRange{From: rune(67705), To: rune(67711)}, + &CodePointRange{From: rune(67751), To: rune(67759)}, + &CodePointRange{From: rune(67835), To: rune(67839)}, + &CodePointRange{From: rune(67862), To: rune(67867)}, + &CodePointRange{From: rune(68028), To: rune(68029)}, + &CodePointRange{From: rune(68032), To: rune(68047)}, + &CodePointRange{From: rune(68050), To: rune(68095)}, + &CodePointRange{From: rune(68160), To: rune(68168)}, + &CodePointRange{From: rune(68221), To: rune(68222)}, + &CodePointRange{From: rune(68253), To: rune(68255)}, + &CodePointRange{From: rune(68331), To: rune(68335)}, + &CodePointRange{From: rune(68440), To: rune(68447)}, + &CodePointRange{From: rune(68472), To: rune(68479)}, + &CodePointRange{From: rune(68521), To: rune(68527)}, + &CodePointRange{From: rune(68858), To: rune(68863)}, + &CodePointRange{From: rune(69216), To: rune(69246)}, + &CodePointRange{From: rune(69405), To: rune(69414)}, + &CodePointRange{From: rune(69457), To: rune(69460)}, + &CodePointRange{From: rune(69573), To: rune(69579)}, + &CodePointRange{From: rune(69714), To: rune(69733)}, + &CodePointRange{From: rune(70113), To: rune(70132)}, + &CodePointRange{From: rune(71482), To: rune(71483)}, + &CodePointRange{From: rune(71914), To: rune(71922)}, + &CodePointRange{From: rune(72794), To: rune(72812)}, + &CodePointRange{From: rune(73664), To: rune(73684)}, + &CodePointRange{From: rune(93019), To: rune(93025)}, + &CodePointRange{From: rune(93824), To: rune(93846)}, + &CodePointRange{From: rune(119520), To: rune(119539)}, + &CodePointRange{From: rune(119648), To: rune(119672)}, + &CodePointRange{From: rune(125127), To: rune(125135)}, + &CodePointRange{From: rune(126065), To: rune(126123)}, + &CodePointRange{From: rune(126125), To: rune(126127)}, + &CodePointRange{From: rune(126129), To: rune(126132)}, + &CodePointRange{From: rune(126209), To: rune(126253)}, + &CodePointRange{From: rune(126255), To: rune(126269)}, + &CodePointRange{From: rune(127232), To: rune(127244)}, + }, + "pc": { + &CodePointRange{From: rune(95), To: rune(95)}, + &CodePointRange{From: rune(8255), To: rune(8256)}, + &CodePointRange{From: rune(8276), To: rune(8276)}, + &CodePointRange{From: rune(65075), To: rune(65076)}, + &CodePointRange{From: rune(65101), To: rune(65103)}, + &CodePointRange{From: rune(65343), To: rune(65343)}, + }, + "pd": { + &CodePointRange{From: rune(45), To: rune(45)}, + &CodePointRange{From: rune(1418), To: rune(1418)}, + &CodePointRange{From: rune(1470), To: rune(1470)}, + &CodePointRange{From: rune(5120), To: rune(5120)}, + &CodePointRange{From: rune(6150), To: rune(6150)}, + &CodePointRange{From: rune(8208), To: rune(8213)}, + &CodePointRange{From: rune(11799), To: rune(11799)}, + &CodePointRange{From: rune(11802), To: rune(11802)}, + &CodePointRange{From: rune(11834), To: rune(11835)}, + &CodePointRange{From: rune(11840), To: rune(11840)}, + &CodePointRange{From: rune(12316), To: rune(12316)}, + &CodePointRange{From: rune(12336), To: rune(12336)}, + &CodePointRange{From: rune(12448), To: rune(12448)}, + &CodePointRange{From: rune(65073), To: rune(65074)}, + &CodePointRange{From: rune(65112), To: rune(65112)}, + &CodePointRange{From: rune(65123), To: rune(65123)}, + &CodePointRange{From: rune(65293), To: rune(65293)}, + &CodePointRange{From: rune(69293), To: rune(69293)}, + }, + "pe": { + &CodePointRange{From: rune(41), To: rune(41)}, + &CodePointRange{From: rune(93), To: rune(93)}, + &CodePointRange{From: rune(125), To: rune(125)}, + &CodePointRange{From: rune(3899), To: rune(3899)}, + &CodePointRange{From: rune(3901), To: rune(3901)}, + &CodePointRange{From: rune(5788), To: rune(5788)}, + &CodePointRange{From: rune(8262), To: rune(8262)}, + &CodePointRange{From: rune(8318), To: rune(8318)}, + &CodePointRange{From: rune(8334), To: rune(8334)}, + &CodePointRange{From: rune(8969), To: rune(8969)}, + &CodePointRange{From: rune(8971), To: rune(8971)}, + &CodePointRange{From: rune(9002), To: rune(9002)}, + &CodePointRange{From: rune(10089), To: rune(10089)}, + &CodePointRange{From: rune(10091), To: rune(10091)}, + &CodePointRange{From: rune(10093), To: rune(10093)}, + &CodePointRange{From: rune(10095), To: rune(10095)}, + &CodePointRange{From: rune(10097), To: rune(10097)}, + &CodePointRange{From: rune(10099), To: rune(10099)}, + &CodePointRange{From: rune(10101), To: rune(10101)}, + &CodePointRange{From: rune(10182), To: rune(10182)}, + &CodePointRange{From: rune(10215), To: rune(10215)}, + &CodePointRange{From: rune(10217), To: rune(10217)}, + &CodePointRange{From: rune(10219), To: rune(10219)}, + &CodePointRange{From: rune(10221), To: rune(10221)}, + &CodePointRange{From: rune(10223), To: rune(10223)}, + &CodePointRange{From: rune(10628), To: rune(10628)}, + &CodePointRange{From: rune(10630), To: rune(10630)}, + &CodePointRange{From: rune(10632), To: rune(10632)}, + &CodePointRange{From: rune(10634), To: rune(10634)}, + &CodePointRange{From: rune(10636), To: rune(10636)}, + &CodePointRange{From: rune(10638), To: rune(10638)}, + &CodePointRange{From: rune(10640), To: rune(10640)}, + &CodePointRange{From: rune(10642), To: rune(10642)}, + &CodePointRange{From: rune(10644), To: rune(10644)}, + &CodePointRange{From: rune(10646), To: rune(10646)}, + &CodePointRange{From: rune(10648), To: rune(10648)}, + &CodePointRange{From: rune(10713), To: rune(10713)}, + &CodePointRange{From: rune(10715), To: rune(10715)}, + &CodePointRange{From: rune(10749), To: rune(10749)}, + &CodePointRange{From: rune(11811), To: rune(11811)}, + &CodePointRange{From: rune(11813), To: rune(11813)}, + &CodePointRange{From: rune(11815), To: rune(11815)}, + &CodePointRange{From: rune(11817), To: rune(11817)}, + &CodePointRange{From: rune(12297), To: rune(12297)}, + &CodePointRange{From: rune(12299), To: rune(12299)}, + &CodePointRange{From: rune(12301), To: rune(12301)}, + &CodePointRange{From: rune(12303), To: rune(12303)}, + &CodePointRange{From: rune(12305), To: rune(12305)}, + &CodePointRange{From: rune(12309), To: rune(12309)}, + &CodePointRange{From: rune(12311), To: rune(12311)}, + &CodePointRange{From: rune(12313), To: rune(12313)}, + &CodePointRange{From: rune(12315), To: rune(12315)}, + &CodePointRange{From: rune(12318), To: rune(12319)}, + &CodePointRange{From: rune(64830), To: rune(64830)}, + &CodePointRange{From: rune(65048), To: rune(65048)}, + &CodePointRange{From: rune(65078), To: rune(65078)}, + &CodePointRange{From: rune(65080), To: rune(65080)}, + &CodePointRange{From: rune(65082), To: rune(65082)}, + &CodePointRange{From: rune(65084), To: rune(65084)}, + &CodePointRange{From: rune(65086), To: rune(65086)}, + &CodePointRange{From: rune(65088), To: rune(65088)}, + &CodePointRange{From: rune(65090), To: rune(65090)}, + &CodePointRange{From: rune(65092), To: rune(65092)}, + &CodePointRange{From: rune(65096), To: rune(65096)}, + &CodePointRange{From: rune(65114), To: rune(65114)}, + &CodePointRange{From: rune(65116), To: rune(65116)}, + &CodePointRange{From: rune(65118), To: rune(65118)}, + &CodePointRange{From: rune(65289), To: rune(65289)}, + &CodePointRange{From: rune(65341), To: rune(65341)}, + &CodePointRange{From: rune(65373), To: rune(65373)}, + &CodePointRange{From: rune(65376), To: rune(65376)}, + &CodePointRange{From: rune(65379), To: rune(65379)}, + }, + "pf": { + &CodePointRange{From: rune(187), To: rune(187)}, + &CodePointRange{From: rune(8217), To: rune(8217)}, + &CodePointRange{From: rune(8221), To: rune(8221)}, + &CodePointRange{From: rune(8250), To: rune(8250)}, + &CodePointRange{From: rune(11779), To: rune(11779)}, + &CodePointRange{From: rune(11781), To: rune(11781)}, + &CodePointRange{From: rune(11786), To: rune(11786)}, + &CodePointRange{From: rune(11789), To: rune(11789)}, + &CodePointRange{From: rune(11805), To: rune(11805)}, + &CodePointRange{From: rune(11809), To: rune(11809)}, + }, + "pi": { + &CodePointRange{From: rune(171), To: rune(171)}, + &CodePointRange{From: rune(8216), To: rune(8216)}, + &CodePointRange{From: rune(8219), To: rune(8220)}, + &CodePointRange{From: rune(8223), To: rune(8223)}, + &CodePointRange{From: rune(8249), To: rune(8249)}, + &CodePointRange{From: rune(11778), To: rune(11778)}, + &CodePointRange{From: rune(11780), To: rune(11780)}, + &CodePointRange{From: rune(11785), To: rune(11785)}, + &CodePointRange{From: rune(11788), To: rune(11788)}, + &CodePointRange{From: rune(11804), To: rune(11804)}, + &CodePointRange{From: rune(11808), To: rune(11808)}, + }, + "po": { + &CodePointRange{From: rune(33), To: rune(35)}, + &CodePointRange{From: rune(37), To: rune(39)}, + &CodePointRange{From: rune(42), To: rune(42)}, + &CodePointRange{From: rune(44), To: rune(44)}, + &CodePointRange{From: rune(46), To: rune(47)}, + &CodePointRange{From: rune(58), To: rune(59)}, + &CodePointRange{From: rune(63), To: rune(64)}, + &CodePointRange{From: rune(92), To: rune(92)}, + &CodePointRange{From: rune(161), To: rune(161)}, + &CodePointRange{From: rune(167), To: rune(167)}, + &CodePointRange{From: rune(182), To: rune(183)}, + &CodePointRange{From: rune(191), To: rune(191)}, + &CodePointRange{From: rune(894), To: rune(894)}, + &CodePointRange{From: rune(903), To: rune(903)}, + &CodePointRange{From: rune(1370), To: rune(1375)}, + &CodePointRange{From: rune(1417), To: rune(1417)}, + &CodePointRange{From: rune(1472), To: rune(1472)}, + &CodePointRange{From: rune(1475), To: rune(1475)}, + &CodePointRange{From: rune(1478), To: rune(1478)}, + &CodePointRange{From: rune(1523), To: rune(1524)}, + &CodePointRange{From: rune(1545), To: rune(1546)}, + &CodePointRange{From: rune(1548), To: rune(1549)}, + &CodePointRange{From: rune(1563), To: rune(1563)}, + &CodePointRange{From: rune(1566), To: rune(1567)}, + &CodePointRange{From: rune(1642), To: rune(1645)}, + &CodePointRange{From: rune(1748), To: rune(1748)}, + &CodePointRange{From: rune(1792), To: rune(1805)}, + &CodePointRange{From: rune(2039), To: rune(2041)}, + &CodePointRange{From: rune(2096), To: rune(2110)}, + &CodePointRange{From: rune(2142), To: rune(2142)}, + &CodePointRange{From: rune(2404), To: rune(2405)}, + &CodePointRange{From: rune(2416), To: rune(2416)}, + &CodePointRange{From: rune(2557), To: rune(2557)}, + &CodePointRange{From: rune(2678), To: rune(2678)}, + &CodePointRange{From: rune(2800), To: rune(2800)}, + &CodePointRange{From: rune(3191), To: rune(3191)}, + &CodePointRange{From: rune(3204), To: rune(3204)}, + &CodePointRange{From: rune(3572), To: rune(3572)}, + &CodePointRange{From: rune(3663), To: rune(3663)}, + &CodePointRange{From: rune(3674), To: rune(3675)}, + &CodePointRange{From: rune(3844), To: rune(3858)}, + &CodePointRange{From: rune(3860), To: rune(3860)}, + &CodePointRange{From: rune(3973), To: rune(3973)}, + &CodePointRange{From: rune(4048), To: rune(4052)}, + &CodePointRange{From: rune(4057), To: rune(4058)}, + &CodePointRange{From: rune(4170), To: rune(4175)}, + &CodePointRange{From: rune(4347), To: rune(4347)}, + &CodePointRange{From: rune(4960), To: rune(4968)}, + &CodePointRange{From: rune(5742), To: rune(5742)}, + &CodePointRange{From: rune(5867), To: rune(5869)}, + &CodePointRange{From: rune(5941), To: rune(5942)}, + &CodePointRange{From: rune(6100), To: rune(6102)}, + &CodePointRange{From: rune(6104), To: rune(6106)}, + &CodePointRange{From: rune(6144), To: rune(6149)}, + &CodePointRange{From: rune(6151), To: rune(6154)}, + &CodePointRange{From: rune(6468), To: rune(6469)}, + &CodePointRange{From: rune(6686), To: rune(6687)}, + &CodePointRange{From: rune(6816), To: rune(6822)}, + &CodePointRange{From: rune(6824), To: rune(6829)}, + &CodePointRange{From: rune(7002), To: rune(7008)}, + &CodePointRange{From: rune(7164), To: rune(7167)}, + &CodePointRange{From: rune(7227), To: rune(7231)}, + &CodePointRange{From: rune(7294), To: rune(7295)}, + &CodePointRange{From: rune(7360), To: rune(7367)}, + &CodePointRange{From: rune(7379), To: rune(7379)}, + &CodePointRange{From: rune(8214), To: rune(8215)}, + &CodePointRange{From: rune(8224), To: rune(8231)}, + &CodePointRange{From: rune(8240), To: rune(8248)}, + &CodePointRange{From: rune(8251), To: rune(8254)}, + &CodePointRange{From: rune(8257), To: rune(8259)}, + &CodePointRange{From: rune(8263), To: rune(8273)}, + &CodePointRange{From: rune(8275), To: rune(8275)}, + &CodePointRange{From: rune(8277), To: rune(8286)}, + &CodePointRange{From: rune(11513), To: rune(11516)}, + &CodePointRange{From: rune(11518), To: rune(11519)}, + &CodePointRange{From: rune(11632), To: rune(11632)}, + &CodePointRange{From: rune(11776), To: rune(11777)}, + &CodePointRange{From: rune(11782), To: rune(11784)}, + &CodePointRange{From: rune(11787), To: rune(11787)}, + &CodePointRange{From: rune(11790), To: rune(11798)}, + &CodePointRange{From: rune(11800), To: rune(11801)}, + &CodePointRange{From: rune(11803), To: rune(11803)}, + &CodePointRange{From: rune(11806), To: rune(11807)}, + &CodePointRange{From: rune(11818), To: rune(11822)}, + &CodePointRange{From: rune(11824), To: rune(11833)}, + &CodePointRange{From: rune(11836), To: rune(11839)}, + &CodePointRange{From: rune(11841), To: rune(11841)}, + &CodePointRange{From: rune(11843), To: rune(11855)}, + &CodePointRange{From: rune(11858), To: rune(11858)}, + &CodePointRange{From: rune(12289), To: rune(12291)}, + &CodePointRange{From: rune(12349), To: rune(12349)}, + &CodePointRange{From: rune(12539), To: rune(12539)}, + &CodePointRange{From: rune(42238), To: rune(42239)}, + &CodePointRange{From: rune(42509), To: rune(42511)}, + &CodePointRange{From: rune(42611), To: rune(42611)}, + &CodePointRange{From: rune(42622), To: rune(42622)}, + &CodePointRange{From: rune(42738), To: rune(42743)}, + &CodePointRange{From: rune(43124), To: rune(43127)}, + &CodePointRange{From: rune(43214), To: rune(43215)}, + &CodePointRange{From: rune(43256), To: rune(43258)}, + &CodePointRange{From: rune(43260), To: rune(43260)}, + &CodePointRange{From: rune(43310), To: rune(43311)}, + &CodePointRange{From: rune(43359), To: rune(43359)}, + &CodePointRange{From: rune(43457), To: rune(43469)}, + &CodePointRange{From: rune(43486), To: rune(43487)}, + &CodePointRange{From: rune(43612), To: rune(43615)}, + &CodePointRange{From: rune(43742), To: rune(43743)}, + &CodePointRange{From: rune(43760), To: rune(43761)}, + &CodePointRange{From: rune(44011), To: rune(44011)}, + &CodePointRange{From: rune(65040), To: rune(65046)}, + &CodePointRange{From: rune(65049), To: rune(65049)}, + &CodePointRange{From: rune(65072), To: rune(65072)}, + &CodePointRange{From: rune(65093), To: rune(65094)}, + &CodePointRange{From: rune(65097), To: rune(65100)}, + &CodePointRange{From: rune(65104), To: rune(65106)}, + &CodePointRange{From: rune(65108), To: rune(65111)}, + &CodePointRange{From: rune(65119), To: rune(65121)}, + &CodePointRange{From: rune(65128), To: rune(65128)}, + &CodePointRange{From: rune(65130), To: rune(65131)}, + &CodePointRange{From: rune(65281), To: rune(65283)}, + &CodePointRange{From: rune(65285), To: rune(65287)}, + &CodePointRange{From: rune(65290), To: rune(65290)}, + &CodePointRange{From: rune(65292), To: rune(65292)}, + &CodePointRange{From: rune(65294), To: rune(65295)}, + &CodePointRange{From: rune(65306), To: rune(65307)}, + &CodePointRange{From: rune(65311), To: rune(65312)}, + &CodePointRange{From: rune(65340), To: rune(65340)}, + &CodePointRange{From: rune(65377), To: rune(65377)}, + &CodePointRange{From: rune(65380), To: rune(65381)}, + &CodePointRange{From: rune(65792), To: rune(65794)}, + &CodePointRange{From: rune(66463), To: rune(66463)}, + &CodePointRange{From: rune(66512), To: rune(66512)}, + &CodePointRange{From: rune(66927), To: rune(66927)}, + &CodePointRange{From: rune(67671), To: rune(67671)}, + &CodePointRange{From: rune(67871), To: rune(67871)}, + &CodePointRange{From: rune(67903), To: rune(67903)}, + &CodePointRange{From: rune(68176), To: rune(68184)}, + &CodePointRange{From: rune(68223), To: rune(68223)}, + &CodePointRange{From: rune(68336), To: rune(68342)}, + &CodePointRange{From: rune(68409), To: rune(68415)}, + &CodePointRange{From: rune(68505), To: rune(68508)}, + &CodePointRange{From: rune(69461), To: rune(69465)}, + &CodePointRange{From: rune(69703), To: rune(69709)}, + &CodePointRange{From: rune(69819), To: rune(69820)}, + &CodePointRange{From: rune(69822), To: rune(69825)}, + &CodePointRange{From: rune(69952), To: rune(69955)}, + &CodePointRange{From: rune(70004), To: rune(70005)}, + &CodePointRange{From: rune(70085), To: rune(70088)}, + &CodePointRange{From: rune(70093), To: rune(70093)}, + &CodePointRange{From: rune(70107), To: rune(70107)}, + &CodePointRange{From: rune(70109), To: rune(70111)}, + &CodePointRange{From: rune(70200), To: rune(70205)}, + &CodePointRange{From: rune(70313), To: rune(70313)}, + &CodePointRange{From: rune(70731), To: rune(70735)}, + &CodePointRange{From: rune(70746), To: rune(70747)}, + &CodePointRange{From: rune(70749), To: rune(70749)}, + &CodePointRange{From: rune(70854), To: rune(70854)}, + &CodePointRange{From: rune(71105), To: rune(71127)}, + &CodePointRange{From: rune(71233), To: rune(71235)}, + &CodePointRange{From: rune(71264), To: rune(71276)}, + &CodePointRange{From: rune(71484), To: rune(71486)}, + &CodePointRange{From: rune(71739), To: rune(71739)}, + &CodePointRange{From: rune(72004), To: rune(72006)}, + &CodePointRange{From: rune(72162), To: rune(72162)}, + &CodePointRange{From: rune(72255), To: rune(72262)}, + &CodePointRange{From: rune(72346), To: rune(72348)}, + &CodePointRange{From: rune(72350), To: rune(72354)}, + &CodePointRange{From: rune(72769), To: rune(72773)}, + &CodePointRange{From: rune(72816), To: rune(72817)}, + &CodePointRange{From: rune(73463), To: rune(73464)}, + &CodePointRange{From: rune(73727), To: rune(73727)}, + &CodePointRange{From: rune(74864), To: rune(74868)}, + &CodePointRange{From: rune(92782), To: rune(92783)}, + &CodePointRange{From: rune(92917), To: rune(92917)}, + &CodePointRange{From: rune(92983), To: rune(92987)}, + &CodePointRange{From: rune(92996), To: rune(92996)}, + &CodePointRange{From: rune(93847), To: rune(93850)}, + &CodePointRange{From: rune(94178), To: rune(94178)}, + &CodePointRange{From: rune(113823), To: rune(113823)}, + &CodePointRange{From: rune(121479), To: rune(121483)}, + &CodePointRange{From: rune(125278), To: rune(125279)}, + }, + "ps": { + &CodePointRange{From: rune(40), To: rune(40)}, + &CodePointRange{From: rune(91), To: rune(91)}, + &CodePointRange{From: rune(123), To: rune(123)}, + &CodePointRange{From: rune(3898), To: rune(3898)}, + &CodePointRange{From: rune(3900), To: rune(3900)}, + &CodePointRange{From: rune(5787), To: rune(5787)}, + &CodePointRange{From: rune(8218), To: rune(8218)}, + &CodePointRange{From: rune(8222), To: rune(8222)}, + &CodePointRange{From: rune(8261), To: rune(8261)}, + &CodePointRange{From: rune(8317), To: rune(8317)}, + &CodePointRange{From: rune(8333), To: rune(8333)}, + &CodePointRange{From: rune(8968), To: rune(8968)}, + &CodePointRange{From: rune(8970), To: rune(8970)}, + &CodePointRange{From: rune(9001), To: rune(9001)}, + &CodePointRange{From: rune(10088), To: rune(10088)}, + &CodePointRange{From: rune(10090), To: rune(10090)}, + &CodePointRange{From: rune(10092), To: rune(10092)}, + &CodePointRange{From: rune(10094), To: rune(10094)}, + &CodePointRange{From: rune(10096), To: rune(10096)}, + &CodePointRange{From: rune(10098), To: rune(10098)}, + &CodePointRange{From: rune(10100), To: rune(10100)}, + &CodePointRange{From: rune(10181), To: rune(10181)}, + &CodePointRange{From: rune(10214), To: rune(10214)}, + &CodePointRange{From: rune(10216), To: rune(10216)}, + &CodePointRange{From: rune(10218), To: rune(10218)}, + &CodePointRange{From: rune(10220), To: rune(10220)}, + &CodePointRange{From: rune(10222), To: rune(10222)}, + &CodePointRange{From: rune(10627), To: rune(10627)}, + &CodePointRange{From: rune(10629), To: rune(10629)}, + &CodePointRange{From: rune(10631), To: rune(10631)}, + &CodePointRange{From: rune(10633), To: rune(10633)}, + &CodePointRange{From: rune(10635), To: rune(10635)}, + &CodePointRange{From: rune(10637), To: rune(10637)}, + &CodePointRange{From: rune(10639), To: rune(10639)}, + &CodePointRange{From: rune(10641), To: rune(10641)}, + &CodePointRange{From: rune(10643), To: rune(10643)}, + &CodePointRange{From: rune(10645), To: rune(10645)}, + &CodePointRange{From: rune(10647), To: rune(10647)}, + &CodePointRange{From: rune(10712), To: rune(10712)}, + &CodePointRange{From: rune(10714), To: rune(10714)}, + &CodePointRange{From: rune(10748), To: rune(10748)}, + &CodePointRange{From: rune(11810), To: rune(11810)}, + &CodePointRange{From: rune(11812), To: rune(11812)}, + &CodePointRange{From: rune(11814), To: rune(11814)}, + &CodePointRange{From: rune(11816), To: rune(11816)}, + &CodePointRange{From: rune(11842), To: rune(11842)}, + &CodePointRange{From: rune(12296), To: rune(12296)}, + &CodePointRange{From: rune(12298), To: rune(12298)}, + &CodePointRange{From: rune(12300), To: rune(12300)}, + &CodePointRange{From: rune(12302), To: rune(12302)}, + &CodePointRange{From: rune(12304), To: rune(12304)}, + &CodePointRange{From: rune(12308), To: rune(12308)}, + &CodePointRange{From: rune(12310), To: rune(12310)}, + &CodePointRange{From: rune(12312), To: rune(12312)}, + &CodePointRange{From: rune(12314), To: rune(12314)}, + &CodePointRange{From: rune(12317), To: rune(12317)}, + &CodePointRange{From: rune(64831), To: rune(64831)}, + &CodePointRange{From: rune(65047), To: rune(65047)}, + &CodePointRange{From: rune(65077), To: rune(65077)}, + &CodePointRange{From: rune(65079), To: rune(65079)}, + &CodePointRange{From: rune(65081), To: rune(65081)}, + &CodePointRange{From: rune(65083), To: rune(65083)}, + &CodePointRange{From: rune(65085), To: rune(65085)}, + &CodePointRange{From: rune(65087), To: rune(65087)}, + &CodePointRange{From: rune(65089), To: rune(65089)}, + &CodePointRange{From: rune(65091), To: rune(65091)}, + &CodePointRange{From: rune(65095), To: rune(65095)}, + &CodePointRange{From: rune(65113), To: rune(65113)}, + &CodePointRange{From: rune(65115), To: rune(65115)}, + &CodePointRange{From: rune(65117), To: rune(65117)}, + &CodePointRange{From: rune(65288), To: rune(65288)}, + &CodePointRange{From: rune(65339), To: rune(65339)}, + &CodePointRange{From: rune(65371), To: rune(65371)}, + &CodePointRange{From: rune(65375), To: rune(65375)}, + &CodePointRange{From: rune(65378), To: rune(65378)}, + }, + "sc": { + &CodePointRange{From: rune(36), To: rune(36)}, + &CodePointRange{From: rune(162), To: rune(165)}, + &CodePointRange{From: rune(1423), To: rune(1423)}, + &CodePointRange{From: rune(1547), To: rune(1547)}, + &CodePointRange{From: rune(2046), To: rune(2047)}, + &CodePointRange{From: rune(2546), To: rune(2547)}, + &CodePointRange{From: rune(2555), To: rune(2555)}, + &CodePointRange{From: rune(2801), To: rune(2801)}, + &CodePointRange{From: rune(3065), To: rune(3065)}, + &CodePointRange{From: rune(3647), To: rune(3647)}, + &CodePointRange{From: rune(6107), To: rune(6107)}, + &CodePointRange{From: rune(8352), To: rune(8383)}, + &CodePointRange{From: rune(43064), To: rune(43064)}, + &CodePointRange{From: rune(65020), To: rune(65020)}, + &CodePointRange{From: rune(65129), To: rune(65129)}, + &CodePointRange{From: rune(65284), To: rune(65284)}, + &CodePointRange{From: rune(65504), To: rune(65505)}, + &CodePointRange{From: rune(65509), To: rune(65510)}, + &CodePointRange{From: rune(73693), To: rune(73696)}, + &CodePointRange{From: rune(123647), To: rune(123647)}, + &CodePointRange{From: rune(126128), To: rune(126128)}, + }, + "sk": { + &CodePointRange{From: rune(94), To: rune(94)}, + &CodePointRange{From: rune(96), To: rune(96)}, + &CodePointRange{From: rune(168), To: rune(168)}, + &CodePointRange{From: rune(175), To: rune(175)}, + &CodePointRange{From: rune(180), To: rune(180)}, + &CodePointRange{From: rune(184), To: rune(184)}, + &CodePointRange{From: rune(706), To: rune(709)}, + &CodePointRange{From: rune(722), To: rune(735)}, + &CodePointRange{From: rune(741), To: rune(747)}, + &CodePointRange{From: rune(749), To: rune(749)}, + &CodePointRange{From: rune(751), To: rune(767)}, + &CodePointRange{From: rune(885), To: rune(885)}, + &CodePointRange{From: rune(900), To: rune(901)}, + &CodePointRange{From: rune(8125), To: rune(8125)}, + &CodePointRange{From: rune(8127), To: rune(8129)}, + &CodePointRange{From: rune(8141), To: rune(8143)}, + &CodePointRange{From: rune(8157), To: rune(8159)}, + &CodePointRange{From: rune(8173), To: rune(8175)}, + &CodePointRange{From: rune(8189), To: rune(8190)}, + &CodePointRange{From: rune(12443), To: rune(12444)}, + &CodePointRange{From: rune(42752), To: rune(42774)}, + &CodePointRange{From: rune(42784), To: rune(42785)}, + &CodePointRange{From: rune(42889), To: rune(42890)}, + &CodePointRange{From: rune(43867), To: rune(43867)}, + &CodePointRange{From: rune(43882), To: rune(43883)}, + &CodePointRange{From: rune(64434), To: rune(64449)}, + &CodePointRange{From: rune(65342), To: rune(65342)}, + &CodePointRange{From: rune(65344), To: rune(65344)}, + &CodePointRange{From: rune(65507), To: rune(65507)}, + &CodePointRange{From: rune(127995), To: rune(127999)}, + }, + "sm": { + &CodePointRange{From: rune(43), To: rune(43)}, + &CodePointRange{From: rune(60), To: rune(62)}, + &CodePointRange{From: rune(124), To: rune(124)}, + &CodePointRange{From: rune(126), To: rune(126)}, + &CodePointRange{From: rune(172), To: rune(172)}, + &CodePointRange{From: rune(177), To: rune(177)}, + &CodePointRange{From: rune(215), To: rune(215)}, + &CodePointRange{From: rune(247), To: rune(247)}, + &CodePointRange{From: rune(1014), To: rune(1014)}, + &CodePointRange{From: rune(1542), To: rune(1544)}, + &CodePointRange{From: rune(8260), To: rune(8260)}, + &CodePointRange{From: rune(8274), To: rune(8274)}, + &CodePointRange{From: rune(8314), To: rune(8316)}, + &CodePointRange{From: rune(8330), To: rune(8332)}, + &CodePointRange{From: rune(8472), To: rune(8472)}, + &CodePointRange{From: rune(8512), To: rune(8516)}, + &CodePointRange{From: rune(8523), To: rune(8523)}, + &CodePointRange{From: rune(8592), To: rune(8596)}, + &CodePointRange{From: rune(8602), To: rune(8603)}, + &CodePointRange{From: rune(8608), To: rune(8608)}, + &CodePointRange{From: rune(8611), To: rune(8611)}, + &CodePointRange{From: rune(8614), To: rune(8614)}, + &CodePointRange{From: rune(8622), To: rune(8622)}, + &CodePointRange{From: rune(8654), To: rune(8655)}, + &CodePointRange{From: rune(8658), To: rune(8658)}, + &CodePointRange{From: rune(8660), To: rune(8660)}, + &CodePointRange{From: rune(8692), To: rune(8959)}, + &CodePointRange{From: rune(8992), To: rune(8993)}, + &CodePointRange{From: rune(9084), To: rune(9084)}, + &CodePointRange{From: rune(9115), To: rune(9139)}, + &CodePointRange{From: rune(9180), To: rune(9185)}, + &CodePointRange{From: rune(9655), To: rune(9655)}, + &CodePointRange{From: rune(9665), To: rune(9665)}, + &CodePointRange{From: rune(9720), To: rune(9727)}, + &CodePointRange{From: rune(9839), To: rune(9839)}, + &CodePointRange{From: rune(10176), To: rune(10180)}, + &CodePointRange{From: rune(10183), To: rune(10213)}, + &CodePointRange{From: rune(10224), To: rune(10239)}, + &CodePointRange{From: rune(10496), To: rune(10626)}, + &CodePointRange{From: rune(10649), To: rune(10711)}, + &CodePointRange{From: rune(10716), To: rune(10747)}, + &CodePointRange{From: rune(10750), To: rune(11007)}, + &CodePointRange{From: rune(11056), To: rune(11076)}, + &CodePointRange{From: rune(11079), To: rune(11084)}, + &CodePointRange{From: rune(64297), To: rune(64297)}, + &CodePointRange{From: rune(65122), To: rune(65122)}, + &CodePointRange{From: rune(65124), To: rune(65126)}, + &CodePointRange{From: rune(65291), To: rune(65291)}, + &CodePointRange{From: rune(65308), To: rune(65310)}, + &CodePointRange{From: rune(65372), To: rune(65372)}, + &CodePointRange{From: rune(65374), To: rune(65374)}, + &CodePointRange{From: rune(65506), To: rune(65506)}, + &CodePointRange{From: rune(65513), To: rune(65516)}, + &CodePointRange{From: rune(120513), To: rune(120513)}, + &CodePointRange{From: rune(120539), To: rune(120539)}, + &CodePointRange{From: rune(120571), To: rune(120571)}, + &CodePointRange{From: rune(120597), To: rune(120597)}, + &CodePointRange{From: rune(120629), To: rune(120629)}, + &CodePointRange{From: rune(120655), To: rune(120655)}, + &CodePointRange{From: rune(120687), To: rune(120687)}, + &CodePointRange{From: rune(120713), To: rune(120713)}, + &CodePointRange{From: rune(120745), To: rune(120745)}, + &CodePointRange{From: rune(120771), To: rune(120771)}, + &CodePointRange{From: rune(126704), To: rune(126705)}, + }, + "so": { + &CodePointRange{From: rune(166), To: rune(166)}, + &CodePointRange{From: rune(169), To: rune(169)}, + &CodePointRange{From: rune(174), To: rune(174)}, + &CodePointRange{From: rune(176), To: rune(176)}, + &CodePointRange{From: rune(1154), To: rune(1154)}, + &CodePointRange{From: rune(1421), To: rune(1422)}, + &CodePointRange{From: rune(1550), To: rune(1551)}, + &CodePointRange{From: rune(1758), To: rune(1758)}, + &CodePointRange{From: rune(1769), To: rune(1769)}, + &CodePointRange{From: rune(1789), To: rune(1790)}, + &CodePointRange{From: rune(2038), To: rune(2038)}, + &CodePointRange{From: rune(2554), To: rune(2554)}, + &CodePointRange{From: rune(2928), To: rune(2928)}, + &CodePointRange{From: rune(3059), To: rune(3064)}, + &CodePointRange{From: rune(3066), To: rune(3066)}, + &CodePointRange{From: rune(3199), To: rune(3199)}, + &CodePointRange{From: rune(3407), To: rune(3407)}, + &CodePointRange{From: rune(3449), To: rune(3449)}, + &CodePointRange{From: rune(3841), To: rune(3843)}, + &CodePointRange{From: rune(3859), To: rune(3859)}, + &CodePointRange{From: rune(3861), To: rune(3863)}, + &CodePointRange{From: rune(3866), To: rune(3871)}, + &CodePointRange{From: rune(3892), To: rune(3892)}, + &CodePointRange{From: rune(3894), To: rune(3894)}, + &CodePointRange{From: rune(3896), To: rune(3896)}, + &CodePointRange{From: rune(4030), To: rune(4037)}, + &CodePointRange{From: rune(4039), To: rune(4044)}, + &CodePointRange{From: rune(4046), To: rune(4047)}, + &CodePointRange{From: rune(4053), To: rune(4056)}, + &CodePointRange{From: rune(4254), To: rune(4255)}, + &CodePointRange{From: rune(5008), To: rune(5017)}, + &CodePointRange{From: rune(5741), To: rune(5741)}, + &CodePointRange{From: rune(6464), To: rune(6464)}, + &CodePointRange{From: rune(6622), To: rune(6655)}, + &CodePointRange{From: rune(7009), To: rune(7018)}, + &CodePointRange{From: rune(7028), To: rune(7036)}, + &CodePointRange{From: rune(8448), To: rune(8449)}, + &CodePointRange{From: rune(8451), To: rune(8454)}, + &CodePointRange{From: rune(8456), To: rune(8457)}, + &CodePointRange{From: rune(8468), To: rune(8468)}, + &CodePointRange{From: rune(8470), To: rune(8471)}, + &CodePointRange{From: rune(8478), To: rune(8483)}, + &CodePointRange{From: rune(8485), To: rune(8485)}, + &CodePointRange{From: rune(8487), To: rune(8487)}, + &CodePointRange{From: rune(8489), To: rune(8489)}, + &CodePointRange{From: rune(8494), To: rune(8494)}, + &CodePointRange{From: rune(8506), To: rune(8507)}, + &CodePointRange{From: rune(8522), To: rune(8522)}, + &CodePointRange{From: rune(8524), To: rune(8525)}, + &CodePointRange{From: rune(8527), To: rune(8527)}, + &CodePointRange{From: rune(8586), To: rune(8587)}, + &CodePointRange{From: rune(8597), To: rune(8601)}, + &CodePointRange{From: rune(8604), To: rune(8607)}, + &CodePointRange{From: rune(8609), To: rune(8610)}, + &CodePointRange{From: rune(8612), To: rune(8613)}, + &CodePointRange{From: rune(8615), To: rune(8621)}, + &CodePointRange{From: rune(8623), To: rune(8653)}, + &CodePointRange{From: rune(8656), To: rune(8657)}, + &CodePointRange{From: rune(8659), To: rune(8659)}, + &CodePointRange{From: rune(8661), To: rune(8691)}, + &CodePointRange{From: rune(8960), To: rune(8967)}, + &CodePointRange{From: rune(8972), To: rune(8991)}, + &CodePointRange{From: rune(8994), To: rune(9000)}, + &CodePointRange{From: rune(9003), To: rune(9083)}, + &CodePointRange{From: rune(9085), To: rune(9114)}, + &CodePointRange{From: rune(9140), To: rune(9179)}, + &CodePointRange{From: rune(9186), To: rune(9254)}, + &CodePointRange{From: rune(9280), To: rune(9290)}, + &CodePointRange{From: rune(9372), To: rune(9449)}, + &CodePointRange{From: rune(9472), To: rune(9654)}, + &CodePointRange{From: rune(9656), To: rune(9664)}, + &CodePointRange{From: rune(9666), To: rune(9719)}, + &CodePointRange{From: rune(9728), To: rune(9838)}, + &CodePointRange{From: rune(9840), To: rune(10087)}, + &CodePointRange{From: rune(10132), To: rune(10175)}, + &CodePointRange{From: rune(10240), To: rune(10495)}, + &CodePointRange{From: rune(11008), To: rune(11055)}, + &CodePointRange{From: rune(11077), To: rune(11078)}, + &CodePointRange{From: rune(11085), To: rune(11123)}, + &CodePointRange{From: rune(11126), To: rune(11157)}, + &CodePointRange{From: rune(11159), To: rune(11263)}, + &CodePointRange{From: rune(11493), To: rune(11498)}, + &CodePointRange{From: rune(11856), To: rune(11857)}, + &CodePointRange{From: rune(11904), To: rune(11929)}, + &CodePointRange{From: rune(11931), To: rune(12019)}, + &CodePointRange{From: rune(12032), To: rune(12245)}, + &CodePointRange{From: rune(12272), To: rune(12283)}, + &CodePointRange{From: rune(12292), To: rune(12292)}, + &CodePointRange{From: rune(12306), To: rune(12307)}, + &CodePointRange{From: rune(12320), To: rune(12320)}, + &CodePointRange{From: rune(12342), To: rune(12343)}, + &CodePointRange{From: rune(12350), To: rune(12351)}, + &CodePointRange{From: rune(12688), To: rune(12689)}, + &CodePointRange{From: rune(12694), To: rune(12703)}, + &CodePointRange{From: rune(12736), To: rune(12771)}, + &CodePointRange{From: rune(12800), To: rune(12830)}, + &CodePointRange{From: rune(12842), To: rune(12871)}, + &CodePointRange{From: rune(12880), To: rune(12880)}, + &CodePointRange{From: rune(12896), To: rune(12927)}, + &CodePointRange{From: rune(12938), To: rune(12976)}, + &CodePointRange{From: rune(12992), To: rune(13311)}, + &CodePointRange{From: rune(19904), To: rune(19967)}, + &CodePointRange{From: rune(42128), To: rune(42182)}, + &CodePointRange{From: rune(43048), To: rune(43051)}, + &CodePointRange{From: rune(43062), To: rune(43063)}, + &CodePointRange{From: rune(43065), To: rune(43065)}, + &CodePointRange{From: rune(43639), To: rune(43641)}, + &CodePointRange{From: rune(65021), To: rune(65021)}, + &CodePointRange{From: rune(65508), To: rune(65508)}, + &CodePointRange{From: rune(65512), To: rune(65512)}, + &CodePointRange{From: rune(65517), To: rune(65518)}, + &CodePointRange{From: rune(65532), To: rune(65533)}, + &CodePointRange{From: rune(65847), To: rune(65855)}, + &CodePointRange{From: rune(65913), To: rune(65929)}, + &CodePointRange{From: rune(65932), To: rune(65934)}, + &CodePointRange{From: rune(65936), To: rune(65948)}, + &CodePointRange{From: rune(65952), To: rune(65952)}, + &CodePointRange{From: rune(66000), To: rune(66044)}, + &CodePointRange{From: rune(67703), To: rune(67704)}, + &CodePointRange{From: rune(68296), To: rune(68296)}, + &CodePointRange{From: rune(71487), To: rune(71487)}, + &CodePointRange{From: rune(73685), To: rune(73692)}, + &CodePointRange{From: rune(73697), To: rune(73713)}, + &CodePointRange{From: rune(92988), To: rune(92991)}, + &CodePointRange{From: rune(92997), To: rune(92997)}, + &CodePointRange{From: rune(113820), To: rune(113820)}, + &CodePointRange{From: rune(118784), To: rune(119029)}, + &CodePointRange{From: rune(119040), To: rune(119078)}, + &CodePointRange{From: rune(119081), To: rune(119140)}, + &CodePointRange{From: rune(119146), To: rune(119148)}, + &CodePointRange{From: rune(119171), To: rune(119172)}, + &CodePointRange{From: rune(119180), To: rune(119209)}, + &CodePointRange{From: rune(119214), To: rune(119272)}, + &CodePointRange{From: rune(119296), To: rune(119361)}, + &CodePointRange{From: rune(119365), To: rune(119365)}, + &CodePointRange{From: rune(119552), To: rune(119638)}, + &CodePointRange{From: rune(120832), To: rune(121343)}, + &CodePointRange{From: rune(121399), To: rune(121402)}, + &CodePointRange{From: rune(121453), To: rune(121460)}, + &CodePointRange{From: rune(121462), To: rune(121475)}, + &CodePointRange{From: rune(121477), To: rune(121478)}, + &CodePointRange{From: rune(123215), To: rune(123215)}, + &CodePointRange{From: rune(126124), To: rune(126124)}, + &CodePointRange{From: rune(126254), To: rune(126254)}, + &CodePointRange{From: rune(126976), To: rune(127019)}, + &CodePointRange{From: rune(127024), To: rune(127123)}, + &CodePointRange{From: rune(127136), To: rune(127150)}, + &CodePointRange{From: rune(127153), To: rune(127167)}, + &CodePointRange{From: rune(127169), To: rune(127183)}, + &CodePointRange{From: rune(127185), To: rune(127221)}, + &CodePointRange{From: rune(127245), To: rune(127405)}, + &CodePointRange{From: rune(127462), To: rune(127490)}, + &CodePointRange{From: rune(127504), To: rune(127547)}, + &CodePointRange{From: rune(127552), To: rune(127560)}, + &CodePointRange{From: rune(127568), To: rune(127569)}, + &CodePointRange{From: rune(127584), To: rune(127589)}, + &CodePointRange{From: rune(127744), To: rune(127994)}, + &CodePointRange{From: rune(128000), To: rune(128727)}, + &CodePointRange{From: rune(128736), To: rune(128748)}, + &CodePointRange{From: rune(128752), To: rune(128764)}, + &CodePointRange{From: rune(128768), To: rune(128883)}, + &CodePointRange{From: rune(128896), To: rune(128984)}, + &CodePointRange{From: rune(128992), To: rune(129003)}, + &CodePointRange{From: rune(129024), To: rune(129035)}, + &CodePointRange{From: rune(129040), To: rune(129095)}, + &CodePointRange{From: rune(129104), To: rune(129113)}, + &CodePointRange{From: rune(129120), To: rune(129159)}, + &CodePointRange{From: rune(129168), To: rune(129197)}, + &CodePointRange{From: rune(129200), To: rune(129201)}, + &CodePointRange{From: rune(129280), To: rune(129400)}, + &CodePointRange{From: rune(129402), To: rune(129483)}, + &CodePointRange{From: rune(129485), To: rune(129619)}, + &CodePointRange{From: rune(129632), To: rune(129645)}, + &CodePointRange{From: rune(129648), To: rune(129652)}, + &CodePointRange{From: rune(129656), To: rune(129658)}, + &CodePointRange{From: rune(129664), To: rune(129670)}, + &CodePointRange{From: rune(129680), To: rune(129704)}, + &CodePointRange{From: rune(129712), To: rune(129718)}, + &CodePointRange{From: rune(129728), To: rune(129730)}, + &CodePointRange{From: rune(129744), To: rune(129750)}, + &CodePointRange{From: rune(129792), To: rune(129938)}, + &CodePointRange{From: rune(129940), To: rune(129994)}, + }, + "zl": { + &CodePointRange{From: rune(8232), To: rune(8232)}, + }, + "zp": { + &CodePointRange{From: rune(8233), To: rune(8233)}, + }, + "zs": { + &CodePointRange{From: rune(32), To: rune(32)}, + &CodePointRange{From: rune(160), To: rune(160)}, + &CodePointRange{From: rune(5760), To: rune(5760)}, + &CodePointRange{From: rune(8192), To: rune(8202)}, + &CodePointRange{From: rune(8239), To: rune(8239)}, + &CodePointRange{From: rune(8287), To: rune(8287)}, + &CodePointRange{From: rune(12288), To: rune(12288)}, + }, +} + +// https://www.unicode.org/Public/13.0.0/ucd/Scripts.txt +var ( + scriptDefaultRange = &CodePointRange{ + From: rune(0), + To: rune(1114111), + } + scriptDefaultValue = "unknown" +) + +// https://www.unicode.org/Public/13.0.0/ucd/Scripts.txt +var scriptCodepoints = map[string][]*CodePointRange{ + "adlm": { + &CodePointRange{From: rune(125184), To: rune(125251)}, + &CodePointRange{From: rune(125252), To: rune(125258)}, + &CodePointRange{From: rune(125259), To: rune(125259)}, + &CodePointRange{From: rune(125264), To: rune(125273)}, + &CodePointRange{From: rune(125278), To: rune(125279)}, + }, + "aghb": { + &CodePointRange{From: rune(66864), To: rune(66915)}, + &CodePointRange{From: rune(66927), To: rune(66927)}, + }, + "ahom": { + &CodePointRange{From: rune(71424), To: rune(71450)}, + &CodePointRange{From: rune(71453), To: rune(71455)}, + &CodePointRange{From: rune(71456), To: rune(71457)}, + &CodePointRange{From: rune(71458), To: rune(71461)}, + &CodePointRange{From: rune(71462), To: rune(71462)}, + &CodePointRange{From: rune(71463), To: rune(71467)}, + &CodePointRange{From: rune(71472), To: rune(71481)}, + &CodePointRange{From: rune(71482), To: rune(71483)}, + &CodePointRange{From: rune(71484), To: rune(71486)}, + &CodePointRange{From: rune(71487), To: rune(71487)}, + }, + "arab": { + &CodePointRange{From: rune(1536), To: rune(1540)}, + &CodePointRange{From: rune(1542), To: rune(1544)}, + &CodePointRange{From: rune(1545), To: rune(1546)}, + &CodePointRange{From: rune(1547), To: rune(1547)}, + &CodePointRange{From: rune(1549), To: rune(1549)}, + &CodePointRange{From: rune(1550), To: rune(1551)}, + &CodePointRange{From: rune(1552), To: rune(1562)}, + &CodePointRange{From: rune(1564), To: rune(1564)}, + &CodePointRange{From: rune(1566), To: rune(1566)}, + &CodePointRange{From: rune(1568), To: rune(1599)}, + &CodePointRange{From: rune(1601), To: rune(1610)}, + &CodePointRange{From: rune(1622), To: rune(1631)}, + &CodePointRange{From: rune(1632), To: rune(1641)}, + &CodePointRange{From: rune(1642), To: rune(1645)}, + &CodePointRange{From: rune(1646), To: rune(1647)}, + &CodePointRange{From: rune(1649), To: rune(1747)}, + &CodePointRange{From: rune(1748), To: rune(1748)}, + &CodePointRange{From: rune(1749), To: rune(1749)}, + &CodePointRange{From: rune(1750), To: rune(1756)}, + &CodePointRange{From: rune(1758), To: rune(1758)}, + &CodePointRange{From: rune(1759), To: rune(1764)}, + &CodePointRange{From: rune(1765), To: rune(1766)}, + &CodePointRange{From: rune(1767), To: rune(1768)}, + &CodePointRange{From: rune(1769), To: rune(1769)}, + &CodePointRange{From: rune(1770), To: rune(1773)}, + &CodePointRange{From: rune(1774), To: rune(1775)}, + &CodePointRange{From: rune(1776), To: rune(1785)}, + &CodePointRange{From: rune(1786), To: rune(1788)}, + &CodePointRange{From: rune(1789), To: rune(1790)}, + &CodePointRange{From: rune(1791), To: rune(1791)}, + &CodePointRange{From: rune(1872), To: rune(1919)}, + &CodePointRange{From: rune(2208), To: rune(2228)}, + &CodePointRange{From: rune(2230), To: rune(2247)}, + &CodePointRange{From: rune(2259), To: rune(2273)}, + &CodePointRange{From: rune(2275), To: rune(2303)}, + &CodePointRange{From: rune(64336), To: rune(64433)}, + &CodePointRange{From: rune(64434), To: rune(64449)}, + &CodePointRange{From: rune(64467), To: rune(64829)}, + &CodePointRange{From: rune(64848), To: rune(64911)}, + &CodePointRange{From: rune(64914), To: rune(64967)}, + &CodePointRange{From: rune(65008), To: rune(65019)}, + &CodePointRange{From: rune(65020), To: rune(65020)}, + &CodePointRange{From: rune(65021), To: rune(65021)}, + &CodePointRange{From: rune(65136), To: rune(65140)}, + &CodePointRange{From: rune(65142), To: rune(65276)}, + &CodePointRange{From: rune(69216), To: rune(69246)}, + &CodePointRange{From: rune(126464), To: rune(126467)}, + &CodePointRange{From: rune(126469), To: rune(126495)}, + &CodePointRange{From: rune(126497), To: rune(126498)}, + &CodePointRange{From: rune(126500), To: rune(126500)}, + &CodePointRange{From: rune(126503), To: rune(126503)}, + &CodePointRange{From: rune(126505), To: rune(126514)}, + &CodePointRange{From: rune(126516), To: rune(126519)}, + &CodePointRange{From: rune(126521), To: rune(126521)}, + &CodePointRange{From: rune(126523), To: rune(126523)}, + &CodePointRange{From: rune(126530), To: rune(126530)}, + &CodePointRange{From: rune(126535), To: rune(126535)}, + &CodePointRange{From: rune(126537), To: rune(126537)}, + &CodePointRange{From: rune(126539), To: rune(126539)}, + &CodePointRange{From: rune(126541), To: rune(126543)}, + &CodePointRange{From: rune(126545), To: rune(126546)}, + &CodePointRange{From: rune(126548), To: rune(126548)}, + &CodePointRange{From: rune(126551), To: rune(126551)}, + &CodePointRange{From: rune(126553), To: rune(126553)}, + &CodePointRange{From: rune(126555), To: rune(126555)}, + &CodePointRange{From: rune(126557), To: rune(126557)}, + &CodePointRange{From: rune(126559), To: rune(126559)}, + &CodePointRange{From: rune(126561), To: rune(126562)}, + &CodePointRange{From: rune(126564), To: rune(126564)}, + &CodePointRange{From: rune(126567), To: rune(126570)}, + &CodePointRange{From: rune(126572), To: rune(126578)}, + &CodePointRange{From: rune(126580), To: rune(126583)}, + &CodePointRange{From: rune(126585), To: rune(126588)}, + &CodePointRange{From: rune(126590), To: rune(126590)}, + &CodePointRange{From: rune(126592), To: rune(126601)}, + &CodePointRange{From: rune(126603), To: rune(126619)}, + &CodePointRange{From: rune(126625), To: rune(126627)}, + &CodePointRange{From: rune(126629), To: rune(126633)}, + &CodePointRange{From: rune(126635), To: rune(126651)}, + &CodePointRange{From: rune(126704), To: rune(126705)}, + }, + "armi": { + &CodePointRange{From: rune(67648), To: rune(67669)}, + &CodePointRange{From: rune(67671), To: rune(67671)}, + &CodePointRange{From: rune(67672), To: rune(67679)}, + }, + "armn": { + &CodePointRange{From: rune(1329), To: rune(1366)}, + &CodePointRange{From: rune(1369), To: rune(1369)}, + &CodePointRange{From: rune(1370), To: rune(1375)}, + &CodePointRange{From: rune(1376), To: rune(1416)}, + &CodePointRange{From: rune(1417), To: rune(1417)}, + &CodePointRange{From: rune(1418), To: rune(1418)}, + &CodePointRange{From: rune(1421), To: rune(1422)}, + &CodePointRange{From: rune(1423), To: rune(1423)}, + &CodePointRange{From: rune(64275), To: rune(64279)}, + }, + "avst": { + &CodePointRange{From: rune(68352), To: rune(68405)}, + &CodePointRange{From: rune(68409), To: rune(68415)}, + }, + "bali": { + &CodePointRange{From: rune(6912), To: rune(6915)}, + &CodePointRange{From: rune(6916), To: rune(6916)}, + &CodePointRange{From: rune(6917), To: rune(6963)}, + &CodePointRange{From: rune(6964), To: rune(6964)}, + &CodePointRange{From: rune(6965), To: rune(6965)}, + &CodePointRange{From: rune(6966), To: rune(6970)}, + &CodePointRange{From: rune(6971), To: rune(6971)}, + &CodePointRange{From: rune(6972), To: rune(6972)}, + &CodePointRange{From: rune(6973), To: rune(6977)}, + &CodePointRange{From: rune(6978), To: rune(6978)}, + &CodePointRange{From: rune(6979), To: rune(6980)}, + &CodePointRange{From: rune(6981), To: rune(6987)}, + &CodePointRange{From: rune(6992), To: rune(7001)}, + &CodePointRange{From: rune(7002), To: rune(7008)}, + &CodePointRange{From: rune(7009), To: rune(7018)}, + &CodePointRange{From: rune(7019), To: rune(7027)}, + &CodePointRange{From: rune(7028), To: rune(7036)}, + }, + "bamu": { + &CodePointRange{From: rune(42656), To: rune(42725)}, + &CodePointRange{From: rune(42726), To: rune(42735)}, + &CodePointRange{From: rune(42736), To: rune(42737)}, + &CodePointRange{From: rune(42738), To: rune(42743)}, + &CodePointRange{From: rune(92160), To: rune(92728)}, + }, + "bass": { + &CodePointRange{From: rune(92880), To: rune(92909)}, + &CodePointRange{From: rune(92912), To: rune(92916)}, + &CodePointRange{From: rune(92917), To: rune(92917)}, + }, + "batk": { + &CodePointRange{From: rune(7104), To: rune(7141)}, + &CodePointRange{From: rune(7142), To: rune(7142)}, + &CodePointRange{From: rune(7143), To: rune(7143)}, + &CodePointRange{From: rune(7144), To: rune(7145)}, + &CodePointRange{From: rune(7146), To: rune(7148)}, + &CodePointRange{From: rune(7149), To: rune(7149)}, + &CodePointRange{From: rune(7150), To: rune(7150)}, + &CodePointRange{From: rune(7151), To: rune(7153)}, + &CodePointRange{From: rune(7154), To: rune(7155)}, + &CodePointRange{From: rune(7164), To: rune(7167)}, + }, + "beng": { + &CodePointRange{From: rune(2432), To: rune(2432)}, + &CodePointRange{From: rune(2433), To: rune(2433)}, + &CodePointRange{From: rune(2434), To: rune(2435)}, + &CodePointRange{From: rune(2437), To: rune(2444)}, + &CodePointRange{From: rune(2447), To: rune(2448)}, + &CodePointRange{From: rune(2451), To: rune(2472)}, + &CodePointRange{From: rune(2474), To: rune(2480)}, + &CodePointRange{From: rune(2482), To: rune(2482)}, + &CodePointRange{From: rune(2486), To: rune(2489)}, + &CodePointRange{From: rune(2492), To: rune(2492)}, + &CodePointRange{From: rune(2493), To: rune(2493)}, + &CodePointRange{From: rune(2494), To: rune(2496)}, + &CodePointRange{From: rune(2497), To: rune(2500)}, + &CodePointRange{From: rune(2503), To: rune(2504)}, + &CodePointRange{From: rune(2507), To: rune(2508)}, + &CodePointRange{From: rune(2509), To: rune(2509)}, + &CodePointRange{From: rune(2510), To: rune(2510)}, + &CodePointRange{From: rune(2519), To: rune(2519)}, + &CodePointRange{From: rune(2524), To: rune(2525)}, + &CodePointRange{From: rune(2527), To: rune(2529)}, + &CodePointRange{From: rune(2530), To: rune(2531)}, + &CodePointRange{From: rune(2534), To: rune(2543)}, + &CodePointRange{From: rune(2544), To: rune(2545)}, + &CodePointRange{From: rune(2546), To: rune(2547)}, + &CodePointRange{From: rune(2548), To: rune(2553)}, + &CodePointRange{From: rune(2554), To: rune(2554)}, + &CodePointRange{From: rune(2555), To: rune(2555)}, + &CodePointRange{From: rune(2556), To: rune(2556)}, + &CodePointRange{From: rune(2557), To: rune(2557)}, + &CodePointRange{From: rune(2558), To: rune(2558)}, + }, + "bhks": { + &CodePointRange{From: rune(72704), To: rune(72712)}, + &CodePointRange{From: rune(72714), To: rune(72750)}, + &CodePointRange{From: rune(72751), To: rune(72751)}, + &CodePointRange{From: rune(72752), To: rune(72758)}, + &CodePointRange{From: rune(72760), To: rune(72765)}, + &CodePointRange{From: rune(72766), To: rune(72766)}, + &CodePointRange{From: rune(72767), To: rune(72767)}, + &CodePointRange{From: rune(72768), To: rune(72768)}, + &CodePointRange{From: rune(72769), To: rune(72773)}, + &CodePointRange{From: rune(72784), To: rune(72793)}, + &CodePointRange{From: rune(72794), To: rune(72812)}, + }, + "bopo": { + &CodePointRange{From: rune(746), To: rune(747)}, + &CodePointRange{From: rune(12549), To: rune(12591)}, + &CodePointRange{From: rune(12704), To: rune(12735)}, + }, + "brah": { + &CodePointRange{From: rune(69632), To: rune(69632)}, + &CodePointRange{From: rune(69633), To: rune(69633)}, + &CodePointRange{From: rune(69634), To: rune(69634)}, + &CodePointRange{From: rune(69635), To: rune(69687)}, + &CodePointRange{From: rune(69688), To: rune(69702)}, + &CodePointRange{From: rune(69703), To: rune(69709)}, + &CodePointRange{From: rune(69714), To: rune(69733)}, + &CodePointRange{From: rune(69734), To: rune(69743)}, + &CodePointRange{From: rune(69759), To: rune(69759)}, + }, + "brai": { + &CodePointRange{From: rune(10240), To: rune(10495)}, + }, + "bugi": { + &CodePointRange{From: rune(6656), To: rune(6678)}, + &CodePointRange{From: rune(6679), To: rune(6680)}, + &CodePointRange{From: rune(6681), To: rune(6682)}, + &CodePointRange{From: rune(6683), To: rune(6683)}, + &CodePointRange{From: rune(6686), To: rune(6687)}, + }, + "buhd": { + &CodePointRange{From: rune(5952), To: rune(5969)}, + &CodePointRange{From: rune(5970), To: rune(5971)}, + }, + "cakm": { + &CodePointRange{From: rune(69888), To: rune(69890)}, + &CodePointRange{From: rune(69891), To: rune(69926)}, + &CodePointRange{From: rune(69927), To: rune(69931)}, + &CodePointRange{From: rune(69932), To: rune(69932)}, + &CodePointRange{From: rune(69933), To: rune(69940)}, + &CodePointRange{From: rune(69942), To: rune(69951)}, + &CodePointRange{From: rune(69952), To: rune(69955)}, + &CodePointRange{From: rune(69956), To: rune(69956)}, + &CodePointRange{From: rune(69957), To: rune(69958)}, + &CodePointRange{From: rune(69959), To: rune(69959)}, + }, + "cans": { + &CodePointRange{From: rune(5120), To: rune(5120)}, + &CodePointRange{From: rune(5121), To: rune(5740)}, + &CodePointRange{From: rune(5741), To: rune(5741)}, + &CodePointRange{From: rune(5742), To: rune(5742)}, + &CodePointRange{From: rune(5743), To: rune(5759)}, + &CodePointRange{From: rune(6320), To: rune(6389)}, + }, + "cari": { + &CodePointRange{From: rune(66208), To: rune(66256)}, + }, + "cham": { + &CodePointRange{From: rune(43520), To: rune(43560)}, + &CodePointRange{From: rune(43561), To: rune(43566)}, + &CodePointRange{From: rune(43567), To: rune(43568)}, + &CodePointRange{From: rune(43569), To: rune(43570)}, + &CodePointRange{From: rune(43571), To: rune(43572)}, + &CodePointRange{From: rune(43573), To: rune(43574)}, + &CodePointRange{From: rune(43584), To: rune(43586)}, + &CodePointRange{From: rune(43587), To: rune(43587)}, + &CodePointRange{From: rune(43588), To: rune(43595)}, + &CodePointRange{From: rune(43596), To: rune(43596)}, + &CodePointRange{From: rune(43597), To: rune(43597)}, + &CodePointRange{From: rune(43600), To: rune(43609)}, + &CodePointRange{From: rune(43612), To: rune(43615)}, + }, + "cher": { + &CodePointRange{From: rune(5024), To: rune(5109)}, + &CodePointRange{From: rune(5112), To: rune(5117)}, + &CodePointRange{From: rune(43888), To: rune(43967)}, + }, + "chrs": { + &CodePointRange{From: rune(69552), To: rune(69572)}, + &CodePointRange{From: rune(69573), To: rune(69579)}, + }, + "copt": { + &CodePointRange{From: rune(994), To: rune(1007)}, + &CodePointRange{From: rune(11392), To: rune(11492)}, + &CodePointRange{From: rune(11493), To: rune(11498)}, + &CodePointRange{From: rune(11499), To: rune(11502)}, + &CodePointRange{From: rune(11503), To: rune(11505)}, + &CodePointRange{From: rune(11506), To: rune(11507)}, + &CodePointRange{From: rune(11513), To: rune(11516)}, + &CodePointRange{From: rune(11517), To: rune(11517)}, + &CodePointRange{From: rune(11518), To: rune(11519)}, + }, + "cprt": { + &CodePointRange{From: rune(67584), To: rune(67589)}, + &CodePointRange{From: rune(67592), To: rune(67592)}, + &CodePointRange{From: rune(67594), To: rune(67637)}, + &CodePointRange{From: rune(67639), To: rune(67640)}, + &CodePointRange{From: rune(67644), To: rune(67644)}, + &CodePointRange{From: rune(67647), To: rune(67647)}, + }, + "cyrl": { + &CodePointRange{From: rune(1024), To: rune(1153)}, + &CodePointRange{From: rune(1154), To: rune(1154)}, + &CodePointRange{From: rune(1155), To: rune(1156)}, + &CodePointRange{From: rune(1159), To: rune(1159)}, + &CodePointRange{From: rune(1160), To: rune(1161)}, + &CodePointRange{From: rune(1162), To: rune(1327)}, + &CodePointRange{From: rune(7296), To: rune(7304)}, + &CodePointRange{From: rune(7467), To: rune(7467)}, + &CodePointRange{From: rune(7544), To: rune(7544)}, + &CodePointRange{From: rune(11744), To: rune(11775)}, + &CodePointRange{From: rune(42560), To: rune(42605)}, + &CodePointRange{From: rune(42606), To: rune(42606)}, + &CodePointRange{From: rune(42607), To: rune(42607)}, + &CodePointRange{From: rune(42608), To: rune(42610)}, + &CodePointRange{From: rune(42611), To: rune(42611)}, + &CodePointRange{From: rune(42612), To: rune(42621)}, + &CodePointRange{From: rune(42622), To: rune(42622)}, + &CodePointRange{From: rune(42623), To: rune(42623)}, + &CodePointRange{From: rune(42624), To: rune(42651)}, + &CodePointRange{From: rune(42652), To: rune(42653)}, + &CodePointRange{From: rune(42654), To: rune(42655)}, + &CodePointRange{From: rune(65070), To: rune(65071)}, + }, + "deva": { + &CodePointRange{From: rune(2304), To: rune(2306)}, + &CodePointRange{From: rune(2307), To: rune(2307)}, + &CodePointRange{From: rune(2308), To: rune(2361)}, + &CodePointRange{From: rune(2362), To: rune(2362)}, + &CodePointRange{From: rune(2363), To: rune(2363)}, + &CodePointRange{From: rune(2364), To: rune(2364)}, + &CodePointRange{From: rune(2365), To: rune(2365)}, + &CodePointRange{From: rune(2366), To: rune(2368)}, + &CodePointRange{From: rune(2369), To: rune(2376)}, + &CodePointRange{From: rune(2377), To: rune(2380)}, + &CodePointRange{From: rune(2381), To: rune(2381)}, + &CodePointRange{From: rune(2382), To: rune(2383)}, + &CodePointRange{From: rune(2384), To: rune(2384)}, + &CodePointRange{From: rune(2389), To: rune(2391)}, + &CodePointRange{From: rune(2392), To: rune(2401)}, + &CodePointRange{From: rune(2402), To: rune(2403)}, + &CodePointRange{From: rune(2406), To: rune(2415)}, + &CodePointRange{From: rune(2416), To: rune(2416)}, + &CodePointRange{From: rune(2417), To: rune(2417)}, + &CodePointRange{From: rune(2418), To: rune(2431)}, + &CodePointRange{From: rune(43232), To: rune(43249)}, + &CodePointRange{From: rune(43250), To: rune(43255)}, + &CodePointRange{From: rune(43256), To: rune(43258)}, + &CodePointRange{From: rune(43259), To: rune(43259)}, + &CodePointRange{From: rune(43260), To: rune(43260)}, + &CodePointRange{From: rune(43261), To: rune(43262)}, + &CodePointRange{From: rune(43263), To: rune(43263)}, + }, + "diak": { + &CodePointRange{From: rune(71936), To: rune(71942)}, + &CodePointRange{From: rune(71945), To: rune(71945)}, + &CodePointRange{From: rune(71948), To: rune(71955)}, + &CodePointRange{From: rune(71957), To: rune(71958)}, + &CodePointRange{From: rune(71960), To: rune(71983)}, + &CodePointRange{From: rune(71984), To: rune(71989)}, + &CodePointRange{From: rune(71991), To: rune(71992)}, + &CodePointRange{From: rune(71995), To: rune(71996)}, + &CodePointRange{From: rune(71997), To: rune(71997)}, + &CodePointRange{From: rune(71998), To: rune(71998)}, + &CodePointRange{From: rune(71999), To: rune(71999)}, + &CodePointRange{From: rune(72000), To: rune(72000)}, + &CodePointRange{From: rune(72001), To: rune(72001)}, + &CodePointRange{From: rune(72002), To: rune(72002)}, + &CodePointRange{From: rune(72003), To: rune(72003)}, + &CodePointRange{From: rune(72004), To: rune(72006)}, + &CodePointRange{From: rune(72016), To: rune(72025)}, + }, + "dogr": { + &CodePointRange{From: rune(71680), To: rune(71723)}, + &CodePointRange{From: rune(71724), To: rune(71726)}, + &CodePointRange{From: rune(71727), To: rune(71735)}, + &CodePointRange{From: rune(71736), To: rune(71736)}, + &CodePointRange{From: rune(71737), To: rune(71738)}, + &CodePointRange{From: rune(71739), To: rune(71739)}, + }, + "dsrt": { + &CodePointRange{From: rune(66560), To: rune(66639)}, + }, + "dupl": { + &CodePointRange{From: rune(113664), To: rune(113770)}, + &CodePointRange{From: rune(113776), To: rune(113788)}, + &CodePointRange{From: rune(113792), To: rune(113800)}, + &CodePointRange{From: rune(113808), To: rune(113817)}, + &CodePointRange{From: rune(113820), To: rune(113820)}, + &CodePointRange{From: rune(113821), To: rune(113822)}, + &CodePointRange{From: rune(113823), To: rune(113823)}, + }, + "egyp": { + &CodePointRange{From: rune(77824), To: rune(78894)}, + &CodePointRange{From: rune(78896), To: rune(78904)}, + }, + "elba": { + &CodePointRange{From: rune(66816), To: rune(66855)}, + }, + "elym": { + &CodePointRange{From: rune(69600), To: rune(69622)}, + }, + "ethi": { + &CodePointRange{From: rune(4608), To: rune(4680)}, + &CodePointRange{From: rune(4682), To: rune(4685)}, + &CodePointRange{From: rune(4688), To: rune(4694)}, + &CodePointRange{From: rune(4696), To: rune(4696)}, + &CodePointRange{From: rune(4698), To: rune(4701)}, + &CodePointRange{From: rune(4704), To: rune(4744)}, + &CodePointRange{From: rune(4746), To: rune(4749)}, + &CodePointRange{From: rune(4752), To: rune(4784)}, + &CodePointRange{From: rune(4786), To: rune(4789)}, + &CodePointRange{From: rune(4792), To: rune(4798)}, + &CodePointRange{From: rune(4800), To: rune(4800)}, + &CodePointRange{From: rune(4802), To: rune(4805)}, + &CodePointRange{From: rune(4808), To: rune(4822)}, + &CodePointRange{From: rune(4824), To: rune(4880)}, + &CodePointRange{From: rune(4882), To: rune(4885)}, + &CodePointRange{From: rune(4888), To: rune(4954)}, + &CodePointRange{From: rune(4957), To: rune(4959)}, + &CodePointRange{From: rune(4960), To: rune(4968)}, + &CodePointRange{From: rune(4969), To: rune(4988)}, + &CodePointRange{From: rune(4992), To: rune(5007)}, + &CodePointRange{From: rune(5008), To: rune(5017)}, + &CodePointRange{From: rune(11648), To: rune(11670)}, + &CodePointRange{From: rune(11680), To: rune(11686)}, + &CodePointRange{From: rune(11688), To: rune(11694)}, + &CodePointRange{From: rune(11696), To: rune(11702)}, + &CodePointRange{From: rune(11704), To: rune(11710)}, + &CodePointRange{From: rune(11712), To: rune(11718)}, + &CodePointRange{From: rune(11720), To: rune(11726)}, + &CodePointRange{From: rune(11728), To: rune(11734)}, + &CodePointRange{From: rune(11736), To: rune(11742)}, + &CodePointRange{From: rune(43777), To: rune(43782)}, + &CodePointRange{From: rune(43785), To: rune(43790)}, + &CodePointRange{From: rune(43793), To: rune(43798)}, + &CodePointRange{From: rune(43808), To: rune(43814)}, + &CodePointRange{From: rune(43816), To: rune(43822)}, + }, + "geor": { + &CodePointRange{From: rune(4256), To: rune(4293)}, + &CodePointRange{From: rune(4295), To: rune(4295)}, + &CodePointRange{From: rune(4301), To: rune(4301)}, + &CodePointRange{From: rune(4304), To: rune(4346)}, + &CodePointRange{From: rune(4348), To: rune(4348)}, + &CodePointRange{From: rune(4349), To: rune(4351)}, + &CodePointRange{From: rune(7312), To: rune(7354)}, + &CodePointRange{From: rune(7357), To: rune(7359)}, + &CodePointRange{From: rune(11520), To: rune(11557)}, + &CodePointRange{From: rune(11559), To: rune(11559)}, + &CodePointRange{From: rune(11565), To: rune(11565)}, + }, + "glag": { + &CodePointRange{From: rune(11264), To: rune(11310)}, + &CodePointRange{From: rune(11312), To: rune(11358)}, + &CodePointRange{From: rune(122880), To: rune(122886)}, + &CodePointRange{From: rune(122888), To: rune(122904)}, + &CodePointRange{From: rune(122907), To: rune(122913)}, + &CodePointRange{From: rune(122915), To: rune(122916)}, + &CodePointRange{From: rune(122918), To: rune(122922)}, + }, + "gong": { + &CodePointRange{From: rune(73056), To: rune(73061)}, + &CodePointRange{From: rune(73063), To: rune(73064)}, + &CodePointRange{From: rune(73066), To: rune(73097)}, + &CodePointRange{From: rune(73098), To: rune(73102)}, + &CodePointRange{From: rune(73104), To: rune(73105)}, + &CodePointRange{From: rune(73107), To: rune(73108)}, + &CodePointRange{From: rune(73109), To: rune(73109)}, + &CodePointRange{From: rune(73110), To: rune(73110)}, + &CodePointRange{From: rune(73111), To: rune(73111)}, + &CodePointRange{From: rune(73112), To: rune(73112)}, + &CodePointRange{From: rune(73120), To: rune(73129)}, + }, + "gonm": { + &CodePointRange{From: rune(72960), To: rune(72966)}, + &CodePointRange{From: rune(72968), To: rune(72969)}, + &CodePointRange{From: rune(72971), To: rune(73008)}, + &CodePointRange{From: rune(73009), To: rune(73014)}, + &CodePointRange{From: rune(73018), To: rune(73018)}, + &CodePointRange{From: rune(73020), To: rune(73021)}, + &CodePointRange{From: rune(73023), To: rune(73029)}, + &CodePointRange{From: rune(73030), To: rune(73030)}, + &CodePointRange{From: rune(73031), To: rune(73031)}, + &CodePointRange{From: rune(73040), To: rune(73049)}, + }, + "goth": { + &CodePointRange{From: rune(66352), To: rune(66368)}, + &CodePointRange{From: rune(66369), To: rune(66369)}, + &CodePointRange{From: rune(66370), To: rune(66377)}, + &CodePointRange{From: rune(66378), To: rune(66378)}, + }, + "gran": { + &CodePointRange{From: rune(70400), To: rune(70401)}, + &CodePointRange{From: rune(70402), To: rune(70403)}, + &CodePointRange{From: rune(70405), To: rune(70412)}, + &CodePointRange{From: rune(70415), To: rune(70416)}, + &CodePointRange{From: rune(70419), To: rune(70440)}, + &CodePointRange{From: rune(70442), To: rune(70448)}, + &CodePointRange{From: rune(70450), To: rune(70451)}, + &CodePointRange{From: rune(70453), To: rune(70457)}, + &CodePointRange{From: rune(70460), To: rune(70460)}, + &CodePointRange{From: rune(70461), To: rune(70461)}, + &CodePointRange{From: rune(70462), To: rune(70463)}, + &CodePointRange{From: rune(70464), To: rune(70464)}, + &CodePointRange{From: rune(70465), To: rune(70468)}, + &CodePointRange{From: rune(70471), To: rune(70472)}, + &CodePointRange{From: rune(70475), To: rune(70477)}, + &CodePointRange{From: rune(70480), To: rune(70480)}, + &CodePointRange{From: rune(70487), To: rune(70487)}, + &CodePointRange{From: rune(70493), To: rune(70497)}, + &CodePointRange{From: rune(70498), To: rune(70499)}, + &CodePointRange{From: rune(70502), To: rune(70508)}, + &CodePointRange{From: rune(70512), To: rune(70516)}, + }, + "grek": { + &CodePointRange{From: rune(880), To: rune(883)}, + &CodePointRange{From: rune(885), To: rune(885)}, + &CodePointRange{From: rune(886), To: rune(887)}, + &CodePointRange{From: rune(890), To: rune(890)}, + &CodePointRange{From: rune(891), To: rune(893)}, + &CodePointRange{From: rune(895), To: rune(895)}, + &CodePointRange{From: rune(900), To: rune(900)}, + &CodePointRange{From: rune(902), To: rune(902)}, + &CodePointRange{From: rune(904), To: rune(906)}, + &CodePointRange{From: rune(908), To: rune(908)}, + &CodePointRange{From: rune(910), To: rune(929)}, + &CodePointRange{From: rune(931), To: rune(993)}, + &CodePointRange{From: rune(1008), To: rune(1013)}, + &CodePointRange{From: rune(1014), To: rune(1014)}, + &CodePointRange{From: rune(1015), To: rune(1023)}, + &CodePointRange{From: rune(7462), To: rune(7466)}, + &CodePointRange{From: rune(7517), To: rune(7521)}, + &CodePointRange{From: rune(7526), To: rune(7530)}, + &CodePointRange{From: rune(7615), To: rune(7615)}, + &CodePointRange{From: rune(7936), To: rune(7957)}, + &CodePointRange{From: rune(7960), To: rune(7965)}, + &CodePointRange{From: rune(7968), To: rune(8005)}, + &CodePointRange{From: rune(8008), To: rune(8013)}, + &CodePointRange{From: rune(8016), To: rune(8023)}, + &CodePointRange{From: rune(8025), To: rune(8025)}, + &CodePointRange{From: rune(8027), To: rune(8027)}, + &CodePointRange{From: rune(8029), To: rune(8029)}, + &CodePointRange{From: rune(8031), To: rune(8061)}, + &CodePointRange{From: rune(8064), To: rune(8116)}, + &CodePointRange{From: rune(8118), To: rune(8124)}, + &CodePointRange{From: rune(8125), To: rune(8125)}, + &CodePointRange{From: rune(8126), To: rune(8126)}, + &CodePointRange{From: rune(8127), To: rune(8129)}, + &CodePointRange{From: rune(8130), To: rune(8132)}, + &CodePointRange{From: rune(8134), To: rune(8140)}, + &CodePointRange{From: rune(8141), To: rune(8143)}, + &CodePointRange{From: rune(8144), To: rune(8147)}, + &CodePointRange{From: rune(8150), To: rune(8155)}, + &CodePointRange{From: rune(8157), To: rune(8159)}, + &CodePointRange{From: rune(8160), To: rune(8172)}, + &CodePointRange{From: rune(8173), To: rune(8175)}, + &CodePointRange{From: rune(8178), To: rune(8180)}, + &CodePointRange{From: rune(8182), To: rune(8188)}, + &CodePointRange{From: rune(8189), To: rune(8190)}, + &CodePointRange{From: rune(8486), To: rune(8486)}, + &CodePointRange{From: rune(43877), To: rune(43877)}, + &CodePointRange{From: rune(65856), To: rune(65908)}, + &CodePointRange{From: rune(65909), To: rune(65912)}, + &CodePointRange{From: rune(65913), To: rune(65929)}, + &CodePointRange{From: rune(65930), To: rune(65931)}, + &CodePointRange{From: rune(65932), To: rune(65934)}, + &CodePointRange{From: rune(65952), To: rune(65952)}, + &CodePointRange{From: rune(119296), To: rune(119361)}, + &CodePointRange{From: rune(119362), To: rune(119364)}, + &CodePointRange{From: rune(119365), To: rune(119365)}, + }, + "gujr": { + &CodePointRange{From: rune(2689), To: rune(2690)}, + &CodePointRange{From: rune(2691), To: rune(2691)}, + &CodePointRange{From: rune(2693), To: rune(2701)}, + &CodePointRange{From: rune(2703), To: rune(2705)}, + &CodePointRange{From: rune(2707), To: rune(2728)}, + &CodePointRange{From: rune(2730), To: rune(2736)}, + &CodePointRange{From: rune(2738), To: rune(2739)}, + &CodePointRange{From: rune(2741), To: rune(2745)}, + &CodePointRange{From: rune(2748), To: rune(2748)}, + &CodePointRange{From: rune(2749), To: rune(2749)}, + &CodePointRange{From: rune(2750), To: rune(2752)}, + &CodePointRange{From: rune(2753), To: rune(2757)}, + &CodePointRange{From: rune(2759), To: rune(2760)}, + &CodePointRange{From: rune(2761), To: rune(2761)}, + &CodePointRange{From: rune(2763), To: rune(2764)}, + &CodePointRange{From: rune(2765), To: rune(2765)}, + &CodePointRange{From: rune(2768), To: rune(2768)}, + &CodePointRange{From: rune(2784), To: rune(2785)}, + &CodePointRange{From: rune(2786), To: rune(2787)}, + &CodePointRange{From: rune(2790), To: rune(2799)}, + &CodePointRange{From: rune(2800), To: rune(2800)}, + &CodePointRange{From: rune(2801), To: rune(2801)}, + &CodePointRange{From: rune(2809), To: rune(2809)}, + &CodePointRange{From: rune(2810), To: rune(2815)}, + }, + "guru": { + &CodePointRange{From: rune(2561), To: rune(2562)}, + &CodePointRange{From: rune(2563), To: rune(2563)}, + &CodePointRange{From: rune(2565), To: rune(2570)}, + &CodePointRange{From: rune(2575), To: rune(2576)}, + &CodePointRange{From: rune(2579), To: rune(2600)}, + &CodePointRange{From: rune(2602), To: rune(2608)}, + &CodePointRange{From: rune(2610), To: rune(2611)}, + &CodePointRange{From: rune(2613), To: rune(2614)}, + &CodePointRange{From: rune(2616), To: rune(2617)}, + &CodePointRange{From: rune(2620), To: rune(2620)}, + &CodePointRange{From: rune(2622), To: rune(2624)}, + &CodePointRange{From: rune(2625), To: rune(2626)}, + &CodePointRange{From: rune(2631), To: rune(2632)}, + &CodePointRange{From: rune(2635), To: rune(2637)}, + &CodePointRange{From: rune(2641), To: rune(2641)}, + &CodePointRange{From: rune(2649), To: rune(2652)}, + &CodePointRange{From: rune(2654), To: rune(2654)}, + &CodePointRange{From: rune(2662), To: rune(2671)}, + &CodePointRange{From: rune(2672), To: rune(2673)}, + &CodePointRange{From: rune(2674), To: rune(2676)}, + &CodePointRange{From: rune(2677), To: rune(2677)}, + &CodePointRange{From: rune(2678), To: rune(2678)}, + }, + "hang": { + &CodePointRange{From: rune(4352), To: rune(4607)}, + &CodePointRange{From: rune(12334), To: rune(12335)}, + &CodePointRange{From: rune(12593), To: rune(12686)}, + &CodePointRange{From: rune(12800), To: rune(12830)}, + &CodePointRange{From: rune(12896), To: rune(12926)}, + &CodePointRange{From: rune(43360), To: rune(43388)}, + &CodePointRange{From: rune(44032), To: rune(55203)}, + &CodePointRange{From: rune(55216), To: rune(55238)}, + &CodePointRange{From: rune(55243), To: rune(55291)}, + &CodePointRange{From: rune(65440), To: rune(65470)}, + &CodePointRange{From: rune(65474), To: rune(65479)}, + &CodePointRange{From: rune(65482), To: rune(65487)}, + &CodePointRange{From: rune(65490), To: rune(65495)}, + &CodePointRange{From: rune(65498), To: rune(65500)}, + }, + "hani": { + &CodePointRange{From: rune(11904), To: rune(11929)}, + &CodePointRange{From: rune(11931), To: rune(12019)}, + &CodePointRange{From: rune(12032), To: rune(12245)}, + &CodePointRange{From: rune(12293), To: rune(12293)}, + &CodePointRange{From: rune(12295), To: rune(12295)}, + &CodePointRange{From: rune(12321), To: rune(12329)}, + &CodePointRange{From: rune(12344), To: rune(12346)}, + &CodePointRange{From: rune(12347), To: rune(12347)}, + &CodePointRange{From: rune(13312), To: rune(19903)}, + &CodePointRange{From: rune(19968), To: rune(40956)}, + &CodePointRange{From: rune(63744), To: rune(64109)}, + &CodePointRange{From: rune(64112), To: rune(64217)}, + &CodePointRange{From: rune(94192), To: rune(94193)}, + &CodePointRange{From: rune(131072), To: rune(173789)}, + &CodePointRange{From: rune(173824), To: rune(177972)}, + &CodePointRange{From: rune(177984), To: rune(178205)}, + &CodePointRange{From: rune(178208), To: rune(183969)}, + &CodePointRange{From: rune(183984), To: rune(191456)}, + &CodePointRange{From: rune(194560), To: rune(195101)}, + &CodePointRange{From: rune(196608), To: rune(201546)}, + }, + "hano": { + &CodePointRange{From: rune(5920), To: rune(5937)}, + &CodePointRange{From: rune(5938), To: rune(5940)}, + }, + "hatr": { + &CodePointRange{From: rune(67808), To: rune(67826)}, + &CodePointRange{From: rune(67828), To: rune(67829)}, + &CodePointRange{From: rune(67835), To: rune(67839)}, + }, + "hebr": { + &CodePointRange{From: rune(1425), To: rune(1469)}, + &CodePointRange{From: rune(1470), To: rune(1470)}, + &CodePointRange{From: rune(1471), To: rune(1471)}, + &CodePointRange{From: rune(1472), To: rune(1472)}, + &CodePointRange{From: rune(1473), To: rune(1474)}, + &CodePointRange{From: rune(1475), To: rune(1475)}, + &CodePointRange{From: rune(1476), To: rune(1477)}, + &CodePointRange{From: rune(1478), To: rune(1478)}, + &CodePointRange{From: rune(1479), To: rune(1479)}, + &CodePointRange{From: rune(1488), To: rune(1514)}, + &CodePointRange{From: rune(1519), To: rune(1522)}, + &CodePointRange{From: rune(1523), To: rune(1524)}, + &CodePointRange{From: rune(64285), To: rune(64285)}, + &CodePointRange{From: rune(64286), To: rune(64286)}, + &CodePointRange{From: rune(64287), To: rune(64296)}, + &CodePointRange{From: rune(64297), To: rune(64297)}, + &CodePointRange{From: rune(64298), To: rune(64310)}, + &CodePointRange{From: rune(64312), To: rune(64316)}, + &CodePointRange{From: rune(64318), To: rune(64318)}, + &CodePointRange{From: rune(64320), To: rune(64321)}, + &CodePointRange{From: rune(64323), To: rune(64324)}, + &CodePointRange{From: rune(64326), To: rune(64335)}, + }, + "hira": { + &CodePointRange{From: rune(12353), To: rune(12438)}, + &CodePointRange{From: rune(12445), To: rune(12446)}, + &CodePointRange{From: rune(12447), To: rune(12447)}, + &CodePointRange{From: rune(110593), To: rune(110878)}, + &CodePointRange{From: rune(110928), To: rune(110930)}, + &CodePointRange{From: rune(127488), To: rune(127488)}, + }, + "hluw": { + &CodePointRange{From: rune(82944), To: rune(83526)}, + }, + "hmng": { + &CodePointRange{From: rune(92928), To: rune(92975)}, + &CodePointRange{From: rune(92976), To: rune(92982)}, + &CodePointRange{From: rune(92983), To: rune(92987)}, + &CodePointRange{From: rune(92988), To: rune(92991)}, + &CodePointRange{From: rune(92992), To: rune(92995)}, + &CodePointRange{From: rune(92996), To: rune(92996)}, + &CodePointRange{From: rune(92997), To: rune(92997)}, + &CodePointRange{From: rune(93008), To: rune(93017)}, + &CodePointRange{From: rune(93019), To: rune(93025)}, + &CodePointRange{From: rune(93027), To: rune(93047)}, + &CodePointRange{From: rune(93053), To: rune(93071)}, + }, + "hmnp": { + &CodePointRange{From: rune(123136), To: rune(123180)}, + &CodePointRange{From: rune(123184), To: rune(123190)}, + &CodePointRange{From: rune(123191), To: rune(123197)}, + &CodePointRange{From: rune(123200), To: rune(123209)}, + &CodePointRange{From: rune(123214), To: rune(123214)}, + &CodePointRange{From: rune(123215), To: rune(123215)}, + }, + "hung": { + &CodePointRange{From: rune(68736), To: rune(68786)}, + &CodePointRange{From: rune(68800), To: rune(68850)}, + &CodePointRange{From: rune(68858), To: rune(68863)}, + }, + "ital": { + &CodePointRange{From: rune(66304), To: rune(66335)}, + &CodePointRange{From: rune(66336), To: rune(66339)}, + &CodePointRange{From: rune(66349), To: rune(66351)}, + }, + "java": { + &CodePointRange{From: rune(43392), To: rune(43394)}, + &CodePointRange{From: rune(43395), To: rune(43395)}, + &CodePointRange{From: rune(43396), To: rune(43442)}, + &CodePointRange{From: rune(43443), To: rune(43443)}, + &CodePointRange{From: rune(43444), To: rune(43445)}, + &CodePointRange{From: rune(43446), To: rune(43449)}, + &CodePointRange{From: rune(43450), To: rune(43451)}, + &CodePointRange{From: rune(43452), To: rune(43453)}, + &CodePointRange{From: rune(43454), To: rune(43456)}, + &CodePointRange{From: rune(43457), To: rune(43469)}, + &CodePointRange{From: rune(43472), To: rune(43481)}, + &CodePointRange{From: rune(43486), To: rune(43487)}, + }, + "kali": { + &CodePointRange{From: rune(43264), To: rune(43273)}, + &CodePointRange{From: rune(43274), To: rune(43301)}, + &CodePointRange{From: rune(43302), To: rune(43309)}, + &CodePointRange{From: rune(43311), To: rune(43311)}, + }, + "kana": { + &CodePointRange{From: rune(12449), To: rune(12538)}, + &CodePointRange{From: rune(12541), To: rune(12542)}, + &CodePointRange{From: rune(12543), To: rune(12543)}, + &CodePointRange{From: rune(12784), To: rune(12799)}, + &CodePointRange{From: rune(13008), To: rune(13054)}, + &CodePointRange{From: rune(13056), To: rune(13143)}, + &CodePointRange{From: rune(65382), To: rune(65391)}, + &CodePointRange{From: rune(65393), To: rune(65437)}, + &CodePointRange{From: rune(110592), To: rune(110592)}, + &CodePointRange{From: rune(110948), To: rune(110951)}, + }, + "khar": { + &CodePointRange{From: rune(68096), To: rune(68096)}, + &CodePointRange{From: rune(68097), To: rune(68099)}, + &CodePointRange{From: rune(68101), To: rune(68102)}, + &CodePointRange{From: rune(68108), To: rune(68111)}, + &CodePointRange{From: rune(68112), To: rune(68115)}, + &CodePointRange{From: rune(68117), To: rune(68119)}, + &CodePointRange{From: rune(68121), To: rune(68149)}, + &CodePointRange{From: rune(68152), To: rune(68154)}, + &CodePointRange{From: rune(68159), To: rune(68159)}, + &CodePointRange{From: rune(68160), To: rune(68168)}, + &CodePointRange{From: rune(68176), To: rune(68184)}, + }, + "khmr": { + &CodePointRange{From: rune(6016), To: rune(6067)}, + &CodePointRange{From: rune(6068), To: rune(6069)}, + &CodePointRange{From: rune(6070), To: rune(6070)}, + &CodePointRange{From: rune(6071), To: rune(6077)}, + &CodePointRange{From: rune(6078), To: rune(6085)}, + &CodePointRange{From: rune(6086), To: rune(6086)}, + &CodePointRange{From: rune(6087), To: rune(6088)}, + &CodePointRange{From: rune(6089), To: rune(6099)}, + &CodePointRange{From: rune(6100), To: rune(6102)}, + &CodePointRange{From: rune(6103), To: rune(6103)}, + &CodePointRange{From: rune(6104), To: rune(6106)}, + &CodePointRange{From: rune(6107), To: rune(6107)}, + &CodePointRange{From: rune(6108), To: rune(6108)}, + &CodePointRange{From: rune(6109), To: rune(6109)}, + &CodePointRange{From: rune(6112), To: rune(6121)}, + &CodePointRange{From: rune(6128), To: rune(6137)}, + &CodePointRange{From: rune(6624), To: rune(6655)}, + }, + "khoj": { + &CodePointRange{From: rune(70144), To: rune(70161)}, + &CodePointRange{From: rune(70163), To: rune(70187)}, + &CodePointRange{From: rune(70188), To: rune(70190)}, + &CodePointRange{From: rune(70191), To: rune(70193)}, + &CodePointRange{From: rune(70194), To: rune(70195)}, + &CodePointRange{From: rune(70196), To: rune(70196)}, + &CodePointRange{From: rune(70197), To: rune(70197)}, + &CodePointRange{From: rune(70198), To: rune(70199)}, + &CodePointRange{From: rune(70200), To: rune(70205)}, + &CodePointRange{From: rune(70206), To: rune(70206)}, + }, + "kits": { + &CodePointRange{From: rune(94180), To: rune(94180)}, + &CodePointRange{From: rune(101120), To: rune(101589)}, + }, + "knda": { + &CodePointRange{From: rune(3200), To: rune(3200)}, + &CodePointRange{From: rune(3201), To: rune(3201)}, + &CodePointRange{From: rune(3202), To: rune(3203)}, + &CodePointRange{From: rune(3204), To: rune(3204)}, + &CodePointRange{From: rune(3205), To: rune(3212)}, + &CodePointRange{From: rune(3214), To: rune(3216)}, + &CodePointRange{From: rune(3218), To: rune(3240)}, + &CodePointRange{From: rune(3242), To: rune(3251)}, + &CodePointRange{From: rune(3253), To: rune(3257)}, + &CodePointRange{From: rune(3260), To: rune(3260)}, + &CodePointRange{From: rune(3261), To: rune(3261)}, + &CodePointRange{From: rune(3262), To: rune(3262)}, + &CodePointRange{From: rune(3263), To: rune(3263)}, + &CodePointRange{From: rune(3264), To: rune(3268)}, + &CodePointRange{From: rune(3270), To: rune(3270)}, + &CodePointRange{From: rune(3271), To: rune(3272)}, + &CodePointRange{From: rune(3274), To: rune(3275)}, + &CodePointRange{From: rune(3276), To: rune(3277)}, + &CodePointRange{From: rune(3285), To: rune(3286)}, + &CodePointRange{From: rune(3294), To: rune(3294)}, + &CodePointRange{From: rune(3296), To: rune(3297)}, + &CodePointRange{From: rune(3298), To: rune(3299)}, + &CodePointRange{From: rune(3302), To: rune(3311)}, + &CodePointRange{From: rune(3313), To: rune(3314)}, + }, + "kthi": { + &CodePointRange{From: rune(69760), To: rune(69761)}, + &CodePointRange{From: rune(69762), To: rune(69762)}, + &CodePointRange{From: rune(69763), To: rune(69807)}, + &CodePointRange{From: rune(69808), To: rune(69810)}, + &CodePointRange{From: rune(69811), To: rune(69814)}, + &CodePointRange{From: rune(69815), To: rune(69816)}, + &CodePointRange{From: rune(69817), To: rune(69818)}, + &CodePointRange{From: rune(69819), To: rune(69820)}, + &CodePointRange{From: rune(69821), To: rune(69821)}, + &CodePointRange{From: rune(69822), To: rune(69825)}, + &CodePointRange{From: rune(69837), To: rune(69837)}, + }, + "lana": { + &CodePointRange{From: rune(6688), To: rune(6740)}, + &CodePointRange{From: rune(6741), To: rune(6741)}, + &CodePointRange{From: rune(6742), To: rune(6742)}, + &CodePointRange{From: rune(6743), To: rune(6743)}, + &CodePointRange{From: rune(6744), To: rune(6750)}, + &CodePointRange{From: rune(6752), To: rune(6752)}, + &CodePointRange{From: rune(6753), To: rune(6753)}, + &CodePointRange{From: rune(6754), To: rune(6754)}, + &CodePointRange{From: rune(6755), To: rune(6756)}, + &CodePointRange{From: rune(6757), To: rune(6764)}, + &CodePointRange{From: rune(6765), To: rune(6770)}, + &CodePointRange{From: rune(6771), To: rune(6780)}, + &CodePointRange{From: rune(6783), To: rune(6783)}, + &CodePointRange{From: rune(6784), To: rune(6793)}, + &CodePointRange{From: rune(6800), To: rune(6809)}, + &CodePointRange{From: rune(6816), To: rune(6822)}, + &CodePointRange{From: rune(6823), To: rune(6823)}, + &CodePointRange{From: rune(6824), To: rune(6829)}, + }, + "laoo": { + &CodePointRange{From: rune(3713), To: rune(3714)}, + &CodePointRange{From: rune(3716), To: rune(3716)}, + &CodePointRange{From: rune(3718), To: rune(3722)}, + &CodePointRange{From: rune(3724), To: rune(3747)}, + &CodePointRange{From: rune(3749), To: rune(3749)}, + &CodePointRange{From: rune(3751), To: rune(3760)}, + &CodePointRange{From: rune(3761), To: rune(3761)}, + &CodePointRange{From: rune(3762), To: rune(3763)}, + &CodePointRange{From: rune(3764), To: rune(3772)}, + &CodePointRange{From: rune(3773), To: rune(3773)}, + &CodePointRange{From: rune(3776), To: rune(3780)}, + &CodePointRange{From: rune(3782), To: rune(3782)}, + &CodePointRange{From: rune(3784), To: rune(3789)}, + &CodePointRange{From: rune(3792), To: rune(3801)}, + &CodePointRange{From: rune(3804), To: rune(3807)}, + }, + "latn": { + &CodePointRange{From: rune(65), To: rune(90)}, + &CodePointRange{From: rune(97), To: rune(122)}, + &CodePointRange{From: rune(170), To: rune(170)}, + &CodePointRange{From: rune(186), To: rune(186)}, + &CodePointRange{From: rune(192), To: rune(214)}, + &CodePointRange{From: rune(216), To: rune(246)}, + &CodePointRange{From: rune(248), To: rune(442)}, + &CodePointRange{From: rune(443), To: rune(443)}, + &CodePointRange{From: rune(444), To: rune(447)}, + &CodePointRange{From: rune(448), To: rune(451)}, + &CodePointRange{From: rune(452), To: rune(659)}, + &CodePointRange{From: rune(660), To: rune(660)}, + &CodePointRange{From: rune(661), To: rune(687)}, + &CodePointRange{From: rune(688), To: rune(696)}, + &CodePointRange{From: rune(736), To: rune(740)}, + &CodePointRange{From: rune(7424), To: rune(7461)}, + &CodePointRange{From: rune(7468), To: rune(7516)}, + &CodePointRange{From: rune(7522), To: rune(7525)}, + &CodePointRange{From: rune(7531), To: rune(7543)}, + &CodePointRange{From: rune(7545), To: rune(7578)}, + &CodePointRange{From: rune(7579), To: rune(7614)}, + &CodePointRange{From: rune(7680), To: rune(7935)}, + &CodePointRange{From: rune(8305), To: rune(8305)}, + &CodePointRange{From: rune(8319), To: rune(8319)}, + &CodePointRange{From: rune(8336), To: rune(8348)}, + &CodePointRange{From: rune(8490), To: rune(8491)}, + &CodePointRange{From: rune(8498), To: rune(8498)}, + &CodePointRange{From: rune(8526), To: rune(8526)}, + &CodePointRange{From: rune(8544), To: rune(8578)}, + &CodePointRange{From: rune(8579), To: rune(8580)}, + &CodePointRange{From: rune(8581), To: rune(8584)}, + &CodePointRange{From: rune(11360), To: rune(11387)}, + &CodePointRange{From: rune(11388), To: rune(11389)}, + &CodePointRange{From: rune(11390), To: rune(11391)}, + &CodePointRange{From: rune(42786), To: rune(42863)}, + &CodePointRange{From: rune(42864), To: rune(42864)}, + &CodePointRange{From: rune(42865), To: rune(42887)}, + &CodePointRange{From: rune(42891), To: rune(42894)}, + &CodePointRange{From: rune(42895), To: rune(42895)}, + &CodePointRange{From: rune(42896), To: rune(42943)}, + &CodePointRange{From: rune(42946), To: rune(42954)}, + &CodePointRange{From: rune(42997), To: rune(42998)}, + &CodePointRange{From: rune(42999), To: rune(42999)}, + &CodePointRange{From: rune(43000), To: rune(43001)}, + &CodePointRange{From: rune(43002), To: rune(43002)}, + &CodePointRange{From: rune(43003), To: rune(43007)}, + &CodePointRange{From: rune(43824), To: rune(43866)}, + &CodePointRange{From: rune(43868), To: rune(43871)}, + &CodePointRange{From: rune(43872), To: rune(43876)}, + &CodePointRange{From: rune(43878), To: rune(43880)}, + &CodePointRange{From: rune(43881), To: rune(43881)}, + &CodePointRange{From: rune(64256), To: rune(64262)}, + &CodePointRange{From: rune(65313), To: rune(65338)}, + &CodePointRange{From: rune(65345), To: rune(65370)}, + }, + "lepc": { + &CodePointRange{From: rune(7168), To: rune(7203)}, + &CodePointRange{From: rune(7204), To: rune(7211)}, + &CodePointRange{From: rune(7212), To: rune(7219)}, + &CodePointRange{From: rune(7220), To: rune(7221)}, + &CodePointRange{From: rune(7222), To: rune(7223)}, + &CodePointRange{From: rune(7227), To: rune(7231)}, + &CodePointRange{From: rune(7232), To: rune(7241)}, + &CodePointRange{From: rune(7245), To: rune(7247)}, + }, + "limb": { + &CodePointRange{From: rune(6400), To: rune(6430)}, + &CodePointRange{From: rune(6432), To: rune(6434)}, + &CodePointRange{From: rune(6435), To: rune(6438)}, + &CodePointRange{From: rune(6439), To: rune(6440)}, + &CodePointRange{From: rune(6441), To: rune(6443)}, + &CodePointRange{From: rune(6448), To: rune(6449)}, + &CodePointRange{From: rune(6450), To: rune(6450)}, + &CodePointRange{From: rune(6451), To: rune(6456)}, + &CodePointRange{From: rune(6457), To: rune(6459)}, + &CodePointRange{From: rune(6464), To: rune(6464)}, + &CodePointRange{From: rune(6468), To: rune(6469)}, + &CodePointRange{From: rune(6470), To: rune(6479)}, + }, + "lina": { + &CodePointRange{From: rune(67072), To: rune(67382)}, + &CodePointRange{From: rune(67392), To: rune(67413)}, + &CodePointRange{From: rune(67424), To: rune(67431)}, + }, + "linb": { + &CodePointRange{From: rune(65536), To: rune(65547)}, + &CodePointRange{From: rune(65549), To: rune(65574)}, + &CodePointRange{From: rune(65576), To: rune(65594)}, + &CodePointRange{From: rune(65596), To: rune(65597)}, + &CodePointRange{From: rune(65599), To: rune(65613)}, + &CodePointRange{From: rune(65616), To: rune(65629)}, + &CodePointRange{From: rune(65664), To: rune(65786)}, + }, + "lisu": { + &CodePointRange{From: rune(42192), To: rune(42231)}, + &CodePointRange{From: rune(42232), To: rune(42237)}, + &CodePointRange{From: rune(42238), To: rune(42239)}, + &CodePointRange{From: rune(73648), To: rune(73648)}, + }, + "lyci": { + &CodePointRange{From: rune(66176), To: rune(66204)}, + }, + "lydi": { + &CodePointRange{From: rune(67872), To: rune(67897)}, + &CodePointRange{From: rune(67903), To: rune(67903)}, + }, + "mahj": { + &CodePointRange{From: rune(69968), To: rune(70002)}, + &CodePointRange{From: rune(70003), To: rune(70003)}, + &CodePointRange{From: rune(70004), To: rune(70005)}, + &CodePointRange{From: rune(70006), To: rune(70006)}, + }, + "maka": { + &CodePointRange{From: rune(73440), To: rune(73458)}, + &CodePointRange{From: rune(73459), To: rune(73460)}, + &CodePointRange{From: rune(73461), To: rune(73462)}, + &CodePointRange{From: rune(73463), To: rune(73464)}, + }, + "mand": { + &CodePointRange{From: rune(2112), To: rune(2136)}, + &CodePointRange{From: rune(2137), To: rune(2139)}, + &CodePointRange{From: rune(2142), To: rune(2142)}, + }, + "mani": { + &CodePointRange{From: rune(68288), To: rune(68295)}, + &CodePointRange{From: rune(68296), To: rune(68296)}, + &CodePointRange{From: rune(68297), To: rune(68324)}, + &CodePointRange{From: rune(68325), To: rune(68326)}, + &CodePointRange{From: rune(68331), To: rune(68335)}, + &CodePointRange{From: rune(68336), To: rune(68342)}, + }, + "marc": { + &CodePointRange{From: rune(72816), To: rune(72817)}, + &CodePointRange{From: rune(72818), To: rune(72847)}, + &CodePointRange{From: rune(72850), To: rune(72871)}, + &CodePointRange{From: rune(72873), To: rune(72873)}, + &CodePointRange{From: rune(72874), To: rune(72880)}, + &CodePointRange{From: rune(72881), To: rune(72881)}, + &CodePointRange{From: rune(72882), To: rune(72883)}, + &CodePointRange{From: rune(72884), To: rune(72884)}, + &CodePointRange{From: rune(72885), To: rune(72886)}, + }, + "medf": { + &CodePointRange{From: rune(93760), To: rune(93823)}, + &CodePointRange{From: rune(93824), To: rune(93846)}, + &CodePointRange{From: rune(93847), To: rune(93850)}, + }, + "mend": { + &CodePointRange{From: rune(124928), To: rune(125124)}, + &CodePointRange{From: rune(125127), To: rune(125135)}, + &CodePointRange{From: rune(125136), To: rune(125142)}, + }, + "merc": { + &CodePointRange{From: rune(68000), To: rune(68023)}, + &CodePointRange{From: rune(68028), To: rune(68029)}, + &CodePointRange{From: rune(68030), To: rune(68031)}, + &CodePointRange{From: rune(68032), To: rune(68047)}, + &CodePointRange{From: rune(68050), To: rune(68095)}, + }, + "mero": { + &CodePointRange{From: rune(67968), To: rune(67999)}, + }, + "mlym": { + &CodePointRange{From: rune(3328), To: rune(3329)}, + &CodePointRange{From: rune(3330), To: rune(3331)}, + &CodePointRange{From: rune(3332), To: rune(3340)}, + &CodePointRange{From: rune(3342), To: rune(3344)}, + &CodePointRange{From: rune(3346), To: rune(3386)}, + &CodePointRange{From: rune(3387), To: rune(3388)}, + &CodePointRange{From: rune(3389), To: rune(3389)}, + &CodePointRange{From: rune(3390), To: rune(3392)}, + &CodePointRange{From: rune(3393), To: rune(3396)}, + &CodePointRange{From: rune(3398), To: rune(3400)}, + &CodePointRange{From: rune(3402), To: rune(3404)}, + &CodePointRange{From: rune(3405), To: rune(3405)}, + &CodePointRange{From: rune(3406), To: rune(3406)}, + &CodePointRange{From: rune(3407), To: rune(3407)}, + &CodePointRange{From: rune(3412), To: rune(3414)}, + &CodePointRange{From: rune(3415), To: rune(3415)}, + &CodePointRange{From: rune(3416), To: rune(3422)}, + &CodePointRange{From: rune(3423), To: rune(3425)}, + &CodePointRange{From: rune(3426), To: rune(3427)}, + &CodePointRange{From: rune(3430), To: rune(3439)}, + &CodePointRange{From: rune(3440), To: rune(3448)}, + &CodePointRange{From: rune(3449), To: rune(3449)}, + &CodePointRange{From: rune(3450), To: rune(3455)}, + }, + "modi": { + &CodePointRange{From: rune(71168), To: rune(71215)}, + &CodePointRange{From: rune(71216), To: rune(71218)}, + &CodePointRange{From: rune(71219), To: rune(71226)}, + &CodePointRange{From: rune(71227), To: rune(71228)}, + &CodePointRange{From: rune(71229), To: rune(71229)}, + &CodePointRange{From: rune(71230), To: rune(71230)}, + &CodePointRange{From: rune(71231), To: rune(71232)}, + &CodePointRange{From: rune(71233), To: rune(71235)}, + &CodePointRange{From: rune(71236), To: rune(71236)}, + &CodePointRange{From: rune(71248), To: rune(71257)}, + }, + "mong": { + &CodePointRange{From: rune(6144), To: rune(6145)}, + &CodePointRange{From: rune(6148), To: rune(6148)}, + &CodePointRange{From: rune(6150), To: rune(6150)}, + &CodePointRange{From: rune(6151), To: rune(6154)}, + &CodePointRange{From: rune(6155), To: rune(6157)}, + &CodePointRange{From: rune(6158), To: rune(6158)}, + &CodePointRange{From: rune(6160), To: rune(6169)}, + &CodePointRange{From: rune(6176), To: rune(6210)}, + &CodePointRange{From: rune(6211), To: rune(6211)}, + &CodePointRange{From: rune(6212), To: rune(6264)}, + &CodePointRange{From: rune(6272), To: rune(6276)}, + &CodePointRange{From: rune(6277), To: rune(6278)}, + &CodePointRange{From: rune(6279), To: rune(6312)}, + &CodePointRange{From: rune(6313), To: rune(6313)}, + &CodePointRange{From: rune(6314), To: rune(6314)}, + &CodePointRange{From: rune(71264), To: rune(71276)}, + }, + "mroo": { + &CodePointRange{From: rune(92736), To: rune(92766)}, + &CodePointRange{From: rune(92768), To: rune(92777)}, + &CodePointRange{From: rune(92782), To: rune(92783)}, + }, + "mtei": { + &CodePointRange{From: rune(43744), To: rune(43754)}, + &CodePointRange{From: rune(43755), To: rune(43755)}, + &CodePointRange{From: rune(43756), To: rune(43757)}, + &CodePointRange{From: rune(43758), To: rune(43759)}, + &CodePointRange{From: rune(43760), To: rune(43761)}, + &CodePointRange{From: rune(43762), To: rune(43762)}, + &CodePointRange{From: rune(43763), To: rune(43764)}, + &CodePointRange{From: rune(43765), To: rune(43765)}, + &CodePointRange{From: rune(43766), To: rune(43766)}, + &CodePointRange{From: rune(43968), To: rune(44002)}, + &CodePointRange{From: rune(44003), To: rune(44004)}, + &CodePointRange{From: rune(44005), To: rune(44005)}, + &CodePointRange{From: rune(44006), To: rune(44007)}, + &CodePointRange{From: rune(44008), To: rune(44008)}, + &CodePointRange{From: rune(44009), To: rune(44010)}, + &CodePointRange{From: rune(44011), To: rune(44011)}, + &CodePointRange{From: rune(44012), To: rune(44012)}, + &CodePointRange{From: rune(44013), To: rune(44013)}, + &CodePointRange{From: rune(44016), To: rune(44025)}, + }, + "mult": { + &CodePointRange{From: rune(70272), To: rune(70278)}, + &CodePointRange{From: rune(70280), To: rune(70280)}, + &CodePointRange{From: rune(70282), To: rune(70285)}, + &CodePointRange{From: rune(70287), To: rune(70301)}, + &CodePointRange{From: rune(70303), To: rune(70312)}, + &CodePointRange{From: rune(70313), To: rune(70313)}, + }, + "mymr": { + &CodePointRange{From: rune(4096), To: rune(4138)}, + &CodePointRange{From: rune(4139), To: rune(4140)}, + &CodePointRange{From: rune(4141), To: rune(4144)}, + &CodePointRange{From: rune(4145), To: rune(4145)}, + &CodePointRange{From: rune(4146), To: rune(4151)}, + &CodePointRange{From: rune(4152), To: rune(4152)}, + &CodePointRange{From: rune(4153), To: rune(4154)}, + &CodePointRange{From: rune(4155), To: rune(4156)}, + &CodePointRange{From: rune(4157), To: rune(4158)}, + &CodePointRange{From: rune(4159), To: rune(4159)}, + &CodePointRange{From: rune(4160), To: rune(4169)}, + &CodePointRange{From: rune(4170), To: rune(4175)}, + &CodePointRange{From: rune(4176), To: rune(4181)}, + &CodePointRange{From: rune(4182), To: rune(4183)}, + &CodePointRange{From: rune(4184), To: rune(4185)}, + &CodePointRange{From: rune(4186), To: rune(4189)}, + &CodePointRange{From: rune(4190), To: rune(4192)}, + &CodePointRange{From: rune(4193), To: rune(4193)}, + &CodePointRange{From: rune(4194), To: rune(4196)}, + &CodePointRange{From: rune(4197), To: rune(4198)}, + &CodePointRange{From: rune(4199), To: rune(4205)}, + &CodePointRange{From: rune(4206), To: rune(4208)}, + &CodePointRange{From: rune(4209), To: rune(4212)}, + &CodePointRange{From: rune(4213), To: rune(4225)}, + &CodePointRange{From: rune(4226), To: rune(4226)}, + &CodePointRange{From: rune(4227), To: rune(4228)}, + &CodePointRange{From: rune(4229), To: rune(4230)}, + &CodePointRange{From: rune(4231), To: rune(4236)}, + &CodePointRange{From: rune(4237), To: rune(4237)}, + &CodePointRange{From: rune(4238), To: rune(4238)}, + &CodePointRange{From: rune(4239), To: rune(4239)}, + &CodePointRange{From: rune(4240), To: rune(4249)}, + &CodePointRange{From: rune(4250), To: rune(4252)}, + &CodePointRange{From: rune(4253), To: rune(4253)}, + &CodePointRange{From: rune(4254), To: rune(4255)}, + &CodePointRange{From: rune(43488), To: rune(43492)}, + &CodePointRange{From: rune(43493), To: rune(43493)}, + &CodePointRange{From: rune(43494), To: rune(43494)}, + &CodePointRange{From: rune(43495), To: rune(43503)}, + &CodePointRange{From: rune(43504), To: rune(43513)}, + &CodePointRange{From: rune(43514), To: rune(43518)}, + &CodePointRange{From: rune(43616), To: rune(43631)}, + &CodePointRange{From: rune(43632), To: rune(43632)}, + &CodePointRange{From: rune(43633), To: rune(43638)}, + &CodePointRange{From: rune(43639), To: rune(43641)}, + &CodePointRange{From: rune(43642), To: rune(43642)}, + &CodePointRange{From: rune(43643), To: rune(43643)}, + &CodePointRange{From: rune(43644), To: rune(43644)}, + &CodePointRange{From: rune(43645), To: rune(43645)}, + &CodePointRange{From: rune(43646), To: rune(43647)}, + }, + "nand": { + &CodePointRange{From: rune(72096), To: rune(72103)}, + &CodePointRange{From: rune(72106), To: rune(72144)}, + &CodePointRange{From: rune(72145), To: rune(72147)}, + &CodePointRange{From: rune(72148), To: rune(72151)}, + &CodePointRange{From: rune(72154), To: rune(72155)}, + &CodePointRange{From: rune(72156), To: rune(72159)}, + &CodePointRange{From: rune(72160), To: rune(72160)}, + &CodePointRange{From: rune(72161), To: rune(72161)}, + &CodePointRange{From: rune(72162), To: rune(72162)}, + &CodePointRange{From: rune(72163), To: rune(72163)}, + &CodePointRange{From: rune(72164), To: rune(72164)}, + }, + "narb": { + &CodePointRange{From: rune(68224), To: rune(68252)}, + &CodePointRange{From: rune(68253), To: rune(68255)}, + }, + "nbat": { + &CodePointRange{From: rune(67712), To: rune(67742)}, + &CodePointRange{From: rune(67751), To: rune(67759)}, + }, + "newa": { + &CodePointRange{From: rune(70656), To: rune(70708)}, + &CodePointRange{From: rune(70709), To: rune(70711)}, + &CodePointRange{From: rune(70712), To: rune(70719)}, + &CodePointRange{From: rune(70720), To: rune(70721)}, + &CodePointRange{From: rune(70722), To: rune(70724)}, + &CodePointRange{From: rune(70725), To: rune(70725)}, + &CodePointRange{From: rune(70726), To: rune(70726)}, + &CodePointRange{From: rune(70727), To: rune(70730)}, + &CodePointRange{From: rune(70731), To: rune(70735)}, + &CodePointRange{From: rune(70736), To: rune(70745)}, + &CodePointRange{From: rune(70746), To: rune(70747)}, + &CodePointRange{From: rune(70749), To: rune(70749)}, + &CodePointRange{From: rune(70750), To: rune(70750)}, + &CodePointRange{From: rune(70751), To: rune(70753)}, + }, + "nkoo": { + &CodePointRange{From: rune(1984), To: rune(1993)}, + &CodePointRange{From: rune(1994), To: rune(2026)}, + &CodePointRange{From: rune(2027), To: rune(2035)}, + &CodePointRange{From: rune(2036), To: rune(2037)}, + &CodePointRange{From: rune(2038), To: rune(2038)}, + &CodePointRange{From: rune(2039), To: rune(2041)}, + &CodePointRange{From: rune(2042), To: rune(2042)}, + &CodePointRange{From: rune(2045), To: rune(2045)}, + &CodePointRange{From: rune(2046), To: rune(2047)}, + }, + "nshu": { + &CodePointRange{From: rune(94177), To: rune(94177)}, + &CodePointRange{From: rune(110960), To: rune(111355)}, + }, + "ogam": { + &CodePointRange{From: rune(5760), To: rune(5760)}, + &CodePointRange{From: rune(5761), To: rune(5786)}, + &CodePointRange{From: rune(5787), To: rune(5787)}, + &CodePointRange{From: rune(5788), To: rune(5788)}, + }, + "olck": { + &CodePointRange{From: rune(7248), To: rune(7257)}, + &CodePointRange{From: rune(7258), To: rune(7287)}, + &CodePointRange{From: rune(7288), To: rune(7293)}, + &CodePointRange{From: rune(7294), To: rune(7295)}, + }, + "orkh": { + &CodePointRange{From: rune(68608), To: rune(68680)}, + }, + "orya": { + &CodePointRange{From: rune(2817), To: rune(2817)}, + &CodePointRange{From: rune(2818), To: rune(2819)}, + &CodePointRange{From: rune(2821), To: rune(2828)}, + &CodePointRange{From: rune(2831), To: rune(2832)}, + &CodePointRange{From: rune(2835), To: rune(2856)}, + &CodePointRange{From: rune(2858), To: rune(2864)}, + &CodePointRange{From: rune(2866), To: rune(2867)}, + &CodePointRange{From: rune(2869), To: rune(2873)}, + &CodePointRange{From: rune(2876), To: rune(2876)}, + &CodePointRange{From: rune(2877), To: rune(2877)}, + &CodePointRange{From: rune(2878), To: rune(2878)}, + &CodePointRange{From: rune(2879), To: rune(2879)}, + &CodePointRange{From: rune(2880), To: rune(2880)}, + &CodePointRange{From: rune(2881), To: rune(2884)}, + &CodePointRange{From: rune(2887), To: rune(2888)}, + &CodePointRange{From: rune(2891), To: rune(2892)}, + &CodePointRange{From: rune(2893), To: rune(2893)}, + &CodePointRange{From: rune(2901), To: rune(2902)}, + &CodePointRange{From: rune(2903), To: rune(2903)}, + &CodePointRange{From: rune(2908), To: rune(2909)}, + &CodePointRange{From: rune(2911), To: rune(2913)}, + &CodePointRange{From: rune(2914), To: rune(2915)}, + &CodePointRange{From: rune(2918), To: rune(2927)}, + &CodePointRange{From: rune(2928), To: rune(2928)}, + &CodePointRange{From: rune(2929), To: rune(2929)}, + &CodePointRange{From: rune(2930), To: rune(2935)}, + }, + "osge": { + &CodePointRange{From: rune(66736), To: rune(66771)}, + &CodePointRange{From: rune(66776), To: rune(66811)}, + }, + "osma": { + &CodePointRange{From: rune(66688), To: rune(66717)}, + &CodePointRange{From: rune(66720), To: rune(66729)}, + }, + "palm": { + &CodePointRange{From: rune(67680), To: rune(67702)}, + &CodePointRange{From: rune(67703), To: rune(67704)}, + &CodePointRange{From: rune(67705), To: rune(67711)}, + }, + "pauc": { + &CodePointRange{From: rune(72384), To: rune(72440)}, + }, + "perm": { + &CodePointRange{From: rune(66384), To: rune(66421)}, + &CodePointRange{From: rune(66422), To: rune(66426)}, + }, + "phag": { + &CodePointRange{From: rune(43072), To: rune(43123)}, + &CodePointRange{From: rune(43124), To: rune(43127)}, + }, + "phli": { + &CodePointRange{From: rune(68448), To: rune(68466)}, + &CodePointRange{From: rune(68472), To: rune(68479)}, + }, + "phlp": { + &CodePointRange{From: rune(68480), To: rune(68497)}, + &CodePointRange{From: rune(68505), To: rune(68508)}, + &CodePointRange{From: rune(68521), To: rune(68527)}, + }, + "phnx": { + &CodePointRange{From: rune(67840), To: rune(67861)}, + &CodePointRange{From: rune(67862), To: rune(67867)}, + &CodePointRange{From: rune(67871), To: rune(67871)}, + }, + "plrd": { + &CodePointRange{From: rune(93952), To: rune(94026)}, + &CodePointRange{From: rune(94031), To: rune(94031)}, + &CodePointRange{From: rune(94032), To: rune(94032)}, + &CodePointRange{From: rune(94033), To: rune(94087)}, + &CodePointRange{From: rune(94095), To: rune(94098)}, + &CodePointRange{From: rune(94099), To: rune(94111)}, + }, + "prti": { + &CodePointRange{From: rune(68416), To: rune(68437)}, + &CodePointRange{From: rune(68440), To: rune(68447)}, + }, + "rjng": { + &CodePointRange{From: rune(43312), To: rune(43334)}, + &CodePointRange{From: rune(43335), To: rune(43345)}, + &CodePointRange{From: rune(43346), To: rune(43347)}, + &CodePointRange{From: rune(43359), To: rune(43359)}, + }, + "rohg": { + &CodePointRange{From: rune(68864), To: rune(68899)}, + &CodePointRange{From: rune(68900), To: rune(68903)}, + &CodePointRange{From: rune(68912), To: rune(68921)}, + }, + "runr": { + &CodePointRange{From: rune(5792), To: rune(5866)}, + &CodePointRange{From: rune(5870), To: rune(5872)}, + &CodePointRange{From: rune(5873), To: rune(5880)}, + }, + "samr": { + &CodePointRange{From: rune(2048), To: rune(2069)}, + &CodePointRange{From: rune(2070), To: rune(2073)}, + &CodePointRange{From: rune(2074), To: rune(2074)}, + &CodePointRange{From: rune(2075), To: rune(2083)}, + &CodePointRange{From: rune(2084), To: rune(2084)}, + &CodePointRange{From: rune(2085), To: rune(2087)}, + &CodePointRange{From: rune(2088), To: rune(2088)}, + &CodePointRange{From: rune(2089), To: rune(2093)}, + &CodePointRange{From: rune(2096), To: rune(2110)}, + }, + "sarb": { + &CodePointRange{From: rune(68192), To: rune(68220)}, + &CodePointRange{From: rune(68221), To: rune(68222)}, + &CodePointRange{From: rune(68223), To: rune(68223)}, + }, + "saur": { + &CodePointRange{From: rune(43136), To: rune(43137)}, + &CodePointRange{From: rune(43138), To: rune(43187)}, + &CodePointRange{From: rune(43188), To: rune(43203)}, + &CodePointRange{From: rune(43204), To: rune(43205)}, + &CodePointRange{From: rune(43214), To: rune(43215)}, + &CodePointRange{From: rune(43216), To: rune(43225)}, + }, + "sgnw": { + &CodePointRange{From: rune(120832), To: rune(121343)}, + &CodePointRange{From: rune(121344), To: rune(121398)}, + &CodePointRange{From: rune(121399), To: rune(121402)}, + &CodePointRange{From: rune(121403), To: rune(121452)}, + &CodePointRange{From: rune(121453), To: rune(121460)}, + &CodePointRange{From: rune(121461), To: rune(121461)}, + &CodePointRange{From: rune(121462), To: rune(121475)}, + &CodePointRange{From: rune(121476), To: rune(121476)}, + &CodePointRange{From: rune(121477), To: rune(121478)}, + &CodePointRange{From: rune(121479), To: rune(121483)}, + &CodePointRange{From: rune(121499), To: rune(121503)}, + &CodePointRange{From: rune(121505), To: rune(121519)}, + }, + "shaw": { + &CodePointRange{From: rune(66640), To: rune(66687)}, + }, + "shrd": { + &CodePointRange{From: rune(70016), To: rune(70017)}, + &CodePointRange{From: rune(70018), To: rune(70018)}, + &CodePointRange{From: rune(70019), To: rune(70066)}, + &CodePointRange{From: rune(70067), To: rune(70069)}, + &CodePointRange{From: rune(70070), To: rune(70078)}, + &CodePointRange{From: rune(70079), To: rune(70080)}, + &CodePointRange{From: rune(70081), To: rune(70084)}, + &CodePointRange{From: rune(70085), To: rune(70088)}, + &CodePointRange{From: rune(70089), To: rune(70092)}, + &CodePointRange{From: rune(70093), To: rune(70093)}, + &CodePointRange{From: rune(70094), To: rune(70094)}, + &CodePointRange{From: rune(70095), To: rune(70095)}, + &CodePointRange{From: rune(70096), To: rune(70105)}, + &CodePointRange{From: rune(70106), To: rune(70106)}, + &CodePointRange{From: rune(70107), To: rune(70107)}, + &CodePointRange{From: rune(70108), To: rune(70108)}, + &CodePointRange{From: rune(70109), To: rune(70111)}, + }, + "sidd": { + &CodePointRange{From: rune(71040), To: rune(71086)}, + &CodePointRange{From: rune(71087), To: rune(71089)}, + &CodePointRange{From: rune(71090), To: rune(71093)}, + &CodePointRange{From: rune(71096), To: rune(71099)}, + &CodePointRange{From: rune(71100), To: rune(71101)}, + &CodePointRange{From: rune(71102), To: rune(71102)}, + &CodePointRange{From: rune(71103), To: rune(71104)}, + &CodePointRange{From: rune(71105), To: rune(71127)}, + &CodePointRange{From: rune(71128), To: rune(71131)}, + &CodePointRange{From: rune(71132), To: rune(71133)}, + }, + "sind": { + &CodePointRange{From: rune(70320), To: rune(70366)}, + &CodePointRange{From: rune(70367), To: rune(70367)}, + &CodePointRange{From: rune(70368), To: rune(70370)}, + &CodePointRange{From: rune(70371), To: rune(70378)}, + &CodePointRange{From: rune(70384), To: rune(70393)}, + }, + "sinh": { + &CodePointRange{From: rune(3457), To: rune(3457)}, + &CodePointRange{From: rune(3458), To: rune(3459)}, + &CodePointRange{From: rune(3461), To: rune(3478)}, + &CodePointRange{From: rune(3482), To: rune(3505)}, + &CodePointRange{From: rune(3507), To: rune(3515)}, + &CodePointRange{From: rune(3517), To: rune(3517)}, + &CodePointRange{From: rune(3520), To: rune(3526)}, + &CodePointRange{From: rune(3530), To: rune(3530)}, + &CodePointRange{From: rune(3535), To: rune(3537)}, + &CodePointRange{From: rune(3538), To: rune(3540)}, + &CodePointRange{From: rune(3542), To: rune(3542)}, + &CodePointRange{From: rune(3544), To: rune(3551)}, + &CodePointRange{From: rune(3558), To: rune(3567)}, + &CodePointRange{From: rune(3570), To: rune(3571)}, + &CodePointRange{From: rune(3572), To: rune(3572)}, + &CodePointRange{From: rune(70113), To: rune(70132)}, + }, + "sogd": { + &CodePointRange{From: rune(69424), To: rune(69445)}, + &CodePointRange{From: rune(69446), To: rune(69456)}, + &CodePointRange{From: rune(69457), To: rune(69460)}, + &CodePointRange{From: rune(69461), To: rune(69465)}, + }, + "sogo": { + &CodePointRange{From: rune(69376), To: rune(69404)}, + &CodePointRange{From: rune(69405), To: rune(69414)}, + &CodePointRange{From: rune(69415), To: rune(69415)}, + }, + "sora": { + &CodePointRange{From: rune(69840), To: rune(69864)}, + &CodePointRange{From: rune(69872), To: rune(69881)}, + }, + "soyo": { + &CodePointRange{From: rune(72272), To: rune(72272)}, + &CodePointRange{From: rune(72273), To: rune(72278)}, + &CodePointRange{From: rune(72279), To: rune(72280)}, + &CodePointRange{From: rune(72281), To: rune(72283)}, + &CodePointRange{From: rune(72284), To: rune(72329)}, + &CodePointRange{From: rune(72330), To: rune(72342)}, + &CodePointRange{From: rune(72343), To: rune(72343)}, + &CodePointRange{From: rune(72344), To: rune(72345)}, + &CodePointRange{From: rune(72346), To: rune(72348)}, + &CodePointRange{From: rune(72349), To: rune(72349)}, + &CodePointRange{From: rune(72350), To: rune(72354)}, + }, + "sund": { + &CodePointRange{From: rune(7040), To: rune(7041)}, + &CodePointRange{From: rune(7042), To: rune(7042)}, + &CodePointRange{From: rune(7043), To: rune(7072)}, + &CodePointRange{From: rune(7073), To: rune(7073)}, + &CodePointRange{From: rune(7074), To: rune(7077)}, + &CodePointRange{From: rune(7078), To: rune(7079)}, + &CodePointRange{From: rune(7080), To: rune(7081)}, + &CodePointRange{From: rune(7082), To: rune(7082)}, + &CodePointRange{From: rune(7083), To: rune(7085)}, + &CodePointRange{From: rune(7086), To: rune(7087)}, + &CodePointRange{From: rune(7088), To: rune(7097)}, + &CodePointRange{From: rune(7098), To: rune(7103)}, + &CodePointRange{From: rune(7360), To: rune(7367)}, + }, + "sylo": { + &CodePointRange{From: rune(43008), To: rune(43009)}, + &CodePointRange{From: rune(43010), To: rune(43010)}, + &CodePointRange{From: rune(43011), To: rune(43013)}, + &CodePointRange{From: rune(43014), To: rune(43014)}, + &CodePointRange{From: rune(43015), To: rune(43018)}, + &CodePointRange{From: rune(43019), To: rune(43019)}, + &CodePointRange{From: rune(43020), To: rune(43042)}, + &CodePointRange{From: rune(43043), To: rune(43044)}, + &CodePointRange{From: rune(43045), To: rune(43046)}, + &CodePointRange{From: rune(43047), To: rune(43047)}, + &CodePointRange{From: rune(43048), To: rune(43051)}, + &CodePointRange{From: rune(43052), To: rune(43052)}, + }, + "syrc": { + &CodePointRange{From: rune(1792), To: rune(1805)}, + &CodePointRange{From: rune(1807), To: rune(1807)}, + &CodePointRange{From: rune(1808), To: rune(1808)}, + &CodePointRange{From: rune(1809), To: rune(1809)}, + &CodePointRange{From: rune(1810), To: rune(1839)}, + &CodePointRange{From: rune(1840), To: rune(1866)}, + &CodePointRange{From: rune(1869), To: rune(1871)}, + &CodePointRange{From: rune(2144), To: rune(2154)}, + }, + "tagb": { + &CodePointRange{From: rune(5984), To: rune(5996)}, + &CodePointRange{From: rune(5998), To: rune(6000)}, + &CodePointRange{From: rune(6002), To: rune(6003)}, + }, + "takr": { + &CodePointRange{From: rune(71296), To: rune(71338)}, + &CodePointRange{From: rune(71339), To: rune(71339)}, + &CodePointRange{From: rune(71340), To: rune(71340)}, + &CodePointRange{From: rune(71341), To: rune(71341)}, + &CodePointRange{From: rune(71342), To: rune(71343)}, + &CodePointRange{From: rune(71344), To: rune(71349)}, + &CodePointRange{From: rune(71350), To: rune(71350)}, + &CodePointRange{From: rune(71351), To: rune(71351)}, + &CodePointRange{From: rune(71352), To: rune(71352)}, + &CodePointRange{From: rune(71360), To: rune(71369)}, + }, + "tale": { + &CodePointRange{From: rune(6480), To: rune(6509)}, + &CodePointRange{From: rune(6512), To: rune(6516)}, + }, + "talu": { + &CodePointRange{From: rune(6528), To: rune(6571)}, + &CodePointRange{From: rune(6576), To: rune(6601)}, + &CodePointRange{From: rune(6608), To: rune(6617)}, + &CodePointRange{From: rune(6618), To: rune(6618)}, + &CodePointRange{From: rune(6622), To: rune(6623)}, + }, + "taml": { + &CodePointRange{From: rune(2946), To: rune(2946)}, + &CodePointRange{From: rune(2947), To: rune(2947)}, + &CodePointRange{From: rune(2949), To: rune(2954)}, + &CodePointRange{From: rune(2958), To: rune(2960)}, + &CodePointRange{From: rune(2962), To: rune(2965)}, + &CodePointRange{From: rune(2969), To: rune(2970)}, + &CodePointRange{From: rune(2972), To: rune(2972)}, + &CodePointRange{From: rune(2974), To: rune(2975)}, + &CodePointRange{From: rune(2979), To: rune(2980)}, + &CodePointRange{From: rune(2984), To: rune(2986)}, + &CodePointRange{From: rune(2990), To: rune(3001)}, + &CodePointRange{From: rune(3006), To: rune(3007)}, + &CodePointRange{From: rune(3008), To: rune(3008)}, + &CodePointRange{From: rune(3009), To: rune(3010)}, + &CodePointRange{From: rune(3014), To: rune(3016)}, + &CodePointRange{From: rune(3018), To: rune(3020)}, + &CodePointRange{From: rune(3021), To: rune(3021)}, + &CodePointRange{From: rune(3024), To: rune(3024)}, + &CodePointRange{From: rune(3031), To: rune(3031)}, + &CodePointRange{From: rune(3046), To: rune(3055)}, + &CodePointRange{From: rune(3056), To: rune(3058)}, + &CodePointRange{From: rune(3059), To: rune(3064)}, + &CodePointRange{From: rune(3065), To: rune(3065)}, + &CodePointRange{From: rune(3066), To: rune(3066)}, + &CodePointRange{From: rune(73664), To: rune(73684)}, + &CodePointRange{From: rune(73685), To: rune(73692)}, + &CodePointRange{From: rune(73693), To: rune(73696)}, + &CodePointRange{From: rune(73697), To: rune(73713)}, + &CodePointRange{From: rune(73727), To: rune(73727)}, + }, + "tang": { + &CodePointRange{From: rune(94176), To: rune(94176)}, + &CodePointRange{From: rune(94208), To: rune(100343)}, + &CodePointRange{From: rune(100352), To: rune(101119)}, + &CodePointRange{From: rune(101632), To: rune(101640)}, + }, + "tavt": { + &CodePointRange{From: rune(43648), To: rune(43695)}, + &CodePointRange{From: rune(43696), To: rune(43696)}, + &CodePointRange{From: rune(43697), To: rune(43697)}, + &CodePointRange{From: rune(43698), To: rune(43700)}, + &CodePointRange{From: rune(43701), To: rune(43702)}, + &CodePointRange{From: rune(43703), To: rune(43704)}, + &CodePointRange{From: rune(43705), To: rune(43709)}, + &CodePointRange{From: rune(43710), To: rune(43711)}, + &CodePointRange{From: rune(43712), To: rune(43712)}, + &CodePointRange{From: rune(43713), To: rune(43713)}, + &CodePointRange{From: rune(43714), To: rune(43714)}, + &CodePointRange{From: rune(43739), To: rune(43740)}, + &CodePointRange{From: rune(43741), To: rune(43741)}, + &CodePointRange{From: rune(43742), To: rune(43743)}, + }, + "telu": { + &CodePointRange{From: rune(3072), To: rune(3072)}, + &CodePointRange{From: rune(3073), To: rune(3075)}, + &CodePointRange{From: rune(3076), To: rune(3076)}, + &CodePointRange{From: rune(3077), To: rune(3084)}, + &CodePointRange{From: rune(3086), To: rune(3088)}, + &CodePointRange{From: rune(3090), To: rune(3112)}, + &CodePointRange{From: rune(3114), To: rune(3129)}, + &CodePointRange{From: rune(3133), To: rune(3133)}, + &CodePointRange{From: rune(3134), To: rune(3136)}, + &CodePointRange{From: rune(3137), To: rune(3140)}, + &CodePointRange{From: rune(3142), To: rune(3144)}, + &CodePointRange{From: rune(3146), To: rune(3149)}, + &CodePointRange{From: rune(3157), To: rune(3158)}, + &CodePointRange{From: rune(3160), To: rune(3162)}, + &CodePointRange{From: rune(3168), To: rune(3169)}, + &CodePointRange{From: rune(3170), To: rune(3171)}, + &CodePointRange{From: rune(3174), To: rune(3183)}, + &CodePointRange{From: rune(3191), To: rune(3191)}, + &CodePointRange{From: rune(3192), To: rune(3198)}, + &CodePointRange{From: rune(3199), To: rune(3199)}, + }, + "tfng": { + &CodePointRange{From: rune(11568), To: rune(11623)}, + &CodePointRange{From: rune(11631), To: rune(11631)}, + &CodePointRange{From: rune(11632), To: rune(11632)}, + &CodePointRange{From: rune(11647), To: rune(11647)}, + }, + "tglg": { + &CodePointRange{From: rune(5888), To: rune(5900)}, + &CodePointRange{From: rune(5902), To: rune(5905)}, + &CodePointRange{From: rune(5906), To: rune(5908)}, + }, + "thaa": { + &CodePointRange{From: rune(1920), To: rune(1957)}, + &CodePointRange{From: rune(1958), To: rune(1968)}, + &CodePointRange{From: rune(1969), To: rune(1969)}, + }, + "thai": { + &CodePointRange{From: rune(3585), To: rune(3632)}, + &CodePointRange{From: rune(3633), To: rune(3633)}, + &CodePointRange{From: rune(3634), To: rune(3635)}, + &CodePointRange{From: rune(3636), To: rune(3642)}, + &CodePointRange{From: rune(3648), To: rune(3653)}, + &CodePointRange{From: rune(3654), To: rune(3654)}, + &CodePointRange{From: rune(3655), To: rune(3662)}, + &CodePointRange{From: rune(3663), To: rune(3663)}, + &CodePointRange{From: rune(3664), To: rune(3673)}, + &CodePointRange{From: rune(3674), To: rune(3675)}, + }, + "tibt": { + &CodePointRange{From: rune(3840), To: rune(3840)}, + &CodePointRange{From: rune(3841), To: rune(3843)}, + &CodePointRange{From: rune(3844), To: rune(3858)}, + &CodePointRange{From: rune(3859), To: rune(3859)}, + &CodePointRange{From: rune(3860), To: rune(3860)}, + &CodePointRange{From: rune(3861), To: rune(3863)}, + &CodePointRange{From: rune(3864), To: rune(3865)}, + &CodePointRange{From: rune(3866), To: rune(3871)}, + &CodePointRange{From: rune(3872), To: rune(3881)}, + &CodePointRange{From: rune(3882), To: rune(3891)}, + &CodePointRange{From: rune(3892), To: rune(3892)}, + &CodePointRange{From: rune(3893), To: rune(3893)}, + &CodePointRange{From: rune(3894), To: rune(3894)}, + &CodePointRange{From: rune(3895), To: rune(3895)}, + &CodePointRange{From: rune(3896), To: rune(3896)}, + &CodePointRange{From: rune(3897), To: rune(3897)}, + &CodePointRange{From: rune(3898), To: rune(3898)}, + &CodePointRange{From: rune(3899), To: rune(3899)}, + &CodePointRange{From: rune(3900), To: rune(3900)}, + &CodePointRange{From: rune(3901), To: rune(3901)}, + &CodePointRange{From: rune(3902), To: rune(3903)}, + &CodePointRange{From: rune(3904), To: rune(3911)}, + &CodePointRange{From: rune(3913), To: rune(3948)}, + &CodePointRange{From: rune(3953), To: rune(3966)}, + &CodePointRange{From: rune(3967), To: rune(3967)}, + &CodePointRange{From: rune(3968), To: rune(3972)}, + &CodePointRange{From: rune(3973), To: rune(3973)}, + &CodePointRange{From: rune(3974), To: rune(3975)}, + &CodePointRange{From: rune(3976), To: rune(3980)}, + &CodePointRange{From: rune(3981), To: rune(3991)}, + &CodePointRange{From: rune(3993), To: rune(4028)}, + &CodePointRange{From: rune(4030), To: rune(4037)}, + &CodePointRange{From: rune(4038), To: rune(4038)}, + &CodePointRange{From: rune(4039), To: rune(4044)}, + &CodePointRange{From: rune(4046), To: rune(4047)}, + &CodePointRange{From: rune(4048), To: rune(4052)}, + &CodePointRange{From: rune(4057), To: rune(4058)}, + }, + "tirh": { + &CodePointRange{From: rune(70784), To: rune(70831)}, + &CodePointRange{From: rune(70832), To: rune(70834)}, + &CodePointRange{From: rune(70835), To: rune(70840)}, + &CodePointRange{From: rune(70841), To: rune(70841)}, + &CodePointRange{From: rune(70842), To: rune(70842)}, + &CodePointRange{From: rune(70843), To: rune(70846)}, + &CodePointRange{From: rune(70847), To: rune(70848)}, + &CodePointRange{From: rune(70849), To: rune(70849)}, + &CodePointRange{From: rune(70850), To: rune(70851)}, + &CodePointRange{From: rune(70852), To: rune(70853)}, + &CodePointRange{From: rune(70854), To: rune(70854)}, + &CodePointRange{From: rune(70855), To: rune(70855)}, + &CodePointRange{From: rune(70864), To: rune(70873)}, + }, + "ugar": { + &CodePointRange{From: rune(66432), To: rune(66461)}, + &CodePointRange{From: rune(66463), To: rune(66463)}, + }, + "vaii": { + &CodePointRange{From: rune(42240), To: rune(42507)}, + &CodePointRange{From: rune(42508), To: rune(42508)}, + &CodePointRange{From: rune(42509), To: rune(42511)}, + &CodePointRange{From: rune(42512), To: rune(42527)}, + &CodePointRange{From: rune(42528), To: rune(42537)}, + &CodePointRange{From: rune(42538), To: rune(42539)}, + }, + "wara": { + &CodePointRange{From: rune(71840), To: rune(71903)}, + &CodePointRange{From: rune(71904), To: rune(71913)}, + &CodePointRange{From: rune(71914), To: rune(71922)}, + &CodePointRange{From: rune(71935), To: rune(71935)}, + }, + "wcho": { + &CodePointRange{From: rune(123584), To: rune(123627)}, + &CodePointRange{From: rune(123628), To: rune(123631)}, + &CodePointRange{From: rune(123632), To: rune(123641)}, + &CodePointRange{From: rune(123647), To: rune(123647)}, + }, + "xpeo": { + &CodePointRange{From: rune(66464), To: rune(66499)}, + &CodePointRange{From: rune(66504), To: rune(66511)}, + &CodePointRange{From: rune(66512), To: rune(66512)}, + &CodePointRange{From: rune(66513), To: rune(66517)}, + }, + "xsux": { + &CodePointRange{From: rune(73728), To: rune(74649)}, + &CodePointRange{From: rune(74752), To: rune(74862)}, + &CodePointRange{From: rune(74864), To: rune(74868)}, + &CodePointRange{From: rune(74880), To: rune(75075)}, + }, + "yezi": { + &CodePointRange{From: rune(69248), To: rune(69289)}, + &CodePointRange{From: rune(69291), To: rune(69292)}, + &CodePointRange{From: rune(69293), To: rune(69293)}, + &CodePointRange{From: rune(69296), To: rune(69297)}, + }, + "yiii": { + &CodePointRange{From: rune(40960), To: rune(40980)}, + &CodePointRange{From: rune(40981), To: rune(40981)}, + &CodePointRange{From: rune(40982), To: rune(42124)}, + &CodePointRange{From: rune(42128), To: rune(42182)}, + }, + "zanb": { + &CodePointRange{From: rune(72192), To: rune(72192)}, + &CodePointRange{From: rune(72193), To: rune(72202)}, + &CodePointRange{From: rune(72203), To: rune(72242)}, + &CodePointRange{From: rune(72243), To: rune(72248)}, + &CodePointRange{From: rune(72249), To: rune(72249)}, + &CodePointRange{From: rune(72250), To: rune(72250)}, + &CodePointRange{From: rune(72251), To: rune(72254)}, + &CodePointRange{From: rune(72255), To: rune(72262)}, + &CodePointRange{From: rune(72263), To: rune(72263)}, + }, + "zinh": { + &CodePointRange{From: rune(768), To: rune(879)}, + &CodePointRange{From: rune(1157), To: rune(1158)}, + &CodePointRange{From: rune(1611), To: rune(1621)}, + &CodePointRange{From: rune(1648), To: rune(1648)}, + &CodePointRange{From: rune(2385), To: rune(2388)}, + &CodePointRange{From: rune(6832), To: rune(6845)}, + &CodePointRange{From: rune(6846), To: rune(6846)}, + &CodePointRange{From: rune(6847), To: rune(6848)}, + &CodePointRange{From: rune(7376), To: rune(7378)}, + &CodePointRange{From: rune(7380), To: rune(7392)}, + &CodePointRange{From: rune(7394), To: rune(7400)}, + &CodePointRange{From: rune(7405), To: rune(7405)}, + &CodePointRange{From: rune(7412), To: rune(7412)}, + &CodePointRange{From: rune(7416), To: rune(7417)}, + &CodePointRange{From: rune(7616), To: rune(7673)}, + &CodePointRange{From: rune(7675), To: rune(7679)}, + &CodePointRange{From: rune(8204), To: rune(8205)}, + &CodePointRange{From: rune(8400), To: rune(8412)}, + &CodePointRange{From: rune(8413), To: rune(8416)}, + &CodePointRange{From: rune(8417), To: rune(8417)}, + &CodePointRange{From: rune(8418), To: rune(8420)}, + &CodePointRange{From: rune(8421), To: rune(8432)}, + &CodePointRange{From: rune(12330), To: rune(12333)}, + &CodePointRange{From: rune(12441), To: rune(12442)}, + &CodePointRange{From: rune(65024), To: rune(65039)}, + &CodePointRange{From: rune(65056), To: rune(65069)}, + &CodePointRange{From: rune(66045), To: rune(66045)}, + &CodePointRange{From: rune(66272), To: rune(66272)}, + &CodePointRange{From: rune(70459), To: rune(70459)}, + &CodePointRange{From: rune(119143), To: rune(119145)}, + &CodePointRange{From: rune(119163), To: rune(119170)}, + &CodePointRange{From: rune(119173), To: rune(119179)}, + &CodePointRange{From: rune(119210), To: rune(119213)}, + &CodePointRange{From: rune(917760), To: rune(917999)}, + }, + "zyyy": { + &CodePointRange{From: rune(0), To: rune(31)}, + &CodePointRange{From: rune(32), To: rune(32)}, + &CodePointRange{From: rune(33), To: rune(35)}, + &CodePointRange{From: rune(36), To: rune(36)}, + &CodePointRange{From: rune(37), To: rune(39)}, + &CodePointRange{From: rune(40), To: rune(40)}, + &CodePointRange{From: rune(41), To: rune(41)}, + &CodePointRange{From: rune(42), To: rune(42)}, + &CodePointRange{From: rune(43), To: rune(43)}, + &CodePointRange{From: rune(44), To: rune(44)}, + &CodePointRange{From: rune(45), To: rune(45)}, + &CodePointRange{From: rune(46), To: rune(47)}, + &CodePointRange{From: rune(48), To: rune(57)}, + &CodePointRange{From: rune(58), To: rune(59)}, + &CodePointRange{From: rune(60), To: rune(62)}, + &CodePointRange{From: rune(63), To: rune(64)}, + &CodePointRange{From: rune(91), To: rune(91)}, + &CodePointRange{From: rune(92), To: rune(92)}, + &CodePointRange{From: rune(93), To: rune(93)}, + &CodePointRange{From: rune(94), To: rune(94)}, + &CodePointRange{From: rune(95), To: rune(95)}, + &CodePointRange{From: rune(96), To: rune(96)}, + &CodePointRange{From: rune(123), To: rune(123)}, + &CodePointRange{From: rune(124), To: rune(124)}, + &CodePointRange{From: rune(125), To: rune(125)}, + &CodePointRange{From: rune(126), To: rune(126)}, + &CodePointRange{From: rune(127), To: rune(159)}, + &CodePointRange{From: rune(160), To: rune(160)}, + &CodePointRange{From: rune(161), To: rune(161)}, + &CodePointRange{From: rune(162), To: rune(165)}, + &CodePointRange{From: rune(166), To: rune(166)}, + &CodePointRange{From: rune(167), To: rune(167)}, + &CodePointRange{From: rune(168), To: rune(168)}, + &CodePointRange{From: rune(169), To: rune(169)}, + &CodePointRange{From: rune(171), To: rune(171)}, + &CodePointRange{From: rune(172), To: rune(172)}, + &CodePointRange{From: rune(173), To: rune(173)}, + &CodePointRange{From: rune(174), To: rune(174)}, + &CodePointRange{From: rune(175), To: rune(175)}, + &CodePointRange{From: rune(176), To: rune(176)}, + &CodePointRange{From: rune(177), To: rune(177)}, + &CodePointRange{From: rune(178), To: rune(179)}, + &CodePointRange{From: rune(180), To: rune(180)}, + &CodePointRange{From: rune(181), To: rune(181)}, + &CodePointRange{From: rune(182), To: rune(183)}, + &CodePointRange{From: rune(184), To: rune(184)}, + &CodePointRange{From: rune(185), To: rune(185)}, + &CodePointRange{From: rune(187), To: rune(187)}, + &CodePointRange{From: rune(188), To: rune(190)}, + &CodePointRange{From: rune(191), To: rune(191)}, + &CodePointRange{From: rune(215), To: rune(215)}, + &CodePointRange{From: rune(247), To: rune(247)}, + &CodePointRange{From: rune(697), To: rune(705)}, + &CodePointRange{From: rune(706), To: rune(709)}, + &CodePointRange{From: rune(710), To: rune(721)}, + &CodePointRange{From: rune(722), To: rune(735)}, + &CodePointRange{From: rune(741), To: rune(745)}, + &CodePointRange{From: rune(748), To: rune(748)}, + &CodePointRange{From: rune(749), To: rune(749)}, + &CodePointRange{From: rune(750), To: rune(750)}, + &CodePointRange{From: rune(751), To: rune(767)}, + &CodePointRange{From: rune(884), To: rune(884)}, + &CodePointRange{From: rune(894), To: rune(894)}, + &CodePointRange{From: rune(901), To: rune(901)}, + &CodePointRange{From: rune(903), To: rune(903)}, + &CodePointRange{From: rune(1541), To: rune(1541)}, + &CodePointRange{From: rune(1548), To: rune(1548)}, + &CodePointRange{From: rune(1563), To: rune(1563)}, + &CodePointRange{From: rune(1567), To: rune(1567)}, + &CodePointRange{From: rune(1600), To: rune(1600)}, + &CodePointRange{From: rune(1757), To: rune(1757)}, + &CodePointRange{From: rune(2274), To: rune(2274)}, + &CodePointRange{From: rune(2404), To: rune(2405)}, + &CodePointRange{From: rune(3647), To: rune(3647)}, + &CodePointRange{From: rune(4053), To: rune(4056)}, + &CodePointRange{From: rune(4347), To: rune(4347)}, + &CodePointRange{From: rune(5867), To: rune(5869)}, + &CodePointRange{From: rune(5941), To: rune(5942)}, + &CodePointRange{From: rune(6146), To: rune(6147)}, + &CodePointRange{From: rune(6149), To: rune(6149)}, + &CodePointRange{From: rune(7379), To: rune(7379)}, + &CodePointRange{From: rune(7393), To: rune(7393)}, + &CodePointRange{From: rune(7401), To: rune(7404)}, + &CodePointRange{From: rune(7406), To: rune(7411)}, + &CodePointRange{From: rune(7413), To: rune(7414)}, + &CodePointRange{From: rune(7415), To: rune(7415)}, + &CodePointRange{From: rune(7418), To: rune(7418)}, + &CodePointRange{From: rune(8192), To: rune(8202)}, + &CodePointRange{From: rune(8203), To: rune(8203)}, + &CodePointRange{From: rune(8206), To: rune(8207)}, + &CodePointRange{From: rune(8208), To: rune(8213)}, + &CodePointRange{From: rune(8214), To: rune(8215)}, + &CodePointRange{From: rune(8216), To: rune(8216)}, + &CodePointRange{From: rune(8217), To: rune(8217)}, + &CodePointRange{From: rune(8218), To: rune(8218)}, + &CodePointRange{From: rune(8219), To: rune(8220)}, + &CodePointRange{From: rune(8221), To: rune(8221)}, + &CodePointRange{From: rune(8222), To: rune(8222)}, + &CodePointRange{From: rune(8223), To: rune(8223)}, + &CodePointRange{From: rune(8224), To: rune(8231)}, + &CodePointRange{From: rune(8232), To: rune(8232)}, + &CodePointRange{From: rune(8233), To: rune(8233)}, + &CodePointRange{From: rune(8234), To: rune(8238)}, + &CodePointRange{From: rune(8239), To: rune(8239)}, + &CodePointRange{From: rune(8240), To: rune(8248)}, + &CodePointRange{From: rune(8249), To: rune(8249)}, + &CodePointRange{From: rune(8250), To: rune(8250)}, + &CodePointRange{From: rune(8251), To: rune(8254)}, + &CodePointRange{From: rune(8255), To: rune(8256)}, + &CodePointRange{From: rune(8257), To: rune(8259)}, + &CodePointRange{From: rune(8260), To: rune(8260)}, + &CodePointRange{From: rune(8261), To: rune(8261)}, + &CodePointRange{From: rune(8262), To: rune(8262)}, + &CodePointRange{From: rune(8263), To: rune(8273)}, + &CodePointRange{From: rune(8274), To: rune(8274)}, + &CodePointRange{From: rune(8275), To: rune(8275)}, + &CodePointRange{From: rune(8276), To: rune(8276)}, + &CodePointRange{From: rune(8277), To: rune(8286)}, + &CodePointRange{From: rune(8287), To: rune(8287)}, + &CodePointRange{From: rune(8288), To: rune(8292)}, + &CodePointRange{From: rune(8294), To: rune(8303)}, + &CodePointRange{From: rune(8304), To: rune(8304)}, + &CodePointRange{From: rune(8308), To: rune(8313)}, + &CodePointRange{From: rune(8314), To: rune(8316)}, + &CodePointRange{From: rune(8317), To: rune(8317)}, + &CodePointRange{From: rune(8318), To: rune(8318)}, + &CodePointRange{From: rune(8320), To: rune(8329)}, + &CodePointRange{From: rune(8330), To: rune(8332)}, + &CodePointRange{From: rune(8333), To: rune(8333)}, + &CodePointRange{From: rune(8334), To: rune(8334)}, + &CodePointRange{From: rune(8352), To: rune(8383)}, + &CodePointRange{From: rune(8448), To: rune(8449)}, + &CodePointRange{From: rune(8450), To: rune(8450)}, + &CodePointRange{From: rune(8451), To: rune(8454)}, + &CodePointRange{From: rune(8455), To: rune(8455)}, + &CodePointRange{From: rune(8456), To: rune(8457)}, + &CodePointRange{From: rune(8458), To: rune(8467)}, + &CodePointRange{From: rune(8468), To: rune(8468)}, + &CodePointRange{From: rune(8469), To: rune(8469)}, + &CodePointRange{From: rune(8470), To: rune(8471)}, + &CodePointRange{From: rune(8472), To: rune(8472)}, + &CodePointRange{From: rune(8473), To: rune(8477)}, + &CodePointRange{From: rune(8478), To: rune(8483)}, + &CodePointRange{From: rune(8484), To: rune(8484)}, + &CodePointRange{From: rune(8485), To: rune(8485)}, + &CodePointRange{From: rune(8487), To: rune(8487)}, + &CodePointRange{From: rune(8488), To: rune(8488)}, + &CodePointRange{From: rune(8489), To: rune(8489)}, + &CodePointRange{From: rune(8492), To: rune(8493)}, + &CodePointRange{From: rune(8494), To: rune(8494)}, + &CodePointRange{From: rune(8495), To: rune(8497)}, + &CodePointRange{From: rune(8499), To: rune(8500)}, + &CodePointRange{From: rune(8501), To: rune(8504)}, + &CodePointRange{From: rune(8505), To: rune(8505)}, + &CodePointRange{From: rune(8506), To: rune(8507)}, + &CodePointRange{From: rune(8508), To: rune(8511)}, + &CodePointRange{From: rune(8512), To: rune(8516)}, + &CodePointRange{From: rune(8517), To: rune(8521)}, + &CodePointRange{From: rune(8522), To: rune(8522)}, + &CodePointRange{From: rune(8523), To: rune(8523)}, + &CodePointRange{From: rune(8524), To: rune(8525)}, + &CodePointRange{From: rune(8527), To: rune(8527)}, + &CodePointRange{From: rune(8528), To: rune(8543)}, + &CodePointRange{From: rune(8585), To: rune(8585)}, + &CodePointRange{From: rune(8586), To: rune(8587)}, + &CodePointRange{From: rune(8592), To: rune(8596)}, + &CodePointRange{From: rune(8597), To: rune(8601)}, + &CodePointRange{From: rune(8602), To: rune(8603)}, + &CodePointRange{From: rune(8604), To: rune(8607)}, + &CodePointRange{From: rune(8608), To: rune(8608)}, + &CodePointRange{From: rune(8609), To: rune(8610)}, + &CodePointRange{From: rune(8611), To: rune(8611)}, + &CodePointRange{From: rune(8612), To: rune(8613)}, + &CodePointRange{From: rune(8614), To: rune(8614)}, + &CodePointRange{From: rune(8615), To: rune(8621)}, + &CodePointRange{From: rune(8622), To: rune(8622)}, + &CodePointRange{From: rune(8623), To: rune(8653)}, + &CodePointRange{From: rune(8654), To: rune(8655)}, + &CodePointRange{From: rune(8656), To: rune(8657)}, + &CodePointRange{From: rune(8658), To: rune(8658)}, + &CodePointRange{From: rune(8659), To: rune(8659)}, + &CodePointRange{From: rune(8660), To: rune(8660)}, + &CodePointRange{From: rune(8661), To: rune(8691)}, + &CodePointRange{From: rune(8692), To: rune(8959)}, + &CodePointRange{From: rune(8960), To: rune(8967)}, + &CodePointRange{From: rune(8968), To: rune(8968)}, + &CodePointRange{From: rune(8969), To: rune(8969)}, + &CodePointRange{From: rune(8970), To: rune(8970)}, + &CodePointRange{From: rune(8971), To: rune(8971)}, + &CodePointRange{From: rune(8972), To: rune(8991)}, + &CodePointRange{From: rune(8992), To: rune(8993)}, + &CodePointRange{From: rune(8994), To: rune(9000)}, + &CodePointRange{From: rune(9001), To: rune(9001)}, + &CodePointRange{From: rune(9002), To: rune(9002)}, + &CodePointRange{From: rune(9003), To: rune(9083)}, + &CodePointRange{From: rune(9084), To: rune(9084)}, + &CodePointRange{From: rune(9085), To: rune(9114)}, + &CodePointRange{From: rune(9115), To: rune(9139)}, + &CodePointRange{From: rune(9140), To: rune(9179)}, + &CodePointRange{From: rune(9180), To: rune(9185)}, + &CodePointRange{From: rune(9186), To: rune(9254)}, + &CodePointRange{From: rune(9280), To: rune(9290)}, + &CodePointRange{From: rune(9312), To: rune(9371)}, + &CodePointRange{From: rune(9372), To: rune(9449)}, + &CodePointRange{From: rune(9450), To: rune(9471)}, + &CodePointRange{From: rune(9472), To: rune(9654)}, + &CodePointRange{From: rune(9655), To: rune(9655)}, + &CodePointRange{From: rune(9656), To: rune(9664)}, + &CodePointRange{From: rune(9665), To: rune(9665)}, + &CodePointRange{From: rune(9666), To: rune(9719)}, + &CodePointRange{From: rune(9720), To: rune(9727)}, + &CodePointRange{From: rune(9728), To: rune(9838)}, + &CodePointRange{From: rune(9839), To: rune(9839)}, + &CodePointRange{From: rune(9840), To: rune(10087)}, + &CodePointRange{From: rune(10088), To: rune(10088)}, + &CodePointRange{From: rune(10089), To: rune(10089)}, + &CodePointRange{From: rune(10090), To: rune(10090)}, + &CodePointRange{From: rune(10091), To: rune(10091)}, + &CodePointRange{From: rune(10092), To: rune(10092)}, + &CodePointRange{From: rune(10093), To: rune(10093)}, + &CodePointRange{From: rune(10094), To: rune(10094)}, + &CodePointRange{From: rune(10095), To: rune(10095)}, + &CodePointRange{From: rune(10096), To: rune(10096)}, + &CodePointRange{From: rune(10097), To: rune(10097)}, + &CodePointRange{From: rune(10098), To: rune(10098)}, + &CodePointRange{From: rune(10099), To: rune(10099)}, + &CodePointRange{From: rune(10100), To: rune(10100)}, + &CodePointRange{From: rune(10101), To: rune(10101)}, + &CodePointRange{From: rune(10102), To: rune(10131)}, + &CodePointRange{From: rune(10132), To: rune(10175)}, + &CodePointRange{From: rune(10176), To: rune(10180)}, + &CodePointRange{From: rune(10181), To: rune(10181)}, + &CodePointRange{From: rune(10182), To: rune(10182)}, + &CodePointRange{From: rune(10183), To: rune(10213)}, + &CodePointRange{From: rune(10214), To: rune(10214)}, + &CodePointRange{From: rune(10215), To: rune(10215)}, + &CodePointRange{From: rune(10216), To: rune(10216)}, + &CodePointRange{From: rune(10217), To: rune(10217)}, + &CodePointRange{From: rune(10218), To: rune(10218)}, + &CodePointRange{From: rune(10219), To: rune(10219)}, + &CodePointRange{From: rune(10220), To: rune(10220)}, + &CodePointRange{From: rune(10221), To: rune(10221)}, + &CodePointRange{From: rune(10222), To: rune(10222)}, + &CodePointRange{From: rune(10223), To: rune(10223)}, + &CodePointRange{From: rune(10224), To: rune(10239)}, + &CodePointRange{From: rune(10496), To: rune(10626)}, + &CodePointRange{From: rune(10627), To: rune(10627)}, + &CodePointRange{From: rune(10628), To: rune(10628)}, + &CodePointRange{From: rune(10629), To: rune(10629)}, + &CodePointRange{From: rune(10630), To: rune(10630)}, + &CodePointRange{From: rune(10631), To: rune(10631)}, + &CodePointRange{From: rune(10632), To: rune(10632)}, + &CodePointRange{From: rune(10633), To: rune(10633)}, + &CodePointRange{From: rune(10634), To: rune(10634)}, + &CodePointRange{From: rune(10635), To: rune(10635)}, + &CodePointRange{From: rune(10636), To: rune(10636)}, + &CodePointRange{From: rune(10637), To: rune(10637)}, + &CodePointRange{From: rune(10638), To: rune(10638)}, + &CodePointRange{From: rune(10639), To: rune(10639)}, + &CodePointRange{From: rune(10640), To: rune(10640)}, + &CodePointRange{From: rune(10641), To: rune(10641)}, + &CodePointRange{From: rune(10642), To: rune(10642)}, + &CodePointRange{From: rune(10643), To: rune(10643)}, + &CodePointRange{From: rune(10644), To: rune(10644)}, + &CodePointRange{From: rune(10645), To: rune(10645)}, + &CodePointRange{From: rune(10646), To: rune(10646)}, + &CodePointRange{From: rune(10647), To: rune(10647)}, + &CodePointRange{From: rune(10648), To: rune(10648)}, + &CodePointRange{From: rune(10649), To: rune(10711)}, + &CodePointRange{From: rune(10712), To: rune(10712)}, + &CodePointRange{From: rune(10713), To: rune(10713)}, + &CodePointRange{From: rune(10714), To: rune(10714)}, + &CodePointRange{From: rune(10715), To: rune(10715)}, + &CodePointRange{From: rune(10716), To: rune(10747)}, + &CodePointRange{From: rune(10748), To: rune(10748)}, + &CodePointRange{From: rune(10749), To: rune(10749)}, + &CodePointRange{From: rune(10750), To: rune(11007)}, + &CodePointRange{From: rune(11008), To: rune(11055)}, + &CodePointRange{From: rune(11056), To: rune(11076)}, + &CodePointRange{From: rune(11077), To: rune(11078)}, + &CodePointRange{From: rune(11079), To: rune(11084)}, + &CodePointRange{From: rune(11085), To: rune(11123)}, + &CodePointRange{From: rune(11126), To: rune(11157)}, + &CodePointRange{From: rune(11159), To: rune(11263)}, + &CodePointRange{From: rune(11776), To: rune(11777)}, + &CodePointRange{From: rune(11778), To: rune(11778)}, + &CodePointRange{From: rune(11779), To: rune(11779)}, + &CodePointRange{From: rune(11780), To: rune(11780)}, + &CodePointRange{From: rune(11781), To: rune(11781)}, + &CodePointRange{From: rune(11782), To: rune(11784)}, + &CodePointRange{From: rune(11785), To: rune(11785)}, + &CodePointRange{From: rune(11786), To: rune(11786)}, + &CodePointRange{From: rune(11787), To: rune(11787)}, + &CodePointRange{From: rune(11788), To: rune(11788)}, + &CodePointRange{From: rune(11789), To: rune(11789)}, + &CodePointRange{From: rune(11790), To: rune(11798)}, + &CodePointRange{From: rune(11799), To: rune(11799)}, + &CodePointRange{From: rune(11800), To: rune(11801)}, + &CodePointRange{From: rune(11802), To: rune(11802)}, + &CodePointRange{From: rune(11803), To: rune(11803)}, + &CodePointRange{From: rune(11804), To: rune(11804)}, + &CodePointRange{From: rune(11805), To: rune(11805)}, + &CodePointRange{From: rune(11806), To: rune(11807)}, + &CodePointRange{From: rune(11808), To: rune(11808)}, + &CodePointRange{From: rune(11809), To: rune(11809)}, + &CodePointRange{From: rune(11810), To: rune(11810)}, + &CodePointRange{From: rune(11811), To: rune(11811)}, + &CodePointRange{From: rune(11812), To: rune(11812)}, + &CodePointRange{From: rune(11813), To: rune(11813)}, + &CodePointRange{From: rune(11814), To: rune(11814)}, + &CodePointRange{From: rune(11815), To: rune(11815)}, + &CodePointRange{From: rune(11816), To: rune(11816)}, + &CodePointRange{From: rune(11817), To: rune(11817)}, + &CodePointRange{From: rune(11818), To: rune(11822)}, + &CodePointRange{From: rune(11823), To: rune(11823)}, + &CodePointRange{From: rune(11824), To: rune(11833)}, + &CodePointRange{From: rune(11834), To: rune(11835)}, + &CodePointRange{From: rune(11836), To: rune(11839)}, + &CodePointRange{From: rune(11840), To: rune(11840)}, + &CodePointRange{From: rune(11841), To: rune(11841)}, + &CodePointRange{From: rune(11842), To: rune(11842)}, + &CodePointRange{From: rune(11843), To: rune(11855)}, + &CodePointRange{From: rune(11856), To: rune(11857)}, + &CodePointRange{From: rune(11858), To: rune(11858)}, + &CodePointRange{From: rune(12272), To: rune(12283)}, + &CodePointRange{From: rune(12288), To: rune(12288)}, + &CodePointRange{From: rune(12289), To: rune(12291)}, + &CodePointRange{From: rune(12292), To: rune(12292)}, + &CodePointRange{From: rune(12294), To: rune(12294)}, + &CodePointRange{From: rune(12296), To: rune(12296)}, + &CodePointRange{From: rune(12297), To: rune(12297)}, + &CodePointRange{From: rune(12298), To: rune(12298)}, + &CodePointRange{From: rune(12299), To: rune(12299)}, + &CodePointRange{From: rune(12300), To: rune(12300)}, + &CodePointRange{From: rune(12301), To: rune(12301)}, + &CodePointRange{From: rune(12302), To: rune(12302)}, + &CodePointRange{From: rune(12303), To: rune(12303)}, + &CodePointRange{From: rune(12304), To: rune(12304)}, + &CodePointRange{From: rune(12305), To: rune(12305)}, + &CodePointRange{From: rune(12306), To: rune(12307)}, + &CodePointRange{From: rune(12308), To: rune(12308)}, + &CodePointRange{From: rune(12309), To: rune(12309)}, + &CodePointRange{From: rune(12310), To: rune(12310)}, + &CodePointRange{From: rune(12311), To: rune(12311)}, + &CodePointRange{From: rune(12312), To: rune(12312)}, + &CodePointRange{From: rune(12313), To: rune(12313)}, + &CodePointRange{From: rune(12314), To: rune(12314)}, + &CodePointRange{From: rune(12315), To: rune(12315)}, + &CodePointRange{From: rune(12316), To: rune(12316)}, + &CodePointRange{From: rune(12317), To: rune(12317)}, + &CodePointRange{From: rune(12318), To: rune(12319)}, + &CodePointRange{From: rune(12320), To: rune(12320)}, + &CodePointRange{From: rune(12336), To: rune(12336)}, + &CodePointRange{From: rune(12337), To: rune(12341)}, + &CodePointRange{From: rune(12342), To: rune(12343)}, + &CodePointRange{From: rune(12348), To: rune(12348)}, + &CodePointRange{From: rune(12349), To: rune(12349)}, + &CodePointRange{From: rune(12350), To: rune(12351)}, + &CodePointRange{From: rune(12443), To: rune(12444)}, + &CodePointRange{From: rune(12448), To: rune(12448)}, + &CodePointRange{From: rune(12539), To: rune(12539)}, + &CodePointRange{From: rune(12540), To: rune(12540)}, + &CodePointRange{From: rune(12688), To: rune(12689)}, + &CodePointRange{From: rune(12690), To: rune(12693)}, + &CodePointRange{From: rune(12694), To: rune(12703)}, + &CodePointRange{From: rune(12736), To: rune(12771)}, + &CodePointRange{From: rune(12832), To: rune(12841)}, + &CodePointRange{From: rune(12842), To: rune(12871)}, + &CodePointRange{From: rune(12872), To: rune(12879)}, + &CodePointRange{From: rune(12880), To: rune(12880)}, + &CodePointRange{From: rune(12881), To: rune(12895)}, + &CodePointRange{From: rune(12927), To: rune(12927)}, + &CodePointRange{From: rune(12928), To: rune(12937)}, + &CodePointRange{From: rune(12938), To: rune(12976)}, + &CodePointRange{From: rune(12977), To: rune(12991)}, + &CodePointRange{From: rune(12992), To: rune(13007)}, + &CodePointRange{From: rune(13055), To: rune(13055)}, + &CodePointRange{From: rune(13144), To: rune(13311)}, + &CodePointRange{From: rune(19904), To: rune(19967)}, + &CodePointRange{From: rune(42752), To: rune(42774)}, + &CodePointRange{From: rune(42775), To: rune(42783)}, + &CodePointRange{From: rune(42784), To: rune(42785)}, + &CodePointRange{From: rune(42888), To: rune(42888)}, + &CodePointRange{From: rune(42889), To: rune(42890)}, + &CodePointRange{From: rune(43056), To: rune(43061)}, + &CodePointRange{From: rune(43062), To: rune(43063)}, + &CodePointRange{From: rune(43064), To: rune(43064)}, + &CodePointRange{From: rune(43065), To: rune(43065)}, + &CodePointRange{From: rune(43310), To: rune(43310)}, + &CodePointRange{From: rune(43471), To: rune(43471)}, + &CodePointRange{From: rune(43867), To: rune(43867)}, + &CodePointRange{From: rune(43882), To: rune(43883)}, + &CodePointRange{From: rune(64830), To: rune(64830)}, + &CodePointRange{From: rune(64831), To: rune(64831)}, + &CodePointRange{From: rune(65040), To: rune(65046)}, + &CodePointRange{From: rune(65047), To: rune(65047)}, + &CodePointRange{From: rune(65048), To: rune(65048)}, + &CodePointRange{From: rune(65049), To: rune(65049)}, + &CodePointRange{From: rune(65072), To: rune(65072)}, + &CodePointRange{From: rune(65073), To: rune(65074)}, + &CodePointRange{From: rune(65075), To: rune(65076)}, + &CodePointRange{From: rune(65077), To: rune(65077)}, + &CodePointRange{From: rune(65078), To: rune(65078)}, + &CodePointRange{From: rune(65079), To: rune(65079)}, + &CodePointRange{From: rune(65080), To: rune(65080)}, + &CodePointRange{From: rune(65081), To: rune(65081)}, + &CodePointRange{From: rune(65082), To: rune(65082)}, + &CodePointRange{From: rune(65083), To: rune(65083)}, + &CodePointRange{From: rune(65084), To: rune(65084)}, + &CodePointRange{From: rune(65085), To: rune(65085)}, + &CodePointRange{From: rune(65086), To: rune(65086)}, + &CodePointRange{From: rune(65087), To: rune(65087)}, + &CodePointRange{From: rune(65088), To: rune(65088)}, + &CodePointRange{From: rune(65089), To: rune(65089)}, + &CodePointRange{From: rune(65090), To: rune(65090)}, + &CodePointRange{From: rune(65091), To: rune(65091)}, + &CodePointRange{From: rune(65092), To: rune(65092)}, + &CodePointRange{From: rune(65093), To: rune(65094)}, + &CodePointRange{From: rune(65095), To: rune(65095)}, + &CodePointRange{From: rune(65096), To: rune(65096)}, + &CodePointRange{From: rune(65097), To: rune(65100)}, + &CodePointRange{From: rune(65101), To: rune(65103)}, + &CodePointRange{From: rune(65104), To: rune(65106)}, + &CodePointRange{From: rune(65108), To: rune(65111)}, + &CodePointRange{From: rune(65112), To: rune(65112)}, + &CodePointRange{From: rune(65113), To: rune(65113)}, + &CodePointRange{From: rune(65114), To: rune(65114)}, + &CodePointRange{From: rune(65115), To: rune(65115)}, + &CodePointRange{From: rune(65116), To: rune(65116)}, + &CodePointRange{From: rune(65117), To: rune(65117)}, + &CodePointRange{From: rune(65118), To: rune(65118)}, + &CodePointRange{From: rune(65119), To: rune(65121)}, + &CodePointRange{From: rune(65122), To: rune(65122)}, + &CodePointRange{From: rune(65123), To: rune(65123)}, + &CodePointRange{From: rune(65124), To: rune(65126)}, + &CodePointRange{From: rune(65128), To: rune(65128)}, + &CodePointRange{From: rune(65129), To: rune(65129)}, + &CodePointRange{From: rune(65130), To: rune(65131)}, + &CodePointRange{From: rune(65279), To: rune(65279)}, + &CodePointRange{From: rune(65281), To: rune(65283)}, + &CodePointRange{From: rune(65284), To: rune(65284)}, + &CodePointRange{From: rune(65285), To: rune(65287)}, + &CodePointRange{From: rune(65288), To: rune(65288)}, + &CodePointRange{From: rune(65289), To: rune(65289)}, + &CodePointRange{From: rune(65290), To: rune(65290)}, + &CodePointRange{From: rune(65291), To: rune(65291)}, + &CodePointRange{From: rune(65292), To: rune(65292)}, + &CodePointRange{From: rune(65293), To: rune(65293)}, + &CodePointRange{From: rune(65294), To: rune(65295)}, + &CodePointRange{From: rune(65296), To: rune(65305)}, + &CodePointRange{From: rune(65306), To: rune(65307)}, + &CodePointRange{From: rune(65308), To: rune(65310)}, + &CodePointRange{From: rune(65311), To: rune(65312)}, + &CodePointRange{From: rune(65339), To: rune(65339)}, + &CodePointRange{From: rune(65340), To: rune(65340)}, + &CodePointRange{From: rune(65341), To: rune(65341)}, + &CodePointRange{From: rune(65342), To: rune(65342)}, + &CodePointRange{From: rune(65343), To: rune(65343)}, + &CodePointRange{From: rune(65344), To: rune(65344)}, + &CodePointRange{From: rune(65371), To: rune(65371)}, + &CodePointRange{From: rune(65372), To: rune(65372)}, + &CodePointRange{From: rune(65373), To: rune(65373)}, + &CodePointRange{From: rune(65374), To: rune(65374)}, + &CodePointRange{From: rune(65375), To: rune(65375)}, + &CodePointRange{From: rune(65376), To: rune(65376)}, + &CodePointRange{From: rune(65377), To: rune(65377)}, + &CodePointRange{From: rune(65378), To: rune(65378)}, + &CodePointRange{From: rune(65379), To: rune(65379)}, + &CodePointRange{From: rune(65380), To: rune(65381)}, + &CodePointRange{From: rune(65392), To: rune(65392)}, + &CodePointRange{From: rune(65438), To: rune(65439)}, + &CodePointRange{From: rune(65504), To: rune(65505)}, + &CodePointRange{From: rune(65506), To: rune(65506)}, + &CodePointRange{From: rune(65507), To: rune(65507)}, + &CodePointRange{From: rune(65508), To: rune(65508)}, + &CodePointRange{From: rune(65509), To: rune(65510)}, + &CodePointRange{From: rune(65512), To: rune(65512)}, + &CodePointRange{From: rune(65513), To: rune(65516)}, + &CodePointRange{From: rune(65517), To: rune(65518)}, + &CodePointRange{From: rune(65529), To: rune(65531)}, + &CodePointRange{From: rune(65532), To: rune(65533)}, + &CodePointRange{From: rune(65792), To: rune(65794)}, + &CodePointRange{From: rune(65799), To: rune(65843)}, + &CodePointRange{From: rune(65847), To: rune(65855)}, + &CodePointRange{From: rune(65936), To: rune(65948)}, + &CodePointRange{From: rune(66000), To: rune(66044)}, + &CodePointRange{From: rune(66273), To: rune(66299)}, + &CodePointRange{From: rune(94178), To: rune(94178)}, + &CodePointRange{From: rune(94179), To: rune(94179)}, + &CodePointRange{From: rune(113824), To: rune(113827)}, + &CodePointRange{From: rune(118784), To: rune(119029)}, + &CodePointRange{From: rune(119040), To: rune(119078)}, + &CodePointRange{From: rune(119081), To: rune(119140)}, + &CodePointRange{From: rune(119141), To: rune(119142)}, + &CodePointRange{From: rune(119146), To: rune(119148)}, + &CodePointRange{From: rune(119149), To: rune(119154)}, + &CodePointRange{From: rune(119155), To: rune(119162)}, + &CodePointRange{From: rune(119171), To: rune(119172)}, + &CodePointRange{From: rune(119180), To: rune(119209)}, + &CodePointRange{From: rune(119214), To: rune(119272)}, + &CodePointRange{From: rune(119520), To: rune(119539)}, + &CodePointRange{From: rune(119552), To: rune(119638)}, + &CodePointRange{From: rune(119648), To: rune(119672)}, + &CodePointRange{From: rune(119808), To: rune(119892)}, + &CodePointRange{From: rune(119894), To: rune(119964)}, + &CodePointRange{From: rune(119966), To: rune(119967)}, + &CodePointRange{From: rune(119970), To: rune(119970)}, + &CodePointRange{From: rune(119973), To: rune(119974)}, + &CodePointRange{From: rune(119977), To: rune(119980)}, + &CodePointRange{From: rune(119982), To: rune(119993)}, + &CodePointRange{From: rune(119995), To: rune(119995)}, + &CodePointRange{From: rune(119997), To: rune(120003)}, + &CodePointRange{From: rune(120005), To: rune(120069)}, + &CodePointRange{From: rune(120071), To: rune(120074)}, + &CodePointRange{From: rune(120077), To: rune(120084)}, + &CodePointRange{From: rune(120086), To: rune(120092)}, + &CodePointRange{From: rune(120094), To: rune(120121)}, + &CodePointRange{From: rune(120123), To: rune(120126)}, + &CodePointRange{From: rune(120128), To: rune(120132)}, + &CodePointRange{From: rune(120134), To: rune(120134)}, + &CodePointRange{From: rune(120138), To: rune(120144)}, + &CodePointRange{From: rune(120146), To: rune(120485)}, + &CodePointRange{From: rune(120488), To: rune(120512)}, + &CodePointRange{From: rune(120513), To: rune(120513)}, + &CodePointRange{From: rune(120514), To: rune(120538)}, + &CodePointRange{From: rune(120539), To: rune(120539)}, + &CodePointRange{From: rune(120540), To: rune(120570)}, + &CodePointRange{From: rune(120571), To: rune(120571)}, + &CodePointRange{From: rune(120572), To: rune(120596)}, + &CodePointRange{From: rune(120597), To: rune(120597)}, + &CodePointRange{From: rune(120598), To: rune(120628)}, + &CodePointRange{From: rune(120629), To: rune(120629)}, + &CodePointRange{From: rune(120630), To: rune(120654)}, + &CodePointRange{From: rune(120655), To: rune(120655)}, + &CodePointRange{From: rune(120656), To: rune(120686)}, + &CodePointRange{From: rune(120687), To: rune(120687)}, + &CodePointRange{From: rune(120688), To: rune(120712)}, + &CodePointRange{From: rune(120713), To: rune(120713)}, + &CodePointRange{From: rune(120714), To: rune(120744)}, + &CodePointRange{From: rune(120745), To: rune(120745)}, + &CodePointRange{From: rune(120746), To: rune(120770)}, + &CodePointRange{From: rune(120771), To: rune(120771)}, + &CodePointRange{From: rune(120772), To: rune(120779)}, + &CodePointRange{From: rune(120782), To: rune(120831)}, + &CodePointRange{From: rune(126065), To: rune(126123)}, + &CodePointRange{From: rune(126124), To: rune(126124)}, + &CodePointRange{From: rune(126125), To: rune(126127)}, + &CodePointRange{From: rune(126128), To: rune(126128)}, + &CodePointRange{From: rune(126129), To: rune(126132)}, + &CodePointRange{From: rune(126209), To: rune(126253)}, + &CodePointRange{From: rune(126254), To: rune(126254)}, + &CodePointRange{From: rune(126255), To: rune(126269)}, + &CodePointRange{From: rune(126976), To: rune(127019)}, + &CodePointRange{From: rune(127024), To: rune(127123)}, + &CodePointRange{From: rune(127136), To: rune(127150)}, + &CodePointRange{From: rune(127153), To: rune(127167)}, + &CodePointRange{From: rune(127169), To: rune(127183)}, + &CodePointRange{From: rune(127185), To: rune(127221)}, + &CodePointRange{From: rune(127232), To: rune(127244)}, + &CodePointRange{From: rune(127245), To: rune(127405)}, + &CodePointRange{From: rune(127462), To: rune(127487)}, + &CodePointRange{From: rune(127489), To: rune(127490)}, + &CodePointRange{From: rune(127504), To: rune(127547)}, + &CodePointRange{From: rune(127552), To: rune(127560)}, + &CodePointRange{From: rune(127568), To: rune(127569)}, + &CodePointRange{From: rune(127584), To: rune(127589)}, + &CodePointRange{From: rune(127744), To: rune(127994)}, + &CodePointRange{From: rune(127995), To: rune(127999)}, + &CodePointRange{From: rune(128000), To: rune(128727)}, + &CodePointRange{From: rune(128736), To: rune(128748)}, + &CodePointRange{From: rune(128752), To: rune(128764)}, + &CodePointRange{From: rune(128768), To: rune(128883)}, + &CodePointRange{From: rune(128896), To: rune(128984)}, + &CodePointRange{From: rune(128992), To: rune(129003)}, + &CodePointRange{From: rune(129024), To: rune(129035)}, + &CodePointRange{From: rune(129040), To: rune(129095)}, + &CodePointRange{From: rune(129104), To: rune(129113)}, + &CodePointRange{From: rune(129120), To: rune(129159)}, + &CodePointRange{From: rune(129168), To: rune(129197)}, + &CodePointRange{From: rune(129200), To: rune(129201)}, + &CodePointRange{From: rune(129280), To: rune(129400)}, + &CodePointRange{From: rune(129402), To: rune(129483)}, + &CodePointRange{From: rune(129485), To: rune(129619)}, + &CodePointRange{From: rune(129632), To: rune(129645)}, + &CodePointRange{From: rune(129648), To: rune(129652)}, + &CodePointRange{From: rune(129656), To: rune(129658)}, + &CodePointRange{From: rune(129664), To: rune(129670)}, + &CodePointRange{From: rune(129680), To: rune(129704)}, + &CodePointRange{From: rune(129712), To: rune(129718)}, + &CodePointRange{From: rune(129728), To: rune(129730)}, + &CodePointRange{From: rune(129744), To: rune(129750)}, + &CodePointRange{From: rune(129792), To: rune(129938)}, + &CodePointRange{From: rune(129940), To: rune(129994)}, + &CodePointRange{From: rune(130032), To: rune(130041)}, + &CodePointRange{From: rune(917505), To: rune(917505)}, + &CodePointRange{From: rune(917536), To: rune(917631)}, + }, +} + +// https://www.unicode.org/Public/13.0.0/ucd/PropList.txt +var otherAlphabeticCodePoints = []*CodePointRange{ + &CodePointRange{From: rune(837), To: rune(837)}, + &CodePointRange{From: rune(1456), To: rune(1469)}, + &CodePointRange{From: rune(1471), To: rune(1471)}, + &CodePointRange{From: rune(1473), To: rune(1474)}, + &CodePointRange{From: rune(1476), To: rune(1477)}, + &CodePointRange{From: rune(1479), To: rune(1479)}, + &CodePointRange{From: rune(1552), To: rune(1562)}, + &CodePointRange{From: rune(1611), To: rune(1623)}, + &CodePointRange{From: rune(1625), To: rune(1631)}, + &CodePointRange{From: rune(1648), To: rune(1648)}, + &CodePointRange{From: rune(1750), To: rune(1756)}, + &CodePointRange{From: rune(1761), To: rune(1764)}, + &CodePointRange{From: rune(1767), To: rune(1768)}, + &CodePointRange{From: rune(1773), To: rune(1773)}, + &CodePointRange{From: rune(1809), To: rune(1809)}, + &CodePointRange{From: rune(1840), To: rune(1855)}, + &CodePointRange{From: rune(1958), To: rune(1968)}, + &CodePointRange{From: rune(2070), To: rune(2071)}, + &CodePointRange{From: rune(2075), To: rune(2083)}, + &CodePointRange{From: rune(2085), To: rune(2087)}, + &CodePointRange{From: rune(2089), To: rune(2092)}, + &CodePointRange{From: rune(2260), To: rune(2271)}, + &CodePointRange{From: rune(2275), To: rune(2281)}, + &CodePointRange{From: rune(2288), To: rune(2306)}, + &CodePointRange{From: rune(2307), To: rune(2307)}, + &CodePointRange{From: rune(2362), To: rune(2362)}, + &CodePointRange{From: rune(2363), To: rune(2363)}, + &CodePointRange{From: rune(2366), To: rune(2368)}, + &CodePointRange{From: rune(2369), To: rune(2376)}, + &CodePointRange{From: rune(2377), To: rune(2380)}, + &CodePointRange{From: rune(2382), To: rune(2383)}, + &CodePointRange{From: rune(2389), To: rune(2391)}, + &CodePointRange{From: rune(2402), To: rune(2403)}, + &CodePointRange{From: rune(2433), To: rune(2433)}, + &CodePointRange{From: rune(2434), To: rune(2435)}, + &CodePointRange{From: rune(2494), To: rune(2496)}, + &CodePointRange{From: rune(2497), To: rune(2500)}, + &CodePointRange{From: rune(2503), To: rune(2504)}, + &CodePointRange{From: rune(2507), To: rune(2508)}, + &CodePointRange{From: rune(2519), To: rune(2519)}, + &CodePointRange{From: rune(2530), To: rune(2531)}, + &CodePointRange{From: rune(2561), To: rune(2562)}, + &CodePointRange{From: rune(2563), To: rune(2563)}, + &CodePointRange{From: rune(2622), To: rune(2624)}, + &CodePointRange{From: rune(2625), To: rune(2626)}, + &CodePointRange{From: rune(2631), To: rune(2632)}, + &CodePointRange{From: rune(2635), To: rune(2636)}, + &CodePointRange{From: rune(2641), To: rune(2641)}, + &CodePointRange{From: rune(2672), To: rune(2673)}, + &CodePointRange{From: rune(2677), To: rune(2677)}, + &CodePointRange{From: rune(2689), To: rune(2690)}, + &CodePointRange{From: rune(2691), To: rune(2691)}, + &CodePointRange{From: rune(2750), To: rune(2752)}, + &CodePointRange{From: rune(2753), To: rune(2757)}, + &CodePointRange{From: rune(2759), To: rune(2760)}, + &CodePointRange{From: rune(2761), To: rune(2761)}, + &CodePointRange{From: rune(2763), To: rune(2764)}, + &CodePointRange{From: rune(2786), To: rune(2787)}, + &CodePointRange{From: rune(2810), To: rune(2812)}, + &CodePointRange{From: rune(2817), To: rune(2817)}, + &CodePointRange{From: rune(2818), To: rune(2819)}, + &CodePointRange{From: rune(2878), To: rune(2878)}, + &CodePointRange{From: rune(2879), To: rune(2879)}, + &CodePointRange{From: rune(2880), To: rune(2880)}, + &CodePointRange{From: rune(2881), To: rune(2884)}, + &CodePointRange{From: rune(2887), To: rune(2888)}, + &CodePointRange{From: rune(2891), To: rune(2892)}, + &CodePointRange{From: rune(2902), To: rune(2902)}, + &CodePointRange{From: rune(2903), To: rune(2903)}, + &CodePointRange{From: rune(2914), To: rune(2915)}, + &CodePointRange{From: rune(2946), To: rune(2946)}, + &CodePointRange{From: rune(3006), To: rune(3007)}, + &CodePointRange{From: rune(3008), To: rune(3008)}, + &CodePointRange{From: rune(3009), To: rune(3010)}, + &CodePointRange{From: rune(3014), To: rune(3016)}, + &CodePointRange{From: rune(3018), To: rune(3020)}, + &CodePointRange{From: rune(3031), To: rune(3031)}, + &CodePointRange{From: rune(3072), To: rune(3072)}, + &CodePointRange{From: rune(3073), To: rune(3075)}, + &CodePointRange{From: rune(3134), To: rune(3136)}, + &CodePointRange{From: rune(3137), To: rune(3140)}, + &CodePointRange{From: rune(3142), To: rune(3144)}, + &CodePointRange{From: rune(3146), To: rune(3148)}, + &CodePointRange{From: rune(3157), To: rune(3158)}, + &CodePointRange{From: rune(3170), To: rune(3171)}, + &CodePointRange{From: rune(3201), To: rune(3201)}, + &CodePointRange{From: rune(3202), To: rune(3203)}, + &CodePointRange{From: rune(3262), To: rune(3262)}, + &CodePointRange{From: rune(3263), To: rune(3263)}, + &CodePointRange{From: rune(3264), To: rune(3268)}, + &CodePointRange{From: rune(3270), To: rune(3270)}, + &CodePointRange{From: rune(3271), To: rune(3272)}, + &CodePointRange{From: rune(3274), To: rune(3275)}, + &CodePointRange{From: rune(3276), To: rune(3276)}, + &CodePointRange{From: rune(3285), To: rune(3286)}, + &CodePointRange{From: rune(3298), To: rune(3299)}, + &CodePointRange{From: rune(3328), To: rune(3329)}, + &CodePointRange{From: rune(3330), To: rune(3331)}, + &CodePointRange{From: rune(3390), To: rune(3392)}, + &CodePointRange{From: rune(3393), To: rune(3396)}, + &CodePointRange{From: rune(3398), To: rune(3400)}, + &CodePointRange{From: rune(3402), To: rune(3404)}, + &CodePointRange{From: rune(3415), To: rune(3415)}, + &CodePointRange{From: rune(3426), To: rune(3427)}, + &CodePointRange{From: rune(3457), To: rune(3457)}, + &CodePointRange{From: rune(3458), To: rune(3459)}, + &CodePointRange{From: rune(3535), To: rune(3537)}, + &CodePointRange{From: rune(3538), To: rune(3540)}, + &CodePointRange{From: rune(3542), To: rune(3542)}, + &CodePointRange{From: rune(3544), To: rune(3551)}, + &CodePointRange{From: rune(3570), To: rune(3571)}, + &CodePointRange{From: rune(3633), To: rune(3633)}, + &CodePointRange{From: rune(3636), To: rune(3642)}, + &CodePointRange{From: rune(3661), To: rune(3661)}, + &CodePointRange{From: rune(3761), To: rune(3761)}, + &CodePointRange{From: rune(3764), To: rune(3769)}, + &CodePointRange{From: rune(3771), To: rune(3772)}, + &CodePointRange{From: rune(3789), To: rune(3789)}, + &CodePointRange{From: rune(3953), To: rune(3966)}, + &CodePointRange{From: rune(3967), To: rune(3967)}, + &CodePointRange{From: rune(3968), To: rune(3969)}, + &CodePointRange{From: rune(3981), To: rune(3991)}, + &CodePointRange{From: rune(3993), To: rune(4028)}, + &CodePointRange{From: rune(4139), To: rune(4140)}, + &CodePointRange{From: rune(4141), To: rune(4144)}, + &CodePointRange{From: rune(4145), To: rune(4145)}, + &CodePointRange{From: rune(4146), To: rune(4150)}, + &CodePointRange{From: rune(4152), To: rune(4152)}, + &CodePointRange{From: rune(4155), To: rune(4156)}, + &CodePointRange{From: rune(4157), To: rune(4158)}, + &CodePointRange{From: rune(4182), To: rune(4183)}, + &CodePointRange{From: rune(4184), To: rune(4185)}, + &CodePointRange{From: rune(4190), To: rune(4192)}, + &CodePointRange{From: rune(4194), To: rune(4196)}, + &CodePointRange{From: rune(4199), To: rune(4205)}, + &CodePointRange{From: rune(4209), To: rune(4212)}, + &CodePointRange{From: rune(4226), To: rune(4226)}, + &CodePointRange{From: rune(4227), To: rune(4228)}, + &CodePointRange{From: rune(4229), To: rune(4230)}, + &CodePointRange{From: rune(4231), To: rune(4236)}, + &CodePointRange{From: rune(4237), To: rune(4237)}, + &CodePointRange{From: rune(4239), To: rune(4239)}, + &CodePointRange{From: rune(4250), To: rune(4252)}, + &CodePointRange{From: rune(4253), To: rune(4253)}, + &CodePointRange{From: rune(5906), To: rune(5907)}, + &CodePointRange{From: rune(5938), To: rune(5939)}, + &CodePointRange{From: rune(5970), To: rune(5971)}, + &CodePointRange{From: rune(6002), To: rune(6003)}, + &CodePointRange{From: rune(6070), To: rune(6070)}, + &CodePointRange{From: rune(6071), To: rune(6077)}, + &CodePointRange{From: rune(6078), To: rune(6085)}, + &CodePointRange{From: rune(6086), To: rune(6086)}, + &CodePointRange{From: rune(6087), To: rune(6088)}, + &CodePointRange{From: rune(6277), To: rune(6278)}, + &CodePointRange{From: rune(6313), To: rune(6313)}, + &CodePointRange{From: rune(6432), To: rune(6434)}, + &CodePointRange{From: rune(6435), To: rune(6438)}, + &CodePointRange{From: rune(6439), To: rune(6440)}, + &CodePointRange{From: rune(6441), To: rune(6443)}, + &CodePointRange{From: rune(6448), To: rune(6449)}, + &CodePointRange{From: rune(6450), To: rune(6450)}, + &CodePointRange{From: rune(6451), To: rune(6456)}, + &CodePointRange{From: rune(6679), To: rune(6680)}, + &CodePointRange{From: rune(6681), To: rune(6682)}, + &CodePointRange{From: rune(6683), To: rune(6683)}, + &CodePointRange{From: rune(6741), To: rune(6741)}, + &CodePointRange{From: rune(6742), To: rune(6742)}, + &CodePointRange{From: rune(6743), To: rune(6743)}, + &CodePointRange{From: rune(6744), To: rune(6750)}, + &CodePointRange{From: rune(6753), To: rune(6753)}, + &CodePointRange{From: rune(6754), To: rune(6754)}, + &CodePointRange{From: rune(6755), To: rune(6756)}, + &CodePointRange{From: rune(6757), To: rune(6764)}, + &CodePointRange{From: rune(6765), To: rune(6770)}, + &CodePointRange{From: rune(6771), To: rune(6772)}, + &CodePointRange{From: rune(6847), To: rune(6848)}, + &CodePointRange{From: rune(6912), To: rune(6915)}, + &CodePointRange{From: rune(6916), To: rune(6916)}, + &CodePointRange{From: rune(6965), To: rune(6965)}, + &CodePointRange{From: rune(6966), To: rune(6970)}, + &CodePointRange{From: rune(6971), To: rune(6971)}, + &CodePointRange{From: rune(6972), To: rune(6972)}, + &CodePointRange{From: rune(6973), To: rune(6977)}, + &CodePointRange{From: rune(6978), To: rune(6978)}, + &CodePointRange{From: rune(6979), To: rune(6979)}, + &CodePointRange{From: rune(7040), To: rune(7041)}, + &CodePointRange{From: rune(7042), To: rune(7042)}, + &CodePointRange{From: rune(7073), To: rune(7073)}, + &CodePointRange{From: rune(7074), To: rune(7077)}, + &CodePointRange{From: rune(7078), To: rune(7079)}, + &CodePointRange{From: rune(7080), To: rune(7081)}, + &CodePointRange{From: rune(7084), To: rune(7085)}, + &CodePointRange{From: rune(7143), To: rune(7143)}, + &CodePointRange{From: rune(7144), To: rune(7145)}, + &CodePointRange{From: rune(7146), To: rune(7148)}, + &CodePointRange{From: rune(7149), To: rune(7149)}, + &CodePointRange{From: rune(7150), To: rune(7150)}, + &CodePointRange{From: rune(7151), To: rune(7153)}, + &CodePointRange{From: rune(7204), To: rune(7211)}, + &CodePointRange{From: rune(7212), To: rune(7219)}, + &CodePointRange{From: rune(7220), To: rune(7221)}, + &CodePointRange{From: rune(7222), To: rune(7222)}, + &CodePointRange{From: rune(7655), To: rune(7668)}, + &CodePointRange{From: rune(9398), To: rune(9449)}, + &CodePointRange{From: rune(11744), To: rune(11775)}, + &CodePointRange{From: rune(42612), To: rune(42619)}, + &CodePointRange{From: rune(42654), To: rune(42655)}, + &CodePointRange{From: rune(43010), To: rune(43010)}, + &CodePointRange{From: rune(43019), To: rune(43019)}, + &CodePointRange{From: rune(43043), To: rune(43044)}, + &CodePointRange{From: rune(43045), To: rune(43046)}, + &CodePointRange{From: rune(43047), To: rune(43047)}, + &CodePointRange{From: rune(43136), To: rune(43137)}, + &CodePointRange{From: rune(43188), To: rune(43203)}, + &CodePointRange{From: rune(43205), To: rune(43205)}, + &CodePointRange{From: rune(43263), To: rune(43263)}, + &CodePointRange{From: rune(43302), To: rune(43306)}, + &CodePointRange{From: rune(43335), To: rune(43345)}, + &CodePointRange{From: rune(43346), To: rune(43346)}, + &CodePointRange{From: rune(43392), To: rune(43394)}, + &CodePointRange{From: rune(43395), To: rune(43395)}, + &CodePointRange{From: rune(43444), To: rune(43445)}, + &CodePointRange{From: rune(43446), To: rune(43449)}, + &CodePointRange{From: rune(43450), To: rune(43451)}, + &CodePointRange{From: rune(43452), To: rune(43453)}, + &CodePointRange{From: rune(43454), To: rune(43455)}, + &CodePointRange{From: rune(43493), To: rune(43493)}, + &CodePointRange{From: rune(43561), To: rune(43566)}, + &CodePointRange{From: rune(43567), To: rune(43568)}, + &CodePointRange{From: rune(43569), To: rune(43570)}, + &CodePointRange{From: rune(43571), To: rune(43572)}, + &CodePointRange{From: rune(43573), To: rune(43574)}, + &CodePointRange{From: rune(43587), To: rune(43587)}, + &CodePointRange{From: rune(43596), To: rune(43596)}, + &CodePointRange{From: rune(43597), To: rune(43597)}, + &CodePointRange{From: rune(43643), To: rune(43643)}, + &CodePointRange{From: rune(43644), To: rune(43644)}, + &CodePointRange{From: rune(43645), To: rune(43645)}, + &CodePointRange{From: rune(43696), To: rune(43696)}, + &CodePointRange{From: rune(43698), To: rune(43700)}, + &CodePointRange{From: rune(43703), To: rune(43704)}, + &CodePointRange{From: rune(43710), To: rune(43710)}, + &CodePointRange{From: rune(43755), To: rune(43755)}, + &CodePointRange{From: rune(43756), To: rune(43757)}, + &CodePointRange{From: rune(43758), To: rune(43759)}, + &CodePointRange{From: rune(43765), To: rune(43765)}, + &CodePointRange{From: rune(44003), To: rune(44004)}, + &CodePointRange{From: rune(44005), To: rune(44005)}, + &CodePointRange{From: rune(44006), To: rune(44007)}, + &CodePointRange{From: rune(44008), To: rune(44008)}, + &CodePointRange{From: rune(44009), To: rune(44010)}, + &CodePointRange{From: rune(64286), To: rune(64286)}, + &CodePointRange{From: rune(66422), To: rune(66426)}, + &CodePointRange{From: rune(68097), To: rune(68099)}, + &CodePointRange{From: rune(68101), To: rune(68102)}, + &CodePointRange{From: rune(68108), To: rune(68111)}, + &CodePointRange{From: rune(68900), To: rune(68903)}, + &CodePointRange{From: rune(69291), To: rune(69292)}, + &CodePointRange{From: rune(69632), To: rune(69632)}, + &CodePointRange{From: rune(69633), To: rune(69633)}, + &CodePointRange{From: rune(69634), To: rune(69634)}, + &CodePointRange{From: rune(69688), To: rune(69701)}, + &CodePointRange{From: rune(69762), To: rune(69762)}, + &CodePointRange{From: rune(69808), To: rune(69810)}, + &CodePointRange{From: rune(69811), To: rune(69814)}, + &CodePointRange{From: rune(69815), To: rune(69816)}, + &CodePointRange{From: rune(69888), To: rune(69890)}, + &CodePointRange{From: rune(69927), To: rune(69931)}, + &CodePointRange{From: rune(69932), To: rune(69932)}, + &CodePointRange{From: rune(69933), To: rune(69938)}, + &CodePointRange{From: rune(69957), To: rune(69958)}, + &CodePointRange{From: rune(70016), To: rune(70017)}, + &CodePointRange{From: rune(70018), To: rune(70018)}, + &CodePointRange{From: rune(70067), To: rune(70069)}, + &CodePointRange{From: rune(70070), To: rune(70078)}, + &CodePointRange{From: rune(70079), To: rune(70079)}, + &CodePointRange{From: rune(70094), To: rune(70094)}, + &CodePointRange{From: rune(70095), To: rune(70095)}, + &CodePointRange{From: rune(70188), To: rune(70190)}, + &CodePointRange{From: rune(70191), To: rune(70193)}, + &CodePointRange{From: rune(70194), To: rune(70195)}, + &CodePointRange{From: rune(70196), To: rune(70196)}, + &CodePointRange{From: rune(70199), To: rune(70199)}, + &CodePointRange{From: rune(70206), To: rune(70206)}, + &CodePointRange{From: rune(70367), To: rune(70367)}, + &CodePointRange{From: rune(70368), To: rune(70370)}, + &CodePointRange{From: rune(70371), To: rune(70376)}, + &CodePointRange{From: rune(70400), To: rune(70401)}, + &CodePointRange{From: rune(70402), To: rune(70403)}, + &CodePointRange{From: rune(70462), To: rune(70463)}, + &CodePointRange{From: rune(70464), To: rune(70464)}, + &CodePointRange{From: rune(70465), To: rune(70468)}, + &CodePointRange{From: rune(70471), To: rune(70472)}, + &CodePointRange{From: rune(70475), To: rune(70476)}, + &CodePointRange{From: rune(70487), To: rune(70487)}, + &CodePointRange{From: rune(70498), To: rune(70499)}, + &CodePointRange{From: rune(70709), To: rune(70711)}, + &CodePointRange{From: rune(70712), To: rune(70719)}, + &CodePointRange{From: rune(70720), To: rune(70721)}, + &CodePointRange{From: rune(70723), To: rune(70724)}, + &CodePointRange{From: rune(70725), To: rune(70725)}, + &CodePointRange{From: rune(70832), To: rune(70834)}, + &CodePointRange{From: rune(70835), To: rune(70840)}, + &CodePointRange{From: rune(70841), To: rune(70841)}, + &CodePointRange{From: rune(70842), To: rune(70842)}, + &CodePointRange{From: rune(70843), To: rune(70846)}, + &CodePointRange{From: rune(70847), To: rune(70848)}, + &CodePointRange{From: rune(70849), To: rune(70849)}, + &CodePointRange{From: rune(71087), To: rune(71089)}, + &CodePointRange{From: rune(71090), To: rune(71093)}, + &CodePointRange{From: rune(71096), To: rune(71099)}, + &CodePointRange{From: rune(71100), To: rune(71101)}, + &CodePointRange{From: rune(71102), To: rune(71102)}, + &CodePointRange{From: rune(71132), To: rune(71133)}, + &CodePointRange{From: rune(71216), To: rune(71218)}, + &CodePointRange{From: rune(71219), To: rune(71226)}, + &CodePointRange{From: rune(71227), To: rune(71228)}, + &CodePointRange{From: rune(71229), To: rune(71229)}, + &CodePointRange{From: rune(71230), To: rune(71230)}, + &CodePointRange{From: rune(71232), To: rune(71232)}, + &CodePointRange{From: rune(71339), To: rune(71339)}, + &CodePointRange{From: rune(71340), To: rune(71340)}, + &CodePointRange{From: rune(71341), To: rune(71341)}, + &CodePointRange{From: rune(71342), To: rune(71343)}, + &CodePointRange{From: rune(71344), To: rune(71349)}, + &CodePointRange{From: rune(71453), To: rune(71455)}, + &CodePointRange{From: rune(71456), To: rune(71457)}, + &CodePointRange{From: rune(71458), To: rune(71461)}, + &CodePointRange{From: rune(71462), To: rune(71462)}, + &CodePointRange{From: rune(71463), To: rune(71466)}, + &CodePointRange{From: rune(71724), To: rune(71726)}, + &CodePointRange{From: rune(71727), To: rune(71735)}, + &CodePointRange{From: rune(71736), To: rune(71736)}, + &CodePointRange{From: rune(71984), To: rune(71989)}, + &CodePointRange{From: rune(71991), To: rune(71992)}, + &CodePointRange{From: rune(71995), To: rune(71996)}, + &CodePointRange{From: rune(72000), To: rune(72000)}, + &CodePointRange{From: rune(72002), To: rune(72002)}, + &CodePointRange{From: rune(72145), To: rune(72147)}, + &CodePointRange{From: rune(72148), To: rune(72151)}, + &CodePointRange{From: rune(72154), To: rune(72155)}, + &CodePointRange{From: rune(72156), To: rune(72159)}, + &CodePointRange{From: rune(72164), To: rune(72164)}, + &CodePointRange{From: rune(72193), To: rune(72202)}, + &CodePointRange{From: rune(72245), To: rune(72248)}, + &CodePointRange{From: rune(72249), To: rune(72249)}, + &CodePointRange{From: rune(72251), To: rune(72254)}, + &CodePointRange{From: rune(72273), To: rune(72278)}, + &CodePointRange{From: rune(72279), To: rune(72280)}, + &CodePointRange{From: rune(72281), To: rune(72283)}, + &CodePointRange{From: rune(72330), To: rune(72342)}, + &CodePointRange{From: rune(72343), To: rune(72343)}, + &CodePointRange{From: rune(72751), To: rune(72751)}, + &CodePointRange{From: rune(72752), To: rune(72758)}, + &CodePointRange{From: rune(72760), To: rune(72765)}, + &CodePointRange{From: rune(72766), To: rune(72766)}, + &CodePointRange{From: rune(72850), To: rune(72871)}, + &CodePointRange{From: rune(72873), To: rune(72873)}, + &CodePointRange{From: rune(72874), To: rune(72880)}, + &CodePointRange{From: rune(72881), To: rune(72881)}, + &CodePointRange{From: rune(72882), To: rune(72883)}, + &CodePointRange{From: rune(72884), To: rune(72884)}, + &CodePointRange{From: rune(72885), To: rune(72886)}, + &CodePointRange{From: rune(73009), To: rune(73014)}, + &CodePointRange{From: rune(73018), To: rune(73018)}, + &CodePointRange{From: rune(73020), To: rune(73021)}, + &CodePointRange{From: rune(73023), To: rune(73025)}, + &CodePointRange{From: rune(73027), To: rune(73027)}, + &CodePointRange{From: rune(73031), To: rune(73031)}, + &CodePointRange{From: rune(73098), To: rune(73102)}, + &CodePointRange{From: rune(73104), To: rune(73105)}, + &CodePointRange{From: rune(73107), To: rune(73108)}, + &CodePointRange{From: rune(73109), To: rune(73109)}, + &CodePointRange{From: rune(73110), To: rune(73110)}, + &CodePointRange{From: rune(73459), To: rune(73460)}, + &CodePointRange{From: rune(73461), To: rune(73462)}, + &CodePointRange{From: rune(94031), To: rune(94031)}, + &CodePointRange{From: rune(94033), To: rune(94087)}, + &CodePointRange{From: rune(94095), To: rune(94098)}, + &CodePointRange{From: rune(94192), To: rune(94193)}, + &CodePointRange{From: rune(113822), To: rune(113822)}, + &CodePointRange{From: rune(122880), To: rune(122886)}, + &CodePointRange{From: rune(122888), To: rune(122904)}, + &CodePointRange{From: rune(122907), To: rune(122913)}, + &CodePointRange{From: rune(122915), To: rune(122916)}, + &CodePointRange{From: rune(122918), To: rune(122922)}, + &CodePointRange{From: rune(125255), To: rune(125255)}, + &CodePointRange{From: rune(127280), To: rune(127305)}, + &CodePointRange{From: rune(127312), To: rune(127337)}, + &CodePointRange{From: rune(127344), To: rune(127369)}, +} + +// https://www.unicode.org/Public/13.0.0/ucd/PropList.txt +var otherLowercaseCodePoints = []*CodePointRange{ + &CodePointRange{From: rune(170), To: rune(170)}, + &CodePointRange{From: rune(186), To: rune(186)}, + &CodePointRange{From: rune(688), To: rune(696)}, + &CodePointRange{From: rune(704), To: rune(705)}, + &CodePointRange{From: rune(736), To: rune(740)}, + &CodePointRange{From: rune(837), To: rune(837)}, + &CodePointRange{From: rune(890), To: rune(890)}, + &CodePointRange{From: rune(7468), To: rune(7530)}, + &CodePointRange{From: rune(7544), To: rune(7544)}, + &CodePointRange{From: rune(7579), To: rune(7615)}, + &CodePointRange{From: rune(8305), To: rune(8305)}, + &CodePointRange{From: rune(8319), To: rune(8319)}, + &CodePointRange{From: rune(8336), To: rune(8348)}, + &CodePointRange{From: rune(8560), To: rune(8575)}, + &CodePointRange{From: rune(9424), To: rune(9449)}, + &CodePointRange{From: rune(11388), To: rune(11389)}, + &CodePointRange{From: rune(42652), To: rune(42653)}, + &CodePointRange{From: rune(42864), To: rune(42864)}, + &CodePointRange{From: rune(43000), To: rune(43001)}, + &CodePointRange{From: rune(43868), To: rune(43871)}, +} + +// https://www.unicode.org/Public/13.0.0/ucd/PropList.txt +var otherUppercaseCodePoints = []*CodePointRange{ + &CodePointRange{From: rune(8544), To: rune(8559)}, + &CodePointRange{From: rune(9398), To: rune(9423)}, + &CodePointRange{From: rune(127280), To: rune(127305)}, + &CodePointRange{From: rune(127312), To: rune(127337)}, + &CodePointRange{From: rune(127344), To: rune(127369)}, +} + +// https://www.unicode.org/Public/13.0.0/ucd/PropList.txt +var whiteSpaceCodePoints = []*CodePointRange{ + &CodePointRange{From: rune(9), To: rune(13)}, + &CodePointRange{From: rune(32), To: rune(32)}, + &CodePointRange{From: rune(133), To: rune(133)}, + &CodePointRange{From: rune(160), To: rune(160)}, + &CodePointRange{From: rune(5760), To: rune(5760)}, + &CodePointRange{From: rune(8192), To: rune(8202)}, + &CodePointRange{From: rune(8232), To: rune(8232)}, + &CodePointRange{From: rune(8233), To: rune(8233)}, + &CodePointRange{From: rune(8239), To: rune(8239)}, + &CodePointRange{From: rune(8287), To: rune(8287)}, + &CodePointRange{From: rune(12288), To: rune(12288)}, +} diff --git a/ucd/codepoint.go.tmpl b/ucd/codepoint.go.tmpl new file mode 100644 index 0000000..cc0d48e --- /dev/null +++ b/ucd/codepoint.go.tmpl @@ -0,0 +1,65 @@ +// Code generated by {{ .GeneratorName }}; DO NOT EDIT. + +package ucd + +// https://www.unicode.org/Public/13.0.0/ucd/PropertyValueAliases.txt +var generalCategoryValueAbbs = map[string]string{ {{ range $long, $abb := .PropertyValueAliases.GeneralCategory }} + "{{ $long }}": "{{ $abb }}",{{ end }} +} + +// https://www.unicode.org/Public/13.0.0/ucd/PropertyValueAliases.txt +var scriptValueAbbs = map[string]string{ {{ range $long, $abb := .PropertyValueAliases.Script }} + "{{ $long }}": "{{ $abb }}",{{ end }} +} + +// https://www.unicode.org/Public/13.0.0/ucd/PropertyValueAliases.txt +var ( + generalCategoryDefaultRange = &CodePointRange{ + From: rune({{ .PropertyValueAliases.GeneralCategoryDefaultRange.From }}), + To: rune({{ .PropertyValueAliases.GeneralCategoryDefaultRange.To }}), + } + generalCategoryDefaultValue = "{{ .PropertyValueAliases.GeneralCategoryDefaultValue }}" +) + +// https://www.unicode.org/Public/13.0.0/ucd/UnicodeData.txt +var generalCategoryCodePoints = map[string][]*CodePointRange{ {{ range $propName, $codePoints := .UnicodeData.GeneralCategory }} + "{{ $propName }}": { {{ range $codePoints }} + &CodePointRange{From: rune({{ .From }}), To: rune({{ .To }})},{{ end }} + },{{ end }} +} + +// https://www.unicode.org/Public/13.0.0/ucd/Scripts.txt +var ( + scriptDefaultRange = &CodePointRange{ + From: rune({{ .Scripts.ScriptDefaultRange.From }}), + To: rune({{ .Scripts.ScriptDefaultRange.To }}), + } + scriptDefaultValue = "{{ .Scripts.ScriptDefaultValue }}" +) + +// https://www.unicode.org/Public/13.0.0/ucd/Scripts.txt +var scriptCodepoints = map[string][]*CodePointRange{ {{ range $script, $codePoints := .Scripts.Script }} + "{{ $script }}": { {{ range $codePoints }} + &CodePointRange{From: rune({{ .From }}), To: rune({{ .To }})},{{ end }} + },{{ end }} +} + +// https://www.unicode.org/Public/13.0.0/ucd/PropList.txt +var otherAlphabeticCodePoints = []*CodePointRange{ {{ range .PropList.OtherAlphabetic }} + &CodePointRange{From: rune({{ .From }}), To: rune({{ .To }})},{{ end }} +} + +// https://www.unicode.org/Public/13.0.0/ucd/PropList.txt +var otherLowercaseCodePoints = []*CodePointRange{ {{ range .PropList.OtherLowercase }} + &CodePointRange{From: rune({{ .From }}), To: rune({{ .To }})},{{ end }} +} + +// https://www.unicode.org/Public/13.0.0/ucd/PropList.txt +var otherUppercaseCodePoints = []*CodePointRange{ {{ range .PropList.OtherUppercase }} + &CodePointRange{From: rune({{ .From }}), To: rune({{ .To }})},{{ end }} +} + +// https://www.unicode.org/Public/13.0.0/ucd/PropList.txt +var whiteSpaceCodePoints = []*CodePointRange{ {{ range .PropList.WhiteSpace }} + &CodePointRange{From: rune({{ .From }}), To: rune({{ .To }})},{{ end }} +} diff --git a/ucd/parser.go b/ucd/parser.go new file mode 100644 index 0000000..88d7134 --- /dev/null +++ b/ucd/parser.go @@ -0,0 +1,155 @@ +package ucd + +import ( + "bufio" + "encoding/binary" + "encoding/hex" + "io" + "regexp" + "strings" +) + +type CodePointRange struct { + From rune + To rune +} + +var codePointRangeNil = &CodePointRange{ + From: 0, + To: 0, +} + +type field string + +func (f field) codePointRange() (*CodePointRange, error) { + var from, to rune + var err error + cp := reCodePointRange.FindStringSubmatch(string(f)) + from, err = decodeHexToRune(cp[1]) + if err != nil { + return codePointRangeNil, err + } + if cp[2] != "" { + to, err = decodeHexToRune(cp[2]) + if err != nil { + return codePointRangeNil, err + } + } else { + to = from + } + return &CodePointRange{ + From: from, + To: to, + }, nil +} + +func decodeHexToRune(hexCodePoint string) (rune, error) { + h := hexCodePoint + if len(h)%2 != 0 { + h = "0" + h + } + b, err := hex.DecodeString(h) + if err != nil { + return 0, err + } + l := len(b) + for i := 0; i < 4-l; i++ { + b = append([]byte{0}, b...) + } + n := binary.BigEndian.Uint32(b) + return rune(n), nil +} + +func (f field) symbol() string { + return string(f) +} + +func (f field) normalizedSymbol() string { + return normalizeSymbolicValue(string(f)) +} + +var symValReplacer = strings.NewReplacer("_", "", "-", "", "\x20", "") + +// normalizeSymbolicValue normalizes a symbolic value. The normalized value meets UAX44-LM3. +// +// https://www.unicode.org/reports/tr44/#UAX44-LM3 +func normalizeSymbolicValue(s string) string { + v := strings.ToLower(symValReplacer.Replace(s)) + if strings.HasPrefix(v, "is") && v != "is" { + return v[2:] + } + return v +} + +var ( + reLine = regexp.MustCompile(`^\s*(.*?)\s*(#.*)?$`) + reCodePointRange = regexp.MustCompile(`^([[:xdigit:]]+)(?:..([[:xdigit:]]+))?$`) + + specialCommentPrefix = "# @missing:" +) + +// This parser can parse data files of Unicode Character Database (UCD). +// Specifically, it has the following two functions: +// - Converts each line of the data files into a slice of fields. +// - Recognizes specially-formatted comments starting `@missing` and generates a slice of fields. +// +// However, for practical purposes, each field needs to be analyzed more specifically. +// For instance, in UnicodeData.txt, the first field represents a range of code points, +// so it needs to be recognized as a hexadecimal string. +// You can perform more specific parsing for each file by implementing a dedicated parser that wraps this parser. +// +// https://www.unicode.org/reports/tr44/#Format_Conventions +type parser struct { + scanner *bufio.Scanner + fields []field + defaultFields []field + err error + + fieldBuf []field + defaultFieldBuf []field +} + +func newParser(r io.Reader) *parser { + return &parser{ + scanner: bufio.NewScanner(r), + fieldBuf: make([]field, 50), + defaultFieldBuf: make([]field, 50), + } +} + +func (p *parser) parse() bool { + for p.scanner.Scan() { + p.parseRecord(p.scanner.Text()) + if p.fields != nil || p.defaultFields != nil { + return true + } + } + p.err = p.scanner.Err() + return false +} + +func (p *parser) parseRecord(src string) { + ms := reLine.FindStringSubmatch(src) + mFields := ms[1] + mComment := ms[2] + if mFields != "" { + p.fields = parseFields(p.fieldBuf, mFields) + } else { + p.fields = nil + } + if strings.HasPrefix(mComment, specialCommentPrefix) { + p.defaultFields = parseFields(p.defaultFieldBuf, strings.Replace(mComment, specialCommentPrefix, "", -1)) + } else { + p.defaultFields = nil + } +} + +func parseFields(buf []field, src string) []field { + n := 0 + for _, f := range strings.Split(src, ";") { + buf[n] = field(strings.TrimSpace(f)) + n++ + } + + return buf[:n] +} diff --git a/ucd/prop_list.go b/ucd/prop_list.go new file mode 100644 index 0000000..31db70c --- /dev/null +++ b/ucd/prop_list.go @@ -0,0 +1,50 @@ +package ucd + +import "io" + +type PropList struct { + OtherAlphabetic []*CodePointRange + OtherLowercase []*CodePointRange + OtherUppercase []*CodePointRange + WhiteSpace []*CodePointRange +} + +// ParsePropList parses the PropList.txt. +func ParsePropList(r io.Reader) (*PropList, error) { + var oa []*CodePointRange + var ol []*CodePointRange + var ou []*CodePointRange + var ws []*CodePointRange + p := newParser(r) + for p.parse() { + if len(p.fields) == 0 { + continue + } + + cp, err := p.fields[0].codePointRange() + if err != nil { + return nil, err + } + + switch p.fields[1].symbol() { + case "Other_Alphabetic": + oa = append(oa, cp) + case "Other_Lowercase": + ol = append(ol, cp) + case "Other_Uppercase": + ou = append(ou, cp) + case "White_Space": + ws = append(ws, cp) + } + } + if p.err != nil { + return nil, p.err + } + + return &PropList{ + OtherAlphabetic: oa, + OtherLowercase: ol, + OtherUppercase: ou, + WhiteSpace: ws, + }, nil +} diff --git a/ucd/property.go b/ucd/property.go new file mode 100644 index 0000000..ba60e80 --- /dev/null +++ b/ucd/property.go @@ -0,0 +1,95 @@ +package ucd + +// contributoryProperties is a set of contributory properties vartan uses internally. +// Property statuses are defined in the following table. +// +// https://unicode.org/reports/tr44/#Property_List_Table +var contributoryProperties = []string{ + "oalpha", + "olower", + "oupper", +} + +func ContributoryProperties() []string { + return contributoryProperties +} + +// https://www.unicode.org/reports/tr44/#GC_Values_Table +var compositGeneralCategories = map[string][]string{ + // Cased_Letter + "lc": {"lu", "ll", "lt"}, + // Letter + "l": {"lu", "ll", "lt", "lm", "lo"}, + // Mark + "m": {"mm", "mc", "me"}, + // Number + "n": {"nd", "nl", "no"}, + // Punctuation + "p": {"pc", "pd", "ps", "pi", "pe", "pf", "po"}, + // Symbol + "s": {"sm", "sc", "sk", "so"}, + // Separator + "z": {"zs", "zl", "zp"}, + // Other + "c": {"cc", "cf", "cs", "co", "cn"}, +} + +// https://www.unicode.org/Public/13.0.0/ucd/DerivedCoreProperties.txt +var derivedCoreProperties = map[string][]string{ + // Alphabetic + "alpha": { + `\p{Lowercase=yes}`, + `\p{Uppercase=yes}`, + `\p{Lt}`, + `\p{Lm}`, + `\p{Lo}`, + `\p{Nl}`, + `\p{Other_Alphabetic=yes}`, + }, + // Lowercase + "lower": { + `\p{Ll}`, + `\p{Other_Lowercase=yes}`, + }, + // Uppercase + "upper": { + `\p{Lu}`, + `\p{Other_Uppercase=yes}`, + }, +} + +// https://www.unicode.org/Public/13.0.0/ucd/PropertyAliases.txt +var propertyNameAbbs = map[string]string{ + "generalcategory": "gc", + "gc": "gc", + "script": "sc", + "sc": "sc", + "alphabetic": "alpha", + "alpha": "alpha", + "otheralphabetic": "oalpha", + "oalpha": "oalpha", + "lowercase": "lower", + "lower": "lower", + "uppercase": "upper", + "upper": "upper", + "otherlowercase": "olower", + "olower": "olower", + "otheruppercase": "oupper", + "oupper": "oupper", + "whitespace": "wspace", + "wspace": "wspace", + "space": "wspace", +} + +// https://www.unicode.org/reports/tr44/#Type_Key_Table +// https://www.unicode.org/reports/tr44/#Binary_Values_Table +var binaryValues = map[string]bool{ + "yes": true, + "y": true, + "true": true, + "t": true, + "no": false, + "n": false, + "false": false, + "f": false, +} diff --git a/ucd/property_value_aliases.go b/ucd/property_value_aliases.go new file mode 100644 index 0000000..4bc69db --- /dev/null +++ b/ucd/property_value_aliases.go @@ -0,0 +1,82 @@ +package ucd + +import "io" + +type PropertyValueAliases struct { + GeneralCategory map[string]string + GeneralCategoryDefaultRange *CodePointRange + GeneralCategoryDefaultValue string + + Script map[string]string +} + +// ParsePropertyValueAliases parses the PropertyValueAliases.txt. +func ParsePropertyValueAliases(r io.Reader) (*PropertyValueAliases, error) { + gcAbbs := map[string]string{} + var defaultGCCPRange *CodePointRange + var defaultGCVal string + scAbbs := map[string]string{} + p := newParser(r) + for p.parse() { + // https://www.unicode.org/reports/tr44/#Property_Value_Aliases + // > In PropertyValueAliases.txt, the first field contains the abbreviated alias for a Unicode property, + // > the second field specifies an abbreviated symbolic name for a value of that property, and the third + // > field specifies the long symbolic name for that value of that property. These are the preferred + // > aliases. Additional aliases for some property values may be specified in the fourth or subsequent + // > fields. + if len(p.fields) > 0 { + switch p.fields[0].symbol() { + case "gc": + gcShort := p.fields[1].normalizedSymbol() + gcLong := p.fields[2].normalizedSymbol() + gcAbbs[gcShort] = gcShort + gcAbbs[gcLong] = gcShort + for _, f := range p.fields[3:] { + gcShortOther := f.normalizedSymbol() + gcAbbs[gcShortOther] = gcShort + } + case "sc": + scShort := p.fields[1].normalizedSymbol() + scLong := p.fields[2].normalizedSymbol() + scAbbs[scShort] = scShort + scAbbs[scLong] = scShort + for _, f := range p.fields[3:] { + scShortOther := f.normalizedSymbol() + scAbbs[scShortOther] = scShort + } + } + } + + // https://www.unicode.org/reports/tr44/#Missing_Conventions + // > @missing lines are also supplied for many properties in the file PropertyValueAliases.txt. + // > ... + // > there are currently two syntactic patterns used for @missing lines, as summarized schematically below: + // > 1. code_point_range; default_prop_val + // > 2. code_point_range; property_name; default_prop_val + // > ... + // > Pattern #2 is used in PropertyValueAliases.txt and in DerivedNormalizationProps.txt, both of which + // > contain values associated with many properties. For example: + // > # @missing: 0000..10FFFF; NFD_QC; Yes + if len(p.defaultFields) > 0 && p.defaultFields[1].symbol() == "General_Category" { + var err error + defaultGCCPRange, err = p.defaultFields[0].codePointRange() + if err != nil { + return nil, err + } + defaultGCVal = p.defaultFields[2].normalizedSymbol() + } + } + if p.err != nil { + return nil, p.err + } + return &PropertyValueAliases{ + GeneralCategory: gcAbbs, + GeneralCategoryDefaultRange: defaultGCCPRange, + GeneralCategoryDefaultValue: defaultGCVal, + Script: scAbbs, + }, nil +} + +func (a *PropertyValueAliases) gcAbb(gc string) string { + return a.GeneralCategory[gc] +} diff --git a/ucd/scripts.go b/ucd/scripts.go new file mode 100644 index 0000000..5040283 --- /dev/null +++ b/ucd/scripts.go @@ -0,0 +1,52 @@ +package ucd + +import ( + "fmt" + "io" +) + +type Scripts struct { + Script map[string][]*CodePointRange + ScriptDefaultRange *CodePointRange + ScriptDefaultValue string +} + +// ParseScripts parses the Scripts.txt. +func ParseScripts(r io.Reader, propValAliases *PropertyValueAliases) (*Scripts, error) { + ss := map[string][]*CodePointRange{} + var defaultRange *CodePointRange + var defaultValue string + p := newParser(r) + for p.parse() { + if len(p.fields) > 0 { + cp, err := p.fields[0].codePointRange() + if err != nil { + return nil, err + } + + name, ok := propValAliases.Script[p.fields[1].normalizedSymbol()] + if !ok { + return nil, fmt.Errorf("unknown property: %v", p.fields[1].symbol()) + } + ss[name] = append(ss[name], cp) + } + + if len(p.defaultFields) > 0 { + var err error + defaultRange, err = p.defaultFields[0].codePointRange() + if err != nil { + return nil, err + } + defaultValue = p.defaultFields[1].normalizedSymbol() + } + } + if p.err != nil { + return nil, p.err + } + + return &Scripts{ + Script: ss, + ScriptDefaultRange: defaultRange, + ScriptDefaultValue: defaultValue, + }, nil +} diff --git a/ucd/unicode_data.go b/ucd/unicode_data.go new file mode 100644 index 0000000..e2a8e87 --- /dev/null +++ b/ucd/unicode_data.go @@ -0,0 +1,56 @@ +package ucd + +import "io" + +type UnicodeData struct { + GeneralCategory map[string][]*CodePointRange + + propValAliases *PropertyValueAliases +} + +// ParseUnicodeData parses the UnicodeData.txt. +func ParseUnicodeData(r io.Reader, propValAliases *PropertyValueAliases) (*UnicodeData, error) { + unicodeData := &UnicodeData{ + GeneralCategory: map[string][]*CodePointRange{}, + propValAliases: propValAliases, + } + + p := newParser(r) + for p.parse() { + if len(p.fields) == 0 { + continue + } + cp, err := p.fields[0].codePointRange() + if err != nil { + return nil, err + } + gc := p.fields[2].normalizedSymbol() + unicodeData.addGC(gc, cp) + } + if p.err != nil { + return nil, p.err + } + + return unicodeData, nil +} + +func (u *UnicodeData) addGC(gc string, cp *CodePointRange) { + // https://www.unicode.org/reports/tr44/#Empty_Fields + // > The data file UnicodeData.txt defines many property values in each record. When a field in a data line + // > for a code point is empty, that indicates that the property takes the default value for that code point. + if gc == "" { + return + } + + cps, ok := u.GeneralCategory[u.propValAliases.gcAbb(gc)] + if ok { + c := cps[len(cps)-1] + if cp.From-c.To == 1 { + c.To = cp.To + } else { + u.GeneralCategory[u.propValAliases.gcAbb(gc)] = append(cps, cp) + } + } else { + u.GeneralCategory[u.propValAliases.gcAbb(gc)] = []*CodePointRange{cp} + } +} diff --git a/utf8/utf8.go b/utf8/utf8.go new file mode 100644 index 0000000..4f52bd4 --- /dev/null +++ b/utf8/utf8.go @@ -0,0 +1,112 @@ +package utf8 + +import ( + "fmt" + "strings" +) + +type CharBlock struct { + From []byte + To []byte +} + +func (b *CharBlock) String() string { + var s strings.Builder + fmt.Fprint(&s, "<") + fmt.Fprintf(&s, "%X", b.From[0]) + for i := 1; i < len(b.From); i++ { + fmt.Fprintf(&s, " %X", b.From[i]) + } + fmt.Fprint(&s, "..") + fmt.Fprintf(&s, "%X", b.To[0]) + for i := 1; i < len(b.To); i++ { + fmt.Fprintf(&s, " %X", b.To[i]) + } + fmt.Fprint(&s, ">") + return s.String() +} + +func GenCharBlocks(from, to rune) ([]*CharBlock, error) { + rs, err := splitCodePoint(from, to) + if err != nil { + return nil, err + } + + blks := make([]*CharBlock, len(rs)) + for i, r := range rs { + blks[i] = &CharBlock{ + From: []byte(string(r.from)), + To: []byte(string(r.to)), + } + } + + return blks, nil +} + +type cpRange struct { + from rune + to rune +} + +// splitCodePoint splits a code point range represented by <from..to> into some blocks. The code points that +// the block contains will be a continuous byte sequence when encoded into UTF-8. For instance, this function +// splits <U+0000..U+07FF> into <U+0000..U+007F> and <U+0080..U+07FF> because <U+0000..U+07FF> is continuous on +// the code point but non-continuous in the UTF-8 byte sequence (In UTF-8, <U+0000..U+007F> is encoded <00..7F>, +// and <U+0080..U+07FF> is encoded <C2 80..DF BF>). +// +// The blocks don't contain surrogate code points <U+D800..U+DFFF> because byte sequences encoding them are +// ill-formed in UTF-8. For instance, <U+D000..U+FFFF> is split into <U+D000..U+D7FF> and <U+E000..U+FFFF>. +// However, when `from` or `to` itself is the surrogate code point, this function returns an error. +func splitCodePoint(from, to rune) ([]*cpRange, error) { + if from > to { + return nil, fmt.Errorf("code point range must be from <= to: U+%X..U+%X", from, to) + } + if from < 0x0000 || from > 0x10ffff || to < 0x0000 || to > 0x10ffff { + return nil, fmt.Errorf("code point must be >=U+0000 and <=U+10FFFF: U+%X..U+%X", from, to) + } + // https://www.unicode.org/versions/Unicode13.0.0/ch03.pdf > 3.9 Unicode Encoding Forms > UTF-8 D92 + // > Because surrogate code points are not Unicode scalar values, any UTF-8 byte sequence that would otherwise + // > map to code points U+D800..U+DFFF is ill-formed. + if from >= 0xd800 && from <= 0xdfff || to >= 0xd800 && to <= 0xdfff { + return nil, fmt.Errorf("surrogate code points U+D800..U+DFFF are not allowed in UTF-8: U+%X..U+%X", from, to) + } + + in := &cpRange{ + from: from, + to: to, + } + var rs []*cpRange + for in.from <= in.to { + r := &cpRange{ + from: in.from, + to: in.to, + } + // https://www.unicode.org/versions/Unicode13.0.0/ch03.pdf > 3.9 Unicode Encoding Forms > UTF-8 Table 3-7. Well-Formed UTF-8 Byte Sequences + switch { + case in.from <= 0x007f && in.to > 0x007f: + r.to = 0x007f + case in.from <= 0x07ff && in.to > 0x07ff: + r.to = 0x07ff + case in.from <= 0x0fff && in.to > 0x0fff: + r.to = 0x0fff + case in.from <= 0xcfff && in.to > 0xcfff: + r.to = 0xcfff + case in.from <= 0xd7ff && in.to > 0xd7ff: + r.to = 0xd7ff + case in.from <= 0xffff && in.to > 0xffff: + r.to = 0xffff + case in.from <= 0x3ffff && in.to > 0x3ffff: + r.to = 0x3ffff + case in.from <= 0xfffff && in.to > 0xfffff: + r.to = 0xfffff + } + rs = append(rs, r) + in.from = r.to + 1 + + // Skip surrogate code points U+D800..U+DFFF. + if in.from >= 0xd800 && in.from <= 0xdfff { + in.from = 0xe000 + } + } + return rs, nil +} diff --git a/utf8/utf8_test.go b/utf8/utf8_test.go new file mode 100644 index 0000000..2dc8093 --- /dev/null +++ b/utf8/utf8_test.go @@ -0,0 +1,181 @@ +package utf8 + +import ( + "fmt" + "testing" +) + +func TestGenCharBlocks_WellFormed(t *testing.T) { + cBlk := func(from []byte, to []byte) *CharBlock { + return &CharBlock{ + From: from, + To: to, + } + } + + seq := func(b ...byte) []byte { + return b + } + + tests := []struct { + from rune + to rune + blocks []*CharBlock + }{ + { + from: '\u0000', + to: '\u007f', + blocks: []*CharBlock{ + cBlk(seq(0x00), seq(0x7f)), + }, + }, + { + from: '\u0080', + to: '\u07ff', + blocks: []*CharBlock{ + cBlk(seq(0xc2, 0x80), seq(0xdf, 0xbf)), + }, + }, + { + from: '\u0800', + to: '\u0fff', + blocks: []*CharBlock{ + cBlk(seq(0xe0, 0xa0, 0x80), seq(0xe0, 0xbf, 0xbf)), + }, + }, + { + from: '\u1000', + to: '\ucfff', + blocks: []*CharBlock{ + cBlk(seq(0xe1, 0x80, 0x80), seq(0xec, 0xbf, 0xbf)), + }, + }, + { + from: '\ud000', + to: '\ud7ff', + blocks: []*CharBlock{ + cBlk(seq(0xed, 0x80, 0x80), seq(0xed, 0x9f, 0xbf)), + }, + }, + { + from: '\ue000', + to: '\uffff', + blocks: []*CharBlock{ + cBlk(seq(0xee, 0x80, 0x80), seq(0xef, 0xbf, 0xbf)), + }, + }, + { + from: '\U00010000', + to: '\U0003ffff', + blocks: []*CharBlock{ + cBlk(seq(0xf0, 0x90, 0x80, 0x80), seq(0xf0, 0xbf, 0xbf, 0xbf)), + }, + }, + { + from: '\U00040000', + to: '\U000fffff', + blocks: []*CharBlock{ + cBlk(seq(0xf1, 0x80, 0x80, 0x80), seq(0xf3, 0xbf, 0xbf, 0xbf)), + }, + }, + { + from: '\U00100000', + to: '\U0010ffff', + blocks: []*CharBlock{ + cBlk(seq(0xf4, 0x80, 0x80, 0x80), seq(0xf4, 0x8f, 0xbf, 0xbf)), + }, + }, + { + from: '\u0000', + to: '\U0010ffff', + blocks: []*CharBlock{ + cBlk(seq(0x00), seq(0x7f)), + cBlk(seq(0xc2, 0x80), seq(0xdf, 0xbf)), + cBlk(seq(0xe0, 0xa0, 0x80), seq(0xe0, 0xbf, 0xbf)), + cBlk(seq(0xe1, 0x80, 0x80), seq(0xec, 0xbf, 0xbf)), + cBlk(seq(0xed, 0x80, 0x80), seq(0xed, 0x9f, 0xbf)), + cBlk(seq(0xee, 0x80, 0x80), seq(0xef, 0xbf, 0xbf)), + cBlk(seq(0xf0, 0x90, 0x80, 0x80), seq(0xf0, 0xbf, 0xbf, 0xbf)), + cBlk(seq(0xf1, 0x80, 0x80, 0x80), seq(0xf3, 0xbf, 0xbf, 0xbf)), + cBlk(seq(0xf4, 0x80, 0x80, 0x80), seq(0xf4, 0x8f, 0xbf, 0xbf)), + }, + }, + } + for _, tt := range tests { + t.Run(fmt.Sprintf("%v..%v", tt.from, tt.to), func(t *testing.T) { + blks, err := GenCharBlocks(tt.from, tt.to) + if err != nil { + t.Fatal(err) + } + if len(blks) != len(tt.blocks) { + t.Fatalf("unexpected character block: want: %+v, got: %+v", tt.blocks, blks) + } + for i, blk := range blks { + if len(blk.From) != len(tt.blocks[i].From) || len(blk.To) != len(tt.blocks[i].To) { + t.Fatalf("unexpected character block: want: %+v, got: %+v", tt.blocks, blks) + } + for j := 0; j < len(blk.From); j++ { + if blk.From[j] != tt.blocks[i].From[j] || blk.To[j] != tt.blocks[i].To[j] { + t.Fatalf("unexpected character block: want: %+v, got: %+v", tt.blocks, blks) + } + } + } + }) + } +} + +func TestGenCharBlocks_IllFormed(t *testing.T) { + tests := []struct { + from rune + to rune + }{ + { + // from > to + from: '\u0001', + to: '\u0000', + }, + { + from: -1, // <U+0000 + to: '\u0000', + }, + { + from: '\u0000', + to: -1, // <U+0000 + }, + { + from: 0x110000, // >U+10FFFF + to: '\u0000', + }, + { + from: '\u0000', + to: 0x110000, // >U+10FFFF + }, + { + from: 0xd800, // U+D800 (surrogate code point) + to: '\ue000', + }, + { + from: 0xdfff, // U+DFFF (surrogate code point) + to: '\ue000', + }, + { + from: '\ucfff', + to: 0xd800, // U+D800 (surrogate code point) + }, + { + from: '\ucfff', + to: 0xdfff, // U+DFFF (surrogate code point) + }, + } + for _, tt := range tests { + t.Run(fmt.Sprintf("%v..%v", tt.from, tt.to), func(t *testing.T) { + blks, err := GenCharBlocks(tt.from, tt.to) + if err == nil { + t.Fatal("expected error didn't occur") + } + if blks != nil { + t.Fatal("character blocks must be nil") + } + }) + } +} |