1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
|
package compiler
import (
"reflect"
"strings"
"testing"
)
func TestLexer(t *testing.T) {
tests := []struct {
caption string
src string
tokens []*token
err error
}{
{
caption: "lexer can recognize ordinaly characters",
src: "123abcいろは",
tokens: []*token{
newToken(tokenKindChar, '1'),
newToken(tokenKindChar, '2'),
newToken(tokenKindChar, '3'),
newToken(tokenKindChar, 'a'),
newToken(tokenKindChar, 'b'),
newToken(tokenKindChar, 'c'),
newToken(tokenKindChar, 'い'),
newToken(tokenKindChar, 'ろ'),
newToken(tokenKindChar, 'は'),
newToken(tokenKindEOF, nullChar),
},
},
{
caption: "lexer can recognize the special characters",
src: "*|()",
tokens: []*token{
newToken(tokenKindRepeat, nullChar),
newToken(tokenKindAlt, nullChar),
newToken(tokenKindGroupOpen, nullChar),
newToken(tokenKindGroupClose, nullChar),
newToken(tokenKindEOF, nullChar),
},
},
{
caption: "lexer can recognize the escape sequences",
src: "\\\\\\*\\|\\(\\)",
tokens: []*token{
newToken(tokenKindChar, '\\'),
newToken(tokenKindChar, '*'),
newToken(tokenKindChar, '|'),
newToken(tokenKindChar, '('),
newToken(tokenKindChar, ')'),
newToken(tokenKindEOF, nullChar),
},
},
{
caption: "lexer raises an error when an invalid escape sequence appears",
src: "\\@",
err: &SyntaxError{},
},
{
caption: "lexer raises an error when the incomplete escape sequence (EOF following \\) appears",
src: "\\",
err: &SyntaxError{},
},
}
for _, tt := range tests {
t.Run(tt.caption, func(t *testing.T) {
lex := newLexer(strings.NewReader(tt.src))
var err error
var tok *token
i := 0
for {
tok, err = lex.next()
if err != nil {
break
}
if i >= len(tt.tokens) {
break
}
eTok := tt.tokens[i]
i++
testToken(t, tok, eTok)
if tok.kind == tokenKindEOF {
break
}
}
ty := reflect.TypeOf(err)
eTy := reflect.TypeOf(tt.err)
if ty != eTy {
t.Fatalf("unexpected error type; want: %v, got: %v", eTy, ty)
}
if i < len(tt.tokens) {
t.Fatalf("expecte more tokens")
}
})
}
}
func testToken(t *testing.T, a, e *token) {
t.Helper()
if e.kind != a.kind || e.char != a.char {
t.Fatalf("unexpected token; want: %v, got: %v", e, a)
}
}
|
