aboutsummaryrefslogtreecommitdiff
path: root/driver/lexer_test.go
diff options
context:
space:
mode:
Diffstat (limited to 'driver/lexer_test.go')
-rw-r--r--driver/lexer_test.go63
1 files changed, 31 insertions, 32 deletions
diff --git a/driver/lexer_test.go b/driver/lexer_test.go
index 1f3841b..68830a5 100644
--- a/driver/lexer_test.go
+++ b/driver/lexer_test.go
@@ -10,6 +10,13 @@ import (
"github.com/nihei9/maleeni/spec"
)
+func newLexEntry(kind string, pattern string) *spec.LexEntry {
+ return &spec.LexEntry{
+ Kind: spec.LexKind(kind),
+ Pattern: spec.LexPattern(pattern),
+ }
+}
+
func TestLexer_Next(t *testing.T) {
test := []struct {
lspec *spec.LexSpec
@@ -19,8 +26,8 @@ func TestLexer_Next(t *testing.T) {
{
lspec: &spec.LexSpec{
Entries: []*spec.LexEntry{
- spec.NewLexEntry("t1", "(a|b)*abb"),
- spec.NewLexEntry("t2", " +"),
+ newLexEntry("t1", "(a|b)*abb"),
+ newLexEntry("t2", " +"),
},
},
src: "abb aabb aaabb babb bbabb abbbabb",
@@ -42,9 +49,9 @@ func TestLexer_Next(t *testing.T) {
{
lspec: &spec.LexSpec{
Entries: []*spec.LexEntry{
- spec.NewLexEntry("t1", "b?a+"),
- spec.NewLexEntry("t2", "(ab)?(cd)+"),
- spec.NewLexEntry("t3", " +"),
+ newLexEntry("t1", "b?a+"),
+ newLexEntry("t2", "(ab)?(cd)+"),
+ newLexEntry("t3", " +"),
},
},
src: "ba baaa a aaa abcd abcdcdcd cd cdcdcd",
@@ -70,7 +77,7 @@ func TestLexer_Next(t *testing.T) {
{
lspec: &spec.LexSpec{
Entries: []*spec.LexEntry{
- spec.NewLexEntry("t1", "."),
+ newLexEntry("t1", "."),
},
},
src: string([]byte{
@@ -114,7 +121,7 @@ func TestLexer_Next(t *testing.T) {
{
lspec: &spec.LexSpec{
Entries: []*spec.LexEntry{
- spec.NewLexEntry("t1", "[ab.*+?|()[\\]]"),
+ newLexEntry("t1", "[ab.*+?|()[\\]]"),
},
},
src: "ab.*+?|()[]",
@@ -142,7 +149,7 @@ func TestLexer_Next(t *testing.T) {
// maleeni cannot handle the null character in patterns because compiler.lexer,
// specifically read() and restore(), recognizes the null characters as that a symbol doesn't exist.
// There is room for improvement in this behavior of the lexer.
- spec.NewLexEntry("1ByteChar", "[\x01-\x7f]"),
+ newLexEntry("1ByteChar", "[\x01-\x7f]"),
},
},
src: string([]byte{
@@ -163,7 +170,7 @@ func TestLexer_Next(t *testing.T) {
lspec: &spec.LexSpec{
Entries: []*spec.LexEntry{
// all 2 byte characters
- spec.NewLexEntry("2ByteChar", "[\xc2\x80-\xdf\xbf]"),
+ newLexEntry("2ByteChar", "[\xc2\x80-\xdf\xbf]"),
},
},
src: string([]byte{
@@ -184,7 +191,7 @@ func TestLexer_Next(t *testing.T) {
lspec: &spec.LexSpec{
Entries: []*spec.LexEntry{
// All bytes are the same.
- spec.NewLexEntry("3ByteChar", "[\xe0\xa0\x80-\xe0\xa0\x80]"),
+ newLexEntry("3ByteChar", "[\xe0\xa0\x80-\xe0\xa0\x80]"),
},
},
src: string([]byte{
@@ -199,7 +206,7 @@ func TestLexer_Next(t *testing.T) {
lspec: &spec.LexSpec{
Entries: []*spec.LexEntry{
// The first two bytes are the same.
- spec.NewLexEntry("3ByteChar", "[\xe0\xa0\x80-\xe0\xa0\xbf]"),
+ newLexEntry("3ByteChar", "[\xe0\xa0\x80-\xe0\xa0\xbf]"),
},
},
src: string([]byte{
@@ -220,7 +227,7 @@ func TestLexer_Next(t *testing.T) {
lspec: &spec.LexSpec{
Entries: []*spec.LexEntry{
// The first byte are the same.
- spec.NewLexEntry("3ByteChar", "[\xe0\xa0\x80-\xe0\xbf\xbf]"),
+ newLexEntry("3ByteChar", "[\xe0\xa0\x80-\xe0\xbf\xbf]"),
},
},
src: string([]byte{
@@ -241,7 +248,7 @@ func TestLexer_Next(t *testing.T) {
lspec: &spec.LexSpec{
Entries: []*spec.LexEntry{
// all 3 byte characters
- spec.NewLexEntry("3ByteChar", "[\xe0\xa0\x80-\xef\xbf\xbf]"),
+ newLexEntry("3ByteChar", "[\xe0\xa0\x80-\xef\xbf\xbf]"),
},
},
src: string([]byte{
@@ -286,7 +293,7 @@ func TestLexer_Next(t *testing.T) {
lspec: &spec.LexSpec{
Entries: []*spec.LexEntry{
// All bytes are the same.
- spec.NewLexEntry("4ByteChar", "[\xf0\x90\x80\x80-\xf0\x90\x80\x80]"),
+ newLexEntry("4ByteChar", "[\xf0\x90\x80\x80-\xf0\x90\x80\x80]"),
},
},
src: string([]byte{
@@ -301,7 +308,7 @@ func TestLexer_Next(t *testing.T) {
lspec: &spec.LexSpec{
Entries: []*spec.LexEntry{
// The first 3 bytes are the same.
- spec.NewLexEntry("4ByteChar", "[\xf0\x90\x80\x80-\xf0\x90\x80\xbf]"),
+ newLexEntry("4ByteChar", "[\xf0\x90\x80\x80-\xf0\x90\x80\xbf]"),
},
},
src: string([]byte{
@@ -322,7 +329,7 @@ func TestLexer_Next(t *testing.T) {
lspec: &spec.LexSpec{
Entries: []*spec.LexEntry{
// The first 2 bytes are the same.
- spec.NewLexEntry("4ByteChar", "[\xf0\x90\x80\x80-\xf0\x90\xbf\xbf]"),
+ newLexEntry("4ByteChar", "[\xf0\x90\x80\x80-\xf0\x90\xbf\xbf]"),
},
},
src: string([]byte{
@@ -343,7 +350,7 @@ func TestLexer_Next(t *testing.T) {
lspec: &spec.LexSpec{
Entries: []*spec.LexEntry{
// The first byte are the same.
- spec.NewLexEntry("4ByteChar", "[\xf0\x90\x80\x80-\xf0\xbf\xbf\xbf]"),
+ newLexEntry("4ByteChar", "[\xf0\x90\x80\x80-\xf0\xbf\xbf\xbf]"),
},
},
src: string([]byte{
@@ -364,7 +371,7 @@ func TestLexer_Next(t *testing.T) {
lspec: &spec.LexSpec{
Entries: []*spec.LexEntry{
// all 4 byte characters
- spec.NewLexEntry("4ByteChar", "[\xf0\x90\x80\x80-\xf4\x8f\xbf\xbf]"),
+ newLexEntry("4ByteChar", "[\xf0\x90\x80\x80-\xf4\x8f\xbf\xbf]"),
},
},
src: string([]byte{
@@ -400,7 +407,7 @@ func TestLexer_Next(t *testing.T) {
{
lspec: &spec.LexSpec{
Entries: []*spec.LexEntry{
- spec.NewLexEntry("NonNumber", "[^0-9]+[0-9]"),
+ newLexEntry("NonNumber", "[^0-9]+[0-9]"),
},
},
src: "foo9",
@@ -439,8 +446,8 @@ func TestLexer_Next(t *testing.T) {
func TestLexer_PeekN(t *testing.T) {
clspec, err := compiler.Compile(&spec.LexSpec{
Entries: []*spec.LexEntry{
- spec.NewLexEntry("", "foo"),
- spec.NewLexEntry("", "bar"),
+ newLexEntry("t1", "foo"),
+ newLexEntry("t2", "bar"),
},
})
if err != nil {
@@ -452,17 +459,9 @@ func TestLexer_PeekN(t *testing.T) {
}
expectedTokens := []*Token{
- {
- ID: 1,
- Match: newByteSequence([]byte("foo")),
- },
- {
- ID: 2,
- Match: newByteSequence([]byte("bar")),
- },
- {
- EOF: true,
- },
+ newToken(1, "t1", []byte("foo")),
+ newToken(2, "t2", []byte("bar")),
+ newEOFToken(),
}
tok, err := lex.Peek1()
generated by cgit v1.2.3 (git 2.49.0) at 2025-07-13 19:57:54 +0000