aboutsummaryrefslogtreecommitdiff
path: root/driver/lexer_test.go
diff options
context:
space:
mode:
Diffstat (limited to 'driver/lexer_test.go')
-rw-r--r--driver/lexer_test.go168
1 files changed, 84 insertions, 84 deletions
diff --git a/driver/lexer_test.go b/driver/lexer_test.go
index 8af3817..549800a 100644
--- a/driver/lexer_test.go
+++ b/driver/lexer_test.go
@@ -232,7 +232,7 @@ func TestLexer_Next(t *testing.T) {
// maleeni cannot handle the null character in patterns because compiler.lexer,
// specifically read() and restore(), recognizes the null characters as that a symbol doesn't exist.
// If a pattern needs a null character, use code point expression \u{0000}.
- newLexEntryDefaultNOP("char1Byte", "[\x01-\x7f]"),
+ newLexEntryDefaultNOP("char_1_byte", "[\x01-\x7f]"),
},
},
src: string([]byte{
@@ -242,10 +242,10 @@ func TestLexer_Next(t *testing.T) {
0x7f,
}),
tokens: []*Token{
- newTokenDefault(1, 1, "char1Byte", []byte{0x01}),
- newTokenDefault(1, 1, "char1Byte", []byte{0x02}),
- newTokenDefault(1, 1, "char1Byte", []byte{0x7e}),
- newTokenDefault(1, 1, "char1Byte", []byte{0x7f}),
+ newTokenDefault(1, 1, "char_1_byte", []byte{0x01}),
+ newTokenDefault(1, 1, "char_1_byte", []byte{0x02}),
+ newTokenDefault(1, 1, "char_1_byte", []byte{0x7e}),
+ newTokenDefault(1, 1, "char_1_byte", []byte{0x7f}),
newEOFTokenDefault(),
},
},
@@ -254,7 +254,7 @@ func TestLexer_Next(t *testing.T) {
Name: "test",
Entries: []*spec.LexEntry{
// all 2 byte characters
- newLexEntryDefaultNOP("char2Byte", "[\xc2\x80-\xdf\xbf]"),
+ newLexEntryDefaultNOP("char_2_byte", "[\xc2\x80-\xdf\xbf]"),
},
},
src: string([]byte{
@@ -264,10 +264,10 @@ func TestLexer_Next(t *testing.T) {
0xdf, 0xbf,
}),
tokens: []*Token{
- newTokenDefault(1, 1, "char2Byte", []byte{0xc2, 0x80}),
- newTokenDefault(1, 1, "char2Byte", []byte{0xc2, 0x81}),
- newTokenDefault(1, 1, "char2Byte", []byte{0xdf, 0xbe}),
- newTokenDefault(1, 1, "char2Byte", []byte{0xdf, 0xbf}),
+ newTokenDefault(1, 1, "char_2_byte", []byte{0xc2, 0x80}),
+ newTokenDefault(1, 1, "char_2_byte", []byte{0xc2, 0x81}),
+ newTokenDefault(1, 1, "char_2_byte", []byte{0xdf, 0xbe}),
+ newTokenDefault(1, 1, "char_2_byte", []byte{0xdf, 0xbf}),
newEOFTokenDefault(),
},
},
@@ -276,14 +276,14 @@ func TestLexer_Next(t *testing.T) {
Name: "test",
Entries: []*spec.LexEntry{
// All bytes are the same.
- newLexEntryDefaultNOP("char3Byte", "[\xe0\xa0\x80-\xe0\xa0\x80]"),
+ newLexEntryDefaultNOP("char_3_byte", "[\xe0\xa0\x80-\xe0\xa0\x80]"),
},
},
src: string([]byte{
0xe0, 0xa0, 0x80,
}),
tokens: []*Token{
- newTokenDefault(1, 1, "char3Byte", []byte{0xe0, 0xa0, 0x80}),
+ newTokenDefault(1, 1, "char_3_byte", []byte{0xe0, 0xa0, 0x80}),
newEOFTokenDefault(),
},
},
@@ -292,7 +292,7 @@ func TestLexer_Next(t *testing.T) {
Name: "test",
Entries: []*spec.LexEntry{
// The first two bytes are the same.
- newLexEntryDefaultNOP("char3Byte", "[\xe0\xa0\x80-\xe0\xa0\xbf]"),
+ newLexEntryDefaultNOP("char_3_byte", "[\xe0\xa0\x80-\xe0\xa0\xbf]"),
},
},
src: string([]byte{
@@ -302,10 +302,10 @@ func TestLexer_Next(t *testing.T) {
0xe0, 0xa0, 0xbf,
}),
tokens: []*Token{
- newTokenDefault(1, 1, "char3Byte", []byte{0xe0, 0xa0, 0x80}),
- newTokenDefault(1, 1, "char3Byte", []byte{0xe0, 0xa0, 0x81}),
- newTokenDefault(1, 1, "char3Byte", []byte{0xe0, 0xa0, 0xbe}),
- newTokenDefault(1, 1, "char3Byte", []byte{0xe0, 0xa0, 0xbf}),
+ newTokenDefault(1, 1, "char_3_byte", []byte{0xe0, 0xa0, 0x80}),
+ newTokenDefault(1, 1, "char_3_byte", []byte{0xe0, 0xa0, 0x81}),
+ newTokenDefault(1, 1, "char_3_byte", []byte{0xe0, 0xa0, 0xbe}),
+ newTokenDefault(1, 1, "char_3_byte", []byte{0xe0, 0xa0, 0xbf}),
newEOFTokenDefault(),
},
},
@@ -314,7 +314,7 @@ func TestLexer_Next(t *testing.T) {
Name: "test",
Entries: []*spec.LexEntry{
// The first byte are the same.
- newLexEntryDefaultNOP("char3Byte", "[\xe0\xa0\x80-\xe0\xbf\xbf]"),
+ newLexEntryDefaultNOP("char_3_byte", "[\xe0\xa0\x80-\xe0\xbf\xbf]"),
},
},
src: string([]byte{
@@ -324,10 +324,10 @@ func TestLexer_Next(t *testing.T) {
0xe0, 0xbf, 0xbf,
}),
tokens: []*Token{
- newTokenDefault(1, 1, "char3Byte", []byte{0xe0, 0xa0, 0x80}),
- newTokenDefault(1, 1, "char3Byte", []byte{0xe0, 0xa0, 0x81}),
- newTokenDefault(1, 1, "char3Byte", []byte{0xe0, 0xbf, 0xbe}),
- newTokenDefault(1, 1, "char3Byte", []byte{0xe0, 0xbf, 0xbf}),
+ newTokenDefault(1, 1, "char_3_byte", []byte{0xe0, 0xa0, 0x80}),
+ newTokenDefault(1, 1, "char_3_byte", []byte{0xe0, 0xa0, 0x81}),
+ newTokenDefault(1, 1, "char_3_byte", []byte{0xe0, 0xbf, 0xbe}),
+ newTokenDefault(1, 1, "char_3_byte", []byte{0xe0, 0xbf, 0xbf}),
newEOFTokenDefault(),
},
},
@@ -336,7 +336,7 @@ func TestLexer_Next(t *testing.T) {
Name: "test",
Entries: []*spec.LexEntry{
// all 3 byte characters
- newLexEntryDefaultNOP("char3Byte", "[\xe0\xa0\x80-\xef\xbf\xbf]"),
+ newLexEntryDefaultNOP("char_3_byte", "[\xe0\xa0\x80-\xef\xbf\xbf]"),
},
},
src: string([]byte{
@@ -358,22 +358,22 @@ func TestLexer_Next(t *testing.T) {
0xef, 0xbf, 0xbf,
}),
tokens: []*Token{
- newTokenDefault(1, 1, "char3Byte", []byte{0xe0, 0xa0, 0x80}),
- newTokenDefault(1, 1, "char3Byte", []byte{0xe0, 0xa0, 0x81}),
- newTokenDefault(1, 1, "char3Byte", []byte{0xe0, 0xbf, 0xbe}),
- newTokenDefault(1, 1, "char3Byte", []byte{0xe0, 0xbf, 0xbf}),
- newTokenDefault(1, 1, "char3Byte", []byte{0xe1, 0x80, 0x80}),
- newTokenDefault(1, 1, "char3Byte", []byte{0xe1, 0x80, 0x81}),
- newTokenDefault(1, 1, "char3Byte", []byte{0xec, 0xbf, 0xbe}),
- newTokenDefault(1, 1, "char3Byte", []byte{0xec, 0xbf, 0xbf}),
- newTokenDefault(1, 1, "char3Byte", []byte{0xed, 0x80, 0x80}),
- newTokenDefault(1, 1, "char3Byte", []byte{0xed, 0x80, 0x81}),
- newTokenDefault(1, 1, "char3Byte", []byte{0xed, 0x9f, 0xbe}),
- newTokenDefault(1, 1, "char3Byte", []byte{0xed, 0x9f, 0xbf}),
- newTokenDefault(1, 1, "char3Byte", []byte{0xee, 0x80, 0x80}),
- newTokenDefault(1, 1, "char3Byte", []byte{0xee, 0x80, 0x81}),
- newTokenDefault(1, 1, "char3Byte", []byte{0xef, 0xbf, 0xbe}),
- newTokenDefault(1, 1, "char3Byte", []byte{0xef, 0xbf, 0xbf}),
+ newTokenDefault(1, 1, "char_3_byte", []byte{0xe0, 0xa0, 0x80}),
+ newTokenDefault(1, 1, "char_3_byte", []byte{0xe0, 0xa0, 0x81}),
+ newTokenDefault(1, 1, "char_3_byte", []byte{0xe0, 0xbf, 0xbe}),
+ newTokenDefault(1, 1, "char_3_byte", []byte{0xe0, 0xbf, 0xbf}),
+ newTokenDefault(1, 1, "char_3_byte", []byte{0xe1, 0x80, 0x80}),
+ newTokenDefault(1, 1, "char_3_byte", []byte{0xe1, 0x80, 0x81}),
+ newTokenDefault(1, 1, "char_3_byte", []byte{0xec, 0xbf, 0xbe}),
+ newTokenDefault(1, 1, "char_3_byte", []byte{0xec, 0xbf, 0xbf}),
+ newTokenDefault(1, 1, "char_3_byte", []byte{0xed, 0x80, 0x80}),
+ newTokenDefault(1, 1, "char_3_byte", []byte{0xed, 0x80, 0x81}),
+ newTokenDefault(1, 1, "char_3_byte", []byte{0xed, 0x9f, 0xbe}),
+ newTokenDefault(1, 1, "char_3_byte", []byte{0xed, 0x9f, 0xbf}),
+ newTokenDefault(1, 1, "char_3_byte", []byte{0xee, 0x80, 0x80}),
+ newTokenDefault(1, 1, "char_3_byte", []byte{0xee, 0x80, 0x81}),
+ newTokenDefault(1, 1, "char_3_byte", []byte{0xef, 0xbf, 0xbe}),
+ newTokenDefault(1, 1, "char_3_byte", []byte{0xef, 0xbf, 0xbf}),
newEOFTokenDefault(),
},
},
@@ -382,14 +382,14 @@ func TestLexer_Next(t *testing.T) {
Name: "test",
Entries: []*spec.LexEntry{
// All bytes are the same.
- newLexEntryDefaultNOP("char4Byte", "[\xf0\x90\x80\x80-\xf0\x90\x80\x80]"),
+ newLexEntryDefaultNOP("char_4_byte", "[\xf0\x90\x80\x80-\xf0\x90\x80\x80]"),
},
},
src: string([]byte{
0xf0, 0x90, 0x80, 0x80,
}),
tokens: []*Token{
- newTokenDefault(1, 1, "char4Byte", []byte{0xf0, 0x90, 0x80, 0x80}),
+ newTokenDefault(1, 1, "char_4_byte", []byte{0xf0, 0x90, 0x80, 0x80}),
newEOFTokenDefault(),
},
},
@@ -398,7 +398,7 @@ func TestLexer_Next(t *testing.T) {
Name: "test",
Entries: []*spec.LexEntry{
// The first 3 bytes are the same.
- newLexEntryDefaultNOP("char4Byte", "[\xf0\x90\x80\x80-\xf0\x90\x80\xbf]"),
+ newLexEntryDefaultNOP("char_4_byte", "[\xf0\x90\x80\x80-\xf0\x90\x80\xbf]"),
},
},
src: string([]byte{
@@ -408,10 +408,10 @@ func TestLexer_Next(t *testing.T) {
0xf0, 0x90, 0x80, 0xbf,
}),
tokens: []*Token{
- newTokenDefault(1, 1, "char4Byte", []byte{0xf0, 0x90, 0x80, 0x80}),
- newTokenDefault(1, 1, "char4Byte", []byte{0xf0, 0x90, 0x80, 0x81}),
- newTokenDefault(1, 1, "char4Byte", []byte{0xf0, 0x90, 0x80, 0xbe}),
- newTokenDefault(1, 1, "char4Byte", []byte{0xf0, 0x90, 0x80, 0xbf}),
+ newTokenDefault(1, 1, "char_4_byte", []byte{0xf0, 0x90, 0x80, 0x80}),
+ newTokenDefault(1, 1, "char_4_byte", []byte{0xf0, 0x90, 0x80, 0x81}),
+ newTokenDefault(1, 1, "char_4_byte", []byte{0xf0, 0x90, 0x80, 0xbe}),
+ newTokenDefault(1, 1, "char_4_byte", []byte{0xf0, 0x90, 0x80, 0xbf}),
newEOFTokenDefault(),
},
},
@@ -420,7 +420,7 @@ func TestLexer_Next(t *testing.T) {
Name: "test",
Entries: []*spec.LexEntry{
// The first 2 bytes are the same.
- newLexEntryDefaultNOP("char4Byte", "[\xf0\x90\x80\x80-\xf0\x90\xbf\xbf]"),
+ newLexEntryDefaultNOP("char_4_byte", "[\xf0\x90\x80\x80-\xf0\x90\xbf\xbf]"),
},
},
src: string([]byte{
@@ -430,10 +430,10 @@ func TestLexer_Next(t *testing.T) {
0xf0, 0x90, 0xbf, 0xbf,
}),
tokens: []*Token{
- newTokenDefault(1, 1, "char4Byte", []byte{0xf0, 0x90, 0x80, 0x80}),
- newTokenDefault(1, 1, "char4Byte", []byte{0xf0, 0x90, 0x80, 0x81}),
- newTokenDefault(1, 1, "char4Byte", []byte{0xf0, 0x90, 0xbf, 0xbe}),
- newTokenDefault(1, 1, "char4Byte", []byte{0xf0, 0x90, 0xbf, 0xbf}),
+ newTokenDefault(1, 1, "char_4_byte", []byte{0xf0, 0x90, 0x80, 0x80}),
+ newTokenDefault(1, 1, "char_4_byte", []byte{0xf0, 0x90, 0x80, 0x81}),
+ newTokenDefault(1, 1, "char_4_byte", []byte{0xf0, 0x90, 0xbf, 0xbe}),
+ newTokenDefault(1, 1, "char_4_byte", []byte{0xf0, 0x90, 0xbf, 0xbf}),
newEOFTokenDefault(),
},
},
@@ -442,7 +442,7 @@ func TestLexer_Next(t *testing.T) {
Name: "test",
Entries: []*spec.LexEntry{
// The first byte are the same.
- newLexEntryDefaultNOP("char4Byte", "[\xf0\x90\x80\x80-\xf0\xbf\xbf\xbf]"),
+ newLexEntryDefaultNOP("char_4_byte", "[\xf0\x90\x80\x80-\xf0\xbf\xbf\xbf]"),
},
},
src: string([]byte{
@@ -452,10 +452,10 @@ func TestLexer_Next(t *testing.T) {
0xf0, 0xbf, 0xbf, 0xbf,
}),
tokens: []*Token{
- newTokenDefault(1, 1, "char4Byte", []byte{0xf0, 0x90, 0x80, 0x80}),
- newTokenDefault(1, 1, "char4Byte", []byte{0xf0, 0x90, 0x80, 0x81}),
- newTokenDefault(1, 1, "char4Byte", []byte{0xf0, 0xbf, 0xbf, 0xbe}),
- newTokenDefault(1, 1, "char4Byte", []byte{0xf0, 0xbf, 0xbf, 0xbf}),
+ newTokenDefault(1, 1, "char_4_byte", []byte{0xf0, 0x90, 0x80, 0x80}),
+ newTokenDefault(1, 1, "char_4_byte", []byte{0xf0, 0x90, 0x80, 0x81}),
+ newTokenDefault(1, 1, "char_4_byte", []byte{0xf0, 0xbf, 0xbf, 0xbe}),
+ newTokenDefault(1, 1, "char_4_byte", []byte{0xf0, 0xbf, 0xbf, 0xbf}),
newEOFTokenDefault(),
},
},
@@ -464,7 +464,7 @@ func TestLexer_Next(t *testing.T) {
Name: "test",
Entries: []*spec.LexEntry{
// all 4 byte characters
- newLexEntryDefaultNOP("char4Byte", "[\xf0\x90\x80\x80-\xf4\x8f\xbf\xbf]"),
+ newLexEntryDefaultNOP("char_4_byte", "[\xf0\x90\x80\x80-\xf4\x8f\xbf\xbf]"),
},
},
src: string([]byte{
@@ -482,18 +482,18 @@ func TestLexer_Next(t *testing.T) {
0xf4, 0x8f, 0xbf, 0xbf,
}),
tokens: []*Token{
- newTokenDefault(1, 1, "char4Byte", []byte{0xf0, 0x90, 0x80, 0x80}),
- newTokenDefault(1, 1, "char4Byte", []byte{0xf0, 0x90, 0x80, 0x81}),
- newTokenDefault(1, 1, "char4Byte", []byte{0xf0, 0xbf, 0xbf, 0xbe}),
- newTokenDefault(1, 1, "char4Byte", []byte{0xf0, 0xbf, 0xbf, 0xbf}),
- newTokenDefault(1, 1, "char4Byte", []byte{0xf1, 0x80, 0x80, 0x80}),
- newTokenDefault(1, 1, "char4Byte", []byte{0xf1, 0x80, 0x80, 0x81}),
- newTokenDefault(1, 1, "char4Byte", []byte{0xf3, 0xbf, 0xbf, 0xbe}),
- newTokenDefault(1, 1, "char4Byte", []byte{0xf3, 0xbf, 0xbf, 0xbf}),
- newTokenDefault(1, 1, "char4Byte", []byte{0xf4, 0x80, 0x80, 0x80}),
- newTokenDefault(1, 1, "char4Byte", []byte{0xf4, 0x80, 0x80, 0x81}),
- newTokenDefault(1, 1, "char4Byte", []byte{0xf4, 0x8f, 0xbf, 0xbe}),
- newTokenDefault(1, 1, "char4Byte", []byte{0xf4, 0x8f, 0xbf, 0xbf}),
+ newTokenDefault(1, 1, "char_4_byte", []byte{0xf0, 0x90, 0x80, 0x80}),
+ newTokenDefault(1, 1, "char_4_byte", []byte{0xf0, 0x90, 0x80, 0x81}),
+ newTokenDefault(1, 1, "char_4_byte", []byte{0xf0, 0xbf, 0xbf, 0xbe}),
+ newTokenDefault(1, 1, "char_4_byte", []byte{0xf0, 0xbf, 0xbf, 0xbf}),
+ newTokenDefault(1, 1, "char_4_byte", []byte{0xf1, 0x80, 0x80, 0x80}),
+ newTokenDefault(1, 1, "char_4_byte", []byte{0xf1, 0x80, 0x80, 0x81}),
+ newTokenDefault(1, 1, "char_4_byte", []byte{0xf3, 0xbf, 0xbf, 0xbe}),
+ newTokenDefault(1, 1, "char_4_byte", []byte{0xf3, 0xbf, 0xbf, 0xbf}),
+ newTokenDefault(1, 1, "char_4_byte", []byte{0xf4, 0x80, 0x80, 0x80}),
+ newTokenDefault(1, 1, "char_4_byte", []byte{0xf4, 0x80, 0x80, 0x81}),
+ newTokenDefault(1, 1, "char_4_byte", []byte{0xf4, 0x8f, 0xbf, 0xbe}),
+ newTokenDefault(1, 1, "char_4_byte", []byte{0xf4, 0x8f, 0xbf, 0xbf}),
newEOFTokenDefault(),
},
},
@@ -501,12 +501,12 @@ func TestLexer_Next(t *testing.T) {
lspec: &spec.LexSpec{
Name: "test",
Entries: []*spec.LexEntry{
- newLexEntryDefaultNOP("NonNumber", "[^0-9]+[0-9]"),
+ newLexEntryDefaultNOP("non_number", "[^0-9]+[0-9]"),
},
},
src: "foo9",
tokens: []*Token{
- newTokenDefault(1, 1, "NonNumber", []byte("foo9")),
+ newTokenDefault(1, 1, "non_number", []byte("foo9")),
newEOFTokenDefault(),
},
},
@@ -514,18 +514,18 @@ func TestLexer_Next(t *testing.T) {
lspec: &spec.LexSpec{
Name: "test",
Entries: []*spec.LexEntry{
- newLexEntryDefaultNOP("char1Byte", "\\u{006E}"),
- newLexEntryDefaultNOP("char2Byte", "\\u{03BD}"),
- newLexEntryDefaultNOP("char3Byte", "\\u{306B}"),
- newLexEntryDefaultNOP("char4Byte", "\\u{01F638}"),
+ newLexEntryDefaultNOP("char_1_byte", "\\u{006E}"),
+ newLexEntryDefaultNOP("char_2_byte", "\\u{03BD}"),
+ newLexEntryDefaultNOP("char_3_byte", "\\u{306B}"),
+ newLexEntryDefaultNOP("char_4_byte", "\\u{01F638}"),
},
},
src: "nνに😸",
tokens: []*Token{
- newTokenDefault(1, 1, "char1Byte", []byte{0x6E}),
- newTokenDefault(2, 2, "char2Byte", []byte{0xCE, 0xBD}),
- newTokenDefault(3, 3, "char3Byte", []byte{0xE3, 0x81, 0xAB}),
- newTokenDefault(4, 4, "char4Byte", []byte{0xF0, 0x9F, 0x98, 0xB8}),
+ newTokenDefault(1, 1, "char_1_byte", []byte{0x6E}),
+ newTokenDefault(2, 2, "char_2_byte", []byte{0xCE, 0xBD}),
+ newTokenDefault(3, 3, "char_3_byte", []byte{0xE3, 0x81, 0xAB}),
+ newTokenDefault(4, 4, "char_4_byte", []byte{0xF0, 0x9F, 0x98, 0xB8}),
newEOFTokenDefault(),
},
},
@@ -533,15 +533,15 @@ func TestLexer_Next(t *testing.T) {
lspec: &spec.LexSpec{
Name: "test",
Entries: []*spec.LexEntry{
- newLexEntryDefaultNOP("codePointsAlt", "[\\u{006E}\\u{03BD}\\u{306B}\\u{01F638}]"),
+ newLexEntryDefaultNOP("code_points_alt", "[\\u{006E}\\u{03BD}\\u{306B}\\u{01F638}]"),
},
},
src: "nνに😸",
tokens: []*Token{
- newTokenDefault(1, 1, "codePointsAlt", []byte{0x6E}),
- newTokenDefault(1, 1, "codePointsAlt", []byte{0xCE, 0xBD}),
- newTokenDefault(1, 1, "codePointsAlt", []byte{0xE3, 0x81, 0xAB}),
- newTokenDefault(1, 1, "codePointsAlt", []byte{0xF0, 0x9F, 0x98, 0xB8}),
+ newTokenDefault(1, 1, "code_points_alt", []byte{0x6E}),
+ newTokenDefault(1, 1, "code_points_alt", []byte{0xCE, 0xBD}),
+ newTokenDefault(1, 1, "code_points_alt", []byte{0xE3, 0x81, 0xAB}),
+ newTokenDefault(1, 1, "code_points_alt", []byte{0xF0, 0x9F, 0x98, 0xB8}),
newEOFTokenDefault(),
},
},
generated by cgit v1.2.3 (git 2.49.0) at 2025-07-13 19:39:27 +0000