From 10d0c5dfeb9749f4226f86d5ac915718c5bec5c9 Mon Sep 17 00:00:00 2001 From: Ryo Nihei Date: Thu, 25 Nov 2021 23:11:50 +0900 Subject: Make character properties available in an inverse expression (Make [^\p{...}] available) --- compiler/parser.go | 4 ++++ driver/lexer_test.go | 16 ++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/compiler/parser.go b/compiler/parser.go index d64b79c..5d1a38d 100644 --- a/compiler/parser.go +++ b/compiler/parser.go @@ -652,6 +652,10 @@ func genNormalCharAST(c rune) astNode { } func exclude(symbol, base astNode) astNode { + if alt, ok := symbol.(*altNode); ok { + return exclude(alt.right, exclude(alt.left, base)) + } + switch base.(type) { case *altNode: left, right := base.children() diff --git a/driver/lexer_test.go b/driver/lexer_test.go index 36a3e31..275e992 100644 --- a/driver/lexer_test.go +++ b/driver/lexer_test.go @@ -756,6 +756,22 @@ func TestLexer_Next(t *testing.T) { newEOFTokenDefault(), }, }, + // Character properties are available in a bracket expression. + { + lspec: &spec.LexSpec{ + Name: "test", + Entries: []*spec.LexEntry{ + newLexEntryDefaultNOP("letter", `[\p{Letter}]+`), + newLexEntryDefaultNOP("non_letter", `[^\p{Letter}]+`), + }, + }, + src: `foo123`, + tokens: []*Token{ + newTokenDefault(1, 1, []byte(`foo`)), + newTokenDefault(2, 2, []byte(`123`)), + newEOFTokenDefault(), + }, + }, } for i, tt := range test { for compLv := compiler.CompressionLevelMin; compLv <= compiler.CompressionLevelMax; compLv++ { -- cgit v1.2.3