From b8b71860d7df5d06dd8a2b9ac62035e8ba8778a7 Mon Sep 17 00:00:00 2001
From: Ryo Nihei <nihei.dev@gmail.com>
Date: Wed, 30 Jun 2021 23:44:50 +0900
Subject: Simplify syntax of modifiers and semantic actions

Modifiers and semantic actions are represented by directives following the '#' symbol.
---
 spec/lexer_test.go | 29 ++++++++++++++++++++---------
 1 file changed, 20 insertions(+), 9 deletions(-)

(limited to 'spec/lexer_test.go')

diff --git a/spec/lexer_test.go b/spec/lexer_test.go
index bd7fbf0..2dc346e 100644
--- a/spec/lexer_test.go
+++ b/spec/lexer_test.go
@@ -14,15 +14,14 @@ func TestLexer_Run(t *testing.T) {
 	}{
 		{
 			caption: "the lexer can recognize all kinds of tokens",
-			src:     `id"terminal":|;@#'()$1...`,
+			src:     `id"terminal":|;#'()$1...`,
 			tokens: []*token{
 				newIDToken("id"),
 				newTerminalPatternToken("terminal"),
 				newSymbolToken(tokenKindColon),
 				newSymbolToken(tokenKindOr),
 				newSymbolToken(tokenKindSemicolon),
-				newSymbolToken(tokenKindModifierMarker),
-				newSymbolToken(tokenKindActionLeader),
+				newSymbolToken(tokenKindDirectiveMarker),
 				newSymbolToken(tokenKindTreeNodeOpen),
 				newSymbolToken(tokenKindTreeNodeClose),
 				newPositionToken(1),
@@ -46,6 +45,20 @@ func TestLexer_Run(t *testing.T) {
 				newEOFToken(),
 			},
 		},
+		{
+			caption: "the lexer can recognize newlines and combine consecutive newlines into one",
+			src:     "\u000A | \u000D | \u000D\u000A | \u000A\u000A \u000D\u000D \u000D\u000A\u000D\u000A",
+			tokens: []*token{
+				newSymbolToken(tokenKindNewline),
+				newSymbolToken(tokenKindOr),
+				newSymbolToken(tokenKindNewline),
+				newSymbolToken(tokenKindOr),
+				newSymbolToken(tokenKindNewline),
+				newSymbolToken(tokenKindOr),
+				newSymbolToken(tokenKindNewline),
+				newEOFToken(),
+			},
+		},
 		{
 			caption: "the lexer ignores line comments",
 			src: `
@@ -56,8 +69,11 @@ foo
 bar // This is the fourth comment.
 `,
 			tokens: []*token{
+				newSymbolToken(tokenKindNewline),
 				newIDToken("foo"),
+				newSymbolToken(tokenKindNewline),
 				newIDToken("bar"),
+				newSymbolToken(tokenKindNewline),
 				newEOFToken(),
 			},
 		},
@@ -89,17 +105,12 @@ bar // This is the fourth comment.
 		{
 			caption: "the lexer skips white spaces",
 			// \u0009: HT
-			// \u000A: LF
-			// \u000D: CR
 			// \u0020: SP
-			src: "a\u0020b\u000Ac\u000Dd\u000D\u000Ae\u0009f",
+			src: "a\u0009b\u0020c",
 			tokens: []*token{
 				newIDToken("a"),
 				newIDToken("b"),
 				newIDToken("c"),
-				newIDToken("d"),
-				newIDToken("e"),
-				newIDToken("f"),
 				newEOFToken(),
 			},
 		},
-- 
cgit v1.2.3