Add ordered symbol notation

7 files changed, 134 insertions, 79 deletions

diff --git a/spec/lexer.go b/spec/lexer.go
index 2459c40..c1c4b0d 100644
--- a/spec/lexer.go
+++ b/spec/lexer.go
@@ -15,21 +15,22 @@ import (
 type tokenKind string
 
 const (
-	tokenKindKWFragment      = tokenKind("fragment")
-	tokenKindID              = tokenKind("id")
-	tokenKindTerminalPattern = tokenKind("terminal pattern")
-	tokenKindStringLiteral   = tokenKind("string")
-	tokenKindColon           = tokenKind(":")
-	tokenKindOr              = tokenKind("|")
-	tokenKindSemicolon       = tokenKind(";")
-	tokenKindLabelMarker     = tokenKind("@")
-	tokenKindDirectiveMarker = tokenKind("#")
-	tokenKindExpantion       = tokenKind("...")
-	tokenKindLParen          = tokenKind("(")
-	tokenKindRParen          = tokenKind(")")
-	tokenKindNewline         = tokenKind("newline")
-	tokenKindEOF             = tokenKind("eof")
-	tokenKindInvalid         = tokenKind("invalid")
+	tokenKindKWFragment          = tokenKind("fragment")
+	tokenKindID                  = tokenKind("id")
+	tokenKindTerminalPattern     = tokenKind("terminal pattern")
+	tokenKindStringLiteral       = tokenKind("string")
+	tokenKindColon               = tokenKind(":")
+	tokenKindOr                  = tokenKind("|")
+	tokenKindSemicolon           = tokenKind(";")
+	tokenKindLabelMarker         = tokenKind("@")
+	tokenKindDirectiveMarker     = tokenKind("#")
+	tokenKindExpantion           = tokenKind("...")
+	tokenKindOrderedSymbolMarker = tokenKind("$")
+	tokenKindLParen              = tokenKind("(")
+	tokenKindRParen              = tokenKind(")")
+	tokenKindNewline             = tokenKind("newline")
+	tokenKindEOF                 = tokenKind("eof")
+	tokenKindInvalid             = tokenKind("invalid")
 )
 
 type Position struct {
@@ -266,6 +267,8 @@ func (l *lexer) lexAndSkipWSs() (*token, error) {
 		return newSymbolToken(tokenKindDirectiveMarker, newPosition(tok.Row+1, tok.Col+1)), nil
 	case KindIDExpansion:
 		return newSymbolToken(tokenKindExpantion, newPosition(tok.Row+1, tok.Col+1)), nil
+	case KindIDOrderedSymbolMarker:
+		return newSymbolToken(tokenKindOrderedSymbolMarker, newPosition(tok.Row+1, tok.Col+1)), nil
 	case KindIDLParen:
 		return newSymbolToken(tokenKindLParen, newPosition(tok.Row+1, tok.Col+1)), nil
 	case KindIDRParen:
diff --git a/spec/lexer_test.go b/spec/lexer_test.go
index 621eff5..0e7cc89 100644
--- a/spec/lexer_test.go
+++ b/spec/lexer_test.go
@@ -36,7 +36,7 @@ func TestLexer_Run(t *testing.T) {
 	}{
 		{
 			caption: "the lexer can recognize all kinds of tokens",
-			src:     `id"terminal"'string':|;@...#()`,
+			src:     `id"terminal"'string':|;@...#$()`,
 			tokens: []*token{
 				idTok("id"),
 				termPatTok("terminal"),
@@ -47,6 +47,7 @@ func TestLexer_Run(t *testing.T) {
 				symTok(tokenKindLabelMarker),
 				symTok(tokenKindExpantion),
 				symTok(tokenKindDirectiveMarker),
+				symTok(tokenKindOrderedSymbolMarker),
 				symTok(tokenKindLParen),
 				symTok(tokenKindRParen),
 				newEOFToken(),
diff --git a/spec/lexspec.json b/spec/lexspec.json
index 6a11a4a..7222be0 100644
--- a/spec/lexspec.json
+++ b/spec/lexspec.json
@@ -118,6 +118,10 @@
 			"pattern": "#"
 		},
 		{
+			"kind": "ordered_symbol_marker",
+			"pattern": "$"
+		},
+		{
 			"kind": "l_paren",
 			"pattern": "\\("
 		},
diff --git a/spec/parser.go b/spec/parser.go
index 3b5907e..9c66bfb 100644
--- a/spec/parser.go
+++ b/spec/parser.go
@@ -55,12 +55,13 @@ type DirectiveNode struct {
 }
 
 type ParameterNode struct {
-	ID        string
-	Pattern   string
-	String    string
-	Group     []*DirectiveNode
-	Expansion bool
-	Pos       Position
+	ID            string
+	Pattern       string
+	String        string
+	OrderedSymbol string
+	Group         []*DirectiveNode
+	Expansion     bool
+	Pos           Position
 }
 
 type FragmentNode struct {
@@ -461,6 +462,14 @@ func (p *parser) parseParameter() *ParameterNode {
 			String: p.lastTok.text,
 			Pos:    p.lastTok.pos,
 		}
+	case p.consume(tokenKindOrderedSymbolMarker):
+		if !p.consume(tokenKindID) {
+			raiseSyntaxError(p.pos.Row, synErrNoOrderedSymbolName)
+		}
+		param = &ParameterNode{
+			OrderedSymbol: p.lastTok.text,
+			Pos:           p.lastTok.pos,
+		}
 	case p.consume(tokenKindLParen):
 		pos := p.lastTok.pos
 		var g []*DirectiveNode
diff --git a/spec/parser_test.go b/spec/parser_test.go
index 3fe950f..15f1330 100644
--- a/spec/parser_test.go
+++ b/spec/parser_test.go
@@ -14,14 +14,12 @@ func TestParse(t *testing.T) {
 			Parameters: []*ParameterNode{param},
 		}
 	}
-
 	prec := func(param *ParameterNode) *DirectiveNode {
 		return &DirectiveNode{
 			Name:       "prec",
 			Parameters: []*ParameterNode{param},
 		}
 	}
-
 	leftAssoc := func(params ...*ParameterNode) *DirectiveNode {
 		return &DirectiveNode{
 			Name:       "left",
@@ -82,6 +80,11 @@ func TestParse(t *testing.T) {
 			ID: id,
 		}
 	}
+	ordSymParam := func(id string) *ParameterNode {
+		return &ParameterNode{
+			OrderedSymbol: id,
+		}
+	}
 	exp := func(param *ParameterNode) *ParameterNode {
 		param.Expansion = true
 		return param
@@ -152,9 +155,9 @@ func TestParse(t *testing.T) {
 #name test;
 
 #prec (
-    #left a b
-    #right c d
-    #assign e f
+    #left a b $x1
+    #right c d $x2
+    #assign e f $x3
 );
 `,
 			ast: &RootNode{
@@ -182,6 +185,10 @@ func TestParse(t *testing.T) {
 												idParam("b"),
 												newPos(5),
 											),
+											withParamPos(
+												ordSymParam("x1"),
+												newPos(5),
+											),
 										),
 										newPos(5),
 									),
@@ -195,6 +202,10 @@ func TestParse(t *testing.T) {
 												idParam("d"),
 												newPos(6),
 											),
+											withParamPos(
+												ordSymParam("x2"),
+												newPos(6),
+											),
 										),
 										newPos(6),
 									),
@@ -208,6 +219,10 @@ func TestParse(t *testing.T) {
 												idParam("f"),
 												newPos(7),
 											),
+											withParamPos(
+												ordSymParam("x3"),
+												newPos(7),
+											),
 										),
 										newPos(7),
 									),
@@ -237,6 +252,15 @@ func TestParse(t *testing.T) {
 			synErr: synErrUnclosedDirGroup,
 		},
 		{
+			caption: "an ordered symbol marker '$' must be followed by and ID",
+			src: `
+#prec (
+    #assign $
+);
+`,
+			synErr: synErrNoOrderedSymbolName,
+		},
+		{
 			caption: "single production is a valid grammar",
 			src:     `a: "a";`,
 			ast: &RootNode{
@@ -299,6 +323,13 @@ c: ;
 			},
 		},
 		{
+			caption: "a production cannot contain an ordered symbol",
+			src: `
+a: $x;
+`,
+			synErr: synErrNoSemicolon,
+		},
+		{
 			caption: "`fragment` is a reserved word",
 			src:     `fragment: 'fragment';`,
 			synErr:  synErrNoProductionName,
diff --git a/spec/syntax_error.go b/spec/syntax_error.go
index 3b44d2d..ad847a2 100644
--- a/spec/syntax_error.go
+++ b/spec/syntax_error.go
@@ -32,6 +32,7 @@ var (
 	synErrLabelWithNoSymbol      = newSyntaxError("a label must follow a symbol")
 	synErrNoLabel                = newSyntaxError("an identifier that represents a label is missing after the label marker @")
 	synErrNoDirectiveName        = newSyntaxError("a directive needs a name")
+	synErrNoOrderedSymbolName    = newSyntaxError("an ordered symbol name is missing")
 	synErrUnclosedDirGroup       = newSyntaxError("a directive group must be closed by )")
 	synErrSemicolonNoNewline     = newSyntaxError("a semicolon must be followed by a newline")
 	synErrFragmentNoPattern      = newSyntaxError("a fragment needs one pattern element")
diff --git a/spec/vartan_lexer.go b/spec/vartan_lexer.go
index 146748a..7c0dfd4 100644
--- a/spec/vartan_lexer.go
+++ b/spec/vartan_lexer.go
@@ -342,55 +342,57 @@ func ModeIDToName(id ModeID) string {
 }
 
 const (
-	KindIDNil                KindID = 0
-	KindIDWhiteSpace         KindID = 1
-	KindIDNewline            KindID = 2
-	KindIDLineComment        KindID = 3
-	KindIDKwFragment         KindID = 4
-	KindIDIdentifier         KindID = 5
-	KindIDTerminalOpen       KindID = 6
-	KindIDStringLiteralOpen  KindID = 7
-	KindIDColon              KindID = 8
-	KindIDOr                 KindID = 9
-	KindIDSemicolon          KindID = 10
-	KindIDLabelMarker        KindID = 11
-	KindIDExpansion          KindID = 12
-	KindIDDirectiveMarker    KindID = 13
-	KindIDLParen             KindID = 14
-	KindIDRParen             KindID = 15
-	KindIDPattern            KindID = 16
-	KindIDTerminalClose      KindID = 17
-	KindIDEscapeSymbol       KindID = 18
-	KindIDCharSeq            KindID = 19
-	KindIDEscapedQuot        KindID = 20
-	KindIDEscapedBackSlash   KindID = 21
-	KindIDStringLiteralClose KindID = 22
+	KindIDNil                 KindID = 0
+	KindIDWhiteSpace          KindID = 1
+	KindIDNewline             KindID = 2
+	KindIDLineComment         KindID = 3
+	KindIDKwFragment          KindID = 4
+	KindIDIdentifier          KindID = 5
+	KindIDTerminalOpen        KindID = 6
+	KindIDStringLiteralOpen   KindID = 7
+	KindIDColon               KindID = 8
+	KindIDOr                  KindID = 9
+	KindIDSemicolon           KindID = 10
+	KindIDLabelMarker         KindID = 11
+	KindIDExpansion           KindID = 12
+	KindIDDirectiveMarker     KindID = 13
+	KindIDOrderedSymbolMarker KindID = 14
+	KindIDLParen              KindID = 15
+	KindIDRParen              KindID = 16
+	KindIDPattern             KindID = 17
+	KindIDTerminalClose       KindID = 18
+	KindIDEscapeSymbol        KindID = 19
+	KindIDCharSeq             KindID = 20
+	KindIDEscapedQuot         KindID = 21
+	KindIDEscapedBackSlash    KindID = 22
+	KindIDStringLiteralClose  KindID = 23
 )
 
 const (
-	KindNameNil                = ""
-	KindNameWhiteSpace         = "white_space"
-	KindNameNewline            = "newline"
-	KindNameLineComment        = "line_comment"
-	KindNameKwFragment         = "kw_fragment"
-	KindNameIdentifier         = "identifier"
-	KindNameTerminalOpen       = "terminal_open"
-	KindNameStringLiteralOpen  = "string_literal_open"
-	KindNameColon              = "colon"
-	KindNameOr                 = "or"
-	KindNameSemicolon          = "semicolon"
-	KindNameLabelMarker        = "label_marker"
-	KindNameExpansion          = "expansion"
-	KindNameDirectiveMarker    = "directive_marker"
-	KindNameLParen             = "l_paren"
-	KindNameRParen             = "r_paren"
-	KindNamePattern            = "pattern"
-	KindNameTerminalClose      = "terminal_close"
-	KindNameEscapeSymbol       = "escape_symbol"
-	KindNameCharSeq            = "char_seq"
-	KindNameEscapedQuot        = "escaped_quot"
-	KindNameEscapedBackSlash   = "escaped_back_slash"
-	KindNameStringLiteralClose = "string_literal_close"
+	KindNameNil                 = ""
+	KindNameWhiteSpace          = "white_space"
+	KindNameNewline             = "newline"
+	KindNameLineComment         = "line_comment"
+	KindNameKwFragment          = "kw_fragment"
+	KindNameIdentifier          = "identifier"
+	KindNameTerminalOpen        = "terminal_open"
+	KindNameStringLiteralOpen   = "string_literal_open"
+	KindNameColon               = "colon"
+	KindNameOr                  = "or"
+	KindNameSemicolon           = "semicolon"
+	KindNameLabelMarker         = "label_marker"
+	KindNameExpansion           = "expansion"
+	KindNameDirectiveMarker     = "directive_marker"
+	KindNameOrderedSymbolMarker = "ordered_symbol_marker"
+	KindNameLParen              = "l_paren"
+	KindNameRParen              = "r_paren"
+	KindNamePattern             = "pattern"
+	KindNameTerminalClose       = "terminal_close"
+	KindNameEscapeSymbol        = "escape_symbol"
+	KindNameCharSeq             = "char_seq"
+	KindNameEscapedQuot         = "escaped_quot"
+	KindNameEscapedBackSlash    = "escaped_back_slash"
+	KindNameStringLiteralClose  = "string_literal_close"
 )
 
 // KindIDToName converts a kind ID to a name.
@@ -424,6 +426,8 @@ func KindIDToName(id KindID) string {
 		return KindNameExpansion
 	case KindIDDirectiveMarker:
 		return KindNameDirectiveMarker
+	case KindIDOrderedSymbolMarker:
+		return KindNameOrderedSymbolMarker
 	case KindIDLParen:
 		return KindNameLParen
 	case KindIDRParen:
@@ -471,7 +475,7 @@ func NewLexSpec() *lexSpec {
 		pop: [][]bool{
 			nil,
 			{
-				false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false,
+				false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false,
 			},
 			{
 				false, false, true, false,
@@ -483,7 +487,7 @@ func NewLexSpec() *lexSpec {
 		push: [][]ModeID{
 			nil,
 			{
-				0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0,
+				0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 			},
 			{
 				0, 0, 0, 0,
@@ -509,7 +513,7 @@ func NewLexSpec() *lexSpec {
 			{
 				0, 0, 1, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 				0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 4, 5, 0, 0, 2, 6, 7, 8, 9,
-				10, 11, 12, 13, 14, 15,
+				10, 11, 12, 13, 14, 15, 16,
 			},
 			{
 				0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -540,6 +544,7 @@ func NewLexSpec() *lexSpec {
 				KindIDLabelMarker,
 				KindIDExpansion,
 				KindIDDirectiveMarker,
+				KindIDOrderedSymbolMarker,
 				KindIDLParen,
 				KindIDRParen,
 			},
@@ -573,6 +578,7 @@ func NewLexSpec() *lexSpec {
 			KindNameLabelMarker,
 			KindNameExpansion,
 			KindNameDirectiveMarker,
+			KindNameOrderedSymbolMarker,
 			KindNameLParen,
 			KindNameRParen,
 			KindNamePattern,
@@ -593,7 +599,7 @@ func NewLexSpec() *lexSpec {
 			{
 				0, 1, 2, 3, 4, 5, 6, 7, 6, 8, 6, 9, 6, 10, 6, 11, 12, 6, 13, 14,
 				6, 15, 16, 6, 17, 18, 19, 20, 21, 22, 23, 24, 24, 25, 26, 0, 0, 0, 0, 0,
-				0, 0, 0, 0, 0, 0,
+				0, 0, 0, 0, 0, 0, 0,
 			},
 			{
 				0, 1, 2, 3, 2, 4, 2, 5, 2, 6, 2, 7, 8, 2, 9, 10, 2, 11, 12, 2,
@@ -639,7 +645,7 @@ func NewLexSpec() *lexSpec {
 				5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
 				5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
 				5, 5, 5, 5, 5, 1, 1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-				-1, -1, -1, -1, -1, -1, -1, -1, 1, -1, 1, 1, -1, -1, -1, 1, 1, 1, -1, -1,
+				-1, -1, -1, -1, -1, -1, -1, -1, 1, -1, 1, 1, 1, -1, -1, 1, 1, 1, -1, -1,
 				-1, -1, 1, 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, 1, -1, -1, -1, -1,
 				1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
 				1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, 1, -1, 1, 1, 1, 1, 1, 1, 1,
@@ -984,7 +990,7 @@ func NewLexSpec() *lexSpec {
 				6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
 				6, 6, 6, 6, 7, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 11, 13, 13,
 				15, 18, 18, 18, 21, 2, 35, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-				0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 36, 43, 0, 0, 0, 37, 44, 45, 0, 0,
+				0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 36, 43, 44, 0, 0, 37, 45, 46, 0, 0,
 				0, 0, 33, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 40, 0, 0, 0, 0,
 				41, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32,
 				32, 32, 32, 32, 32, 32, 32, 0, 0, 0, 0, 32, 0, 32, 32, 32, 32, 32, 24, 32,