aboutsummaryrefslogtreecommitdiff
path: root/spec
diff options
context:
space:
mode:
Diffstat (limited to 'spec')
-rw-r--r--spec/lexer.go33
-rw-r--r--spec/lexer_test.go3
-rw-r--r--spec/lexspec.json4
-rw-r--r--spec/parser.go21
-rw-r--r--spec/parser_test.go41
-rw-r--r--spec/syntax_error.go1
-rw-r--r--spec/vartan_lexer.go110
7 files changed, 134 insertions, 79 deletions
diff --git a/spec/lexer.go b/spec/lexer.go
index 2459c40..c1c4b0d 100644
--- a/spec/lexer.go
+++ b/spec/lexer.go
@@ -15,21 +15,22 @@ import (
type tokenKind string
const (
- tokenKindKWFragment = tokenKind("fragment")
- tokenKindID = tokenKind("id")
- tokenKindTerminalPattern = tokenKind("terminal pattern")
- tokenKindStringLiteral = tokenKind("string")
- tokenKindColon = tokenKind(":")
- tokenKindOr = tokenKind("|")
- tokenKindSemicolon = tokenKind(";")
- tokenKindLabelMarker = tokenKind("@")
- tokenKindDirectiveMarker = tokenKind("#")
- tokenKindExpantion = tokenKind("...")
- tokenKindLParen = tokenKind("(")
- tokenKindRParen = tokenKind(")")
- tokenKindNewline = tokenKind("newline")
- tokenKindEOF = tokenKind("eof")
- tokenKindInvalid = tokenKind("invalid")
+ tokenKindKWFragment = tokenKind("fragment")
+ tokenKindID = tokenKind("id")
+ tokenKindTerminalPattern = tokenKind("terminal pattern")
+ tokenKindStringLiteral = tokenKind("string")
+ tokenKindColon = tokenKind(":")
+ tokenKindOr = tokenKind("|")
+ tokenKindSemicolon = tokenKind(";")
+ tokenKindLabelMarker = tokenKind("@")
+ tokenKindDirectiveMarker = tokenKind("#")
+ tokenKindExpantion = tokenKind("...")
+ tokenKindOrderedSymbolMarker = tokenKind("$")
+ tokenKindLParen = tokenKind("(")
+ tokenKindRParen = tokenKind(")")
+ tokenKindNewline = tokenKind("newline")
+ tokenKindEOF = tokenKind("eof")
+ tokenKindInvalid = tokenKind("invalid")
)
type Position struct {
@@ -266,6 +267,8 @@ func (l *lexer) lexAndSkipWSs() (*token, error) {
return newSymbolToken(tokenKindDirectiveMarker, newPosition(tok.Row+1, tok.Col+1)), nil
case KindIDExpansion:
return newSymbolToken(tokenKindExpantion, newPosition(tok.Row+1, tok.Col+1)), nil
+ case KindIDOrderedSymbolMarker:
+ return newSymbolToken(tokenKindOrderedSymbolMarker, newPosition(tok.Row+1, tok.Col+1)), nil
case KindIDLParen:
return newSymbolToken(tokenKindLParen, newPosition(tok.Row+1, tok.Col+1)), nil
case KindIDRParen:
diff --git a/spec/lexer_test.go b/spec/lexer_test.go
index 621eff5..0e7cc89 100644
--- a/spec/lexer_test.go
+++ b/spec/lexer_test.go
@@ -36,7 +36,7 @@ func TestLexer_Run(t *testing.T) {
}{
{
caption: "the lexer can recognize all kinds of tokens",
- src: `id"terminal"'string':|;@...#()`,
+ src: `id"terminal"'string':|;@...#$()`,
tokens: []*token{
idTok("id"),
termPatTok("terminal"),
@@ -47,6 +47,7 @@ func TestLexer_Run(t *testing.T) {
symTok(tokenKindLabelMarker),
symTok(tokenKindExpantion),
symTok(tokenKindDirectiveMarker),
+ symTok(tokenKindOrderedSymbolMarker),
symTok(tokenKindLParen),
symTok(tokenKindRParen),
newEOFToken(),
diff --git a/spec/lexspec.json b/spec/lexspec.json
index 6a11a4a..7222be0 100644
--- a/spec/lexspec.json
+++ b/spec/lexspec.json
@@ -118,6 +118,10 @@
"pattern": "#"
},
{
+ "kind": "ordered_symbol_marker",
+ "pattern": "$"
+ },
+ {
"kind": "l_paren",
"pattern": "\\("
},
diff --git a/spec/parser.go b/spec/parser.go
index 3b5907e..9c66bfb 100644
--- a/spec/parser.go
+++ b/spec/parser.go
@@ -55,12 +55,13 @@ type DirectiveNode struct {
}
type ParameterNode struct {
- ID string
- Pattern string
- String string
- Group []*DirectiveNode
- Expansion bool
- Pos Position
+ ID string
+ Pattern string
+ String string
+ OrderedSymbol string
+ Group []*DirectiveNode
+ Expansion bool
+ Pos Position
}
type FragmentNode struct {
@@ -461,6 +462,14 @@ func (p *parser) parseParameter() *ParameterNode {
String: p.lastTok.text,
Pos: p.lastTok.pos,
}
+ case p.consume(tokenKindOrderedSymbolMarker):
+ if !p.consume(tokenKindID) {
+ raiseSyntaxError(p.pos.Row, synErrNoOrderedSymbolName)
+ }
+ param = &ParameterNode{
+ OrderedSymbol: p.lastTok.text,
+ Pos: p.lastTok.pos,
+ }
case p.consume(tokenKindLParen):
pos := p.lastTok.pos
var g []*DirectiveNode
diff --git a/spec/parser_test.go b/spec/parser_test.go
index 3fe950f..15f1330 100644
--- a/spec/parser_test.go
+++ b/spec/parser_test.go
@@ -14,14 +14,12 @@ func TestParse(t *testing.T) {
Parameters: []*ParameterNode{param},
}
}
-
prec := func(param *ParameterNode) *DirectiveNode {
return &DirectiveNode{
Name: "prec",
Parameters: []*ParameterNode{param},
}
}
-
leftAssoc := func(params ...*ParameterNode) *DirectiveNode {
return &DirectiveNode{
Name: "left",
@@ -82,6 +80,11 @@ func TestParse(t *testing.T) {
ID: id,
}
}
+ ordSymParam := func(id string) *ParameterNode {
+ return &ParameterNode{
+ OrderedSymbol: id,
+ }
+ }
exp := func(param *ParameterNode) *ParameterNode {
param.Expansion = true
return param
@@ -152,9 +155,9 @@ func TestParse(t *testing.T) {
#name test;
#prec (
- #left a b
- #right c d
- #assign e f
+ #left a b $x1
+ #right c d $x2
+ #assign e f $x3
);
`,
ast: &RootNode{
@@ -182,6 +185,10 @@ func TestParse(t *testing.T) {
idParam("b"),
newPos(5),
),
+ withParamPos(
+ ordSymParam("x1"),
+ newPos(5),
+ ),
),
newPos(5),
),
@@ -195,6 +202,10 @@ func TestParse(t *testing.T) {
idParam("d"),
newPos(6),
),
+ withParamPos(
+ ordSymParam("x2"),
+ newPos(6),
+ ),
),
newPos(6),
),
@@ -208,6 +219,10 @@ func TestParse(t *testing.T) {
idParam("f"),
newPos(7),
),
+ withParamPos(
+ ordSymParam("x3"),
+ newPos(7),
+ ),
),
newPos(7),
),
@@ -237,6 +252,15 @@ func TestParse(t *testing.T) {
synErr: synErrUnclosedDirGroup,
},
{
+ caption: "an ordered symbol marker '$' must be followed by and ID",
+ src: `
+#prec (
+ #assign $
+);
+`,
+ synErr: synErrNoOrderedSymbolName,
+ },
+ {
caption: "single production is a valid grammar",
src: `a: "a";`,
ast: &RootNode{
@@ -299,6 +323,13 @@ c: ;
},
},
{
+ caption: "a production cannot contain an ordered symbol",
+ src: `
+a: $x;
+`,
+ synErr: synErrNoSemicolon,
+ },
+ {
caption: "`fragment` is a reserved word",
src: `fragment: 'fragment';`,
synErr: synErrNoProductionName,
diff --git a/spec/syntax_error.go b/spec/syntax_error.go
index 3b44d2d..ad847a2 100644
--- a/spec/syntax_error.go
+++ b/spec/syntax_error.go
@@ -32,6 +32,7 @@ var (
synErrLabelWithNoSymbol = newSyntaxError("a label must follow a symbol")
synErrNoLabel = newSyntaxError("an identifier that represents a label is missing after the label marker @")
synErrNoDirectiveName = newSyntaxError("a directive needs a name")
+ synErrNoOrderedSymbolName = newSyntaxError("an ordered symbol name is missing")
synErrUnclosedDirGroup = newSyntaxError("a directive group must be closed by )")
synErrSemicolonNoNewline = newSyntaxError("a semicolon must be followed by a newline")
synErrFragmentNoPattern = newSyntaxError("a fragment needs one pattern element")
diff --git a/spec/vartan_lexer.go b/spec/vartan_lexer.go
index 146748a..7c0dfd4 100644
--- a/spec/vartan_lexer.go
+++ b/spec/vartan_lexer.go
@@ -342,55 +342,57 @@ func ModeIDToName(id ModeID) string {
}
const (
- KindIDNil KindID = 0
- KindIDWhiteSpace KindID = 1
- KindIDNewline KindID = 2
- KindIDLineComment KindID = 3
- KindIDKwFragment KindID = 4
- KindIDIdentifier KindID = 5
- KindIDTerminalOpen KindID = 6
- KindIDStringLiteralOpen KindID = 7
- KindIDColon KindID = 8
- KindIDOr KindID = 9
- KindIDSemicolon KindID = 10
- KindIDLabelMarker KindID = 11
- KindIDExpansion KindID = 12
- KindIDDirectiveMarker KindID = 13
- KindIDLParen KindID = 14
- KindIDRParen KindID = 15
- KindIDPattern KindID = 16
- KindIDTerminalClose KindID = 17
- KindIDEscapeSymbol KindID = 18
- KindIDCharSeq KindID = 19
- KindIDEscapedQuot KindID = 20
- KindIDEscapedBackSlash KindID = 21
- KindIDStringLiteralClose KindID = 22
+ KindIDNil KindID = 0
+ KindIDWhiteSpace KindID = 1
+ KindIDNewline KindID = 2
+ KindIDLineComment KindID = 3
+ KindIDKwFragment KindID = 4
+ KindIDIdentifier KindID = 5
+ KindIDTerminalOpen KindID = 6
+ KindIDStringLiteralOpen KindID = 7
+ KindIDColon KindID = 8
+ KindIDOr KindID = 9
+ KindIDSemicolon KindID = 10
+ KindIDLabelMarker KindID = 11
+ KindIDExpansion KindID = 12
+ KindIDDirectiveMarker KindID = 13
+ KindIDOrderedSymbolMarker KindID = 14
+ KindIDLParen KindID = 15
+ KindIDRParen KindID = 16
+ KindIDPattern KindID = 17
+ KindIDTerminalClose KindID = 18
+ KindIDEscapeSymbol KindID = 19
+ KindIDCharSeq KindID = 20
+ KindIDEscapedQuot KindID = 21
+ KindIDEscapedBackSlash KindID = 22
+ KindIDStringLiteralClose KindID = 23
)
const (
- KindNameNil = ""
- KindNameWhiteSpace = "white_space"
- KindNameNewline = "newline"
- KindNameLineComment = "line_comment"
- KindNameKwFragment = "kw_fragment"
- KindNameIdentifier = "identifier"
- KindNameTerminalOpen = "terminal_open"
- KindNameStringLiteralOpen = "string_literal_open"
- KindNameColon = "colon"
- KindNameOr = "or"
- KindNameSemicolon = "semicolon"
- KindNameLabelMarker = "label_marker"
- KindNameExpansion = "expansion"
- KindNameDirectiveMarker = "directive_marker"
- KindNameLParen = "l_paren"
- KindNameRParen = "r_paren"
- KindNamePattern = "pattern"
- KindNameTerminalClose = "terminal_close"
- KindNameEscapeSymbol = "escape_symbol"
- KindNameCharSeq = "char_seq"
- KindNameEscapedQuot = "escaped_quot"
- KindNameEscapedBackSlash = "escaped_back_slash"
- KindNameStringLiteralClose = "string_literal_close"
+ KindNameNil = ""
+ KindNameWhiteSpace = "white_space"
+ KindNameNewline = "newline"
+ KindNameLineComment = "line_comment"
+ KindNameKwFragment = "kw_fragment"
+ KindNameIdentifier = "identifier"
+ KindNameTerminalOpen = "terminal_open"
+ KindNameStringLiteralOpen = "string_literal_open"
+ KindNameColon = "colon"
+ KindNameOr = "or"
+ KindNameSemicolon = "semicolon"
+ KindNameLabelMarker = "label_marker"
+ KindNameExpansion = "expansion"
+ KindNameDirectiveMarker = "directive_marker"
+ KindNameOrderedSymbolMarker = "ordered_symbol_marker"
+ KindNameLParen = "l_paren"
+ KindNameRParen = "r_paren"
+ KindNamePattern = "pattern"
+ KindNameTerminalClose = "terminal_close"
+ KindNameEscapeSymbol = "escape_symbol"
+ KindNameCharSeq = "char_seq"
+ KindNameEscapedQuot = "escaped_quot"
+ KindNameEscapedBackSlash = "escaped_back_slash"
+ KindNameStringLiteralClose = "string_literal_close"
)
// KindIDToName converts a kind ID to a name.
@@ -424,6 +426,8 @@ func KindIDToName(id KindID) string {
return KindNameExpansion
case KindIDDirectiveMarker:
return KindNameDirectiveMarker
+ case KindIDOrderedSymbolMarker:
+ return KindNameOrderedSymbolMarker
case KindIDLParen:
return KindNameLParen
case KindIDRParen:
@@ -471,7 +475,7 @@ func NewLexSpec() *lexSpec {
pop: [][]bool{
nil,
{
- false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false,
+ false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false,
},
{
false, false, true, false,
@@ -483,7 +487,7 @@ func NewLexSpec() *lexSpec {
push: [][]ModeID{
nil,
{
- 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0,
},
{
0, 0, 0, 0,
@@ -509,7 +513,7 @@ func NewLexSpec() *lexSpec {
{
0, 0, 1, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 4, 5, 0, 0, 2, 6, 7, 8, 9,
- 10, 11, 12, 13, 14, 15,
+ 10, 11, 12, 13, 14, 15, 16,
},
{
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -540,6 +544,7 @@ func NewLexSpec() *lexSpec {
KindIDLabelMarker,
KindIDExpansion,
KindIDDirectiveMarker,
+ KindIDOrderedSymbolMarker,
KindIDLParen,
KindIDRParen,
},
@@ -573,6 +578,7 @@ func NewLexSpec() *lexSpec {
KindNameLabelMarker,
KindNameExpansion,
KindNameDirectiveMarker,
+ KindNameOrderedSymbolMarker,
KindNameLParen,
KindNameRParen,
KindNamePattern,
@@ -593,7 +599,7 @@ func NewLexSpec() *lexSpec {
{
0, 1, 2, 3, 4, 5, 6, 7, 6, 8, 6, 9, 6, 10, 6, 11, 12, 6, 13, 14,
6, 15, 16, 6, 17, 18, 19, 20, 21, 22, 23, 24, 24, 25, 26, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0,
},
{
0, 1, 2, 3, 2, 4, 2, 5, 2, 6, 2, 7, 8, 2, 9, 10, 2, 11, 12, 2,
@@ -639,7 +645,7 @@ func NewLexSpec() *lexSpec {
5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
5, 5, 5, 5, 5, 1, 1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
- -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, 1, 1, -1, -1, -1, 1, 1, 1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, 1, 1, 1, -1, -1, 1, 1, 1, -1, -1,
-1, -1, 1, 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, 1, -1, -1, -1, -1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, 1, -1, 1, 1, 1, 1, 1, 1, 1,
@@ -984,7 +990,7 @@ func NewLexSpec() *lexSpec {
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
6, 6, 6, 6, 7, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 11, 13, 13,
15, 18, 18, 18, 21, 2, 35, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 36, 43, 0, 0, 0, 37, 44, 45, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 36, 43, 44, 0, 0, 37, 45, 46, 0, 0,
0, 0, 33, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 40, 0, 0, 0, 0,
41, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32,
32, 32, 32, 32, 32, 32, 32, 0, 0, 0, 0, 32, 0, 32, 32, 32, 32, 32, 24, 32,
generated by cgit v1.2.3 (git 2.49.0) at 2025-07-13 15:59:48 +0000