diff options
Diffstat (limited to 'spec')
-rw-r--r-- | spec/lexer.go | 25 | ||||
-rw-r--r-- | spec/lexer_test.go | 14 | ||||
-rw-r--r-- | spec/lexspec.json | 4 | ||||
-rw-r--r-- | spec/parser.go | 35 | ||||
-rw-r--r-- | spec/parser_test.go | 62 | ||||
-rw-r--r-- | spec/syntax_error.go | 1 | ||||
-rw-r--r-- | spec/vartan_lexer.go | 64 |
7 files changed, 56 insertions, 149 deletions
diff --git a/spec/lexer.go b/spec/lexer.go index 51791be..6e9279d 100644 --- a/spec/lexer.go +++ b/spec/lexer.go @@ -7,7 +7,6 @@ import ( _ "embed" "fmt" "io" - "strconv" "strings" verr "github.com/nihei9/vartan/error" @@ -25,7 +24,6 @@ const ( tokenKindSemicolon = tokenKind(";") tokenKindLabelMarker = tokenKind("@") tokenKindDirectiveMarker = tokenKind("#") - tokenKindPosition = tokenKind("$") tokenKindExpantion = tokenKind("...") tokenKindMetaDataMarker = tokenKind("%") tokenKindNewline = tokenKind("newline") @@ -48,7 +46,6 @@ func newPosition(row, col int) Position { type token struct { kind tokenKind text string - num int pos Position } @@ -83,14 +80,6 @@ func newStringLiteralToken(text string, pos Position) *token { } } -func newPositionToken(num int, pos Position) *token { - return &token{ - kind: tokenKindPosition, - num: num, - pos: pos, - } -} - func newEOFToken() *token { return &token{ kind: tokenKindEOF, @@ -274,20 +263,6 @@ func (l *lexer) lexAndSkipWSs() (*token, error) { return newSymbolToken(tokenKindLabelMarker, newPosition(tok.Row+1, tok.Col+1)), nil case KindIDDirectiveMarker: return newSymbolToken(tokenKindDirectiveMarker, newPosition(tok.Row+1, tok.Col+1)), nil - case KindIDPosition: - // Remove '$' character and convert to an integer. - num, err := strconv.Atoi(string(tok.Lexeme)[1:]) - if err != nil { - return nil, err - } - if num == 0 { - return nil, &verr.SpecError{ - Cause: synErrZeroPos, - Row: tok.Row + 1, - Col: tok.Col + 1, - } - } - return newPositionToken(num, newPosition(tok.Row+1, tok.Col+1)), nil case KindIDExpansion: return newSymbolToken(tokenKindExpantion, newPosition(tok.Row+1, tok.Col+1)), nil case KindIDMetadataMarker: diff --git a/spec/lexer_test.go b/spec/lexer_test.go index 5588d0b..4fab8db 100644 --- a/spec/lexer_test.go +++ b/spec/lexer_test.go @@ -24,10 +24,6 @@ func TestLexer_Run(t *testing.T) { return newSymbolToken(kind, newPosition(1, 0)) } - posTok := func(num int) *token { - return newPositionToken(num, newPosition(1, 0)) - } - invalidTok := func(text string) *token { return newInvalidToken(text, newPosition(1, 0)) } @@ -40,7 +36,7 @@ func TestLexer_Run(t *testing.T) { }{ { caption: "the lexer can recognize all kinds of tokens", - src: `id"terminal"'string':|;@$1...#%`, + src: `id"terminal"'string':|;@...#%`, tokens: []*token{ idTok("id"), termPatTok("terminal"), @@ -49,7 +45,6 @@ func TestLexer_Run(t *testing.T) { symTok(tokenKindOr), symTok(tokenKindSemicolon), symTok(tokenKindLabelMarker), - posTok(1), symTok(tokenKindExpantion), symTok(tokenKindDirectiveMarker), symTok(tokenKindMetaDataMarker), @@ -148,11 +143,6 @@ bar // This is the fourth comment. err: synErrIncompletedEscSeq, }, { - caption: "a position must be greater than or equal to 1", - src: `$0`, - err: synErrZeroPos, - }, - { caption: "the lexer can recognize valid tokens following an invalid token", src: `abc!!!def`, tokens: []*token{ @@ -213,7 +203,7 @@ bar // This is the fourth comment. func testToken(t *testing.T, tok, expected *token) { t.Helper() - if tok.kind != expected.kind || tok.text != expected.text || tok.num != expected.num { + if tok.kind != expected.kind || tok.text != expected.text { t.Fatalf("unexpected token; want: %+v, got: %+v", expected, tok) } } diff --git a/spec/lexspec.json b/spec/lexspec.json index 838763b..ff8ff0d 100644 --- a/spec/lexspec.json +++ b/spec/lexspec.json @@ -110,10 +110,6 @@ "pattern": "@" }, { - "kind": "position", - "pattern": "$(0|[1-9][0-9]*)" - }, - { "kind": "expansion", "pattern": "\\.\\.\\." }, diff --git a/spec/parser.go b/spec/parser.go index a9e4d62..e092f59 100644 --- a/spec/parser.go +++ b/spec/parser.go @@ -55,14 +55,8 @@ type DirectiveNode struct { } type ParameterNode struct { - ID string - String string - SymbolPosition *SymbolPositionNode - Pos Position -} - -type SymbolPositionNode struct { - Position int + ID string + String string Expansion bool Pos Position } @@ -453,32 +447,25 @@ func (p *parser) parseDirective() *DirectiveNode { } func (p *parser) parseParameter() *ParameterNode { + var param *ParameterNode switch { case p.consume(tokenKindID): - return &ParameterNode{ + param = &ParameterNode{ ID: p.lastTok.text, Pos: p.lastTok.pos, } case p.consume(tokenKindStringLiteral): - return &ParameterNode{ + param = &ParameterNode{ String: p.lastTok.text, Pos: p.lastTok.pos, } - case p.consume(tokenKindPosition): - symPos := &SymbolPositionNode{ - Position: p.lastTok.num, - Pos: p.lastTok.pos, - } - if p.consume(tokenKindExpantion) { - symPos.Expansion = true - } - return &ParameterNode{ - SymbolPosition: symPos, - Pos: symPos.Pos, - } + default: + return nil } - - return nil + if p.consume(tokenKindExpantion) { + param.Expansion = true + } + return param } func (p *parser) consume(expected tokenKind) bool { diff --git a/spec/parser_test.go b/spec/parser_test.go index e81f95e..3cc772d 100644 --- a/spec/parser_test.go +++ b/spec/parser_test.go @@ -62,20 +62,9 @@ func TestParse(t *testing.T) { ID: id, } } - symPosParam := func(symPos *SymbolPositionNode) *ParameterNode { - return &ParameterNode{ - SymbolPosition: symPos, - } - } - symPos := func(symPos int, exp bool) *SymbolPositionNode { - return &SymbolPositionNode{ - Position: symPos, - Expansion: exp, - } - } - withSymPosPos := func(symPos *SymbolPositionNode, pos Position) *SymbolPositionNode { - symPos.Pos = pos - return symPos + exp := func(param *ParameterNode) *ParameterNode { + param.Expansion = true + return param } withParamPos := func(param *ParameterNode, pos Position) *ParameterNode { param.Pos = pos @@ -393,14 +382,14 @@ s: foo; foo: "foo"; synErr: synErrSemicolonNoNewline, }, { - caption: "a grammar can contain 'ast' directives", + caption: "a grammar can contain 'ast' directives and expansion operator", src: ` s - : foo bar_list #ast $1 $2 + : foo bar_list #ast foo bar_list ; bar_list - : bar_list bar #ast $1... $2 - | bar #ast $1 + : bar_list bar #ast bar_list... bar + | bar #ast bar ; foo: "foo"; bar: "bar"; @@ -410,17 +399,17 @@ bar: "bar"; prod("s", withAltDir( alt(id("foo"), id("bar_list")), - dir("ast", symPosParam(symPos(1, false)), symPosParam(symPos(2, false))), + dir("ast", idParam("foo"), idParam("bar_list")), ), ), prod("bar_list", withAltDir( alt(id("bar_list"), id("bar")), - dir("ast", symPosParam(symPos(1, true)), symPosParam(symPos(2, false))), + dir("ast", exp(idParam("bar_list")), idParam("bar")), ), withAltDir( alt(id("bar")), - dir("ast", symPosParam(symPos(1, false))), + dir("ast", idParam("bar")), ), ), }, @@ -439,7 +428,7 @@ bar: "bar"; src: ` #mode default exp - : exp "\+" id #ast $1 $2 + : exp "\+" id #ast exp id | id ; whitespace: "\u{0020}+" #skip; @@ -462,18 +451,8 @@ fragment number: "[0-9]"; ), withDirPos( dir("ast", - withParamPos( - symPosParam( - withSymPosPos(symPos(1, false), newPos(4)), - ), - newPos(4), - ), - withParamPos( - symPosParam( - withSymPosPos(symPos(2, false), newPos(4)), - ), - newPos(4), - ), + withParamPos(idParam("exp"), newPos(4)), + withParamPos(idParam("id"), newPos(4)), ), newPos(4), ), @@ -861,19 +840,8 @@ func testParameter(t *testing.T, param, expected *ParameterNode, checkPosition b if param.String != expected.String { t.Fatalf("unexpected string parameter; want: %v, got: %v", expected.ID, param.ID) } - if expected.SymbolPosition == nil && param.SymbolPosition != nil { - t.Fatalf("unexpected symbol position parameter; want: nil, got: %+v", param.SymbolPosition) - } - if expected.SymbolPosition != nil { - if param.SymbolPosition == nil { - t.Fatalf("unexpected symbol position parameter; want: %+v, got: nil", expected.SymbolPosition) - } - if param.SymbolPosition.Position != expected.SymbolPosition.Position { - t.Fatalf("unexpected symbol position; want: %v, got: %v", expected.SymbolPosition.Position, param.SymbolPosition.Position) - } - if checkPosition { - testPosition(t, param.Pos, expected.Pos) - } + if param.Expansion != expected.Expansion { + t.Fatalf("unexpected expansion; want: %v, got: %v", expected.Expansion, param.Expansion) } if checkPosition { testPosition(t, param.Pos, expected.Pos) diff --git a/spec/syntax_error.go b/spec/syntax_error.go index 741d578..92348db 100644 --- a/spec/syntax_error.go +++ b/spec/syntax_error.go @@ -22,7 +22,6 @@ var ( synErrIncompletedEscSeq = newSyntaxError("incompleted escape sequence; unexpected EOF following a backslash") synErrEmptyPattern = newSyntaxError("a pattern must include at least one character") synErrEmptyString = newSyntaxError("a string must include at least one character") - synErrZeroPos = newSyntaxError("a position must be greater than or equal to 1") // syntax errors synErrInvalidToken = newSyntaxError("invalid token") diff --git a/spec/vartan_lexer.go b/spec/vartan_lexer.go index 3042243..8c9e7ad 100644 --- a/spec/vartan_lexer.go +++ b/spec/vartan_lexer.go @@ -354,17 +354,16 @@ const ( KindIDOr KindID = 9 KindIDSemicolon KindID = 10 KindIDLabelMarker KindID = 11 - KindIDPosition KindID = 12 - KindIDExpansion KindID = 13 - KindIDDirectiveMarker KindID = 14 - KindIDMetadataMarker KindID = 15 - KindIDPattern KindID = 16 - KindIDTerminalClose KindID = 17 - KindIDEscapeSymbol KindID = 18 - KindIDCharSeq KindID = 19 - KindIDEscapedQuot KindID = 20 - KindIDEscapedBackSlash KindID = 21 - KindIDStringLiteralClose KindID = 22 + KindIDExpansion KindID = 12 + KindIDDirectiveMarker KindID = 13 + KindIDMetadataMarker KindID = 14 + KindIDPattern KindID = 15 + KindIDTerminalClose KindID = 16 + KindIDEscapeSymbol KindID = 17 + KindIDCharSeq KindID = 18 + KindIDEscapedQuot KindID = 19 + KindIDEscapedBackSlash KindID = 20 + KindIDStringLiteralClose KindID = 21 ) const ( @@ -380,7 +379,6 @@ const ( KindNameOr = "or" KindNameSemicolon = "semicolon" KindNameLabelMarker = "label_marker" - KindNamePosition = "position" KindNameExpansion = "expansion" KindNameDirectiveMarker = "directive_marker" KindNameMetadataMarker = "metadata_marker" @@ -420,8 +418,6 @@ func KindIDToName(id KindID) string { return KindNameSemicolon case KindIDLabelMarker: return KindNameLabelMarker - case KindIDPosition: - return KindNamePosition case KindIDExpansion: return KindNameExpansion case KindIDDirectiveMarker: @@ -471,7 +467,7 @@ func NewLexSpec() *lexSpec { pop: [][]bool{ nil, { - false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, + false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, }, { false, false, true, false, @@ -483,7 +479,7 @@ func NewLexSpec() *lexSpec { push: [][]ModeID{ nil, { - 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, }, { 0, 0, 0, 0, @@ -508,8 +504,8 @@ func NewLexSpec() *lexSpec { nil, { 0, 0, 1, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 4, 5, 0, 12, 0, 0, 2, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 15, + 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 4, 5, 0, 0, 2, 6, 7, 8, 9, + 10, 11, 12, 13, 14, }, { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -538,7 +534,6 @@ func NewLexSpec() *lexSpec { KindIDOr, KindIDSemicolon, KindIDLabelMarker, - KindIDPosition, KindIDExpansion, KindIDDirectiveMarker, KindIDMetadataMarker, @@ -571,7 +566,6 @@ func NewLexSpec() *lexSpec { KindNameOr, KindNameSemicolon, KindNameLabelMarker, - KindNamePosition, KindNameExpansion, KindNameDirectiveMarker, KindNameMetadataMarker, @@ -592,8 +586,8 @@ func NewLexSpec() *lexSpec { nil, { 0, 1, 2, 3, 4, 5, 6, 7, 6, 8, 6, 9, 6, 10, 6, 11, 12, 6, 13, 14, - 6, 15, 16, 6, 17, 18, 19, 20, 21, 22, 23, 24, 24, 25, 26, 27, 28, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, + 6, 15, 16, 6, 17, 18, 19, 20, 21, 22, 23, 24, 24, 25, 26, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, }, { 0, 1, 2, 3, 2, 4, 2, 5, 2, 6, 2, 7, 8, 2, 9, 10, 2, 11, 12, 2, @@ -610,8 +604,8 @@ func NewLexSpec() *lexSpec { rowDisplacements: [][]int{ nil, { - 0, 236, 1574, 1575, 1576, 0, 237, 1323, 301, 1387, 365, 1291, 429, 493, 557, 1419, 621, 765, 840, 915, - 990, 1065, 1140, 1215, 1290, 1515, 1525, 1578, 1579, + 0, 236, 1554, 1555, 1556, 0, 237, 1323, 301, 1387, 365, 1291, 429, 493, 557, 1419, 621, 765, 840, 915, + 990, 1065, 1140, 1215, 1290, 1558, 1559, }, { 0, 0, 736, 2548, 852, 2612, 916, 2372, 980, 1044, 1108, 2839, 1172, 245, 2613, 1236, 2677, 1300, 2420, 1364, @@ -639,7 +633,7 @@ func NewLexSpec() *lexSpec { 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 1, 1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, 1, 1, 1, 1, -1, 1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, 1, 1, -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, 1, 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, 1, -1, -1, -1, -1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, 1, -1, 1, 1, 1, 1, 1, 1, 1, @@ -704,10 +698,9 @@ func NewLexSpec() *lexSpec { 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, - 15, 15, 15, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 2, -1, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 15, 15, 15, 2, -1, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 2, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 4, 27, 28, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 4, 25, 26, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, @@ -984,13 +977,13 @@ func NewLexSpec() *lexSpec { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 11, 13, 13, - 15, 18, 18, 18, 21, 2, 37, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 38, 46, 33, 47, 0, 39, 0, 0, 0, 0, - 0, 0, 35, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 42, 0, 0, 0, 0, - 43, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 15, 18, 18, 18, 21, 2, 35, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 36, 43, 0, 44, 0, 37, 0, 0, 0, 0, + 0, 0, 33, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 40, 0, 0, 0, 0, + 41, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 0, 0, 0, 0, 32, 0, 32, 32, 32, 32, 32, 24, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 0, - 41, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 39, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, @@ -1050,10 +1043,9 @@ func NewLexSpec() *lexSpec { 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, - 22, 22, 22, 44, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 2, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 22, 22, 22, 2, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 5, 36, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 5, 34, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |