diff options
author | Ryo Nihei <nihei.dev@gmail.com> | 2022-05-09 00:36:06 +0900 |
---|---|---|
committer | Ryo Nihei <nihei.dev@gmail.com> | 2022-05-10 23:14:52 +0900 |
commit | dd5fd3372cdb53e7a3a36b5ef61b0b0c35023798 (patch) | |
tree | e29796e3c0aee95e443aeabe6b24e2ed4c81dac0 | |
parent | Add #assign directive (diff) | |
download | urubu-dd5fd3372cdb53e7a3a36b5ef61b0b0c35023798.tar.gz urubu-dd5fd3372cdb53e7a3a36b5ef61b0b0c35023798.tar.xz |
Add ordered symbol notation
-rw-r--r-- | README.md | 17 | ||||
-rw-r--r-- | driver/parser_test.go | 14 | ||||
-rw-r--r-- | grammar/grammar.go | 266 | ||||
-rw-r--r-- | grammar/grammar_test.go | 825 | ||||
-rw-r--r-- | grammar/semantic_error.go | 1 | ||||
-rw-r--r-- | spec/lexer.go | 33 | ||||
-rw-r--r-- | spec/lexer_test.go | 3 | ||||
-rw-r--r-- | spec/lexspec.json | 4 | ||||
-rw-r--r-- | spec/parser.go | 21 | ||||
-rw-r--r-- | spec/parser_test.go | 41 | ||||
-rw-r--r-- | spec/syntax_error.go | 1 | ||||
-rw-r--r-- | spec/vartan_lexer.go | 110 |
12 files changed, 1120 insertions, 216 deletions
@@ -28,8 +28,8 @@ vartan uses BNF-like DSL to define your grammar. As an example, let's write a gr #name expr; #prec ( - #left mul div - #left add sub + #left mul div + #left add sub ); expr @@ -505,7 +505,9 @@ If you want to change precedence, `#assign` directive helps you. `#assign` direc When the right-most terminal symbol of an alternative has precedence or associativity defined explicitly, the alternative inherits its precedence and associativity. -`#prec` directive assigns the same precedence as a specified symbol to an alternative. +`#prec` directive assigns the same precedence as a specified symbol to an alternative and disables associativity. + +You can define an ordered symbol with the form `$<ID>`. The ordered symbol is an identifier having only precedence, and you can use it in `#prec` directive applied to an alternative. The ordered symbol helps you to resolve shift/reduce conflicts without terminal symbol definitions. The grammar for simple four arithmetic operations and assignment expression can be defined as follows: @@ -513,9 +515,10 @@ The grammar for simple four arithmetic operations and assignment expression can #name example; #prec ( - #left mul div - #left add sub - #right assign + #assign $uminus + #left mul div + #left add sub + #right assign ); expr @@ -525,7 +528,7 @@ expr | expr div expr | id assign expr | int - | sub int #prec mul // This `sub` means a unary minus symbol. + | sub int #prec $uminus // This `sub` means a unary minus symbol. | id ; diff --git a/driver/parser_test.go b/driver/parser_test.go index c37d268..60dd3f4 100644 --- a/driver/parser_test.go +++ b/driver/parser_test.go @@ -168,6 +168,7 @@ bar_text: "bar"; #name test; #prec ( + #assign $uminus #left mul div #left add sub ); @@ -178,7 +179,7 @@ expr | expr mul expr | expr div expr | int - | sub int #prec mul #ast int sub // This 'sub' means the unary minus symbol. + | sub int #prec $uminus // This 'sub' means the unary minus symbol. ; int @@ -197,13 +198,13 @@ div nonTermNode("expr", nonTermNode("expr", nonTermNode("expr", - termNode("int", "1"), termNode("sub", "-"), + termNode("int", "1"), ), termNode("mul", "*"), nonTermNode("expr", - termNode("int", "2"), termNode("sub", "-"), + termNode("int", "2"), ), ), termNode("add", "+"), @@ -553,6 +554,7 @@ add #name test; #prec ( + #assign $uminus #left mul div #left add sub ); @@ -562,8 +564,8 @@ expr | expr sub expr | expr mul expr | expr div expr - | sub expr #prec mul // This 'sub' means a unary minus symbol. | int + | sub int #prec $uminus // This 'sub' means a unary minus symbol. ; ws #skip @@ -593,9 +595,7 @@ div nonTermNode("expr", nonTermNode("expr", termNode("sub", "-"), - nonTermNode("expr", - termNode("int", "1"), - ), + termNode("int", "1"), ), termNode("mul", "*"), nonTermNode("expr", diff --git a/grammar/grammar.go b/grammar/grammar.go index da1c77c..a5d6cb7 100644 --- a/grammar/grammar.go +++ b/grammar/grammar.go @@ -150,7 +150,7 @@ func (b *GrammarBuilder) Build() (*Grammar, error) { return nil, b.errs } - pa, err := b.genPrecAndAssoc(symTabAndLexSpec.symTab, prodsAndActs.prods, prodsAndActs.prodPrecs, prodsAndActs.prodPrecPoss) + pa, err := b.genPrecAndAssoc(symTabAndLexSpec.symTab, prodsAndActs) if err != nil { return nil, err } @@ -597,12 +597,13 @@ func genLexEntry(prod *spec.ProductionNode) (*mlspec.LexEntry, bool, string, *ve } type productionsAndActions struct { - prods *productionSet - augStartSym symbol - astActs map[productionID][]*astActionEntry - prodPrecs map[productionID]symbol - prodPrecPoss map[productionID]*spec.Position - recoverProds map[productionID]struct{} + prods *productionSet + augStartSym symbol + astActs map[productionID][]*astActionEntry + prodPrecsTerm map[productionID]symbol + prodPrecsOrdSym map[productionID]string + prodPrecPoss map[productionID]*spec.Position + recoverProds map[productionID]struct{} } func (b *GrammarBuilder) genProductionsAndActions(root *spec.RootNode, symTabAndLexSpec *symbolTableAndLexSpec) (*productionsAndActions, error) { @@ -620,7 +621,8 @@ func (b *GrammarBuilder) genProductionsAndActions(root *spec.RootNode, symTabAnd prods := newProductionSet() var augStartSym symbol astActs := map[productionID][]*astActionEntry{} - prodPrecs := map[productionID]symbol{} + prodPrecsTerm := map[productionID]symbol{} + prodPrecsOrdSym := map[productionID]string{} prodPrecPoss := map[productionID]*spec.Position{} recoverProds := map[productionID]struct{}{} @@ -918,36 +920,42 @@ func (b *GrammarBuilder) genProductionsAndActions(root *spec.RootNode, symTabAnd } astActs[p.id] = astAct case "prec": - if len(dir.Parameters) != 1 || dir.Parameters[0].ID == "" { + if len(dir.Parameters) != 1 || (dir.Parameters[0].ID == "" && dir.Parameters[0].OrderedSymbol == "") { b.errs = append(b.errs, &verr.SpecError{ Cause: semErrDirInvalidParam, - Detail: "'prec' directive needs an ID parameter", + Detail: "'prec' directive needs just one ID parameter or ordered symbol", Row: dir.Pos.Row, Col: dir.Pos.Col, }) continue LOOP_RHS } - sym, ok := symTab.toSymbol(dir.Parameters[0].ID) - if !ok { - b.errs = append(b.errs, &verr.SpecError{ - Cause: semErrDirInvalidParam, - Detail: fmt.Sprintf("unknown terminal symbol: %v", dir.Parameters[0].ID), - Row: dir.Pos.Row, - Col: dir.Pos.Col, - }) - continue LOOP_RHS - } - if !sym.isTerminal() { - b.errs = append(b.errs, &verr.SpecError{ - Cause: semErrDirInvalidParam, - Detail: fmt.Sprintf("the symbol must be a terminal: %v", dir.Parameters[0].ID), - Row: dir.Pos.Row, - Col: dir.Pos.Col, - }) - continue LOOP_RHS + switch { + case dir.Parameters[0].ID != "": + sym, ok := symTab.toSymbol(dir.Parameters[0].ID) + if !ok { + b.errs = append(b.errs, &verr.SpecError{ + Cause: semErrDirInvalidParam, + Detail: fmt.Sprintf("unknown terminal symbol: %v", dir.Parameters[0].ID), + Row: dir.Pos.Row, + Col: dir.Pos.Col, + }) + continue LOOP_RHS + } + if !sym.isTerminal() { + b.errs = append(b.errs, &verr.SpecError{ + Cause: semErrDirInvalidParam, + Detail: fmt.Sprintf("the symbol must be a terminal: %v", dir.Parameters[0].ID), + Row: dir.Pos.Row, + Col: dir.Pos.Col, + }) + continue LOOP_RHS + } + prodPrecsTerm[p.id] = sym + prodPrecPoss[p.id] = &dir.Parameters[0].Pos + case dir.Parameters[0].OrderedSymbol != "": + prodPrecsOrdSym[p.id] = dir.Parameters[0].OrderedSymbol + prodPrecPoss[p.id] = &dir.Parameters[0].Pos } - prodPrecs[p.id] = sym - prodPrecPoss[p.id] = &dir.Parameters[0].Pos case "recover": if len(dir.Parameters) > 0 { b.errs = append(b.errs, &verr.SpecError{ @@ -973,18 +981,20 @@ func (b *GrammarBuilder) genProductionsAndActions(root *spec.RootNode, symTabAnd } return &productionsAndActions{ - prods: prods, - augStartSym: augStartSym, - astActs: astActs, - prodPrecs: prodPrecs, - prodPrecPoss: prodPrecPoss, - recoverProds: recoverProds, + prods: prods, + augStartSym: augStartSym, + astActs: astActs, + prodPrecsTerm: prodPrecsTerm, + prodPrecsOrdSym: prodPrecsOrdSym, + prodPrecPoss: prodPrecPoss, + recoverProds: recoverProds, }, nil } -func (b *GrammarBuilder) genPrecAndAssoc(symTab *symbolTable, prods *productionSet, prodPrecs map[productionID]symbol, prodPrecPoss map[productionID]*spec.Position) (*precAndAssoc, error) { +func (b *GrammarBuilder) genPrecAndAssoc(symTab *symbolTable, prodsAndActs *productionsAndActions) (*precAndAssoc, error) { termPrec := map[symbolNum]int{} termAssoc := map[symbolNum]assocType{} + ordSymPrec := map[string]int{} { var precGroup []*spec.DirectiveNode for _, dir := range b.AST.Directives { @@ -1004,9 +1014,10 @@ func (b *GrammarBuilder) genPrecAndAssoc(symTab *symbolTable, prods *productionS if dir.Name != "name" && dir.Name != "prec" { b.errs = append(b.errs, &verr.SpecError{ - Cause: semErrDirInvalidName, - Row: dir.Pos.Row, - Col: dir.Pos.Col, + Cause: semErrDirInvalidName, + Detail: dir.Name, + Row: dir.Pos.Row, + Col: dir.Pos.Col, }) continue } @@ -1024,9 +1035,10 @@ func (b *GrammarBuilder) genPrecAndAssoc(symTab *symbolTable, prods *productionS assocTy = assocTypeNil default: b.errs = append(b.errs, &verr.SpecError{ - Cause: semErrDirInvalidName, - Row: dir.Pos.Row, - Col: dir.Pos.Col, + Cause: semErrDirInvalidName, + Detail: dir.Name, + Row: dir.Pos.Row, + Col: dir.Pos.Col, }) return nil, nil } @@ -1042,63 +1054,85 @@ func (b *GrammarBuilder) genPrecAndAssoc(symTab *symbolTable, prods *productionS } ASSOC_PARAM_LOOP: for _, p := range dir.Parameters { - if p.ID == "" { - b.errs = append(b.errs, &verr.SpecError{ - Cause: semErrDirInvalidParam, - Detail: "a parameter must be an ID", - Row: p.Pos.Row, - Col: p.Pos.Col, - }) - return nil, nil - } - - sym, ok := symTab.toSymbol(p.ID) - if !ok { - b.errs = append(b.errs, &verr.SpecError{ - Cause: semErrDirInvalidParam, - Detail: fmt.Sprintf("'%v' is undefined", p.ID), - Row: p.Pos.Row, - Col: p.Pos.Col, - }) - return nil, nil - } - if !sym.isTerminal() { - b.errs = append(b.errs, &verr.SpecError{ - Cause: semErrDirInvalidParam, - Detail: fmt.Sprintf("associativity can take only terminal symbol ('%v' is a non-terminal)", p.ID), - Row: p.Pos.Row, - Col: p.Pos.Col, - }) - return nil, nil - } - if prec, alreadySet := termPrec[sym.num()]; alreadySet { - if prec == precN { - b.errs = append(b.errs, &verr.SpecError{ - Cause: semErrDuplicateAssoc, - Detail: fmt.Sprintf("'%v' already has the same associativity and precedence", p.ID), - Row: p.Pos.Row, - Col: p.Pos.Col, - }) - } else if assoc := termAssoc[sym.num()]; assoc == assocTy { + switch { + case p.ID != "": + sym, ok := symTab.toSymbol(p.ID) + if !ok { b.errs = append(b.errs, &verr.SpecError{ - Cause: semErrDuplicateAssoc, - Detail: fmt.Sprintf("'%v' already has different precedence", p.ID), + Cause: semErrDirInvalidParam, + Detail: fmt.Sprintf("'%v' is undefined", p.ID), Row: p.Pos.Row, Col: p.Pos.Col, }) - } else { + return nil, nil + } + if !sym.isTerminal() { b.errs = append(b.errs, &verr.SpecError{ - Cause: semErrDuplicateAssoc, - Detail: fmt.Sprintf("'%v' already has different associativity and precedence", p.ID), + Cause: semErrDirInvalidParam, + Detail: fmt.Sprintf("associativity can take only terminal symbol ('%v' is a non-terminal)", p.ID), Row: p.Pos.Row, Col: p.Pos.Col, }) + return nil, nil + } + if prec, alreadySet := termPrec[sym.num()]; alreadySet { + if prec == precN { + b.errs = append(b.errs, &verr.SpecError{ + Cause: semErrDuplicateAssoc, + Detail: fmt.Sprintf("'%v' already has the same associativity and precedence", p.ID), + Row: p.Pos.Row, + Col: p.Pos.Col, + }) + } else if assoc := termAssoc[sym.num()]; assoc == assocTy { + b.errs = append(b.errs, &verr.SpecError{ + Cause: semErrDuplicateAssoc, + Detail: fmt.Sprintf("'%v' already has different precedence", p.ID), + Row: p.Pos.Row, + Col: p.Pos.Col, + }) + } else { + b.errs = append(b.errs, &verr.SpecError{ + Cause: semErrDuplicateAssoc, + Detail: fmt.Sprintf("'%v' already has different associativity and precedence", p.ID), + Row: p.Pos.Row, + Col: p.Pos.Col, + }) + } + break ASSOC_PARAM_LOOP + } + + termPrec[sym.num()] = precN + termAssoc[sym.num()] = assocTy + case p.OrderedSymbol != "": + if prec, alreadySet := ordSymPrec[p.OrderedSymbol]; alreadySet { + if prec == precN { + b.errs = append(b.errs, &verr.SpecError{ + Cause: semErrDuplicateAssoc, + Detail: fmt.Sprintf("'$%v' already has the same precedence", p.OrderedSymbol), + Row: p.Pos.Row, + Col: p.Pos.Col, + }) + } else { + b.errs = append(b.errs, &verr.SpecError{ + Cause: semErrDuplicateAssoc, + Detail: fmt.Sprintf("'$%v' already has different precedence", p.OrderedSymbol), + Row: p.Pos.Row, + Col: p.Pos.Col, + }) + } + break ASSOC_PARAM_LOOP } - break ASSOC_PARAM_LOOP - } - termPrec[sym.num()] = precN - termAssoc[sym.num()] = assocTy + ordSymPrec[p.OrderedSymbol] = precN + default: + b.errs = append(b.errs, &verr.SpecError{ + Cause: semErrDirInvalidParam, + Detail: "a parameter must be an ID or an ordered symbol", + Row: p.Pos.Row, + Col: p.Pos.Col, + }) + return nil, nil + } } precN++ @@ -1110,36 +1144,46 @@ func (b *GrammarBuilder) genPrecAndAssoc(symTab *symbolTable, prods *productionS prodPrec := map[productionNum]int{} prodAssoc := map[productionNum]assocType{} - for _, prod := range prods.getAllProductions() { - mostrightTerm := symbolNil - for _, sym := range prod.rhs { - if !sym.isTerminal() { - continue - } - mostrightTerm = sym - } - if !mostrightTerm.isNil() { - if prec, ok := termPrec[mostrightTerm.num()]; ok { - prodPrec[prod.num] = prec - } - if assoc, ok := termAssoc[mostrightTerm.num()]; ok { - prodAssoc[prod.num] = assoc - } - } - - // #prec directive changes only precedence, not associativity. - if term, ok := prodPrecs[prod.id]; ok { + for _, prod := range prodsAndActs.prods.getAllProductions() { + // A #prec directive changes only precedence, not associativity. + if term, ok := prodsAndActs.prodPrecsTerm[prod.id]; ok { if prec, ok := termPrec[term.num()]; ok { prodPrec[prod.num] = prec + prodAssoc[prod.num] = assocTypeNil } else { text, _ := symTab.toText(term) b.errs = append(b.errs, &verr.SpecError{ Cause: semErrUndefinedPrec, Detail: text, - Row: prodPrecPoss[prod.id].Row, - Col: prodPrecPoss[prod.id].Col, + Row: prodsAndActs.prodPrecPoss[prod.id].Row, + Col: prodsAndActs.prodPrecPoss[prod.id].Col, }) } + } else if ordSym, ok := prodsAndActs.prodPrecsOrdSym[prod.id]; ok { + if prec, ok := ordSymPrec[ordSym]; ok { + prodPrec[prod.num] = prec + prodAssoc[prod.num] = assocTypeNil + } else { + b.errs = append(b.errs, &verr.SpecError{ + Cause: semErrUndefinedOrdSym, + Detail: fmt.Sprintf("$%v", ordSym), + Row: prodsAndActs.prodPrecPoss[prod.id].Row, + Col: prodsAndActs.prodPrecPoss[prod.id].Col, + }) + } + } else { + // A production inherits precedence and associativity from the right-most terminal symbol. + mostrightTerm := symbolNil + for _, sym := range prod.rhs { + if !sym.isTerminal() { + continue + } + mostrightTerm = sym + } + if !mostrightTerm.isNil() { + prodPrec[prod.num] = termPrec[mostrightTerm.num()] + prodAssoc[prod.num] = termAssoc[mostrightTerm.num()] + } } } if len(b.errs) > 0 { diff --git a/grammar/grammar_test.go b/grammar/grammar_test.go index d4a361d..62823e1 100644 --- a/grammar/grammar_test.go +++ b/grammar/grammar_test.go @@ -464,6 +464,189 @@ bar }, }, { + caption: "even if a non-terminal symbol apears to a terminal symbol, a production inherits precedence and associativity from the right-most terminal symbol, not from the non-terminal symbol", + specSrc: ` +#name test; + +#prec ( + #left foo + #right bar +); + +s + : foo a // This alternative has the same precedence and associativity as the right-most terminal symbol 'foo', not 'a'. + ; +a + : bar + ; + +foo + : 'foo'; +bar + : 'bar'; +`, + validate: func(t *testing.T, g *Grammar) { + var fooPrec int + var fooAssoc assocType + { + s, _ := g.symbolTable.toSymbol("foo") + fooPrec = g.precAndAssoc.terminalPrecedence(s.num()) + fooAssoc = g.precAndAssoc.terminalAssociativity(s.num()) + } + var barPrec int + var barAssoc assocType + { + s, _ := g.symbolTable.toSymbol("bar") + barPrec = g.precAndAssoc.terminalPrecedence(s.num()) + barAssoc = g.precAndAssoc.terminalAssociativity(s.num()) + } + var aPrec int + var aAssoc assocType + { + s, _ := g.symbolTable.toSymbol("a") + ps, _ := g.productionSet.findByLHS(s) + aPrec = g.precAndAssoc.productionPredence(ps[0].num) + aAssoc = g.precAndAssoc.productionAssociativity(ps[0].num) + } + var sPrec int + var sAssoc assocType + { + s, _ := g.symbolTable.toSymbol("s") + ps, _ := g.productionSet.findByLHS(s) + sPrec = g.precAndAssoc.productionPredence(ps[0].num) + sAssoc = g.precAndAssoc.productionAssociativity(ps[0].num) + } + if fooPrec != 1 || fooAssoc != assocTypeLeft { + t.Fatalf("unexpected terminal precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", 1, assocTypeLeft, fooPrec, fooAssoc) + } + if barPrec != 2 || barAssoc != assocTypeRight { + t.Fatalf("unexpected terminal precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", 2, assocTypeRight, barPrec, barAssoc) + } + if aPrec != barPrec || aAssoc != barAssoc { + t.Fatalf("unexpected production precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", barPrec, barAssoc, aPrec, aAssoc) + } + if sPrec != fooPrec || sAssoc != fooAssoc { + t.Fatalf("unexpected production precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", fooPrec, fooAssoc, sPrec, sAssoc) + } + }, + }, + { + caption: "each alternative in the same production can have its own precedence and associativity", + specSrc: ` +#name test; + +#prec ( + #left foo + #right bar + #assign baz +); + +s + : foo + | bar + | baz + | bra + ; + +foo + : 'foo'; +bar + : 'bar'; +baz + : 'baz'; +bra + : 'bra'; +`, + validate: func(t *testing.T, g *Grammar) { + var alt1Prec int + var alt1Assoc assocType + var alt2Prec int + var alt2Assoc assocType + var alt3Prec int + var alt3Assoc assocType + var alt4Prec int + var alt4Assoc assocType + { + s, _ := g.symbolTable.toSymbol("s") + ps, _ := g.productionSet.findByLHS(s) + alt1Prec = g.precAndAssoc.productionPredence(ps[0].num) + alt1Assoc = g.precAndAssoc.productionAssociativity(ps[0].num) + alt2Prec = g.precAndAssoc.productionPredence(ps[1].num) + alt2Assoc = g.precAndAssoc.productionAssociativity(ps[1].num) + alt3Prec = g.precAndAssoc.productionPredence(ps[2].num) + alt3Assoc = g.precAndAssoc.productionAssociativity(ps[2].num) + alt4Prec = g.precAndAssoc.productionPredence(ps[3].num) + alt4Assoc = g.precAndAssoc.productionAssociativity(ps[3].num) + } + if alt1Prec != 1 || alt1Assoc != assocTypeLeft { + t.Fatalf("unexpected production precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", 1, assocTypeLeft, alt1Prec, alt1Assoc) + } + if alt2Prec != 2 || alt2Assoc != assocTypeRight { + t.Fatalf("unexpected production precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", 2, assocTypeRight, alt2Prec, alt2Assoc) + } + if alt3Prec != 3 || alt3Assoc != assocTypeNil { + t.Fatalf("unexpected production precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", 3, assocTypeNil, alt3Prec, alt3Assoc) + } + if alt4Prec != precNil || alt4Assoc != assocTypeNil { + t.Fatalf("unexpected production precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", precNil, assocTypeNil, alt4Prec, alt4Assoc) + } + }, + }, + { + caption: "when a production contains no terminal symbols, the production will not have precedence and associativiry", + specSrc: ` +#name test; + +#prec ( + #left foo +); + +s + : a + ; +a + : foo + ; + +foo + : 'foo'; +`, + validate: func(t *testing.T, g *Grammar) { + var fooPrec int + var fooAssoc assocType + { + s, _ := g.symbolTable.toSymbol("foo") + fooPrec = g.precAndAssoc.terminalPrecedence(s.num()) + fooAssoc = g.precAndAssoc.terminalAssociativity(s.num()) + } + var aPrec int + var aAssoc assocType + { + s, _ := g.symbolTable.toSymbol("a") + ps, _ := g.productionSet.findByLHS(s) + aPrec = g.precAndAssoc.productionPredence(ps[0].num) + aAssoc = g.precAndAssoc.productionAssociativity(ps[0].num) + } + var sPrec int + var sAssoc assocType + { + s, _ := g.symbolTable.toSymbol("s") + ps, _ := g.productionSet.findByLHS(s) + sPrec = g.precAndAssoc.productionPredence(ps[0].num) + sAssoc = g.precAndAssoc.productionAssociativity(ps[0].num) + } + if fooPrec != 1 || fooAssoc != assocTypeLeft { + t.Fatalf("unexpected terminal precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", 1, assocTypeLeft, fooPrec, fooAssoc) + } + if aPrec != fooPrec || aAssoc != fooAssoc { + t.Fatalf("unexpected production precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", fooPrec, fooAssoc, aPrec, aAssoc) + } + if sPrec != precNil || sAssoc != assocTypeNil { + t.Fatalf("unexpected production precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", precNil, assocTypeNil, sPrec, sAssoc) + } + }, + }, + { caption: "the `#prec` directive applied to an alternative changes only precedence, not associativity", specSrc: ` #name test; @@ -553,8 +736,310 @@ bar if barPrec != 2 || barAssoc != assocTypeRight { t.Fatalf("unexpected terminal precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", 2, assocTypeRight, barPrec, barAssoc) } - if sPrec != fooPrec || sAssoc != barAssoc { - t.Fatalf("unexpected production precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", fooPrec, barAssoc, sPrec, sAssoc) + if sPrec != fooPrec || sAssoc != assocTypeNil { + t.Fatalf("unexpected production precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", fooPrec, assocTypeNil, sPrec, sAssoc) + } + }, + }, + { + caption: "an ordered symbol can appear in a `#left` directive", + specSrc: ` +#name test; + +#prec ( + #left $high + #right foo bar + #left $low +); + +s + : foo #prec $high + | bar #prec $low + ; + +foo + : 'foo'; +bar + : 'bar'; +`, + validate: func(t *testing.T, g *Grammar) { + var fooPrec int + var fooAssoc assocType + { + s, _ := g.symbolTable.toSymbol("foo") + fooPrec = g.precAndAssoc.terminalPrecedence(s.num()) + fooAssoc = g.precAndAssoc.terminalAssociativity(s.num()) + } + if fooPrec != 2 || fooAssoc != assocTypeRight { + t.Fatalf("unexpected terminal precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", 2, assocTypeRight, fooPrec, fooAssoc) + } + var barPrec int + var barAssoc assocType + { + s, _ := g.symbolTable.toSymbol("bar") + barPrec = g.precAndAssoc.terminalPrecedence(s.num()) + barAssoc = g.precAndAssoc.terminalAssociativity(s.num()) + } + if barPrec != 2 || barAssoc != assocTypeRight { + t.Fatalf("unexpected terminal precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", 2, assocTypeRight, barPrec, barAssoc) + } + var alt1Prec int + var alt1Assoc assocType + var alt2Prec int + var alt2Assoc assocType + { + s, _ := g.symbolTable.toSymbol("s") + ps, _ := g.productionSet.findByLHS(s) + alt1Prec = g.precAndAssoc.productionPredence(ps[0].num) + alt1Assoc = g.precAndAssoc.productionAssociativity(ps[0].num) + alt2Prec = g.precAndAssoc.productionPredence(ps[1].num) + alt2Assoc = g.precAndAssoc.productionAssociativity(ps[1].num) + } + if alt1Prec != 1 || alt1Assoc != assocTypeNil { + t.Fatalf("unexpected terminal precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", 1, assocTypeNil, alt1Prec, alt1Assoc) + } + if alt2Prec != 3 || alt2Assoc != assocTypeNil { + t.Fatalf("unexpected terminal precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", 3, assocTypeNil, alt2Prec, alt2Assoc) + } + }, + }, + { + caption: "an ordered symbol can appear in a `#right` directive", + specSrc: ` +#name test; + +#prec ( + #right $high + #left foo bar + #right $low +); + +s + : foo #prec $high + | bar #prec $low + ; + +foo + : 'foo'; +bar + : 'bar'; +`, + validate: func(t *testing.T, g *Grammar) { + var fooPrec int + var fooAssoc assocType + { + s, _ := g.symbolTable.toSymbol("foo") + fooPrec = g.precAndAssoc.terminalPrecedence(s.num()) + fooAssoc = g.precAndAssoc.terminalAssociativity(s.num()) + } + if fooPrec != 2 || fooAssoc != assocTypeLeft { + t.Fatalf("unexpected terminal precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", 2, assocTypeLeft, fooPrec, fooAssoc) + } + var barPrec int + var barAssoc assocType + { + s, _ := g.symbolTable.toSymbol("bar") + barPrec = g.precAndAssoc.terminalPrecedence(s.num()) + barAssoc = g.precAndAssoc.terminalAssociativity(s.num()) + } + if barPrec != 2 || barAssoc != assocTypeLeft { + t.Fatalf("unexpected terminal precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", 2, assocTypeLeft, barPrec, barAssoc) + } + var alt1Prec int + var alt1Assoc assocType + var alt2Prec int + var alt2Assoc assocType + { + s, _ := g.symbolTable.toSymbol("s") + ps, _ := g.productionSet.findByLHS(s) + alt1Prec = g.precAndAssoc.productionPredence(ps[0].num) + alt1Assoc = g.precAndAssoc.productionAssociativity(ps[0].num) + alt2Prec = g.precAndAssoc.productionPredence(ps[1].num) + alt2Assoc = g.precAndAssoc.productionAssociativity(ps[1].num) + } + if alt1Prec != 1 || alt1Assoc != assocTypeNil { + t.Fatalf("unexpected terminal precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", 1, assocTypeNil, alt1Prec, alt1Assoc) + } + if alt2Prec != 3 || alt2Assoc != assocTypeNil { + t.Fatalf("unexpected terminal precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", 3, assocTypeNil, alt2Prec, alt2Assoc) + } + }, + }, + { + caption: "an ordered symbol can appear in a `#assign` directive", + specSrc: ` +#name test; + +#prec ( + #assign $high + #left foo + #right bar + #assign $low +); + +s + : foo #prec $high + | bar #prec $low + ; + +foo + : 'foo'; +bar + : 'bar'; +`, + validate: func(t *testing.T, g *Grammar) { + var fooPrec int + var fooAssoc assocType + { + s, _ := g.symbolTable.toSymbol("foo") + fooPrec = g.precAndAssoc.terminalPrecedence(s.num()) + fooAssoc = g.precAndAssoc.terminalAssociativity(s.num()) + } + if fooPrec != 2 || fooAssoc != assocTypeLeft { + t.Fatalf("unexpected terminal precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", 2, assocTypeLeft, fooPrec, fooAssoc) + } + var barPrec int + var barAssoc assocType + { + s, _ := g.symbolTable.toSymbol("bar") + barPrec = g.precAndAssoc.terminalPrecedence(s.num()) + barAssoc = g.precAndAssoc.terminalAssociativity(s.num()) + } + if barPrec != 3 || barAssoc != assocTypeRight { + t.Fatalf("unexpected terminal precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", 3, assocTypeRight, barPrec, barAssoc) + } + var alt1Prec int + var alt1Assoc assocType + var alt2Prec int + var alt2Assoc assocType + { + s, _ := g.symbolTable.toSymbol("s") + ps, _ := g.productionSet.findByLHS(s) + alt1Prec = g.precAndAssoc.productionPredence(ps[0].num) + alt1Assoc = g.precAndAssoc.productionAssociativity(ps[0].num) + alt2Prec = g.precAndAssoc.productionPredence(ps[1].num) + alt2Assoc = g.precAndAssoc.productionAssociativity(ps[1].num) + } + if alt1Prec != 1 || alt1Assoc != assocTypeNil { + t.Fatalf("unexpected terminal precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", 1, assocTypeNil, alt1Prec, alt1Assoc) + } + if alt2Prec != 4 || alt2Assoc != assocTypeNil { + t.Fatalf("unexpected terminal precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", 4, assocTypeNil, alt2Prec, alt2Assoc) + } + }, + }, + { + caption: "names of an ordered symbol and a terminal symbol can duplicate", + specSrc: ` +#name test; + +#prec ( + #left foo bar + #right $foo +); + +s + : foo + | bar #prec $foo + ; + +foo + : 'foo'; +bar + : 'bar'; +`, + validate: func(t *testing.T, g *Grammar) { + var fooPrec int + var fooAssoc assocType + { + s, _ := g.symbolTable.toSymbol("foo") + fooPrec = g.precAndAssoc.terminalPrecedence(s.num()) + fooAssoc = g.precAndAssoc.terminalAssociativity(s.num()) + } + var barPrec int + var barAssoc assocType + { + s, _ := g.symbolTable.toSymbol("bar") + barPrec = g.precAndAssoc.terminalPrecedence(s.num()) + barAssoc = g.precAndAssoc.terminalAssociativity(s.num()) + } + if fooPrec != 1 || fooAssoc != assocTypeLeft { + t.Fatalf("unexpected terminal precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", 1, assocTypeLeft, fooPrec, fooAssoc) + } + if barPrec != 1 || barAssoc != assocTypeLeft { + t.Fatalf("unexpected terminal precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", 1, assocTypeLeft, barPrec, barAssoc) + } + var alt1Prec int + var alt1Assoc assocType + var alt2Prec int + var alt2Assoc assocType + { + s, _ := g.symbolTable.toSymbol("s") + ps, _ := g.productionSet.findByLHS(s) + alt1Prec = g.precAndAssoc.productionPredence(ps[0].num) + alt1Assoc = g.precAndAssoc.productionAssociativity(ps[0].num) + alt2Prec = g.precAndAssoc.productionPredence(ps[1].num) + alt2Assoc = g.precAndAssoc.productionAssociativity(ps[1].num) + } + if alt1Prec != fooPrec || alt1Assoc != fooAssoc { + t.Fatalf("unexpected terminal precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", fooPrec, fooAssoc, alt1Prec, alt1Assoc) + } + if alt2Prec != 2 || alt2Assoc != assocTypeNil { + t.Fatalf("unexpected terminal precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", 2, assocTypeNil, alt2Prec, alt2Assoc) + } + }, + }, + { + caption: "names of an ordered symbol and a non-terminal symbol can duplicate", + specSrc: ` +#name test; + +#prec ( + #left foo bar + #right $a +); + +s + : a + | bar #prec $a + ; +a + : foo + ; + +foo + : 'foo'; +bar + : 'bar'; +`, + validate: func(t *testing.T, g *Grammar) { + var barPrec int + var barAssoc assocType + { + s, _ := g.symbolTable.toSymbol("bar") + barPrec = g.precAndAssoc.terminalPrecedence(s.num()) + barAssoc = g.precAndAssoc.terminalAssociativity(s.num()) + } + if barPrec != 1 || barAssoc != assocTypeLeft { + t.Fatalf("unexpected terminal precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", 1, assocTypeLeft, barPrec, barAssoc) + } + var alt1Prec int + var alt1Assoc assocType + var alt2Prec int + var alt2Assoc assocType + { + s, _ := g.symbolTable.toSymbol("s") + ps, _ := g.productionSet.findByLHS(s) + alt1Prec = g.precAndAssoc.productionPredence(ps[0].num) + alt1Assoc = g.precAndAssoc.productionAssociativity(ps[0].num) + alt2Prec = g.precAndAssoc.productionPredence(ps[1].num) + alt2Assoc = g.precAndAssoc.productionAssociativity(ps[1].num) + } + if alt1Prec != precNil || alt1Assoc != assocTypeNil { + t.Fatalf("unexpected terminal precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", precNil, assocTypeNil, alt1Prec, alt1Assoc) + } + if alt2Prec != 2 || alt2Assoc != assocTypeNil { + t.Fatalf("unexpected terminal precedence and associativity: want: (prec: %v, assoc: %v), got: (prec: %v, assoc: %v)", 2, assocTypeNil, alt2Prec, alt2Assoc) } }, }, @@ -1001,6 +1486,22 @@ foo errs: []*SemanticError{semErrDirInvalidParam}, }, { + caption: "the `#prec` directive cannot take an ordered symbol parameter", + specSrc: ` +#name test; + +#prec $x; + +s + : foo + ; + +foo + : 'foo'; +`, + errs: []*SemanticError{semErrDirInvalidParam}, + }, + { caption: "the `#prec` directive cannot take a pattern parameter", specSrc: ` #name test; @@ -1160,7 +1661,7 @@ foo errs: []*SemanticError{semErrDirInvalidParam}, }, { - caption: "the `#left` dirctive cannot be specified multiple times for a symbol", + caption: "the `#left` dirctive cannot be specified multiple times for a terminal symbol", specSrc: ` #name test; @@ -1178,7 +1679,25 @@ foo errs: []*SemanticError{semErrDuplicateAssoc}, }, { - caption: "a symbol cannot have different precedence", + caption: "the `#left` dirctive cannot be specified multiple times for an ordered symbol", + specSrc: ` +#name test; + +#prec ( + #left $x $x +); + +s + : foo #prec $x + ; + +foo + : 'foo'; +`, + errs: []*SemanticError{semErrDuplicateAssoc}, + }, + { + caption: "a terminal symbol cannot have different precedence", specSrc: ` #name test; @@ -1197,7 +1716,26 @@ foo errs: []*SemanticError{semErrDuplicateAssoc}, }, { - caption: "a symbol cannot have different associativity", + caption: "an ordered symbol cannot have different precedence", + specSrc: ` +#name test; + +#prec ( + #left $x + #left $x +); + +s + : foo #prec $x + ; + +foo + : 'foo'; +`, + errs: []*SemanticError{semErrDuplicateAssoc}, + }, + { + caption: "a terminal symbol cannot have different associativity", specSrc: ` #name test; @@ -1215,6 +1753,25 @@ foo `, errs: []*SemanticError{semErrDuplicateAssoc}, }, + { + caption: "an ordered symbol cannot have different associativity", + specSrc: ` +#name test; + +#prec ( + #right $x + #left $x +); + +s + : foo #prec $x + ; + +foo + : 'foo'; +`, + errs: []*SemanticError{semErrDuplicateAssoc}, + }, } rightDirTests := []*specErrTest{ @@ -1327,7 +1884,7 @@ foo errs: []*SemanticError{semErrDirInvalidParam}, }, { - caption: "the `#right` directive cannot be specified multiple times for a symbol", + caption: "the `#right` directive cannot be specified multiple times for a terminal symbol", specSrc: ` #name test; @@ -1345,7 +1902,25 @@ foo errs: []*SemanticError{semErrDuplicateAssoc}, }, { - caption: "a symbol cannot have different precedence", + caption: "the `#right` directive cannot be specified multiple times for an ordered symbol", + specSrc: ` +#name test; + +#prec ( + #right $x $x +); + +s + : foo #prec $x + ; + +foo + : 'foo'; +`, + errs: []*SemanticError{semErrDuplicateAssoc}, + }, + { + caption: "a terminal symbol cannot have different precedence", specSrc: ` #name test; @@ -1364,7 +1939,26 @@ foo errs: []*SemanticError{semErrDuplicateAssoc}, }, { - caption: "a symbol cannot have different associativity", + caption: "an ordered symbol cannot have different precedence", + specSrc: ` +#name test; + +#prec ( + #right $x + #right $x +); + +s + : foo #prec $x + ; + +foo + : 'foo'; +`, + errs: []*SemanticError{semErrDuplicateAssoc}, + }, + { + caption: "a terminal symbol cannot have different associativity", specSrc: ` #name test; @@ -1382,6 +1976,25 @@ foo `, errs: []*SemanticError{semErrDuplicateAssoc}, }, + { + caption: "an ordered symbol cannot have different associativity", + specSrc: ` +#name test; + +#prec ( + #left $x + #right $x +); + +s + : foo #prec $x + ; + +foo + : 'foo'; +`, + errs: []*SemanticError{semErrDuplicateAssoc}, + }, } assignDirTests := []*specErrTest{ @@ -1494,7 +2107,7 @@ foo errs: []*SemanticError{semErrDirInvalidParam}, }, { - caption: "the `#assign` dirctive cannot be specified multiple times for a symbol", + caption: "the `#assign` dirctive cannot be specified multiple times for a terminal symbol", specSrc: ` #name test; @@ -1512,7 +2125,25 @@ foo errs: []*SemanticError{semErrDuplicateAssoc}, }, { - caption: "a symbol cannot have different precedence", + caption: "the `#assign` dirctive cannot be specified multiple times for an ordered symbol", + specSrc: ` +#name test; + +#prec ( + #assign $x $x +); + +s + : foo #prec $x + ; + +foo + : 'foo'; +`, + errs: []*SemanticError{semErrDuplicateAssoc}, + }, + { + caption: "a terminal symbol cannot have different precedence", specSrc: ` #name test; @@ -1531,7 +2162,26 @@ foo errs: []*SemanticError{semErrDuplicateAssoc}, }, { - caption: "a symbol cannot have different associativity", + caption: "an ordered symbol cannot have different precedence", + specSrc: ` +#name test; + +#prec ( + #assign $x + #assign $x +); + +s + : foo #prec $x + ; + +foo + : 'foo'; +`, + errs: []*SemanticError{semErrDuplicateAssoc}, + }, + { + caption: "a terminal symbol cannot have different associativity", specSrc: ` #name test; @@ -1549,6 +2199,25 @@ foo `, errs: []*SemanticError{semErrDuplicateAssoc}, }, + { + caption: "an ordered symbol cannot have different associativity", + specSrc: ` +#name test; + +#prec ( + #assign $x + #left $x +); + +s + : foo #prec $x + ; + +foo + : 'foo'; +`, + errs: []*SemanticError{semErrDuplicateAssoc}, + }, } errorSymTests := []*specErrTest{ @@ -1618,6 +2287,24 @@ foo errs: []*SemanticError{semErrDirInvalidParam}, }, { + caption: "the `#ast` directive cannot take an ordered symbol parameter", + specSrc: ` +#name test; + +#prec ( + #assign $x +); + +s + : foo #ast $x + ; + +foo + : "foo"; +`, + errs: []*SemanticError{semErrDirInvalidParam}, + }, + { caption: "the `#ast` directive cannot take a pattern parameter", specSrc: ` #name test; @@ -1825,7 +2512,7 @@ foo altPrecDirTests := []*specErrTest{ { - caption: "the `#prec` directive needs an ID parameter", + caption: "the `#prec` directive needs an ID parameter or an ordered symbol parameter", specSrc: ` #name test; @@ -1876,6 +2563,20 @@ bar errs: []*SemanticError{semErrDirInvalidParam}, }, { + caption: "the `#prec` directive cannot take an undefined ordered symbol parameter", + specSrc: ` +#name test; + +s + : foo #prec $x + ; + +foo + : 'foo'; +`, + errs: []*SemanticError{semErrUndefinedOrdSym}, + }, + { caption: "the `#prec` directive cannot take a pattern parameter", specSrc: ` #name test; @@ -1951,6 +2652,24 @@ foo errs: []*SemanticError{semErrDirInvalidParam}, }, { + caption: "the `#recover` directive cannot take an ordered symbol parameter", + specSrc: ` +#name test; + +#prec ( + #assign $x +); + +s + : foo #recover $x + ; + +foo + : 'foo'; +`, + errs: []*SemanticError{semErrDirInvalidParam}, + }, + { caption: "the `#recover` directive cannot take a pattern parameter", specSrc: ` #name test; @@ -2120,6 +2839,26 @@ bar #mode errs: []*SemanticError{semErrDirInvalidParam}, }, { + caption: "the `#mode` directive cannot take an ordered symbol parameter", + specSrc: ` +#name test; + +#prec ( + #assign $x +); + +s + : foo bar + ; + +foo + : 'foo'; +bar #mode $x + : 'bar'; +`, + errs: []*SemanticError{semErrDirInvalidParam}, + }, + { caption: "the `#mode` directive cannot take a pattern parameter", specSrc: ` #name test; @@ -2203,6 +2942,26 @@ bar #mode mode_1 errs: []*SemanticError{semErrDirInvalidParam}, }, { + caption: "the `#push` directive cannot take an ordered symbol parameter", + specSrc: ` +#name test; + +#prec ( + #assign $x +); + +s + : foo bar + ; + +foo #push $x + : 'foo'; +bar + : 'bar'; +`, + errs: []*SemanticError{semErrDirInvalidParam}, + }, + { caption: "the `#push` directive cannot take a pattern parameter", specSrc: ` #name test; @@ -2272,6 +3031,28 @@ baz #pop mode_1 errs: []*SemanticError{semErrDirInvalidParam}, }, { + caption: "the `#pop` directive cannot take an ordered symbol parameter", + specSrc: ` +#name test; + +#prec ( + #assign $x +); + +s + : foo bar baz + ; + +foo #push mode_1 + : 'foo'; +bar #mode mode_1 + : 'bar'; +baz #pop $x + : 'baz'; +`, + errs: []*SemanticError{semErrDirInvalidParam}, + }, + { caption: "the `#pop` directive cannot take a pattern parameter", specSrc: ` #name test; @@ -2345,6 +3126,26 @@ bar errs: []*SemanticError{semErrDirInvalidParam}, }, { + caption: "the `#skip` directive cannot take an ordered symbol parameter", + specSrc: ` +#name test; + +#prec ( + #assign $x +); + +s + : foo bar + ; + +foo #skip $x + : 'foo'; +bar + : 'bar'; +`, + errs: []*SemanticError{semErrDirInvalidParam}, + }, + { caption: "the `#skip` directive cannot take a pattern parameter", specSrc: ` #name test; diff --git a/grammar/semantic_error.go b/grammar/semantic_error.go index a843719..794d8da 100644 --- a/grammar/semantic_error.go +++ b/grammar/semantic_error.go @@ -18,6 +18,7 @@ var ( semErrNoGrammarName = newSemanticError("name is missing") semErrDuplicateAssoc = newSemanticError("associativity and precedence cannot be specified multiple times for a symbol") semErrUndefinedPrec = newSemanticError("symbol must has precedence") + semErrUndefinedOrdSym = newSemanticError("undefined ordered symbol") semErrUnusedProduction = newSemanticError("unused production") semErrUnusedTerminal = newSemanticError("unused terminal") semErrTermCannotBeSkipped = newSemanticError("a terminal used in productions cannot be skipped") diff --git a/spec/lexer.go b/spec/lexer.go index 2459c40..c1c4b0d 100644 --- a/spec/lexer.go +++ b/spec/lexer.go @@ -15,21 +15,22 @@ import ( type tokenKind string const ( - tokenKindKWFragment = tokenKind("fragment") - tokenKindID = tokenKind("id") - tokenKindTerminalPattern = tokenKind("terminal pattern") - tokenKindStringLiteral = tokenKind("string") - tokenKindColon = tokenKind(":") - tokenKindOr = tokenKind("|") - tokenKindSemicolon = tokenKind(";") - tokenKindLabelMarker = tokenKind("@") - tokenKindDirectiveMarker = tokenKind("#") - tokenKindExpantion = tokenKind("...") - tokenKindLParen = tokenKind("(") - tokenKindRParen = tokenKind(")") - tokenKindNewline = tokenKind("newline") - tokenKindEOF = tokenKind("eof") - tokenKindInvalid = tokenKind("invalid") + tokenKindKWFragment = tokenKind("fragment") + tokenKindID = tokenKind("id") + tokenKindTerminalPattern = tokenKind("terminal pattern") + tokenKindStringLiteral = tokenKind("string") + tokenKindColon = tokenKind(":") + tokenKindOr = tokenKind("|") + tokenKindSemicolon = tokenKind(";") + tokenKindLabelMarker = tokenKind("@") + tokenKindDirectiveMarker = tokenKind("#") + tokenKindExpantion = tokenKind("...") + tokenKindOrderedSymbolMarker = tokenKind("$") + tokenKindLParen = tokenKind("(") + tokenKindRParen = tokenKind(")") + tokenKindNewline = tokenKind("newline") + tokenKindEOF = tokenKind("eof") + tokenKindInvalid = tokenKind("invalid") ) type Position struct { @@ -266,6 +267,8 @@ func (l *lexer) lexAndSkipWSs() (*token, error) { return newSymbolToken(tokenKindDirectiveMarker, newPosition(tok.Row+1, tok.Col+1)), nil case KindIDExpansion: return newSymbolToken(tokenKindExpantion, newPosition(tok.Row+1, tok.Col+1)), nil + case KindIDOrderedSymbolMarker: + return newSymbolToken(tokenKindOrderedSymbolMarker, newPosition(tok.Row+1, tok.Col+1)), nil case KindIDLParen: return newSymbolToken(tokenKindLParen, newPosition(tok.Row+1, tok.Col+1)), nil case KindIDRParen: diff --git a/spec/lexer_test.go b/spec/lexer_test.go index 621eff5..0e7cc89 100644 --- a/spec/lexer_test.go +++ b/spec/lexer_test.go @@ -36,7 +36,7 @@ func TestLexer_Run(t *testing.T) { }{ { caption: "the lexer can recognize all kinds of tokens", - src: `id"terminal"'string':|;@...#()`, + src: `id"terminal"'string':|;@...#$()`, tokens: []*token{ idTok("id"), termPatTok("terminal"), @@ -47,6 +47,7 @@ func TestLexer_Run(t *testing.T) { symTok(tokenKindLabelMarker), symTok(tokenKindExpantion), symTok(tokenKindDirectiveMarker), + symTok(tokenKindOrderedSymbolMarker), symTok(tokenKindLParen), symTok(tokenKindRParen), newEOFToken(), diff --git a/spec/lexspec.json b/spec/lexspec.json index 6a11a4a..7222be0 100644 --- a/spec/lexspec.json +++ b/spec/lexspec.json @@ -118,6 +118,10 @@ "pattern": "#" }, { + "kind": "ordered_symbol_marker", + "pattern": "$" + }, + { "kind": "l_paren", "pattern": "\\(" }, diff --git a/spec/parser.go b/spec/parser.go index 3b5907e..9c66bfb 100644 --- a/spec/parser.go +++ b/spec/parser.go @@ -55,12 +55,13 @@ type DirectiveNode struct { } type ParameterNode struct { - ID string - Pattern string - String string - Group []*DirectiveNode - Expansion bool - Pos Position + ID string + Pattern string + String string + OrderedSymbol string + Group []*DirectiveNode + Expansion bool + Pos Position } type FragmentNode struct { @@ -461,6 +462,14 @@ func (p *parser) parseParameter() *ParameterNode { String: p.lastTok.text, Pos: p.lastTok.pos, } + case p.consume(tokenKindOrderedSymbolMarker): + if !p.consume(tokenKindID) { + raiseSyntaxError(p.pos.Row, synErrNoOrderedSymbolName) + } + param = &ParameterNode{ + OrderedSymbol: p.lastTok.text, + Pos: p.lastTok.pos, + } case p.consume(tokenKindLParen): pos := p.lastTok.pos var g []*DirectiveNode diff --git a/spec/parser_test.go b/spec/parser_test.go index 3fe950f..15f1330 100644 --- a/spec/parser_test.go +++ b/spec/parser_test.go @@ -14,14 +14,12 @@ func TestParse(t *testing.T) { Parameters: []*ParameterNode{param}, } } - prec := func(param *ParameterNode) *DirectiveNode { return &DirectiveNode{ Name: "prec", Parameters: []*ParameterNode{param}, } } - leftAssoc := func(params ...*ParameterNode) *DirectiveNode { return &DirectiveNode{ Name: "left", @@ -82,6 +80,11 @@ func TestParse(t *testing.T) { ID: id, } } + ordSymParam := func(id string) *ParameterNode { + return &ParameterNode{ + OrderedSymbol: id, + } + } exp := func(param *ParameterNode) *ParameterNode { param.Expansion = true return param @@ -152,9 +155,9 @@ func TestParse(t *testing.T) { #name test; #prec ( - #left a b - #right c d - #assign e f + #left a b $x1 + #right c d $x2 + #assign e f $x3 ); `, ast: &RootNode{ @@ -182,6 +185,10 @@ func TestParse(t *testing.T) { idParam("b"), newPos(5), ), + withParamPos( + ordSymParam("x1"), + newPos(5), + ), ), newPos(5), ), @@ -195,6 +202,10 @@ func TestParse(t *testing.T) { idParam("d"), newPos(6), ), + withParamPos( + ordSymParam("x2"), + newPos(6), + ), ), newPos(6), ), @@ -208,6 +219,10 @@ func TestParse(t *testing.T) { idParam("f"), newPos(7), ), + withParamPos( + ordSymParam("x3"), + newPos(7), + ), ), newPos(7), ), @@ -237,6 +252,15 @@ func TestParse(t *testing.T) { synErr: synErrUnclosedDirGroup, }, { + caption: "an ordered symbol marker '$' must be followed by and ID", + src: ` +#prec ( + #assign $ +); +`, + synErr: synErrNoOrderedSymbolName, + }, + { caption: "single production is a valid grammar", src: `a: "a";`, ast: &RootNode{ @@ -299,6 +323,13 @@ c: ; }, }, { + caption: "a production cannot contain an ordered symbol", + src: ` +a: $x; +`, + synErr: synErrNoSemicolon, + }, + { caption: "`fragment` is a reserved word", src: `fragment: 'fragment';`, synErr: synErrNoProductionName, diff --git a/spec/syntax_error.go b/spec/syntax_error.go index 3b44d2d..ad847a2 100644 --- a/spec/syntax_error.go +++ b/spec/syntax_error.go @@ -32,6 +32,7 @@ var ( synErrLabelWithNoSymbol = newSyntaxError("a label must follow a symbol") synErrNoLabel = newSyntaxError("an identifier that represents a label is missing after the label marker @") synErrNoDirectiveName = newSyntaxError("a directive needs a name") + synErrNoOrderedSymbolName = newSyntaxError("an ordered symbol name is missing") synErrUnclosedDirGroup = newSyntaxError("a directive group must be closed by )") synErrSemicolonNoNewline = newSyntaxError("a semicolon must be followed by a newline") synErrFragmentNoPattern = newSyntaxError("a fragment needs one pattern element") diff --git a/spec/vartan_lexer.go b/spec/vartan_lexer.go index 146748a..7c0dfd4 100644 --- a/spec/vartan_lexer.go +++ b/spec/vartan_lexer.go @@ -342,55 +342,57 @@ func ModeIDToName(id ModeID) string { } const ( - KindIDNil KindID = 0 - KindIDWhiteSpace KindID = 1 - KindIDNewline KindID = 2 - KindIDLineComment KindID = 3 - KindIDKwFragment KindID = 4 - KindIDIdentifier KindID = 5 - KindIDTerminalOpen KindID = 6 - KindIDStringLiteralOpen KindID = 7 - KindIDColon KindID = 8 - KindIDOr KindID = 9 - KindIDSemicolon KindID = 10 - KindIDLabelMarker KindID = 11 - KindIDExpansion KindID = 12 - KindIDDirectiveMarker KindID = 13 - KindIDLParen KindID = 14 - KindIDRParen KindID = 15 - KindIDPattern KindID = 16 - KindIDTerminalClose KindID = 17 - KindIDEscapeSymbol KindID = 18 - KindIDCharSeq KindID = 19 - KindIDEscapedQuot KindID = 20 - KindIDEscapedBackSlash KindID = 21 - KindIDStringLiteralClose KindID = 22 + KindIDNil KindID = 0 + KindIDWhiteSpace KindID = 1 + KindIDNewline KindID = 2 + KindIDLineComment KindID = 3 + KindIDKwFragment KindID = 4 + KindIDIdentifier KindID = 5 + KindIDTerminalOpen KindID = 6 + KindIDStringLiteralOpen KindID = 7 + KindIDColon KindID = 8 + KindIDOr KindID = 9 + KindIDSemicolon KindID = 10 + KindIDLabelMarker KindID = 11 + KindIDExpansion KindID = 12 + KindIDDirectiveMarker KindID = 13 + KindIDOrderedSymbolMarker KindID = 14 + KindIDLParen KindID = 15 + KindIDRParen KindID = 16 + KindIDPattern KindID = 17 + KindIDTerminalClose KindID = 18 + KindIDEscapeSymbol KindID = 19 + KindIDCharSeq KindID = 20 + KindIDEscapedQuot KindID = 21 + KindIDEscapedBackSlash KindID = 22 + KindIDStringLiteralClose KindID = 23 ) const ( - KindNameNil = "" - KindNameWhiteSpace = "white_space" - KindNameNewline = "newline" - KindNameLineComment = "line_comment" - KindNameKwFragment = "kw_fragment" - KindNameIdentifier = "identifier" - KindNameTerminalOpen = "terminal_open" - KindNameStringLiteralOpen = "string_literal_open" - KindNameColon = "colon" - KindNameOr = "or" - KindNameSemicolon = "semicolon" - KindNameLabelMarker = "label_marker" - KindNameExpansion = "expansion" - KindNameDirectiveMarker = "directive_marker" - KindNameLParen = "l_paren" - KindNameRParen = "r_paren" - KindNamePattern = "pattern" - KindNameTerminalClose = "terminal_close" - KindNameEscapeSymbol = "escape_symbol" - KindNameCharSeq = "char_seq" - KindNameEscapedQuot = "escaped_quot" - KindNameEscapedBackSlash = "escaped_back_slash" - KindNameStringLiteralClose = "string_literal_close" + KindNameNil = "" + KindNameWhiteSpace = "white_space" + KindNameNewline = "newline" + KindNameLineComment = "line_comment" + KindNameKwFragment = "kw_fragment" + KindNameIdentifier = "identifier" + KindNameTerminalOpen = "terminal_open" + KindNameStringLiteralOpen = "string_literal_open" + KindNameColon = "colon" + KindNameOr = "or" + KindNameSemicolon = "semicolon" + KindNameLabelMarker = "label_marker" + KindNameExpansion = "expansion" + KindNameDirectiveMarker = "directive_marker" + KindNameOrderedSymbolMarker = "ordered_symbol_marker" + KindNameLParen = "l_paren" + KindNameRParen = "r_paren" + KindNamePattern = "pattern" + KindNameTerminalClose = "terminal_close" + KindNameEscapeSymbol = "escape_symbol" + KindNameCharSeq = "char_seq" + KindNameEscapedQuot = "escaped_quot" + KindNameEscapedBackSlash = "escaped_back_slash" + KindNameStringLiteralClose = "string_literal_close" ) // KindIDToName converts a kind ID to a name. @@ -424,6 +426,8 @@ func KindIDToName(id KindID) string { return KindNameExpansion case KindIDDirectiveMarker: return KindNameDirectiveMarker + case KindIDOrderedSymbolMarker: + return KindNameOrderedSymbolMarker case KindIDLParen: return KindNameLParen case KindIDRParen: @@ -471,7 +475,7 @@ func NewLexSpec() *lexSpec { pop: [][]bool{ nil, { - false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, + false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, }, { false, false, true, false, @@ -483,7 +487,7 @@ func NewLexSpec() *lexSpec { push: [][]ModeID{ nil, { - 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, }, { 0, 0, 0, 0, @@ -509,7 +513,7 @@ func NewLexSpec() *lexSpec { { 0, 0, 1, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 4, 5, 0, 0, 2, 6, 7, 8, 9, - 10, 11, 12, 13, 14, 15, + 10, 11, 12, 13, 14, 15, 16, }, { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -540,6 +544,7 @@ func NewLexSpec() *lexSpec { KindIDLabelMarker, KindIDExpansion, KindIDDirectiveMarker, + KindIDOrderedSymbolMarker, KindIDLParen, KindIDRParen, }, @@ -573,6 +578,7 @@ func NewLexSpec() *lexSpec { KindNameLabelMarker, KindNameExpansion, KindNameDirectiveMarker, + KindNameOrderedSymbolMarker, KindNameLParen, KindNameRParen, KindNamePattern, @@ -593,7 +599,7 @@ func NewLexSpec() *lexSpec { { 0, 1, 2, 3, 4, 5, 6, 7, 6, 8, 6, 9, 6, 10, 6, 11, 12, 6, 13, 14, 6, 15, 16, 6, 17, 18, 19, 20, 21, 22, 23, 24, 24, 25, 26, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, }, { 0, 1, 2, 3, 2, 4, 2, 5, 2, 6, 2, 7, 8, 2, 9, 10, 2, 11, 12, 2, @@ -639,7 +645,7 @@ func NewLexSpec() *lexSpec { 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 1, 1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, 1, 1, -1, -1, -1, 1, 1, 1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, 1, 1, 1, -1, -1, 1, 1, 1, -1, -1, -1, -1, 1, 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, 1, -1, -1, -1, -1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, 1, -1, 1, 1, 1, 1, 1, 1, 1, @@ -984,7 +990,7 @@ func NewLexSpec() *lexSpec { 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 11, 13, 13, 15, 18, 18, 18, 21, 2, 35, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 36, 43, 0, 0, 0, 37, 44, 45, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 36, 43, 44, 0, 0, 37, 45, 46, 0, 0, 0, 0, 33, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 40, 0, 0, 0, 0, 41, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 0, 0, 0, 0, 32, 0, 32, 32, 32, 32, 32, 24, 32, |