diff options
-rw-r--r-- | driver/parser/parser.go | 4 | ||||
-rw-r--r-- | driver/parser/semantic_action.go | 15 | ||||
-rw-r--r-- | driver/parser/template.go | 4 | ||||
-rw-r--r-- | driver/parser/token_stream.go | 4 |
4 files changed, 22 insertions, 5 deletions
diff --git a/driver/parser/parser.go b/driver/parser/parser.go index 05f7d38..2eaa678 100644 --- a/driver/parser/parser.go +++ b/driver/parser/parser.go @@ -64,6 +64,10 @@ type VToken interface { // Invalid returns true when a token is invalid. Invalid() bool + // BytePosition returns (position, length) pair. + // `position` is a byte position where a token appears and `length` is a length in bytes. + BytePosition() (int, int) + // Position returns (row, column) pair. Position() (int, int) } diff --git a/driver/parser/semantic_action.go b/driver/parser/semantic_action.go index 1e8079c..6bb78cf 100644 --- a/driver/parser/semantic_action.go +++ b/driver/parser/semantic_action.go @@ -49,7 +49,7 @@ var _ SyntaxTreeNode = &Node{} // SyntaxTreeBuilder allows you to construct a syntax tree containing arbitrary user-defined node types. // The parser uses SyntaxTreeBuilder interface as a part of semantic actions via SyntaxTreeActionSet interface. type SyntaxTreeBuilder interface { - Shift(kindName string, text string, row, col int) SyntaxTreeNode + Shift(kindName string, tok VToken) SyntaxTreeNode ShiftError(kindName string) SyntaxTreeNode Reduce(kindName string, children []SyntaxTreeNode) SyntaxTreeNode Accept(f SyntaxTreeNode) @@ -68,11 +68,15 @@ func NewDefaultSyntaxTreeBuilder() *DefaultSyntaxTreeBuilder { } // Shift is a implementation of SyntaxTreeBuilder.Shift. -func (b *DefaultSyntaxTreeBuilder) Shift(kindName string, text string, row, col int) SyntaxTreeNode { +func (b *DefaultSyntaxTreeBuilder) Shift(kindName string, tok VToken) SyntaxTreeNode { + bytePos, byteLen := tok.BytePosition() + row, col := tok.Position() return &Node{ Type: NodeTypeTerminal, KindName: kindName, - Text: text, + Text: string(tok.Lexeme()), + BytePos: bytePos, + ByteLen: byteLen, Row: row, Col: col, } @@ -141,8 +145,7 @@ func NewCSTActionSet(gram Grammar, builder SyntaxTreeBuilder) *SyntaxTreeActionS // Shift is a implementation of SemanticActionSet.Shift method. func (a *SyntaxTreeActionSet) Shift(tok VToken, recovered bool) { term := a.tokenToTerminal(tok) - row, col := tok.Position() - a.semStack.push(a.builder.Shift(a.gram.Terminal(term), string(tok.Lexeme()), row, col)) + a.semStack.push(a.builder.Shift(a.gram.Terminal(term), tok)) } // Reduce is a implementation of SemanticActionSet.Reduce method. @@ -255,6 +258,8 @@ type Node struct { Type NodeType KindName string Text string + BytePos int + ByteLen int Row int Col int Children []*Node diff --git a/driver/parser/template.go b/driver/parser/template.go index 96eb71f..e9620dc 100644 --- a/driver/parser/template.go +++ b/driver/parser/template.go @@ -433,6 +433,10 @@ func (t *vToken) Invalid() bool { return t.tok.Invalid } +func (t *vToken) BytePosition() (int, int) { + return t.tok.BytePos, t.tok.ByteLen +} + func (t *vToken) Position() (int, int) { return t.tok.Row, t.tok.Col } diff --git a/driver/parser/token_stream.go b/driver/parser/token_stream.go index 0bc9e32..929b733 100644 --- a/driver/parser/token_stream.go +++ b/driver/parser/token_stream.go @@ -28,6 +28,10 @@ func (t *vToken) Invalid() bool { return t.tok.Invalid } +func (t *vToken) BytePosition() (int, int) { + return t.tok.BytePos, t.tok.ByteLen +} + func (t *vToken) Position() (int, int) { return t.tok.Row, t.tok.Col } |