Add byte position to nodes of a syntax tree

4 files changed, 22 insertions, 5 deletions

diff --git a/driver/parser/parser.go b/driver/parser/parser.go
index 05f7d38..2eaa678 100644
--- a/driver/parser/parser.go
+++ b/driver/parser/parser.go
@@ -64,6 +64,10 @@ type VToken interface {
 	// Invalid returns true when a token is invalid.
 	Invalid() bool
 
+	// BytePosition returns (position, length) pair.
+	// `position` is a byte position where a token appears and `length` is a length in bytes.
+	BytePosition() (int, int)
+
 	// Position returns (row, column) pair.
 	Position() (int, int)
 }
diff --git a/driver/parser/semantic_action.go b/driver/parser/semantic_action.go
index 1e8079c..6bb78cf 100644
--- a/driver/parser/semantic_action.go
+++ b/driver/parser/semantic_action.go
@@ -49,7 +49,7 @@ var _ SyntaxTreeNode = &Node{}
 // SyntaxTreeBuilder allows you to construct a syntax tree containing arbitrary user-defined node types.
 // The parser uses SyntaxTreeBuilder interface as a part of semantic actions via SyntaxTreeActionSet interface.
 type SyntaxTreeBuilder interface {
-	Shift(kindName string, text string, row, col int) SyntaxTreeNode
+	Shift(kindName string, tok VToken) SyntaxTreeNode
 	ShiftError(kindName string) SyntaxTreeNode
 	Reduce(kindName string, children []SyntaxTreeNode) SyntaxTreeNode
 	Accept(f SyntaxTreeNode)
@@ -68,11 +68,15 @@ func NewDefaultSyntaxTreeBuilder() *DefaultSyntaxTreeBuilder {
 }
 
 // Shift is a implementation of SyntaxTreeBuilder.Shift.
-func (b *DefaultSyntaxTreeBuilder) Shift(kindName string, text string, row, col int) SyntaxTreeNode {
+func (b *DefaultSyntaxTreeBuilder) Shift(kindName string, tok VToken) SyntaxTreeNode {
+	bytePos, byteLen := tok.BytePosition()
+	row, col := tok.Position()
 	return &Node{
 		Type:     NodeTypeTerminal,
 		KindName: kindName,
-		Text:     text,
+		Text:     string(tok.Lexeme()),
+		BytePos:  bytePos,
+		ByteLen:  byteLen,
 		Row:      row,
 		Col:      col,
 	}
@@ -141,8 +145,7 @@ func NewCSTActionSet(gram Grammar, builder SyntaxTreeBuilder) *SyntaxTreeActionS
 // Shift is a implementation of SemanticActionSet.Shift method.
 func (a *SyntaxTreeActionSet) Shift(tok VToken, recovered bool) {
 	term := a.tokenToTerminal(tok)
-	row, col := tok.Position()
-	a.semStack.push(a.builder.Shift(a.gram.Terminal(term), string(tok.Lexeme()), row, col))
+	a.semStack.push(a.builder.Shift(a.gram.Terminal(term), tok))
 }
 
 // Reduce is a implementation of SemanticActionSet.Reduce method.
@@ -255,6 +258,8 @@ type Node struct {
 	Type     NodeType
 	KindName string
 	Text     string
+	BytePos  int
+	ByteLen  int
 	Row      int
 	Col      int
 	Children []*Node
diff --git a/driver/parser/template.go b/driver/parser/template.go
index 96eb71f..e9620dc 100644
--- a/driver/parser/template.go
+++ b/driver/parser/template.go
@@ -433,6 +433,10 @@ func (t *vToken) Invalid() bool {
 	return t.tok.Invalid
 }
 
+func (t *vToken) BytePosition() (int, int) {
+	return t.tok.BytePos, t.tok.ByteLen
+}
+
 func (t *vToken) Position() (int, int) {
 	return t.tok.Row, t.tok.Col
 }
diff --git a/driver/parser/token_stream.go b/driver/parser/token_stream.go
index 0bc9e32..929b733 100644
--- a/driver/parser/token_stream.go
+++ b/driver/parser/token_stream.go
@@ -28,6 +28,10 @@ func (t *vToken) Invalid() bool {
 	return t.tok.Invalid
 }
 
+func (t *vToken) BytePosition() (int, int) {
+	return t.tok.BytePos, t.tok.ByteLen
+}
+
 func (t *vToken) Position() (int, int) {
 	return t.tok.Row, t.tok.Col
 }