aboutsummaryrefslogtreecommitdiff
path: root/compiler
diff options
context:
space:
mode:
Diffstat (limited to 'compiler')
-rw-r--r--compiler/ast.go6
-rw-r--r--compiler/compiler.go112
-rw-r--r--compiler/dfa.go18
-rw-r--r--compiler/dfa_test.go6
-rw-r--r--compiler/parser.go12
-rw-r--r--compiler/parser_test.go8
-rw-r--r--compiler/test_util_test.go4
7 files changed, 96 insertions, 70 deletions
diff --git a/compiler/ast.go b/compiler/ast.go
index 7d3965a..046662e 100644
--- a/compiler/ast.go
+++ b/compiler/ast.go
@@ -3,6 +3,8 @@ package compiler
import (
"fmt"
"io"
+
+ "github.com/nihei9/maleeni/spec"
)
type astNode interface {
@@ -78,13 +80,13 @@ func (n *symbolNode) last() *symbolPositionSet {
}
type endMarkerNode struct {
- id int
+ id spec.LexModeKindID
pos symbolPosition
firstMemo *symbolPositionSet
lastMemo *symbolPositionSet
}
-func newEndMarkerNode(id int) *endMarkerNode {
+func newEndMarkerNode(id spec.LexModeKindID) *endMarkerNode {
return &endMarkerNode{
id: id,
pos: symbolPositionNil,
diff --git a/compiler/compiler.go b/compiler/compiler.go
index 5d3e52f..5d1a1d5 100644
--- a/compiler/compiler.go
+++ b/compiler/compiler.go
@@ -54,28 +54,28 @@ func Compile(lexspec *spec.LexSpec, opts ...CompilerOption) (*spec.CompiledLexSp
}
}
- modeEntries, modes, modeNums, fragmetns := groupEntriesByLexMode(lexspec.Entries)
+ modeEntries, modeNames, modeName2ID, fragmetns := groupEntriesByLexMode(lexspec.Entries)
modeSpecs := []*spec.CompiledLexModeSpec{
nil,
}
for i, es := range modeEntries[1:] {
- modeName := modes[i+1]
+ modeName := modeNames[i+1]
config.logger.Log("Compile %v mode:", modeName)
- modeSpec, err := compile(es, modeNums, fragmetns, config)
+ modeSpec, err := compile(es, modeName2ID, fragmetns, config)
if err != nil {
return nil, fmt.Errorf("failed to compile in %v mode: %w", modeName, err)
}
modeSpecs = append(modeSpecs, modeSpec)
}
- var kindNames []spec.LexKind
- var name2ID map[spec.LexKind]spec.LexKindID
+ var kindNames []spec.LexKindName
+ var name2ID map[spec.LexKindName]spec.LexKindID
{
- name2ID = map[spec.LexKind]spec.LexKindID{}
+ name2ID = map[spec.LexKindName]spec.LexKindID{}
id := spec.LexKindIDMin
for _, modeSpec := range modeSpecs[1:] {
- for _, name := range modeSpec.Kinds[1:] {
+ for _, name := range modeSpec.KindNames[1:] {
if _, ok := name2ID[name]; ok {
continue
}
@@ -84,7 +84,7 @@ func Compile(lexspec *spec.LexSpec, opts ...CompilerOption) (*spec.CompiledLexSp
}
}
- kindNames = make([]spec.LexKind, len(name2ID)+1)
+ kindNames = make([]spec.LexKindName, len(name2ID)+1)
for name, id := range name2ID {
kindNames[id] = name
}
@@ -94,8 +94,8 @@ func Compile(lexspec *spec.LexSpec, opts ...CompilerOption) (*spec.CompiledLexSp
{
kindIDs = make([][]spec.LexKindID, len(modeSpecs))
for i, modeSpec := range modeSpecs[1:] {
- ids := make([]spec.LexKindID, len(modeSpec.Kinds))
- for modeID, name := range modeSpec.Kinds {
+ ids := make([]spec.LexKindID, len(modeSpec.KindNames))
+ for modeID, name := range modeSpec.KindNames {
if modeID == 0 {
continue
}
@@ -106,25 +106,25 @@ func Compile(lexspec *spec.LexSpec, opts ...CompilerOption) (*spec.CompiledLexSp
}
return &spec.CompiledLexSpec{
- InitialMode: spec.LexModeNumDefault,
- Modes: modes,
- Kinds: kindNames,
+ InitialModeID: spec.LexModeIDDefault,
+ ModeNames: modeNames,
+ KindNames: kindNames,
KindIDs: kindIDs,
CompressionLevel: config.compLv,
Specs: modeSpecs,
}, nil
}
-func groupEntriesByLexMode(entries []*spec.LexEntry) ([][]*spec.LexEntry, []spec.LexModeName, map[spec.LexModeName]spec.LexModeNum, map[string]*spec.LexEntry) {
- modes := []spec.LexModeName{
+func groupEntriesByLexMode(entries []*spec.LexEntry) ([][]*spec.LexEntry, []spec.LexModeName, map[spec.LexModeName]spec.LexModeID, map[string]*spec.LexEntry) {
+ modeNames := []spec.LexModeName{
spec.LexModeNameNil,
spec.LexModeNameDefault,
}
- modeNums := map[spec.LexModeName]spec.LexModeNum{
- spec.LexModeNameNil: spec.LexModeNumNil,
- spec.LexModeNameDefault: spec.LexModeNumDefault,
+ modeName2ID := map[spec.LexModeName]spec.LexModeID{
+ spec.LexModeNameNil: spec.LexModeIDNil,
+ spec.LexModeNameDefault: spec.LexModeIDDefault,
}
- lastModeNum := spec.LexModeNumDefault
+ lastModeID := spec.LexModeIDDefault
modeEntries := [][]*spec.LexEntry{
nil,
{},
@@ -141,30 +141,30 @@ func groupEntriesByLexMode(entries []*spec.LexEntry) ([][]*spec.LexEntry, []spec
spec.LexModeNameDefault,
}
}
- for _, mode := range ms {
- num, ok := modeNums[mode]
+ for _, modeName := range ms {
+ modeID, ok := modeName2ID[modeName]
if !ok {
- num = lastModeNum.Succ()
- lastModeNum = num
- modeNums[mode] = num
- modes = append(modes, mode)
+ modeID = lastModeID + 1
+ lastModeID = modeID
+ modeName2ID[modeName] = modeID
+ modeNames = append(modeNames, modeName)
modeEntries = append(modeEntries, []*spec.LexEntry{})
}
- modeEntries[num] = append(modeEntries[num], e)
+ modeEntries[modeID] = append(modeEntries[modeID], e)
}
}
- return modeEntries, modes, modeNums, fragments
+ return modeEntries, modeNames, modeName2ID, fragments
}
-func compile(entries []*spec.LexEntry, modeNums map[spec.LexModeName]spec.LexModeNum, fragments map[string]*spec.LexEntry, config *compilerConfig) (*spec.CompiledLexModeSpec, error) {
- var kinds []spec.LexKind
- var patterns map[int][]byte
+func compile(entries []*spec.LexEntry, modeName2ID map[spec.LexModeName]spec.LexModeID, fragments map[string]*spec.LexEntry, config *compilerConfig) (*spec.CompiledLexModeSpec, error) {
+ var kindNames []spec.LexKindName
+ var patterns map[spec.LexModeKindID][]byte
{
- kinds = append(kinds, spec.LexKindNil)
- patterns = map[int][]byte{}
+ kindNames = append(kindNames, spec.LexKindNameNil)
+ patterns = map[spec.LexModeKindID][]byte{}
for i, e := range entries {
- kinds = append(kinds, e.Kind)
- patterns[i+1] = []byte(e.Pattern)
+ kindNames = append(kindNames, e.Kind)
+ patterns[spec.LexModeKindID(i+1)] = []byte(e.Pattern)
}
config.logger.Log("Patterns:")
@@ -173,16 +173,16 @@ func compile(entries []*spec.LexEntry, modeNums map[spec.LexModeName]spec.LexMod
}
}
- push := []spec.LexModeNum{
- spec.LexModeNumNil,
+ push := []spec.LexModeID{
+ spec.LexModeIDNil,
}
pop := []int{
0,
}
for _, e := range entries {
- pushV := spec.LexModeNumNil
+ pushV := spec.LexModeIDNil
if e.Push != "" {
- pushV = modeNums[e.Push]
+ pushV = modeName2ID[e.Push]
}
push = append(push, pushV)
popV := 0
@@ -222,7 +222,7 @@ func compile(entries []*spec.LexEntry, modeNums map[spec.LexModeName]spec.LexMod
config.logger.Log(`DFA:
States: %v states (%v entries)
- Initial State: %v`, tranTab.RowCount, tranTab.RowCount*tranTab.ColCount, tranTab.InitialState)
+ Initial State ID: %v`, tranTab.RowCount, tranTab.RowCount*tranTab.ColCount, tranTab.InitialStateID)
config.logger.Log(" Accepting States:")
for state, symbol := range tranTab.AcceptingStates {
config.logger.Log(" %v: %v", state, symbol)
@@ -244,10 +244,10 @@ func compile(entries []*spec.LexEntry, modeNums map[spec.LexModeName]spec.LexMod
}
return &spec.CompiledLexModeSpec{
- Kinds: kinds,
- Push: push,
- Pop: pop,
- DFA: tranTab,
+ KindNames: kindNames,
+ Push: push,
+ Pop: pop,
+ DFA: tranTab,
}, nil
}
@@ -259,7 +259,7 @@ const (
func compressTransitionTableLv2(tranTab *spec.TransitionTable) (*spec.TransitionTable, error) {
ueTab := compressor.NewUniqueEntriesTable()
{
- orig, err := compressor.NewOriginalTable(tranTab.UncompressedTransition, tranTab.ColCount)
+ orig, err := compressor.NewOriginalTable(convertStateIDSliceToIntSlice(tranTab.UncompressedTransition), tranTab.ColCount)
if err != nil {
return nil, err
}
@@ -285,8 +285,8 @@ func compressTransitionTableLv2(tranTab *spec.TransitionTable) (*spec.Transition
UniqueEntries: &spec.RowDisplacementTable{
OriginalRowCount: rdTab.OriginalRowCount,
OriginalColCount: rdTab.OriginalColCount,
- EmptyValue: rdTab.EmptyValue,
- Entries: rdTab.Entries,
+ EmptyValue: spec.StateIDNil,
+ Entries: convertIntSliceToStateIDSlice(rdTab.Entries),
Bounds: rdTab.Bounds,
RowDisplacement: rdTab.RowDisplacement,
},
@@ -302,7 +302,7 @@ func compressTransitionTableLv2(tranTab *spec.TransitionTable) (*spec.Transition
func compressTransitionTableLv1(tranTab *spec.TransitionTable) (*spec.TransitionTable, error) {
ueTab := compressor.NewUniqueEntriesTable()
{
- orig, err := compressor.NewOriginalTable(tranTab.UncompressedTransition, tranTab.ColCount)
+ orig, err := compressor.NewOriginalTable(convertStateIDSliceToIntSlice(tranTab.UncompressedTransition), tranTab.ColCount)
if err != nil {
return nil, err
}
@@ -313,7 +313,7 @@ func compressTransitionTableLv1(tranTab *spec.TransitionTable) (*spec.Transition
}
tranTab.Transition = &spec.UniqueEntriesTable{
- UncompressedUniqueEntries: ueTab.UniqueEntries,
+ UncompressedUniqueEntries: convertIntSliceToStateIDSlice(ueTab.UniqueEntries),
RowNums: ueTab.RowNums,
OriginalRowCount: ueTab.OriginalRowCount,
OriginalColCount: ueTab.OriginalColCount,
@@ -322,3 +322,19 @@ func compressTransitionTableLv1(tranTab *spec.TransitionTable) (*spec.Transition
return tranTab, nil
}
+
+func convertStateIDSliceToIntSlice(s []spec.StateID) []int {
+ is := make([]int, len(s))
+ for i, v := range s {
+ is[i] = v.Int()
+ }
+ return is
+}
+
+func convertIntSliceToStateIDSlice(s []int) []spec.StateID {
+ ss := make([]spec.StateID, len(s))
+ for i, v := range s {
+ ss[i] = spec.StateID(v)
+ }
+ return ss
+}
diff --git a/compiler/dfa.go b/compiler/dfa.go
index b94fca8..1d0b26a 100644
--- a/compiler/dfa.go
+++ b/compiler/dfa.go
@@ -9,7 +9,7 @@ import (
type DFA struct {
States []string
InitialState string
- AcceptingStatesTable map[string]int
+ AcceptingStatesTable map[string]spec.LexModeKindID
TransitionTable map[string][256]string
}
@@ -65,7 +65,7 @@ func genDFA(root astNode, symTab *symbolTable) *DFA {
}
}
- accTab := map[string]int{}
+ accTab := map[string]spec.LexModeKindID{}
{
for h, s := range stateMap {
for _, pos := range s.set() {
@@ -104,33 +104,33 @@ func genDFA(root astNode, symTab *symbolTable) *DFA {
}
func genTransitionTable(dfa *DFA) (*spec.TransitionTable, error) {
- state2Num := map[string]int{}
+ stateHash2ID := map[string]spec.StateID{}
for i, s := range dfa.States {
// Since 0 represents an invalid value in a transition table,
// assign a number greater than or equal to 1 to states.
- state2Num[s] = i + 1
+ stateHash2ID[s] = spec.StateID(i + spec.StateIDMin.Int())
}
- acc := make([]int, len(dfa.States)+1)
+ acc := make([]spec.LexModeKindID, len(dfa.States)+1)
for _, s := range dfa.States {
id, ok := dfa.AcceptingStatesTable[s]
if !ok {
continue
}
- acc[state2Num[s]] = id
+ acc[stateHash2ID[s]] = id
}
rowCount := len(dfa.States) + 1
colCount := 256
- tran := make([]int, rowCount*colCount)
+ tran := make([]spec.StateID, rowCount*colCount)
for s, tab := range dfa.TransitionTable {
for v, to := range tab {
- tran[state2Num[s]*256+v] = state2Num[to]
+ tran[stateHash2ID[s].Int()*256+v] = stateHash2ID[to]
}
}
return &spec.TransitionTable{
- InitialState: state2Num[dfa.InitialState],
+ InitialStateID: stateHash2ID[dfa.InitialState],
AcceptingStates: acc,
UncompressedTransition: tran,
RowCount: rowCount,
diff --git a/compiler/dfa_test.go b/compiler/dfa_test.go
index a797313..2683815 100644
--- a/compiler/dfa_test.go
+++ b/compiler/dfa_test.go
@@ -2,10 +2,12 @@ package compiler
import (
"testing"
+
+ "github.com/nihei9/maleeni/spec"
)
func TestGenDFA(t *testing.T) {
- root, symTab, err := parse(map[int][]byte{
+ root, symTab, err := parse(map[spec.LexModeKindID][]byte{
1: []byte("(a|b)*abb"),
}, nil)
if err != nil {
@@ -94,7 +96,7 @@ func TestGenDFA(t *testing.T) {
t.Errorf("initial state is mismatched; want: %v, got: %v", s0.hash(), dfa.InitialState)
}
- accTab := map[string]int{
+ accTab := map[string]spec.LexModeKindID{
s3.hash(): 1,
}
if len(dfa.AcceptingStatesTable) != len(accTab) {
diff --git a/compiler/parser.go b/compiler/parser.go
index 06a762d..1c9126a 100644
--- a/compiler/parser.go
+++ b/compiler/parser.go
@@ -7,6 +7,8 @@ import (
"fmt"
"io"
"strings"
+
+ "github.com/nihei9/maleeni/spec"
)
type ParseErrors struct {
@@ -23,7 +25,7 @@ func (e *ParseErrors) Error() string {
}
type ParseError struct {
- ID int
+ ID spec.LexModeKindID
Pattern []byte
Cause error
Details string
@@ -44,13 +46,13 @@ func raiseSyntaxError(synErr *SyntaxError) {
type symbolTable struct {
symPos2Byte map[symbolPosition]byteRange
- endPos2ID map[symbolPosition]int
+ endPos2ID map[symbolPosition]spec.LexModeKindID
}
func genSymbolTable(root astNode) *symbolTable {
symTab := &symbolTable{
symPos2Byte: map[symbolPosition]byteRange{},
- endPos2ID: map[symbolPosition]int{},
+ endPos2ID: map[symbolPosition]spec.LexModeKindID{},
}
return genSymTab(symTab, root)
}
@@ -76,7 +78,7 @@ func genSymTab(symTab *symbolTable, node astNode) *symbolTable {
return symTab
}
-func parse(regexps map[int][]byte, fragments map[string][]byte) (astNode, *symbolTable, error) {
+func parse(regexps map[spec.LexModeKindID][]byte, fragments map[string][]byte) (astNode, *symbolTable, error) {
if len(regexps) == 0 {
return nil, nil, fmt.Errorf("parse() needs at least one token entry")
}
@@ -159,7 +161,7 @@ func parseFragments(fragments map[string][]byte) (map[string]astNode, error) {
return fragmentASTs, nil
}
-func parseRegexp(regexps map[int][]byte, fragmentASTs map[string]astNode) (astNode, error) {
+func parseRegexp(regexps map[spec.LexModeKindID][]byte, fragmentASTs map[string]astNode) (astNode, error) {
symPos := symbolPositionMin
var root astNode
var perrs []*ParseError
diff --git a/compiler/parser_test.go b/compiler/parser_test.go
index 3394845..30e6130 100644
--- a/compiler/parser_test.go
+++ b/compiler/parser_test.go
@@ -4,6 +4,8 @@ import (
"fmt"
"reflect"
"testing"
+
+ "github.com/nihei9/maleeni/spec"
)
func symPos(n uint16) symbolPosition {
@@ -1197,7 +1199,7 @@ func TestParse(t *testing.T) {
for kind, pattern := range tt.fragments {
fragments[kind] = []byte(pattern)
}
- ast, _, err := parse(map[int][]byte{
+ ast, _, err := parse(map[spec.LexModeKindID][]byte{
1: []byte(tt.pattern),
}, fragments)
if tt.syntaxError != nil {
@@ -1237,7 +1239,7 @@ func TestParse(t *testing.T) {
}
func TestParse_FollowAndSymbolTable(t *testing.T) {
- root, symTab, err := parse(map[int][]byte{
+ root, symTab, err := parse(map[spec.LexModeKindID][]byte{
1: []byte("(a|b)*abb"),
}, nil)
if err != nil {
@@ -1295,7 +1297,7 @@ func TestParse_FollowAndSymbolTable(t *testing.T) {
symPos(4): entry(byte('b')),
symPos(5): entry(byte('b')),
},
- endPos2ID: map[symbolPosition]int{
+ endPos2ID: map[symbolPosition]spec.LexModeKindID{
endPos(6): 1,
},
}
diff --git a/compiler/test_util_test.go b/compiler/test_util_test.go
index 2ead2c9..72e150b 100644
--- a/compiler/test_util_test.go
+++ b/compiler/test_util_test.go
@@ -1,5 +1,7 @@
package compiler
+import "github.com/nihei9/maleeni/spec"
+
func newRangeSymbolNodeWithPos(from, to byte, pos symbolPosition) *symbolNode {
n := newRangeSymbolNode(from, to)
n.pos = pos
@@ -13,7 +15,7 @@ func newSymbolNodeWithPos(v byte, pos symbolPosition) *symbolNode {
}
func newEndMarkerNodeWithPos(id int, pos symbolPosition) *endMarkerNode {
- n := newEndMarkerNode(id)
+ n := newEndMarkerNode(spec.LexModeKindID(id))
n.pos = pos
return n
}
generated by cgit v1.2.3 (git 2.49.0) at 2025-07-13 16:59:06 +0000