diff options
author | Ryo Nihei <nihei.dev@gmail.com> | 2021-08-01 15:29:18 +0900 |
---|---|---|
committer | Ryo Nihei <nihei.dev@gmail.com> | 2021-08-01 15:38:54 +0900 |
commit | 03e3688e3928c88c12107ea734c35281c814e0c0 (patch) | |
tree | 7f57554aec423098c8325238aef72cffdae7651e /compiler/compiler.go | |
parent | Fix CHANGELOG (diff) | |
download | tre-03e3688e3928c88c12107ea734c35281c814e0c0.tar.gz tre-03e3688e3928c88c12107ea734c35281c814e0c0.tar.xz |
Add unique kind IDs to tokens
Diffstat (limited to 'compiler/compiler.go')
-rw-r--r-- | compiler/compiler.go | 38 |
1 files changed, 38 insertions, 0 deletions
diff --git a/compiler/compiler.go b/compiler/compiler.go index f382d16..5d3e52f 100644 --- a/compiler/compiler.go +++ b/compiler/compiler.go @@ -69,9 +69,47 @@ func Compile(lexspec *spec.LexSpec, opts ...CompilerOption) (*spec.CompiledLexSp modeSpecs = append(modeSpecs, modeSpec) } + var kindNames []spec.LexKind + var name2ID map[spec.LexKind]spec.LexKindID + { + name2ID = map[spec.LexKind]spec.LexKindID{} + id := spec.LexKindIDMin + for _, modeSpec := range modeSpecs[1:] { + for _, name := range modeSpec.Kinds[1:] { + if _, ok := name2ID[name]; ok { + continue + } + name2ID[name] = id + id++ + } + } + + kindNames = make([]spec.LexKind, len(name2ID)+1) + for name, id := range name2ID { + kindNames[id] = name + } + } + + var kindIDs [][]spec.LexKindID + { + kindIDs = make([][]spec.LexKindID, len(modeSpecs)) + for i, modeSpec := range modeSpecs[1:] { + ids := make([]spec.LexKindID, len(modeSpec.Kinds)) + for modeID, name := range modeSpec.Kinds { + if modeID == 0 { + continue + } + ids[modeID] = name2ID[name] + } + kindIDs[i+1] = ids + } + } + return &spec.CompiledLexSpec{ InitialMode: spec.LexModeNumDefault, Modes: modes, + Kinds: kindNames, + KindIDs: kindIDs, CompressionLevel: config.compLv, Specs: modeSpecs, }, nil |