aboutsummaryrefslogtreecommitdiff
path: root/src/urubu/driver/lexer
diff options
context:
space:
mode:
Diffstat (limited to 'src/urubu/driver/lexer')
-rw-r--r--src/urubu/driver/lexer/lexer.go335
-rw-r--r--src/urubu/driver/lexer/spec.go71
-rw-r--r--src/urubu/driver/lexer/template.go760
3 files changed, 0 insertions, 1166 deletions
diff --git a/src/urubu/driver/lexer/lexer.go b/src/urubu/driver/lexer/lexer.go
deleted file mode 100644
index 3f9712e..0000000
--- a/src/urubu/driver/lexer/lexer.go
+++ /dev/null
@@ -1,335 +0,0 @@
-package lexer
-
-import (
- "fmt"
- "io"
-)
-
-type ModeID int
-
-func (id ModeID) Int() int {
- return int(id)
-}
-
-type StateID int
-
-func (id StateID) Int() int {
- return int(id)
-}
-
-type KindID int
-
-func (id KindID) Int() int {
- return int(id)
-}
-
-type ModeKindID int
-
-func (id ModeKindID) Int() int {
- return int(id)
-}
-
-type LexSpec interface {
- InitialMode() ModeID
- Pop(mode ModeID, modeKind ModeKindID) bool
- Push(mode ModeID, modeKind ModeKindID) (ModeID, bool)
- ModeName(mode ModeID) string
- InitialState(mode ModeID) StateID
- NextState(mode ModeID, state StateID, v int) (StateID, bool)
- Accept(mode ModeID, state StateID) (ModeKindID, bool)
- KindIDAndName(mode ModeID, modeKind ModeKindID) (KindID, string)
-}
-
-// Token representes a token.
-type Token struct {
- // ModeID is an ID of a lex mode.
- ModeID ModeID
-
- // KindID is an ID of a kind. This is unique among all modes.
- KindID KindID
-
- // ModeKindID is an ID of a lexical kind. This is unique only within a mode.
- // Note that you need to use KindID field if you want to identify a kind across all modes.
- ModeKindID ModeKindID
-
- // BytePos is a byte position where a token appears.
- BytePos int
-
- // ByteLen is a length of a token.
- ByteLen int
-
- // Row is a row number where a token appears.
- Row int
-
- // Col is a column number where a token appears.
- // Note that Col is counted in code points, not bytes.
- Col int
-
- // Lexeme is a byte sequence matched a pattern of a lexical specification.
- Lexeme []byte
-
- // When this field is true, it means the token is the EOF token.
- EOF bool
-
- // When this field is true, it means the token is an error token.
- Invalid bool
-}
-
-type LexerOption func(l *Lexer) error
-
-// DisableModeTransition disables the active mode transition. Thus, even if the lexical specification has the push and pop
-// operations, the lexer doesn't perform these operations. When the lexical specification has multiple modes, and this option is
-// enabled, you need to call the Lexer.Push and Lexer.Pop methods to perform the mode transition. You can use the Lexer.Mode method
-// to know the current lex mode.
-func DisableModeTransition() LexerOption {
- return func(l *Lexer) error {
- l.passiveModeTran = true
- return nil
- }
-}
-
-type lexerState struct {
- srcPtr int
- row int
- col int
-}
-
-type Lexer struct {
- spec LexSpec
- src []byte
- state lexerState
- lastAcceptedState lexerState
- tokBuf []*Token
- modeStack []ModeID
- passiveModeTran bool
-}
-
-// NewLexer returns a new lexer.
-func NewLexer(spec LexSpec, src io.Reader, opts ...LexerOption) (*Lexer, error) {
- b, err := io.ReadAll(src)
- if err != nil {
- return nil, err
- }
- l := &Lexer{
- spec: spec,
- src: b,
- state: lexerState{
- srcPtr: 0,
- row: 0,
- col: 0,
- },
- lastAcceptedState: lexerState{
- srcPtr: 0,
- row: 0,
- col: 0,
- },
- modeStack: []ModeID{
- spec.InitialMode(),
- },
- passiveModeTran: false,
- }
- for _, opt := range opts {
- err := opt(l)
- if err != nil {
- return nil, err
- }
- }
-
- return l, nil
-}
-
-// Next returns a next token.
-func (l *Lexer) Next() (*Token, error) {
- if len(l.tokBuf) > 0 {
- tok := l.tokBuf[0]
- l.tokBuf = l.tokBuf[1:]
- return tok, nil
- }
-
- tok, err := l.nextAndTransition()
- if err != nil {
- return nil, err
- }
- if !tok.Invalid {
- return tok, nil
- }
- errTok := tok
- for {
- tok, err = l.nextAndTransition()
- if err != nil {
- return nil, err
- }
- if !tok.Invalid {
- break
- }
- errTok.ByteLen += tok.ByteLen
- errTok.Lexeme = append(errTok.Lexeme, tok.Lexeme...)
- }
- l.tokBuf = append(l.tokBuf, tok)
-
- return errTok, nil
-}
-
-func (l *Lexer) nextAndTransition() (*Token, error) {
- tok, err := l.next()
- if err != nil {
- return nil, err
- }
- if tok.EOF || tok.Invalid {
- return tok, nil
- }
- if l.passiveModeTran {
- return tok, nil
- }
- mode := l.Mode()
- if l.spec.Pop(mode, tok.ModeKindID) {
- err := l.PopMode()
- if err != nil {
- return nil, err
- }
- }
- if mode, ok := l.spec.Push(mode, tok.ModeKindID); ok {
- l.PushMode(mode)
- }
- // The checking length of the mode stack must be at after pop and push operations because those operations can be performed
- // at the same time. When the mode stack has just one element and popped it, the mode stack will be temporarily emptied.
- // However, since a push operation may be performed immediately after it, the lexer allows the stack to be temporarily empty.
- if len(l.modeStack) == 0 {
- return nil, fmt.Errorf("a mode stack must have at least one element")
- }
- return tok, nil
-}
-
-func (l *Lexer) next() (*Token, error) {
- mode := l.Mode()
- state := l.spec.InitialState(mode)
- buf := []byte{}
- startPos := l.state.srcPtr
- row := l.state.row
- col := l.state.col
- var tok *Token
- for {
- v, eof := l.read()
- if eof {
- if tok != nil {
- l.revert()
- return tok, nil
- }
- // When `buf` has unaccepted data and reads the EOF, the lexer treats the buffered data as an invalid token.
- if len(buf) > 0 {
- return &Token{
- ModeID: mode,
- ModeKindID: 0,
- BytePos: startPos,
- ByteLen: l.state.srcPtr - startPos,
- Lexeme: buf,
- Row: row,
- Col: col,
- Invalid: true,
- }, nil
- }
- return &Token{
- ModeID: mode,
- ModeKindID: 0,
- BytePos: startPos,
- Row: row,
- Col: col,
- EOF: true,
- }, nil
- }
- buf = append(buf, v)
- nextState, ok := l.spec.NextState(mode, state, int(v))
- if !ok {
- if tok != nil {
- l.revert()
- return tok, nil
- }
- return &Token{
- ModeID: mode,
- ModeKindID: 0,
- BytePos: startPos,
- ByteLen: l.state.srcPtr - startPos,
- Lexeme: buf,
- Row: row,
- Col: col,
- Invalid: true,
- }, nil
- }
- state = nextState
- if modeKindID, ok := l.spec.Accept(mode, state); ok {
- kindID, _ := l.spec.KindIDAndName(mode, modeKindID)
- tok = &Token{
- ModeID: mode,
- KindID: kindID,
- ModeKindID: modeKindID,
- BytePos: startPos,
- ByteLen: l.state.srcPtr - startPos,
- Lexeme: buf,
- Row: row,
- Col: col,
- }
- l.accept()
- }
- }
-}
-
-// Mode returns the current lex mode.
-func (l *Lexer) Mode() ModeID {
- return l.modeStack[len(l.modeStack)-1]
-}
-
-// PushMode adds a lex mode onto the mode stack.
-func (l *Lexer) PushMode(mode ModeID) {
- l.modeStack = append(l.modeStack, mode)
-}
-
-// PopMode removes a lex mode from the top of the mode stack.
-func (l *Lexer) PopMode() error {
- sLen := len(l.modeStack)
- if sLen == 0 {
- return fmt.Errorf("cannot pop a lex mode from a lex mode stack any more")
- }
- l.modeStack = l.modeStack[:sLen-1]
- return nil
-}
-
-func (l *Lexer) read() (byte, bool) {
- if l.state.srcPtr >= len(l.src) {
- return 0, true
- }
-
- b := l.src[l.state.srcPtr]
- l.state.srcPtr++
-
- // Count the token positions.
- // The driver treats LF as the end of lines and counts columns in code points, not bytes.
- // To count in code points, we refer to the First Byte column in the Table 3-6.
- //
- // Reference:
- // - [Table 3-6] https://www.unicode.org/versions/Unicode13.0.0/ch03.pdf > Table 3-6. UTF-8 Bit Distribution
- if b < 128 {
- // 0x0A is LF.
- if b == 0x0A {
- l.state.row++
- l.state.col = 0
- } else {
- l.state.col++
- }
- } else if b>>5 == 6 || b>>4 == 14 || b>>3 == 30 {
- l.state.col++
- }
-
- return b, false
-}
-
-// accept saves the current state.
-func (l *Lexer) accept() {
- l.lastAcceptedState = l.state
-}
-
-// revert reverts the lexer state to the last accepted state.
-//
-// We must not call this function consecutively.
-func (l *Lexer) revert() {
- l.state = l.lastAcceptedState
-}
diff --git a/src/urubu/driver/lexer/spec.go b/src/urubu/driver/lexer/spec.go
deleted file mode 100644
index 75c74af..0000000
--- a/src/urubu/driver/lexer/spec.go
+++ /dev/null
@@ -1,71 +0,0 @@
-package lexer
-
-import spec "urubu/spec/grammar"
-
-type lexSpec struct {
- spec *spec.LexicalSpec
-}
-
-func NewLexSpec(spec *spec.LexicalSpec) *lexSpec {
- return &lexSpec{
- spec: spec,
- }
-}
-
-func (s *lexSpec) InitialMode() ModeID {
- return ModeID(s.spec.InitialModeID.Int())
-}
-
-func (s *lexSpec) Pop(mode ModeID, modeKind ModeKindID) bool {
- return s.spec.Specs[mode].Pop[modeKind] == 1
-}
-
-func (s *lexSpec) Push(mode ModeID, modeKind ModeKindID) (ModeID, bool) {
- modeID := s.spec.Specs[mode].Push[modeKind]
- return ModeID(modeID.Int()), !modeID.IsNil()
-}
-
-func (s *lexSpec) ModeName(mode ModeID) string {
- return s.spec.ModeNames[mode].String()
-}
-
-func (s *lexSpec) InitialState(mode ModeID) StateID {
- return StateID(s.spec.Specs[mode].DFA.InitialStateID.Int())
-}
-
-func (s *lexSpec) NextState(mode ModeID, state StateID, v int) (StateID, bool) {
- switch s.spec.CompressionLevel {
- case 2:
- tran := s.spec.Specs[mode].DFA.Transition
- rowNum := tran.RowNums[state]
- d := tran.UniqueEntries.RowDisplacement[rowNum]
- if tran.UniqueEntries.Bounds[d+v] != rowNum {
- return StateID(tran.UniqueEntries.EmptyValue.Int()), false
- }
- return StateID(tran.UniqueEntries.Entries[d+v].Int()), true
- case 1:
- tran := s.spec.Specs[mode].DFA.Transition
- next := tran.UncompressedUniqueEntries[tran.RowNums[state]*tran.OriginalColCount+v]
- if next == spec.StateIDNil {
- return StateID(spec.StateIDNil.Int()), false
- }
- return StateID(next.Int()), true
- }
-
- modeSpec := s.spec.Specs[mode]
- next := modeSpec.DFA.UncompressedTransition[state.Int()*modeSpec.DFA.ColCount+v]
- if next == spec.StateIDNil {
- return StateID(spec.StateIDNil), false
- }
- return StateID(next.Int()), true
-}
-
-func (s *lexSpec) Accept(mode ModeID, state StateID) (ModeKindID, bool) {
- modeKindID := s.spec.Specs[mode].DFA.AcceptingStates[state]
- return ModeKindID(modeKindID.Int()), modeKindID != spec.LexModeKindIDNil
-}
-
-func (s *lexSpec) KindIDAndName(mode ModeID, modeKind ModeKindID) (KindID, string) {
- kindID := s.spec.KindIDs[mode][modeKind]
- return KindID(kindID.Int()), s.spec.KindNames[kindID].String()
-}
diff --git a/src/urubu/driver/lexer/template.go b/src/urubu/driver/lexer/template.go
deleted file mode 100644
index 35dfd93..0000000
--- a/src/urubu/driver/lexer/template.go
+++ /dev/null
@@ -1,760 +0,0 @@
-package lexer
-
-import (
- "bytes"
- _ "embed"
- "fmt"
- "go/ast"
- "go/format"
- "go/parser"
- "go/token"
- "strings"
- "text/template"
-
- "urubu/grammar/lexical"
- spec "urubu/spec/grammar"
-)
-
-// go:embed lexer.go
-var lexerCoreSrc string
-
-func GenLexer(lexSpec *spec.LexicalSpec, pkgName string) ([]byte, error) {
- var lexerSrc string
- {
- fset := token.NewFileSet()
- f, err := parser.ParseFile(fset, "lexer.go", lexerCoreSrc, parser.ParseComments)
- if err != nil {
- return nil, err
- }
-
- var b strings.Builder
- err = format.Node(&b, fset, f)
- if err != nil {
- return nil, err
- }
-
- lexerSrc = b.String()
- }
-
- var modeIDsSrc string
- {
- var b strings.Builder
- fmt.Fprintf(&b, "const (\n")
- for i, k := range lexSpec.ModeNames {
- if i == spec.LexModeIDNil.Int() {
- fmt.Fprintf(&b, " ModeIDNil ModeID = %v\n", i)
- continue
- }
- fmt.Fprintf(&b, " ModeID%v ModeID = %v\n", lexical.SnakeCaseToUpperCamelCase(k.String()), i)
- }
- fmt.Fprintf(&b, ")")
-
- modeIDsSrc = b.String()
- }
-
- var modeNamesSrc string
- {
- var b strings.Builder
- fmt.Fprintf(&b, "const (\n")
- for i, k := range lexSpec.ModeNames {
- if i == spec.LexModeIDNil.Int() {
- fmt.Fprintf(&b, " ModeNameNil = %#v\n", "")
- continue
- }
- fmt.Fprintf(&b, " ModeName%v = %#v\n", lexical.SnakeCaseToUpperCamelCase(k.String()), k)
- }
- fmt.Fprintf(&b, ")")
-
- modeNamesSrc = b.String()
- }
-
- var modeIDToNameSrc string
- {
- var b strings.Builder
- fmt.Fprintf(&b, `
-// ModeIDToName converts a mode ID to a name.
-func ModeIDToName(id ModeID) string {
- switch id {`)
- for i, k := range lexSpec.ModeNames {
- if i == spec.LexModeIDNil.Int() {
- fmt.Fprintf(&b, `
- case ModeIDNil:
- return ModeNameNil`)
- continue
- }
- name := lexical.SnakeCaseToUpperCamelCase(k.String())
- fmt.Fprintf(&b, `
- case ModeID%v:
- return ModeName%v`, name, name)
- }
- fmt.Fprintf(&b, `
- }
- return ""
-}
-`)
-
- modeIDToNameSrc = b.String()
- }
-
- var kindIDsSrc string
- {
- var b strings.Builder
- fmt.Fprintf(&b, "const (\n")
- for i, k := range lexSpec.KindNames {
- if i == spec.LexKindIDNil.Int() {
- fmt.Fprintf(&b, " KindIDNil KindID = %v\n", i)
- continue
- }
- fmt.Fprintf(&b, " KindID%v KindID = %v\n", lexical.SnakeCaseToUpperCamelCase(k.String()), i)
- }
- fmt.Fprintf(&b, ")")
-
- kindIDsSrc = b.String()
- }
-
- var kindNamesSrc string
- {
- var b strings.Builder
- fmt.Fprintf(&b, "const (\n")
- fmt.Fprintf(&b, " KindNameNil = %#v\n", "")
- for _, k := range lexSpec.KindNames[1:] {
- fmt.Fprintf(&b, " KindName%v = %#v\n", lexical.SnakeCaseToUpperCamelCase(k.String()), k)
- }
- fmt.Fprintf(&b, ")")
-
- kindNamesSrc = b.String()
- }
-
- var kindIDToNameSrc string
- {
- var b strings.Builder
- fmt.Fprintf(&b, `
-// KindIDToName converts a kind ID to a name.
-func KindIDToName(id KindID) string {
- switch id {`)
- for i, k := range lexSpec.KindNames {
- if i == spec.LexModeIDNil.Int() {
- fmt.Fprintf(&b, `
- case KindIDNil:
- return KindNameNil`)
- continue
- }
- name := lexical.SnakeCaseToUpperCamelCase(k.String())
- fmt.Fprintf(&b, `
- case KindID%v:
- return KindName%v`, name, name)
- }
- fmt.Fprintf(&b, `
- }
- return ""
-}
-`)
-
- kindIDToNameSrc = b.String()
- }
-
- var specSrc string
- {
- t, err := template.New("").Funcs(genTemplateFuncs(lexSpec)).Parse(lexSpecTemplate)
- if err != nil {
- return nil, err
- }
-
- var b strings.Builder
- err = t.Execute(&b, map[string]interface{}{
- "initialModeID": "ModeID" + lexical.SnakeCaseToUpperCamelCase(lexSpec.ModeNames[lexSpec.InitialModeID].String()),
- "modeIDNil": "ModeIDNil",
- "modeKindIDNil": spec.LexModeKindIDNil,
- "stateIDNil": spec.StateIDNil,
- "compressionLevel": lexSpec.CompressionLevel,
- })
- if err != nil {
- return nil, err
- }
-
- specSrc = b.String()
- }
-
- var src string
- {
- tmpl := `// Code generated by vartan-go. DO NOT EDIT.
-{{ .lexerSrc }}
-
-{{ .modeIDsSrc }}
-
-{{ .modeNamesSrc }}
-
-{{ .modeIDToNameSrc }}
-
-{{ .kindIDsSrc }}
-
-{{ .kindNamesSrc }}
-
-{{ .kindIDToNameSrc }}
-
-{{ .specSrc }}
-`
-
- t, err := template.New("").Parse(tmpl)
- if err != nil {
- return nil, err
- }
-
- var b strings.Builder
- err = t.Execute(&b, map[string]string{
- "lexerSrc": lexerSrc,
- "modeIDsSrc": modeIDsSrc,
- "modeNamesSrc": modeNamesSrc,
- "modeIDToNameSrc": modeIDToNameSrc,
- "kindIDsSrc": kindIDsSrc,
- "kindNamesSrc": kindNamesSrc,
- "kindIDToNameSrc": kindIDToNameSrc,
- "specSrc": specSrc,
- })
- if err != nil {
- return nil, err
- }
-
- src = b.String()
- }
-
- fset := token.NewFileSet()
- f, err := parser.ParseFile(fset, "", src, parser.ParseComments)
- if err != nil {
- return nil, err
- }
-
- f.Name = ast.NewIdent(pkgName)
-
- var b bytes.Buffer
- err = format.Node(&b, fset, f)
- if err != nil {
- return nil, err
- }
-
- return b.Bytes(), nil
-}
-
-const lexSpecTemplate = `
-type lexSpec struct {
- pop [][]bool
- push [][]ModeID
- modeNames []string
- initialStates []StateID
- acceptances [][]ModeKindID
- kindIDs [][]KindID
- kindNames []string
- initialModeID ModeID
- modeIDNil ModeID
- modeKindIDNil ModeKindID
- stateIDNil StateID
-
- rowNums [][]int
- rowDisplacements [][]int
- bounds [][]int
- entries [][]StateID
- originalColCounts []int
-}
-
-func NewLexSpec() *lexSpec {
- return &lexSpec{
- pop: {{ genPopTable }},
- push: {{ genPushTable }},
- modeNames: {{ genModeNameTable }},
- initialStates: {{ genInitialStateTable }},
- acceptances: {{ genAcceptTable }},
- kindIDs: {{ genKindIDTable }},
- kindNames: {{ genKindNameTable }},
- initialModeID: {{ .initialModeID }},
- modeIDNil: {{ .modeIDNil }},
- modeKindIDNil: {{ .modeKindIDNil }},
- stateIDNil: {{ .stateIDNil }},
-
- rowNums: {{ genRowNums }},
- rowDisplacements: {{ genRowDisplacements }},
- bounds: {{ genBounds }},
- entries: {{ genEntries }},
- originalColCounts: {{ genOriginalColCounts }},
- }
-}
-
-func (s *lexSpec) InitialMode() ModeID {
- return s.initialModeID
-}
-
-func (s *lexSpec) Pop(mode ModeID, modeKind ModeKindID) bool {
- return s.pop[mode][modeKind]
-}
-
-func (s *lexSpec) Push(mode ModeID, modeKind ModeKindID) (ModeID, bool) {
- id := s.push[mode][modeKind]
- return id, id != s.modeIDNil
-}
-
-func (s *lexSpec) ModeName(mode ModeID) string {
- return s.modeNames[mode]
-}
-
-func (s *lexSpec) InitialState(mode ModeID) StateID {
- return s.initialStates[mode]
-}
-
-func (s *lexSpec) NextState(mode ModeID, state StateID, v int) (StateID, bool) {
-{{ if eq .compressionLevel 2 -}}
- rowNum := s.rowNums[mode][state]
- d := s.rowDisplacements[mode][rowNum]
- if s.bounds[mode][d+v] != rowNum {
- return s.stateIDNil, false
- }
- return s.entries[mode][d+v], true
-{{ else if eq .compressionLevel 1 -}}
- rowNum := s.rowNums[mode][state]
- colCount := s.originalColCounts[mode]
- next := s.entries[mode][rowNum*colCount+v]
- if next == s.stateIDNil {
- return s.stateIDNil, false
- }
- return next, true
-{{ else -}}
- colCount := s.originalColCounts[mode]
- next := s.entries[mode][int(state)*colCount+v]
- if next == s.stateIDNil {
- return s.stateIDNil, false
- }
- return next, true
-{{ end -}}
-}
-
-func (s *lexSpec) Accept(mode ModeID, state StateID) (ModeKindID, bool) {
- id := s.acceptances[mode][state]
- return id, id != s.modeKindIDNil
-}
-
-func (s *lexSpec) KindIDAndName(mode ModeID, modeKind ModeKindID) (KindID, string) {
- id := s.kindIDs[mode][modeKind]
- return id, s.kindNames[id]
-}
-`
-
-func genTemplateFuncs(lexSpec *spec.LexicalSpec) template.FuncMap {
- fns := template.FuncMap{
- "genPopTable": func() string {
- var b strings.Builder
- fmt.Fprintf(&b, "[][]bool{\n")
- for i, s := range lexSpec.Specs {
- if i == spec.LexModeIDNil.Int() {
- fmt.Fprintf(&b, "nil,\n")
- continue
- }
-
- c := 1
- fmt.Fprintf(&b, "{\n")
- for _, v := range s.Pop {
- fmt.Fprintf(&b, "%v, ", v != 0)
-
- if c == 20 {
- fmt.Fprintf(&b, "\n")
- c = 1
- } else {
- c++
- }
- }
- if c > 1 {
- fmt.Fprintf(&b, "\n")
- }
- fmt.Fprintf(&b, "},\n")
- }
- fmt.Fprintf(&b, "}")
- return b.String()
- },
- "genPushTable": func() string {
- var b strings.Builder
- fmt.Fprintf(&b, "[][]ModeID{\n")
- for i, s := range lexSpec.Specs {
- if i == spec.LexModeIDNil.Int() {
- fmt.Fprintf(&b, "nil,\n")
- continue
- }
-
- c := 1
- fmt.Fprintf(&b, "{\n")
- for _, v := range s.Push {
- fmt.Fprintf(&b, "%v,", v)
-
- if c == 20 {
- fmt.Fprintf(&b, "\n")
- c = 1
- } else {
- c++
- }
- }
- if c > 1 {
- fmt.Fprintf(&b, "\n")
- }
- fmt.Fprintf(&b, "},\n")
- }
- fmt.Fprintf(&b, "}")
- return b.String()
- },
- "genModeNameTable": func() string {
- var b strings.Builder
- fmt.Fprintf(&b, "[]string{\n")
- for i, name := range lexSpec.ModeNames {
- if i == spec.LexModeIDNil.Int() {
- fmt.Fprintf(&b, "ModeNameNil,\n")
- continue
- }
- fmt.Fprintf(&b, "ModeName%v,\n", lexical.SnakeCaseToUpperCamelCase(name.String()))
- }
- fmt.Fprintf(&b, "}")
- return b.String()
- },
- "genInitialStateTable": func() string {
- var b strings.Builder
- fmt.Fprintf(&b, "[]StateID{\n")
- for i, s := range lexSpec.Specs {
- if i == spec.LexModeIDNil.Int() {
- fmt.Fprintf(&b, "%v,\n", spec.StateIDNil)
- continue
- }
-
- fmt.Fprintf(&b, "%v,\n", s.DFA.InitialStateID)
- }
- fmt.Fprintf(&b, "}")
- return b.String()
- },
- "genAcceptTable": func() string {
- var b strings.Builder
- fmt.Fprintf(&b, "[][]ModeKindID{\n")
- for i, s := range lexSpec.Specs {
- if i == spec.LexModeIDNil.Int() {
- fmt.Fprintf(&b, "nil,\n")
- continue
- }
-
- c := 1
- fmt.Fprintf(&b, "{\n")
- for _, v := range s.DFA.AcceptingStates {
- fmt.Fprintf(&b, "%v,", v)
-
- if c == 20 {
- fmt.Fprintf(&b, "\n")
- c = 1
- } else {
- c++
- }
- }
- if c > 1 {
- fmt.Fprintf(&b, "\n")
- }
- fmt.Fprintf(&b, "},\n")
- }
- fmt.Fprintf(&b, "}")
- return b.String()
- },
- "genKindIDTable": func() string {
- var b strings.Builder
- fmt.Fprintf(&b, "[][]KindID{\n")
- for i, ids := range lexSpec.KindIDs {
- if i == spec.LexModeIDNil.Int() {
- fmt.Fprintf(&b, "nil,\n")
- continue
- }
-
- fmt.Fprintf(&b, "{\n")
- for j, id := range ids {
- if j == spec.LexModeKindIDNil.Int() {
- fmt.Fprintf(&b, "KindIDNil,\n")
- continue
- }
- fmt.Fprintf(&b, "KindID%v,\n", lexical.SnakeCaseToUpperCamelCase(string(lexSpec.KindNames[id].String())))
- }
- fmt.Fprintf(&b, "},\n")
- }
- fmt.Fprintf(&b, "}")
- return b.String()
- },
- "genKindNameTable": func() string {
- var b strings.Builder
- fmt.Fprintf(&b, "[]string{\n")
- for i, name := range lexSpec.KindNames {
- if i == spec.LexKindIDNil.Int() {
- fmt.Fprintf(&b, "KindNameNil,\n")
- continue
- }
- fmt.Fprintf(&b, "KindName%v,\n", lexical.SnakeCaseToUpperCamelCase(name.String()))
- }
- fmt.Fprintf(&b, "}")
- return b.String()
- },
- }
-
- switch lexSpec.CompressionLevel {
- case 2:
- fns["genRowNums"] = func() string {
- var b strings.Builder
- fmt.Fprintf(&b, "[][]int{\n")
- for i, s := range lexSpec.Specs {
- if i == spec.LexModeIDNil.Int() {
- fmt.Fprintf(&b, "nil,\n")
- continue
- }
-
- c := 1
- fmt.Fprintf(&b, "{\n")
- for _, v := range s.DFA.Transition.RowNums {
- fmt.Fprintf(&b, "%v,", v)
-
- if c == 20 {
- fmt.Fprintf(&b, "\n")
- c = 1
- } else {
- c++
- }
- }
- if c > 1 {
- fmt.Fprintf(&b, "\n")
- }
- fmt.Fprintf(&b, "},\n")
- }
- fmt.Fprintf(&b, "}")
- return b.String()
- }
-
- fns["genRowDisplacements"] = func() string {
- var b strings.Builder
- fmt.Fprintf(&b, "[][]int{\n")
- for i, s := range lexSpec.Specs {
- if i == spec.LexModeIDNil.Int() {
- fmt.Fprintf(&b, "nil,\n")
- continue
- }
-
- c := 1
- fmt.Fprintf(&b, "{\n")
- for _, d := range s.DFA.Transition.UniqueEntries.RowDisplacement {
- fmt.Fprintf(&b, "%v,", d)
-
- if c == 20 {
- fmt.Fprintf(&b, "\n")
- c = 1
- } else {
- c++
- }
- }
- if c > 1 {
- fmt.Fprintf(&b, "\n")
- }
- fmt.Fprintf(&b, "},\n")
- }
- fmt.Fprintf(&b, "}")
- return b.String()
- }
-
- fns["genBounds"] = func() string {
- var b strings.Builder
- fmt.Fprintf(&b, "[][]int{\n")
- for i, s := range lexSpec.Specs {
- if i == spec.LexModeIDNil.Int() {
- fmt.Fprintf(&b, "nil,\n")
- continue
- }
-
- c := 1
- fmt.Fprintf(&b, "{\n")
- for _, v := range s.DFA.Transition.UniqueEntries.Bounds {
- fmt.Fprintf(&b, "%v,", v)
-
- if c == 20 {
- fmt.Fprintf(&b, "\n")
- c = 1
- } else {
- c++
- }
- }
- if c > 1 {
- fmt.Fprintf(&b, "\n")
- }
- fmt.Fprintf(&b, "},\n")
- }
- fmt.Fprintf(&b, "}")
- return b.String()
- }
-
- fns["genEntries"] = func() string {
- var b strings.Builder
- fmt.Fprintf(&b, "[][]StateID{\n")
- for i, s := range lexSpec.Specs {
- if i == spec.LexModeIDNil.Int() {
- fmt.Fprintf(&b, "nil,\n")
- continue
- }
-
- c := 1
- fmt.Fprintf(&b, "{\n")
- for _, v := range s.DFA.Transition.UniqueEntries.Entries {
- fmt.Fprintf(&b, "%v,", v)
-
- if c == 20 {
- fmt.Fprintf(&b, "\n")
- c = 1
- } else {
- c++
- }
- }
- if c > 1 {
- fmt.Fprintf(&b, "\n")
- }
- fmt.Fprintf(&b, "},\n")
- }
- fmt.Fprintf(&b, "}")
- return b.String()
- }
-
- fns["genOriginalColCounts"] = func() string {
- return "nil"
- }
- case 1:
- fns["genRowNums"] = func() string {
- var b strings.Builder
- fmt.Fprintf(&b, "[][]int{\n")
- for i, s := range lexSpec.Specs {
- if i == spec.LexModeIDNil.Int() {
- fmt.Fprintf(&b, "nil,\n")
- continue
- }
-
- c := 1
- fmt.Fprintf(&b, "{\n")
- for _, v := range s.DFA.Transition.RowNums {
- fmt.Fprintf(&b, "%v,", v)
-
- if c == 20 {
- fmt.Fprintf(&b, "\n")
- c = 1
- } else {
- c++
- }
- }
- if c > 1 {
- fmt.Fprintf(&b, "\n")
- }
- fmt.Fprintf(&b, "},\n")
- }
- fmt.Fprintf(&b, "}")
- return b.String()
- }
-
- fns["genRowDisplacements"] = func() string {
- return "nil"
- }
-
- fns["genBounds"] = func() string {
- return "nil"
- }
-
- fns["genEntries"] = func() string {
- var b strings.Builder
- fmt.Fprintf(&b, "[][]StateID{\n")
- for i, s := range lexSpec.Specs {
- if i == spec.LexModeIDNil.Int() {
- fmt.Fprintf(&b, "nil,\n")
- continue
- }
-
- c := 1
- fmt.Fprintf(&b, "{\n")
- for _, v := range s.DFA.Transition.UncompressedUniqueEntries {
- fmt.Fprintf(&b, "%v,", v)
-
- if c == 20 {
- fmt.Fprintf(&b, "\n")
- c = 1
- } else {
- c++
- }
- }
- if c > 1 {
- fmt.Fprintf(&b, "\n")
- }
- fmt.Fprintf(&b, "},\n")
- }
- fmt.Fprintf(&b, "}")
- return b.String()
- }
-
- fns["genOriginalColCounts"] = func() string {
- var b strings.Builder
- fmt.Fprintf(&b, "[]int{\n")
- for i, s := range lexSpec.Specs {
- if i == spec.LexModeIDNil.Int() {
- fmt.Fprintf(&b, "0,\n")
- continue
- }
-
- fmt.Fprintf(&b, "%v,\n", s.DFA.Transition.OriginalColCount)
- }
- fmt.Fprintf(&b, "}")
- return b.String()
- }
- default:
- fns["genRowNums"] = func() string {
- return "nil"
- }
-
- fns["genRowDisplacements"] = func() string {
- return "nil"
- }
-
- fns["genBounds"] = func() string {
- return "nil"
- }
-
- fns["genEntries"] = func() string {
- var b strings.Builder
- fmt.Fprintf(&b, "[][]StateID{\n")
- for i, s := range lexSpec.Specs {
- if i == spec.LexModeIDNil.Int() {
- fmt.Fprintf(&b, "nil,\n")
- continue
- }
-
- c := 1
- fmt.Fprintf(&b, "{\n")
- for _, v := range s.DFA.UncompressedTransition {
- fmt.Fprintf(&b, "%v,", v)
-
- if c == 20 {
- fmt.Fprintf(&b, "\n")
- c = 1
- } else {
- c++
- }
- }
- if c > 1 {
- fmt.Fprintf(&b, "\n")
- }
- fmt.Fprintf(&b, "},\n")
- }
- fmt.Fprintf(&b, "}")
- return b.String()
- }
-
- fns["genOriginalColCounts"] = func() string {
- var b strings.Builder
- fmt.Fprintf(&b, "[]int{\n")
- for i, s := range lexSpec.Specs {
- if i == spec.LexModeIDNil.Int() {
- fmt.Fprintf(&b, "0,\n")
- continue
- }
-
- fmt.Fprintf(&b, "%v,\n", s.DFA.ColCount)
- }
- fmt.Fprintf(&b, "}")
- return b.String()
- }
- }
-
- return fns
-}
generated by cgit v1.2.3 (git 2.49.0) at 2025-07-13 18:09:50 +0000