package lexer import ( "bytes" _ "embed" "fmt" "go/ast" "go/format" "go/parser" "go/token" "io" "strings" "text/template" "urubu/grammar/lexical" spec "urubu/spec/grammar" ) type ModeID int func (id ModeID) Int() int { return int(id) } type StateID int func (id StateID) Int() int { return int(id) } type KindID int func (id KindID) Int() int { return int(id) } type ModeKindID int func (id ModeKindID) Int() int { return int(id) } type LexSpec interface { InitialMode() ModeID Pop(mode ModeID, modeKind ModeKindID) bool Push(mode ModeID, modeKind ModeKindID) (ModeID, bool) ModeName(mode ModeID) string InitialState(mode ModeID) StateID NextState(mode ModeID, state StateID, v int) (StateID, bool) Accept(mode ModeID, state StateID) (ModeKindID, bool) KindIDAndName(mode ModeID, modeKind ModeKindID) (KindID, string) } // Token representes a token. type Token struct { // ModeID is an ID of a lex mode. ModeID ModeID // KindID is an ID of a kind. This is unique among all modes. KindID KindID // ModeKindID is an ID of a lexical kind. This is unique only within a mode. // Note that you need to use KindID field if you want to identify a kind across all modes. ModeKindID ModeKindID // BytePos is a byte position where a token appears. BytePos int // ByteLen is a length of a token. ByteLen int // Row is a row number where a token appears. Row int // Col is a column number where a token appears. // Note that Col is counted in code points, not bytes. Col int // Lexeme is a byte sequence matched a pattern of a lexical specification. Lexeme []byte // When this field is true, it means the token is the EOF token. EOF bool // When this field is true, it means the token is an error token. Invalid bool } type LexerOption func(l *Lexer) error // DisableModeTransition disables the active mode transition. Thus, even if the lexical specification has the push and pop // operations, the lexer doesn't perform these operations. When the lexical specification has multiple modes, and this option is // enabled, you need to call the Lexer.Push and Lexer.Pop methods to perform the mode transition. You can use the Lexer.Mode method // to know the current lex mode. func DisableModeTransition() LexerOption { return func(l *Lexer) error { l.passiveModeTran = true return nil } } type lexerState struct { srcPtr int row int col int } type Lexer struct { spec LexSpec src []byte state lexerState lastAcceptedState lexerState tokBuf []*Token modeStack []ModeID passiveModeTran bool } // NewLexer returns a new lexer. func NewLexer(spec LexSpec, src io.Reader, opts ...LexerOption) (*Lexer, error) { b, err := io.ReadAll(src) if err != nil { return nil, err } l := &Lexer{ spec: spec, src: b, state: lexerState{ srcPtr: 0, row: 0, col: 0, }, lastAcceptedState: lexerState{ srcPtr: 0, row: 0, col: 0, }, modeStack: []ModeID{ spec.InitialMode(), }, passiveModeTran: false, } for _, opt := range opts { err := opt(l) if err != nil { return nil, err } } return l, nil } // Next returns a next token. func (l *Lexer) Next() (*Token, error) { if len(l.tokBuf) > 0 { tok := l.tokBuf[0] l.tokBuf = l.tokBuf[1:] return tok, nil } tok, err := l.nextAndTransition() if err != nil { return nil, err } if !tok.Invalid { return tok, nil } errTok := tok for { tok, err = l.nextAndTransition() if err != nil { return nil, err } if !tok.Invalid { break } errTok.ByteLen += tok.ByteLen errTok.Lexeme = append(errTok.Lexeme, tok.Lexeme...) } l.tokBuf = append(l.tokBuf, tok) return errTok, nil } func (l *Lexer) nextAndTransition() (*Token, error) { tok, err := l.next() if err != nil { return nil, err } if tok.EOF || tok.Invalid { return tok, nil } if l.passiveModeTran { return tok, nil } mode := l.Mode() if l.spec.Pop(mode, tok.ModeKindID) { err := l.PopMode() if err != nil { return nil, err } } if mode, ok := l.spec.Push(mode, tok.ModeKindID); ok { l.PushMode(mode) } // The checking length of the mode stack must be at after pop and push operations because those operations can be performed // at the same time. When the mode stack has just one element and popped it, the mode stack will be temporarily emptied. // However, since a push operation may be performed immediately after it, the lexer allows the stack to be temporarily empty. if len(l.modeStack) == 0 { return nil, fmt.Errorf("a mode stack must have at least one element") } return tok, nil } func (l *Lexer) next() (*Token, error) { mode := l.Mode() state := l.spec.InitialState(mode) buf := []byte{} startPos := l.state.srcPtr row := l.state.row col := l.state.col var tok *Token for { v, eof := l.read() if eof { if tok != nil { l.revert() return tok, nil } // When `buf` has unaccepted data and reads the EOF, the lexer treats the buffered data as an invalid token. if len(buf) > 0 { return &Token{ ModeID: mode, ModeKindID: 0, BytePos: startPos, ByteLen: l.state.srcPtr - startPos, Lexeme: buf, Row: row, Col: col, Invalid: true, }, nil } return &Token{ ModeID: mode, ModeKindID: 0, BytePos: startPos, Row: row, Col: col, EOF: true, }, nil } buf = append(buf, v) nextState, ok := l.spec.NextState(mode, state, int(v)) if !ok { if tok != nil { l.revert() return tok, nil } return &Token{ ModeID: mode, ModeKindID: 0, BytePos: startPos, ByteLen: l.state.srcPtr - startPos, Lexeme: buf, Row: row, Col: col, Invalid: true, }, nil } state = nextState if modeKindID, ok := l.spec.Accept(mode, state); ok { kindID, _ := l.spec.KindIDAndName(mode, modeKindID) tok = &Token{ ModeID: mode, KindID: kindID, ModeKindID: modeKindID, BytePos: startPos, ByteLen: l.state.srcPtr - startPos, Lexeme: buf, Row: row, Col: col, } l.accept() } } } // Mode returns the current lex mode. func (l *Lexer) Mode() ModeID { return l.modeStack[len(l.modeStack)-1] } // PushMode adds a lex mode onto the mode stack. func (l *Lexer) PushMode(mode ModeID) { l.modeStack = append(l.modeStack, mode) } // PopMode removes a lex mode from the top of the mode stack. func (l *Lexer) PopMode() error { sLen := len(l.modeStack) if sLen == 0 { return fmt.Errorf("cannot pop a lex mode from a lex mode stack any more") } l.modeStack = l.modeStack[:sLen-1] return nil } func (l *Lexer) read() (byte, bool) { if l.state.srcPtr >= len(l.src) { return 0, true } b := l.src[l.state.srcPtr] l.state.srcPtr++ // Count the token positions. // The driver treats LF as the end of lines and counts columns in code points, not bytes. // To count in code points, we refer to the First Byte column in the Table 3-6. // // Reference: // - [Table 3-6] https://www.unicode.org/versions/Unicode13.0.0/ch03.pdf > Table 3-6. UTF-8 Bit Distribution if b < 128 { // 0x0A is LF. if b == 0x0A { l.state.row++ l.state.col = 0 } else { l.state.col++ } } else if b>>5 == 6 || b>>4 == 14 || b>>3 == 30 { l.state.col++ } return b, false } // accept saves the current state. func (l *Lexer) accept() { l.lastAcceptedState = l.state } // revert reverts the lexer state to the last accepted state. // // We must not call this function consecutively. func (l *Lexer) revert() { l.state = l.lastAcceptedState } type lexSpec struct { spec *spec.LexicalSpec } func NewLexSpec(spec *spec.LexicalSpec) *lexSpec { return &lexSpec{ spec: spec, } } func (s *lexSpec) InitialMode() ModeID { return ModeID(s.spec.InitialModeID.Int()) } func (s *lexSpec) Pop(mode ModeID, modeKind ModeKindID) bool { return s.spec.Specs[mode].Pop[modeKind] == 1 } func (s *lexSpec) Push(mode ModeID, modeKind ModeKindID) (ModeID, bool) { modeID := s.spec.Specs[mode].Push[modeKind] return ModeID(modeID.Int()), !modeID.IsNil() } func (s *lexSpec) ModeName(mode ModeID) string { return s.spec.ModeNames[mode].String() } func (s *lexSpec) InitialState(mode ModeID) StateID { return StateID(s.spec.Specs[mode].DFA.InitialStateID.Int()) } func (s *lexSpec) NextState(mode ModeID, state StateID, v int) (StateID, bool) { switch s.spec.CompressionLevel { case 2: tran := s.spec.Specs[mode].DFA.Transition rowNum := tran.RowNums[state] d := tran.UniqueEntries.RowDisplacement[rowNum] if tran.UniqueEntries.Bounds[d+v] != rowNum { return StateID(tran.UniqueEntries.EmptyValue.Int()), false } return StateID(tran.UniqueEntries.Entries[d+v].Int()), true case 1: tran := s.spec.Specs[mode].DFA.Transition next := tran.UncompressedUniqueEntries[tran.RowNums[state]*tran.OriginalColCount+v] if next == spec.StateIDNil { return StateID(spec.StateIDNil.Int()), false } return StateID(next.Int()), true } modeSpec := s.spec.Specs[mode] next := modeSpec.DFA.UncompressedTransition[state.Int()*modeSpec.DFA.ColCount+v] if next == spec.StateIDNil { return StateID(spec.StateIDNil), false } return StateID(next.Int()), true } func (s *lexSpec) Accept(mode ModeID, state StateID) (ModeKindID, bool) { modeKindID := s.spec.Specs[mode].DFA.AcceptingStates[state] return ModeKindID(modeKindID.Int()), modeKindID != spec.LexModeKindIDNil } func (s *lexSpec) KindIDAndName(mode ModeID, modeKind ModeKindID) (KindID, string) { kindID := s.spec.KindIDs[mode][modeKind] return KindID(kindID.Int()), s.spec.KindNames[kindID].String() } // go:embed lexer.go var lexerCoreSrc string func GenLexer(lexSpec *spec.LexicalSpec, pkgName string) ([]byte, error) { var lexerSrc string { fset := token.NewFileSet() f, err := parser.ParseFile(fset, "lexer.go", lexerCoreSrc, parser.ParseComments) if err != nil { return nil, err } var b strings.Builder err = format.Node(&b, fset, f) if err != nil { return nil, err } lexerSrc = b.String() } var modeIDsSrc string { var b strings.Builder fmt.Fprintf(&b, "const (\n") for i, k := range lexSpec.ModeNames { if i == spec.LexModeIDNil.Int() { fmt.Fprintf(&b, " ModeIDNil ModeID = %v\n", i) continue } fmt.Fprintf(&b, " ModeID%v ModeID = %v\n", lexical.SnakeCaseToUpperCamelCase(k.String()), i) } fmt.Fprintf(&b, ")") modeIDsSrc = b.String() } var modeNamesSrc string { var b strings.Builder fmt.Fprintf(&b, "const (\n") for i, k := range lexSpec.ModeNames { if i == spec.LexModeIDNil.Int() { fmt.Fprintf(&b, " ModeNameNil = %#v\n", "") continue } fmt.Fprintf(&b, " ModeName%v = %#v\n", lexical.SnakeCaseToUpperCamelCase(k.String()), k) } fmt.Fprintf(&b, ")") modeNamesSrc = b.String() } var modeIDToNameSrc string { var b strings.Builder fmt.Fprintf(&b, ` // ModeIDToName converts a mode ID to a name. func ModeIDToName(id ModeID) string { switch id {`) for i, k := range lexSpec.ModeNames { if i == spec.LexModeIDNil.Int() { fmt.Fprintf(&b, ` case ModeIDNil: return ModeNameNil`) continue } name := lexical.SnakeCaseToUpperCamelCase(k.String()) fmt.Fprintf(&b, ` case ModeID%v: return ModeName%v`, name, name) } fmt.Fprintf(&b, ` } return "" } `) modeIDToNameSrc = b.String() } var kindIDsSrc string { var b strings.Builder fmt.Fprintf(&b, "const (\n") for i, k := range lexSpec.KindNames { if i == spec.LexKindIDNil.Int() { fmt.Fprintf(&b, " KindIDNil KindID = %v\n", i) continue } fmt.Fprintf(&b, " KindID%v KindID = %v\n", lexical.SnakeCaseToUpperCamelCase(k.String()), i) } fmt.Fprintf(&b, ")") kindIDsSrc = b.String() } var kindNamesSrc string { var b strings.Builder fmt.Fprintf(&b, "const (\n") fmt.Fprintf(&b, " KindNameNil = %#v\n", "") for _, k := range lexSpec.KindNames[1:] { fmt.Fprintf(&b, " KindName%v = %#v\n", lexical.SnakeCaseToUpperCamelCase(k.String()), k) } fmt.Fprintf(&b, ")") kindNamesSrc = b.String() } var kindIDToNameSrc string { var b strings.Builder fmt.Fprintf(&b, ` // KindIDToName converts a kind ID to a name. func KindIDToName(id KindID) string { switch id {`) for i, k := range lexSpec.KindNames { if i == spec.LexModeIDNil.Int() { fmt.Fprintf(&b, ` case KindIDNil: return KindNameNil`) continue } name := lexical.SnakeCaseToUpperCamelCase(k.String()) fmt.Fprintf(&b, ` case KindID%v: return KindName%v`, name, name) } fmt.Fprintf(&b, ` } return "" } `) kindIDToNameSrc = b.String() } var specSrc string { t, err := template.New("").Funcs(genTemplateFuncs(lexSpec)).Parse(lexSpecTemplate) if err != nil { return nil, err } var b strings.Builder err = t.Execute(&b, map[string]interface{}{ "initialModeID": "ModeID" + lexical.SnakeCaseToUpperCamelCase(lexSpec.ModeNames[lexSpec.InitialModeID].String()), "modeIDNil": "ModeIDNil", "modeKindIDNil": spec.LexModeKindIDNil, "stateIDNil": spec.StateIDNil, "compressionLevel": lexSpec.CompressionLevel, }) if err != nil { return nil, err } specSrc = b.String() } var src string { tmpl := `// Code generated by vartan-go. DO NOT EDIT. {{ .lexerSrc }} {{ .modeIDsSrc }} {{ .modeNamesSrc }} {{ .modeIDToNameSrc }} {{ .kindIDsSrc }} {{ .kindNamesSrc }} {{ .kindIDToNameSrc }} {{ .specSrc }} ` t, err := template.New("").Parse(tmpl) if err != nil { return nil, err } var b strings.Builder err = t.Execute(&b, map[string]string{ "lexerSrc": lexerSrc, "modeIDsSrc": modeIDsSrc, "modeNamesSrc": modeNamesSrc, "modeIDToNameSrc": modeIDToNameSrc, "kindIDsSrc": kindIDsSrc, "kindNamesSrc": kindNamesSrc, "kindIDToNameSrc": kindIDToNameSrc, "specSrc": specSrc, }) if err != nil { return nil, err } src = b.String() } fset := token.NewFileSet() f, err := parser.ParseFile(fset, "", src, parser.ParseComments) if err != nil { return nil, err } f.Name = ast.NewIdent(pkgName) var b bytes.Buffer err = format.Node(&b, fset, f) if err != nil { return nil, err } return b.Bytes(), nil } const lexSpecTemplate = ` type lexSpec struct { pop [][]bool push [][]ModeID modeNames []string initialStates []StateID acceptances [][]ModeKindID kindIDs [][]KindID kindNames []string initialModeID ModeID modeIDNil ModeID modeKindIDNil ModeKindID stateIDNil StateID rowNums [][]int rowDisplacements [][]int bounds [][]int entries [][]StateID originalColCounts []int } func NewLexSpec() *lexSpec { return &lexSpec{ pop: {{ genPopTable }}, push: {{ genPushTable }}, modeNames: {{ genModeNameTable }}, initialStates: {{ genInitialStateTable }}, acceptances: {{ genAcceptTable }}, kindIDs: {{ genKindIDTable }}, kindNames: {{ genKindNameTable }}, initialModeID: {{ .initialModeID }}, modeIDNil: {{ .modeIDNil }}, modeKindIDNil: {{ .modeKindIDNil }}, stateIDNil: {{ .stateIDNil }}, rowNums: {{ genRowNums }}, rowDisplacements: {{ genRowDisplacements }}, bounds: {{ genBounds }}, entries: {{ genEntries }}, originalColCounts: {{ genOriginalColCounts }}, } } func (s *lexSpec) InitialMode() ModeID { return s.initialModeID } func (s *lexSpec) Pop(mode ModeID, modeKind ModeKindID) bool { return s.pop[mode][modeKind] } func (s *lexSpec) Push(mode ModeID, modeKind ModeKindID) (ModeID, bool) { id := s.push[mode][modeKind] return id, id != s.modeIDNil } func (s *lexSpec) ModeName(mode ModeID) string { return s.modeNames[mode] } func (s *lexSpec) InitialState(mode ModeID) StateID { return s.initialStates[mode] } func (s *lexSpec) NextState(mode ModeID, state StateID, v int) (StateID, bool) { {{ if eq .compressionLevel 2 -}} rowNum := s.rowNums[mode][state] d := s.rowDisplacements[mode][rowNum] if s.bounds[mode][d+v] != rowNum { return s.stateIDNil, false } return s.entries[mode][d+v], true {{ else if eq .compressionLevel 1 -}} rowNum := s.rowNums[mode][state] colCount := s.originalColCounts[mode] next := s.entries[mode][rowNum*colCount+v] if next == s.stateIDNil { return s.stateIDNil, false } return next, true {{ else -}} colCount := s.originalColCounts[mode] next := s.entries[mode][int(state)*colCount+v] if next == s.stateIDNil { return s.stateIDNil, false } return next, true {{ end -}} } func (s *lexSpec) Accept(mode ModeID, state StateID) (ModeKindID, bool) { id := s.acceptances[mode][state] return id, id != s.modeKindIDNil } func (s *lexSpec) KindIDAndName(mode ModeID, modeKind ModeKindID) (KindID, string) { id := s.kindIDs[mode][modeKind] return id, s.kindNames[id] } ` func genTemplateFuncs(lexSpec *spec.LexicalSpec) template.FuncMap { fns := template.FuncMap{ "genPopTable": func() string { var b strings.Builder fmt.Fprintf(&b, "[][]bool{\n") for i, s := range lexSpec.Specs { if i == spec.LexModeIDNil.Int() { fmt.Fprintf(&b, "nil,\n") continue } c := 1 fmt.Fprintf(&b, "{\n") for _, v := range s.Pop { fmt.Fprintf(&b, "%v, ", v != 0) if c == 20 { fmt.Fprintf(&b, "\n") c = 1 } else { c++ } } if c > 1 { fmt.Fprintf(&b, "\n") } fmt.Fprintf(&b, "},\n") } fmt.Fprintf(&b, "}") return b.String() }, "genPushTable": func() string { var b strings.Builder fmt.Fprintf(&b, "[][]ModeID{\n") for i, s := range lexSpec.Specs { if i == spec.LexModeIDNil.Int() { fmt.Fprintf(&b, "nil,\n") continue } c := 1 fmt.Fprintf(&b, "{\n") for _, v := range s.Push { fmt.Fprintf(&b, "%v,", v) if c == 20 { fmt.Fprintf(&b, "\n") c = 1 } else { c++ } } if c > 1 { fmt.Fprintf(&b, "\n") } fmt.Fprintf(&b, "},\n") } fmt.Fprintf(&b, "}") return b.String() }, "genModeNameTable": func() string { var b strings.Builder fmt.Fprintf(&b, "[]string{\n") for i, name := range lexSpec.ModeNames { if i == spec.LexModeIDNil.Int() { fmt.Fprintf(&b, "ModeNameNil,\n") continue } fmt.Fprintf(&b, "ModeName%v,\n", lexical.SnakeCaseToUpperCamelCase(name.String())) } fmt.Fprintf(&b, "}") return b.String() }, "genInitialStateTable": func() string { var b strings.Builder fmt.Fprintf(&b, "[]StateID{\n") for i, s := range lexSpec.Specs { if i == spec.LexModeIDNil.Int() { fmt.Fprintf(&b, "%v,\n", spec.StateIDNil) continue } fmt.Fprintf(&b, "%v,\n", s.DFA.InitialStateID) } fmt.Fprintf(&b, "}") return b.String() }, "genAcceptTable": func() string { var b strings.Builder fmt.Fprintf(&b, "[][]ModeKindID{\n") for i, s := range lexSpec.Specs { if i == spec.LexModeIDNil.Int() { fmt.Fprintf(&b, "nil,\n") continue } c := 1 fmt.Fprintf(&b, "{\n") for _, v := range s.DFA.AcceptingStates { fmt.Fprintf(&b, "%v,", v) if c == 20 { fmt.Fprintf(&b, "\n") c = 1 } else { c++ } } if c > 1 { fmt.Fprintf(&b, "\n") } fmt.Fprintf(&b, "},\n") } fmt.Fprintf(&b, "}") return b.String() }, "genKindIDTable": func() string { var b strings.Builder fmt.Fprintf(&b, "[][]KindID{\n") for i, ids := range lexSpec.KindIDs { if i == spec.LexModeIDNil.Int() { fmt.Fprintf(&b, "nil,\n") continue } fmt.Fprintf(&b, "{\n") for j, id := range ids { if j == spec.LexModeKindIDNil.Int() { fmt.Fprintf(&b, "KindIDNil,\n") continue } fmt.Fprintf(&b, "KindID%v,\n", lexical.SnakeCaseToUpperCamelCase(string(lexSpec.KindNames[id].String()))) } fmt.Fprintf(&b, "},\n") } fmt.Fprintf(&b, "}") return b.String() }, "genKindNameTable": func() string { var b strings.Builder fmt.Fprintf(&b, "[]string{\n") for i, name := range lexSpec.KindNames { if i == spec.LexKindIDNil.Int() { fmt.Fprintf(&b, "KindNameNil,\n") continue } fmt.Fprintf(&b, "KindName%v,\n", lexical.SnakeCaseToUpperCamelCase(name.String())) } fmt.Fprintf(&b, "}") return b.String() }, } switch lexSpec.CompressionLevel { case 2: fns["genRowNums"] = func() string { var b strings.Builder fmt.Fprintf(&b, "[][]int{\n") for i, s := range lexSpec.Specs { if i == spec.LexModeIDNil.Int() { fmt.Fprintf(&b, "nil,\n") continue } c := 1 fmt.Fprintf(&b, "{\n") for _, v := range s.DFA.Transition.RowNums { fmt.Fprintf(&b, "%v,", v) if c == 20 { fmt.Fprintf(&b, "\n") c = 1 } else { c++ } } if c > 1 { fmt.Fprintf(&b, "\n") } fmt.Fprintf(&b, "},\n") } fmt.Fprintf(&b, "}") return b.String() } fns["genRowDisplacements"] = func() string { var b strings.Builder fmt.Fprintf(&b, "[][]int{\n") for i, s := range lexSpec.Specs { if i == spec.LexModeIDNil.Int() { fmt.Fprintf(&b, "nil,\n") continue } c := 1 fmt.Fprintf(&b, "{\n") for _, d := range s.DFA.Transition.UniqueEntries.RowDisplacement { fmt.Fprintf(&b, "%v,", d) if c == 20 { fmt.Fprintf(&b, "\n") c = 1 } else { c++ } } if c > 1 { fmt.Fprintf(&b, "\n") } fmt.Fprintf(&b, "},\n") } fmt.Fprintf(&b, "}") return b.String() } fns["genBounds"] = func() string { var b strings.Builder fmt.Fprintf(&b, "[][]int{\n") for i, s := range lexSpec.Specs { if i == spec.LexModeIDNil.Int() { fmt.Fprintf(&b, "nil,\n") continue } c := 1 fmt.Fprintf(&b, "{\n") for _, v := range s.DFA.Transition.UniqueEntries.Bounds { fmt.Fprintf(&b, "%v,", v) if c == 20 { fmt.Fprintf(&b, "\n") c = 1 } else { c++ } } if c > 1 { fmt.Fprintf(&b, "\n") } fmt.Fprintf(&b, "},\n") } fmt.Fprintf(&b, "}") return b.String() } fns["genEntries"] = func() string { var b strings.Builder fmt.Fprintf(&b, "[][]StateID{\n") for i, s := range lexSpec.Specs { if i == spec.LexModeIDNil.Int() { fmt.Fprintf(&b, "nil,\n") continue } c := 1 fmt.Fprintf(&b, "{\n") for _, v := range s.DFA.Transition.UniqueEntries.Entries { fmt.Fprintf(&b, "%v,", v) if c == 20 { fmt.Fprintf(&b, "\n") c = 1 } else { c++ } } if c > 1 { fmt.Fprintf(&b, "\n") } fmt.Fprintf(&b, "},\n") } fmt.Fprintf(&b, "}") return b.String() } fns["genOriginalColCounts"] = func() string { return "nil" } case 1: fns["genRowNums"] = func() string { var b strings.Builder fmt.Fprintf(&b, "[][]int{\n") for i, s := range lexSpec.Specs { if i == spec.LexModeIDNil.Int() { fmt.Fprintf(&b, "nil,\n") continue } c := 1 fmt.Fprintf(&b, "{\n") for _, v := range s.DFA.Transition.RowNums { fmt.Fprintf(&b, "%v,", v) if c == 20 { fmt.Fprintf(&b, "\n") c = 1 } else { c++ } } if c > 1 { fmt.Fprintf(&b, "\n") } fmt.Fprintf(&b, "},\n") } fmt.Fprintf(&b, "}") return b.String() } fns["genRowDisplacements"] = func() string { return "nil" } fns["genBounds"] = func() string { return "nil" } fns["genEntries"] = func() string { var b strings.Builder fmt.Fprintf(&b, "[][]StateID{\n") for i, s := range lexSpec.Specs { if i == spec.LexModeIDNil.Int() { fmt.Fprintf(&b, "nil,\n") continue } c := 1 fmt.Fprintf(&b, "{\n") for _, v := range s.DFA.Transition.UncompressedUniqueEntries { fmt.Fprintf(&b, "%v,", v) if c == 20 { fmt.Fprintf(&b, "\n") c = 1 } else { c++ } } if c > 1 { fmt.Fprintf(&b, "\n") } fmt.Fprintf(&b, "},\n") } fmt.Fprintf(&b, "}") return b.String() } fns["genOriginalColCounts"] = func() string { var b strings.Builder fmt.Fprintf(&b, "[]int{\n") for i, s := range lexSpec.Specs { if i == spec.LexModeIDNil.Int() { fmt.Fprintf(&b, "0,\n") continue } fmt.Fprintf(&b, "%v,\n", s.DFA.Transition.OriginalColCount) } fmt.Fprintf(&b, "}") return b.String() } default: fns["genRowNums"] = func() string { return "nil" } fns["genRowDisplacements"] = func() string { return "nil" } fns["genBounds"] = func() string { return "nil" } fns["genEntries"] = func() string { var b strings.Builder fmt.Fprintf(&b, "[][]StateID{\n") for i, s := range lexSpec.Specs { if i == spec.LexModeIDNil.Int() { fmt.Fprintf(&b, "nil,\n") continue } c := 1 fmt.Fprintf(&b, "{\n") for _, v := range s.DFA.UncompressedTransition { fmt.Fprintf(&b, "%v,", v) if c == 20 { fmt.Fprintf(&b, "\n") c = 1 } else { c++ } } if c > 1 { fmt.Fprintf(&b, "\n") } fmt.Fprintf(&b, "},\n") } fmt.Fprintf(&b, "}") return b.String() } fns["genOriginalColCounts"] = func() string { var b strings.Builder fmt.Fprintf(&b, "[]int{\n") for i, s := range lexSpec.Specs { if i == spec.LexModeIDNil.Int() { fmt.Fprintf(&b, "0,\n") continue } fmt.Fprintf(&b, "%v,\n", s.DFA.ColCount) } fmt.Fprintf(&b, "}") return b.String() } } return fns }