diff options
author | Ryo Nihei <nihei.dev@gmail.com> | 2021-02-16 00:07:40 +0900 |
---|---|---|
committer | Ryo Nihei <nihei.dev@gmail.com> | 2021-02-16 00:07:40 +0900 |
commit | c313f7870bd547534c7c7bb0ad01003ab9983b34 (patch) | |
tree | f1b6a443c4372f9eb69f314009d721703e208a0b /driver/lexer.go | |
parent | Add bracket expression matching specified character (diff) | |
download | tre-c313f7870bd547534c7c7bb0ad01003ab9983b34.tar.gz tre-c313f7870bd547534c7c7bb0ad01003ab9983b34.tar.xz |
Add types of lexical specifications
APIs of compiler and driver packages use these types. Because CompiledLexSpec struct a lexer takes has kind names of lexical specification entries, the lexer sets them to tokens.
Diffstat (limited to 'driver/lexer.go')
-rw-r--r-- | driver/lexer.go | 30 |
1 files changed, 16 insertions, 14 deletions
diff --git a/driver/lexer.go b/driver/lexer.go index 710d54d..3a6f039 100644 --- a/driver/lexer.go +++ b/driver/lexer.go @@ -5,19 +5,21 @@ import ( "io" "io/ioutil" - "github.com/nihei9/maleeni/compiler" + "github.com/nihei9/maleeni/spec" ) type Token struct { ID int + Kind string Match []byte EOF bool Invalid bool } -func newToken(id int, match []byte) *Token { +func newToken(id int, kind string, match []byte) *Token { return &Token{ ID: id, + Kind: kind, Match: match, } } @@ -38,21 +40,21 @@ func newInvalidToken(match []byte) *Token { } type lexer struct { - tranTab *compiler.TransitionTable - src []byte - srcPtr int - tokBuf []*Token + clspec *spec.CompiledLexSpec + src []byte + srcPtr int + tokBuf []*Token } -func NewLexer(tranTab *compiler.TransitionTable, src io.Reader) (*lexer, error) { +func NewLexer(clspec *spec.CompiledLexSpec, src io.Reader) (*lexer, error) { b, err := ioutil.ReadAll(src) if err != nil { return nil, err } return &lexer{ - tranTab: tranTab, - src: b, - srcPtr: 0, + clspec: clspec, + src: b, + srcPtr: 0, }, nil } @@ -112,7 +114,7 @@ func (l *lexer) peekN(n int) (*Token, error) { } func (l *lexer) next() (*Token, error) { - state := l.tranTab.InitialState + state := l.clspec.DFA.InitialState buf := []byte{} unfixedBufLen := 0 var tok *Token @@ -127,7 +129,7 @@ func (l *lexer) next() (*Token, error) { } buf = append(buf, v) unfixedBufLen++ - entry := l.tranTab.Transition[state] + entry := l.clspec.DFA.Transition[state] if len(entry) == 0 { return nil, fmt.Errorf("no transition entry; state: %v", state) } @@ -140,9 +142,9 @@ func (l *lexer) next() (*Token, error) { return newInvalidToken(buf), nil } state = nextState - id, ok := l.tranTab.AcceptingStates[state] + id, ok := l.clspec.DFA.AcceptingStates[state] if ok { - tok = newToken(id, buf) + tok = newToken(id, l.clspec.Kinds[id], buf) unfixedBufLen = 0 } } |