aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorRyo Nihei <nihei.dev@gmail.com>2021-05-12 21:31:33 +0900
committerRyo Nihei <nihei.dev@gmail.com>2021-05-13 18:45:54 +0900
commit67742a9fa833e82ac06bc664c5d0495ce7408b08 (patch)
tree2bf08e8672d24cbf3f33b429d1a93ecb20c75544
parentUse go fmt instead of gofmt (diff)
downloadtre-67742a9fa833e82ac06bc664c5d0495ce7408b08.tar.gz
tre-67742a9fa833e82ac06bc664c5d0495ce7408b08.tar.xz
Rename fields of driver.Token
-rw-r--r--README.md4
-rw-r--r--driver/lexer.go45
-rw-r--r--driver/lexer_test.go2
3 files changed, 25 insertions, 26 deletions
diff --git a/README.md b/README.md
index c142bab..6fc098d 100644
--- a/README.md
+++ b/README.md
@@ -43,7 +43,7 @@ If you want to make sure that the lexical specification behaves as expected, you
`maleeni lex` command outputs tokens in JSON format. For simplicity, print significant fields of the tokens in CSV format using jq command.
```sh
-$ echo -n 'The truth is out there.' | maleeni lex clexspec.json | jq -r '[.kind, .text, .eof] | @csv'
+$ echo -n 'The truth is out there.' | maleeni lex clexspec.json | jq -r '[.kind_name, .text, .eof] | @csv'
"word","The",false
"whitespace"," ",false
"word","truth",false
@@ -115,7 +115,7 @@ entry object:
| Field | Type | Nullable | Description |
|---------|------------------|----------|-----------------------------------------------------------------------------------------------|
-| kinds | string | false | A name of a token kind |
+| kind | string | false | A name of a token kind |
| pattern | string | false | A pattern in a regular expression |
| modes | array of strings | true | Mode names that an entry is enabled in (default: "default") |
| push | string | true | A mode name that the lexer pushes to own mode stack when a token matching the pattern appears |
diff --git a/driver/lexer.go b/driver/lexer.go
index 278a104..8d4a10f 100644
--- a/driver/lexer.go
+++ b/driver/lexer.go
@@ -62,11 +62,11 @@ type Token struct {
// `ModeName` is a mode name that represents in which mode the lexer detected the token.
ModeName spec.LexModeName
- // `ID` represents an ID that corresponds to a `Kind`.
- ID int
+ // `Kind` represents a number that corresponds to a `KindName`.
+ Kind int
- // `Kind` is a kind name that represents what kind the token has.
- Kind string
+ // `KindName` is a kind name that represents what kind the token has.
+ KindName string
// If `EOF` is true, it means the token is the EOF token.
EOF bool
@@ -78,12 +78,12 @@ type Token struct {
match byteSequence
}
-func newToken(mode spec.LexModeNum, modeName spec.LexModeName, id int, kind string, match byteSequence) *Token {
+func newToken(mode spec.LexModeNum, modeName spec.LexModeName, kind int, kindName string, match byteSequence) *Token {
return &Token{
Mode: mode,
ModeName: modeName,
- ID: id,
Kind: kind,
+ KindName: kindName,
match: match,
}
}
@@ -92,7 +92,7 @@ func newEOFToken(mode spec.LexModeNum, modeName spec.LexModeName) *Token {
return &Token{
Mode: mode,
ModeName: modeName,
- ID: 0,
+ Kind: 0,
EOF: true,
}
}
@@ -101,7 +101,7 @@ func newInvalidToken(mode spec.LexModeNum, modeName spec.LexModeName, match byte
return &Token{
Mode: mode,
ModeName: modeName,
- ID: 0,
+ Kind: 0,
match: match,
Invalid: true,
}
@@ -114,7 +114,7 @@ func (t *Token) String() string {
if t.EOF {
return "{eof}"
}
- return fmt.Sprintf("{mode: %v, mode name: %v, id: %v, kind: %v, text: %v, byte: %v}", t.Mode, t.ModeName, t.ID, t.Kind, t.Text(), t.Match())
+ return fmt.Sprintf("{mode: %v, mode name: %v, kind: %v, kind name: %v, text: %v, byte: %v}", t.Mode, t.ModeName, t.Kind, t.KindName, t.Text(), t.Match())
}
// Match returns a byte slice matched a pattern of a lexical specification.
@@ -128,23 +128,22 @@ func (t *Token) Text() string {
}
func (t *Token) MarshalJSON() ([]byte, error) {
- m := t.match.ByteSlice()
return json.Marshal(struct {
- Mode int `json:"mode"`
- ModeName string `json:"mode_name"`
- ID int `json:"id"`
- Kind string `json:"kind"`
- Match []byte `json:"match"`
- Text string `json:"text"`
- EOF bool `json:"eof"`
- Invalid bool `json:"invalid"`
+ Mode int `json:"mode"`
+ ModeName string `json:"mode_name"`
+ Kind int `json:"kind"`
+ KindName string `json:"kind_name"`
+ Match byteSequence `json:"match"`
+ Text string `json:"text"`
+ EOF bool `json:"eof"`
+ Invalid bool `json:"invalid"`
}{
Mode: t.Mode.Int(),
ModeName: t.ModeName.String(),
- ID: t.ID,
Kind: t.Kind,
- Match: m,
- Text: string(m),
+ KindName: t.KindName,
+ Match: t.match,
+ Text: t.Text(),
EOF: t.EOF,
Invalid: t.Invalid,
})
@@ -256,13 +255,13 @@ func (l *Lexer) nextAndTranMode() (*Token, error) {
return tok, nil
}
spec := l.clspec.Specs[l.mode()]
- if spec.Pop[tok.ID] == 1 {
+ if spec.Pop[tok.Kind] == 1 {
err := l.popMode()
if err != nil {
return nil, err
}
}
- mode := spec.Push[tok.ID]
+ mode := spec.Push[tok.Kind]
if !mode.IsNil() {
l.pushMode(mode)
}
diff --git a/driver/lexer_test.go b/driver/lexer_test.go
index 0156ef8..1d0e887 100644
--- a/driver/lexer_test.go
+++ b/driver/lexer_test.go
@@ -558,8 +558,8 @@ func testToken(t *testing.T, expected, actual *Token) {
if actual.Mode != expected.Mode ||
actual.ModeName != actual.ModeName ||
- actual.ID != expected.ID ||
actual.Kind != expected.Kind ||
+ actual.KindName != expected.KindName ||
!bytes.Equal(actual.Match(), expected.Match()) ||
actual.EOF != expected.EOF ||
actual.Invalid != expected.Invalid {