Update README and godoc

1 files changed, 23 insertions, 7 deletions

diff --git a/driver/lexer.go b/driver/lexer.go
index 76f8092..8a9e04a 100644
--- a/driver/lexer.go
+++ b/driver/lexer.go
@@ -53,15 +53,31 @@ func (s byteSequence) merge(a byteSequence) byteSequence {
 	return append([]byte(s), []byte(a)...)
 }
 
+// Token representes a token.
 type Token struct {
-	Mode     spec.LexModeNum  `json:"mode"`
+	// `Mode` represents a number that corresponds to a `ModeName`.
+	Mode spec.LexModeNum `json:"mode"`
+
+	// `ModeName` is a mode name that represents in which mode the lexer detected the token.
 	ModeName spec.LexModeName `json:"mode_name"`
-	ID       int              `json:"id"`
-	Kind     string           `json:"kind"`
-	Match    byteSequence     `json:"match"`
-	Text     string           `json:"text"`
-	EOF      bool             `json:"eof"`
-	Invalid  bool             `json:"invalid"`
+
+	// `ID` represents an ID that corresponds to a `Kind`.
+	ID int `json:"id"`
+
+	// `Kind` is a kind name that represents what kind the token has.
+	Kind string `json:"kind"`
+
+	// `Match` is a byte sequence matched a pattern of a lexical specification.
+	Match byteSequence `json:"match"`
+
+	// `Text` is a string representation of the `Match`.
+	Text string `json:"text"`
+
+	// If `EOF` is true, it means the token is the EOF token.
+	EOF bool `json:"eof"`
+
+	// If `Invalid` is true, it means the token is an error token.
+	Invalid bool `json:"invalid"`
 }
 
 func newToken(mode spec.LexModeNum, modeName spec.LexModeName, id int, kind string, match byteSequence) *Token {