aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--.github/workflows/ci.yaml (renamed from .github/workflows/test.yml)15
-rw-r--r--README.md2
-rw-r--r--cmd/generator/main.go2
-rw-r--r--cmd/maleeni/compile.go16
-rw-r--r--compiler/dfa/dfa.go2
-rw-r--r--compiler/dfa/symbol_position.go22
-rw-r--r--compiler/dfa/tree.go22
-rw-r--r--compiler/dfa/tree_test.go9
-rw-r--r--compiler/parser/lexer.go6
-rw-r--r--compiler/parser/parser.go5
-rw-r--r--compiler/parser/tree.go5
-rw-r--r--driver/lexer_test.go20
12 files changed, 64 insertions, 62 deletions
diff --git a/.github/workflows/test.yml b/.github/workflows/ci.yaml
index 47bc688..fc0b475 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/ci.yaml
@@ -1,4 +1,4 @@
-name: Test
+name: ci
on:
push:
@@ -8,7 +8,7 @@ on:
jobs:
- build:
+ test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
@@ -20,3 +20,14 @@ jobs:
- name: Test
run: go test -v ./...
+
+ golangci:
+ name: lint
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2
+ - name: golangci-lint
+ uses: golangci/golangci-lint-action@v2
+ with:
+ version: latest
+ args: -E gofmt
diff --git a/README.md b/README.md
index 805e50c..7289fa1 100644
--- a/README.md
+++ b/README.md
@@ -2,7 +2,7 @@
maleeni is a lexer generator for golang. maleeni also provides a command to perform lexical analysis to allow easy debugging of your lexical specification.
-[![Test](https://github.com/nihei9/maleeni/actions/workflows/test.yml/badge.svg?branch=main)](https://github.com/nihei9/maleeni/actions/workflows/test.yml)
+[![ci](https://github.com/nihei9/maleeni/actions/workflows/ci.yaml/badge.svg)](https://github.com/nihei9/maleeni/actions/workflows/ci.yaml)
## Installation
diff --git a/cmd/generator/main.go b/cmd/generator/main.go
index 20dbc5f..75b7eea 100644
--- a/cmd/generator/main.go
+++ b/cmd/generator/main.go
@@ -93,6 +93,6 @@ func gen() error {
return err
}
defer f.Close()
- fmt.Fprintf(f, b.String())
+ fmt.Fprint(f, b.String())
return nil
}
diff --git a/cmd/maleeni/compile.go b/cmd/maleeni/compile.go
index 2fcb8ad..e353875 100644
--- a/cmd/maleeni/compile.go
+++ b/cmd/maleeni/compile.go
@@ -14,22 +14,22 @@ import (
)
var compileFlags = struct {
- debug *bool
- compLv *int
- output *string
+ debug *bool
+ compLv *int
+ output *string
}{}
func init() {
cmd := &cobra.Command{
- Use: "compile",
- Short: "Compile a lexical specification into a DFA",
- Long: `compile takes a lexical specification and generates a DFA accepting the tokens described in the specification.`,
+ Use: "compile",
+ Short: "Compile a lexical specification into a DFA",
+ Long: `compile takes a lexical specification and generates a DFA accepting the tokens described in the specification.`,
Example: ` Read from/Write to the specified file:
maleeni compile lexspec.json -o clexspec.json
Read from stdin and write to stdout:
cat lexspec.json | maleeni compile`,
- Args: cobra.MaximumNArgs(1),
- RunE: runCompile,
+ Args: cobra.MaximumNArgs(1),
+ RunE: runCompile,
}
compileFlags.compLv = cmd.Flags().Int("compression-level", compiler.CompressionLevelMax, "compression level")
compileFlags.output = cmd.Flags().StringP("output", "o", "", "output file path (default stdout)")
diff --git a/compiler/dfa/dfa.go b/compiler/dfa/dfa.go
index c56af90..777bd24 100644
--- a/compiler/dfa/dfa.go
+++ b/compiler/dfa/dfa.go
@@ -47,7 +47,7 @@ type DFA struct {
TransitionTable map[string][256]string
}
-func GenDFA(root byteTree, symTab *symbolTable) *DFA {
+func GenDFA(root byteTree, symTab *symbolTable) *DFA {
initialState := root.first()
initialStateHash := initialState.hash()
stateMap := map[string]*symbolPositionSet{
diff --git a/compiler/dfa/symbol_position.go b/compiler/dfa/symbol_position.go
index 10d5a1e..f154251 100644
--- a/compiler/dfa/symbol_position.go
+++ b/compiler/dfa/symbol_position.go
@@ -38,10 +38,7 @@ func (p symbolPosition) String() string {
}
func (p symbolPosition) isEndMark() bool {
- if uint16(p)&symbolPositionMaskEndMark > 1 {
- return true
- }
- return false
+ return uint16(p)&symbolPositionMaskEndMark > 1
}
func (p symbolPosition) describe() (uint16, bool) {
@@ -102,19 +99,6 @@ func (s *symbolPositionSet) merge(t *symbolPositionSet) *symbolPositionSet {
return s
}
-func (s *symbolPositionSet) intersect(set *symbolPositionSet) *symbolPositionSet {
- in := newSymbolPositionSet()
- for _, p1 := range s.s {
- for _, p2 := range set.s {
- if p1 != p2 {
- continue
- }
- in.add(p1)
- }
- }
- return in
-}
-
func (s *symbolPositionSet) hash() string {
if len(s.s) <= 0 {
return ""
@@ -171,9 +155,9 @@ func sortSymbolPositions(ps []symbolPosition, left, right int) {
p1, p2 = p2, p1
}
if p2 > p3 {
- p2, p3 = p3, p2
+ p2 = p3
if p1 > p2 {
- p1, p2 = p2, p1
+ p2 = p1
}
}
pivot = p2
diff --git a/compiler/dfa/tree.go b/compiler/dfa/tree.go
index a3442f8..bd93a26 100644
--- a/compiler/dfa/tree.go
+++ b/compiler/dfa/tree.go
@@ -153,7 +153,7 @@ func newConcatNode(left, right byteTree) *concatNode {
}
func (n *concatNode) String() string {
- return fmt.Sprintf("concat")
+ return "concat"
}
func (n *concatNode) children() (byteTree, byteTree) {
@@ -207,7 +207,7 @@ func newAltNode(left, right byteTree) *altNode {
}
func (n *altNode) String() string {
- return fmt.Sprintf("alt")
+ return "alt"
}
func (n *altNode) children() (byteTree, byteTree) {
@@ -255,7 +255,7 @@ func newRepeatNode(left byteTree) *repeatNode {
}
func (n *repeatNode) String() string {
- return fmt.Sprintf("repeat")
+ return "repeat"
}
func (n *repeatNode) children() (byteTree, byteTree) {
@@ -288,14 +288,6 @@ func (n *repeatNode) clone() byteTree {
return newRepeatNode(n.left.clone())
}
-func newRepeatOneOrMoreNode(left byteTree) *concatNode {
- return newConcatNode(
- left,
- &repeatNode{
- left: left.clone(),
- })
-}
-
type optionNode struct {
left byteTree
firstMemo *symbolPositionSet
@@ -309,7 +301,7 @@ func newOptionNode(left byteTree) *optionNode {
}
func (n *optionNode) String() string {
- return fmt.Sprintf("option")
+ return "option"
}
func (n *optionNode) children() (byteTree, byteTree) {
@@ -452,6 +444,7 @@ func oneOf(ts ...byteTree) byteTree {
return alt
}
+//nolint:unused
func printByteTree(w io.Writer, t byteTree, ruledLine string, childRuledLinePrefix string, withAttrs bool) {
if t == nil {
return
@@ -505,7 +498,10 @@ func ConvertCPTreeToByteTree(cpTrees map[spec.LexModeKindID]parser.CPTree) (byte
}
bt = oneOf(bt, concat(t, newEndMarkerNode(id)))
}
- positionSymbols(bt, symbolPositionMin)
+ _, err := positionSymbols(bt, symbolPositionMin)
+ if err != nil {
+ return nil, nil, err
+ }
return bt, genSymbolTable(bt), nil
}
diff --git a/compiler/dfa/tree_test.go b/compiler/dfa/tree_test.go
index e37a69c..1c1fd1d 100644
--- a/compiler/dfa/tree_test.go
+++ b/compiler/dfa/tree_test.go
@@ -128,12 +128,6 @@ func TestByteTree(t *testing.T) {
}
}
-func newRangeSymbolNodeWithPos(from, to byte, pos symbolPosition) *symbolNode {
- n := newRangeSymbolNode(from, to)
- n.pos = pos
- return n
-}
-
func newSymbolNodeWithPos(v byte, pos symbolPosition) *symbolNode {
n := newSymbolNode(v)
n.pos = pos
@@ -172,6 +166,9 @@ func TestFollowAndSymbolTable(t *testing.T) {
bt, symTab, err := ConvertCPTreeToByteTree(map[spec.LexModeKindID]parser.CPTree{
spec.LexModeKindIDMin: cpt,
})
+ if err != nil {
+ t.Fatal(err)
+ }
{
followTab := genFollowTable(bt)
diff --git a/compiler/parser/lexer.go b/compiler/parser/lexer.go
index 01cebeb..3861825 100644
--- a/compiler/parser/lexer.go
+++ b/compiler/parser/lexer.go
@@ -416,7 +416,7 @@ func (l *lexer) nextInCodePoint(c rune) (*token, error) {
return nil, err
}
if eof {
- l.restore()
+ err := l.restore()
if err != nil {
return nil, err
}
@@ -471,7 +471,7 @@ func (l *lexer) nextInCharProp(c rune) (*token, error) {
return nil, err
}
if eof {
- l.restore()
+ err := l.restore()
if err != nil {
return nil, err
}
@@ -512,7 +512,7 @@ func (l *lexer) nextInFragment(c rune) (*token, error) {
return nil, err
}
if eof {
- l.restore()
+ err := l.restore()
if err != nil {
return nil, err
}
diff --git a/compiler/parser/parser.go b/compiler/parser/parser.go
index fd85fab..3706525 100644
--- a/compiler/parser/parser.go
+++ b/compiler/parser/parser.go
@@ -446,10 +446,7 @@ func genAnyCharAST() CPTree {
}
func isValidOrder(from, to rune) bool {
- if from <= to {
- return true
- }
- return false
+ return from <= to
}
func genConcatNode(cs ...CPTree) CPTree {
diff --git a/compiler/parser/tree.go b/compiler/parser/tree.go
index b5fb723..04ba723 100644
--- a/compiler/parser/tree.go
+++ b/compiler/parser/tree.go
@@ -207,7 +207,7 @@ func newConcatNode(left, right CPTree) *concatNode {
}
func (n *concatNode) String() string {
- return fmt.Sprintf("concat")
+ return "concat"
}
func (n *concatNode) Range() (rune, rune, bool) {
@@ -258,7 +258,7 @@ func newAltNode(left, right CPTree) *altNode {
}
func (n *altNode) String() string {
- return fmt.Sprintf("alt")
+ return "alt"
}
func (n *altNode) Range() (rune, rune, bool) {
@@ -419,6 +419,7 @@ func (n *fragmentNode) clone() CPTree {
return newFragmentNode(n.kind, n.tree.clone())
}
+//nolint:unused
func printCPTree(w io.Writer, t CPTree, ruledLine string, childRuledLinePrefix string) {
if t == nil {
return
diff --git a/driver/lexer_test.go b/driver/lexer_test.go
index e0b82b1..9e5148c 100644
--- a/driver/lexer_test.go
+++ b/driver/lexer_test.go
@@ -72,9 +72,9 @@ func newEOFTokenDefault() *Token {
return newEOFToken(ModeID(spec.LexModeIDDefault.Int()), spec.LexModeNameDefault.String())
}
-func newInvalidToken(modeID ModeID, lexeme []byte) *Token {
+func newInvalidTokenDefault(lexeme []byte) *Token {
return &Token{
- ModeID: modeID,
+ ModeID: ModeID(spec.LexModeIDDefault.Int()),
ModeKindID: 0,
Lexeme: lexeme,
Invalid: true,
@@ -772,6 +772,22 @@ func TestLexer_Next(t *testing.T) {
newEOFTokenDefault(),
},
},
+ // The driver can continue lexical analysis even after it detects an invalid token.
+ {
+ lspec: &spec.LexSpec{
+ Name: "test",
+ Entries: []*spec.LexEntry{
+ newLexEntryDefaultNOP("lower", `[a-z]+`),
+ },
+ },
+ src: `foo123bar`,
+ tokens: []*Token{
+ newTokenDefault(1, 1, []byte(`foo`)),
+ newInvalidTokenDefault([]byte(`123`)),
+ newTokenDefault(1, 1, []byte(`bar`)),
+ newEOFTokenDefault(),
+ },
+ },
}
for i, tt := range test {
for compLv := compiler.CompressionLevelMin; compLv <= compiler.CompressionLevelMax; compLv++ {
generated by cgit v1.2.3 (git 2.49.0) at 2025-07-13 01:15:21 +0000