diff options
-rw-r--r-- | .github/workflows/ci.yaml (renamed from .github/workflows/test.yml) | 15 | ||||
-rw-r--r-- | README.md | 2 | ||||
-rw-r--r-- | cmd/generator/main.go | 2 | ||||
-rw-r--r-- | cmd/maleeni/compile.go | 16 | ||||
-rw-r--r-- | compiler/dfa/dfa.go | 2 | ||||
-rw-r--r-- | compiler/dfa/symbol_position.go | 22 | ||||
-rw-r--r-- | compiler/dfa/tree.go | 22 | ||||
-rw-r--r-- | compiler/dfa/tree_test.go | 9 | ||||
-rw-r--r-- | compiler/parser/lexer.go | 6 | ||||
-rw-r--r-- | compiler/parser/parser.go | 5 | ||||
-rw-r--r-- | compiler/parser/tree.go | 5 | ||||
-rw-r--r-- | driver/lexer_test.go | 20 |
12 files changed, 64 insertions, 62 deletions
diff --git a/.github/workflows/test.yml b/.github/workflows/ci.yaml index 47bc688..fc0b475 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/ci.yaml @@ -1,4 +1,4 @@ -name: Test +name: ci on: push: @@ -8,7 +8,7 @@ on: jobs: - build: + test: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -20,3 +20,14 @@ jobs: - name: Test run: go test -v ./... + + golangci: + name: lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: golangci-lint + uses: golangci/golangci-lint-action@v2 + with: + version: latest + args: -E gofmt @@ -2,7 +2,7 @@ maleeni is a lexer generator for golang. maleeni also provides a command to perform lexical analysis to allow easy debugging of your lexical specification. -[](https://github.com/nihei9/maleeni/actions/workflows/test.yml) +[](https://github.com/nihei9/maleeni/actions/workflows/ci.yaml) ## Installation diff --git a/cmd/generator/main.go b/cmd/generator/main.go index 20dbc5f..75b7eea 100644 --- a/cmd/generator/main.go +++ b/cmd/generator/main.go @@ -93,6 +93,6 @@ func gen() error { return err } defer f.Close() - fmt.Fprintf(f, b.String()) + fmt.Fprint(f, b.String()) return nil } diff --git a/cmd/maleeni/compile.go b/cmd/maleeni/compile.go index 2fcb8ad..e353875 100644 --- a/cmd/maleeni/compile.go +++ b/cmd/maleeni/compile.go @@ -14,22 +14,22 @@ import ( ) var compileFlags = struct { - debug *bool - compLv *int - output *string + debug *bool + compLv *int + output *string }{} func init() { cmd := &cobra.Command{ - Use: "compile", - Short: "Compile a lexical specification into a DFA", - Long: `compile takes a lexical specification and generates a DFA accepting the tokens described in the specification.`, + Use: "compile", + Short: "Compile a lexical specification into a DFA", + Long: `compile takes a lexical specification and generates a DFA accepting the tokens described in the specification.`, Example: ` Read from/Write to the specified file: maleeni compile lexspec.json -o clexspec.json Read from stdin and write to stdout: cat lexspec.json | maleeni compile`, - Args: cobra.MaximumNArgs(1), - RunE: runCompile, + Args: cobra.MaximumNArgs(1), + RunE: runCompile, } compileFlags.compLv = cmd.Flags().Int("compression-level", compiler.CompressionLevelMax, "compression level") compileFlags.output = cmd.Flags().StringP("output", "o", "", "output file path (default stdout)") diff --git a/compiler/dfa/dfa.go b/compiler/dfa/dfa.go index c56af90..777bd24 100644 --- a/compiler/dfa/dfa.go +++ b/compiler/dfa/dfa.go @@ -47,7 +47,7 @@ type DFA struct { TransitionTable map[string][256]string } -func GenDFA(root byteTree, symTab *symbolTable) *DFA { +func GenDFA(root byteTree, symTab *symbolTable) *DFA { initialState := root.first() initialStateHash := initialState.hash() stateMap := map[string]*symbolPositionSet{ diff --git a/compiler/dfa/symbol_position.go b/compiler/dfa/symbol_position.go index 10d5a1e..f154251 100644 --- a/compiler/dfa/symbol_position.go +++ b/compiler/dfa/symbol_position.go @@ -38,10 +38,7 @@ func (p symbolPosition) String() string { } func (p symbolPosition) isEndMark() bool { - if uint16(p)&symbolPositionMaskEndMark > 1 { - return true - } - return false + return uint16(p)&symbolPositionMaskEndMark > 1 } func (p symbolPosition) describe() (uint16, bool) { @@ -102,19 +99,6 @@ func (s *symbolPositionSet) merge(t *symbolPositionSet) *symbolPositionSet { return s } -func (s *symbolPositionSet) intersect(set *symbolPositionSet) *symbolPositionSet { - in := newSymbolPositionSet() - for _, p1 := range s.s { - for _, p2 := range set.s { - if p1 != p2 { - continue - } - in.add(p1) - } - } - return in -} - func (s *symbolPositionSet) hash() string { if len(s.s) <= 0 { return "" @@ -171,9 +155,9 @@ func sortSymbolPositions(ps []symbolPosition, left, right int) { p1, p2 = p2, p1 } if p2 > p3 { - p2, p3 = p3, p2 + p2 = p3 if p1 > p2 { - p1, p2 = p2, p1 + p2 = p1 } } pivot = p2 diff --git a/compiler/dfa/tree.go b/compiler/dfa/tree.go index a3442f8..bd93a26 100644 --- a/compiler/dfa/tree.go +++ b/compiler/dfa/tree.go @@ -153,7 +153,7 @@ func newConcatNode(left, right byteTree) *concatNode { } func (n *concatNode) String() string { - return fmt.Sprintf("concat") + return "concat" } func (n *concatNode) children() (byteTree, byteTree) { @@ -207,7 +207,7 @@ func newAltNode(left, right byteTree) *altNode { } func (n *altNode) String() string { - return fmt.Sprintf("alt") + return "alt" } func (n *altNode) children() (byteTree, byteTree) { @@ -255,7 +255,7 @@ func newRepeatNode(left byteTree) *repeatNode { } func (n *repeatNode) String() string { - return fmt.Sprintf("repeat") + return "repeat" } func (n *repeatNode) children() (byteTree, byteTree) { @@ -288,14 +288,6 @@ func (n *repeatNode) clone() byteTree { return newRepeatNode(n.left.clone()) } -func newRepeatOneOrMoreNode(left byteTree) *concatNode { - return newConcatNode( - left, - &repeatNode{ - left: left.clone(), - }) -} - type optionNode struct { left byteTree firstMemo *symbolPositionSet @@ -309,7 +301,7 @@ func newOptionNode(left byteTree) *optionNode { } func (n *optionNode) String() string { - return fmt.Sprintf("option") + return "option" } func (n *optionNode) children() (byteTree, byteTree) { @@ -452,6 +444,7 @@ func oneOf(ts ...byteTree) byteTree { return alt } +//nolint:unused func printByteTree(w io.Writer, t byteTree, ruledLine string, childRuledLinePrefix string, withAttrs bool) { if t == nil { return @@ -505,7 +498,10 @@ func ConvertCPTreeToByteTree(cpTrees map[spec.LexModeKindID]parser.CPTree) (byte } bt = oneOf(bt, concat(t, newEndMarkerNode(id))) } - positionSymbols(bt, symbolPositionMin) + _, err := positionSymbols(bt, symbolPositionMin) + if err != nil { + return nil, nil, err + } return bt, genSymbolTable(bt), nil } diff --git a/compiler/dfa/tree_test.go b/compiler/dfa/tree_test.go index e37a69c..1c1fd1d 100644 --- a/compiler/dfa/tree_test.go +++ b/compiler/dfa/tree_test.go @@ -128,12 +128,6 @@ func TestByteTree(t *testing.T) { } } -func newRangeSymbolNodeWithPos(from, to byte, pos symbolPosition) *symbolNode { - n := newRangeSymbolNode(from, to) - n.pos = pos - return n -} - func newSymbolNodeWithPos(v byte, pos symbolPosition) *symbolNode { n := newSymbolNode(v) n.pos = pos @@ -172,6 +166,9 @@ func TestFollowAndSymbolTable(t *testing.T) { bt, symTab, err := ConvertCPTreeToByteTree(map[spec.LexModeKindID]parser.CPTree{ spec.LexModeKindIDMin: cpt, }) + if err != nil { + t.Fatal(err) + } { followTab := genFollowTable(bt) diff --git a/compiler/parser/lexer.go b/compiler/parser/lexer.go index 01cebeb..3861825 100644 --- a/compiler/parser/lexer.go +++ b/compiler/parser/lexer.go @@ -416,7 +416,7 @@ func (l *lexer) nextInCodePoint(c rune) (*token, error) { return nil, err } if eof { - l.restore() + err := l.restore() if err != nil { return nil, err } @@ -471,7 +471,7 @@ func (l *lexer) nextInCharProp(c rune) (*token, error) { return nil, err } if eof { - l.restore() + err := l.restore() if err != nil { return nil, err } @@ -512,7 +512,7 @@ func (l *lexer) nextInFragment(c rune) (*token, error) { return nil, err } if eof { - l.restore() + err := l.restore() if err != nil { return nil, err } diff --git a/compiler/parser/parser.go b/compiler/parser/parser.go index fd85fab..3706525 100644 --- a/compiler/parser/parser.go +++ b/compiler/parser/parser.go @@ -446,10 +446,7 @@ func genAnyCharAST() CPTree { } func isValidOrder(from, to rune) bool { - if from <= to { - return true - } - return false + return from <= to } func genConcatNode(cs ...CPTree) CPTree { diff --git a/compiler/parser/tree.go b/compiler/parser/tree.go index b5fb723..04ba723 100644 --- a/compiler/parser/tree.go +++ b/compiler/parser/tree.go @@ -207,7 +207,7 @@ func newConcatNode(left, right CPTree) *concatNode { } func (n *concatNode) String() string { - return fmt.Sprintf("concat") + return "concat" } func (n *concatNode) Range() (rune, rune, bool) { @@ -258,7 +258,7 @@ func newAltNode(left, right CPTree) *altNode { } func (n *altNode) String() string { - return fmt.Sprintf("alt") + return "alt" } func (n *altNode) Range() (rune, rune, bool) { @@ -419,6 +419,7 @@ func (n *fragmentNode) clone() CPTree { return newFragmentNode(n.kind, n.tree.clone()) } +//nolint:unused func printCPTree(w io.Writer, t CPTree, ruledLine string, childRuledLinePrefix string) { if t == nil { return diff --git a/driver/lexer_test.go b/driver/lexer_test.go index e0b82b1..9e5148c 100644 --- a/driver/lexer_test.go +++ b/driver/lexer_test.go @@ -72,9 +72,9 @@ func newEOFTokenDefault() *Token { return newEOFToken(ModeID(spec.LexModeIDDefault.Int()), spec.LexModeNameDefault.String()) } -func newInvalidToken(modeID ModeID, lexeme []byte) *Token { +func newInvalidTokenDefault(lexeme []byte) *Token { return &Token{ - ModeID: modeID, + ModeID: ModeID(spec.LexModeIDDefault.Int()), ModeKindID: 0, Lexeme: lexeme, Invalid: true, @@ -772,6 +772,22 @@ func TestLexer_Next(t *testing.T) { newEOFTokenDefault(), }, }, + // The driver can continue lexical analysis even after it detects an invalid token. + { + lspec: &spec.LexSpec{ + Name: "test", + Entries: []*spec.LexEntry{ + newLexEntryDefaultNOP("lower", `[a-z]+`), + }, + }, + src: `foo123bar`, + tokens: []*Token{ + newTokenDefault(1, 1, []byte(`foo`)), + newInvalidTokenDefault([]byte(`123`)), + newTokenDefault(1, 1, []byte(`bar`)), + newEOFTokenDefault(), + }, + }, } for i, tt := range test { for compLv := compiler.CompressionLevelMin; compLv <= compiler.CompressionLevelMax; compLv++ { |