diff options
-rw-r--r-- | cmd/maleeni/compile.go | 10 | ||||
-rw-r--r-- | cmd/maleeni/lex.go | 4 | ||||
-rw-r--r-- | compiler/compiler.go | 64 | ||||
-rw-r--r-- | driver/lexer.go | 28 | ||||
-rw-r--r-- | driver/lexer_test.go | 40 | ||||
-rw-r--r-- | spec/spec.go | 18 |
6 files changed, 119 insertions, 45 deletions
diff --git a/cmd/maleeni/compile.go b/cmd/maleeni/compile.go index 6a5e303..fd24afe 100644 --- a/cmd/maleeni/compile.go +++ b/cmd/maleeni/compile.go @@ -15,6 +15,7 @@ import ( var compileFlags = struct { debug *bool lexSpec *string + compLv *int output *string }{} @@ -27,8 +28,9 @@ func init() { RunE: runCompile, } compileFlags.debug = cmd.Flags().BoolP("debug", "d", false, "enable logging") - compileFlags.lexSpec = cmd.Flags().StringP("lex-spec", "l", "", "lexical specification file path (default: stdin)") - compileFlags.output = cmd.Flags().StringP("output", "o", "", "output file path (default: stdout)") + compileFlags.lexSpec = cmd.Flags().StringP("lex-spec", "l", "", "lexical specification file path (default stdin)") + compileFlags.compLv = cmd.Flags().IntP("compression-level", "c", compiler.CompressionLevelMax, "compression level") + compileFlags.output = cmd.Flags().StringP("output", "o", "", "output file path (default stdout)") rootCmd.AddCommand(cmd) } @@ -38,7 +40,9 @@ func runCompile(cmd *cobra.Command, args []string) (retErr error) { return fmt.Errorf("Cannot read a lexical specification: %w", err) } - var opts []compiler.CompilerOption + opts := []compiler.CompilerOption{ + compiler.CompressionLevel(*compileFlags.compLv), + } if *compileFlags.debug { fileName := "maleeni-compile.log" f, err := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) diff --git a/cmd/maleeni/lex.go b/cmd/maleeni/lex.go index c111370..e6eab18 100644 --- a/cmd/maleeni/lex.go +++ b/cmd/maleeni/lex.go @@ -30,8 +30,8 @@ As use ` + "`maleeni compile`" + `, you can generate the specification.`, RunE: runLex, } lexFlags.debug = cmd.Flags().BoolP("debug", "d", false, "enable logging") - lexFlags.source = cmd.Flags().StringP("source", "s", "", "source file path (default: stdin)") - lexFlags.output = cmd.Flags().StringP("output", "o", "", "output file path (default: stdout)") + lexFlags.source = cmd.Flags().StringP("source", "s", "", "source file path (default stdin)") + lexFlags.output = cmd.Flags().StringP("output", "o", "", "output file path (default stdout)") lexFlags.breakOnError = cmd.Flags().BoolP("break-on-error", "b", false, "break lexical analysis with exit status 1 immediately when an error token appears.") rootCmd.AddCommand(cmd) } diff --git a/compiler/compiler.go b/compiler/compiler.go index d1720de..2fdbce1 100644 --- a/compiler/compiler.go +++ b/compiler/compiler.go @@ -23,8 +23,19 @@ func EnableLogging(w io.Writer) CompilerOption { } } +func CompressionLevel(lv int) CompilerOption { + return func(c *compilerConfig) error { + if lv < CompressionLevelMin || lv > CompressionLevelMax { + return fmt.Errorf("compression level must be %v to %v", CompressionLevelMin, CompressionLevelMax) + } + c.compLv = lv + return nil + } +} + type compilerConfig struct { logger log.Logger + compLv int } func Compile(lexspec *spec.LexSpec, opts ...CompilerOption) (*spec.CompiledLexSpec, error) { @@ -59,9 +70,10 @@ func Compile(lexspec *spec.LexSpec, opts ...CompilerOption) (*spec.CompiledLexSp } return &spec.CompiledLexSpec{ - InitialMode: spec.LexModeNumDefault, - Modes: modes, - Specs: modeSpecs, + InitialMode: spec.LexModeNumDefault, + Modes: modes, + CompressionLevel: config.compLv, + Specs: modeSpecs, }, nil } @@ -169,9 +181,18 @@ func compile(entries []*spec.LexEntry, modeNums map[spec.LexModeName]spec.LexMod } } - tranTab, err := compressTransitionTable(tranTab) - if err != nil { - return nil, err + var err error + switch config.compLv { + case 2: + tranTab, err = compressTransitionTableLv2(tranTab) + if err != nil { + return nil, err + } + case 1: + tranTab, err = compressTransitionTableLv1(tranTab) + if err != nil { + return nil, err + } } return &spec.CompiledLexModeSpec{ @@ -182,7 +203,12 @@ func compile(entries []*spec.LexEntry, modeNums map[spec.LexModeName]spec.LexMod }, nil } -func compressTransitionTable(tranTab *spec.TransitionTable) (*spec.TransitionTable, error) { +const ( + CompressionLevelMin = 0 + CompressionLevelMax = 2 +) + +func compressTransitionTableLv2(tranTab *spec.TransitionTable) (*spec.TransitionTable, error) { ueTab := compressor.NewUniqueEntriesTable() { orig, err := compressor.NewOriginalTable(tranTab.UncompressedTransition, tranTab.ColCount) @@ -224,3 +250,27 @@ func compressTransitionTable(tranTab *spec.TransitionTable) (*spec.TransitionTab return tranTab, nil } + +func compressTransitionTableLv1(tranTab *spec.TransitionTable) (*spec.TransitionTable, error) { + ueTab := compressor.NewUniqueEntriesTable() + { + orig, err := compressor.NewOriginalTable(tranTab.UncompressedTransition, tranTab.ColCount) + if err != nil { + return nil, err + } + err = ueTab.Compress(orig) + if err != nil { + return nil, err + } + } + + tranTab.Transition = &spec.UniqueEntriesTable{ + UncompressedUniqueEntries: ueTab.UniqueEntries, + RowNums: ueTab.RowNums, + OriginalRowCount: ueTab.OriginalRowCount, + OriginalColCount: ueTab.OriginalColCount, + } + tranTab.UncompressedTransition = nil + + return tranTab, nil +} diff --git a/driver/lexer.go b/driver/lexer.go index bac478f..278a104 100644 --- a/driver/lexer.go +++ b/driver/lexer.go @@ -319,13 +319,29 @@ func (l *Lexer) next() (*Token, error) { } func (l *Lexer) lookupNextState(mode spec.LexModeNum, state int, v int) (int, bool) { - tab := l.clspec.Specs[mode].DFA.Transition - rowNum := tab.RowNums[state] - d := tab.UniqueEntries.RowDisplacement[rowNum] - if tab.UniqueEntries.Bounds[d+v] != rowNum { - return tab.UniqueEntries.EmptyValue, false + switch l.clspec.CompressionLevel { + case 2: + tab := l.clspec.Specs[mode].DFA.Transition + rowNum := tab.RowNums[state] + d := tab.UniqueEntries.RowDisplacement[rowNum] + if tab.UniqueEntries.Bounds[d+v] != rowNum { + return tab.UniqueEntries.EmptyValue, false + } + return tab.UniqueEntries.Entries[d+v], true + case 1: + tab := l.clspec.Specs[mode].DFA.Transition + next := tab.UncompressedUniqueEntries[tab.RowNums[state]*tab.OriginalColCount+v] + if next == 0 { + return 0, false + } + return next, true + } + spec := l.clspec.Specs[mode] + next := spec.DFA.UncompressedTransition[state*spec.DFA.ColCount+v] + if next == 0 { + return 0, false } - return tab.UniqueEntries.Entries[d+v], true + return next, true } func (l *Lexer) mode() spec.LexModeNum { diff --git a/driver/lexer_test.go b/driver/lexer_test.go index f11a9c0..0156ef8 100644 --- a/driver/lexer_test.go +++ b/driver/lexer_test.go @@ -526,28 +526,30 @@ func TestLexer_Next(t *testing.T) { }, } for i, tt := range test { - t.Run(fmt.Sprintf("#%v", i), func(t *testing.T) { - clspec, err := compiler.Compile(tt.lspec) - if err != nil { - t.Fatalf("unexpected error occurred: %v", err) - } - lexer, err := NewLexer(clspec, strings.NewReader(tt.src)) - if err != nil { - t.Fatalf("unexpecated error occurred; %v", err) - } - for _, eTok := range tt.tokens { - tok, err := lexer.Next() + for compLv := compiler.CompressionLevelMin; compLv <= compiler.CompressionLevelMax; compLv++ { + t.Run(fmt.Sprintf("#%v-%v", i, compLv), func(t *testing.T) { + clspec, err := compiler.Compile(tt.lspec, compiler.CompressionLevel(compLv)) if err != nil { - t.Log(err) - break + t.Fatalf("unexpected error occurred: %v", err) } - testToken(t, eTok, tok) - // t.Logf("token: ID: %v, Match: %+v Text: \"%v\", EOF: %v, Invalid: %v", tok.ID, tok.Match(), tok.Text(), tok.EOF, tok.Invalid) - if tok.EOF { - break + lexer, err := NewLexer(clspec, strings.NewReader(tt.src)) + if err != nil { + t.Fatalf("unexpecated error occurred; %v", err) + } + for _, eTok := range tt.tokens { + tok, err := lexer.Next() + if err != nil { + t.Log(err) + break + } + testToken(t, eTok, tok) + // t.Logf("token: ID: %v, Match: %+v Text: \"%v\", EOF: %v, Invalid: %v", tok.ID, tok.Match(), tok.Text(), tok.EOF, tok.Invalid) + if tok.EOF { + break + } } - } - }) + }) + } } } diff --git a/spec/spec.go b/spec/spec.go index 7398ea3..e3c318e 100644 --- a/spec/spec.go +++ b/spec/spec.go @@ -162,11 +162,12 @@ type RowDisplacementTable struct { } type UniqueEntriesTable struct { - UniqueEntries *RowDisplacementTable `json:"unique_entries"` - RowNums []int `json:"row_nums"` - OriginalRowCount int `json:"original_row_count"` - OriginalColCount int `json:"original_col_count"` - EmptyValue int `json:"empty_value"` + UniqueEntries *RowDisplacementTable `json:"unique_entries,omitempty"` + UncompressedUniqueEntries []int `json:"uncompressed_unique_entries,omitempty"` + RowNums []int `json:"row_nums"` + OriginalRowCount int `json:"original_row_count"` + OriginalColCount int `json:"original_col_count"` + EmptyValue int `json:"empty_value"` } type TransitionTable struct { @@ -186,7 +187,8 @@ type CompiledLexModeSpec struct { } type CompiledLexSpec struct { - InitialMode LexModeNum `json:"initial_mode"` - Modes []LexModeName `json:"modes"` - Specs []*CompiledLexModeSpec `json:"specs"` + InitialMode LexModeNum `json:"initial_mode"` + Modes []LexModeName `json:"modes"` + CompressionLevel int `json:"compression_level"` + Specs []*CompiledLexModeSpec `json:"specs"` } |