aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--cmd/maleeni/compile.go10
-rw-r--r--cmd/maleeni/lex.go4
-rw-r--r--compiler/compiler.go64
-rw-r--r--driver/lexer.go28
-rw-r--r--driver/lexer_test.go40
-rw-r--r--spec/spec.go18
6 files changed, 119 insertions, 45 deletions
diff --git a/cmd/maleeni/compile.go b/cmd/maleeni/compile.go
index 6a5e303..fd24afe 100644
--- a/cmd/maleeni/compile.go
+++ b/cmd/maleeni/compile.go
@@ -15,6 +15,7 @@ import (
var compileFlags = struct {
debug *bool
lexSpec *string
+ compLv *int
output *string
}{}
@@ -27,8 +28,9 @@ func init() {
RunE: runCompile,
}
compileFlags.debug = cmd.Flags().BoolP("debug", "d", false, "enable logging")
- compileFlags.lexSpec = cmd.Flags().StringP("lex-spec", "l", "", "lexical specification file path (default: stdin)")
- compileFlags.output = cmd.Flags().StringP("output", "o", "", "output file path (default: stdout)")
+ compileFlags.lexSpec = cmd.Flags().StringP("lex-spec", "l", "", "lexical specification file path (default stdin)")
+ compileFlags.compLv = cmd.Flags().IntP("compression-level", "c", compiler.CompressionLevelMax, "compression level")
+ compileFlags.output = cmd.Flags().StringP("output", "o", "", "output file path (default stdout)")
rootCmd.AddCommand(cmd)
}
@@ -38,7 +40,9 @@ func runCompile(cmd *cobra.Command, args []string) (retErr error) {
return fmt.Errorf("Cannot read a lexical specification: %w", err)
}
- var opts []compiler.CompilerOption
+ opts := []compiler.CompilerOption{
+ compiler.CompressionLevel(*compileFlags.compLv),
+ }
if *compileFlags.debug {
fileName := "maleeni-compile.log"
f, err := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
diff --git a/cmd/maleeni/lex.go b/cmd/maleeni/lex.go
index c111370..e6eab18 100644
--- a/cmd/maleeni/lex.go
+++ b/cmd/maleeni/lex.go
@@ -30,8 +30,8 @@ As use ` + "`maleeni compile`" + `, you can generate the specification.`,
RunE: runLex,
}
lexFlags.debug = cmd.Flags().BoolP("debug", "d", false, "enable logging")
- lexFlags.source = cmd.Flags().StringP("source", "s", "", "source file path (default: stdin)")
- lexFlags.output = cmd.Flags().StringP("output", "o", "", "output file path (default: stdout)")
+ lexFlags.source = cmd.Flags().StringP("source", "s", "", "source file path (default stdin)")
+ lexFlags.output = cmd.Flags().StringP("output", "o", "", "output file path (default stdout)")
lexFlags.breakOnError = cmd.Flags().BoolP("break-on-error", "b", false, "break lexical analysis with exit status 1 immediately when an error token appears.")
rootCmd.AddCommand(cmd)
}
diff --git a/compiler/compiler.go b/compiler/compiler.go
index d1720de..2fdbce1 100644
--- a/compiler/compiler.go
+++ b/compiler/compiler.go
@@ -23,8 +23,19 @@ func EnableLogging(w io.Writer) CompilerOption {
}
}
+func CompressionLevel(lv int) CompilerOption {
+ return func(c *compilerConfig) error {
+ if lv < CompressionLevelMin || lv > CompressionLevelMax {
+ return fmt.Errorf("compression level must be %v to %v", CompressionLevelMin, CompressionLevelMax)
+ }
+ c.compLv = lv
+ return nil
+ }
+}
+
type compilerConfig struct {
logger log.Logger
+ compLv int
}
func Compile(lexspec *spec.LexSpec, opts ...CompilerOption) (*spec.CompiledLexSpec, error) {
@@ -59,9 +70,10 @@ func Compile(lexspec *spec.LexSpec, opts ...CompilerOption) (*spec.CompiledLexSp
}
return &spec.CompiledLexSpec{
- InitialMode: spec.LexModeNumDefault,
- Modes: modes,
- Specs: modeSpecs,
+ InitialMode: spec.LexModeNumDefault,
+ Modes: modes,
+ CompressionLevel: config.compLv,
+ Specs: modeSpecs,
}, nil
}
@@ -169,9 +181,18 @@ func compile(entries []*spec.LexEntry, modeNums map[spec.LexModeName]spec.LexMod
}
}
- tranTab, err := compressTransitionTable(tranTab)
- if err != nil {
- return nil, err
+ var err error
+ switch config.compLv {
+ case 2:
+ tranTab, err = compressTransitionTableLv2(tranTab)
+ if err != nil {
+ return nil, err
+ }
+ case 1:
+ tranTab, err = compressTransitionTableLv1(tranTab)
+ if err != nil {
+ return nil, err
+ }
}
return &spec.CompiledLexModeSpec{
@@ -182,7 +203,12 @@ func compile(entries []*spec.LexEntry, modeNums map[spec.LexModeName]spec.LexMod
}, nil
}
-func compressTransitionTable(tranTab *spec.TransitionTable) (*spec.TransitionTable, error) {
+const (
+ CompressionLevelMin = 0
+ CompressionLevelMax = 2
+)
+
+func compressTransitionTableLv2(tranTab *spec.TransitionTable) (*spec.TransitionTable, error) {
ueTab := compressor.NewUniqueEntriesTable()
{
orig, err := compressor.NewOriginalTable(tranTab.UncompressedTransition, tranTab.ColCount)
@@ -224,3 +250,27 @@ func compressTransitionTable(tranTab *spec.TransitionTable) (*spec.TransitionTab
return tranTab, nil
}
+
+func compressTransitionTableLv1(tranTab *spec.TransitionTable) (*spec.TransitionTable, error) {
+ ueTab := compressor.NewUniqueEntriesTable()
+ {
+ orig, err := compressor.NewOriginalTable(tranTab.UncompressedTransition, tranTab.ColCount)
+ if err != nil {
+ return nil, err
+ }
+ err = ueTab.Compress(orig)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ tranTab.Transition = &spec.UniqueEntriesTable{
+ UncompressedUniqueEntries: ueTab.UniqueEntries,
+ RowNums: ueTab.RowNums,
+ OriginalRowCount: ueTab.OriginalRowCount,
+ OriginalColCount: ueTab.OriginalColCount,
+ }
+ tranTab.UncompressedTransition = nil
+
+ return tranTab, nil
+}
diff --git a/driver/lexer.go b/driver/lexer.go
index bac478f..278a104 100644
--- a/driver/lexer.go
+++ b/driver/lexer.go
@@ -319,13 +319,29 @@ func (l *Lexer) next() (*Token, error) {
}
func (l *Lexer) lookupNextState(mode spec.LexModeNum, state int, v int) (int, bool) {
- tab := l.clspec.Specs[mode].DFA.Transition
- rowNum := tab.RowNums[state]
- d := tab.UniqueEntries.RowDisplacement[rowNum]
- if tab.UniqueEntries.Bounds[d+v] != rowNum {
- return tab.UniqueEntries.EmptyValue, false
+ switch l.clspec.CompressionLevel {
+ case 2:
+ tab := l.clspec.Specs[mode].DFA.Transition
+ rowNum := tab.RowNums[state]
+ d := tab.UniqueEntries.RowDisplacement[rowNum]
+ if tab.UniqueEntries.Bounds[d+v] != rowNum {
+ return tab.UniqueEntries.EmptyValue, false
+ }
+ return tab.UniqueEntries.Entries[d+v], true
+ case 1:
+ tab := l.clspec.Specs[mode].DFA.Transition
+ next := tab.UncompressedUniqueEntries[tab.RowNums[state]*tab.OriginalColCount+v]
+ if next == 0 {
+ return 0, false
+ }
+ return next, true
+ }
+ spec := l.clspec.Specs[mode]
+ next := spec.DFA.UncompressedTransition[state*spec.DFA.ColCount+v]
+ if next == 0 {
+ return 0, false
}
- return tab.UniqueEntries.Entries[d+v], true
+ return next, true
}
func (l *Lexer) mode() spec.LexModeNum {
diff --git a/driver/lexer_test.go b/driver/lexer_test.go
index f11a9c0..0156ef8 100644
--- a/driver/lexer_test.go
+++ b/driver/lexer_test.go
@@ -526,28 +526,30 @@ func TestLexer_Next(t *testing.T) {
},
}
for i, tt := range test {
- t.Run(fmt.Sprintf("#%v", i), func(t *testing.T) {
- clspec, err := compiler.Compile(tt.lspec)
- if err != nil {
- t.Fatalf("unexpected error occurred: %v", err)
- }
- lexer, err := NewLexer(clspec, strings.NewReader(tt.src))
- if err != nil {
- t.Fatalf("unexpecated error occurred; %v", err)
- }
- for _, eTok := range tt.tokens {
- tok, err := lexer.Next()
+ for compLv := compiler.CompressionLevelMin; compLv <= compiler.CompressionLevelMax; compLv++ {
+ t.Run(fmt.Sprintf("#%v-%v", i, compLv), func(t *testing.T) {
+ clspec, err := compiler.Compile(tt.lspec, compiler.CompressionLevel(compLv))
if err != nil {
- t.Log(err)
- break
+ t.Fatalf("unexpected error occurred: %v", err)
}
- testToken(t, eTok, tok)
- // t.Logf("token: ID: %v, Match: %+v Text: \"%v\", EOF: %v, Invalid: %v", tok.ID, tok.Match(), tok.Text(), tok.EOF, tok.Invalid)
- if tok.EOF {
- break
+ lexer, err := NewLexer(clspec, strings.NewReader(tt.src))
+ if err != nil {
+ t.Fatalf("unexpecated error occurred; %v", err)
+ }
+ for _, eTok := range tt.tokens {
+ tok, err := lexer.Next()
+ if err != nil {
+ t.Log(err)
+ break
+ }
+ testToken(t, eTok, tok)
+ // t.Logf("token: ID: %v, Match: %+v Text: \"%v\", EOF: %v, Invalid: %v", tok.ID, tok.Match(), tok.Text(), tok.EOF, tok.Invalid)
+ if tok.EOF {
+ break
+ }
}
- }
- })
+ })
+ }
}
}
diff --git a/spec/spec.go b/spec/spec.go
index 7398ea3..e3c318e 100644
--- a/spec/spec.go
+++ b/spec/spec.go
@@ -162,11 +162,12 @@ type RowDisplacementTable struct {
}
type UniqueEntriesTable struct {
- UniqueEntries *RowDisplacementTable `json:"unique_entries"`
- RowNums []int `json:"row_nums"`
- OriginalRowCount int `json:"original_row_count"`
- OriginalColCount int `json:"original_col_count"`
- EmptyValue int `json:"empty_value"`
+ UniqueEntries *RowDisplacementTable `json:"unique_entries,omitempty"`
+ UncompressedUniqueEntries []int `json:"uncompressed_unique_entries,omitempty"`
+ RowNums []int `json:"row_nums"`
+ OriginalRowCount int `json:"original_row_count"`
+ OriginalColCount int `json:"original_col_count"`
+ EmptyValue int `json:"empty_value"`
}
type TransitionTable struct {
@@ -186,7 +187,8 @@ type CompiledLexModeSpec struct {
}
type CompiledLexSpec struct {
- InitialMode LexModeNum `json:"initial_mode"`
- Modes []LexModeName `json:"modes"`
- Specs []*CompiledLexModeSpec `json:"specs"`
+ InitialMode LexModeNum `json:"initial_mode"`
+ Modes []LexModeName `json:"modes"`
+ CompressionLevel int `json:"compression_level"`
+ Specs []*CompiledLexModeSpec `json:"specs"`
}
generated by cgit v1.2.3 (git 2.49.0) at 2025-07-13 08:33:04 +0000