Add skip action

2 files changed, 33 insertions, 12 deletions

diff --git a/driver/parser.go b/driver/parser.go
index 12081bd..adc2a54 100644
--- a/driver/parser.go
+++ b/driver/parser.go
@@ -57,13 +57,10 @@ func NewParser(gram *spec.CompiledGrammar, src io.Reader) (*Parser, error) {
 func (p *Parser) Parse() error {
 	termCount := p.gram.ParsingTable.TerminalCount
 	p.push(p.gram.ParsingTable.InitialState)
-	tok, err := p.lex.Next()
+	tok, err := p.nextToken()
 	if err != nil {
 		return err
 	}
-	if tok.Invalid {
-		return fmt.Errorf("invalid token: %+v", tok)
-	}
 	for {
 		var tsym int
 		if tok.EOF {
@@ -113,16 +110,28 @@ func (p *Parser) Parse() error {
 	}
 }
 
+func (p *Parser) nextToken() (*mldriver.Token, error) {
+	skip := p.gram.LexicalSpecification.Maleeni.Skip
+	for {
+		tok, err := p.lex.Next()
+		if err != nil {
+			return nil, err
+		}
+		if tok.Invalid {
+			return nil, fmt.Errorf("invalid token: %+v", tok)
+		}
+
+		if skip[tok.Mode.Int()][tok.Kind] > 0 {
+			continue
+		}
+
+		return tok, nil
+	}
+}
+
 func (p *Parser) shift(nextState int) (*mldriver.Token, error) {
 	p.push(nextState)
-	tok, err := p.lex.Next()
-	if err != nil {
-		return nil, err
-	}
-	if tok.Invalid {
-		return nil, fmt.Errorf("invalid token: %+v", tok)
-	}
-	return tok, nil
+	return p.nextToken()
 }
 
 func (p *Parser) reduce(prodNum int) bool {
diff --git a/driver/parser_test.go b/driver/parser_test.go
index 17fee21..75b25ae 100644
--- a/driver/parser_test.go
+++ b/driver/parser_test.go
@@ -64,6 +64,18 @@ bar: "bar";
 `,
 			src: `foobar`,
 		},
+		// The parser can skips specified tokens.
+		{
+			specSrc: `
+s
+    : foo bar
+    ;
+foo: "foo";
+bar: "bar";
+white_space: "[\u{0009}\u{0020}]+" # skip;
+`,
+			src: `foo bar`,
+		},
 	}
 	for _, tt := range tests {
 		ast, err := spec.Parse(strings.NewReader(tt.specSrc))