Add vm target to pegen script to generate the vm parser (python#130)

Author: lysnikolaou

Tools/peg_generator/Makefile

@@ -25,6 +25,9 @@ build: peg_extension/parse.c
 peg_extension/parse.c: $(GRAMMAR) $(TOKENS) pegen/*.py peg_extension/peg_extension.c ../../Parser/pegen/pegen.c ../../Parser/pegen/parse_string.c ../../Parser/pegen/*.h pegen/grammar_parser.py
 	$(PYTHON) -m pegen -q c $(GRAMMAR) $(TOKENS) -o peg_extension/parse.c --compile-extension
 
+generate_vm: $(GRAMMAR) $(TOKENS) pegen/*.py ../../Parser/pegen/pegen.c ../../Parser/pegen/parse_string.c ../../Parser/pegen/*.h
+	$(PYTHON) -m pegen -q vm $(GRAMMAR) $(TOKENS) -o ../../Parser/pegen/vmparse.h
+
 clean:
 	-rm -f peg_extension/*.o peg_extension/*.so peg_extension/parse.c
 	-rm -f data/xxl.py

Tools/peg_generator/pegen/__main__.py

@@ -16,6 +16,31 @@
 from pegen.build import Grammar, Parser, Tokenizer, ParserGenerator
 
 
+def generate_vm_code(
+    args: argparse.Namespace,
+) -> Tuple[Grammar, Parser, Tokenizer, ParserGenerator]:
+    from pegen.build import build_vm_parser_and_generator
+
+    verbose = args.verbose
+    verbose_tokenizer = verbose >= 3
+    verbose_parser = verbose == 2 or verbose >= 4
+    try:
+        grammar, parser, tokenizer, gen = build_vm_parser_and_generator(
+            args.grammar_filename,
+            args.tokens_filename,
+            args.output,
+            verbose_tokenizer,
+            verbose_parser,
+        )
+        return grammar, parser, tokenizer, gen
+    except Exception as err:
+        if args.verbose:
+            raise  # Show traceback
+        traceback.print_exception(err.__class__, err, None)
+        sys.stderr.write("For full traceback, use -v\n")
+        sys.exit(1)
+
+
 def generate_c_code(
     args: argparse.Namespace,
 ) -> Tuple[Grammar, Parser, Tokenizer, ParserGenerator]:
@@ -116,6 +141,18 @@ def generate_python_code(
     "--skip-actions", action="store_true", help="Suppress code emission for rule actions",
 )
 
+vm_parser = subparsers.add_parser("vm", help="Generate the new VM parser generator")
+vm_parser.set_defaults(func=generate_vm_code)
+vm_parser.add_argument("grammar_filename", help="Grammar description")
+vm_parser.add_argument("tokens_filename", help="Tokens description")
+vm_parser.add_argument(
+    "-o",
+    "--output",
+    metavar="OUT",
+    default="vmparse.h",
+    help="Where to write the generated parser",
+)
+
 
 def main() -> None:
     from pegen.testutil import print_memstats

Tools/peg_generator/pegen/build.py

@@ -8,6 +8,7 @@
 from typing import Optional, Tuple, List, IO, Set, Dict
 
 from pegen.c_generator import CParserGenerator
+from pegen.vm_generator import VMParserGenerator
 from pegen.grammar import Grammar
 from pegen.grammar_parser import GeneratedParser as GrammarParser
 from pegen.parser import Parser
@@ -181,6 +182,19 @@ def build_python_generator(
     return gen
 
 
+def build_vm_generator(
+    grammar: Grammar, grammar_file: str, tokens_file: str, output_file: str,
+) -> ParserGenerator:
+    with open(tokens_file, "r") as tok_file:
+        all_tokens, exact_tok, non_exact_tok = generate_token_definitions(tok_file)
+    with open(output_file, "w") as file:
+        gen: ParserGenerator = VMParserGenerator(
+            grammar, all_tokens, exact_tok, non_exact_tok, file
+        )
+        gen.generate(grammar_file)
+    return gen
+
+
 def build_c_parser_and_generator(
     grammar_file: str,
     tokens_file: str,
@@ -246,3 +260,26 @@ def build_python_parser_and_generator(
     grammar, parser, tokenizer = build_parser(grammar_file, verbose_tokenizer, verbose_parser)
     gen = build_python_generator(grammar, grammar_file, output_file, skip_actions=skip_actions,)
     return grammar, parser, tokenizer, gen
+
+
+def build_vm_parser_and_generator(
+    grammar_file: str,
+    tokens_file: str,
+    output_file: str,
+    verbose_tokenizer: bool = False,
+    verbose_parser: bool = False,
+) -> Tuple[Grammar, Parser, Tokenizer, ParserGenerator]:
+    """Generate rules, C parser, tokenizer, parser generator for a given grammar
+
+    Args:
+        grammar_file (string): Path for the grammar file
+        tokens_file (string): Path for the tokens file
+        output_file (string): Path for the output file
+        verbose_tokenizer (bool, optional): Whether to display additional output
+          when generating the tokenizer. Defaults to False.
+        verbose_parser (bool, optional): Whether to display additional output
+          when generating the parser. Defaults to False.
+    """
+    grammar, parser, tokenizer = build_parser(grammar_file, verbose_tokenizer, verbose_parser)
+    gen = build_vm_generator(grammar, grammar_file, tokens_file, output_file)
+    return grammar, parser, tokenizer, gen

Tools/peg_generator/pegen/vm_generator.py

@@ -7,10 +7,9 @@
 import tokenize
 from collections import defaultdict
 from itertools import accumulate
-from typing import Any, Dict, Iterator, List, Optional, Tuple, Union
+from typing import Any, Dict, Iterator, List, Optional, Tuple, Set, IO, Text, Union
 
 from pegen import grammar
-from pegen.build import build_parser
 from pegen.grammar import (
     Alt,
     Cut,
@@ -77,9 +76,14 @@ def __init__(self, name: str, startrulename: str):
 
 class VMCallMakerVisitor(GrammarVisitor):
     def __init__(
-        self, parser_generator: ParserGenerator,
+        self,
+        parser_generator: ParserGenerator,
+        exact_tokens: Dict[str, int],
+        non_exact_tokens: Set[str],
     ):
         self.gen = parser_generator
+        self.exact_tokens = exact_tokens
+        self.non_exact_tokens = non_exact_tokens
         self.cache: Dict[Any, Any] = {}
         self.keyword_cache: Dict[str, int] = {}
         self.soft_keyword_cache: List[str] = []
@@ -101,8 +105,8 @@ def visit_StringLeaf(self, node: StringLeaf) -> Tuple[str, str]:
                 return self.keyword_helper(val)
             else:
                 return self.soft_keyword_helper(val)
-        tok_num: int = token.EXACT_TOKEN_TYPES[val]  # type: ignore [attr-defined]
-        return "OP_TOKEN", token.tok_name[tok_num]
+        tok_num: int = self.exact_tokens[val]
+        return "OP_TOKEN", self.gen.tokens[tok_num]
 
     def visit_Repeat0(self, node: Repeat0) -> str:
         if node in self.cache:
@@ -149,12 +153,19 @@ def can_we_inline(node: Rhs) -> int:
 
 class VMParserGenerator(ParserGenerator, GrammarVisitor):
     def __init__(
-        self, grammar: grammar.Grammar,
+        self,
+        grammar: grammar.Grammar,
+        tokens: Dict[str, int],
+        exact_tokens: Dict[str, int],
+        non_exact_tokens: Set[str],
+        file: Optional[IO[Text]],
     ):
-        super().__init__(grammar, token.tok_name, sys.stdout)
+        super().__init__(grammar, tokens, file)
 
         self.opcode_buffer: Optional[List[Opcode]] = None
-        self.callmakervisitor: VMCallMakerVisitor = VMCallMakerVisitor(self)
+        self.callmakervisitor: VMCallMakerVisitor = VMCallMakerVisitor(
+            self, exact_tokens, non_exact_tokens,
+        )
 
     @contextlib.contextmanager
     def set_opcode_buffer(self, buffer: List[Opcode]) -> Iterator[None]:
@@ -517,6 +528,7 @@ def visit_Gather(self, node: Gather) -> None:
 
 
 def main() -> None:
+    from pegen.build import build_parser
     filename = "../../Grammar/python.gram"
     if sys.argv[1:]:
         filename = sys.argv[1]