1 # Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved.

     2 # Licensed to PSF under a Contributor Agreement.

     3 

     4 # Modifications:

     5 # Copyright 2006 Google, Inc. All Rights Reserved.

     6 # Licensed to PSF under a Contributor Agreement.

     7 

     8 """Parser driver.

     9 

    10 This provides a high-level interface to parse a file into a syntax tree.

    11 

    12 """

    13 

    14 __author__ = "Guido van Rossum <guido@python.org>"

    15 

    16 __all__ = ["Driver", "load_grammar"]

    17 

    18 # Python imports

    19 import os

    20 import logging

    21 import sys

    22 

    23 # Pgen imports

    24 from . import grammar, parse, token, tokenize, pgen

    25 

    26 

    27 class Driver(object):

    28 

    29     def __init__(self, grammar, convert=None, logger=None):

    30         self.grammar = grammar

    31         if logger is None:

    32             logger = logging.getLogger()

    33         self.logger = logger

    34         self.convert = convert

    35 

    36     def parse_tokens(self, tokens, debug=False):

    37         """Parse a series of tokens and return the syntax tree."""

    38         # XXX Move the prefix computation into a wrapper around tokenize.

    39         p = parse.Parser(self.grammar, self.convert)

    40         p.setup()

    41         lineno = 1

    42         column = 0

    43         type = value = start = end = line_text = None

    44         prefix = ""

    45         for quintuple in tokens:

    46             type, value, start, end, line_text = quintuple

    47             if start != (lineno, column):

    48                 assert (lineno, column) <= start, ((lineno, column), start)

    49                 s_lineno, s_column = start

    50                 if lineno < s_lineno:

    51                     prefix += "\n" * (s_lineno - lineno)

    52                     lineno = s_lineno

    53                     column = 0

    54                 if column < s_column:

    55                     prefix += line_text[column:s_column]

    56                     column = s_column

    57             if type in (tokenize.COMMENT, tokenize.NL):

    58                 prefix += value

    59                 lineno, column = end

    60                 if value.endswith("\n"):

    61                     lineno += 1

    62                     column = 0

    63                 continue

    64             if type == token.OP:

    65                 type = grammar.opmap[value]

    66             if debug:

    67                 self.logger.debug("%s %r (prefix=%r)",

    68                                   token.tok_name[type], value, prefix)

    69             if p.addtoken(type, value, (prefix, start)):

    70                 if debug:

    71                     self.logger.debug("Stop.")

    72                 break

    73             prefix = ""

    74             lineno, column = end

    75             if value.endswith("\n"):

    76                 lineno += 1

    77                 column = 0

    78         else:

    79             # We never broke out -- EOF is too soon (how can this happen???)

    80             raise parse.ParseError("incomplete input", t, v, x)

    81         return p.rootnode

    82 

    83     def parse_stream_raw(self, stream, debug=False):

    84         """Parse a stream and return the syntax tree."""

    85         tokens = tokenize.generate_tokens(stream.readline)

    86         return self.parse_tokens(tokens, debug)

    87 

    88     def parse_stream(self, stream, debug=False):

    89         """Parse a stream and return the syntax tree."""

    90         return self.parse_stream_raw(stream, debug)

    91 

    92     def parse_file(self, filename, debug=False):

    93         """Parse a file and return the syntax tree."""

    94         stream = open(filename)

    95         try:

    96             return self.parse_stream(stream, debug)

    97         finally:

    98             stream.close()

    99 

   100     def parse_string(self, text, debug=False):

   101         """Parse a string and return the syntax tree."""

   102         tokens = tokenize.generate_tokens(generate_lines(text).next)

   103         return self.parse_tokens(tokens, debug)

   104 

   105 

   106 def generate_lines(text):

   107     """Generator that behaves like readline without using StringIO."""

   108     for line in text.splitlines(True):

   109         yield line

   110     while True:

   111         yield ""

   112 

   113 

   114 def load_grammar(gt="Grammar.txt", gp=None,

   115                  save=True, force=False, logger=None):

   116     """Load the grammar (maybe from a pickle)."""

   117     if logger is None:

   118         logger = logging.getLogger()

   119     if gp is None:

   120         head, tail = os.path.splitext(gt)

   121         if tail == ".txt":

   122             tail = ""

   123         gp = head + tail + ".".join(map(str, sys.version_info)) + ".pickle"

   124     if force or not _newer(gp, gt):

   125         logger.info("Generating grammar tables from %s", gt)

   126         g = pgen.generate_grammar(gt)

   127         if save:

   128             logger.info("Writing grammar tables to %s", gp)

   129             try:

   130                 g.dump(gp)

   131             except IOError, e:

   132                 logger.info("Writing failed:"+str(e))

   133     else:

   134         g = grammar.Grammar()

   135         g.load(gp)

   136     return g

   137 

   138 

   139 def _newer(a, b):

   140     """Inquire whether file a was written since file b."""

   141     if not os.path.exists(a):

   142         return False

   143     if not os.path.exists(b):

   144         return True

   145     return os.path.getmtime(a) >= os.path.getmtime(b)