1 """Parse a Python module and describe its classes and methods.

     2 

     3 Parse enough of a Python file to recognize imports and class and

     4 method definitions, and to find out the superclasses of a class.

     5 

     6 The interface consists of a single function:

     7         readmodule_ex(module [, path])

     8 where module is the name of a Python module, and path is an optional

     9 list of directories where the module is to be searched.  If present,

    10 path is prepended to the system search path sys.path.  The return

    11 value is a dictionary.  The keys of the dictionary are the names of

    12 the classes defined in the module (including classes that are defined

    13 via the from XXX import YYY construct).  The values are class

    14 instances of the class Class defined here.  One special key/value pair

    15 is present for packages: the key '__path__' has a list as its value

    16 which contains the package search path.

    17 

    18 A class is described by the class Class in this module.  Instances

    19 of this class have the following instance variables:

    20         module -- the module name

    21         name -- the name of the class

    22         super -- a list of super classes (Class instances)

    23         methods -- a dictionary of methods

    24         file -- the file in which the class was defined

    25         lineno -- the line in the file on which the class statement occurred

    26 The dictionary of methods uses the method names as keys and the line

    27 numbers on which the method was defined as values.

    28 If the name of a super class is not recognized, the corresponding

    29 entry in the list of super classes is not a class instance but a

    30 string giving the name of the super class.  Since import statements

    31 are recognized and imported modules are scanned as well, this

    32 shouldn't happen often.

    33 

    34 A function is described by the class Function in this module.

    35 Instances of this class have the following instance variables:

    36         module -- the module name

    37         name -- the name of the class

    38         file -- the file in which the class was defined

    39         lineno -- the line in the file on which the class statement occurred

    40 """

    41 

    42 import sys

    43 import imp

    44 import tokenize

    45 from token import NAME, DEDENT, OP

    46 from operator import itemgetter

    47 

    48 __all__ = ["readmodule", "readmodule_ex", "Class", "Function"]

    49 

    50 _modules = {}                           # cache of modules we've seen

    51 

    52 # each Python class is represented by an instance of this class

    53 class Class:

    54     '''Class to represent a Python class.'''

    55     def __init__(self, module, name, super, file, lineno):

    56         self.module = module

    57         self.name = name

    58         if super is None:

    59             super = []

    60         self.super = super

    61         self.methods = {}

    62         self.file = file

    63         self.lineno = lineno

    64 

    65     def _addmethod(self, name, lineno):

    66         self.methods[name] = lineno

    67 

    68 class Function:

    69     '''Class to represent a top-level Python function'''

    70     def __init__(self, module, name, file, lineno):

    71         self.module = module

    72         self.name = name

    73         self.file = file

    74         self.lineno = lineno

    75 

    76 def readmodule(module, path=None):

    77     '''Backwards compatible interface.

    78 

    79     Call readmodule_ex() and then only keep Class objects from the

    80     resulting dictionary.'''

    81 

    82     res = {}

    83     for key, value in _readmodule(module, path or []).items():

    84         if isinstance(value, Class):

    85             res[key] = value

    86     return res

    87 

    88 def readmodule_ex(module, path=None):

    89     '''Read a module file and return a dictionary of classes.

    90 

    91     Search for MODULE in PATH and sys.path, read and parse the

    92     module and return a dictionary with one entry for each class

    93     found in the module.

    94     '''

    95     return _readmodule(module, path or [])

    96 

    97 def _readmodule(module, path, inpackage=None):

    98     '''Do the hard work for readmodule[_ex].

    99 

   100     If INPACKAGE is given, it must be the dotted name of the package in

   101     which we are searching for a submodule, and then PATH must be the

   102     package search path; otherwise, we are searching for a top-level

   103     module, and PATH is combined with sys.path.

   104     '''

   105     # Compute the full module name (prepending inpackage if set)

   106     if inpackage is not None:

   107         fullmodule = "%s.%s" % (inpackage, module)

   108     else:

   109         fullmodule = module

   110 

   111     # Check in the cache

   112     if fullmodule in _modules:

   113         return _modules[fullmodule]

   114 

   115     # Initialize the dict for this module's contents

   116     dict = {}

   117 

   118     # Check if it is a built-in module; we don't do much for these

   119     if module in sys.builtin_module_names and inpackage is None:

   120         _modules[module] = dict

   121         return dict

   122 

   123     # Check for a dotted module name

   124     i = module.rfind('.')

   125     if i >= 0:

   126         package = module[:i]

   127         submodule = module[i+1:]

   128         parent = _readmodule(package, path, inpackage)

   129         if inpackage is not None:

   130             package = "%s.%s" % (inpackage, package)

   131         return _readmodule(submodule, parent['__path__'], package)

   132 

   133     # Search the path for the module

   134     f = None

   135     if inpackage is not None:

   136         f, fname, (_s, _m, ty) = imp.find_module(module, path)

   137     else:

   138         f, fname, (_s, _m, ty) = imp.find_module(module, path + sys.path)

   139     if ty == imp.PKG_DIRECTORY:

   140         dict['__path__'] = [fname]

   141         path = [fname] + path

   142         f, fname, (_s, _m, ty) = imp.find_module('__init__', [fname])

   143     _modules[fullmodule] = dict

   144     if ty != imp.PY_SOURCE:

   145         # not Python source, can't do anything with this module

   146         f.close()

   147         return dict

   148 

   149     stack = [] # stack of (class, indent) pairs

   150 

   151     g = tokenize.generate_tokens(f.readline)

   152     try:

   153         for tokentype, token, start, _end, _line in g:

   154             if tokentype == DEDENT:

   155                 lineno, thisindent = start

   156                 # close nested classes and defs

   157                 while stack and stack[-1][1] >= thisindent:

   158                     del stack[-1]

   159             elif token == 'def':

   160                 lineno, thisindent = start

   161                 # close previous nested classes and defs

   162                 while stack and stack[-1][1] >= thisindent:

   163                     del stack[-1]

   164                 tokentype, meth_name, start = g.next()[0:3]

   165                 if tokentype != NAME:

   166                     continue # Syntax error

   167                 if stack:

   168                     cur_class = stack[-1][0]

   169                     if isinstance(cur_class, Class):

   170                         # it's a method

   171                         cur_class._addmethod(meth_name, lineno)

   172                     # else it's a nested def

   173                 else:

   174                     # it's a function

   175                     dict[meth_name] = Function(fullmodule, meth_name,

   176                                                fname, lineno)

   177                 stack.append((None, thisindent)) # Marker for nested fns

   178             elif token == 'class':

   179                 lineno, thisindent = start

   180                 # close previous nested classes and defs

   181                 while stack and stack[-1][1] >= thisindent:

   182                     del stack[-1]

   183                 tokentype, class_name, start = g.next()[0:3]

   184                 if tokentype != NAME:

   185                     continue # Syntax error

   186                 # parse what follows the class name

   187                 tokentype, token, start = g.next()[0:3]

   188                 inherit = None

   189                 if token == '(':

   190                     names = [] # List of superclasses

   191                     # there's a list of superclasses

   192                     level = 1

   193                     super = [] # Tokens making up current superclass

   194                     while True:

   195                         tokentype, token, start = g.next()[0:3]

   196                         if token in (')', ',') and level == 1:

   197                             n = "".join(super)

   198                             if n in dict:

   199                                 # we know this super class

   200                                 n = dict[n]

   201                             else:

   202                                 c = n.split('.')

   203                                 if len(c) > 1:

   204                                     # super class is of the form

   205                                     # module.class: look in module for

   206                                     # class

   207                                     m = c[-2]

   208                                     c = c[-1]

   209                                     if m in _modules:

   210                                         d = _modules[m]

   211                                         if c in d:

   212                                             n = d[c]

   213                             names.append(n)

   214                             super = []

   215                         if token == '(':

   216                             level += 1

   217                         elif token == ')':

   218                             level -= 1

   219                             if level == 0:

   220                                 break

   221                         elif token == ',' and level == 1:

   222                             pass

   223                         # only use NAME and OP (== dot) tokens for type name

   224                         elif tokentype in (NAME, OP) and level == 1:

   225                             super.append(token)

   226                         # expressions in the base list are not supported

   227                     inherit = names

   228                 cur_class = Class(fullmodule, class_name, inherit,

   229                                   fname, lineno)

   230                 if not stack:

   231                     dict[class_name] = cur_class

   232                 stack.append((cur_class, thisindent))

   233             elif token == 'import' and start[1] == 0:

   234                 modules = _getnamelist(g)

   235                 for mod, _mod2 in modules:

   236                     try:

   237                         # Recursively read the imported module

   238                         if inpackage is None:

   239                             _readmodule(mod, path)

   240                         else:

   241                             try:

   242                                 _readmodule(mod, path, inpackage)

   243                             except ImportError:

   244                                 _readmodule(mod, [])

   245                     except:

   246                         # If we can't find or parse the imported module,

   247                         # too bad -- don't die here.

   248                         pass

   249             elif token == 'from' and start[1] == 0:

   250                 mod, token = _getname(g)

   251                 if not mod or token != "import":

   252                     continue

   253                 names = _getnamelist(g)

   254                 try:

   255                     # Recursively read the imported module

   256                     d = _readmodule(mod, path, inpackage)

   257                 except:

   258                     # If we can't find or parse the imported module,

   259                     # too bad -- don't die here.

   260                     continue

   261                 # add any classes that were defined in the imported module

   262                 # to our name space if they were mentioned in the list

   263                 for n, n2 in names:

   264                     if n in d:

   265                         dict[n2 or n] = d[n]

   266                     elif n == '*':

   267                         # don't add names that start with _

   268                         for n in d:

   269                             if n[0] != '_':

   270                                 dict[n] = d[n]

   271     except StopIteration:

   272         pass

   273 

   274     f.close()

   275     return dict

   276 

   277 def _getnamelist(g):

   278     # Helper to get a comma-separated list of dotted names plus 'as'

   279     # clauses.  Return a list of pairs (name, name2) where name2 is

   280     # the 'as' name, or None if there is no 'as' clause.

   281     names = []

   282     while True:

   283         name, token = _getname(g)

   284         if not name:

   285             break

   286         if token == 'as':

   287             name2, token = _getname(g)

   288         else:

   289             name2 = None

   290         names.append((name, name2))

   291         while token != "," and "\n" not in token:

   292             token = g.next()[1]

   293         if token != ",":

   294             break

   295     return names

   296 

   297 def _getname(g):

   298     # Helper to get a dotted name, return a pair (name, token) where

   299     # name is the dotted name, or None if there was no dotted name,

   300     # and token is the next input token.

   301     parts = []

   302     tokentype, token = g.next()[0:2]

   303     if tokentype != NAME and token != '*':

   304         return (None, token)

   305     parts.append(token)

   306     while True:

   307         tokentype, token = g.next()[0:2]

   308         if token != '.':

   309             break

   310         tokentype, token = g.next()[0:2]

   311         if tokentype != NAME:

   312             break

   313         parts.append(token)

   314     return (".".join(parts), token)

   315 

   316 def _main():

   317     # Main program for testing.

   318     import os

   319     mod = sys.argv[1]

   320     if os.path.exists(mod):

   321         path = [os.path.dirname(mod)]

   322         mod = os.path.basename(mod)

   323         if mod.lower().endswith(".py"):

   324             mod = mod[:-3]

   325     else:

   326         path = []

   327     dict = readmodule_ex(mod, path)

   328     objs = dict.values()

   329     objs.sort(lambda a, b: cmp(getattr(a, 'lineno', 0),

   330                                getattr(b, 'lineno', 0)))

   331     for obj in objs:

   332         if isinstance(obj, Class):

   333             print "class", obj.name, obj.super, obj.lineno

   334             methods = sorted(obj.methods.iteritems(), key=itemgetter(1))

   335             for name, lineno in methods:

   336                 if name != "__path__":

   337                     print "  def", name, lineno

   338         elif isinstance(obj, Function):

   339             print "def", obj.name, obj.lineno

   340 

   341 if __name__ == "__main__":

   342     _main()