1 """distutils.filelist

     2 

     3 Provides the FileList class, used for poking about the filesystem

     4 and building lists of files.

     5 """

     6 

     7 # This module should be kept compatible with Python 2.1.

     8 

     9 __revision__ = "$Id: filelist.py 61000 2008-02-23 17:40:11Z christian.heimes $"

    10 

    11 import os, string, re

    12 import fnmatch

    13 from types import *

    14 from distutils.util import convert_path

    15 from distutils.errors import DistutilsTemplateError, DistutilsInternalError

    16 from distutils import log

    17 

    18 class FileList:

    19 

    20     """A list of files built by on exploring the filesystem and filtered by

    21     applying various patterns to what we find there.

    22 

    23     Instance attributes:

    24       dir

    25         directory from which files will be taken -- only used if

    26         'allfiles' not supplied to constructor

    27       files

    28         list of filenames currently being built/filtered/manipulated

    29       allfiles

    30         complete list of files under consideration (ie. without any

    31         filtering applied)

    32     """

    33 

    34     def __init__(self,

    35                  warn=None,

    36                  debug_print=None):

    37         # ignore argument to FileList, but keep them for backwards

    38         # compatibility

    39 

    40         self.allfiles = None

    41         self.files = []

    42 

    43     def set_allfiles (self, allfiles):

    44         self.allfiles = allfiles

    45 

    46     def findall (self, dir=os.curdir):

    47         self.allfiles = findall(dir)

    48 

    49     def debug_print (self, msg):

    50         """Print 'msg' to stdout if the global DEBUG (taken from the

    51         DISTUTILS_DEBUG environment variable) flag is true.

    52         """

    53         from distutils.debug import DEBUG

    54         if DEBUG:

    55             print msg

    56 

    57     # -- List-like methods ---------------------------------------------

    58 

    59     def append (self, item):

    60         self.files.append(item)

    61 

    62     def extend (self, items):

    63         self.files.extend(items)

    64 

    65     def sort (self):

    66         # Not a strict lexical sort!

    67         sortable_files = map(os.path.split, self.files)

    68         sortable_files.sort()

    69         self.files = []

    70         for sort_tuple in sortable_files:

    71             self.files.append(apply(os.path.join, sort_tuple))

    72 

    73 

    74     # -- Other miscellaneous utility methods ---------------------------

    75 

    76     def remove_duplicates (self):

    77         # Assumes list has been sorted!

    78         for i in range(len(self.files) - 1, 0, -1):

    79             if self.files[i] == self.files[i - 1]:

    80                 del self.files[i]

    81 

    82 

    83     # -- "File template" methods ---------------------------------------

    84 

    85     def _parse_template_line (self, line):

    86         words = string.split(line)

    87         action = words[0]

    88 

    89         patterns = dir = dir_pattern = None

    90 

    91         if action in ('include', 'exclude',

    92                       'global-include', 'global-exclude'):

    93             if len(words) < 2:

    94                 raise DistutilsTemplateError, \

    95                       "'%s' expects <pattern1> <pattern2> ..." % action

    96 

    97             patterns = map(convert_path, words[1:])

    98 

    99         elif action in ('recursive-include', 'recursive-exclude'):

   100             if len(words) < 3:

   101                 raise DistutilsTemplateError, \

   102                       "'%s' expects <dir> <pattern1> <pattern2> ..." % action

   103 

   104             dir = convert_path(words[1])

   105             patterns = map(convert_path, words[2:])

   106 

   107         elif action in ('graft', 'prune'):

   108             if len(words) != 2:

   109                 raise DistutilsTemplateError, \

   110                      "'%s' expects a single <dir_pattern>" % action

   111 

   112             dir_pattern = convert_path(words[1])

   113 

   114         else:

   115             raise DistutilsTemplateError, "unknown action '%s'" % action

   116 

   117         return (action, patterns, dir, dir_pattern)

   118 

   119     # _parse_template_line ()

   120 

   121 

   122     def process_template_line (self, line):

   123 

   124         # Parse the line: split it up, make sure the right number of words

   125         # is there, and return the relevant words.  'action' is always

   126         # defined: it's the first word of the line.  Which of the other

   127         # three are defined depends on the action; it'll be either

   128         # patterns, (dir and patterns), or (dir_pattern).

   129         (action, patterns, dir, dir_pattern) = self._parse_template_line(line)

   130 

   131         # OK, now we know that the action is valid and we have the

   132         # right number of words on the line for that action -- so we

   133         # can proceed with minimal error-checking.

   134         if action == 'include':

   135             self.debug_print("include " + string.join(patterns))

   136             for pattern in patterns:

   137                 if not self.include_pattern(pattern, anchor=1):

   138                     log.warn("warning: no files found matching '%s'",

   139                              pattern)

   140 

   141         elif action == 'exclude':

   142             self.debug_print("exclude " + string.join(patterns))

   143             for pattern in patterns:

   144                 if not self.exclude_pattern(pattern, anchor=1):

   145                     log.warn(("warning: no previously-included files "

   146                               "found matching '%s'"), pattern)

   147 

   148         elif action == 'global-include':

   149             self.debug_print("global-include " + string.join(patterns))

   150             for pattern in patterns:

   151                 if not self.include_pattern(pattern, anchor=0):

   152                     log.warn(("warning: no files found matching '%s' " +

   153                               "anywhere in distribution"), pattern)

   154 

   155         elif action == 'global-exclude':

   156             self.debug_print("global-exclude " + string.join(patterns))

   157             for pattern in patterns:

   158                 if not self.exclude_pattern(pattern, anchor=0):

   159                     log.warn(("warning: no previously-included files matching "

   160                               "'%s' found anywhere in distribution"),

   161                              pattern)

   162 

   163         elif action == 'recursive-include':

   164             self.debug_print("recursive-include %s %s" %

   165                              (dir, string.join(patterns)))

   166             for pattern in patterns:

   167                 if not self.include_pattern(pattern, prefix=dir):

   168                     log.warn(("warning: no files found matching '%s' " +

   169                                 "under directory '%s'"),

   170                              pattern, dir)

   171 

   172         elif action == 'recursive-exclude':

   173             self.debug_print("recursive-exclude %s %s" %

   174                              (dir, string.join(patterns)))

   175             for pattern in patterns:

   176                 if not self.exclude_pattern(pattern, prefix=dir):

   177                     log.warn(("warning: no previously-included files matching "

   178                               "'%s' found under directory '%s'"),

   179                              pattern, dir)

   180 

   181         elif action == 'graft':

   182             self.debug_print("graft " + dir_pattern)

   183             if not self.include_pattern(None, prefix=dir_pattern):

   184                 log.warn("warning: no directories found matching '%s'",

   185                          dir_pattern)

   186 

   187         elif action == 'prune':

   188             self.debug_print("prune " + dir_pattern)

   189             if not self.exclude_pattern(None, prefix=dir_pattern):

   190                 log.warn(("no previously-included directories found " +

   191                           "matching '%s'"), dir_pattern)

   192         else:

   193             raise DistutilsInternalError, \

   194                   "this cannot happen: invalid action '%s'" % action

   195 

   196     # process_template_line ()

   197 

   198 

   199     # -- Filtering/selection methods -----------------------------------

   200 

   201     def include_pattern (self, pattern,

   202                          anchor=1, prefix=None, is_regex=0):

   203         """Select strings (presumably filenames) from 'self.files' that

   204         match 'pattern', a Unix-style wildcard (glob) pattern.  Patterns

   205         are not quite the same as implemented by the 'fnmatch' module: '*'

   206         and '?'  match non-special characters, where "special" is platform-

   207         dependent: slash on Unix; colon, slash, and backslash on

   208         DOS/Windows; and colon on Mac OS.

   209 

   210         If 'anchor' is true (the default), then the pattern match is more

   211         stringent: "*.py" will match "foo.py" but not "foo/bar.py".  If

   212         'anchor' is false, both of these will match.

   213 

   214         If 'prefix' is supplied, then only filenames starting with 'prefix'

   215         (itself a pattern) and ending with 'pattern', with anything in between

   216         them, will match.  'anchor' is ignored in this case.

   217 

   218         If 'is_regex' is true, 'anchor' and 'prefix' are ignored, and

   219         'pattern' is assumed to be either a string containing a regex or a

   220         regex object -- no translation is done, the regex is just compiled

   221         and used as-is.

   222 

   223         Selected strings will be added to self.files.

   224 

   225         Return 1 if files are found.

   226         """

   227         files_found = 0

   228         pattern_re = translate_pattern(pattern, anchor, prefix, is_regex)

   229         self.debug_print("include_pattern: applying regex r'%s'" %

   230                          pattern_re.pattern)

   231 

   232         # delayed loading of allfiles list

   233         if self.allfiles is None:

   234             self.findall()

   235 

   236         for name in self.allfiles:

   237             if pattern_re.search(name):

   238                 self.debug_print(" adding " + name)

   239                 self.files.append(name)

   240                 files_found = 1

   241 

   242         return files_found

   243 

   244     # include_pattern ()

   245 

   246 

   247     def exclude_pattern (self, pattern,

   248                          anchor=1, prefix=None, is_regex=0):

   249         """Remove strings (presumably filenames) from 'files' that match

   250         'pattern'.  Other parameters are the same as for

   251         'include_pattern()', above.

   252         The list 'self.files' is modified in place.

   253         Return 1 if files are found.

   254         """

   255         files_found = 0

   256         pattern_re = translate_pattern(pattern, anchor, prefix, is_regex)

   257         self.debug_print("exclude_pattern: applying regex r'%s'" %

   258                          pattern_re.pattern)

   259         for i in range(len(self.files)-1, -1, -1):

   260             if pattern_re.search(self.files[i]):

   261                 self.debug_print(" removing " + self.files[i])

   262                 del self.files[i]

   263                 files_found = 1

   264 

   265         return files_found

   266 

   267     # exclude_pattern ()

   268 

   269 # class FileList

   270 

   271 

   272 # ----------------------------------------------------------------------

   273 # Utility functions

   274 

   275 def findall (dir = os.curdir):

   276     """Find all files under 'dir' and return the list of full filenames

   277     (relative to 'dir').

   278     """

   279     from stat import ST_MODE, S_ISREG, S_ISDIR, S_ISLNK

   280 

   281     list = []

   282     stack = [dir]

   283     pop = stack.pop

   284     push = stack.append

   285 

   286     while stack:

   287         dir = pop()

   288         names = os.listdir(dir)

   289 

   290         for name in names:

   291             if dir != os.curdir:        # avoid the dreaded "./" syndrome

   292                 fullname = os.path.join(dir, name)

   293             else:

   294                 fullname = name

   295 

   296             # Avoid excess stat calls -- just one will do, thank you!

   297             stat = os.stat(fullname)

   298             mode = stat[ST_MODE]

   299             if S_ISREG(mode):

   300                 list.append(fullname)

   301             elif S_ISDIR(mode) and not S_ISLNK(mode):

   302                 push(fullname)

   303 

   304     return list

   305 

   306 

   307 def glob_to_re (pattern):

   308     """Translate a shell-like glob pattern to a regular expression; return

   309     a string containing the regex.  Differs from 'fnmatch.translate()' in

   310     that '*' does not match "special characters" (which are

   311     platform-specific).

   312     """

   313     pattern_re = fnmatch.translate(pattern)

   314 

   315     # '?' and '*' in the glob pattern become '.' and '.*' in the RE, which

   316     # IMHO is wrong -- '?' and '*' aren't supposed to match slash in Unix,

   317     # and by extension they shouldn't match such "special characters" under

   318     # any OS.  So change all non-escaped dots in the RE to match any

   319     # character except the special characters.

   320     # XXX currently the "special characters" are just slash -- i.e. this is

   321     # Unix-only.

   322     pattern_re = re.sub(r'(^|[^\\])\.', r'\1[^/]', pattern_re)

   323     return pattern_re

   324 

   325 # glob_to_re ()

   326 

   327 

   328 def translate_pattern (pattern, anchor=1, prefix=None, is_regex=0):

   329     """Translate a shell-like wildcard pattern to a compiled regular

   330     expression.  Return the compiled regex.  If 'is_regex' true,

   331     then 'pattern' is directly compiled to a regex (if it's a string)

   332     or just returned as-is (assumes it's a regex object).

   333     """

   334     if is_regex:

   335         if type(pattern) is StringType:

   336             return re.compile(pattern)

   337         else:

   338             return pattern

   339 

   340     if pattern:

   341         pattern_re = glob_to_re(pattern)

   342     else:

   343         pattern_re = ''

   344 

   345     if prefix is not None:

   346         prefix_re = (glob_to_re(prefix))[0:-1] # ditch trailing $

   347         pattern_re = "^" + os.path.join(prefix_re, ".*" + pattern_re)

   348     else:                               # no prefix -- respect anchor flag

   349         if anchor:

   350             pattern_re = "^" + pattern_re

   351 

   352     return re.compile(pattern_re)

   353 

   354 # translate_pattern ()