1 """Text wrapping and filling.

     2 """

     3 

     4 # Copyright (C) 1999-2001 Gregory P. Ward.

     5 # Copyright (C) 2002, 2003 Python Software Foundation.

     6 # Written by Greg Ward <gward@python.net>

     7 

     8 __revision__ = "$Id: textwrap.py 46863 2006-06-11 19:42:51Z tim.peters $"

     9 

    10 import string, re

    11 

    12 # Do the right thing with boolean values for all known Python versions

    13 # (so this module can be copied to projects that don't depend on Python

    14 # 2.3, e.g. Optik and Docutils).

    15 try:

    16     True, False

    17 except NameError:

    18     (True, False) = (1, 0)

    19 

    20 __all__ = ['TextWrapper', 'wrap', 'fill']

    21 

    22 # Hardcode the recognized whitespace characters to the US-ASCII

    23 # whitespace characters.  The main reason for doing this is that in

    24 # ISO-8859-1, 0xa0 is non-breaking whitespace, so in certain locales

    25 # that character winds up in string.whitespace.  Respecting

    26 # string.whitespace in those cases would 1) make textwrap treat 0xa0 the

    27 # same as any other whitespace char, which is clearly wrong (it's a

    28 # *non-breaking* space), 2) possibly cause problems with Unicode,

    29 # since 0xa0 is not in range(128).

    30 _whitespace = '\t\n\x0b\x0c\r '

    31 

    32 class TextWrapper:

    33     """

    34     Object for wrapping/filling text.  The public interface consists of

    35     the wrap() and fill() methods; the other methods are just there for

    36     subclasses to override in order to tweak the default behaviour.

    37     If you want to completely replace the main wrapping algorithm,

    38     you'll probably have to override _wrap_chunks().

    39 

    40     Several instance attributes control various aspects of wrapping:

    41       width (default: 70)

    42         the maximum width of wrapped lines (unless break_long_words

    43         is false)

    44       initial_indent (default: "")

    45         string that will be prepended to the first line of wrapped

    46         output.  Counts towards the line's width.

    47       subsequent_indent (default: "")

    48         string that will be prepended to all lines save the first

    49         of wrapped output; also counts towards each line's width.

    50       expand_tabs (default: true)

    51         Expand tabs in input text to spaces before further processing.

    52         Each tab will become 1 .. 8 spaces, depending on its position in

    53         its line.  If false, each tab is treated as a single character.

    54       replace_whitespace (default: true)

    55         Replace all whitespace characters in the input text by spaces

    56         after tab expansion.  Note that if expand_tabs is false and

    57         replace_whitespace is true, every tab will be converted to a

    58         single space!

    59       fix_sentence_endings (default: false)

    60         Ensure that sentence-ending punctuation is always followed

    61         by two spaces.  Off by default because the algorithm is

    62         (unavoidably) imperfect.

    63       break_long_words (default: true)

    64         Break words longer than 'width'.  If false, those words will not

    65         be broken, and some lines might be longer than 'width'.

    66     """

    67 

    68     whitespace_trans = string.maketrans(_whitespace, ' ' * len(_whitespace))

    69 

    70     unicode_whitespace_trans = {}

    71     uspace = ord(u' ')

    72     for x in map(ord, _whitespace):

    73         unicode_whitespace_trans[x] = uspace

    74 

    75     # This funky little regex is just the trick for splitting

    76     # text up into word-wrappable chunks.  E.g.

    77     #   "Hello there -- you goof-ball, use the -b option!"

    78     # splits into

    79     #   Hello/ /there/ /--/ /you/ /goof-/ball,/ /use/ /the/ /-b/ /option!

    80     # (after stripping out empty strings).

    81     wordsep_re = re.compile(

    82         r'(\s+|'                                  # any whitespace

    83         r'[^\s\w]*\w+[a-zA-Z]-(?=\w+[a-zA-Z])|'   # hyphenated words

    84         r'(?<=[\w\!\"\'\&\.\,\?])-{2,}(?=\w))')   # em-dash

    85 

    86     # XXX this is not locale- or charset-aware -- string.lowercase

    87     # is US-ASCII only (and therefore English-only)

    88     sentence_end_re = re.compile(r'[%s]'              # lowercase letter

    89                                  r'[\.\!\?]'          # sentence-ending punct.

    90                                  r'[\"\']?'           # optional end-of-quote

    91                                  % string.lowercase)

    92 

    93 

    94     def __init__(self,

    95                  width=70,

    96                  initial_indent="",

    97                  subsequent_indent="",

    98                  expand_tabs=True,

    99                  replace_whitespace=True,

   100                  fix_sentence_endings=False,

   101                  break_long_words=True):

   102         self.width = width

   103         self.initial_indent = initial_indent

   104         self.subsequent_indent = subsequent_indent

   105         self.expand_tabs = expand_tabs

   106         self.replace_whitespace = replace_whitespace

   107         self.fix_sentence_endings = fix_sentence_endings

   108         self.break_long_words = break_long_words

   109 

   110 

   111     # -- Private methods -----------------------------------------------

   112     # (possibly useful for subclasses to override)

   113 

   114     def _munge_whitespace(self, text):

   115         """_munge_whitespace(text : string) -> string

   116 

   117         Munge whitespace in text: expand tabs and convert all other

   118         whitespace characters to spaces.  Eg. " foo\tbar\n\nbaz"

   119         becomes " foo    bar  baz".

   120         """

   121         if self.expand_tabs:

   122             text = text.expandtabs()

   123         if self.replace_whitespace:

   124             if isinstance(text, str):

   125                 text = text.translate(self.whitespace_trans)

   126             elif isinstance(text, unicode):

   127                 text = text.translate(self.unicode_whitespace_trans)

   128         return text

   129 

   130 

   131     def _split(self, text):

   132         """_split(text : string) -> [string]

   133 

   134         Split the text to wrap into indivisible chunks.  Chunks are

   135         not quite the same as words; see wrap_chunks() for full

   136         details.  As an example, the text

   137           Look, goof-ball -- use the -b option!

   138         breaks into the following chunks:

   139           'Look,', ' ', 'goof-', 'ball', ' ', '--', ' ',

   140           'use', ' ', 'the', ' ', '-b', ' ', 'option!'

   141         """

   142         chunks = self.wordsep_re.split(text)

   143         chunks = filter(None, chunks)

   144         return chunks

   145 

   146     def _fix_sentence_endings(self, chunks):

   147         """_fix_sentence_endings(chunks : [string])

   148 

   149         Correct for sentence endings buried in 'chunks'.  Eg. when the

   150         original text contains "... foo.\nBar ...", munge_whitespace()

   151         and split() will convert that to [..., "foo.", " ", "Bar", ...]

   152         which has one too few spaces; this method simply changes the one

   153         space to two.

   154         """

   155         i = 0

   156         pat = self.sentence_end_re

   157         while i < len(chunks)-1:

   158             if chunks[i+1] == " " and pat.search(chunks[i]):

   159                 chunks[i+1] = "  "

   160                 i += 2

   161             else:

   162                 i += 1

   163 

   164     def _handle_long_word(self, reversed_chunks, cur_line, cur_len, width):

   165         """_handle_long_word(chunks : [string],

   166                              cur_line : [string],

   167                              cur_len : int, width : int)

   168 

   169         Handle a chunk of text (most likely a word, not whitespace) that

   170         is too long to fit in any line.

   171         """

   172         space_left = max(width - cur_len, 1)

   173 

   174         # If we're allowed to break long words, then do so: put as much

   175         # of the next chunk onto the current line as will fit.

   176         if self.break_long_words:

   177             cur_line.append(reversed_chunks[-1][:space_left])

   178             reversed_chunks[-1] = reversed_chunks[-1][space_left:]

   179 

   180         # Otherwise, we have to preserve the long word intact.  Only add

   181         # it to the current line if there's nothing already there --

   182         # that minimizes how much we violate the width constraint.

   183         elif not cur_line:

   184             cur_line.append(reversed_chunks.pop())

   185 

   186         # If we're not allowed to break long words, and there's already

   187         # text on the current line, do nothing.  Next time through the

   188         # main loop of _wrap_chunks(), we'll wind up here again, but

   189         # cur_len will be zero, so the next line will be entirely

   190         # devoted to the long word that we can't handle right now.

   191 

   192     def _wrap_chunks(self, chunks):

   193         """_wrap_chunks(chunks : [string]) -> [string]

   194 

   195         Wrap a sequence of text chunks and return a list of lines of

   196         length 'self.width' or less.  (If 'break_long_words' is false,

   197         some lines may be longer than this.)  Chunks correspond roughly

   198         to words and the whitespace between them: each chunk is

   199         indivisible (modulo 'break_long_words'), but a line break can

   200         come between any two chunks.  Chunks should not have internal

   201         whitespace; ie. a chunk is either all whitespace or a "word".

   202         Whitespace chunks will be removed from the beginning and end of

   203         lines, but apart from that whitespace is preserved.

   204         """

   205         lines = []

   206         if self.width <= 0:

   207             raise ValueError("invalid width %r (must be > 0)" % self.width)

   208 

   209         # Arrange in reverse order so items can be efficiently popped

   210         # from a stack of chucks.

   211         chunks.reverse()

   212 

   213         while chunks:

   214 

   215             # Start the list of chunks that will make up the current line.

   216             # cur_len is just the length of all the chunks in cur_line.

   217             cur_line = []

   218             cur_len = 0

   219 

   220             # Figure out which static string will prefix this line.

   221             if lines:

   222                 indent = self.subsequent_indent

   223             else:

   224                 indent = self.initial_indent

   225 

   226             # Maximum width for this line.

   227             width = self.width - len(indent)

   228 

   229             # First chunk on line is whitespace -- drop it, unless this

   230             # is the very beginning of the text (ie. no lines started yet).

   231             if chunks[-1].strip() == '' and lines:

   232                 del chunks[-1]

   233 

   234             while chunks:

   235                 l = len(chunks[-1])

   236 

   237                 # Can at least squeeze this chunk onto the current line.

   238                 if cur_len + l <= width:

   239                     cur_line.append(chunks.pop())

   240                     cur_len += l

   241 

   242                 # Nope, this line is full.

   243                 else:

   244                     break

   245 

   246             # The current line is full, and the next chunk is too big to

   247             # fit on *any* line (not just this one).

   248             if chunks and len(chunks[-1]) > width:

   249                 self._handle_long_word(chunks, cur_line, cur_len, width)

   250 

   251             # If the last chunk on this line is all whitespace, drop it.

   252             if cur_line and cur_line[-1].strip() == '':

   253                 del cur_line[-1]

   254 

   255             # Convert current line back to a string and store it in list

   256             # of all lines (return value).

   257             if cur_line:

   258                 lines.append(indent + ''.join(cur_line))

   259 

   260         return lines

   261 

   262 

   263     # -- Public interface ----------------------------------------------

   264 

   265     def wrap(self, text):

   266         """wrap(text : string) -> [string]

   267 

   268         Reformat the single paragraph in 'text' so it fits in lines of

   269         no more than 'self.width' columns, and return a list of wrapped

   270         lines.  Tabs in 'text' are expanded with string.expandtabs(),

   271         and all other whitespace characters (including newline) are

   272         converted to space.

   273         """

   274         text = self._munge_whitespace(text)

   275         chunks = self._split(text)

   276         if self.fix_sentence_endings:

   277             self._fix_sentence_endings(chunks)

   278         return self._wrap_chunks(chunks)

   279 

   280     def fill(self, text):

   281         """fill(text : string) -> string

   282 

   283         Reformat the single paragraph in 'text' to fit in lines of no

   284         more than 'self.width' columns, and return a new string

   285         containing the entire wrapped paragraph.

   286         """

   287         return "\n".join(self.wrap(text))

   288 

   289 

   290 # -- Convenience interface ---------------------------------------------

   291 

   292 def wrap(text, width=70, **kwargs):

   293     """Wrap a single paragraph of text, returning a list of wrapped lines.

   294 

   295     Reformat the single paragraph in 'text' so it fits in lines of no

   296     more than 'width' columns, and return a list of wrapped lines.  By

   297     default, tabs in 'text' are expanded with string.expandtabs(), and

   298     all other whitespace characters (including newline) are converted to

   299     space.  See TextWrapper class for available keyword args to customize

   300     wrapping behaviour.

   301     """

   302     w = TextWrapper(width=width, **kwargs)

   303     return w.wrap(text)

   304 

   305 def fill(text, width=70, **kwargs):

   306     """Fill a single paragraph of text, returning a new string.

   307 

   308     Reformat the single paragraph in 'text' to fit in lines of no more

   309     than 'width' columns, and return a new string containing the entire

   310     wrapped paragraph.  As with wrap(), tabs are expanded and other

   311     whitespace characters converted to space.  See TextWrapper class for

   312     available keyword args to customize wrapping behaviour.

   313     """

   314     w = TextWrapper(width=width, **kwargs)

   315     return w.fill(text)

   316 

   317 

   318 # -- Loosely related functionality -------------------------------------

   319 

   320 _whitespace_only_re = re.compile('^[ \t]+$', re.MULTILINE)

   321 _leading_whitespace_re = re.compile('(^[ \t]*)(?:[^ \t\n])', re.MULTILINE)

   322 

   323 def dedent(text):

   324     """Remove any common leading whitespace from every line in `text`.

   325 

   326     This can be used to make triple-quoted strings line up with the left

   327     edge of the display, while still presenting them in the source code

   328     in indented form.

   329 

   330     Note that tabs and spaces are both treated as whitespace, but they

   331     are not equal: the lines "  hello" and "\thello" are

   332     considered to have no common leading whitespace.  (This behaviour is

   333     new in Python 2.5; older versions of this module incorrectly

   334     expanded tabs before searching for common leading whitespace.)

   335     """

   336     # Look for the longest leading string of spaces and tabs common to

   337     # all lines.

   338     margin = None

   339     text = _whitespace_only_re.sub('', text)

   340     indents = _leading_whitespace_re.findall(text)

   341     for indent in indents:

   342         if margin is None:

   343             margin = indent

   344 

   345         # Current line more deeply indented than previous winner:

   346         # no change (previous winner is still on top).

   347         elif indent.startswith(margin):

   348             pass

   349 

   350         # Current line consistent with and no deeper than previous winner:

   351         # it's the new winner.

   352         elif margin.startswith(indent):

   353             margin = indent

   354 

   355         # Current line and previous winner have no common whitespace:

   356         # there is no margin.

   357         else:

   358             margin = ""

   359             break

   360 

   361     # sanity check (testing/debugging only)

   362     if 0 and margin:

   363         for line in text.split("\n"):

   364             assert not line or line.startswith(margin), \

   365                    "line = %r, margin = %r" % (line, margin)

   366 

   367     if margin:

   368         text = re.sub(r'(?m)^' + margin, '', text)

   369     return text

   370 

   371 if __name__ == "__main__":

   372     #print dedent("\tfoo\n\tbar")

   373     #print dedent("  \thello there\n  \t  how are you?")

   374     print dedent("Hello there.\n  This is indented.")