1 """HTML 2.0 parser.

     2 

     3 See the HTML 2.0 specification:

     4 http://www.w3.org/hypertext/WWW/MarkUp/html-spec/html-spec_toc.html

     5 """

     6 

     7 import sgmllib

     8 

     9 from formatter import AS_IS

    10 

    11 __all__ = ["HTMLParser", "HTMLParseError"]

    12 

    13 

    14 class HTMLParseError(sgmllib.SGMLParseError):

    15     """Error raised when an HTML document can't be parsed."""

    16 

    17 

    18 class HTMLParser(sgmllib.SGMLParser):

    19     """This is the basic HTML parser class.

    20 

    21     It supports all entity names required by the XHTML 1.0 Recommendation.

    22     It also defines handlers for all HTML 2.0 and many HTML 3.0 and 3.2

    23     elements.

    24 

    25     """

    26 

    27     from htmlentitydefs import entitydefs

    28 

    29     def __init__(self, formatter, verbose=0):

    30         """Creates an instance of the HTMLParser class.

    31 

    32         The formatter parameter is the formatter instance associated with

    33         the parser.

    34 

    35         """

    36         sgmllib.SGMLParser.__init__(self, verbose)

    37         self.formatter = formatter

    38 

    39     def error(self, message):

    40         raise HTMLParseError(message)

    41 

    42     def reset(self):

    43         sgmllib.SGMLParser.reset(self)

    44         self.savedata = None

    45         self.isindex = 0

    46         self.title = None

    47         self.base = None

    48         self.anchor = None

    49         self.anchorlist = []

    50         self.nofill = 0

    51         self.list_stack = []

    52 

    53     # ------ Methods used internally; some may be overridden

    54 

    55     # --- Formatter interface, taking care of 'savedata' mode;

    56     # shouldn't need to be overridden

    57 

    58     def handle_data(self, data):

    59         if self.savedata is not None:

    60             self.savedata = self.savedata + data

    61         else:

    62             if self.nofill:

    63                 self.formatter.add_literal_data(data)

    64             else:

    65                 self.formatter.add_flowing_data(data)

    66 

    67     # --- Hooks to save data; shouldn't need to be overridden

    68 

    69     def save_bgn(self):

    70         """Begins saving character data in a buffer instead of sending it

    71         to the formatter object.

    72 

    73         Retrieve the stored data via the save_end() method.  Use of the

    74         save_bgn() / save_end() pair may not be nested.

    75 

    76         """

    77         self.savedata = ''

    78 

    79     def save_end(self):

    80         """Ends buffering character data and returns all data saved since

    81         the preceding call to the save_bgn() method.

    82 

    83         If the nofill flag is false, whitespace is collapsed to single

    84         spaces.  A call to this method without a preceding call to the

    85         save_bgn() method will raise a TypeError exception.

    86 

    87         """

    88         data = self.savedata

    89         self.savedata = None

    90         if not self.nofill:

    91             data = ' '.join(data.split())

    92         return data

    93 

    94     # --- Hooks for anchors; should probably be overridden

    95 

    96     def anchor_bgn(self, href, name, type):

    97         """This method is called at the start of an anchor region.

    98 

    99         The arguments correspond to the attributes of the <A> tag with

   100         the same names.  The default implementation maintains a list of

   101         hyperlinks (defined by the HREF attribute for <A> tags) within

   102         the document.  The list of hyperlinks is available as the data

   103         attribute anchorlist.

   104 

   105         """

   106         self.anchor = href

   107         if self.anchor:

   108             self.anchorlist.append(href)

   109 

   110     def anchor_end(self):

   111         """This method is called at the end of an anchor region.

   112 

   113         The default implementation adds a textual footnote marker using an

   114         index into the list of hyperlinks created by the anchor_bgn()method.

   115 

   116         """

   117         if self.anchor:

   118             self.handle_data("[%d]" % len(self.anchorlist))

   119             self.anchor = None

   120 

   121     # --- Hook for images; should probably be overridden

   122 

   123     def handle_image(self, src, alt, *args):

   124         """This method is called to handle images.

   125 

   126         The default implementation simply passes the alt value to the

   127         handle_data() method.

   128 

   129         """

   130         self.handle_data(alt)

   131 

   132     # --------- Top level elememts

   133 

   134     def start_html(self, attrs): pass

   135     def end_html(self): pass

   136 

   137     def start_head(self, attrs): pass

   138     def end_head(self): pass

   139 

   140     def start_body(self, attrs): pass

   141     def end_body(self): pass

   142 

   143     # ------ Head elements

   144 

   145     def start_title(self, attrs):

   146         self.save_bgn()

   147 

   148     def end_title(self):

   149         self.title = self.save_end()

   150 

   151     def do_base(self, attrs):

   152         for a, v in attrs:

   153             if a == 'href':

   154                 self.base = v

   155 

   156     def do_isindex(self, attrs):

   157         self.isindex = 1

   158 

   159     def do_link(self, attrs):

   160         pass

   161 

   162     def do_meta(self, attrs):

   163         pass

   164 

   165     def do_nextid(self, attrs): # Deprecated

   166         pass

   167 

   168     # ------ Body elements

   169 

   170     # --- Headings

   171 

   172     def start_h1(self, attrs):

   173         self.formatter.end_paragraph(1)

   174         self.formatter.push_font(('h1', 0, 1, 0))

   175 

   176     def end_h1(self):

   177         self.formatter.end_paragraph(1)

   178         self.formatter.pop_font()

   179 

   180     def start_h2(self, attrs):

   181         self.formatter.end_paragraph(1)

   182         self.formatter.push_font(('h2', 0, 1, 0))

   183 

   184     def end_h2(self):

   185         self.formatter.end_paragraph(1)

   186         self.formatter.pop_font()

   187 

   188     def start_h3(self, attrs):

   189         self.formatter.end_paragraph(1)

   190         self.formatter.push_font(('h3', 0, 1, 0))

   191 

   192     def end_h3(self):

   193         self.formatter.end_paragraph(1)

   194         self.formatter.pop_font()

   195 

   196     def start_h4(self, attrs):

   197         self.formatter.end_paragraph(1)

   198         self.formatter.push_font(('h4', 0, 1, 0))

   199 

   200     def end_h4(self):

   201         self.formatter.end_paragraph(1)

   202         self.formatter.pop_font()

   203 

   204     def start_h5(self, attrs):

   205         self.formatter.end_paragraph(1)

   206         self.formatter.push_font(('h5', 0, 1, 0))

   207 

   208     def end_h5(self):

   209         self.formatter.end_paragraph(1)

   210         self.formatter.pop_font()

   211 

   212     def start_h6(self, attrs):

   213         self.formatter.end_paragraph(1)

   214         self.formatter.push_font(('h6', 0, 1, 0))

   215 

   216     def end_h6(self):

   217         self.formatter.end_paragraph(1)

   218         self.formatter.pop_font()

   219 

   220     # --- Block Structuring Elements

   221 

   222     def do_p(self, attrs):

   223         self.formatter.end_paragraph(1)

   224 

   225     def start_pre(self, attrs):

   226         self.formatter.end_paragraph(1)

   227         self.formatter.push_font((AS_IS, AS_IS, AS_IS, 1))

   228         self.nofill = self.nofill + 1

   229 

   230     def end_pre(self):

   231         self.formatter.end_paragraph(1)

   232         self.formatter.pop_font()

   233         self.nofill = max(0, self.nofill - 1)

   234 

   235     def start_xmp(self, attrs):

   236         self.start_pre(attrs)

   237         self.setliteral('xmp') # Tell SGML parser

   238 

   239     def end_xmp(self):

   240         self.end_pre()

   241 

   242     def start_listing(self, attrs):

   243         self.start_pre(attrs)

   244         self.setliteral('listing') # Tell SGML parser

   245 

   246     def end_listing(self):

   247         self.end_pre()

   248 

   249     def start_address(self, attrs):

   250         self.formatter.end_paragraph(0)

   251         self.formatter.push_font((AS_IS, 1, AS_IS, AS_IS))

   252 

   253     def end_address(self):

   254         self.formatter.end_paragraph(0)

   255         self.formatter.pop_font()

   256 

   257     def start_blockquote(self, attrs):

   258         self.formatter.end_paragraph(1)

   259         self.formatter.push_margin('blockquote')

   260 

   261     def end_blockquote(self):

   262         self.formatter.end_paragraph(1)

   263         self.formatter.pop_margin()

   264 

   265     # --- List Elements

   266 

   267     def start_ul(self, attrs):

   268         self.formatter.end_paragraph(not self.list_stack)

   269         self.formatter.push_margin('ul')

   270         self.list_stack.append(['ul', '*', 0])

   271 

   272     def end_ul(self):

   273         if self.list_stack: del self.list_stack[-1]

   274         self.formatter.end_paragraph(not self.list_stack)

   275         self.formatter.pop_margin()

   276 

   277     def do_li(self, attrs):

   278         self.formatter.end_paragraph(0)

   279         if self.list_stack:

   280             [dummy, label, counter] = top = self.list_stack[-1]

   281             top[2] = counter = counter+1

   282         else:

   283             label, counter = '*', 0

   284         self.formatter.add_label_data(label, counter)

   285 

   286     def start_ol(self, attrs):

   287         self.formatter.end_paragraph(not self.list_stack)

   288         self.formatter.push_margin('ol')

   289         label = '1.'

   290         for a, v in attrs:

   291             if a == 'type':

   292                 if len(v) == 1: v = v + '.'

   293                 label = v

   294         self.list_stack.append(['ol', label, 0])

   295 

   296     def end_ol(self):

   297         if self.list_stack: del self.list_stack[-1]

   298         self.formatter.end_paragraph(not self.list_stack)

   299         self.formatter.pop_margin()

   300 

   301     def start_menu(self, attrs):

   302         self.start_ul(attrs)

   303 

   304     def end_menu(self):

   305         self.end_ul()

   306 

   307     def start_dir(self, attrs):

   308         self.start_ul(attrs)

   309 

   310     def end_dir(self):

   311         self.end_ul()

   312 

   313     def start_dl(self, attrs):

   314         self.formatter.end_paragraph(1)

   315         self.list_stack.append(['dl', '', 0])

   316 

   317     def end_dl(self):

   318         self.ddpop(1)

   319         if self.list_stack: del self.list_stack[-1]

   320 

   321     def do_dt(self, attrs):

   322         self.ddpop()

   323 

   324     def do_dd(self, attrs):

   325         self.ddpop()

   326         self.formatter.push_margin('dd')

   327         self.list_stack.append(['dd', '', 0])

   328 

   329     def ddpop(self, bl=0):

   330         self.formatter.end_paragraph(bl)

   331         if self.list_stack:

   332             if self.list_stack[-1][0] == 'dd':

   333                 del self.list_stack[-1]

   334                 self.formatter.pop_margin()

   335 

   336     # --- Phrase Markup

   337 

   338     # Idiomatic Elements

   339 

   340     def start_cite(self, attrs): self.start_i(attrs)

   341     def end_cite(self): self.end_i()

   342 

   343     def start_code(self, attrs): self.start_tt(attrs)

   344     def end_code(self): self.end_tt()

   345 

   346     def start_em(self, attrs): self.start_i(attrs)

   347     def end_em(self): self.end_i()

   348 

   349     def start_kbd(self, attrs): self.start_tt(attrs)

   350     def end_kbd(self): self.end_tt()

   351 

   352     def start_samp(self, attrs): self.start_tt(attrs)

   353     def end_samp(self): self.end_tt()

   354 

   355     def start_strong(self, attrs): self.start_b(attrs)

   356     def end_strong(self): self.end_b()

   357 

   358     def start_var(self, attrs): self.start_i(attrs)

   359     def end_var(self): self.end_i()

   360 

   361     # Typographic Elements

   362 

   363     def start_i(self, attrs):

   364         self.formatter.push_font((AS_IS, 1, AS_IS, AS_IS))

   365     def end_i(self):

   366         self.formatter.pop_font()

   367 

   368     def start_b(self, attrs):

   369         self.formatter.push_font((AS_IS, AS_IS, 1, AS_IS))

   370     def end_b(self):

   371         self.formatter.pop_font()

   372 

   373     def start_tt(self, attrs):

   374         self.formatter.push_font((AS_IS, AS_IS, AS_IS, 1))

   375     def end_tt(self):

   376         self.formatter.pop_font()

   377 

   378     def start_a(self, attrs):

   379         href = ''

   380         name = ''

   381         type = ''

   382         for attrname, value in attrs:

   383             value = value.strip()

   384             if attrname == 'href':

   385                 href = value

   386             if attrname == 'name':

   387                 name = value

   388             if attrname == 'type':

   389                 type = value.lower()

   390         self.anchor_bgn(href, name, type)

   391 

   392     def end_a(self):

   393         self.anchor_end()

   394 

   395     # --- Line Break

   396 

   397     def do_br(self, attrs):

   398         self.formatter.add_line_break()

   399 

   400     # --- Horizontal Rule

   401 

   402     def do_hr(self, attrs):

   403         self.formatter.add_hor_rule()

   404 

   405     # --- Image

   406 

   407     def do_img(self, attrs):

   408         align = ''

   409         alt = '(image)'

   410         ismap = ''

   411         src = ''

   412         width = 0

   413         height = 0

   414         for attrname, value in attrs:

   415             if attrname == 'align':

   416                 align = value

   417             if attrname == 'alt':

   418                 alt = value

   419             if attrname == 'ismap':

   420                 ismap = value

   421             if attrname == 'src':

   422                 src = value

   423             if attrname == 'width':

   424                 try: width = int(value)

   425                 except ValueError: pass

   426             if attrname == 'height':

   427                 try: height = int(value)

   428                 except ValueError: pass

   429         self.handle_image(src, alt, ismap, align, width, height)

   430 

   431     # --- Really Old Unofficial Deprecated Stuff

   432 

   433     def do_plaintext(self, attrs):

   434         self.start_pre(attrs)

   435         self.setnomoretags() # Tell SGML parser

   436 

   437     # --- Unhandled tags

   438 

   439     def unknown_starttag(self, tag, attrs):

   440         pass

   441 

   442     def unknown_endtag(self, tag):

   443         pass

   444 

   445 

   446 def test(args = None):

   447     import sys, formatter

   448 

   449     if not args:

   450         args = sys.argv[1:]

   451 

   452     silent = args and args[0] == '-s'

   453     if silent:

   454         del args[0]

   455 

   456     if args:

   457         file = args[0]

   458     else:

   459         file = 'test.html'

   460 

   461     if file == '-':

   462         f = sys.stdin

   463     else:

   464         try:

   465             f = open(file, 'r')

   466         except IOError, msg:

   467             print file, ":", msg

   468             sys.exit(1)

   469 

   470     data = f.read()

   471 

   472     if f is not sys.stdin:

   473         f.close()

   474 

   475     if silent:

   476         f = formatter.NullFormatter()

   477     else:

   478         f = formatter.AbstractFormatter(formatter.DumbWriter())

   479 

   480     p = HTMLParser(f)

   481     p.feed(data)

   482     p.close()

   483 

   484 

   485 if __name__ == '__main__':

   486     test()