Package lxml :: Package html :: Module soupparser
[hide private]
[frames] | no frames]

Source Code for Module lxml.html.soupparser

  1  __doc__ = """External interface to the BeautifulSoup HTML parser. 
  2  """ 
  3   
  4  __all__ = ["fromstring", "parse", "convert_tree"] 
  5   
  6  from lxml import etree, html 
  7  from BeautifulSoup import \ 
  8       BeautifulSoup, Tag, Comment, ProcessingInstruction, NavigableString 
  9   
 10   
11 -def fromstring(data, beautifulsoup=None, makeelement=None, **bsargs):
12 """Parse a string of HTML data into an Element tree using the 13 BeautifulSoup parser. 14 15 Returns the root ``<html>`` Element of the tree. 16 17 You can pass a different BeautifulSoup parser through the 18 `beautifulsoup` keyword, and a diffent Element factory function 19 through the `makeelement` keyword. By default, the standard 20 ``BeautifulSoup`` class and the default factory of `lxml.html` are 21 used. 22 """ 23 return _parse(data, beautifulsoup, makeelement, **bsargs)
24
25 -def parse(file, beautifulsoup=None, makeelement=None, **bsargs):
26 """Parse a file into an ElemenTree using the BeautifulSoup parser. 27 28 You can pass a different BeautifulSoup parser through the 29 `beautifulsoup` keyword, and a diffent Element factory function 30 through the `makeelement` keyword. By default, the standard 31 ``BeautifulSoup`` class and the default factory of `lxml.html` are 32 used. 33 """ 34 if not hasattr(file, 'read'): 35 file = open(file) 36 root = _parse(file, beautifulsoup, makeelement, **bsargs) 37 return etree.ElementTree(root)
38
39 -def convert_tree(beautiful_soup_tree, makeelement=None):
40 """Convert a BeautifulSoup tree to a list of Element trees. 41 42 Returns a list instead of a single root Element to support 43 HTML-like soup with more than one root element. 44 45 You can pass a different Element factory through the `makeelement` 46 keyword. 47 """ 48 if makeelement is None: 49 makeelement = html.html_parser.makeelement 50 root = _convert_tree(beautiful_soup_tree, makeelement) 51 children = root.getchildren() 52 for child in children: 53 root.remove(child) 54 return children
55 56 57 # helpers 58
59 -def _parse(source, beautifulsoup, makeelement, **bsargs):
60 if beautifulsoup is None: 61 beautifulsoup = BeautifulSoup 62 if makeelement is None: 63 makeelement = html.html_parser.makeelement 64 if 'convertEntities' not in bsargs: 65 bsargs['convertEntities'] = 'html' 66 tree = beautifulsoup(source, **bsargs) 67 root = _convert_tree(tree, makeelement) 68 # from ET: wrap the document in a html root element, if necessary 69 if len(root) == 1 and root[0].tag == "html": 70 return root[0] 71 root.tag = "html" 72 return root
73
74 -def _convert_tree(beautiful_soup_tree, makeelement):
75 root = makeelement(beautiful_soup_tree.name, 76 attrib=dict(beautiful_soup_tree.attrs)) 77 _convert_children(root, beautiful_soup_tree, makeelement) 78 return root
79
80 -def _convert_children(parent, beautiful_soup_tree, makeelement):
81 SubElement = etree.SubElement 82 et_child = None 83 for child in beautiful_soup_tree: 84 if isinstance(child, Tag): 85 et_child = SubElement(parent, child.name, attrib=dict( 86 [(k, unescape(v)) for (k,v) in child.attrs])) 87 _convert_children(et_child, child, makeelement) 88 elif type(child) is NavigableString: 89 _append_text(parent, et_child, unescape(child)) 90 else: 91 if isinstance(child, Comment): 92 parent.append(etree.Comment(child)) 93 elif isinstance(child, ProcessingInstruction): 94 parent.append(etree.ProcessingInstruction( 95 *child.split(' ', 1))) 96 else: # CData 97 _append_text(parent, et_child, unescape(child))
98
99 -def _append_text(parent, element, text):
100 if element is None: 101 parent.text = (parent.text or '') + text 102 else: 103 element.tail = (element.tail or '') + text
104 105 106 # copied from ET's ElementSoup 107 108 try: 109 from html.entities import name2codepoint # Python 3 110 except ImportError: 111 from htmlentitydefs import name2codepoint 112 import re 113 114 handle_entities = re.compile("&(\w+);").sub 115
116 -def unescape(string):
117 if not string: 118 return '' 119 # work around oddities in BeautifulSoup's entity handling 120 def unescape_entity(m): 121 try: 122 return unichr(name2codepoint[m.group(1)]) 123 except KeyError: 124 return m.group(0) # use as is
125 return handle_entities(unescape_entity, string) 126