lxml.html.soupparser

1 __doc__ = """External interface to the BeautifulSoup HTML parser. 2 """ 3 4 __all__ = ["fromstring", "parse", "convert_tree"] 5 6 from lxml import etree, html 7 from BeautifulSoup import \ 8 BeautifulSoup, Tag, Comment, ProcessingInstruction, NavigableString 9 10

11 -def fromstring(data, beautifulsoup=None, makeelement=None, **bsargs):

12 """Parse a string of HTML data into an Element tree using the 13 BeautifulSoup parser. 14 15 Returns the root ``<html>`` Element of the tree. 16 17 You can pass a different BeautifulSoup parser through the 18 `beautifulsoup` keyword, and a diffent Element factory function 19 through the `makeelement` keyword. By default, the standard 20 ``BeautifulSoup`` class and the default factory of `lxml.html` are 21 used. 22 """ 23 return _parse(data, beautifulsoup, makeelement, **bsargs)

24

25 -def parse(file, beautifulsoup=None, makeelement=None, **bsargs):

26 """Parse a file into an ElemenTree using the BeautifulSoup parser. 27 28 You can pass a different BeautifulSoup parser through the 29 `beautifulsoup` keyword, and a diffent Element factory function 30 through the `makeelement` keyword. By default, the standard 31 ``BeautifulSoup`` class and the default factory of `lxml.html` are 32 used. 33 """ 34 if not hasattr(file, 'read'): 35 file = open(file) 36 root = _parse(file, beautifulsoup, makeelement, **bsargs) 37 return etree.ElementTree(root)

38

39 -def convert_tree(beautiful_soup_tree, makeelement=None):

40 """Convert a BeautifulSoup tree to a list of Element trees. 41 42 Returns a list instead of a single root Element to support 43 HTML-like soup with more than one root element. 44 45 You can pass a different Element factory through the `makeelement` 46 keyword. 47 """ 48 if makeelement is None: 49 makeelement = html.html_parser.makeelement 50 root = _convert_tree(beautiful_soup_tree, makeelement) 51 children = root.getchildren() 52 for child in children: 53 root.remove(child) 54 return children

55 56 57 # helpers 58

59 -def _parse(source, beautifulsoup, makeelement, **bsargs):

60 if beautifulsoup is None: 61 beautifulsoup = BeautifulSoup 62 if makeelement is None: 63 makeelement = html.html_parser.makeelement 64 if 'convertEntities' not in bsargs: 65 bsargs['convertEntities'] = 'html' 66 tree = beautifulsoup(source, **bsargs) 67 root = _convert_tree(tree, makeelement) 68 # from ET: wrap the document in a html root element, if necessary 69 if len(root) == 1 and root[0].tag == "html": 70 return root[0] 71 root.tag = "html" 72 return root

73

74 -def _convert_tree(beautiful_soup_tree, makeelement):

75 root = makeelement(beautiful_soup_tree.name, 76 attrib=dict(beautiful_soup_tree.attrs)) 77 _convert_children(root, beautiful_soup_tree, makeelement) 78 return root

79

80 -def _convert_children(parent, beautiful_soup_tree, makeelement):

81 SubElement = etree.SubElement 82 et_child = None 83 for child in beautiful_soup_tree: 84 if isinstance(child, Tag): 85 et_child = SubElement(parent, child.name, attrib=dict( 86 [(k, unescape(v)) for (k,v) in child.attrs])) 87 _convert_children(et_child, child, makeelement) 88 elif type(child) is NavigableString: 89 _append_text(parent, et_child, unescape(child)) 90 else: 91 if isinstance(child, Comment): 92 parent.append(etree.Comment(child)) 93 elif isinstance(child, ProcessingInstruction): 94 parent.append(etree.ProcessingInstruction( 95 *child.split(' ', 1))) 96 else: # CData 97 _append_text(parent, et_child, unescape(child))

98

99 -def _append_text(parent, element, text):

100 if element is None: 101 parent.text = (parent.text or '') + text 102 else: 103 element.tail = (element.tail or '') + text

104 105 106 # copied from ET's ElementSoup 107 108 try: 109 from html.entities import name2codepoint # Python 3 110 except ImportError: 111 from htmlentitydefs import name2codepoint 112 import re 113 114 handle_entities = re.compile("&(\w+);").sub 115

116 -def unescape(string):

117 if not string: 118 return '' 119 # work around oddities in BeautifulSoup's entity handling 120 def unescape_entity(m): 121 try: 122 return unichr(name2codepoint[m.group(1)]) 123 except KeyError: 124 return m.group(0) # use as is

125 return handle_entities(unescape_entity, string) 126

Source Code for Module lxml.html.soupparser