Package lxml :: Package html :: Module html5parser
[hide private]
[frames] | no frames]

Source Code for Module lxml.html.html5parser

  1  """ 
  2  An interface to html5lib that mimics the lxml.html interface. 
  3  """ 
  4  import sys 
  5  import string 
  6   
  7  from html5lib import HTMLParser as _HTMLParser 
  8  from html5lib.treebuilders.etree_lxml import TreeBuilder 
  9  from lxml import etree 
 10  from lxml.html import Element, XHTML_NAMESPACE, _contains_block_level_tag 
 11   
 12  # python3 compatibility 
 13  try: 
 14      _strings = basestring 
 15  except NameError: 
 16      _strings = (bytes, str) 
 17  try: 
 18      from urllib2 import urlopen 
 19  except ImportError: 
 20      from urllib.request import urlopen 
 21  try: 
 22      from urlparse import urlparse 
 23  except ImportError: 
 24      from urllib.parse import urlparse 
 25   
 26   
27 -class HTMLParser(_HTMLParser):
28 """An html5lib HTML parser with lxml as tree.""" 29
30 - def __init__(self, strict=False, **kwargs):
31 _HTMLParser.__init__(self, strict=strict, tree=TreeBuilder, **kwargs)
32 33 34 try: 35 from html5lib import XHTMLParser as _XHTMLParser 36 except ImportError: 37 pass 38 else:
39 - class XHTMLParser(_XHTMLParser):
40 """An html5lib XHTML Parser with lxml as tree.""" 41
42 - def __init__(self, strict=False, **kwargs):
43 _XHTMLParser.__init__(self, strict=strict, tree=TreeBuilder, **kwargs)
44 45 xhtml_parser = XHTMLParser() 46 47
48 -def _find_tag(tree, tag):
49 elem = tree.find(tag) 50 if elem is not None: 51 return elem 52 return tree.find('{%s}%s' % (XHTML_NAMESPACE, tag))
53 54
55 -def document_fromstring(html, guess_charset=None, parser=None):
56 """ 57 Parse a whole document into a string. 58 59 If `guess_charset` is true, or if the input is not Unicode but a 60 byte string, the `chardet` library will perform charset guessing 61 on the string. 62 """ 63 if not isinstance(html, _strings): 64 raise TypeError('string required') 65 66 if parser is None: 67 parser = html_parser 68 69 options = {} 70 if guess_charset is None and isinstance(html, bytes): 71 # html5lib does not accept useChardet as an argument, if it 72 # detected the html argument would produce unicode objects. 73 guess_charset = True 74 if guess_charset is not None: 75 options['useChardet'] = guess_charset 76 return parser.parse(html, **options).getroot()
77 78
79 -def fragments_fromstring(html, no_leading_text=False, 80 guess_charset=None, parser=None):
81 """Parses several HTML elements, returning a list of elements. 82 83 The first item in the list may be a string. If no_leading_text is true, 84 then it will be an error if there is leading text, and it will always be 85 a list of only elements. 86 87 If `guess_charset` is true, the `chardet` library will perform charset 88 guessing on the string. 89 """ 90 if not isinstance(html, _strings): 91 raise TypeError('string required') 92 93 if parser is None: 94 parser = html_parser 95 96 options = {} 97 if guess_charset is None and isinstance(html, bytes): 98 # html5lib does not accept useChardet as an argument, if it 99 # detected the html argument would produce unicode objects. 100 guess_charset = False 101 if guess_charset is not None: 102 options['useChardet'] = guess_charset 103 children = parser.parseFragment(html, 'div', **options) 104 if children and isinstance(children[0], _strings): 105 if no_leading_text: 106 if children[0].strip(): 107 raise etree.ParserError('There is leading text: %r' % 108 children[0]) 109 del children[0] 110 return children
111 112
113 -def fragment_fromstring(html, create_parent=False, 114 guess_charset=None, parser=None):
115 """Parses a single HTML element; it is an error if there is more than 116 one element, or if anything but whitespace precedes or follows the 117 element. 118 119 If 'create_parent' is true (or is a tag name) then a parent node 120 will be created to encapsulate the HTML in a single element. In 121 this case, leading or trailing text is allowed. 122 123 If `guess_charset` is true, the `chardet` library will perform charset 124 guessing on the string. 125 """ 126 if not isinstance(html, _strings): 127 raise TypeError('string required') 128 129 accept_leading_text = bool(create_parent) 130 131 elements = fragments_fromstring( 132 html, guess_charset=guess_charset, parser=parser, 133 no_leading_text=not accept_leading_text) 134 135 if create_parent: 136 if not isinstance(create_parent, _strings): 137 create_parent = 'div' 138 new_root = Element(create_parent) 139 if elements: 140 if isinstance(elements[0], _strings): 141 new_root.text = elements[0] 142 del elements[0] 143 new_root.extend(elements) 144 return new_root 145 146 if not elements: 147 raise etree.ParserError('No elements found') 148 if len(elements) > 1: 149 raise etree.ParserError('Multiple elements found') 150 result = elements[0] 151 if result.tail and result.tail.strip(): 152 raise etree.ParserError('Element followed by text: %r' % result.tail) 153 result.tail = None 154 return result
155 156
157 -def fromstring(html, guess_charset=None, parser=None):
158 """Parse the html, returning a single element/document. 159 160 This tries to minimally parse the chunk of text, without knowing if it 161 is a fragment or a document. 162 163 'base_url' will set the document's base_url attribute (and the tree's 164 docinfo.URL) 165 166 If `guess_charset` is true, or if the input is not Unicode but a 167 byte string, the `chardet` library will perform charset guessing 168 on the string. 169 """ 170 if not isinstance(html, _strings): 171 raise TypeError('string required') 172 doc = document_fromstring(html, parser=parser, 173 guess_charset=guess_charset) 174 175 # document starts with doctype or <html>, full document! 176 start = html[:50] 177 if isinstance(start, bytes): 178 # Allow text comparison in python3. 179 # Decode as ascii, that also covers latin-1 and utf-8 for the 180 # characters we need. 181 start = start.decode('ascii', 'replace') 182 183 start = start.lstrip().lower() 184 if start.startswith('<html') or start.startswith('<!doctype'): 185 return doc 186 187 head = _find_tag(doc, 'head') 188 189 # if the head is not empty we have a full document 190 if len(head): 191 return doc 192 193 body = _find_tag(doc, 'body') 194 195 # The body has just one element, so it was probably a single 196 # element passed in 197 if (len(body) == 1 and (not body.text or not body.text.strip()) 198 and (not body[-1].tail or not body[-1].tail.strip())): 199 return body[0] 200 201 # Now we have a body which represents a bunch of tags which have the 202 # content that was passed in. We will create a fake container, which 203 # is the body tag, except <body> implies too much structure. 204 if _contains_block_level_tag(body): 205 body.tag = 'div' 206 else: 207 body.tag = 'span' 208 return body
209 210
211 -def parse(filename_url_or_file, guess_charset=None, parser=None):
212 """Parse a filename, URL, or file-like object into an HTML document 213 tree. Note: this returns a tree, not an element. Use 214 ``parse(...).getroot()`` to get the document root. 215 216 If ``guess_charset`` is true, the ``useChardet`` option is passed into 217 html5lib to enable character detection. This option is on by default 218 when parsing from URLs, off by default when parsing from file(-like) 219 objects (which tend to return Unicode more often than not), and on by 220 default when parsing from a file path (which is read in binary mode). 221 """ 222 if parser is None: 223 parser = html_parser 224 if not isinstance(filename_url_or_file, _strings): 225 fp = filename_url_or_file 226 if guess_charset is None: 227 # assume that file-like objects return Unicode more often than bytes 228 guess_charset = False 229 elif _looks_like_url(filename_url_or_file): 230 fp = urlopen(filename_url_or_file) 231 if guess_charset is None: 232 # assume that URLs return bytes 233 guess_charset = True 234 else: 235 fp = open(filename_url_or_file, 'rb') 236 if guess_charset is None: 237 guess_charset = True 238 239 options = {} 240 # html5lib does not accept useChardet as an argument, if it 241 # detected the html argument would produce unicode objects. 242 if guess_charset: 243 options['useChardet'] = guess_charset 244 return parser.parse(fp, **options)
245 246
247 -def _looks_like_url(str):
248 scheme = urlparse(str)[0] 249 if not scheme: 250 return False 251 elif (sys.platform == 'win32' and 252 scheme in string.ascii_letters 253 and len(scheme) == 1): 254 # looks like a 'normal' absolute path 255 return False 256 else: 257 return True
258 259 260 html_parser = HTMLParser() 261