Source Code for Module lxml.html.html5parser

  1  """ 
  2  An interface to html5lib that mimics the lxml.html interface. 
  3  """ 
  4   
  5  import sys 
  6  import string 
  7   
  8  from html5lib import HTMLParser as _HTMLParser 
  9  from html5lib.treebuilders.etree_lxml import TreeBuilder 
 10   
 11  from lxml import etree 
 12  from lxml.html import _contains_block_level_tag, XHTML_NAMESPACE, Element 
 13   
 14  # python3 compatibility 
 15  try: 
 16      _strings = basestring 
 17  except NameError: 
 18      _strings = (bytes, str) 
 19  try: 
 20      from urllib2 import urlopen 
 21  except ImportError: 
 22      from urllib.request import urlopen 
 23  try: 
 24      from urlparse import urlparse 
 25  except ImportError: 
 26      from urllib.parse import urlparse 
 27   
 28 -class HTMLParser(_HTMLParser): 
 29      """An html5lib HTML parser with lxml as tree.""" 
 30   
 31 -    def __init__(self, strict=False, **kwargs): 
 32          _HTMLParser.__init__(self, strict=strict, tree=TreeBuilder, **kwargs) 
 33   
 34   
 35  try: 
 36      from html5lib import XHTMLParser as _XHTMLParser 
 37  except ImportError: 
 38      pass 
 39  else: 
 40 -    class XHTMLParser(_XHTMLParser): 
 41          """An html5lib XHTML Parser with lxml as tree.""" 
 42   
 43 -        def __init__(self, strict=False, **kwargs): 
 44              _XHTMLParser.__init__(self, strict=strict, tree=TreeBuilder, **kwargs) 
 45   
 46      xhtml_parser = XHTMLParser() 
 47   
 48   
 49 -def _find_tag(tree, tag): 
 50      elem = tree.find(tag) 
 51      if elem is not None: 
 52          return elem 
 53      return tree.find('{%s}%s' % (XHTML_NAMESPACE, tag)) 
 54   
 55   
 56 -def document_fromstring(html, guess_charset=True, parser=None): 
 57      """Parse a whole document into a string.""" 
 58      if not isinstance(html, _strings): 
 59          raise TypeError('string required') 
 60   
 61      if parser is None: 
 62          parser = html_parser 
 63   
 64      return parser.parse(html, useChardet=guess_charset).getroot() 
 65   
 66   
 67 -def fragments_fromstring(html, no_leading_text=False, 
 68                           guess_charset=False, parser=None): 
 69      """Parses several HTML elements, returning a list of elements. 
 70   
 71      The first item in the list may be a string.  If no_leading_text is true, 
 72      then it will be an error if there is leading text, and it will always be 
 73      a list of only elements. 
 74   
 75      If `guess_charset` is `True` and the text was not unicode but a 
 76      bytestring, the `chardet` library will perform charset guessing on the 
 77      string. 
 78      """ 
 79      if not isinstance(html, _strings): 
 80          raise TypeError('string required') 
 81   
 82      if parser is None: 
 83          parser = html_parser 
 84   
 85      children = parser.parseFragment(html, 'div', useChardet=guess_charset) 
 86      if children and isinstance(children[0], _strings): 
 87          if no_leading_text: 
 88              if children[0].strip(): 
 89                  raise etree.ParserError('There is leading text: %r' % 
 90                                          children[0]) 
 91              del children[0] 
 92      return children 
 93   
 94   
 95 -def fragment_fromstring(html, create_parent=False, 
 96                          guess_charset=False, parser=None): 
 97      """Parses a single HTML element; it is an error if there is more than 
 98      one element, or if anything but whitespace precedes or follows the 
 99      element. 
100   
101      If create_parent is true (or is a tag name) then a parent node 
102      will be created to encapsulate the HTML in a single element.  In 
103      this case, leading or trailing text is allowed. 
104      """ 
105      if not isinstance(html, _strings): 
106          raise TypeError('string required') 
107   
108      accept_leading_text = bool(create_parent) 
109   
110      elements = fragments_fromstring( 
111          html, guess_charset=guess_charset, parser=parser, 
112          no_leading_text=not accept_leading_text) 
113   
114      if create_parent: 
115          if not isinstance(create_parent, _strings): 
116              create_parent = 'div' 
117          new_root = Element(create_parent) 
118          if elements: 
119              if isinstance(elements[0], _strings): 
120                  new_root.text = elements[0] 
121                  del elements[0] 
122              new_root.extend(elements) 
123          return new_root 
124   
125      if not elements: 
126          raise etree.ParserError('No elements found') 
127      if len(elements) > 1: 
128          raise etree.ParserError('Multiple elements found') 
129      result = elements[0] 
130      if result.tail and result.tail.strip(): 
131          raise etree.ParserError('Element followed by text: %r' % result.tail) 
132      result.tail = None 
133      return result 
134   
135   
136 -def fromstring(html, guess_charset=True, parser=None): 
137      """Parse the html, returning a single element/document. 
138   
139      This tries to minimally parse the chunk of text, without knowing if it 
140      is a fragment or a document. 
141   
142      base_url will set the document's base_url attribute (and the tree's docinfo.URL) 
143      """ 
144      if not isinstance(html, _strings): 
145          raise TypeError('string required') 
146      doc = document_fromstring(html, parser=parser, 
147                                guess_charset=guess_charset) 
148   
149      # document starts with doctype or <html>, full document! 
150      start = html[:50].lstrip().lower() 
151      if start.startswith('<html') or start.startswith('<!doctype'): 
152          return doc 
153   
154      head = _find_tag(doc, 'head') 
155   
156      # if the head is not empty we have a full document 
157      if len(head): 
158          return doc 
159   
160      body = _find_tag(doc, 'body') 
161   
162      # The body has just one element, so it was probably a single 
163      # element passed in 
164      if (len(body) == 1 and (not body.text or not body.text.strip()) 
165          and (not body[-1].tail or not body[-1].tail.strip())): 
166          return body[0] 
167   
168      # Now we have a body which represents a bunch of tags which have the 
169      # content that was passed in.  We will create a fake container, which 
170      # is the body tag, except <body> implies too much structure. 
171      if _contains_block_level_tag(body): 
172          body.tag = 'div' 
173      else: 
174          body.tag = 'span' 
175      return body 
176   
177   
178 -def parse(filename_url_or_file, guess_charset=True, parser=None): 
179      """Parse a filename, URL, or file-like object into an HTML document 
180      tree.  Note: this returns a tree, not an element.  Use 
181      ``parse(...).getroot()`` to get the document root. 
182      """ 
183      if parser is None: 
184          parser = html_parser 
185      if not isinstance(filename_url_or_file, _strings): 
186          fp = filename_url_or_file 
187      elif _looks_like_url(filename_url_or_file): 
188          fp = urlopen(filename_url_or_file) 
189      else: 
190          fp = open(filename_url_or_file, 'rb') 
191      return parser.parse(fp, useChardet=guess_charset) 
192   
193   
194 -def _looks_like_url(str): 
195      scheme = urlparse(str)[0] 
196      if not scheme: 
197          return False 
198      elif (sys.platform == 'win32' and 
199              scheme in string.ascii_letters 
200              and len(scheme) == 1): 
201          # looks like a 'normal' absolute path 
202          return False 
203      else: 
204          return True 
205   
206   
207  html_parser = HTMLParser() 
208