lxml.html

1 # Copyright (c) 2004 Ian Bicking. All rights reserved. 2 # 3 # Redistribution and use in source and binary forms, with or without 4 # modification, are permitted provided that the following conditions are 5 # met: 6 # 7 # 1. Redistributions of source code must retain the above copyright 8 # notice, this list of conditions and the following disclaimer. 9 # 10 # 2. Redistributions in binary form must reproduce the above copyright 11 # notice, this list of conditions and the following disclaimer in 12 # the documentation and/or other materials provided with the 13 # distribution. 14 # 15 # 3. Neither the name of Ian Bicking nor the names of its contributors may 16 # be used to endorse or promote products derived from this software 17 # without specific prior written permission. 18 # 19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL IAN BICKING OR 23 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 24 # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 25 # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 26 # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 27 # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 28 # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 29 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 31 """The ``lxml.html`` tool set for HTML handling. 32 """ 33 34 from __future__ import absolute_import 35 36 __all__ = [ 37 'document_fromstring', 'fragment_fromstring', 'fragments_fromstring', 'fromstring', 38 'tostring', 'Element', 'defs', 'open_in_browser', 'submit_form', 39 'find_rel_links', 'find_class', 'make_links_absolute', 40 'resolve_base_href', 'iterlinks', 'rewrite_links', 'open_in_browser', 'parse'] 41 42 43 import copy 44 import sys 45 import re 46 from functools import partial 47 48 try: 49 # while unnecessary, importing from 'collections.abc' is the right way to do it 50 from collections.abc import MutableMapping, MutableSet 51 except ImportError: 52 from collections import MutableMapping, MutableSet 53 54 from .. import etree 55 from . import defs 56 from ._setmixin import SetMixin 57 58 try: 59 from urlparse import urljoin 60 except ImportError: 61 # Python 3 62 from urllib.parse import urljoin 63 64 try: 65 unicode 66 except NameError: 67 # Python 3 68 unicode = str 69 try: 70 basestring 71 except NameError: 72 # Python 3 73 basestring = (str, bytes)

74 75 76 -def __fix_docstring(s):

77 if not s: 78 return s 79 if sys.version_info[0] >= 3: 80 sub = re.compile(r"^(\s*)u'", re.M).sub 81 else: 82 sub = re.compile(r"^(\s*)b'", re.M).sub 83 return sub(r"\1'", s)

84 85 86 XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml" 87 88 _rel_links_xpath = etree.XPath("descendant-or-self::a[@rel]|descendant-or-self::x:a[@rel]", 89 namespaces={'x':XHTML_NAMESPACE}) 90 _options_xpath = etree.XPath("descendant-or-self::option|descendant-or-self::x:option", 91 namespaces={'x':XHTML_NAMESPACE}) 92 _forms_xpath = etree.XPath("descendant-or-self::form|descendant-or-self::x:form", 93 namespaces={'x':XHTML_NAMESPACE}) 94 #_class_xpath = etree.XPath(r"descendant-or-self::*[regexp:match(@class, concat('\b', $class_name, '\b'))]", {'regexp': 'http://exslt.org/regular-expressions'}) 95 _class_xpath = etree.XPath("descendant-or-self::*[@class and contains(concat(' ', normalize-space(@class), ' '), concat(' ', $class_name, ' '))]") 96 _id_xpath = etree.XPath("descendant-or-self::*[@id=$id]") 97 _collect_string_content = etree.XPath("string()") 98 _iter_css_urls = re.compile(r'url$('+'["][^"]*["]|'+"['][^']*[']|"+r'[^)]*)$', re.I).finditer 99 _iter_css_imports = re.compile(r'@import "(.*?)"').finditer 100 _label_xpath = etree.XPath("//label[@for=$id]|//x:label[@for=$id]", 101 namespaces={'x':XHTML_NAMESPACE}) 102 _archive_re = re.compile(r'[^ ]+') 103 _parse_meta_refresh_url = re.compile( 104 r'[^;=]*;\s*(?:url\s*=\s*)?(?P<url>.*)$', re.I).search

105 106 107 -def _unquote_match(s, pos):

108 if s[:1] == '"' and s[-1:] == '"' or s[:1] == "'" and s[-1:] == "'": 109 return s[1:-1], pos+1 110 else: 111 return s,pos

112

113 114 -def _transform_result(typ, result):

115 """Convert the result back into the input type. 116 """ 117 if issubclass(typ, bytes): 118 return tostring(result, encoding='utf-8') 119 elif issubclass(typ, unicode): 120 return tostring(result, encoding='unicode') 121 else: 122 return result

123

124 125 -def _nons(tag):

126 if isinstance(tag, basestring): 127 if tag[0] == '{' and tag[1:len(XHTML_NAMESPACE)+1] == XHTML_NAMESPACE: 128 return tag.split('}')[-1] 129 return tag

130

131 132 -class Classes(MutableSet):

133 """Provides access to an element's class attribute as a set-like collection. 134 Usage:: 135 136 >>> el = fromstring('<p class="hidden large">Text</p>') 137 >>> classes = el.classes # or: classes = Classes(el.attrib) 138 >>> classes |= ['block', 'paragraph'] 139 >>> el.get('class') 140 'hidden large block paragraph' 141 >>> classes.toggle('hidden') 142 False 143 >>> el.get('class') 144 'large block paragraph' 145 >>> classes -= ('some', 'classes', 'block') 146 >>> el.get('class') 147 'large paragraph' 148 """

149 - def __init__(self, attributes):

150 self._attributes = attributes 151 self._get_class_value = partial(attributes.get, 'class', '')

152

153 - def add(self, value):

154 """ 155 Add a class. 156 157 This has no effect if the class is already present. 158 """ 159 if not value or re.search(r'\s', value): 160 raise ValueError("Invalid class name: %r" % value) 161 classes = self._get_class_value().split() 162 if value in classes: 163 return 164 classes.append(value) 165 self._attributes['class'] = ' '.join(classes)

166

167 - def discard(self, value):

168 """ 169 Remove a class if it is currently present. 170 171 If the class is not present, do nothing. 172 """ 173 if not value or re.search(r'\s', value): 174 raise ValueError("Invalid class name: %r" % value) 175 classes = [name for name in self._get_class_value().split() 176 if name != value] 177 if classes: 178 self._attributes['class'] = ' '.join(classes) 179 elif 'class' in self._attributes: 180 del self._attributes['class']

181

182 - def remove(self, value):

183 """ 184 Remove a class; it must currently be present. 185 186 If the class is not present, raise a KeyError. 187 """ 188 if not value or re.search(r'\s', value): 189 raise ValueError("Invalid class name: %r" % value) 190 super(Classes, self).remove(value)

191

192 - def __contains__(self, name):

193 classes = self._get_class_value() 194 return name in classes and name in classes.split()

195

196 - def __iter__(self):

197 return iter(self._get_class_value().split())

198

199 - def __len__(self):

200 return len(self._get_class_value().split())

201 202 # non-standard methods 203

204 - def update(self, values):

205 """ 206 Add all names from 'values'. 207 """ 208 classes = self._get_class_value().split() 209 extended = False 210 for value in values: 211 if value not in classes: 212 classes.append(value) 213 extended = True 214 if extended: 215 self._attributes['class'] = ' '.join(classes)

216

217 - def toggle(self, value):

218 """ 219 Add a class name if it isn't there yet, or remove it if it exists. 220 221 Returns true if the class was added (and is now enabled) and 222 false if it was removed (and is now disabled). 223 """ 224 if not value or re.search(r'\s', value): 225 raise ValueError("Invalid class name: %r" % value) 226 classes = self._get_class_value().split() 227 try: 228 classes.remove(value) 229 enabled = False 230 except ValueError: 231 classes.append(value) 232 enabled = True 233 if classes: 234 self._attributes['class'] = ' '.join(classes) 235 else: 236 del self._attributes['class'] 237 return enabled

238

239 240 -class HtmlMixin(object):

241

242 - def set(self, key, value=None):

243 """set(self, key, value=None) 244 245 Sets an element attribute. If no value is provided, or if the value is None, 246 creates a 'boolean' attribute without value, e.g. "<form novalidate></form>" 247 for ``form.set('novalidate')``. 248 """ 249 super(HtmlElement, self).set(key, value)

250 251 @property

252 - def classes(self):

253 """ 254 A set-like wrapper around the 'class' attribute. 255 """ 256 return Classes(self.attrib)

257 258 @classes.setter

259 - def classes(self, classes):

260 assert isinstance(classes, Classes) # only allow "el.classes |= ..." etc. 261 value = classes._get_class_value() 262 if value: 263 self.set('class', value) 264 elif self.get('class') is not None: 265 del self.attrib['class']

266 267 @property

268 - def base_url(self):

269 """ 270 Returns the base URL, given when the page was parsed. 271 272 Use with ``urlparse.urljoin(el.base_url, href)`` to get 273 absolute URLs. 274 """ 275 return self.getroottree().docinfo.URL

276 277 @property

278 - def forms(self):

279 """ 280 Return a list of all the forms 281 """ 282 return _forms_xpath(self)

283 284 @property

285 - def body(self):

286 """ 287 Return the <body> element. Can be called from a child element 288 to get the document's head. 289 """ 290 return self.xpath('//body|//x:body', namespaces={'x':XHTML_NAMESPACE})[0]

291 292 @property

293 - def head(self):

294 """ 295 Returns the <head> element. Can be called from a child 296 element to get the document's head. 297 """ 298 return self.xpath('//head|//x:head', namespaces={'x':XHTML_NAMESPACE})[0]

299 300 @property

301 - def label(self):

302 """ 303 Get or set any <label> element associated with this element. 304 """ 305 id = self.get('id') 306 if not id: 307 return None 308 result = _label_xpath(self, id=id) 309 if not result: 310 return None 311 else: 312 return result[0]

313 314 @label.setter

315 - def label(self, label):

316 id = self.get('id') 317 if not id: 318 raise TypeError( 319 "You cannot set a label for an element (%r) that has no id" 320 % self) 321 if _nons(label.tag) != 'label': 322 raise TypeError( 323 "You can only assign label to a label element (not %r)" 324 % label) 325 label.set('for', id)

326 327 @label.deleter

328 - def label(self):

329 label = self.label 330 if label is not None: 331 del label.attrib['for']

332

333 - def drop_tree(self):

334 """ 335 Removes this element from the tree, including its children and 336 text. The tail text is joined to the previous element or 337 parent. 338 """ 339 parent = self.getparent() 340 assert parent is not None 341 if self.tail: 342 previous = self.getprevious() 343 if previous is None: 344 parent.text = (parent.text or '') + self.tail 345 else: 346 previous.tail = (previous.tail or '') + self.tail 347 parent.remove(self)

348

349 - def drop_tag(self):

350 """ 351 Remove the tag, but not its children or text. The children and text 352 are merged into the parent. 353 354 Example:: 355 356 >>> h = fragment_fromstring('<div>Hello <b>World!</b></div>') 357 >>> h.find('.//b').drop_tag() 358 >>> print(tostring(h, encoding='unicode')) 359 <div>Hello World!</div> 360 """ 361 parent = self.getparent() 362 assert parent is not None 363 previous = self.getprevious() 364 if self.text and isinstance(self.tag, basestring): 365 # not a Comment, etc. 366 if previous is None: 367 parent.text = (parent.text or '') + self.text 368 else: 369 previous.tail = (previous.tail or '') + self.text 370 if self.tail: 371 if len(self): 372 last = self[-1] 373 last.tail = (last.tail or '') + self.tail 374 elif previous is None: 375 parent.text = (parent.text or '') + self.tail 376 else: 377 previous.tail = (previous.tail or '') + self.tail 378 index = parent.index(self) 379 parent[index:index+1] = self[:]

380

381 - def find_rel_links(self, rel):

382 """ 383 Find any links like ``<a rel="{rel}">...</a>``; returns a list of elements. 384 """ 385 rel = rel.lower() 386 return [el for el in _rel_links_xpath(self) 387 if el.get('rel').lower() == rel]

388

389 - def find_class(self, class_name):

390 """ 391 Find any elements with the given class name. 392 """ 393 return _class_xpath(self, class_name=class_name)

394

395 - def get_element_by_id(self, id, *default):

396 """ 397 Get the first element in a document with the given id. If none is 398 found, return the default argument if provided or raise KeyError 399 otherwise. 400 401 Note that there can be more than one element with the same id, 402 and this isn't uncommon in HTML documents found in the wild. 403 Browsers return only the first match, and this function does 404 the same. 405 """ 406 try: 407 # FIXME: should this check for multiple matches? 408 # browsers just return the first one 409 return _id_xpath(self, id=id)[0] 410 except IndexError: 411 if default: 412 return default[0] 413 else: 414 raise KeyError(id)

415

416 - def text_content(self):

417 """ 418 Return the text content of the tag (and the text in any children). 419 """ 420 return _collect_string_content(self)

421

422 - def cssselect(self, expr, translator='html'):

423 """ 424 Run the CSS expression on this element and its children, 425 returning a list of the results. 426 427 Equivalent to lxml.cssselect.CSSSelect(expr, translator='html')(self) 428 -- note that pre-compiling the expression can provide a substantial 429 speedup. 430 """ 431 # Do the import here to make the dependency optional. 432 from lxml.cssselect import CSSSelector 433 return CSSSelector(expr, translator=translator)(self)

434 435 ######################################## 436 ## Link functions 437 ######################################## 438

439 - def make_links_absolute(self, base_url=None, resolve_base_href=True, 440 handle_failures=None):

441 """ 442 Make all links in the document absolute, given the 443 ``base_url`` for the document (the full URL where the document 444 came from), or if no ``base_url`` is given, then the ``.base_url`` 445 of the document. 446 447 If ``resolve_base_href`` is true, then any ``<base href>`` 448 tags in the document are used *and* removed from the document. 449 If it is false then any such tag is ignored. 450 451 If ``handle_failures`` is None (default), a failure to process 452 a URL will abort the processing. If set to 'ignore', errors 453 are ignored. If set to 'discard', failing URLs will be removed. 454 """ 455 if base_url is None: 456 base_url = self.base_url 457 if base_url is None: 458 raise TypeError( 459 "No base_url given, and the document has no base_url") 460 if resolve_base_href: 461 self.resolve_base_href() 462 463 if handle_failures == 'ignore': 464 def link_repl(href): 465 try: 466 return urljoin(base_url, href) 467 except ValueError: 468 return href

469 elif handle_failures == 'discard': 470 def link_repl(href): 471 try: 472 return urljoin(base_url, href) 473 except ValueError: 474 return None

475 elif handle_failures is None: 476 def link_repl(href): 477 return urljoin(base_url, href) 478 else: 479 raise ValueError( 480 "unexpected value for handle_failures: %r" % handle_failures) 481 482 self.rewrite_links(link_repl) 483

484 - def resolve_base_href(self, handle_failures=None):

485 """ 486 Find any ``<base href>`` tag in the document, and apply its 487 values to all links found in the document. Also remove the 488 tag once it has been applied. 489 490 If ``handle_failures`` is None (default), a failure to process 491 a URL will abort the processing. If set to 'ignore', errors 492 are ignored. If set to 'discard', failing URLs will be removed. 493 """ 494 base_href = None 495 basetags = self.xpath('//base[@href]|//x:base[@href]', 496 namespaces={'x': XHTML_NAMESPACE}) 497 for b in basetags: 498 base_href = b.get('href') 499 b.drop_tree() 500 if not base_href: 501 return 502 self.make_links_absolute(base_href, resolve_base_href=False, 503 handle_failures=handle_failures)

504

505 - def iterlinks(self):

506 """ 507 Yield (element, attribute, link, pos), where attribute may be None 508 (indicating the link is in the text). ``pos`` is the position 509 where the link occurs; often 0, but sometimes something else in 510 the case of links in stylesheets or style tags. 511 512 Note: <base href> is *not* taken into account in any way. The 513 link you get is exactly the link in the document. 514 515 Note: multiple links inside of a single text string or 516 attribute value are returned in reversed order. This makes it 517 possible to replace or delete them from the text string value 518 based on their reported text positions. Otherwise, a 519 modification at one text position can change the positions of 520 links reported later on. 521 """ 522 link_attrs = defs.link_attrs 523 for el in self.iter(etree.Element): 524 attribs = el.attrib 525 tag = _nons(el.tag) 526 if tag == 'object': 527 codebase = None 528 ## <object> tags have attributes that are relative to 529 ## codebase 530 if 'codebase' in attribs: 531 codebase = el.get('codebase') 532 yield (el, 'codebase', codebase, 0) 533 for attrib in ('classid', 'data'): 534 if attrib in attribs: 535 value = el.get(attrib) 536 if codebase is not None: 537 value = urljoin(codebase, value) 538 yield (el, attrib, value, 0) 539 if 'archive' in attribs: 540 for match in _archive_re.finditer(el.get('archive')): 541 value = match.group(0) 542 if codebase is not None: 543 value = urljoin(codebase, value) 544 yield (el, 'archive', value, match.start()) 545 else: 546 for attrib in link_attrs: 547 if attrib in attribs: 548 yield (el, attrib, attribs[attrib], 0) 549 if tag == 'meta': 550 http_equiv = attribs.get('http-equiv', '').lower() 551 if http_equiv == 'refresh': 552 content = attribs.get('content', '') 553 match = _parse_meta_refresh_url(content) 554 url = (match.group('url') if match else content).strip() 555 # unexpected content means the redirect won't work, but we might 556 # as well be permissive and return the entire string. 557 if url: 558 url, pos = _unquote_match( 559 url, match.start('url') if match else content.find(url)) 560 yield (el, 'content', url, pos) 561 elif tag == 'param': 562 valuetype = el.get('valuetype') or '' 563 if valuetype.lower() == 'ref': 564 ## FIXME: while it's fine we *find* this link, 565 ## according to the spec we aren't supposed to 566 ## actually change the value, including resolving 567 ## it. It can also still be a link, even if it 568 ## doesn't have a valuetype="ref" (which seems to be the norm) 569 ## http://www.w3.org/TR/html401/struct/objects.html#adef-valuetype 570 yield (el, 'value', el.get('value'), 0) 571 elif tag == 'style' and el.text: 572 urls = [ 573 # (start_pos, url) 574 _unquote_match(match.group(1), match.start(1))[::-1] 575 for match in _iter_css_urls(el.text) 576 ] + [ 577 (match.start(1), match.group(1)) 578 for match in _iter_css_imports(el.text) 579 ] 580 if urls: 581 # sort by start pos to bring both match sets back into order 582 # and reverse the list to report correct positions despite 583 # modifications 584 urls.sort(reverse=True) 585 for start, url in urls: 586 yield (el, None, url, start) 587 if 'style' in attribs: 588 urls = list(_iter_css_urls(attribs['style'])) 589 if urls: 590 # return in reversed order to simplify in-place modifications 591 for match in urls[::-1]: 592 url, start = _unquote_match(match.group(1), match.start(1)) 593 yield (el, 'style', url, start)

594

595 - def rewrite_links(self, link_repl_func, resolve_base_href=True, 596 base_href=None):

597 """ 598 Rewrite all the links in the document. For each link 599 ``link_repl_func(link)`` will be called, and the return value 600 will replace the old link. 601 602 Note that links may not be absolute (unless you first called 603 ``make_links_absolute()``), and may be internal (e.g., 604 ``'#anchor'``). They can also be values like 605 ``'mailto:email'`` or ``'javascript:expr'``. 606 607 If you give ``base_href`` then all links passed to 608 ``link_repl_func()`` will take that into account. 609 610 If the ``link_repl_func`` returns None, the attribute or 611 tag text will be removed completely. 612 """ 613 if base_href is not None: 614 # FIXME: this can be done in one pass with a wrapper 615 # around link_repl_func 616 self.make_links_absolute( 617 base_href, resolve_base_href=resolve_base_href) 618 elif resolve_base_href: 619 self.resolve_base_href() 620 621 for el, attrib, link, pos in self.iterlinks(): 622 new_link = link_repl_func(link.strip()) 623 if new_link == link: 624 continue 625 if new_link is None: 626 # Remove the attribute or element content 627 if attrib is None: 628 el.text = '' 629 else: 630 del el.attrib[attrib] 631 continue 632 633 if attrib is None: 634 new = el.text[:pos] + new_link + el.text[pos+len(link):] 635 el.text = new 636 else: 637 cur = el.get(attrib) 638 if not pos and len(cur) == len(link): 639 new = new_link # most common case 640 else: 641 new = cur[:pos] + new_link + cur[pos+len(link):] 642 el.set(attrib, new)

643

644 645 -class _MethodFunc(object):

646 """ 647 An object that represents a method on an element as a function; 648 the function takes either an element or an HTML string. It 649 returns whatever the function normally returns, or if the function 650 works in-place (and so returns None) it returns a serialized form 651 of the resulting document. 652 """

653 - def __init__(self, name, copy=False, source_class=HtmlMixin):

654 self.name = name 655 self.copy = copy 656 self.__doc__ = getattr(source_class, self.name).__doc__

657 - def __call__(self, doc, *args, **kw):

658 result_type = type(doc) 659 if isinstance(doc, basestring): 660 if 'copy' in kw: 661 raise TypeError( 662 "The keyword 'copy' can only be used with element inputs to %s, not a string input" % self.name) 663 doc = fromstring(doc, **kw) 664 else: 665 if 'copy' in kw: 666 make_a_copy = kw.pop('copy') 667 else: 668 make_a_copy = self.copy 669 if make_a_copy: 670 doc = copy.deepcopy(doc) 671 meth = getattr(doc, self.name) 672 result = meth(*args, **kw) 673 # FIXME: this None test is a bit sloppy 674 if result is None: 675 # Then return what we got in 676 return _transform_result(result_type, doc) 677 else: 678 return result

679 680 681 find_rel_links = _MethodFunc('find_rel_links', copy=False) 682 find_class = _MethodFunc('find_class', copy=False) 683 make_links_absolute = _MethodFunc('make_links_absolute', copy=True) 684 resolve_base_href = _MethodFunc('resolve_base_href', copy=True) 685 iterlinks = _MethodFunc('iterlinks', copy=False) 686 rewrite_links = _MethodFunc('rewrite_links', copy=True)

687 688 689 -class HtmlComment(etree.CommentBase, HtmlMixin):

690 pass

691

692 693 -class HtmlElement(etree.ElementBase, HtmlMixin):

694 # Override etree.ElementBase.cssselect() and set(), despite the MRO (FIXME: change base order?) 695 cssselect = HtmlMixin.cssselect 696 set = HtmlMixin.set

697

698 699 -class HtmlProcessingInstruction(etree.PIBase, HtmlMixin):

700 pass

701

702 703 -class HtmlEntity(etree.EntityBase, HtmlMixin):

704 pass

705

706 707 -class HtmlElementClassLookup(etree.CustomElementClassLookup):

708 """A lookup scheme for HTML Element classes. 709 710 To create a lookup instance with different Element classes, pass a tag 711 name mapping of Element classes in the ``classes`` keyword argument and/or 712 a tag name mapping of Mixin classes in the ``mixins`` keyword argument. 713 The special key '*' denotes a Mixin class that should be mixed into all 714 Element classes. 715 """ 716 _default_element_classes = {} 717

718 - def __init__(self, classes=None, mixins=None):

719 etree.CustomElementClassLookup.__init__(self) 720 if classes is None: 721 classes = self._default_element_classes.copy() 722 if mixins: 723 mixers = {} 724 for name, value in mixins: 725 if name == '*': 726 for n in classes.keys(): 727 mixers.setdefault(n, []).append(value) 728 else: 729 mixers.setdefault(name, []).append(value) 730 for name, mix_bases in mixers.items(): 731 cur = classes.get(name, HtmlElement) 732 bases = tuple(mix_bases + [cur]) 733 classes[name] = type(cur.__name__, bases, {}) 734 self._element_classes = classes

735

736 - def lookup(self, node_type, document, namespace, name):

737 if node_type == 'element': 738 return self._element_classes.get(name.lower(), HtmlElement) 739 elif node_type == 'comment': 740 return HtmlComment 741 elif node_type == 'PI': 742 return HtmlProcessingInstruction 743 elif node_type == 'entity': 744 return HtmlEntity 745 # Otherwise normal lookup 746 return None

747 748 749 ################################################################################ 750 # parsing 751 ################################################################################ 752 753 _looks_like_full_html_unicode = re.compile( 754 unicode(r'^\s*<(?:html|!doctype)'), re.I).match 755 _looks_like_full_html_bytes = re.compile( 756 r'^\s*<(?:html|!doctype)'.encode('ascii'), re.I).match

757 758 759 -def document_fromstring(html, parser=None, ensure_head_body=False, **kw):

760 if parser is None: 761 parser = html_parser 762 value = etree.fromstring(html, parser, **kw) 763 if value is None: 764 raise etree.ParserError( 765 "Document is empty") 766 if ensure_head_body and value.find('head') is None: 767 value.insert(0, Element('head')) 768 if ensure_head_body and value.find('body') is None: 769 value.append(Element('body')) 770 return value

771

772 773 -def fragments_fromstring(html, no_leading_text=False, base_url=None, 774 parser=None, **kw):

775 """Parses several HTML elements, returning a list of elements. 776 777 The first item in the list may be a string. 778 If no_leading_text is true, then it will be an error if there is 779 leading text, and it will always be a list of only elements. 780 781 base_url will set the document's base_url attribute 782 (and the tree's docinfo.URL). 783 """ 784 if parser is None: 785 parser = html_parser 786 # FIXME: check what happens when you give html with a body, head, etc. 787 if isinstance(html, bytes): 788 if not _looks_like_full_html_bytes(html): 789 # can't use %-formatting in early Py3 versions 790 html = ('<html><body>'.encode('ascii') + html + 791 '</body></html>'.encode('ascii')) 792 else: 793 if not _looks_like_full_html_unicode(html): 794 html = '<html><body>%s</body></html>' % html 795 doc = document_fromstring(html, parser=parser, base_url=base_url, **kw) 796 assert _nons(doc.tag) == 'html' 797 bodies = [e for e in doc if _nons(e.tag) == 'body'] 798 assert len(bodies) == 1, ("too many bodies: %r in %r" % (bodies, html)) 799 body = bodies[0] 800 elements = [] 801 if no_leading_text and body.text and body.text.strip(): 802 raise etree.ParserError( 803 "There is leading text: %r" % body.text) 804 if body.text and body.text.strip(): 805 elements.append(body.text) 806 elements.extend(body) 807 # FIXME: removing the reference to the parent artificial document 808 # would be nice 809 return elements

810

811 812 -def fragment_fromstring(html, create_parent=False, base_url=None, 813 parser=None, **kw):

814 """ 815 Parses a single HTML element; it is an error if there is more than 816 one element, or if anything but whitespace precedes or follows the 817 element. 818 819 If ``create_parent`` is true (or is a tag name) then a parent node 820 will be created to encapsulate the HTML in a single element. In this 821 case, leading or trailing text is also allowed, as are multiple elements 822 as result of the parsing. 823 824 Passing a ``base_url`` will set the document's ``base_url`` attribute 825 (and the tree's docinfo.URL). 826 """ 827 if parser is None: 828 parser = html_parser 829 830 accept_leading_text = bool(create_parent) 831 832 elements = fragments_fromstring( 833 html, parser=parser, no_leading_text=not accept_leading_text, 834 base_url=base_url, **kw) 835 836 if create_parent: 837 if not isinstance(create_parent, basestring): 838 create_parent = 'div' 839 new_root = Element(create_parent) 840 if elements: 841 if isinstance(elements[0], basestring): 842 new_root.text = elements[0] 843 del elements[0] 844 new_root.extend(elements) 845 return new_root 846 847 if not elements: 848 raise etree.ParserError('No elements found') 849 if len(elements) > 1: 850 raise etree.ParserError( 851 "Multiple elements found (%s)" 852 % ', '.join([_element_name(e) for e in elements])) 853 el = elements[0] 854 if el.tail and el.tail.strip(): 855 raise etree.ParserError( 856 "Element followed by text: %r" % el.tail) 857 el.tail = None 858 return el

859

860 861 -def fromstring(html, base_url=None, parser=None, **kw):

862 """ 863 Parse the html, returning a single element/document. 864 865 This tries to minimally parse the chunk of text, without knowing if it 866 is a fragment or a document. 867 868 base_url will set the document's base_url attribute (and the tree's docinfo.URL) 869 """ 870 if parser is None: 871 parser = html_parser 872 if isinstance(html, bytes): 873 is_full_html = _looks_like_full_html_bytes(html) 874 else: 875 is_full_html = _looks_like_full_html_unicode(html) 876 doc = document_fromstring(html, parser=parser, base_url=base_url, **kw) 877 if is_full_html: 878 return doc 879 # otherwise, lets parse it out... 880 bodies = doc.findall('body') 881 if not bodies: 882 bodies = doc.findall('{%s}body' % XHTML_NAMESPACE) 883 if bodies: 884 body = bodies[0] 885 if len(bodies) > 1: 886 # Somehow there are multiple bodies, which is bad, but just 887 # smash them into one body 888 for other_body in bodies[1:]: 889 if other_body.text: 890 if len(body): 891 body[-1].tail = (body[-1].tail or '') + other_body.text 892 else: 893 body.text = (body.text or '') + other_body.text 894 body.extend(other_body) 895 # We'll ignore tail 896 # I guess we are ignoring attributes too 897 other_body.drop_tree() 898 else: 899 body = None 900 heads = doc.findall('head') 901 if not heads: 902 heads = doc.findall('{%s}head' % XHTML_NAMESPACE) 903 if heads: 904 # Well, we have some sort of structure, so lets keep it all 905 head = heads[0] 906 if len(heads) > 1: 907 for other_head in heads[1:]: 908 head.extend(other_head) 909 # We don't care about text or tail in a head 910 other_head.drop_tree() 911 return doc 912 if body is None: 913 return doc 914 if (len(body) == 1 and (not body.text or not body.text.strip()) 915 and (not body[-1].tail or not body[-1].tail.strip())): 916 # The body has just one element, so it was probably a single 917 # element passed in 918 return body[0] 919 # Now we have a body which represents a bunch of tags which have the 920 # content that was passed in. We will create a fake container, which 921 # is the body tag, except <body> implies too much structure. 922 if _contains_block_level_tag(body): 923 body.tag = 'div' 924 else: 925 body.tag = 'span' 926 return body

927

928 929 -def parse(filename_or_url, parser=None, base_url=None, **kw):

930 """ 931 Parse a filename, URL, or file-like object into an HTML document 932 tree. Note: this returns a tree, not an element. Use 933 ``parse(...).getroot()`` to get the document root. 934 935 You can override the base URL with the ``base_url`` keyword. This 936 is most useful when parsing from a file-like object. 937 """ 938 if parser is None: 939 parser = html_parser 940 return etree.parse(filename_or_url, parser, base_url=base_url, **kw)

941

942 943 -def _contains_block_level_tag(el):

944 # FIXME: I could do this with XPath, but would that just be 945 # unnecessarily slow? 946 for el in el.iter(etree.Element): 947 if _nons(el.tag) in defs.block_tags: 948 return True 949 return False

950

951 952 -def _element_name(el):

953 if isinstance(el, etree.CommentBase): 954 return 'comment' 955 elif isinstance(el, basestring): 956 return 'string' 957 else: 958 return _nons(el.tag)

959

960 961 ################################################################################ 962 # form handling 963 ################################################################################ 964 965 -class FormElement(HtmlElement):

966 """ 967 Represents a <form> element. 968 """ 969 970 @property

971 - def inputs(self):

972 """ 973 Returns an accessor for all the input elements in the form. 974 975 See `InputGetter` for more information about the object. 976 """ 977 return InputGetter(self)

978 979 @property

980 - def fields(self):

981 """ 982 Dictionary-like object that represents all the fields in this 983 form. You can set values in this dictionary to effect the 984 form. 985 """ 986 return FieldsDict(self.inputs)

987 988 @fields.setter

989 - def fields(self, value):

990 fields = self.fields 991 prev_keys = fields.keys() 992 for key, value in value.items(): 993 if key in prev_keys: 994 prev_keys.remove(key) 995 fields[key] = value 996 for key in prev_keys: 997 if key is None: 998 # Case of an unnamed input; these aren't really 999 # expressed in form_values() anyway. 1000 continue 1001 fields[key] = None

1002

1003 - def _name(self):

1004 if self.get('name'): 1005 return self.get('name') 1006 elif self.get('id'): 1007 return '#' + self.get('id') 1008 iter_tags = self.body.iter 1009 forms = list(iter_tags('form')) 1010 if not forms: 1011 forms = list(iter_tags('{%s}form' % XHTML_NAMESPACE)) 1012 return str(forms.index(self))

1013

1014 - def form_values(self):

1015 """ 1016 Return a list of tuples of the field values for the form. 1017 This is suitable to be passed to ``urllib.urlencode()``. 1018 """ 1019 results = [] 1020 for el in self.inputs: 1021 name = el.name 1022 if not name or 'disabled' in el.attrib: 1023 continue 1024 tag = _nons(el.tag) 1025 if tag == 'textarea': 1026 results.append((name, el.value)) 1027 elif tag == 'select': 1028 value = el.value 1029 if el.multiple: 1030 for v in value: 1031 results.append((name, v)) 1032 elif value is not None: 1033 results.append((name, el.value)) 1034 else: 1035 assert tag == 'input', ( 1036 "Unexpected tag: %r" % el) 1037 if el.checkable and not el.checked: 1038 continue 1039 if el.type in ('submit', 'image', 'reset', 'file'): 1040 continue 1041 value = el.value 1042 if value is not None: 1043 results.append((name, el.value)) 1044 return results

1045 1046 @property

1047 - def action(self):

1048 """ 1049 Get/set the form's ``action`` attribute. 1050 """ 1051 base_url = self.base_url 1052 action = self.get('action') 1053 if base_url and action is not None: 1054 return urljoin(base_url, action) 1055 else: 1056 return action

1057 1058 @action.setter

1059 - def action(self, value):

1060 self.set('action', value)

1061 1062 @action.deleter

1063 - def action(self):

1064 attrib = self.attrib 1065 if 'action' in attrib: 1066 del attrib['action']

1067 1068 @property

1069 - def method(self):

1070 """ 1071 Get/set the form's method. Always returns a capitalized 1072 string, and defaults to ``'GET'`` 1073 """ 1074 return self.get('method', 'GET').upper()

1075 1076 @method.setter

1077 - def method(self, value):

1078 self.set('method', value.upper())

1079 1080 1081 HtmlElementClassLookup._default_element_classes['form'] = FormElement

1082 1083 1084 -def submit_form(form, extra_values=None, open_http=None):

1085 """ 1086 Helper function to submit a form. Returns a file-like object, as from 1087 ``urllib.urlopen()``. This object also has a ``.geturl()`` function, 1088 which shows the URL if there were any redirects. 1089 1090 You can use this like:: 1091 1092 form = doc.forms[0] 1093 form.inputs['foo'].value = 'bar' # etc 1094 response = form.submit() 1095 doc = parse(response) 1096 doc.make_links_absolute(response.geturl()) 1097 1098 To change the HTTP requester, pass a function as ``open_http`` keyword 1099 argument that opens the URL for you. The function must have the following 1100 signature:: 1101 1102 open_http(method, URL, values) 1103 1104 The action is one of 'GET' or 'POST', the URL is the target URL as a 1105 string, and the values are a sequence of ``(name, value)`` tuples with the 1106 form data. 1107 """ 1108 values = form.form_values() 1109 if extra_values: 1110 if hasattr(extra_values, 'items'): 1111 extra_values = extra_values.items() 1112 values.extend(extra_values) 1113 if open_http is None: 1114 open_http = open_http_urllib 1115 if form.action: 1116 url = form.action 1117 else: 1118 url = form.base_url 1119 return open_http(form.method, url, values)

1120

1121 1122 -def open_http_urllib(method, url, values):

1123 if not url: 1124 raise ValueError("cannot submit, no URL provided") 1125 ## FIXME: should test that it's not a relative URL or something 1126 try: 1127 from urllib import urlencode, urlopen 1128 except ImportError: # Python 3 1129 from urllib.request import urlopen 1130 from urllib.parse import urlencode 1131 if method == 'GET': 1132 if '?' in url: 1133 url += '&' 1134 else: 1135 url += '?' 1136 url += urlencode(values) 1137 data = None 1138 else: 1139 data = urlencode(values) 1140 if not isinstance(data, bytes): 1141 data = data.encode('ASCII') 1142 return urlopen(url, data)

1143

1144 1145 -class FieldsDict(MutableMapping):

1146

1147 - def __init__(self, inputs):

1148 self.inputs = inputs

1149 - def __getitem__(self, item):

1150 return self.inputs[item].value

1151 - def __setitem__(self, item, value):

1152 self.inputs[item].value = value

1153 - def __delitem__(self, item):

1154 raise KeyError( 1155 "You cannot remove keys from ElementDict")

1156 - def keys(self):

1157 return self.inputs.keys()

1158 - def __contains__(self, item):

1159 return item in self.inputs

1160 - def __iter__(self):

1161 return iter(self.inputs.keys())

1162 - def __len__(self):

1163 return len(self.inputs)

1164

1165 - def __repr__(self):

1166 return '<%s for form %s>' % ( 1167 self.__class__.__name__, 1168 self.inputs.form._name())

1169

1170 1171 -class InputGetter(object):

1172 1173 """ 1174 An accessor that represents all the input fields in a form. 1175 1176 You can get fields by name from this, with 1177 ``form.inputs['field_name']``. If there are a set of checkboxes 1178 with the same name, they are returned as a list (a `CheckboxGroup` 1179 which also allows value setting). Radio inputs are handled 1180 similarly. 1181 1182 You can also iterate over this to get all input elements. This 1183 won't return the same thing as if you get all the names, as 1184 checkboxes and radio elements are returned individually. 1185 """ 1186 1187 _name_xpath = etree.XPath(".//*[@name = $name and (local-name(.) = 'select' or local-name(.) = 'input' or local-name(.) = 'textarea')]") 1188 _all_xpath = etree.XPath(".//*[local-name() = 'select' or local-name() = 'input' or local-name() = 'textarea']") 1189

1190 - def __init__(self, form):

1191 self.form = form

1192

1193 - def __repr__(self):

1194 return '<%s for form %s>' % ( 1195 self.__class__.__name__, 1196 self.form._name())

1197 1198 ## FIXME: there should be more methods, and it's unclear if this is 1199 ## a dictionary-like object or list-like object 1200

1201 - def __getitem__(self, name):

1202 results = self._name_xpath(self.form, name=name) 1203 if results: 1204 type = results[0].get('type') 1205 if type == 'radio' and len(results) > 1: 1206 group = RadioGroup(results) 1207 group.name = name 1208 return group 1209 elif type == 'checkbox' and len(results) > 1: 1210 group = CheckboxGroup(results) 1211 group.name = name 1212 return group 1213 else: 1214 # I don't like throwing away elements like this 1215 return results[0] 1216 else: 1217 raise KeyError( 1218 "No input element with the name %r" % name)

1219

1220 - def __contains__(self, name):

1221 results = self._name_xpath(self.form, name=name) 1222 return bool(results)

1223

1224 - def keys(self):

1225 names = set() 1226 for el in self: 1227 names.add(el.name) 1228 if None in names: 1229 names.remove(None) 1230 return list(names)

1231

1232 - def __iter__(self):

1233 ## FIXME: kind of dumb to turn a list into an iterator, only 1234 ## to have it likely turned back into a list again :( 1235 return iter(self._all_xpath(self.form))

1236

1237 1238 -class InputMixin(object):

1239 """ 1240 Mix-in for all input elements (input, select, and textarea) 1241 """ 1242 @property

1243 - def name(self):

1244 """ 1245 Get/set the name of the element 1246 """ 1247 return self.get('name')

1248 1249 @name.setter

1250 - def name(self, value):

1251 self.set('name', value)

1252 1253 @name.deleter

1254 - def name(self):

1255 attrib = self.attrib 1256 if 'name' in attrib: 1257 del attrib['name']

1258

1259 - def __repr__(self):

1260 type_name = getattr(self, 'type', None) 1261 if type_name: 1262 type_name = ' type=%r' % type_name 1263 else: 1264 type_name = '' 1265 return '<%s %x name=%r%s>' % ( 1266 self.__class__.__name__, id(self), self.name, type_name)

1267

1268 1269 -class TextareaElement(InputMixin, HtmlElement):

1270 """ 1271 ``<textarea>`` element. You can get the name with ``.name`` and 1272 get/set the value with ``.value`` 1273 """ 1274 @property

1275 - def value(self):

1276 """ 1277 Get/set the value (which is the contents of this element) 1278 """ 1279 content = self.text or '' 1280 if self.tag.startswith("{%s}" % XHTML_NAMESPACE): 1281 serialisation_method = 'xml' 1282 else: 1283 serialisation_method = 'html' 1284 for el in self: 1285 # it's rare that we actually get here, so let's not use ''.join() 1286 content += etree.tostring( 1287 el, method=serialisation_method, encoding='unicode') 1288 return content

1289 1290 @value.setter

1291 - def value(self, value):

1292 del self[:] 1293 self.text = value

1294 1295 @value.deleter

1296 - def value(self):

1297 self.text = '' 1298 del self[:]

1299 1300 1301 HtmlElementClassLookup._default_element_classes['textarea'] = TextareaElement

1302 1303 1304 -class SelectElement(InputMixin, HtmlElement):

1305 """ 1306 ``<select>`` element. You can get the name with ``.name``. 1307 1308 ``.value`` will be the value of the selected option, unless this 1309 is a multi-select element (``<select multiple>``), in which case 1310 it will be a set-like object. In either case ``.value_options`` 1311 gives the possible values. 1312 1313 The boolean attribute ``.multiple`` shows if this is a 1314 multi-select. 1315 """ 1316 @property

1317 - def value(self):

1318 """ 1319 Get/set the value of this select (the selected option). 1320 1321 If this is a multi-select, this is a set-like object that 1322 represents all the selected options. 1323 """ 1324 if self.multiple: 1325 return MultipleSelectOptions(self) 1326 for el in _options_xpath(self): 1327 if el.get('selected') is not None: 1328 value = el.get('value') 1329 if value is None: 1330 value = (el.text or '').strip() 1331 return value 1332 return None

1333 1334 @value.setter

1335 - def value(self, value):

1336 if self.multiple: 1337 if isinstance(value, basestring): 1338 raise TypeError("You must pass in a sequence") 1339 values = self.value 1340 values.clear() 1341 values.update(value) 1342 return 1343 checked_option = None 1344 if value is not None: 1345 for el in _options_xpath(self): 1346 opt_value = el.get('value') 1347 if opt_value is None: 1348 opt_value = (el.text or '').strip() 1349 if opt_value == value: 1350 checked_option = el 1351 break 1352 else: 1353 raise ValueError( 1354 "There is no option with the value of %r" % value) 1355 for el in _options_xpath(self): 1356 if 'selected' in el.attrib: 1357 del el.attrib['selected'] 1358 if checked_option is not None: 1359 checked_option.set('selected', '')

1360 1361 @value.deleter

1362 - def value(self):

1363 # FIXME: should del be allowed at all? 1364 if self.multiple: 1365 self.value.clear() 1366 else: 1367 self.value = None

1368 1369 @property

1370 - def value_options(self):

1371 """ 1372 All the possible values this select can have (the ``value`` 1373 attribute of all the ``<option>`` elements. 1374 """ 1375 options = [] 1376 for el in _options_xpath(self): 1377 value = el.get('value') 1378 if value is None: 1379 value = (el.text or '').strip() 1380 options.append(value) 1381 return options

1382 1383 @property

1384 - def multiple(self):

1385 """ 1386 Boolean attribute: is there a ``multiple`` attribute on this element. 1387 """ 1388 return 'multiple' in self.attrib

1389 1390 @multiple.setter

1391 - def multiple(self, value):

1392 if value: 1393 self.set('multiple', '') 1394 elif 'multiple' in self.attrib: 1395 del self.attrib['multiple']

1396 1397 1398 HtmlElementClassLookup._default_element_classes['select'] = SelectElement

1399 1400 1401 -class MultipleSelectOptions(SetMixin):

1402 """ 1403 Represents all the selected options in a ``<select multiple>`` element. 1404 1405 You can add to this set-like option to select an option, or remove 1406 to unselect the option. 1407 """ 1408

1409 - def __init__(self, select):

1410 self.select = select

1411 1412 @property

1413 - def options(self):

1414 """ 1415 Iterator of all the ``<option>`` elements. 1416 """ 1417 return iter(_options_xpath(self.select))

1418

1419 - def __iter__(self):

1420 for option in self.options: 1421 if 'selected' in option.attrib: 1422 opt_value = option.get('value') 1423 if opt_value is None: 1424 opt_value = (option.text or '').strip() 1425 yield opt_value

1426

1427 - def add(self, item):

1428 for option in self.options: 1429 opt_value = option.get('value') 1430 if opt_value is None: 1431 opt_value = (option.text or '').strip() 1432 if opt_value == item: 1433 option.set('selected', '') 1434 break 1435 else: 1436 raise ValueError( 1437 "There is no option with the value %r" % item)

1438

1439 - def remove(self, item):

1440 for option in self.options: 1441 opt_value = option.get('value') 1442 if opt_value is None: 1443 opt_value = (option.text or '').strip() 1444 if opt_value == item: 1445 if 'selected' in option.attrib: 1446 del option.attrib['selected'] 1447 else: 1448 raise ValueError( 1449 "The option %r is not currently selected" % item) 1450 break 1451 else: 1452 raise ValueError( 1453 "There is not option with the value %r" % item)

1454

1455 - def __repr__(self):

1456 return '<%s {%s} for select name=%r>' % ( 1457 self.__class__.__name__, 1458 ', '.join([repr(v) for v in self]), 1459 self.select.name)

1460

1461 1462 -class RadioGroup(list):

1463 """ 1464 This object represents several ``<input type=radio>`` elements 1465 that have the same name. 1466 1467 You can use this like a list, but also use the property 1468 ``.value`` to check/uncheck inputs. Also you can use 1469 ``.value_options`` to get the possible values. 1470 """ 1471 @property

1472 - def value(self):

1473 """ 1474 Get/set the value, which checks the radio with that value (and 1475 unchecks any other value). 1476 """ 1477 for el in self: 1478 if 'checked' in el.attrib: 1479 return el.get('value') 1480 return None

1481 1482 @value.setter

1483 - def value(self, value):

1484 checked_option = None 1485 if value is not None: 1486 for el in self: 1487 if el.get('value') == value: 1488 checked_option = el 1489 break 1490 else: 1491 raise ValueError("There is no radio input with the value %r" % value) 1492 for el in self: 1493 if 'checked' in el.attrib: 1494 del el.attrib['checked'] 1495 if checked_option is not None: 1496 checked_option.set('checked', '')

1497 1498 @value.deleter

1499 - def value(self):

1500 self.value = None

1501 1502 @property

1503 - def value_options(self):

1504 """ 1505 Returns a list of all the possible values. 1506 """ 1507 return [el.get('value') for el in self]

1508

1509 - def __repr__(self):

1510 return '%s(%s)' % ( 1511 self.__class__.__name__, 1512 list.__repr__(self))

1513

1514 1515 -class CheckboxGroup(list):

1516 """ 1517 Represents a group of checkboxes (``<input type=checkbox>``) that 1518 have the same name. 1519 1520 In addition to using this like a list, the ``.value`` attribute 1521 returns a set-like object that you can add to or remove from to 1522 check and uncheck checkboxes. You can also use ``.value_options`` 1523 to get the possible values. 1524 """ 1525 @property

1526 - def value(self):

1527 """ 1528 Return a set-like object that can be modified to check or 1529 uncheck individual checkboxes according to their value. 1530 """ 1531 return CheckboxValues(self)

1532 1533 @value.setter

1534 - def value(self, value):

1535 values = self.value 1536 values.clear() 1537 if not hasattr(value, '__iter__'): 1538 raise ValueError( 1539 "A CheckboxGroup (name=%r) must be set to a sequence (not %r)" 1540 % (self[0].name, value)) 1541 values.update(value)

1542 1543 @value.deleter

1544 - def value(self):

1545 self.value.clear()

1546 1547 @property

1548 - def value_options(self):

1549 """ 1550 Returns a list of all the possible values. 1551 """ 1552 return [el.get('value') for el in self]

1553

1554 - def __repr__(self):

1555 return '%s(%s)' % ( 1556 self.__class__.__name__, list.__repr__(self))

1557

1558 1559 -class CheckboxValues(SetMixin):

1560 """ 1561 Represents the values of the checked checkboxes in a group of 1562 checkboxes with the same name. 1563 """ 1564

1565 - def __init__(self, group):

1566 self.group = group

1567

1568 - def __iter__(self):

1569 return iter([ 1570 el.get('value') 1571 for el in self.group 1572 if 'checked' in el.attrib])

1573

1574 - def add(self, value):

1575 for el in self.group: 1576 if el.get('value') == value: 1577 el.set('checked', '') 1578 break 1579 else: 1580 raise KeyError("No checkbox with value %r" % value)

1581

1582 - def remove(self, value):

1583 for el in self.group: 1584 if el.get('value') == value: 1585 if 'checked' in el.attrib: 1586 del el.attrib['checked'] 1587 else: 1588 raise KeyError( 1589 "The checkbox with value %r was already unchecked" % value) 1590 break 1591 else: 1592 raise KeyError( 1593 "No checkbox with value %r" % value)

1594

1595 - def __repr__(self):

1596 return '<%s {%s} for checkboxes name=%r>' % ( 1597 self.__class__.__name__, 1598 ', '.join([repr(v) for v in self]), 1599 self.group.name)

1600

1601 1602 -class InputElement(InputMixin, HtmlElement):

1603 """ 1604 Represents an ``<input>`` element. 1605 1606 You can get the type with ``.type`` (which is lower-cased and 1607 defaults to ``'text'``). 1608 1609 Also you can get and set the value with ``.value`` 1610 1611 Checkboxes and radios have the attribute ``input.checkable == 1612 True`` (for all others it is false) and a boolean attribute 1613 ``.checked``. 1614 1615 """ 1616 1617 ## FIXME: I'm a little uncomfortable with the use of .checked 1618 @property

1619 - def value(self):

1620 """ 1621 Get/set the value of this element, using the ``value`` attribute. 1622 1623 Also, if this is a checkbox and it has no value, this defaults 1624 to ``'on'``. If it is a checkbox or radio that is not 1625 checked, this returns None. 1626 """ 1627 if self.checkable: 1628 if self.checked: 1629 return self.get('value') or 'on' 1630 else: 1631 return None 1632 return self.get('value')

1633 1634 @value.setter

1635 - def value(self, value):

1636 if self.checkable: 1637 if not value: 1638 self.checked = False 1639 else: 1640 self.checked = True 1641 if isinstance(value, basestring): 1642 self.set('value', value) 1643 else: 1644 self.set('value', value)

1645 1646 @value.deleter

1647 - def value(self):

1648 if self.checkable: 1649 self.checked = False 1650 else: 1651 if 'value' in self.attrib: 1652 del self.attrib['value']

1653 1654 @property

1655 - def type(self):

1656 """ 1657 Return the type of this element (using the type attribute). 1658 """ 1659 return self.get('type', 'text').lower()

1660 1661 @type.setter

1662 - def type(self, value):

1663 self.set('type', value)

1664 1665 @property

1666 - def checkable(self):

1667 """ 1668 Boolean: can this element be checked? 1669 """ 1670 return self.type in ('checkbox', 'radio')

1671 1672 @property

1673 - def checked(self):

1674 """ 1675 Boolean attribute to get/set the presence of the ``checked`` 1676 attribute. 1677 1678 You can only use this on checkable input types. 1679 """ 1680 if not self.checkable: 1681 raise AttributeError('Not a checkable input type') 1682 return 'checked' in self.attrib

1683 1684 @checked.setter

1685 - def checked(self, value):

1686 if not self.checkable: 1687 raise AttributeError('Not a checkable input type') 1688 if value: 1689 self.set('checked', '') 1690 else: 1691 attrib = self.attrib 1692 if 'checked' in attrib: 1693 del attrib['checked']

1694 1695 1696 HtmlElementClassLookup._default_element_classes['input'] = InputElement

1697 1698 1699 -class LabelElement(HtmlElement):

1700 """ 1701 Represents a ``<label>`` element. 1702 1703 Label elements are linked to other elements with their ``for`` 1704 attribute. You can access this element with ``label.for_element``. 1705 """ 1706 @property

1707 - def for_element(self):

1708 """ 1709 Get/set the element this label points to. Return None if it 1710 can't be found. 1711 """ 1712 id = self.get('for') 1713 if not id: 1714 return None 1715 return self.body.get_element_by_id(id)

1716 1717 @for_element.setter

1718 - def for_element(self, other):

1719 id = other.get('id') 1720 if not id: 1721 raise TypeError( 1722 "Element %r has no id attribute" % other) 1723 self.set('for', id)

1724 1725 @for_element.deleter

1726 - def for_element(self):

1727 attrib = self.attrib 1728 if 'id' in attrib: 1729 del attrib['id']

1730 1731 1732 HtmlElementClassLookup._default_element_classes['label'] = LabelElement

1733 1734 1735 ############################################################ 1736 ## Serialization 1737 ############################################################ 1738 1739 -def html_to_xhtml(html):

1740 """Convert all tags in an HTML tree to XHTML by moving them to the 1741 XHTML namespace. 1742 """ 1743 try: 1744 html = html.getroot() 1745 except AttributeError: 1746 pass 1747 prefix = "{%s}" % XHTML_NAMESPACE 1748 for el in html.iter(etree.Element): 1749 tag = el.tag 1750 if tag[0] != '{': 1751 el.tag = prefix + tag

1752

1753 1754 -def xhtml_to_html(xhtml):

1755 """Convert all tags in an XHTML tree to HTML by removing their 1756 XHTML namespace. 1757 """ 1758 try: 1759 xhtml = xhtml.getroot() 1760 except AttributeError: 1761 pass 1762 prefix = "{%s}" % XHTML_NAMESPACE 1763 prefix_len = len(prefix) 1764 for el in xhtml.iter(prefix + "*"): 1765 el.tag = el.tag[prefix_len:]

1766 1767 1768 # This isn't a general match, but it's a match for what libxml2 1769 # specifically serialises: 1770 __str_replace_meta_content_type = re.compile( 1771 r'<meta http-equiv="Content-Type"[^>]*>').sub 1772 __bytes_replace_meta_content_type = re.compile( 1773 r'<meta http-equiv="Content-Type"[^>]*>'.encode('ASCII')).sub

1774 1775 1776 -def tostring(doc, pretty_print=False, include_meta_content_type=False, 1777 encoding=None, method="html", with_tail=True, doctype=None):

1778 """Return an HTML string representation of the document. 1779 1780 Note: if include_meta_content_type is true this will create a 1781 ``<meta http-equiv="Content-Type" ...>`` tag in the head; 1782 regardless of the value of include_meta_content_type any existing 1783 ``<meta http-equiv="Content-Type" ...>`` tag will be removed 1784 1785 The ``encoding`` argument controls the output encoding (defauts to 1786 ASCII, with &#...; character references for any characters outside 1787 of ASCII). Note that you can pass the name ``'unicode'`` as 1788 ``encoding`` argument to serialise to a Unicode string. 1789 1790 The ``method`` argument defines the output method. It defaults to 1791 'html', but can also be 'xml' for xhtml output, or 'text' to 1792 serialise to plain text without markup. 1793 1794 To leave out the tail text of the top-level element that is being 1795 serialised, pass ``with_tail=False``. 1796 1797 The ``doctype`` option allows passing in a plain string that will 1798 be serialised before the XML tree. Note that passing in non 1799 well-formed content here will make the XML output non well-formed. 1800 Also, an existing doctype in the document tree will not be removed 1801 when serialising an ElementTree instance. 1802 1803 Example:: 1804 1805 >>> from lxml import html 1806 >>> root = html.fragment_fromstring('<p>Hello<br>world!</p>') 1807 1808 >>> html.tostring(root) 1809 b'<p>Hello<br>world!</p>' 1810 >>> html.tostring(root, method='html') 1811 b'<p>Hello<br>world!</p>' 1812 1813 >>> html.tostring(root, method='xml') 1814 b'<p>Hello<br/>world!</p>' 1815 1816 >>> html.tostring(root, method='text') 1817 b'Helloworld!' 1818 1819 >>> html.tostring(root, method='text', encoding='unicode') 1820 u'Helloworld!' 1821 1822 >>> root = html.fragment_fromstring('<div><p>Hello<br>world!</p>TAIL</div>') 1823 >>> html.tostring(root[0], method='text', encoding='unicode') 1824 u'Helloworld!TAIL' 1825 1826 >>> html.tostring(root[0], method='text', encoding='unicode', with_tail=False) 1827 u'Helloworld!' 1828 1829 >>> doc = html.document_fromstring('<p>Hello<br>world!</p>') 1830 >>> html.tostring(doc, method='html', encoding='unicode') 1831 u'<html><body><p>Hello<br>world!</p></body></html>' 1832 1833 >>> print(html.tostring(doc, method='html', encoding='unicode', 1834 ... doctype='<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN"' 1835 ... ' "http://www.w3.org/TR/html4/strict.dtd">')) 1836 <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd"> 1837 <html><body><p>Hello<br>world!</p></body></html> 1838 """ 1839 html = etree.tostring(doc, method=method, pretty_print=pretty_print, 1840 encoding=encoding, with_tail=with_tail, 1841 doctype=doctype) 1842 if method == 'html' and not include_meta_content_type: 1843 if isinstance(html, str): 1844 html = __str_replace_meta_content_type('', html) 1845 else: 1846 html = __bytes_replace_meta_content_type(bytes(), html) 1847 return html

1848 1849 1850 tostring.__doc__ = __fix_docstring(tostring.__doc__)

1851 1852 1853 -def open_in_browser(doc, encoding=None):

1854 """ 1855 Open the HTML document in a web browser, saving it to a temporary 1856 file to open it. Note that this does not delete the file after 1857 use. This is mainly meant for debugging. 1858 """ 1859 import os 1860 import webbrowser 1861 import tempfile 1862 if not isinstance(doc, etree._ElementTree): 1863 doc = etree.ElementTree(doc) 1864 handle, fn = tempfile.mkstemp(suffix='.html') 1865 f = os.fdopen(handle, 'wb') 1866 try: 1867 doc.write(f, method="html", encoding=encoding or doc.docinfo.encoding or "UTF-8") 1868 finally: 1869 # we leak the file itself here, but we should at least close it 1870 f.close() 1871 url = 'file://' + fn.replace(os.path.sep, '/') 1872 print(url) 1873 webbrowser.open(url)

1874

1875 1876 ################################################################################ 1877 # configure Element class lookup 1878 ################################################################################ 1879 1880 -class HTMLParser(etree.HTMLParser):

1881 """An HTML parser that is configured to return lxml.html Element 1882 objects. 1883 """

1884 - def __init__(self, **kwargs):

1885 super(HTMLParser, self).__init__(**kwargs) 1886 self.set_element_class_lookup(HtmlElementClassLookup())

1887

1888 1889 -class XHTMLParser(etree.XMLParser):

1890 """An XML parser that is configured to return lxml.html Element 1891 objects. 1892 1893 Note that this parser is not really XHTML aware unless you let it 1894 load a DTD that declares the HTML entities. To do this, make sure 1895 you have the XHTML DTDs installed in your catalogs, and create the 1896 parser like this:: 1897 1898 >>> parser = XHTMLParser(load_dtd=True) 1899 1900 If you additionally want to validate the document, use this:: 1901 1902 >>> parser = XHTMLParser(dtd_validation=True) 1903 1904 For catalog support, see http://www.xmlsoft.org/catalog.html. 1905 """

1906 - def __init__(self, **kwargs):

1907 super(XHTMLParser, self).__init__(**kwargs) 1908 self.set_element_class_lookup(HtmlElementClassLookup())

1909

1910 1911 -def Element(*args, **kw):

1912 """Create a new HTML Element. 1913 1914 This can also be used for XHTML documents. 1915 """ 1916 v = html_parser.makeelement(*args, **kw) 1917 return v

1918 1919 1920 html_parser = HTMLParser() 1921 xhtml_parser = XHTMLParser() 1922

Source Code for Package lxml.html