Package lxml :: Package html :: Module clean
[hide private]
[frames] | no frames]

Source Code for Module lxml.html.clean

  1  # cython: language_level=3str 
  2   
  3  """A cleanup tool for HTML. 
  4   
  5  Removes unwanted tags and content.  See the `Cleaner` class for 
  6  details. 
  7  """ 
  8   
  9  from __future__ import absolute_import 
 10   
 11  import re 
 12  import copy 
 13  try: 
 14      from urlparse import urlsplit 
 15      from urllib import unquote_plus 
 16  except ImportError: 
 17      # Python 3 
 18      from urllib.parse import urlsplit, unquote_plus 
 19  from lxml import etree 
 20  from lxml.html import defs 
 21  from lxml.html import fromstring, XHTML_NAMESPACE 
 22  from lxml.html import xhtml_to_html, _transform_result 
 23   
 24  try: 
 25      unichr 
 26  except NameError: 
 27      # Python 3 
 28      unichr = chr 
 29  try: 
 30      unicode 
 31  except NameError: 
 32      # Python 3 
 33      unicode = str 
 34  try: 
 35      basestring 
 36  except NameError: 
 37      basestring = (str, bytes) 
 38   
 39   
 40  __all__ = ['clean_html', 'clean', 'Cleaner', 'autolink', 'autolink_html', 
 41             'word_break', 'word_break_html'] 
 42   
 43  # Look at http://code.sixapart.com/trac/livejournal/browser/trunk/cgi-bin/cleanhtml.pl 
 44  #   Particularly the CSS cleaning; most of the tag cleaning is integrated now 
 45  # I have multiple kinds of schemes searched; but should schemes be 
 46  #   whitelisted instead? 
 47  # max height? 
 48  # remove images?  Also in CSS?  background attribute? 
 49  # Some way to whitelist object, iframe, etc (e.g., if you want to 
 50  #   allow *just* embedded YouTube movies) 
 51  # Log what was deleted and why? 
 52  # style="behavior: ..." might be bad in IE? 
 53  # Should we have something for just <meta http-equiv>?  That's the worst of the 
 54  #   metas. 
 55  # UTF-7 detections?  Example: 
 56  #     <HEAD><META HTTP-EQUIV="CONTENT-TYPE" CONTENT="text/html; charset=UTF-7"> </HEAD>+ADw-SCRIPT+AD4-alert('XSS');+ADw-/SCRIPT+AD4- 
 57  #   you don't always have to have the charset set, if the page has no charset 
 58  #   and there's UTF7-like code in it. 
 59  # Look at these tests: http://htmlpurifier.org/live/smoketests/xssAttacks.php 
 60   
 61   
 62  # This is an IE-specific construct you can have in a stylesheet to 
 63  # run some Javascript: 
 64  _css_javascript_re = re.compile( 
 65      r'expression\s*\(.*?\)', re.S|re.I) 
 66   
 67  # Do I have to worry about @\nimport? 
 68  _css_import_re = re.compile( 
 69      r'@\s*import', re.I) 
 70   
 71  # All kinds of schemes besides just javascript: that can cause 
 72  # execution: 
 73  _is_image_dataurl = re.compile( 
 74      r'^data:image/.+;base64', re.I).search 
 75  _is_possibly_malicious_scheme = re.compile( 
 76      r'(?:javascript|jscript|livescript|vbscript|data|about|mocha):', 
 77      re.I).search 
78 -def _is_javascript_scheme(s):
79 if _is_image_dataurl(s): 80 return None 81 return _is_possibly_malicious_scheme(s)
82 83 _substitute_whitespace = re.compile(r'[\s\x00-\x08\x0B\x0C\x0E-\x19]+').sub 84 # FIXME: should data: be blocked? 85 86 # FIXME: check against: http://msdn2.microsoft.com/en-us/library/ms537512.aspx 87 _conditional_comment_re = re.compile( 88 r'\[if[\s\n\r]+.*?][\s\n\r]*>', re.I|re.S) 89 90 _find_styled_elements = etree.XPath( 91 "descendant-or-self::*[@style]") 92 93 _find_external_links = etree.XPath( 94 ("descendant-or-self::a [normalize-space(@href) and substring(normalize-space(@href),1,1) != '#'] |" 95 "descendant-or-self::x:a[normalize-space(@href) and substring(normalize-space(@href),1,1) != '#']"), 96 namespaces={'x':XHTML_NAMESPACE}) 97 98
99 -class Cleaner(object):
100 """ 101 Instances cleans the document of each of the possible offending 102 elements. The cleaning is controlled by attributes; you can 103 override attributes in a subclass, or set them in the constructor. 104 105 ``scripts``: 106 Removes any ``<script>`` tags. 107 108 ``javascript``: 109 Removes any Javascript, like an ``onclick`` attribute. Also removes stylesheets 110 as they could contain Javascript. 111 112 ``comments``: 113 Removes any comments. 114 115 ``style``: 116 Removes any style tags. 117 118 ``inline_style`` 119 Removes any style attributes. Defaults to the value of the ``style`` option. 120 121 ``links``: 122 Removes any ``<link>`` tags 123 124 ``meta``: 125 Removes any ``<meta>`` tags 126 127 ``page_structure``: 128 Structural parts of a page: ``<head>``, ``<html>``, ``<title>``. 129 130 ``processing_instructions``: 131 Removes any processing instructions. 132 133 ``embedded``: 134 Removes any embedded objects (flash, iframes) 135 136 ``frames``: 137 Removes any frame-related tags 138 139 ``forms``: 140 Removes any form tags 141 142 ``annoying_tags``: 143 Tags that aren't *wrong*, but are annoying. ``<blink>`` and ``<marquee>`` 144 145 ``remove_tags``: 146 A list of tags to remove. Only the tags will be removed, 147 their content will get pulled up into the parent tag. 148 149 ``kill_tags``: 150 A list of tags to kill. Killing also removes the tag's content, 151 i.e. the whole subtree, not just the tag itself. 152 153 ``allow_tags``: 154 A list of tags to include (default include all). 155 156 ``remove_unknown_tags``: 157 Remove any tags that aren't standard parts of HTML. 158 159 ``safe_attrs_only``: 160 If true, only include 'safe' attributes (specifically the list 161 from the feedparser HTML sanitisation web site). 162 163 ``safe_attrs``: 164 A set of attribute names to override the default list of attributes 165 considered 'safe' (when safe_attrs_only=True). 166 167 ``add_nofollow``: 168 If true, then any <a> tags will have ``rel="nofollow"`` added to them. 169 170 ``host_whitelist``: 171 A list or set of hosts that you can use for embedded content 172 (for content like ``<object>``, ``<link rel="stylesheet">``, etc). 173 You can also implement/override the method 174 ``allow_embedded_url(el, url)`` or ``allow_element(el)`` to 175 implement more complex rules for what can be embedded. 176 Anything that passes this test will be shown, regardless of 177 the value of (for instance) ``embedded``. 178 179 Note that this parameter might not work as intended if you do not 180 make the links absolute before doing the cleaning. 181 182 Note that you may also need to set ``whitelist_tags``. 183 184 ``whitelist_tags``: 185 A set of tags that can be included with ``host_whitelist``. 186 The default is ``iframe`` and ``embed``; you may wish to 187 include other tags like ``script``, or you may want to 188 implement ``allow_embedded_url`` for more control. Set to None to 189 include all tags. 190 191 This modifies the document *in place*. 192 """ 193 194 scripts = True 195 javascript = True 196 comments = True 197 style = False 198 inline_style = None 199 links = True 200 meta = True 201 page_structure = True 202 processing_instructions = True 203 embedded = True 204 frames = True 205 forms = True 206 annoying_tags = True 207 remove_tags = None 208 allow_tags = None 209 kill_tags = None 210 remove_unknown_tags = True 211 safe_attrs_only = True 212 safe_attrs = defs.safe_attrs 213 add_nofollow = False 214 host_whitelist = () 215 whitelist_tags = {'iframe', 'embed'} 216
217 - def __init__(self, **kw):
218 not_an_attribute = object() 219 for name, value in kw.items(): 220 default = getattr(self, name, not_an_attribute) 221 if (default is not None and default is not True and default is not False 222 and not isinstance(default, (frozenset, set, tuple, list))): 223 raise TypeError( 224 "Unknown parameter: %s=%r" % (name, value)) 225 setattr(self, name, value) 226 if self.inline_style is None and 'inline_style' not in kw: 227 self.inline_style = self.style 228 229 if kw.get("allow_tags"): 230 if kw.get("remove_unknown_tags"): 231 raise ValueError("It does not make sense to pass in both " 232 "allow_tags and remove_unknown_tags") 233 self.remove_unknown_tags = False
234 235 # Used to lookup the primary URL for a given tag that is up for 236 # removal: 237 _tag_link_attrs = dict( 238 script='src', 239 link='href', 240 # From: http://java.sun.com/j2se/1.4.2/docs/guide/misc/applet.html 241 # From what I can tell, both attributes can contain a link: 242 applet=['code', 'object'], 243 iframe='src', 244 embed='src', 245 layer='src', 246 # FIXME: there doesn't really seem like a general way to figure out what 247 # links an <object> tag uses; links often go in <param> tags with values 248 # that we don't really know. You'd have to have knowledge about specific 249 # kinds of plugins (probably keyed off classid), and match against those. 250 ##object=?, 251 # FIXME: not looking at the action currently, because it is more complex 252 # than than -- if you keep the form, you should keep the form controls. 253 ##form='action', 254 a='href', 255 ) 256
257 - def __call__(self, doc):
258 """ 259 Cleans the document. 260 """ 261 try: 262 getroot = doc.getroot 263 except AttributeError: 264 pass # Element instance 265 else: 266 doc = getroot() # ElementTree instance, instead of an element 267 # convert XHTML to HTML 268 xhtml_to_html(doc) 269 # Normalize a case that IE treats <image> like <img>, and that 270 # can confuse either this step or later steps. 271 for el in doc.iter('image'): 272 el.tag = 'img' 273 if not self.comments: 274 # Of course, if we were going to kill comments anyway, we don't 275 # need to worry about this 276 self.kill_conditional_comments(doc) 277 278 kill_tags = set(self.kill_tags or ()) 279 remove_tags = set(self.remove_tags or ()) 280 allow_tags = set(self.allow_tags or ()) 281 282 if self.scripts: 283 kill_tags.add('script') 284 if self.safe_attrs_only: 285 safe_attrs = set(self.safe_attrs) 286 for el in doc.iter(etree.Element): 287 attrib = el.attrib 288 for aname in attrib.keys(): 289 if aname not in safe_attrs: 290 del attrib[aname] 291 if self.javascript: 292 if not (self.safe_attrs_only and 293 self.safe_attrs == defs.safe_attrs): 294 # safe_attrs handles events attributes itself 295 for el in doc.iter(etree.Element): 296 attrib = el.attrib 297 for aname in attrib.keys(): 298 if aname.startswith('on'): 299 del attrib[aname] 300 doc.rewrite_links(self._remove_javascript_link, 301 resolve_base_href=False) 302 # If we're deleting style then we don't have to remove JS links 303 # from styles, otherwise... 304 if not self.inline_style: 305 for el in _find_styled_elements(doc): 306 old = el.get('style') 307 new = _css_javascript_re.sub('', old) 308 new = _css_import_re.sub('', new) 309 if self._has_sneaky_javascript(new): 310 # Something tricky is going on... 311 del el.attrib['style'] 312 elif new != old: 313 el.set('style', new) 314 if not self.style: 315 for el in list(doc.iter('style')): 316 if el.get('type', '').lower().strip() == 'text/javascript': 317 el.drop_tree() 318 continue 319 old = el.text or '' 320 new = _css_javascript_re.sub('', old) 321 # The imported CSS can do anything; we just can't allow: 322 new = _css_import_re.sub('', old) 323 if self._has_sneaky_javascript(new): 324 # Something tricky is going on... 325 el.text = '/* deleted */' 326 elif new != old: 327 el.text = new 328 if self.comments: 329 kill_tags.add(etree.Comment) 330 if self.processing_instructions: 331 kill_tags.add(etree.ProcessingInstruction) 332 if self.style: 333 kill_tags.add('style') 334 if self.inline_style: 335 etree.strip_attributes(doc, 'style') 336 if self.links: 337 kill_tags.add('link') 338 elif self.style or self.javascript: 339 # We must get rid of included stylesheets if Javascript is not 340 # allowed, as you can put Javascript in them 341 for el in list(doc.iter('link')): 342 if 'stylesheet' in el.get('rel', '').lower(): 343 # Note this kills alternate stylesheets as well 344 if not self.allow_element(el): 345 el.drop_tree() 346 if self.meta: 347 kill_tags.add('meta') 348 if self.page_structure: 349 remove_tags.update(('head', 'html', 'title')) 350 if self.embedded: 351 # FIXME: is <layer> really embedded? 352 # We should get rid of any <param> tags not inside <applet>; 353 # These are not really valid anyway. 354 for el in list(doc.iter('param')): 355 found_parent = False 356 parent = el.getparent() 357 while parent is not None and parent.tag not in ('applet', 'object'): 358 parent = parent.getparent() 359 if parent is None: 360 el.drop_tree() 361 kill_tags.update(('applet',)) 362 # The alternate contents that are in an iframe are a good fallback: 363 remove_tags.update(('iframe', 'embed', 'layer', 'object', 'param')) 364 if self.frames: 365 # FIXME: ideally we should look at the frame links, but 366 # generally frames don't mix properly with an HTML 367 # fragment anyway. 368 kill_tags.update(defs.frame_tags) 369 if self.forms: 370 remove_tags.add('form') 371 kill_tags.update(('button', 'input', 'select', 'textarea')) 372 if self.annoying_tags: 373 remove_tags.update(('blink', 'marquee')) 374 375 _remove = [] 376 _kill = [] 377 for el in doc.iter(): 378 if el.tag in kill_tags: 379 if self.allow_element(el): 380 continue 381 _kill.append(el) 382 elif el.tag in remove_tags: 383 if self.allow_element(el): 384 continue 385 _remove.append(el) 386 387 if _remove and _remove[0] == doc: 388 # We have to drop the parent-most tag, which we can't 389 # do. Instead we'll rewrite it: 390 el = _remove.pop(0) 391 el.tag = 'div' 392 el.attrib.clear() 393 elif _kill and _kill[0] == doc: 394 # We have to drop the parent-most element, which we can't 395 # do. Instead we'll clear it: 396 el = _kill.pop(0) 397 if el.tag != 'html': 398 el.tag = 'div' 399 el.clear() 400 401 _kill.reverse() # start with innermost tags 402 for el in _kill: 403 el.drop_tree() 404 for el in _remove: 405 el.drop_tag() 406 407 if self.remove_unknown_tags: 408 if allow_tags: 409 raise ValueError( 410 "It does not make sense to pass in both allow_tags and remove_unknown_tags") 411 allow_tags = set(defs.tags) 412 if allow_tags: 413 # make sure we do not remove comments/PIs if users want them (which is rare enough) 414 if not self.comments: 415 allow_tags.add(etree.Comment) 416 if not self.processing_instructions: 417 allow_tags.add(etree.ProcessingInstruction) 418 419 bad = [] 420 for el in doc.iter(): 421 if el.tag not in allow_tags: 422 bad.append(el) 423 if bad: 424 if bad[0] is doc: 425 el = bad.pop(0) 426 el.tag = 'div' 427 el.attrib.clear() 428 for el in bad: 429 el.drop_tag() 430 if self.add_nofollow: 431 for el in _find_external_links(doc): 432 if not self.allow_follow(el): 433 rel = el.get('rel') 434 if rel: 435 if ('nofollow' in rel 436 and ' nofollow ' in (' %s ' % rel)): 437 continue 438 rel = '%s nofollow' % rel 439 else: 440 rel = 'nofollow' 441 el.set('rel', rel)
442
443 - def allow_follow(self, anchor):
444 """ 445 Override to suppress rel="nofollow" on some anchors. 446 """ 447 return False
448
449 - def allow_element(self, el):
450 """ 451 Decide whether an element is configured to be accepted or rejected. 452 453 :param el: an element. 454 :return: true to accept the element or false to reject/discard it. 455 """ 456 if el.tag not in self._tag_link_attrs: 457 return False 458 attr = self._tag_link_attrs[el.tag] 459 if isinstance(attr, (list, tuple)): 460 for one_attr in attr: 461 url = el.get(one_attr) 462 if not url: 463 return False 464 if not self.allow_embedded_url(el, url): 465 return False 466 return True 467 else: 468 url = el.get(attr) 469 if not url: 470 return False 471 return self.allow_embedded_url(el, url)
472
473 - def allow_embedded_url(self, el, url):
474 """ 475 Decide whether a URL that was found in an element's attributes or text 476 if configured to be accepted or rejected. 477 478 :param el: an element. 479 :param url: a URL found on the element. 480 :return: true to accept the URL and false to reject it. 481 """ 482 if self.whitelist_tags is not None and el.tag not in self.whitelist_tags: 483 return False 484 scheme, netloc, path, query, fragment = urlsplit(url) 485 netloc = netloc.lower().split(':', 1)[0] 486 if scheme not in ('http', 'https'): 487 return False 488 if netloc in self.host_whitelist: 489 return True 490 return False
491
492 - def kill_conditional_comments(self, doc):
493 """ 494 IE conditional comments basically embed HTML that the parser 495 doesn't normally see. We can't allow anything like that, so 496 we'll kill any comments that could be conditional. 497 """ 498 has_conditional_comment = _conditional_comment_re.search 499 self._kill_elements( 500 doc, lambda el: has_conditional_comment(el.text), 501 etree.Comment)
502
503 - def _kill_elements(self, doc, condition, iterate=None):
504 bad = [] 505 for el in doc.iter(iterate): 506 if condition(el): 507 bad.append(el) 508 for el in bad: 509 el.drop_tree()
510 518 519 _substitute_comments = re.compile(r'/\*.*?\*/', re.S).sub 520
521 - def _has_sneaky_javascript(self, style):
522 """ 523 Depending on the browser, stuff like ``e x p r e s s i o n(...)`` 524 can get interpreted, or ``expre/* stuff */ssion(...)``. This 525 checks for attempt to do stuff like this. 526 527 Typically the response will be to kill the entire style; if you 528 have just a bit of Javascript in the style another rule will catch 529 that and remove only the Javascript from the style; this catches 530 more sneaky attempts. 531 """ 532 style = self._substitute_comments('', style) 533 style = style.replace('\\', '') 534 style = _substitute_whitespace('', style) 535 style = style.lower() 536 if 'javascript:' in style: 537 return True 538 if 'expression(' in style: 539 return True 540 return False
541
542 - def clean_html(self, html):
543 result_type = type(html) 544 if isinstance(html, basestring): 545 doc = fromstring(html) 546 else: 547 doc = copy.deepcopy(html) 548 self(doc) 549 return _transform_result(result_type, doc)
550 551 clean = Cleaner() 552 clean_html = clean.clean_html 553 554 ############################################################ 555 ## Autolinking 556 ############################################################ 557 558 _link_regexes = [ 559 re.compile(r'(?P<body>https?://(?P<host>[a-z0-9._-]+)(?:/[/\-_.,a-z0-9%&?;=~]*)?(?:\([/\-_.,a-z0-9%&?;=~]*\))?)', re.I), 560 # This is conservative, but autolinking can be a bit conservative: 561 re.compile(r'mailto:(?P<body>[a-z0-9._-]+@(?P<host>[a-z0-9_.-]+[a-z]))', re.I), 562 ] 563 564 _avoid_elements = ['textarea', 'pre', 'code', 'head', 'select', 'a'] 565 566 _avoid_hosts = [ 567 re.compile(r'^localhost', re.I), 568 re.compile(r'\bexample\.(?:com|org|net)$', re.I), 569 re.compile(r'^127\.0\.0\.1$'), 570 ] 571 572 _avoid_classes = ['nolink'] 573 618 676 685 686 autolink_html.__doc__ = autolink.__doc__ 687 688 ############################################################ 689 ## Word wrapping 690 ############################################################ 691 692 _avoid_word_break_elements = ['pre', 'textarea', 'code'] 693 _avoid_word_break_classes = ['nobreak'] 694
695 -def word_break(el, max_width=40, 696 avoid_elements=_avoid_word_break_elements, 697 avoid_classes=_avoid_word_break_classes, 698 break_character=unichr(0x200b)):
699 """ 700 Breaks any long words found in the body of the text (not attributes). 701 702 Doesn't effect any of the tags in avoid_elements, by default 703 ``<textarea>`` and ``<pre>`` 704 705 Breaks words by inserting &#8203;, which is a unicode character 706 for Zero Width Space character. This generally takes up no space 707 in rendering, but does copy as a space, and in monospace contexts 708 usually takes up space. 709 710 See http://www.cs.tut.fi/~jkorpela/html/nobr.html for a discussion 711 """ 712 # Character suggestion of &#8203 comes from: 713 # http://www.cs.tut.fi/~jkorpela/html/nobr.html 714 if el.tag in _avoid_word_break_elements: 715 return 716 class_name = el.get('class') 717 if class_name: 718 dont_break = False 719 class_name = class_name.split() 720 for avoid in avoid_classes: 721 if avoid in class_name: 722 dont_break = True 723 break 724 if dont_break: 725 return 726 if el.text: 727 el.text = _break_text(el.text, max_width, break_character) 728 for child in el: 729 word_break(child, max_width=max_width, 730 avoid_elements=avoid_elements, 731 avoid_classes=avoid_classes, 732 break_character=break_character) 733 if child.tail: 734 child.tail = _break_text(child.tail, max_width, break_character)
735
736 -def word_break_html(html, *args, **kw):
737 result_type = type(html) 738 doc = fromstring(html) 739 word_break(doc, *args, **kw) 740 return _transform_result(result_type, doc)
741
742 -def _break_text(text, max_width, break_character):
743 words = text.split() 744 for word in words: 745 if len(word) > max_width: 746 replacement = _insert_break(word, max_width, break_character) 747 text = text.replace(word, replacement) 748 return text
749 750 _break_prefer_re = re.compile(r'[^a-z]', re.I) 751
752 -def _insert_break(word, width, break_character):
753 orig_word = word 754 result = '' 755 while len(word) > width: 756 start = word[:width] 757 breaks = list(_break_prefer_re.finditer(start)) 758 if breaks: 759 last_break = breaks[-1] 760 # Only walk back up to 10 characters to find a nice break: 761 if last_break.end() > width-10: 762 # FIXME: should the break character be at the end of the 763 # chunk, or the beginning of the next chunk? 764 start = word[:last_break.end()] 765 result += start + break_character 766 word = word[len(start):] 767 result += word 768 return result
769