Package lxml :: Package html :: Module clean
[hide private]
[frames] | no frames]

Source Code for Module lxml.html.clean

  1  """A cleanup tool for HTML. 
  2   
  3  Removes unwanted tags and content.  See the `Cleaner` class for 
  4  details. 
  5  """ 
  6   
  7  import re 
  8  import copy 
  9  try: 
 10      from urlparse import urlsplit 
 11  except ImportError: 
 12      # Python 3 
 13      from urllib.parse import urlsplit 
 14  from lxml import etree 
 15  from lxml.html import defs 
 16  from lxml.html import fromstring, XHTML_NAMESPACE 
 17  from lxml.html import xhtml_to_html, _transform_result 
 18   
 19  try: 
 20      unichr 
 21  except NameError: 
 22      # Python 3 
 23      unichr = chr 
 24  try: 
 25      unicode 
 26  except NameError: 
 27      # Python 3 
 28      unicode = str 
 29  try: 
 30      bytes 
 31  except NameError: 
 32      # Python < 2.6 
 33      bytes = str 
 34  try: 
 35      basestring 
 36  except NameError: 
 37      basestring = (str, bytes) 
 38   
 39   
 40  __all__ = ['clean_html', 'clean', 'Cleaner', 'autolink', 'autolink_html', 
 41             'word_break', 'word_break_html'] 
 42   
 43  # Look at http://code.sixapart.com/trac/livejournal/browser/trunk/cgi-bin/cleanhtml.pl 
 44  #   Particularly the CSS cleaning; most of the tag cleaning is integrated now 
 45  # I have multiple kinds of schemes searched; but should schemes be 
 46  #   whitelisted instead? 
 47  # max height? 
 48  # remove images?  Also in CSS?  background attribute? 
 49  # Some way to whitelist object, iframe, etc (e.g., if you want to 
 50  #   allow *just* embedded YouTube movies) 
 51  # Log what was deleted and why? 
 52  # style="behavior: ..." might be bad in IE? 
 53  # Should we have something for just <meta http-equiv>?  That's the worst of the 
 54  #   metas. 
 55  # UTF-7 detections?  Example: 
 56  #     <HEAD><META HTTP-EQUIV="CONTENT-TYPE" CONTENT="text/html; charset=UTF-7"> </HEAD>+ADw-SCRIPT+AD4-alert('XSS');+ADw-/SCRIPT+AD4- 
 57  #   you don't always have to have the charset set, if the page has no charset 
 58  #   and there's UTF7-like code in it. 
 59  # Look at these tests: http://htmlpurifier.org/live/smoketests/xssAttacks.php 
 60   
 61   
 62  # This is an IE-specific construct you can have in a stylesheet to 
 63  # run some Javascript: 
 64  _css_javascript_re = re.compile( 
 65      r'expression\s*\(.*?\)', re.S|re.I) 
 66   
 67  # Do I have to worry about @\nimport? 
 68  _css_import_re = re.compile( 
 69      r'@\s*import', re.I) 
 70   
 71  # All kinds of schemes besides just javascript: that can cause 
 72  # execution: 
 73  _is_image_dataurl = re.compile( 
 74      r'^data:image/.+;base64', re.I).search 
 75  _is_possibly_malicious_scheme = re.compile( 
 76      r'(?:javascript|jscript|livescript|vbscript|data|about|mocha):', 
 77      re.I).search 
78 -def _is_javascript_scheme(s):
79 if _is_image_dataurl(s): 80 return None 81 return _is_possibly_malicious_scheme(s)
82 83 _substitute_whitespace = re.compile(r'[\s\x00-\x08\x0B\x0C\x0E-\x19]+').sub 84 # FIXME: should data: be blocked? 85 86 # FIXME: check against: http://msdn2.microsoft.com/en-us/library/ms537512.aspx 87 _conditional_comment_re = re.compile( 88 r'\[if[\s\n\r]+.*?][\s\n\r]*>', re.I|re.S) 89 90 _find_styled_elements = etree.XPath( 91 "descendant-or-self::*[@style]") 92 93 _find_external_links = etree.XPath( 94 ("descendant-or-self::a [normalize-space(@href) and substring(normalize-space(@href),1,1) != '#'] |" 95 "descendant-or-self::x:a[normalize-space(@href) and substring(normalize-space(@href),1,1) != '#']"), 96 namespaces={'x':XHTML_NAMESPACE}) 97 98
99 -class Cleaner(object):
100 """ 101 Instances cleans the document of each of the possible offending 102 elements. The cleaning is controlled by attributes; you can 103 override attributes in a subclass, or set them in the constructor. 104 105 ``scripts``: 106 Removes any ``<script>`` tags. 107 108 ``javascript``: 109 Removes any Javascript, like an ``onclick`` attribute. Also removes stylesheets 110 as they could contain Javascript. 111 112 ``comments``: 113 Removes any comments. 114 115 ``style``: 116 Removes any style tags. 117 118 ``inline_style`` 119 Removes any style attributes. Defaults to the value of the ``style`` option. 120 121 ``links``: 122 Removes any ``<link>`` tags 123 124 ``meta``: 125 Removes any ``<meta>`` tags 126 127 ``page_structure``: 128 Structural parts of a page: ``<head>``, ``<html>``, ``<title>``. 129 130 ``processing_instructions``: 131 Removes any processing instructions. 132 133 ``embedded``: 134 Removes any embedded objects (flash, iframes) 135 136 ``frames``: 137 Removes any frame-related tags 138 139 ``forms``: 140 Removes any form tags 141 142 ``annoying_tags``: 143 Tags that aren't *wrong*, but are annoying. ``<blink>`` and ``<marquee>`` 144 145 ``remove_tags``: 146 A list of tags to remove. Only the tags will be removed, 147 their content will get pulled up into the parent tag. 148 149 ``kill_tags``: 150 A list of tags to kill. Killing also removes the tag's content, 151 i.e. the whole subtree, not just the tag itself. 152 153 ``allow_tags``: 154 A list of tags to include (default include all). 155 156 ``remove_unknown_tags``: 157 Remove any tags that aren't standard parts of HTML. 158 159 ``safe_attrs_only``: 160 If true, only include 'safe' attributes (specifically the list 161 from the feedparser HTML sanitisation web site). 162 163 ``safe_attrs``: 164 A set of attribute names to override the default list of attributes 165 considered 'safe' (when safe_attrs_only=True). 166 167 ``add_nofollow``: 168 If true, then any <a> tags will have ``rel="nofollow"`` added to them. 169 170 ``host_whitelist``: 171 A list or set of hosts that you can use for embedded content 172 (for content like ``<object>``, ``<link rel="stylesheet">``, etc). 173 You can also implement/override the method 174 ``allow_embedded_url(el, url)`` or ``allow_element(el)`` to 175 implement more complex rules for what can be embedded. 176 Anything that passes this test will be shown, regardless of 177 the value of (for instance) ``embedded``. 178 179 Note that this parameter might not work as intended if you do not 180 make the links absolute before doing the cleaning. 181 182 Note that you may also need to set ``whitelist_tags``. 183 184 ``whitelist_tags``: 185 A set of tags that can be included with ``host_whitelist``. 186 The default is ``iframe`` and ``embed``; you may wish to 187 include other tags like ``script``, or you may want to 188 implement ``allow_embedded_url`` for more control. Set to None to 189 include all tags. 190 191 This modifies the document *in place*. 192 """ 193 194 scripts = True 195 javascript = True 196 comments = True 197 style = False 198 inline_style = None 199 links = True 200 meta = True 201 page_structure = True 202 processing_instructions = True 203 embedded = True 204 frames = True 205 forms = True 206 annoying_tags = True 207 remove_tags = None 208 allow_tags = None 209 kill_tags = None 210 remove_unknown_tags = True 211 safe_attrs_only = True 212 safe_attrs = defs.safe_attrs 213 add_nofollow = False 214 host_whitelist = () 215 whitelist_tags = set(['iframe', 'embed']) 216
217 - def __init__(self, **kw):
218 for name, value in kw.items(): 219 if not hasattr(self, name): 220 raise TypeError( 221 "Unknown parameter: %s=%r" % (name, value)) 222 setattr(self, name, value) 223 if self.inline_style is None and 'inline_style' not in kw: 224 self.inline_style = self.style
225 226 # Used to lookup the primary URL for a given tag that is up for 227 # removal: 228 _tag_link_attrs = dict( 229 script='src', 230 link='href', 231 # From: http://java.sun.com/j2se/1.4.2/docs/guide/misc/applet.html 232 # From what I can tell, both attributes can contain a link: 233 applet=['code', 'object'], 234 iframe='src', 235 embed='src', 236 layer='src', 237 # FIXME: there doesn't really seem like a general way to figure out what 238 # links an <object> tag uses; links often go in <param> tags with values 239 # that we don't really know. You'd have to have knowledge about specific 240 # kinds of plugins (probably keyed off classid), and match against those. 241 ##object=?, 242 # FIXME: not looking at the action currently, because it is more complex 243 # than than -- if you keep the form, you should keep the form controls. 244 ##form='action', 245 a='href', 246 ) 247
248 - def __call__(self, doc):
249 """ 250 Cleans the document. 251 """ 252 if hasattr(doc, 'getroot'): 253 # ElementTree instance, instead of an element 254 doc = doc.getroot() 255 # convert XHTML to HTML 256 xhtml_to_html(doc) 257 # Normalize a case that IE treats <image> like <img>, and that 258 # can confuse either this step or later steps. 259 for el in doc.iter('image'): 260 el.tag = 'img' 261 if not self.comments: 262 # Of course, if we were going to kill comments anyway, we don't 263 # need to worry about this 264 self.kill_conditional_comments(doc) 265 266 kill_tags = set(self.kill_tags or ()) 267 remove_tags = set(self.remove_tags or ()) 268 allow_tags = set(self.allow_tags or ()) 269 270 if self.scripts: 271 kill_tags.add('script') 272 if self.safe_attrs_only: 273 safe_attrs = set(self.safe_attrs) 274 for el in doc.iter(etree.Element): 275 attrib = el.attrib 276 for aname in attrib.keys(): 277 if aname not in safe_attrs: 278 del attrib[aname] 279 if self.javascript: 280 if not (self.safe_attrs_only and 281 self.safe_attrs == defs.safe_attrs): 282 # safe_attrs handles events attributes itself 283 for el in doc.iter(etree.Element): 284 attrib = el.attrib 285 for aname in attrib.keys(): 286 if aname.startswith('on'): 287 del attrib[aname] 288 doc.rewrite_links(self._remove_javascript_link, 289 resolve_base_href=False) 290 # If we're deleting style then we don't have to remove JS links 291 # from styles, otherwise... 292 if not self.inline_style: 293 for el in _find_styled_elements(doc): 294 old = el.get('style') 295 new = _css_javascript_re.sub('', old) 296 new = _css_import_re.sub('', new) 297 if self._has_sneaky_javascript(new): 298 # Something tricky is going on... 299 del el.attrib['style'] 300 elif new != old: 301 el.set('style', new) 302 if not self.style: 303 for el in list(doc.iter('style')): 304 if el.get('type', '').lower().strip() == 'text/javascript': 305 el.drop_tree() 306 continue 307 old = el.text or '' 308 new = _css_javascript_re.sub('', old) 309 # The imported CSS can do anything; we just can't allow: 310 new = _css_import_re.sub('', old) 311 if self._has_sneaky_javascript(new): 312 # Something tricky is going on... 313 el.text = '/* deleted */' 314 elif new != old: 315 el.text = new 316 if self.comments or self.processing_instructions: 317 # FIXME: why either? I feel like there's some obscure reason 318 # because you can put PIs in comments...? But I've already 319 # forgotten it 320 kill_tags.add(etree.Comment) 321 if self.processing_instructions: 322 kill_tags.add(etree.ProcessingInstruction) 323 if self.style: 324 kill_tags.add('style') 325 if self.inline_style: 326 etree.strip_attributes(doc, 'style') 327 if self.links: 328 kill_tags.add('link') 329 elif self.style or self.javascript: 330 # We must get rid of included stylesheets if Javascript is not 331 # allowed, as you can put Javascript in them 332 for el in list(doc.iter('link')): 333 if 'stylesheet' in el.get('rel', '').lower(): 334 # Note this kills alternate stylesheets as well 335 if not self.allow_element(el): 336 el.drop_tree() 337 if self.meta: 338 kill_tags.add('meta') 339 if self.page_structure: 340 remove_tags.update(('head', 'html', 'title')) 341 if self.embedded: 342 # FIXME: is <layer> really embedded? 343 # We should get rid of any <param> tags not inside <applet>; 344 # These are not really valid anyway. 345 for el in list(doc.iter('param')): 346 found_parent = False 347 parent = el.getparent() 348 while parent is not None and parent.tag not in ('applet', 'object'): 349 parent = parent.getparent() 350 if parent is None: 351 el.drop_tree() 352 kill_tags.update(('applet',)) 353 # The alternate contents that are in an iframe are a good fallback: 354 remove_tags.update(('iframe', 'embed', 'layer', 'object', 'param')) 355 if self.frames: 356 # FIXME: ideally we should look at the frame links, but 357 # generally frames don't mix properly with an HTML 358 # fragment anyway. 359 kill_tags.update(defs.frame_tags) 360 if self.forms: 361 remove_tags.add('form') 362 kill_tags.update(('button', 'input', 'select', 'textarea')) 363 if self.annoying_tags: 364 remove_tags.update(('blink', 'marquee')) 365 366 _remove = [] 367 _kill = [] 368 for el in doc.iter(): 369 if el.tag in kill_tags: 370 if self.allow_element(el): 371 continue 372 _kill.append(el) 373 elif el.tag in remove_tags: 374 if self.allow_element(el): 375 continue 376 _remove.append(el) 377 378 if _remove and _remove[0] == doc: 379 # We have to drop the parent-most tag, which we can't 380 # do. Instead we'll rewrite it: 381 el = _remove.pop(0) 382 el.tag = 'div' 383 el.attrib.clear() 384 elif _kill and _kill[0] == doc: 385 # We have to drop the parent-most element, which we can't 386 # do. Instead we'll clear it: 387 el = _kill.pop(0) 388 if el.tag != 'html': 389 el.tag = 'div' 390 el.clear() 391 392 _kill.reverse() # start with innermost tags 393 for el in _kill: 394 el.drop_tree() 395 for el in _remove: 396 el.drop_tag() 397 398 if self.remove_unknown_tags: 399 if allow_tags: 400 raise ValueError( 401 "It does not make sense to pass in both allow_tags and remove_unknown_tags") 402 allow_tags = set(defs.tags) 403 if allow_tags: 404 bad = [] 405 for el in doc.iter(): 406 if el.tag not in allow_tags: 407 bad.append(el) 408 if bad: 409 if bad[0] is doc: 410 el = bad.pop(0) 411 el.tag = 'div' 412 el.attrib.clear() 413 for el in bad: 414 el.drop_tag() 415 if self.add_nofollow: 416 for el in _find_external_links(doc): 417 if not self.allow_follow(el): 418 rel = el.get('rel') 419 if rel: 420 if ('nofollow' in rel 421 and ' nofollow ' in (' %s ' % rel)): 422 continue 423 rel = '%s nofollow' % rel 424 else: 425 rel = 'nofollow' 426 el.set('rel', rel)
427
428 - def allow_follow(self, anchor):
429 """ 430 Override to suppress rel="nofollow" on some anchors. 431 """ 432 return False
433
434 - def allow_element(self, el):
435 if el.tag not in self._tag_link_attrs: 436 return False 437 attr = self._tag_link_attrs[el.tag] 438 if isinstance(attr, (list, tuple)): 439 for one_attr in attr: 440 url = el.get(one_attr) 441 if not url: 442 return False 443 if not self.allow_embedded_url(el, url): 444 return False 445 return True 446 else: 447 url = el.get(attr) 448 if not url: 449 return False 450 return self.allow_embedded_url(el, url)
451
452 - def allow_embedded_url(self, el, url):
453 if (self.whitelist_tags is not None 454 and el.tag not in self.whitelist_tags): 455 return False 456 scheme, netloc, path, query, fragment = urlsplit(url) 457 netloc = netloc.lower().split(':', 1)[0] 458 if scheme not in ('http', 'https'): 459 return False 460 if netloc in self.host_whitelist: 461 return True 462 return False
463
464 - def kill_conditional_comments(self, doc):
465 """ 466 IE conditional comments basically embed HTML that the parser 467 doesn't normally see. We can't allow anything like that, so 468 we'll kill any comments that could be conditional. 469 """ 470 bad = [] 471 self._kill_elements( 472 doc, lambda el: _conditional_comment_re.search(el.text), 473 etree.Comment)
474
475 - def _kill_elements(self, doc, condition, iterate=None):
476 bad = [] 477 for el in doc.iter(iterate): 478 if condition(el): 479 bad.append(el) 480 for el in bad: 481 el.drop_tree()
482 490 491 _substitute_comments = re.compile(r'/\*.*?\*/', re.S).sub 492
493 - def _has_sneaky_javascript(self, style):
494 """ 495 Depending on the browser, stuff like ``e x p r e s s i o n(...)`` 496 can get interpreted, or ``expre/* stuff */ssion(...)``. This 497 checks for attempt to do stuff like this. 498 499 Typically the response will be to kill the entire style; if you 500 have just a bit of Javascript in the style another rule will catch 501 that and remove only the Javascript from the style; this catches 502 more sneaky attempts. 503 """ 504 style = self._substitute_comments('', style) 505 style = style.replace('\\', '') 506 style = _substitute_whitespace('', style) 507 style = style.lower() 508 if 'javascript:' in style: 509 return True 510 if 'expression(' in style: 511 return True 512 return False
513
514 - def clean_html(self, html):
515 result_type = type(html) 516 if isinstance(html, basestring): 517 doc = fromstring(html) 518 else: 519 doc = copy.deepcopy(html) 520 self(doc) 521 return _transform_result(result_type, doc)
522 523 clean = Cleaner() 524 clean_html = clean.clean_html 525 526 ############################################################ 527 ## Autolinking 528 ############################################################ 529 530 _link_regexes = [ 531 re.compile(r'(?P<body>https?://(?P<host>[a-z0-9._-]+)(?:/[/\-_.,a-z0-9%&?;=~]*)?(?:\([/\-_.,a-z0-9%&?;=~]*\))?)', re.I), 532 # This is conservative, but autolinking can be a bit conservative: 533 re.compile(r'mailto:(?P<body>[a-z0-9._-]+@(?P<host>[a-z0-9_.-]+[a-z]))', re.I), 534 ] 535 536 _avoid_elements = ['textarea', 'pre', 'code', 'head', 'select', 'a'] 537 538 _avoid_hosts = [ 539 re.compile(r'^localhost', re.I), 540 re.compile(r'\bexample\.(?:com|org|net)$', re.I), 541 re.compile(r'^127\.0\.0\.1$'), 542 ] 543 544 _avoid_classes = ['nolink'] 545 590 648 657 658 autolink_html.__doc__ = autolink.__doc__ 659 660 ############################################################ 661 ## Word wrapping 662 ############################################################ 663 664 _avoid_word_break_elements = ['pre', 'textarea', 'code'] 665 _avoid_word_break_classes = ['nobreak'] 666
667 -def word_break(el, max_width=40, 668 avoid_elements=_avoid_word_break_elements, 669 avoid_classes=_avoid_word_break_classes, 670 break_character=unichr(0x200b)):
671 """ 672 Breaks any long words found in the body of the text (not attributes). 673 674 Doesn't effect any of the tags in avoid_elements, by default 675 ``<textarea>`` and ``<pre>`` 676 677 Breaks words by inserting &#8203;, which is a unicode character 678 for Zero Width Space character. This generally takes up no space 679 in rendering, but does copy as a space, and in monospace contexts 680 usually takes up space. 681 682 See http://www.cs.tut.fi/~jkorpela/html/nobr.html for a discussion 683 """ 684 # Character suggestion of &#8203 comes from: 685 # http://www.cs.tut.fi/~jkorpela/html/nobr.html 686 if el.tag in _avoid_word_break_elements: 687 return 688 class_name = el.get('class') 689 if class_name: 690 dont_break = False 691 class_name = class_name.split() 692 for avoid in avoid_classes: 693 if avoid in class_name: 694 dont_break = True 695 break 696 if dont_break: 697 return 698 if el.text: 699 el.text = _break_text(el.text, max_width, break_character) 700 for child in el: 701 word_break(child, max_width=max_width, 702 avoid_elements=avoid_elements, 703 avoid_classes=avoid_classes, 704 break_character=break_character) 705 if child.tail: 706 child.tail = _break_text(child.tail, max_width, break_character)
707
708 -def word_break_html(html, *args, **kw):
709 result_type = type(html) 710 doc = fromstring(html) 711 word_break(doc, *args, **kw) 712 return _transform_result(result_type, doc)
713
714 -def _break_text(text, max_width, break_character):
715 words = text.split() 716 for word in words: 717 if len(word) > max_width: 718 replacement = _insert_break(word, max_width, break_character) 719 text = text.replace(word, replacement) 720 return text
721 722 _break_prefer_re = re.compile(r'[^a-z]', re.I) 723
724 -def _insert_break(word, width, break_character):
725 orig_word = word 726 result = '' 727 while len(word) > width: 728 start = word[:width] 729 breaks = list(_break_prefer_re.finditer(start)) 730 if breaks: 731 last_break = breaks[-1] 732 # Only walk back up to 10 characters to find a nice break: 733 if last_break.end() > width-10: 734 # FIXME: should the break character be at the end of the 735 # chunk, or the beginning of the next chunk? 736 start = word[:last_break.end()] 737 result += start + break_character 738 word = word[len(start):] 739 result += word 740 return result
741