lxml.html.clean

100 """ 101 Instances cleans the document of each of the possible offending 102 elements. The cleaning is controlled by attributes; you can 103 override attributes in a subclass, or set them in the constructor. 104 105 ``scripts``: 106 Removes any ``<script>`` tags. 107 108 ``javascript``: 109 Removes any Javascript, like an ``onclick`` attribute. Also removes stylesheets 110 as they could contain Javascript. 111 112 ``comments``: 113 Removes any comments. 114 115 ``style``: 116 Removes any style tags. 117 118 ``inline_style`` 119 Removes any style attributes. Defaults to the value of the ``style`` option. 120 121 ``links``: 122 Removes any ``<link>`` tags 123 124 ``meta``: 125 Removes any ``<meta>`` tags 126 127 ``page_structure``: 128 Structural parts of a page: ``<head>``, ``<html>``, ``<title>``. 129 130 ``processing_instructions``: 131 Removes any processing instructions. 132 133 ``embedded``: 134 Removes any embedded objects (flash, iframes) 135 136 ``frames``: 137 Removes any frame-related tags 138 139 ``forms``: 140 Removes any form tags 141 142 ``annoying_tags``: 143 Tags that aren't *wrong*, but are annoying. ``<blink>`` and ``<marquee>`` 144 145 ``remove_tags``: 146 A list of tags to remove. Only the tags will be removed, 147 their content will get pulled up into the parent tag. 148 149 ``kill_tags``: 150 A list of tags to kill. Killing also removes the tag's content, 151 i.e. the whole subtree, not just the tag itself. 152 153 ``allow_tags``: 154 A list of tags to include (default include all). 155 156 ``remove_unknown_tags``: 157 Remove any tags that aren't standard parts of HTML. 158 159 ``safe_attrs_only``: 160 If true, only include 'safe' attributes (specifically the list 161 from the feedparser HTML sanitisation web site). 162 163 ``safe_attrs``: 164 A set of attribute names to override the default list of attributes 165 considered 'safe' (when safe_attrs_only=True). 166 167 ``add_nofollow``: 168 If true, then any <a> tags will have ``rel="nofollow"`` added to them. 169 170 ``host_whitelist``: 171 A list or set of hosts that you can use for embedded content 172 (for content like ``<object>``, ``<link rel="stylesheet">``, etc). 173 You can also implement/override the method 174 ``allow_embedded_url(el, url)`` or ``allow_element(el)`` to 175 implement more complex rules for what can be embedded. 176 Anything that passes this test will be shown, regardless of 177 the value of (for instance) ``embedded``. 178 179 Note that this parameter might not work as intended if you do not 180 make the links absolute before doing the cleaning. 181 182 Note that you may also need to set ``whitelist_tags``. 183 184 ``whitelist_tags``: 185 A set of tags that can be included with ``host_whitelist``. 186 The default is ``iframe`` and ``embed``; you may wish to 187 include other tags like ``script``, or you may want to 188 implement ``allow_embedded_url`` for more control. Set to None to 189 include all tags. 190 191 This modifies the document *in place*. 192 """ 193 194 scripts = True 195 javascript = True 196 comments = True 197 style = False 198 inline_style = None 199 links = True 200 meta = True 201 page_structure = True 202 processing_instructions = True 203 embedded = True 204 frames = True 205 forms = True 206 annoying_tags = True 207 remove_tags = None 208 allow_tags = None 209 kill_tags = None 210 remove_unknown_tags = True 211 safe_attrs_only = True 212 safe_attrs = defs.safe_attrs 213 add_nofollow = False 214 host_whitelist = () 215 whitelist_tags = {'iframe', 'embed'} 216

217 - def __init__(self, **kw):

218 not_an_attribute = object() 219 for name, value in kw.items(): 220 default = getattr(self, name, not_an_attribute) 221 if (default is not None and default is not True and default is not False 222 and not isinstance(default, (frozenset, set, tuple, list))): 223 raise TypeError( 224 "Unknown parameter: %s=%r" % (name, value)) 225 setattr(self, name, value) 226 if self.inline_style is None and 'inline_style' not in kw: 227 self.inline_style = self.style 228 229 if kw.get("allow_tags"): 230 if kw.get("remove_unknown_tags"): 231 raise ValueError("It does not make sense to pass in both " 232 "allow_tags and remove_unknown_tags") 233 self.remove_unknown_tags = False

234 235 # Used to lookup the primary URL for a given tag that is up for 236 # removal: 237 _tag_link_attrs = dict( 238 script='src', 239 link='href', 240 # From: http://java.sun.com/j2se/1.4.2/docs/guide/misc/applet.html 241 # From what I can tell, both attributes can contain a link: 242 applet=['code', 'object'], 243 iframe='src', 244 embed='src', 245 layer='src', 246 # FIXME: there doesn't really seem like a general way to figure out what 247 # links an <object> tag uses; links often go in <param> tags with values 248 # that we don't really know. You'd have to have knowledge about specific 249 # kinds of plugins (probably keyed off classid), and match against those. 250 ##object=?, 251 # FIXME: not looking at the action currently, because it is more complex 252 # than than -- if you keep the form, you should keep the form controls. 253 ##form='action', 254 a='href', 255 ) 256

257 - def __call__(self, doc):

258 """ 259 Cleans the document. 260 """ 261 try: 262 getroot = doc.getroot 263 except AttributeError: 264 pass # Element instance 265 else: 266 doc = getroot() # ElementTree instance, instead of an element 267 # convert XHTML to HTML 268 xhtml_to_html(doc) 269 # Normalize a case that IE treats <image> like <img>, and that 270 # can confuse either this step or later steps. 271 for el in doc.iter('image'): 272 el.tag = 'img' 273 if not self.comments: 274 # Of course, if we were going to kill comments anyway, we don't 275 # need to worry about this 276 self.kill_conditional_comments(doc) 277 278 kill_tags = set(self.kill_tags or ()) 279 remove_tags = set(self.remove_tags or ()) 280 allow_tags = set(self.allow_tags or ()) 281 282 if self.scripts: 283 kill_tags.add('script') 284 if self.safe_attrs_only: 285 safe_attrs = set(self.safe_attrs) 286 for el in doc.iter(etree.Element): 287 attrib = el.attrib 288 for aname in attrib.keys(): 289 if aname not in safe_attrs: 290 del attrib[aname] 291 if self.javascript: 292 if not (self.safe_attrs_only and 293 self.safe_attrs == defs.safe_attrs): 294 # safe_attrs handles events attributes itself 295 for el in doc.iter(etree.Element): 296 attrib = el.attrib 297 for aname in attrib.keys(): 298 if aname.startswith('on'): 299 del attrib[aname] 300 doc.rewrite_links(self._remove_javascript_link, 301 resolve_base_href=False) 302 # If we're deleting style then we don't have to remove JS links 303 # from styles, otherwise... 304 if not self.inline_style: 305 for el in _find_styled_elements(doc): 306 old = el.get('style') 307 new = _css_javascript_re.sub('', old) 308 new = _css_import_re.sub('', new) 309 if self._has_sneaky_javascript(new): 310 # Something tricky is going on... 311 del el.attrib['style'] 312 elif new != old: 313 el.set('style', new) 314 if not self.style: 315 for el in list(doc.iter('style')): 316 if el.get('type', '').lower().strip() == 'text/javascript': 317 el.drop_tree() 318 continue 319 old = el.text or '' 320 new = _css_javascript_re.sub('', old) 321 # The imported CSS can do anything; we just can't allow: 322 new = _css_import_re.sub('', old) 323 if self._has_sneaky_javascript(new): 324 # Something tricky is going on... 325 el.text = '/* deleted */' 326 elif new != old: 327 el.text = new 328 if self.comments: 329 kill_tags.add(etree.Comment) 330 if self.processing_instructions: 331 kill_tags.add(etree.ProcessingInstruction) 332 if self.style: 333 kill_tags.add('style') 334 if self.inline_style: 335 etree.strip_attributes(doc, 'style') 336 if self.links: 337 kill_tags.add('link') 338 elif self.style or self.javascript: 339 # We must get rid of included stylesheets if Javascript is not 340 # allowed, as you can put Javascript in them 341 for el in list(doc.iter('link')): 342 if 'stylesheet' in el.get('rel', '').lower(): 343 # Note this kills alternate stylesheets as well 344 if not self.allow_element(el): 345 el.drop_tree() 346 if self.meta: 347 kill_tags.add('meta') 348 if self.page_structure: 349 remove_tags.update(('head', 'html', 'title')) 350 if self.embedded: 351 # FIXME: is <layer> really embedded? 352 # We should get rid of any <param> tags not inside <applet>; 353 # These are not really valid anyway. 354 for el in list(doc.iter('param')): 355 found_parent = False 356 parent = el.getparent() 357 while parent is not None and parent.tag not in ('applet', 'object'): 358 parent = parent.getparent() 359 if parent is None: 360 el.drop_tree() 361 kill_tags.update(('applet',)) 362 # The alternate contents that are in an iframe are a good fallback: 363 remove_tags.update(('iframe', 'embed', 'layer', 'object', 'param')) 364 if self.frames: 365 # FIXME: ideally we should look at the frame links, but 366 # generally frames don't mix properly with an HTML 367 # fragment anyway. 368 kill_tags.update(defs.frame_tags) 369 if self.forms: 370 remove_tags.add('form') 371 kill_tags.update(('button', 'input', 'select', 'textarea')) 372 if self.annoying_tags: 373 remove_tags.update(('blink', 'marquee')) 374 375 _remove = [] 376 _kill = [] 377 for el in doc.iter(): 378 if el.tag in kill_tags: 379 if self.allow_element(el): 380 continue 381 _kill.append(el) 382 elif el.tag in remove_tags: 383 if self.allow_element(el): 384 continue 385 _remove.append(el) 386 387 if _remove and _remove[0] == doc: 388 # We have to drop the parent-most tag, which we can't 389 # do. Instead we'll rewrite it: 390 el = _remove.pop(0) 391 el.tag = 'div' 392 el.attrib.clear() 393 elif _kill and _kill[0] == doc: 394 # We have to drop the parent-most element, which we can't 395 # do. Instead we'll clear it: 396 el = _kill.pop(0) 397 if el.tag != 'html': 398 el.tag = 'div' 399 el.clear() 400 401 _kill.reverse() # start with innermost tags 402 for el in _kill: 403 el.drop_tree() 404 for el in _remove: 405 el.drop_tag() 406 407 if self.remove_unknown_tags: 408 if allow_tags: 409 raise ValueError( 410 "It does not make sense to pass in both allow_tags and remove_unknown_tags") 411 allow_tags = set(defs.tags) 412 if allow_tags: 413 # make sure we do not remove comments/PIs if users want them (which is rare enough) 414 if not self.comments: 415 allow_tags.add(etree.Comment) 416 if not self.processing_instructions: 417 allow_tags.add(etree.ProcessingInstruction) 418 419 bad = [] 420 for el in doc.iter(): 421 if el.tag not in allow_tags: 422 bad.append(el) 423 if bad: 424 if bad[0] is doc: 425 el = bad.pop(0) 426 el.tag = 'div' 427 el.attrib.clear() 428 for el in bad: 429 el.drop_tag() 430 if self.add_nofollow: 431 for el in _find_external_links(doc): 432 if not self.allow_follow(el): 433 rel = el.get('rel') 434 if rel: 435 if ('nofollow' in rel 436 and ' nofollow ' in (' %s ' % rel)): 437 continue 438 rel = '%s nofollow' % rel 439 else: 440 rel = 'nofollow' 441 el.set('rel', rel)

442

443 - def allow_follow(self, anchor):

444 """ 445 Override to suppress rel="nofollow" on some anchors. 446 """ 447 return False

448

449 - def allow_element(self, el):

450 """ 451 Decide whether an element is configured to be accepted or rejected. 452 453 :param el: an element. 454 :return: true to accept the element or false to reject/discard it. 455 """ 456 if el.tag not in self._tag_link_attrs: 457 return False 458 attr = self._tag_link_attrs[el.tag] 459 if isinstance(attr, (list, tuple)): 460 for one_attr in attr: 461 url = el.get(one_attr) 462 if not url: 463 return False 464 if not self.allow_embedded_url(el, url): 465 return False 466 return True 467 else: 468 url = el.get(attr) 469 if not url: 470 return False 471 return self.allow_embedded_url(el, url)

472

473 - def allow_embedded_url(self, el, url):

474 """ 475 Decide whether a URL that was found in an element's attributes or text 476 if configured to be accepted or rejected. 477 478 :param el: an element. 479 :param url: a URL found on the element. 480 :return: true to accept the URL and false to reject it. 481 """ 482 if self.whitelist_tags is not None and el.tag not in self.whitelist_tags: 483 return False 484 scheme, netloc, path, query, fragment = urlsplit(url) 485 netloc = netloc.lower().split(':', 1)[0] 486 if scheme not in ('http', 'https'): 487 return False 488 if netloc in self.host_whitelist: 489 return True 490 return False

491

492 - def kill_conditional_comments(self, doc):

493 """ 494 IE conditional comments basically embed HTML that the parser 495 doesn't normally see. We can't allow anything like that, so 496 we'll kill any comments that could be conditional. 497 """ 498 has_conditional_comment = _conditional_comment_re.search 499 self._kill_elements( 500 doc, lambda el: has_conditional_comment(el.text), 501 etree.Comment)

502

503 - def _kill_elements(self, doc, condition, iterate=None):

504 bad = [] 505 for el in doc.iter(iterate): 506 if condition(el): 507 bad.append(el) 508 for el in bad: 509 el.drop_tree()

510

511 - def _remove_javascript_link(self, link):

512 # links like "j a v a s c r i p t:" might be interpreted in IE 513 new = _substitute_whitespace('', unquote_plus(link)) 514 if _is_javascript_scheme(new): 515 # FIXME: should this be None to delete? 516 return '' 517 return link

518 519 _substitute_comments = re.compile(r'/\*.*?\*/', re.S).sub 520

521 - def _has_sneaky_javascript(self, style):

522 """ 523 Depending on the browser, stuff like ``e x p r e s s i o n(...)`` 524 can get interpreted, or ``expre/* stuff */ssion(...)``. This 525 checks for attempt to do stuff like this. 526 527 Typically the response will be to kill the entire style; if you 528 have just a bit of Javascript in the style another rule will catch 529 that and remove only the Javascript from the style; this catches 530 more sneaky attempts. 531 """ 532 style = self._substitute_comments('', style) 533 style = style.replace('\\', '') 534 style = _substitute_whitespace('', style) 535 style = style.lower() 536 if 'javascript:' in style: 537 return True 538 if 'expression(' in style: 539 return True 540 return False

541

542 - def clean_html(self, html):

543 result_type = type(html) 544 if isinstance(html, basestring): 545 doc = fromstring(html) 546 else: 547 doc = copy.deepcopy(html) 548 self(doc) 549 return _transform_result(result_type, doc)

Source Code for Module lxml.html.clean