lxml.html.clean

100 """ 101 Instances cleans the document of each of the possible offending 102 elements. The cleaning is controlled by attributes; you can 103 override attributes in a subclass, or set them in the constructor. 104 105 ``scripts``: 106 Removes any ``<script>`` tags. 107 108 ``javascript``: 109 Removes any Javascript, like an ``onclick`` attribute. Also removes stylesheets 110 as they could contain Javascript. 111 112 ``comments``: 113 Removes any comments. 114 115 ``style``: 116 Removes any style tags. 117 118 ``inline_style`` 119 Removes any style attributes. Defaults to the value of the ``style`` option. 120 121 ``links``: 122 Removes any ``<link>`` tags 123 124 ``meta``: 125 Removes any ``<meta>`` tags 126 127 ``page_structure``: 128 Structural parts of a page: ``<head>``, ``<html>``, ``<title>``. 129 130 ``processing_instructions``: 131 Removes any processing instructions. 132 133 ``embedded``: 134 Removes any embedded objects (flash, iframes) 135 136 ``frames``: 137 Removes any frame-related tags 138 139 ``forms``: 140 Removes any form tags 141 142 ``annoying_tags``: 143 Tags that aren't *wrong*, but are annoying. ``<blink>`` and ``<marquee>`` 144 145 ``remove_tags``: 146 A list of tags to remove. Only the tags will be removed, 147 their content will get pulled up into the parent tag. 148 149 ``kill_tags``: 150 A list of tags to kill. Killing also removes the tag's content, 151 i.e. the whole subtree, not just the tag itself. 152 153 ``allow_tags``: 154 A list of tags to include (default include all). 155 156 ``remove_unknown_tags``: 157 Remove any tags that aren't standard parts of HTML. 158 159 ``safe_attrs_only``: 160 If true, only include 'safe' attributes (specifically the list 161 from the feedparser HTML sanitisation web site). 162 163 ``safe_attrs``: 164 A set of attribute names to override the default list of attributes 165 considered 'safe' (when safe_attrs_only=True). 166 167 ``add_nofollow``: 168 If true, then any <a> tags will have ``rel="nofollow"`` added to them. 169 170 ``host_whitelist``: 171 A list or set of hosts that you can use for embedded content 172 (for content like ``<object>``, ``<link rel="stylesheet">``, etc). 173 You can also implement/override the method 174 ``allow_embedded_url(el, url)`` or ``allow_element(el)`` to 175 implement more complex rules for what can be embedded. 176 Anything that passes this test will be shown, regardless of 177 the value of (for instance) ``embedded``. 178 179 Note that this parameter might not work as intended if you do not 180 make the links absolute before doing the cleaning. 181 182 Note that you may also need to set ``whitelist_tags``. 183 184 ``whitelist_tags``: 185 A set of tags that can be included with ``host_whitelist``. 186 The default is ``iframe`` and ``embed``; you may wish to 187 include other tags like ``script``, or you may want to 188 implement ``allow_embedded_url`` for more control. Set to None to 189 include all tags. 190 191 This modifies the document *in place*. 192 """ 193 194 scripts = True 195 javascript = True 196 comments = True 197 style = False 198 inline_style = None 199 links = True 200 meta = True 201 page_structure = True 202 processing_instructions = True 203 embedded = True 204 frames = True 205 forms = True 206 annoying_tags = True 207 remove_tags = None 208 allow_tags = None 209 kill_tags = None 210 remove_unknown_tags = True 211 safe_attrs_only = True 212 safe_attrs = defs.safe_attrs 213 add_nofollow = False 214 host_whitelist = () 215 whitelist_tags = {'iframe', 'embed'} 216

217 - def __init__(self, **kw):

218 for name, value in kw.items(): 219 if not hasattr(self, name): 220 raise TypeError( 221 "Unknown parameter: %s=%r" % (name, value)) 222 setattr(self, name, value) 223 if self.inline_style is None and 'inline_style' not in kw: 224 self.inline_style = self.style

225 226 # Used to lookup the primary URL for a given tag that is up for 227 # removal: 228 _tag_link_attrs = dict( 229 script='src', 230 link='href', 231 # From: http://java.sun.com/j2se/1.4.2/docs/guide/misc/applet.html 232 # From what I can tell, both attributes can contain a link: 233 applet=['code', 'object'], 234 iframe='src', 235 embed='src', 236 layer='src', 237 # FIXME: there doesn't really seem like a general way to figure out what 238 # links an <object> tag uses; links often go in <param> tags with values 239 # that we don't really know. You'd have to have knowledge about specific 240 # kinds of plugins (probably keyed off classid), and match against those. 241 ##object=?, 242 # FIXME: not looking at the action currently, because it is more complex 243 # than than -- if you keep the form, you should keep the form controls. 244 ##form='action', 245 a='href', 246 ) 247

248 - def __call__(self, doc):

249 """ 250 Cleans the document. 251 """ 252 if hasattr(doc, 'getroot'): 253 # ElementTree instance, instead of an element 254 doc = doc.getroot() 255 # convert XHTML to HTML 256 xhtml_to_html(doc) 257 # Normalize a case that IE treats <image> like <img>, and that 258 # can confuse either this step or later steps. 259 for el in doc.iter('image'): 260 el.tag = 'img' 261 if not self.comments: 262 # Of course, if we were going to kill comments anyway, we don't 263 # need to worry about this 264 self.kill_conditional_comments(doc) 265 266 kill_tags = set(self.kill_tags or ()) 267 remove_tags = set(self.remove_tags or ()) 268 allow_tags = set(self.allow_tags or ()) 269 270 if self.scripts: 271 kill_tags.add('script') 272 if self.safe_attrs_only: 273 safe_attrs = set(self.safe_attrs) 274 for el in doc.iter(etree.Element): 275 attrib = el.attrib 276 for aname in attrib.keys(): 277 if aname not in safe_attrs: 278 del attrib[aname] 279 if self.javascript: 280 if not (self.safe_attrs_only and 281 self.safe_attrs == defs.safe_attrs): 282 # safe_attrs handles events attributes itself 283 for el in doc.iter(etree.Element): 284 attrib = el.attrib 285 for aname in attrib.keys(): 286 if aname.startswith('on'): 287 del attrib[aname] 288 doc.rewrite_links(self._remove_javascript_link, 289 resolve_base_href=False) 290 # If we're deleting style then we don't have to remove JS links 291 # from styles, otherwise... 292 if not self.inline_style: 293 for el in _find_styled_elements(doc): 294 old = el.get('style') 295 new = _css_javascript_re.sub('', old) 296 new = _css_import_re.sub('', new) 297 if self._has_sneaky_javascript(new): 298 # Something tricky is going on... 299 del el.attrib['style'] 300 elif new != old: 301 el.set('style', new) 302 if not self.style: 303 for el in list(doc.iter('style')): 304 if el.get('type', '').lower().strip() == 'text/javascript': 305 el.drop_tree() 306 continue 307 old = el.text or '' 308 new = _css_javascript_re.sub('', old) 309 # The imported CSS can do anything; we just can't allow: 310 new = _css_import_re.sub('', old) 311 if self._has_sneaky_javascript(new): 312 # Something tricky is going on... 313 el.text = '/* deleted */' 314 elif new != old: 315 el.text = new 316 if self.comments or self.processing_instructions: 317 # FIXME: why either? I feel like there's some obscure reason 318 # because you can put PIs in comments...? But I've already 319 # forgotten it 320 kill_tags.add(etree.Comment) 321 if self.processing_instructions: 322 kill_tags.add(etree.ProcessingInstruction) 323 if self.style: 324 kill_tags.add('style') 325 if self.inline_style: 326 etree.strip_attributes(doc, 'style') 327 if self.links: 328 kill_tags.add('link') 329 elif self.style or self.javascript: 330 # We must get rid of included stylesheets if Javascript is not 331 # allowed, as you can put Javascript in them 332 for el in list(doc.iter('link')): 333 if 'stylesheet' in el.get('rel', '').lower(): 334 # Note this kills alternate stylesheets as well 335 if not self.allow_element(el): 336 el.drop_tree() 337 if self.meta: 338 kill_tags.add('meta') 339 if self.page_structure: 340 remove_tags.update(('head', 'html', 'title')) 341 if self.embedded: 342 # FIXME: is <layer> really embedded? 343 # We should get rid of any <param> tags not inside <applet>; 344 # These are not really valid anyway. 345 for el in list(doc.iter('param')): 346 found_parent = False 347 parent = el.getparent() 348 while parent is not None and parent.tag not in ('applet', 'object'): 349 parent = parent.getparent() 350 if parent is None: 351 el.drop_tree() 352 kill_tags.update(('applet',)) 353 # The alternate contents that are in an iframe are a good fallback: 354 remove_tags.update(('iframe', 'embed', 'layer', 'object', 'param')) 355 if self.frames: 356 # FIXME: ideally we should look at the frame links, but 357 # generally frames don't mix properly with an HTML 358 # fragment anyway. 359 kill_tags.update(defs.frame_tags) 360 if self.forms: 361 remove_tags.add('form') 362 kill_tags.update(('button', 'input', 'select', 'textarea')) 363 if self.annoying_tags: 364 remove_tags.update(('blink', 'marquee')) 365 366 _remove = [] 367 _kill = [] 368 for el in doc.iter(): 369 if el.tag in kill_tags: 370 if self.allow_element(el): 371 continue 372 _kill.append(el) 373 elif el.tag in remove_tags: 374 if self.allow_element(el): 375 continue 376 _remove.append(el) 377 378 if _remove and _remove[0] == doc: 379 # We have to drop the parent-most tag, which we can't 380 # do. Instead we'll rewrite it: 381 el = _remove.pop(0) 382 el.tag = 'div' 383 el.attrib.clear() 384 elif _kill and _kill[0] == doc: 385 # We have to drop the parent-most element, which we can't 386 # do. Instead we'll clear it: 387 el = _kill.pop(0) 388 if el.tag != 'html': 389 el.tag = 'div' 390 el.clear() 391 392 _kill.reverse() # start with innermost tags 393 for el in _kill: 394 el.drop_tree() 395 for el in _remove: 396 el.drop_tag() 397 398 if self.remove_unknown_tags: 399 if allow_tags: 400 raise ValueError( 401 "It does not make sense to pass in both allow_tags and remove_unknown_tags") 402 allow_tags = set(defs.tags) 403 if allow_tags: 404 bad = [] 405 for el in doc.iter(): 406 if el.tag not in allow_tags: 407 bad.append(el) 408 if bad: 409 if bad[0] is doc: 410 el = bad.pop(0) 411 el.tag = 'div' 412 el.attrib.clear() 413 for el in bad: 414 el.drop_tag() 415 if self.add_nofollow: 416 for el in _find_external_links(doc): 417 if not self.allow_follow(el): 418 rel = el.get('rel') 419 if rel: 420 if ('nofollow' in rel 421 and ' nofollow ' in (' %s ' % rel)): 422 continue 423 rel = '%s nofollow' % rel 424 else: 425 rel = 'nofollow' 426 el.set('rel', rel)

427

428 - def allow_follow(self, anchor):

429 """ 430 Override to suppress rel="nofollow" on some anchors. 431 """ 432 return False

433

434 - def allow_element(self, el):

435 """ 436 Decide whether an element is configured to be accepted or rejected. 437 438 :param el: an element. 439 :return: true to accept the element or false to reject/discard it. 440 """ 441 if el.tag not in self._tag_link_attrs: 442 return False 443 attr = self._tag_link_attrs[el.tag] 444 if isinstance(attr, (list, tuple)): 445 for one_attr in attr: 446 url = el.get(one_attr) 447 if not url: 448 return False 449 if not self.allow_embedded_url(el, url): 450 return False 451 return True 452 else: 453 url = el.get(attr) 454 if not url: 455 return False 456 return self.allow_embedded_url(el, url)

457

458 - def allow_embedded_url(self, el, url):

459 """ 460 Decide whether a URL that was found in an element's attributes or text 461 if configured to be accepted or rejected. 462 463 :param el: an element. 464 :param url: a URL found on the element. 465 :return: true to accept the URL and false to reject it. 466 """ 467 if self.whitelist_tags is not None and el.tag not in self.whitelist_tags: 468 return False 469 scheme, netloc, path, query, fragment = urlsplit(url) 470 netloc = netloc.lower().split(':', 1)[0] 471 if scheme not in ('http', 'https'): 472 return False 473 if netloc in self.host_whitelist: 474 return True 475 return False

476

477 - def kill_conditional_comments(self, doc):

478 """ 479 IE conditional comments basically embed HTML that the parser 480 doesn't normally see. We can't allow anything like that, so 481 we'll kill any comments that could be conditional. 482 """ 483 bad = [] 484 self._kill_elements( 485 doc, lambda el: _conditional_comment_re.search(el.text), 486 etree.Comment)

487

488 - def _kill_elements(self, doc, condition, iterate=None):

489 bad = [] 490 for el in doc.iter(iterate): 491 if condition(el): 492 bad.append(el) 493 for el in bad: 494 el.drop_tree()

495

496 - def _remove_javascript_link(self, link):

497 # links like "j a v a s c r i p t:" might be interpreted in IE 498 new = _substitute_whitespace('', unquote_plus(link)) 499 if _is_javascript_scheme(new): 500 # FIXME: should this be None to delete? 501 return '' 502 return link

503 504 _substitute_comments = re.compile(r'/\*.*?\*/', re.S).sub 505

506 - def _has_sneaky_javascript(self, style):

507 """ 508 Depending on the browser, stuff like ``e x p r e s s i o n(...)`` 509 can get interpreted, or ``expre/* stuff */ssion(...)``. This 510 checks for attempt to do stuff like this. 511 512 Typically the response will be to kill the entire style; if you 513 have just a bit of Javascript in the style another rule will catch 514 that and remove only the Javascript from the style; this catches 515 more sneaky attempts. 516 """ 517 style = self._substitute_comments('', style) 518 style = style.replace('\\', '') 519 style = _substitute_whitespace('', style) 520 style = style.lower() 521 if 'javascript:' in style: 522 return True 523 if 'expression(' in style: 524 return True 525 return False

526

527 - def clean_html(self, html):

528 result_type = type(html) 529 if isinstance(html, basestring): 530 doc = fromstring(html) 531 else: 532 doc = copy.deepcopy(html) 533 self(doc) 534 return _transform_result(result_type, doc)

Source Code for Module lxml.html.clean