lxml.html.clean

91 """ 92 Instances cleans the document of each of the possible offending 93 elements. The cleaning is controlled by attributes; you can 94 override attributes in a subclass, or set them in the constructor. 95 96 ``scripts``: 97 Removes any ``<script>`` tags. 98 99 ``javascript``: 100 Removes any Javascript, like an ``onclick`` attribute. Also removes stylesheets 101 as they could contain Javascript. 102 103 ``comments``: 104 Removes any comments. 105 106 ``style``: 107 Removes any style tags or attributes. 108 109 ``links``: 110 Removes any ``<link>`` tags 111 112 ``meta``: 113 Removes any ``<meta>`` tags 114 115 ``page_structure``: 116 Structural parts of a page: ``<head>``, ``<html>``, ``<title>``. 117 118 ``processing_instructions``: 119 Removes any processing instructions. 120 121 ``embedded``: 122 Removes any embedded objects (flash, iframes) 123 124 ``frames``: 125 Removes any frame-related tags 126 127 ``forms``: 128 Removes any form tags 129 130 ``annoying_tags``: 131 Tags that aren't *wrong*, but are annoying. ``<blink>`` and ``<marquee>`` 132 133 ``remove_tags``: 134 A list of tags to remove. Only the tags will be removed, 135 their content will get pulled up into the parent tag. 136 137 ``kill_tags``: 138 A list of tags to kill. Killing also removes the tag's content, 139 i.e. the whole subtree, not just the tag itself. 140 141 ``allow_tags``: 142 A list of tags to include (default include all). 143 144 ``remove_unknown_tags``: 145 Remove any tags that aren't standard parts of HTML. 146 147 ``safe_attrs_only``: 148 If true, only include 'safe' attributes (specifically the list 149 from the feedparser HTML sanitisation web site). 150 151 ``safe_attrs``: 152 A set of attribute names to override the default list of attributes 153 considered 'safe' (when safe_attrs_only=True). 154 155 ``add_nofollow``: 156 If true, then any <a> tags will have ``rel="nofollow"`` added to them. 157 158 ``host_whitelist``: 159 A list or set of hosts that you can use for embedded content 160 (for content like ``<object>``, ``<link rel="stylesheet">``, etc). 161 You can also implement/override the method 162 ``allow_embedded_url(el, url)`` or ``allow_element(el)`` to 163 implement more complex rules for what can be embedded. 164 Anything that passes this test will be shown, regardless of 165 the value of (for instance) ``embedded``. 166 167 Note that this parameter might not work as intended if you do not 168 make the links absolute before doing the cleaning. 169 170 Note that you may also need to set ``whitelist_tags``. 171 172 ``whitelist_tags``: 173 A set of tags that can be included with ``host_whitelist``. 174 The default is ``iframe`` and ``embed``; you may wish to 175 include other tags like ``script``, or you may want to 176 implement ``allow_embedded_url`` for more control. Set to None to 177 include all tags. 178 179 This modifies the document *in place*. 180 """ 181 182 scripts = True 183 javascript = True 184 comments = True 185 style = False 186 links = True 187 meta = True 188 page_structure = True 189 processing_instructions = True 190 embedded = True 191 frames = True 192 forms = True 193 annoying_tags = True 194 remove_tags = None 195 allow_tags = None 196 kill_tags = None 197 remove_unknown_tags = True 198 safe_attrs_only = True 199 safe_attrs = defs.safe_attrs 200 add_nofollow = False 201 host_whitelist = () 202 whitelist_tags = set(['iframe', 'embed']) 203

204 - def __init__(self, **kw):

205 for name, value in kw.items(): 206 if not hasattr(self, name): 207 raise TypeError( 208 "Unknown parameter: %s=%r" % (name, value)) 209 setattr(self, name, value)

210 211 # Used to lookup the primary URL for a given tag that is up for 212 # removal: 213 _tag_link_attrs = dict( 214 script='src', 215 link='href', 216 # From: http://java.sun.com/j2se/1.4.2/docs/guide/misc/applet.html 217 # From what I can tell, both attributes can contain a link: 218 applet=['code', 'object'], 219 iframe='src', 220 embed='src', 221 layer='src', 222 # FIXME: there doesn't really seem like a general way to figure out what 223 # links an <object> tag uses; links often go in <param> tags with values 224 # that we don't really know. You'd have to have knowledge about specific 225 # kinds of plugins (probably keyed off classid), and match against those. 226 ##object=?, 227 # FIXME: not looking at the action currently, because it is more complex 228 # than than -- if you keep the form, you should keep the form controls. 229 ##form='action', 230 a='href', 231 ) 232

233 - def __call__(self, doc):

234 """ 235 Cleans the document. 236 """ 237 if hasattr(doc, 'getroot'): 238 # ElementTree instance, instead of an element 239 doc = doc.getroot() 240 # convert XHTML to HTML 241 xhtml_to_html(doc) 242 # Normalize a case that IE treats <image> like <img>, and that 243 # can confuse either this step or later steps. 244 for el in doc.iter('image'): 245 el.tag = 'img' 246 if not self.comments: 247 # Of course, if we were going to kill comments anyway, we don't 248 # need to worry about this 249 self.kill_conditional_comments(doc) 250 251 kill_tags = set(self.kill_tags or ()) 252 remove_tags = set(self.remove_tags or ()) 253 allow_tags = set(self.allow_tags or ()) 254 255 if self.scripts: 256 kill_tags.add('script') 257 if self.safe_attrs_only: 258 safe_attrs = set(self.safe_attrs) 259 for el in doc.iter(): 260 attrib = el.attrib 261 for aname in attrib.keys(): 262 if aname not in safe_attrs: 263 del attrib[aname] 264 if self.javascript: 265 if not (self.safe_attrs_only and 266 self.safe_attrs == defs.safe_attrs): 267 # safe_attrs handles events attributes itself 268 for el in doc.iter(): 269 attrib = el.attrib 270 for aname in attrib.keys(): 271 if aname.startswith('on'): 272 del attrib[aname] 273 doc.rewrite_links(self._remove_javascript_link, 274 resolve_base_href=False) 275 if not self.style: 276 # If we're deleting style then we don't have to remove JS links 277 # from styles, otherwise... 278 for el in _find_styled_elements(doc): 279 old = el.get('style') 280 new = _css_javascript_re.sub('', old) 281 new = _css_import_re.sub('', new) 282 if self._has_sneaky_javascript(new): 283 # Something tricky is going on... 284 del el.attrib['style'] 285 elif new != old: 286 el.set('style', new) 287 for el in list(doc.iter('style')): 288 if el.get('type', '').lower().strip() == 'text/javascript': 289 el.drop_tree() 290 continue 291 old = el.text or '' 292 new = _css_javascript_re.sub('', old) 293 # The imported CSS can do anything; we just can't allow: 294 new = _css_import_re.sub('', old) 295 if self._has_sneaky_javascript(new): 296 # Something tricky is going on... 297 el.text = '/* deleted */' 298 elif new != old: 299 el.text = new 300 if self.comments or self.processing_instructions: 301 # FIXME: why either? I feel like there's some obscure reason 302 # because you can put PIs in comments...? But I've already 303 # forgotten it 304 kill_tags.add(etree.Comment) 305 if self.processing_instructions: 306 kill_tags.add(etree.ProcessingInstruction) 307 if self.style: 308 kill_tags.add('style') 309 etree.strip_attributes(doc, 'style') 310 if self.links: 311 kill_tags.add('link') 312 elif self.style or self.javascript: 313 # We must get rid of included stylesheets if Javascript is not 314 # allowed, as you can put Javascript in them 315 for el in list(doc.iter('link')): 316 if 'stylesheet' in el.get('rel', '').lower(): 317 # Note this kills alternate stylesheets as well 318 if not self.allow_element(el): 319 el.drop_tree() 320 if self.meta: 321 kill_tags.add('meta') 322 if self.page_structure: 323 remove_tags.update(('head', 'html', 'title')) 324 if self.embedded: 325 # FIXME: is <layer> really embedded? 326 # We should get rid of any <param> tags not inside <applet>; 327 # These are not really valid anyway. 328 for el in list(doc.iter('param')): 329 found_parent = False 330 parent = el.getparent() 331 while parent is not None and parent.tag not in ('applet', 'object'): 332 parent = parent.getparent() 333 if parent is None: 334 el.drop_tree() 335 kill_tags.update(('applet',)) 336 # The alternate contents that are in an iframe are a good fallback: 337 remove_tags.update(('iframe', 'embed', 'layer', 'object', 'param')) 338 if self.frames: 339 # FIXME: ideally we should look at the frame links, but 340 # generally frames don't mix properly with an HTML 341 # fragment anyway. 342 kill_tags.update(defs.frame_tags) 343 if self.forms: 344 remove_tags.add('form') 345 kill_tags.update(('button', 'input', 'select', 'textarea')) 346 if self.annoying_tags: 347 remove_tags.update(('blink', 'marquee')) 348 349 _remove = [] 350 _kill = [] 351 for el in doc.iter(): 352 if el.tag in kill_tags: 353 if self.allow_element(el): 354 continue 355 _kill.append(el) 356 elif el.tag in remove_tags: 357 if self.allow_element(el): 358 continue 359 _remove.append(el) 360 361 if _remove and _remove[0] == doc: 362 # We have to drop the parent-most tag, which we can't 363 # do. Instead we'll rewrite it: 364 el = _remove.pop(0) 365 el.tag = 'div' 366 el.attrib.clear() 367 elif _kill and _kill[0] == doc: 368 # We have to drop the parent-most element, which we can't 369 # do. Instead we'll clear it: 370 el = _kill.pop(0) 371 if el.tag != 'html': 372 el.tag = 'div' 373 el.clear() 374 375 _kill.reverse() # start with innermost tags 376 for el in _kill: 377 el.drop_tree() 378 for el in _remove: 379 el.drop_tag() 380 381 allow_tags = self.allow_tags 382 if self.remove_unknown_tags: 383 if allow_tags: 384 raise ValueError( 385 "It does not make sense to pass in both allow_tags and remove_unknown_tags") 386 allow_tags = set(defs.tags) 387 if allow_tags: 388 bad = [] 389 for el in doc.iter(): 390 if el.tag not in allow_tags: 391 bad.append(el) 392 if bad: 393 if bad[0] is doc: 394 el = bad.pop(0) 395 el.tag = 'div' 396 el.attrib.clear() 397 for el in bad: 398 el.drop_tag() 399 if self.add_nofollow: 400 for el in _find_external_links(doc): 401 if not self.allow_follow(el): 402 rel = el.get('rel') 403 if rel: 404 if ('nofollow' in rel 405 and ' nofollow ' in (' %s ' % rel)): 406 continue 407 rel = '%s nofollow' % rel 408 else: 409 rel = 'nofollow' 410 el.set('rel', rel)

411

412 - def allow_follow(self, anchor):

413 """ 414 Override to suppress rel="nofollow" on some anchors. 415 """ 416 return False

417

418 - def allow_element(self, el):

419 if el.tag not in self._tag_link_attrs: 420 return False 421 attr = self._tag_link_attrs[el.tag] 422 if isinstance(attr, (list, tuple)): 423 for one_attr in attr: 424 url = el.get(one_attr) 425 if not url: 426 return False 427 if not self.allow_embedded_url(el, url): 428 return False 429 return True 430 else: 431 url = el.get(attr) 432 if not url: 433 return False 434 return self.allow_embedded_url(el, url)

435

436 - def allow_embedded_url(self, el, url):

437 if (self.whitelist_tags is not None 438 and el.tag not in self.whitelist_tags): 439 return False 440 scheme, netloc, path, query, fragment = urlsplit(url) 441 netloc = netloc.lower().split(':', 1)[0] 442 if scheme not in ('http', 'https'): 443 return False 444 if netloc in self.host_whitelist: 445 return True 446 return False

447

448 - def kill_conditional_comments(self, doc):

449 """ 450 IE conditional comments basically embed HTML that the parser 451 doesn't normally see. We can't allow anything like that, so 452 we'll kill any comments that could be conditional. 453 """ 454 bad = [] 455 self._kill_elements( 456 doc, lambda el: _conditional_comment_re.search(el.text), 457 etree.Comment)

458

459 - def _kill_elements(self, doc, condition, iterate=None):

460 bad = [] 461 for el in doc.iter(iterate): 462 if condition(el): 463 bad.append(el) 464 for el in bad: 465 el.drop_tree()

466

467 - def _remove_javascript_link(self, link):

468 # links like "j a v a s c r i p t:" might be interpreted in IE 469 new = _substitute_whitespace('', link) 470 if _javascript_scheme_re.search(new): 471 # FIXME: should this be None to delete? 472 return '' 473 return link

474 475 _substitute_comments = re.compile(r'/\*.*?\*/', re.S).sub 476

477 - def _has_sneaky_javascript(self, style):

478 """ 479 Depending on the browser, stuff like ``e x p r e s s i o n(...)`` 480 can get interpreted, or ``expre/* stuff */ssion(...)``. This 481 checks for attempt to do stuff like this. 482 483 Typically the response will be to kill the entire style; if you 484 have just a bit of Javascript in the style another rule will catch 485 that and remove only the Javascript from the style; this catches 486 more sneaky attempts. 487 """ 488 style = self._substitute_comments('', style) 489 style = style.replace('\\', '') 490 style = _substitute_whitespace('', style) 491 style = style.lower() 492 if 'javascript:' in style: 493 return True 494 if 'expression(' in style: 495 return True 496 return False

497

498 - def clean_html(self, html):

499 result_type = type(html) 500 if isinstance(html, basestring): 501 doc = fromstring(html) 502 else: 503 doc = copy.deepcopy(html) 504 self(doc) 505 return _transform_result(result_type, doc)

Source Code for Module lxml.html.clean