lxml.tests.test_htmlparser

22 """HTML parser test cases 23 """ 24 etree = etree 25 26 html_str = _bytes("<html><head><title>test</title></head><body><h1>page title</h1></body></html>") 27 html_str_pretty = _bytes("""\ 28 <html> 29 <head><title>test</title></head> 30 <body><h1>page title</h1></body> 31 </html> 32 """) 33 broken_html_str = _bytes("<html><head><title>test" 34 "<body><h1>page title</h3></p></html>") 35 uhtml_str = _bytes( 36 "<html><head><title>test Ã¡</title></head>" 37 "<body><h1>page Ã¡ title</h1></body></html>").decode('utf8') 38

39 - def tearDown(self):

40 super(HtmlParserTestCase, self).tearDown() 41 self.etree.set_default_parser()

42

43 - def test_module_HTML(self):

44 element = self.etree.HTML(self.html_str) 45 self.assertEqual(self.etree.tostring(element, method="html"), 46 self.html_str)

47

48 - def test_module_HTML_unicode(self):

49 element = self.etree.HTML(self.uhtml_str) 50 self.assertEqual( 51 self.etree.tostring(element, method="html", encoding='unicode'), 52 self.uhtml_str) 53 self.assertEqual(element.findtext('.//h1'), 54 _bytes("page Ã¡ title").decode('utf8'))

55

56 - def test_wide_unicode_xml(self):

57 if sys.maxunicode < 1114111: 58 return # skip test 59 element = self.etree.HTML(_bytes( 60 '<html><body><p>\\U00026007</p></body></html>' 61 ).decode('unicode_escape')) 62 p_text = element.findtext('.//p') 63 self.assertEqual(1, len(p_text)) 64 self.assertEqual(_bytes('\\U00026007').decode('unicode_escape'), 65 p_text)

66

67 - def test_html_ids(self):

68 parser = self.etree.HTMLParser(recover=False) 69 fromstring = self.etree.fromstring 70 html = fromstring(''' 71 <html><body id="bodyID"><p id="pID"></p></body></html> 72 ''', parser=parser) 73 self.assertEqual(len(html.xpath('//p[@id="pID"]')), 1) 74 self.assertEqual(len(html.findall('.//p[@id="pID"]')), 1)

75

76 - def test_html_ids_no_collect_ids(self):

77 parser = self.etree.HTMLParser(recover=False, collect_ids=False) 78 fromstring = self.etree.fromstring 79 html = fromstring(''' 80 <html><body id="bodyID"><p id="pID"></p></body></html> 81 ''', parser=parser) 82 self.assertEqual(len(html.xpath('//p[@id="pID"]')), 1) 83 self.assertEqual(len(html.findall('.//p[@id="pID"]')), 1)

84

85 - def test_module_HTML_pretty_print(self):

86 element = self.etree.HTML(self.html_str) 87 self.assertEqual(self.etree.tostring(element, method="html", pretty_print=True), 88 self.html_str_pretty)

89

90 - def test_module_parse_html_error(self):

91 parser = self.etree.HTMLParser(recover=False) 92 parse = self.etree.parse 93 f = BytesIO("<html></body>") 94 self.assertRaises(self.etree.XMLSyntaxError, 95 parse, f, parser)

96

97 - def test_html_element_name_empty(self):

98 parser = self.etree.HTMLParser() 99 Element = parser.makeelement 100 101 el = Element('name') 102 self.assertRaises(ValueError, Element, '{}') 103 self.assertRaises(ValueError, setattr, el, 'tag', '{}') 104 105 self.assertRaises(ValueError, Element, '{test}') 106 self.assertRaises(ValueError, setattr, el, 'tag', '{test}')

107

108 - def test_html_element_name_colon(self):

109 parser = self.etree.HTMLParser() 110 Element = parser.makeelement 111 112 pname = Element('p:name') 113 self.assertEqual(pname.tag, 'p:name') 114 115 pname = Element('{test}p:name') 116 self.assertEqual(pname.tag, '{test}p:name') 117 118 pname = Element('name') 119 pname.tag = 'p:name' 120 self.assertEqual(pname.tag, 'p:name')

121

122 - def test_html_element_name_quote(self):

123 parser = self.etree.HTMLParser() 124 Element = parser.makeelement 125 126 self.assertRaises(ValueError, Element, 'p"name') 127 self.assertRaises(ValueError, Element, "na'me") 128 self.assertRaises(ValueError, Element, '{test}"name') 129 self.assertRaises(ValueError, Element, "{test}name'") 130 131 el = Element('name') 132 self.assertRaises(ValueError, setattr, el, 'tag', "pname'") 133 self.assertRaises(ValueError, setattr, el, 'tag', '"pname') 134 self.assertEqual(el.tag, "name")

135

136 - def test_html_element_name_space(self):

137 parser = self.etree.HTMLParser() 138 Element = parser.makeelement 139 140 self.assertRaises(ValueError, Element, ' name ') 141 self.assertRaises(ValueError, Element, 'na me') 142 self.assertRaises(ValueError, Element, '{test} name') 143 144 el = Element('name') 145 self.assertRaises(ValueError, setattr, el, 'tag', ' name ') 146 self.assertEqual(el.tag, "name")

147

148 - def test_html_subelement_name_empty(self):

149 parser = self.etree.HTMLParser() 150 Element = parser.makeelement 151 152 SubElement = self.etree.SubElement 153 154 el = Element('name') 155 self.assertRaises(ValueError, SubElement, el, '{}') 156 self.assertRaises(ValueError, SubElement, el, '{test}')

157

158 - def test_html_subelement_name_colon(self):

159 parser = self.etree.HTMLParser() 160 Element = parser.makeelement 161 SubElement = self.etree.SubElement 162 163 el = Element('name') 164 pname = SubElement(el, 'p:name') 165 self.assertEqual(pname.tag, 'p:name') 166 167 pname = SubElement(el, '{test}p:name') 168 self.assertEqual(pname.tag, '{test}p:name')

169

170 - def test_html_subelement_name_quote(self):

171 parser = self.etree.HTMLParser() 172 Element = parser.makeelement 173 SubElement = self.etree.SubElement 174 175 el = Element('name') 176 self.assertRaises(ValueError, SubElement, el, "name'") 177 self.assertRaises(ValueError, SubElement, el, 'na"me') 178 self.assertRaises(ValueError, SubElement, el, "{test}na'me") 179 self.assertRaises(ValueError, SubElement, el, '{test}"name')

180

181 - def test_html_subelement_name_space(self):

182 parser = self.etree.HTMLParser() 183 Element = parser.makeelement 184 SubElement = self.etree.SubElement 185 186 el = Element('name') 187 self.assertRaises(ValueError, SubElement, el, ' name ') 188 self.assertRaises(ValueError, SubElement, el, 'na me') 189 self.assertRaises(ValueError, SubElement, el, '{test} name')

190

191 - def test_module_parse_html_norecover(self):

192 parser = self.etree.HTMLParser(recover=False) 193 parse = self.etree.parse 194 f = BytesIO(self.broken_html_str) 195 self.assertRaises(self.etree.XMLSyntaxError, 196 parse, f, parser)

197

198 - def test_module_parse_html_default_doctype(self):

199 parser = self.etree.HTMLParser(default_doctype=False) 200 d = html.fromstring('<!DOCTYPE html><h1>S</h1></html>', parser=parser) 201 self.assertEqual(d.getroottree().docinfo.doctype, '<!DOCTYPE html>') 202 203 d = html.fromstring('<html><h1>S</h1></html>', parser=parser) 204 self.assertEqual(d.getroottree().docinfo.doctype, '')

205

206 - def test_parse_encoding_8bit_explicit(self):

207 text = _str('Søk på nettet') 208 html_latin1 = (_str('<p>%s</p>') % text).encode('iso-8859-1') 209 210 tree = self.etree.parse( 211 BytesIO(html_latin1), 212 self.etree.HTMLParser(encoding="iso-8859-1")) 213 p = tree.find("//p") 214 self.assertEqual(p.text, text)

215

216 - def test_parse_encoding_8bit_override(self):

217 text = _str('Søk på nettet') 218 wrong_head = _str(''' 219 <head> 220 <meta http-equiv="Content-Type" 221 content="text/html; charset=UTF-8" /> 222 </head>''') 223 html_latin1 = (_str('<html>%s<body><p>%s</p></body></html>') % (wrong_head, 224 text) 225 ).encode('iso-8859-1') 226 227 self.assertRaises(self.etree.ParseError, 228 self.etree.parse, 229 BytesIO(html_latin1)) 230 231 tree = self.etree.parse( 232 BytesIO(html_latin1), 233 self.etree.HTMLParser(encoding="iso-8859-1")) 234 p = tree.find("//p") 235 self.assertEqual(p.text, text)

236

237 - def test_module_HTML_broken(self):

238 element = self.etree.HTML(self.broken_html_str) 239 self.assertEqual(self.etree.tostring(element, method="html"), 240 self.html_str)

241

242 - def test_module_HTML_cdata(self):

243 # by default, libxml2 generates CDATA nodes for <script> content 244 html = _bytes('<html><head><style>foo</style></head></html>') 245 element = self.etree.HTML(html) 246 self.assertEqual(element[0][0].text, "foo")

247

248 - def test_module_HTML_access(self):

249 element = self.etree.HTML(self.html_str) 250 self.assertEqual(element[0][0].tag, 'title')

251

252 - def test_module_parse_html(self):

253 parser = self.etree.HTMLParser() 254 filename = tempfile.mktemp(suffix=".html") 255 write_to_file(filename, self.html_str, 'wb') 256 try: 257 with open(filename, 'rb') as f: 258 tree = self.etree.parse(f, parser) 259 self.assertEqual(self.etree.tostring(tree.getroot(), method="html"), 260 self.html_str) 261 finally: 262 os.remove(filename)

263

264 - def test_module_parse_html_filelike(self):

265 parser = self.etree.HTMLParser() 266 f = SillyFileLike(self.html_str) 267 tree = self.etree.parse(f, parser) 268 html = self.etree.tostring(tree.getroot(), 269 method="html", encoding='UTF-8') 270 self.assertEqual(html, self.html_str)

271 272 ## def test_module_parse_html_filelike_unicode(self): 273 ## parser = self.etree.HTMLParser() 274 ## f = SillyFileLike(self.uhtml_str) 275 ## tree = self.etree.parse(f, parser) 276 ## html = self.etree.tostring(tree.getroot(), encoding='UTF-8') 277 ## self.assertEqual(unicode(html, 'UTF-8'), self.uhtml_str) 278

279 - def test_html_file_error(self):

280 parser = self.etree.HTMLParser() 281 parse = self.etree.parse 282 self.assertRaises(IOError, 283 parse, "__some_hopefully_nonexisting_file__.html", 284 parser)

285

286 - def test_default_parser_HTML_broken(self):

287 self.assertRaises(self.etree.XMLSyntaxError, 288 self.etree.parse, BytesIO(self.broken_html_str)) 289 290 self.etree.set_default_parser( self.etree.HTMLParser() ) 291 292 tree = self.etree.parse(BytesIO(self.broken_html_str)) 293 self.assertEqual(self.etree.tostring(tree.getroot(), method="html"), 294 self.html_str) 295 296 self.etree.set_default_parser() 297 298 self.assertRaises(self.etree.XMLSyntaxError, 299 self.etree.parse, BytesIO(self.broken_html_str))

300

301 - def test_html_iterparse(self):

302 iterparse = self.etree.iterparse 303 f = BytesIO( 304 '<html><head><title>TITLE</title><body><p>P</p></body></html>') 305 306 iterator = iterparse(f, html=True) 307 self.assertEqual(None, iterator.root) 308 309 events = list(iterator) 310 root = iterator.root 311 self.assertTrue(root is not None) 312 self.assertEqual( 313 [('end', root[0][0]), ('end', root[0]), ('end', root[1][0]), 314 ('end', root[1]), ('end', root)], 315 events)

316

317 - def test_html_iterparse_tag(self):

318 iterparse = self.etree.iterparse 319 f = BytesIO( 320 '<html><head><title>TITLE</title><body><p>P</p></body></html>') 321 322 iterator = iterparse(f, html=True, tag=["p", "title"]) 323 self.assertEqual(None, iterator.root) 324 325 events = list(iterator) 326 root = iterator.root 327 self.assertTrue(root is not None) 328 self.assertEqual( 329 [('end', root[0][0]), ('end', root[1][0])], 330 events)

331

332 - def test_html_iterparse_stop_short(self):

333 iterparse = self.etree.iterparse 334 f = BytesIO( 335 '<html><head><title>TITLE</title><body><p>P</p></body></html>') 336 337 iterator = iterparse(f, html=True) 338 self.assertEqual(None, iterator.root) 339 340 event, element = next(iterator) 341 self.assertEqual('end', event) 342 self.assertEqual('title', element.tag) 343 self.assertEqual(None, iterator.root) 344 del element 345 346 event, element = next(iterator) 347 self.assertEqual('end', event) 348 self.assertEqual('head', element.tag) 349 self.assertEqual(None, iterator.root) 350 del element 351 del iterator

352

353 - def test_html_iterparse_broken(self):

354 iterparse = self.etree.iterparse 355 f = BytesIO('<head><title>TEST></head><p>P<br></div>') 356 357 iterator = iterparse(f, html=True) 358 self.assertEqual(None, iterator.root) 359 360 events = list(iterator) 361 root = iterator.root 362 self.assertTrue(root is not None) 363 self.assertEqual('html', root.tag) 364 self.assertEqual('head', root[0].tag) 365 self.assertEqual('body', root[1].tag) 366 self.assertEqual('p', root[1][0].tag) 367 self.assertEqual('br', root[1][0][0].tag) 368 self.assertEqual( 369 [('end', root[0][0]), ('end', root[0]), ('end', root[1][0][0]), 370 ('end', root[1][0]), ('end', root[1]), ('end', root)], 371 events)

372

373 - def test_html_iterparse_broken_no_recover(self):

374 iterparse = self.etree.iterparse 375 f = BytesIO('<p>P<br></div>') 376 iterator = iterparse(f, html=True, recover=False) 377 self.assertRaises(self.etree.XMLSyntaxError, list, iterator)

378

379 - def test_html_iterparse_file(self):

380 iterparse = self.etree.iterparse 381 iterator = iterparse(fileInTestDir("shakespeare.html"), 382 html=True) 383 384 self.assertEqual(None, iterator.root) 385 events = list(iterator) 386 root = iterator.root 387 self.assertTrue(root is not None) 388 self.assertEqual(249, len(events)) 389 self.assertFalse( 390 [event for (event, element) in events if event != 'end'])

391

392 - def test_html_iterparse_start(self):

393 iterparse = self.etree.iterparse 394 f = BytesIO( 395 '<html><head><title>TITLE</title><body><p>P</p></body></html>') 396 397 iterator = iterparse(f, html=True, events=('start',)) 398 self.assertEqual(None, iterator.root) 399 400 events = list(iterator) 401 root = iterator.root 402 self.assertNotEqual(None, root) 403 self.assertEqual( 404 [('start', root), ('start', root[0]), ('start', root[0][0]), 405 ('start', root[1]), ('start', root[1][0])], 406 events)

407

408 - def test_html_feed_parser(self):

409 parser = self.etree.HTMLParser() 410 parser.feed("<html><body></") 411 parser.feed("body></html>") 412 root = parser.close() 413 414 self.assertEqual('html', root.tag) 415 # test that we find all names in the parser dict 416 self.assertEqual([root], list(root.iter('html'))) 417 self.assertEqual([root[0]], list(root.iter('body')))

418

419 - def test_html_feed_parser_chunky(self):

420 parser = self.etree.HTMLParser() 421 parser.feed("<htm") 422 parser.feed("l><body") 423 parser.feed("><") 424 parser.feed("p><") 425 parser.feed("strong") 426 parser.feed(">some ") 427 parser.feed("text</strong></p><") 428 parser.feed("/body></html>") 429 root = parser.close() 430 431 self.assertEqual('html', root.tag) 432 # test that we find all names in the parser dict 433 self.assertEqual([root], list(root.iter('html'))) 434 self.assertEqual([root[0]], list(root.iter('body'))) 435 self.assertEqual([root[0][0]], list(root.iter('p'))) 436 self.assertEqual([root[0][0][0]], list(root.iter('strong')))

437

438 - def test_html_feed_parser_more_tags(self):

439 parser = self.etree.HTMLParser() 440 parser.feed('<html><head>') 441 parser.feed('<title>TITLE</title><body><p>P</p></body><') 442 parser.feed("/html>") 443 root = parser.close() 444 445 self.assertEqual('html', root.tag) 446 # test that we find all names in the parser dict 447 self.assertEqual([root], list(root.iter('html'))) 448 self.assertEqual([root[0]], list(root.iter('head'))) 449 self.assertEqual([root[0][0]], list(root.iter('title'))) 450 self.assertEqual([root[1]], list(root.iter('body'))) 451 self.assertEqual([root[1][0]], list(root.iter('p')))

452

453 - def test_html_parser_target_tag(self):

454 assertFalse = self.assertFalse 455 events = [] 456 class Target(object): 457 def start(self, tag, attrib): 458 events.append(("start", tag)) 459 assertFalse(attrib)

460 def end(self, tag): 461 events.append(("end", tag))

Source Code for Module lxml.tests.test_htmlparser