lxml.tests.test_htmlparser

24 """HTML parser test cases 25 """ 26 etree = etree 27 28 html_str = _bytes("<html><head><title>test</title></head><body><h1>page title</h1></body></html>") 29 html_str_pretty = _bytes("""\ 30 <html> 31 <head><title>test</title></head> 32 <body><h1>page title</h1></body> 33 </html> 34 """) 35 broken_html_str = _bytes("<html><head><title>test" 36 "<body><h1>page title</h3></p></html>") 37 uhtml_str = _bytes( 38 "<html><head><title>test Ã¡</title></head>" 39 "<body><h1>page Ã¡ title</h1></body></html>").decode('utf8') 40

41 - def tearDown(self):

42 super(HtmlParserTestCase, self).tearDown() 43 self.etree.set_default_parser()

44

45 - def test_module_HTML(self):

46 element = self.etree.HTML(self.html_str) 47 self.assertEqual(self.etree.tostring(element, method="html"), 48 self.html_str)

49

50 - def test_module_HTML_unicode(self):

51 element = self.etree.HTML(self.uhtml_str) 52 self.assertEqual( 53 self.etree.tostring(element, method="html", encoding='unicode'), 54 self.uhtml_str) 55 self.assertEqual(element.findtext('.//h1'), 56 _bytes("page Ã¡ title").decode('utf8'))

57

58 - def test_wide_unicode_xml(self):

59 if sys.maxunicode < 1114111: 60 return # skip test 61 element = self.etree.HTML(_bytes( 62 '<html><body><p>\\U00026007</p></body></html>' 63 ).decode('unicode_escape')) 64 p_text = element.findtext('.//p') 65 self.assertEqual(1, len(p_text)) 66 self.assertEqual(_bytes('\\U00026007').decode('unicode_escape'), 67 p_text)

68

69 - def test_module_HTML_pretty_print(self):

70 element = self.etree.HTML(self.html_str) 71 self.assertEqual(self.etree.tostring(element, method="html", pretty_print=True), 72 self.html_str_pretty)

73

74 - def test_module_parse_html_error(self):

75 parser = self.etree.HTMLParser(recover=False) 76 parse = self.etree.parse 77 f = BytesIO("<html></body>") 78 self.assertRaises(self.etree.XMLSyntaxError, 79 parse, f, parser)

80

81 - def test_html_element_name_empty(self):

82 parser = self.etree.HTMLParser() 83 Element = parser.makeelement 84 85 el = Element('name') 86 self.assertRaises(ValueError, Element, '{}') 87 self.assertRaises(ValueError, setattr, el, 'tag', '{}') 88 89 self.assertRaises(ValueError, Element, '{test}') 90 self.assertRaises(ValueError, setattr, el, 'tag', '{test}')

91

92 - def test_html_element_name_colon(self):

93 parser = self.etree.HTMLParser() 94 Element = parser.makeelement 95 96 pname = Element('p:name') 97 self.assertEqual(pname.tag, 'p:name') 98 99 pname = Element('{test}p:name') 100 self.assertEqual(pname.tag, '{test}p:name') 101 102 pname = Element('name') 103 pname.tag = 'p:name' 104 self.assertEqual(pname.tag, 'p:name')

105

106 - def test_html_element_name_quote(self):

107 parser = self.etree.HTMLParser() 108 Element = parser.makeelement 109 110 self.assertRaises(ValueError, Element, 'p"name') 111 self.assertRaises(ValueError, Element, "na'me") 112 self.assertRaises(ValueError, Element, '{test}"name') 113 self.assertRaises(ValueError, Element, "{test}name'") 114 115 el = Element('name') 116 self.assertRaises(ValueError, setattr, el, 'tag', "pname'") 117 self.assertRaises(ValueError, setattr, el, 'tag', '"pname') 118 self.assertEqual(el.tag, "name")

119

120 - def test_html_element_name_space(self):

121 parser = self.etree.HTMLParser() 122 Element = parser.makeelement 123 124 self.assertRaises(ValueError, Element, ' name ') 125 self.assertRaises(ValueError, Element, 'na me') 126 self.assertRaises(ValueError, Element, '{test} name') 127 128 el = Element('name') 129 self.assertRaises(ValueError, setattr, el, 'tag', ' name ') 130 self.assertEqual(el.tag, "name")

131

132 - def test_html_subelement_name_empty(self):

133 parser = self.etree.HTMLParser() 134 Element = parser.makeelement 135 136 SubElement = self.etree.SubElement 137 138 el = Element('name') 139 self.assertRaises(ValueError, SubElement, el, '{}') 140 self.assertRaises(ValueError, SubElement, el, '{test}')

141

142 - def test_html_subelement_name_colon(self):

143 parser = self.etree.HTMLParser() 144 Element = parser.makeelement 145 SubElement = self.etree.SubElement 146 147 el = Element('name') 148 pname = SubElement(el, 'p:name') 149 self.assertEqual(pname.tag, 'p:name') 150 151 pname = SubElement(el, '{test}p:name') 152 self.assertEqual(pname.tag, '{test}p:name')

153

154 - def test_html_subelement_name_quote(self):

155 parser = self.etree.HTMLParser() 156 Element = parser.makeelement 157 SubElement = self.etree.SubElement 158 159 el = Element('name') 160 self.assertRaises(ValueError, SubElement, el, "name'") 161 self.assertRaises(ValueError, SubElement, el, 'na"me') 162 self.assertRaises(ValueError, SubElement, el, "{test}na'me") 163 self.assertRaises(ValueError, SubElement, el, '{test}"name')

164

165 - def test_html_subelement_name_space(self):

166 parser = self.etree.HTMLParser() 167 Element = parser.makeelement 168 SubElement = self.etree.SubElement 169 170 el = Element('name') 171 self.assertRaises(ValueError, SubElement, el, ' name ') 172 self.assertRaises(ValueError, SubElement, el, 'na me') 173 self.assertRaises(ValueError, SubElement, el, '{test} name')

174

175 - def test_module_parse_html_norecover(self):

176 parser = self.etree.HTMLParser(recover=False) 177 parse = self.etree.parse 178 f = BytesIO(self.broken_html_str) 179 self.assertRaises(self.etree.XMLSyntaxError, 180 parse, f, parser)

181

182 - def test_parse_encoding_8bit_explicit(self):

183 text = _str('Søk på nettet') 184 html_latin1 = (_str('<p>%s</p>') % text).encode('iso-8859-1') 185 186 tree = self.etree.parse( 187 BytesIO(html_latin1), 188 self.etree.HTMLParser(encoding="iso-8859-1")) 189 p = tree.find("//p") 190 self.assertEqual(p.text, text)

191

192 - def test_parse_encoding_8bit_override(self):

193 text = _str('Søk på nettet') 194 wrong_head = _str(''' 195 <head> 196 <meta http-equiv="Content-Type" 197 content="text/html; charset=UTF-8" /> 198 </head>''') 199 html_latin1 = (_str('<html>%s<body><p>%s</p></body></html>') % (wrong_head, 200 text) 201 ).encode('iso-8859-1') 202 203 self.assertRaises(self.etree.ParseError, 204 self.etree.parse, 205 BytesIO(html_latin1)) 206 207 tree = self.etree.parse( 208 BytesIO(html_latin1), 209 self.etree.HTMLParser(encoding="iso-8859-1")) 210 p = tree.find("//p") 211 self.assertEqual(p.text, text)

212

213 - def test_module_HTML_broken(self):

214 element = self.etree.HTML(self.broken_html_str) 215 self.assertEqual(self.etree.tostring(element, method="html"), 216 self.html_str)

217

218 - def test_module_HTML_cdata(self):

219 # by default, libxml2 generates CDATA nodes for <script> content 220 html = _bytes('<html><head><style>foo</style></head></html>') 221 element = self.etree.HTML(html) 222 self.assertEqual(element[0][0].text, "foo")

223

224 - def test_module_HTML_access(self):

225 element = self.etree.HTML(self.html_str) 226 self.assertEqual(element[0][0].tag, 'title')

227

228 - def test_module_parse_html(self):

229 parser = self.etree.HTMLParser() 230 filename = tempfile.mktemp(suffix=".html") 231 write_to_file(filename, self.html_str, 'wb') 232 try: 233 f = open(filename, 'rb') 234 tree = self.etree.parse(f, parser) 235 f.close() 236 self.assertEqual(self.etree.tostring(tree.getroot(), method="html"), 237 self.html_str) 238 finally: 239 os.remove(filename)

240

241 - def test_module_parse_html_filelike(self):

242 parser = self.etree.HTMLParser() 243 f = SillyFileLike(self.html_str) 244 tree = self.etree.parse(f, parser) 245 html = self.etree.tostring(tree.getroot(), 246 method="html", encoding='UTF-8') 247 self.assertEqual(html, self.html_str)

248 249 ## def test_module_parse_html_filelike_unicode(self): 250 ## parser = self.etree.HTMLParser() 251 ## f = SillyFileLike(self.uhtml_str) 252 ## tree = self.etree.parse(f, parser) 253 ## html = self.etree.tostring(tree.getroot(), encoding='UTF-8') 254 ## self.assertEqual(unicode(html, 'UTF-8'), self.uhtml_str) 255

256 - def test_html_file_error(self):

257 parser = self.etree.HTMLParser() 258 parse = self.etree.parse 259 self.assertRaises(IOError, 260 parse, "__some_hopefully_nonexisting_file__.html", 261 parser)

262

263 - def test_default_parser_HTML_broken(self):

264 self.assertRaises(self.etree.XMLSyntaxError, 265 self.etree.parse, BytesIO(self.broken_html_str)) 266 267 self.etree.set_default_parser( self.etree.HTMLParser() ) 268 269 tree = self.etree.parse(BytesIO(self.broken_html_str)) 270 self.assertEqual(self.etree.tostring(tree.getroot(), method="html"), 271 self.html_str) 272 273 self.etree.set_default_parser() 274 275 self.assertRaises(self.etree.XMLSyntaxError, 276 self.etree.parse, BytesIO(self.broken_html_str))

277

278 - def test_html_iterparse(self):

279 iterparse = self.etree.iterparse 280 f = BytesIO( 281 '<html><head><title>TITLE</title><body><p>P</p></body></html>') 282 283 iterator = iterparse(f, html=True) 284 self.assertEqual(None, iterator.root) 285 286 events = list(iterator) 287 root = iterator.root 288 self.assertTrue(root is not None) 289 self.assertEqual( 290 [('end', root[0][0]), ('end', root[0]), ('end', root[1][0]), 291 ('end', root[1]), ('end', root)], 292 events)

293

294 - def test_html_iterparse_stop_short(self):

295 iterparse = self.etree.iterparse 296 f = BytesIO( 297 '<html><head><title>TITLE</title><body><p>P</p></body></html>') 298 299 iterator = iterparse(f, html=True) 300 self.assertEqual(None, iterator.root) 301 302 event, element = next(iterator) 303 self.assertEqual('end', event) 304 self.assertEqual('title', element.tag) 305 self.assertEqual(None, iterator.root) 306 del element 307 308 event, element = next(iterator) 309 self.assertEqual('end', event) 310 self.assertEqual('head', element.tag) 311 self.assertEqual(None, iterator.root) 312 del element 313 del iterator

314

315 - def test_html_iterparse_broken(self):

316 iterparse = self.etree.iterparse 317 f = BytesIO('<head><title>TEST></head><p>P<br></div>') 318 319 iterator = iterparse(f, html=True) 320 self.assertEqual(None, iterator.root) 321 322 events = list(iterator) 323 root = iterator.root 324 self.assertTrue(root is not None) 325 self.assertEqual('html', root.tag) 326 self.assertEqual('head', root[0].tag) 327 self.assertEqual('body', root[1].tag) 328 self.assertEqual('p', root[1][0].tag) 329 self.assertEqual('br', root[1][0][0].tag) 330 self.assertEqual( 331 [('end', root[0][0]), ('end', root[0]), ('end', root[1][0][0]), 332 ('end', root[1][0]), ('end', root[1]), ('end', root)], 333 events)

334

335 - def test_html_iterparse_broken_no_recover(self):

336 iterparse = self.etree.iterparse 337 f = BytesIO('<p>P<br></div>') 338 iterator = iterparse(f, html=True, recover=False) 339 self.assertRaises(self.etree.XMLSyntaxError, list, iterator)

340

341 - def test_html_iterparse_file(self):

342 iterparse = self.etree.iterparse 343 iterator = iterparse(fileInTestDir("shakespeare.html"), 344 html=True) 345 346 self.assertEqual(None, iterator.root) 347 events = list(iterator) 348 root = iterator.root 349 self.assertTrue(root is not None) 350 self.assertEqual(249, len(events)) 351 self.assertFalse( 352 [event for (event, element) in events if event != 'end'])

353

354 - def test_html_iterparse_start(self):

355 iterparse = self.etree.iterparse 356 f = BytesIO( 357 '<html><head><title>TITLE</title><body><p>P</p></body></html>') 358 359 iterator = iterparse(f, html=True, events=('start',)) 360 self.assertEqual(None, iterator.root) 361 362 events = list(iterator) 363 root = iterator.root 364 self.assertNotEqual(None, root) 365 self.assertEqual( 366 [('start', root), ('start', root[0]), ('start', root[0][0]), 367 ('start', root[1]), ('start', root[1][0])], 368 events)

369

370 - def test_html_feed_parser(self):

371 parser = self.etree.HTMLParser() 372 parser.feed("<html><body></") 373 parser.feed("body></html>") 374 root = parser.close() 375 376 self.assertEqual('html', root.tag) 377 # test that we find all names in the parser dict 378 self.assertEqual([root], list(root.iter('html'))) 379 self.assertEqual([root[0]], list(root.iter('body')))

380

381 - def test_html_feed_parser_chunky(self):

382 parser = self.etree.HTMLParser() 383 parser.feed("<htm") 384 parser.feed("l><body") 385 parser.feed("><") 386 parser.feed("p><") 387 parser.feed("strong") 388 parser.feed(">some ") 389 parser.feed("text</strong></p><") 390 parser.feed("/body></html>") 391 root = parser.close() 392 393 self.assertEqual('html', root.tag) 394 # test that we find all names in the parser dict 395 self.assertEqual([root], list(root.iter('html'))) 396 self.assertEqual([root[0]], list(root.iter('body'))) 397 self.assertEqual([root[0][0]], list(root.iter('p'))) 398 self.assertEqual([root[0][0][0]], list(root.iter('strong')))

399

400 - def test_html_feed_parser_more_tags(self):

401 parser = self.etree.HTMLParser() 402 parser.feed('<html><head>') 403 parser.feed('<title>TITLE</title><body><p>P</p></body><') 404 parser.feed("/html>") 405 root = parser.close() 406 407 self.assertEqual('html', root.tag) 408 # test that we find all names in the parser dict 409 self.assertEqual([root], list(root.iter('html'))) 410 self.assertEqual([root[0]], list(root.iter('head'))) 411 self.assertEqual([root[0][0]], list(root.iter('title'))) 412 self.assertEqual([root[1]], list(root.iter('body'))) 413 self.assertEqual([root[1][0]], list(root.iter('p')))

414

415 - def test_html_parser_target_tag(self):

416 assertFalse = self.assertFalse 417 events = [] 418 class Target(object): 419 def start(self, tag, attrib): 420 events.append(("start", tag)) 421 assertFalse(attrib)

422 def end(self, tag): 423 events.append(("end", tag))

Source Code for Module lxml.tests.test_htmlparser