lxml.tests.test_htmlparser

23 """HTML parser test cases 24 """ 25 etree = etree 26 27 html_str = _bytes("<html><head><title>test</title></head><body><h1>page title</h1></body></html>") 28 html_str_pretty = _bytes("""\ 29 <html> 30 <head><title>test</title></head> 31 <body><h1>page title</h1></body> 32 </html> 33 """) 34 broken_html_str = _bytes("<html><head><title>test" 35 "<body><h1>page title</h3></p></html>") 36 uhtml_str = _bytes( 37 "<html><head><title>test Ã¡</title></head>" 38 "<body><h1>page Ã¡ title</h1></body></html>").decode('utf8') 39

40 - def tearDown(self):

41 super(HtmlParserTestCase, self).tearDown() 42 self.etree.set_default_parser()

43

44 - def test_module_HTML(self):

45 element = self.etree.HTML(self.html_str) 46 self.assertEqual(self.etree.tostring(element, method="html"), 47 self.html_str)

48

49 - def test_module_HTML_unicode(self):

50 element = self.etree.HTML(self.uhtml_str) 51 self.assertEqual( 52 self.etree.tostring(element, method="html", encoding='unicode'), 53 self.uhtml_str) 54 self.assertEqual(element.findtext('.//h1'), 55 _bytes("page Ã¡ title").decode('utf8'))

56

57 - def test_wide_unicode_xml(self):

58 if sys.maxunicode < 1114111: 59 return # skip test 60 element = self.etree.HTML(_bytes( 61 '<html><body><p>\\U00026007</p></body></html>' 62 ).decode('unicode_escape')) 63 p_text = element.findtext('.//p') 64 self.assertEqual(1, len(p_text)) 65 self.assertEqual(_bytes('\\U00026007').decode('unicode_escape'), 66 p_text)

67

68 - def test_module_HTML_pretty_print(self):

69 element = self.etree.HTML(self.html_str) 70 self.assertEqual(self.etree.tostring(element, method="html", pretty_print=True), 71 self.html_str_pretty)

72

73 - def test_module_parse_html_error(self):

74 parser = self.etree.HTMLParser(recover=False) 75 parse = self.etree.parse 76 f = BytesIO("<html></body>") 77 self.assertRaises(self.etree.XMLSyntaxError, 78 parse, f, parser)

79

80 - def test_html_element_name_empty(self):

81 parser = self.etree.HTMLParser() 82 Element = parser.makeelement 83 84 el = Element('name') 85 self.assertRaises(ValueError, Element, '{}') 86 self.assertRaises(ValueError, setattr, el, 'tag', '{}') 87 88 self.assertRaises(ValueError, Element, '{test}') 89 self.assertRaises(ValueError, setattr, el, 'tag', '{test}')

90

91 - def test_html_element_name_colon(self):

92 parser = self.etree.HTMLParser() 93 Element = parser.makeelement 94 95 pname = Element('p:name') 96 self.assertEqual(pname.tag, 'p:name') 97 98 pname = Element('{test}p:name') 99 self.assertEqual(pname.tag, '{test}p:name') 100 101 pname = Element('name') 102 pname.tag = 'p:name' 103 self.assertEqual(pname.tag, 'p:name')

104

105 - def test_html_element_name_quote(self):

106 parser = self.etree.HTMLParser() 107 Element = parser.makeelement 108 109 self.assertRaises(ValueError, Element, 'p"name') 110 self.assertRaises(ValueError, Element, "na'me") 111 self.assertRaises(ValueError, Element, '{test}"name') 112 self.assertRaises(ValueError, Element, "{test}name'") 113 114 el = Element('name') 115 self.assertRaises(ValueError, setattr, el, 'tag', "pname'") 116 self.assertRaises(ValueError, setattr, el, 'tag', '"pname') 117 self.assertEqual(el.tag, "name")

118

119 - def test_html_element_name_space(self):

120 parser = self.etree.HTMLParser() 121 Element = parser.makeelement 122 123 self.assertRaises(ValueError, Element, ' name ') 124 self.assertRaises(ValueError, Element, 'na me') 125 self.assertRaises(ValueError, Element, '{test} name') 126 127 el = Element('name') 128 self.assertRaises(ValueError, setattr, el, 'tag', ' name ') 129 self.assertEqual(el.tag, "name")

130

131 - def test_html_subelement_name_empty(self):

132 parser = self.etree.HTMLParser() 133 Element = parser.makeelement 134 135 SubElement = self.etree.SubElement 136 137 el = Element('name') 138 self.assertRaises(ValueError, SubElement, el, '{}') 139 self.assertRaises(ValueError, SubElement, el, '{test}')

140

141 - def test_html_subelement_name_colon(self):

142 parser = self.etree.HTMLParser() 143 Element = parser.makeelement 144 SubElement = self.etree.SubElement 145 146 el = Element('name') 147 pname = SubElement(el, 'p:name') 148 self.assertEqual(pname.tag, 'p:name') 149 150 pname = SubElement(el, '{test}p:name') 151 self.assertEqual(pname.tag, '{test}p:name')

152

153 - def test_html_subelement_name_quote(self):

154 parser = self.etree.HTMLParser() 155 Element = parser.makeelement 156 SubElement = self.etree.SubElement 157 158 el = Element('name') 159 self.assertRaises(ValueError, SubElement, el, "name'") 160 self.assertRaises(ValueError, SubElement, el, 'na"me') 161 self.assertRaises(ValueError, SubElement, el, "{test}na'me") 162 self.assertRaises(ValueError, SubElement, el, '{test}"name')

163

164 - def test_html_subelement_name_space(self):

165 parser = self.etree.HTMLParser() 166 Element = parser.makeelement 167 SubElement = self.etree.SubElement 168 169 el = Element('name') 170 self.assertRaises(ValueError, SubElement, el, ' name ') 171 self.assertRaises(ValueError, SubElement, el, 'na me') 172 self.assertRaises(ValueError, SubElement, el, '{test} name')

173

174 - def test_module_parse_html_norecover(self):

175 parser = self.etree.HTMLParser(recover=False) 176 parse = self.etree.parse 177 f = BytesIO(self.broken_html_str) 178 self.assertRaises(self.etree.XMLSyntaxError, 179 parse, f, parser)

180

181 - def test_parse_encoding_8bit_explicit(self):

182 text = _str('Søk på nettet') 183 html_latin1 = (_str('<p>%s</p>') % text).encode('iso-8859-1') 184 185 tree = self.etree.parse( 186 BytesIO(html_latin1), 187 self.etree.HTMLParser(encoding="iso-8859-1")) 188 p = tree.find("//p") 189 self.assertEqual(p.text, text)

190

191 - def test_parse_encoding_8bit_override(self):

192 text = _str('Søk på nettet') 193 wrong_head = _str(''' 194 <head> 195 <meta http-equiv="Content-Type" 196 content="text/html; charset=UTF-8" /> 197 </head>''') 198 html_latin1 = (_str('<html>%s<body><p>%s</p></body></html>') % (wrong_head, 199 text) 200 ).encode('iso-8859-1') 201 202 self.assertRaises(self.etree.ParseError, 203 self.etree.parse, 204 BytesIO(html_latin1)) 205 206 tree = self.etree.parse( 207 BytesIO(html_latin1), 208 self.etree.HTMLParser(encoding="iso-8859-1")) 209 p = tree.find("//p") 210 self.assertEqual(p.text, text)

211

212 - def test_module_HTML_broken(self):

213 element = self.etree.HTML(self.broken_html_str) 214 self.assertEqual(self.etree.tostring(element, method="html"), 215 self.html_str)

216

217 - def test_module_HTML_cdata(self):

218 # by default, libxml2 generates CDATA nodes for <script> content 219 html = _bytes('<html><head><style>foo</style></head></html>') 220 element = self.etree.HTML(html) 221 self.assertEqual(element[0][0].text, "foo")

222

223 - def test_module_HTML_access(self):

224 element = self.etree.HTML(self.html_str) 225 self.assertEqual(element[0][0].tag, 'title')

226

227 - def test_module_parse_html(self):

228 parser = self.etree.HTMLParser() 229 filename = tempfile.mktemp(suffix=".html") 230 write_to_file(filename, self.html_str, 'wb') 231 try: 232 f = open(filename, 'rb') 233 tree = self.etree.parse(f, parser) 234 f.close() 235 self.assertEqual(self.etree.tostring(tree.getroot(), method="html"), 236 self.html_str) 237 finally: 238 os.remove(filename)

239

240 - def test_module_parse_html_filelike(self):

241 parser = self.etree.HTMLParser() 242 f = SillyFileLike(self.html_str) 243 tree = self.etree.parse(f, parser) 244 html = self.etree.tostring(tree.getroot(), 245 method="html", encoding='UTF-8') 246 self.assertEqual(html, self.html_str)

247 248 ## def test_module_parse_html_filelike_unicode(self): 249 ## parser = self.etree.HTMLParser() 250 ## f = SillyFileLike(self.uhtml_str) 251 ## tree = self.etree.parse(f, parser) 252 ## html = self.etree.tostring(tree.getroot(), encoding='UTF-8') 253 ## self.assertEqual(unicode(html, 'UTF-8'), self.uhtml_str) 254

255 - def test_html_file_error(self):

256 parser = self.etree.HTMLParser() 257 parse = self.etree.parse 258 self.assertRaises(IOError, 259 parse, "__some_hopefully_nonexisting_file__.html", 260 parser)

261

262 - def test_default_parser_HTML_broken(self):

263 self.assertRaises(self.etree.XMLSyntaxError, 264 self.etree.parse, BytesIO(self.broken_html_str)) 265 266 self.etree.set_default_parser( self.etree.HTMLParser() ) 267 268 tree = self.etree.parse(BytesIO(self.broken_html_str)) 269 self.assertEqual(self.etree.tostring(tree.getroot(), method="html"), 270 self.html_str) 271 272 self.etree.set_default_parser() 273 274 self.assertRaises(self.etree.XMLSyntaxError, 275 self.etree.parse, BytesIO(self.broken_html_str))

276

277 - def test_html_iterparse(self):

278 iterparse = self.etree.iterparse 279 f = BytesIO( 280 '<html><head><title>TITLE</title><body><p>P</p></body></html>') 281 282 iterator = iterparse(f, html=True) 283 self.assertEqual(None, iterator.root) 284 285 events = list(iterator) 286 root = iterator.root 287 self.assertTrue(root is not None) 288 self.assertEqual( 289 [('end', root[0][0]), ('end', root[0]), ('end', root[1][0]), 290 ('end', root[1]), ('end', root)], 291 events)

292

293 - def test_html_iterparse_stop_short(self):

294 iterparse = self.etree.iterparse 295 f = BytesIO( 296 '<html><head><title>TITLE</title><body><p>P</p></body></html>') 297 298 iterator = iterparse(f, html=True) 299 self.assertEqual(None, iterator.root) 300 301 event, element = next(iterator) 302 self.assertEqual('end', event) 303 self.assertEqual('title', element.tag) 304 self.assertEqual(None, iterator.root) 305 del element 306 307 event, element = next(iterator) 308 self.assertEqual('end', event) 309 self.assertEqual('head', element.tag) 310 self.assertEqual(None, iterator.root) 311 del element 312 del iterator

313

314 - def test_html_iterparse_broken(self):

315 iterparse = self.etree.iterparse 316 f = BytesIO('<head><title>TEST></head><p>P<br></div>') 317 318 iterator = iterparse(f, html=True) 319 self.assertEqual(None, iterator.root) 320 321 events = list(iterator) 322 root = iterator.root 323 self.assertTrue(root is not None) 324 self.assertEqual('html', root.tag) 325 self.assertEqual('head', root[0].tag) 326 self.assertEqual('body', root[1].tag) 327 self.assertEqual('p', root[1][0].tag) 328 self.assertEqual('br', root[1][0][0].tag) 329 self.assertEqual( 330 [('end', root[0][0]), ('end', root[0]), ('end', root[1][0][0]), 331 ('end', root[1][0]), ('end', root[1]), ('end', root)], 332 events)

333

334 - def test_html_iterparse_broken_no_recover(self):

335 iterparse = self.etree.iterparse 336 f = BytesIO('<p>P<br></div>') 337 iterator = iterparse(f, html=True, recover=False) 338 self.assertRaises(self.etree.XMLSyntaxError, list, iterator)

339

340 - def test_html_iterparse_file(self):

341 iterparse = self.etree.iterparse 342 iterator = iterparse(fileInTestDir("shakespeare.html"), 343 html=True) 344 345 self.assertEqual(None, iterator.root) 346 events = list(iterator) 347 root = iterator.root 348 self.assertTrue(root is not None) 349 self.assertEqual(249, len(events)) 350 self.assertFalse( 351 [event for (event, element) in events if event != 'end'])

352

353 - def test_html_iterparse_start(self):

354 iterparse = self.etree.iterparse 355 f = BytesIO( 356 '<html><head><title>TITLE</title><body><p>P</p></body></html>') 357 358 iterator = iterparse(f, html=True, events=('start',)) 359 self.assertEqual(None, iterator.root) 360 361 events = list(iterator) 362 root = iterator.root 363 self.assertNotEqual(None, root) 364 self.assertEqual( 365 [('start', root), ('start', root[0]), ('start', root[0][0]), 366 ('start', root[1]), ('start', root[1][0])], 367 events)

368

369 - def test_html_feed_parser(self):

370 parser = self.etree.HTMLParser() 371 parser.feed("<html><body></") 372 parser.feed("body></html>") 373 root = parser.close() 374 375 self.assertEqual('html', root.tag) 376 # test that we find all names in the parser dict 377 self.assertEqual([root], list(root.iter('html'))) 378 self.assertEqual([root[0]], list(root.iter('body')))

379

380 - def test_html_feed_parser_chunky(self):

381 parser = self.etree.HTMLParser() 382 parser.feed("<htm") 383 parser.feed("l><body") 384 parser.feed("><") 385 parser.feed("p><") 386 parser.feed("strong") 387 parser.feed(">some ") 388 parser.feed("text</strong></p><") 389 parser.feed("/body></html>") 390 root = parser.close() 391 392 self.assertEqual('html', root.tag) 393 # test that we find all names in the parser dict 394 self.assertEqual([root], list(root.iter('html'))) 395 self.assertEqual([root[0]], list(root.iter('body'))) 396 self.assertEqual([root[0][0]], list(root.iter('p'))) 397 self.assertEqual([root[0][0][0]], list(root.iter('strong')))

398

399 - def test_html_feed_parser_more_tags(self):

400 parser = self.etree.HTMLParser() 401 parser.feed('<html><head>') 402 parser.feed('<title>TITLE</title><body><p>P</p></body><') 403 parser.feed("/html>") 404 root = parser.close() 405 406 self.assertEqual('html', root.tag) 407 # test that we find all names in the parser dict 408 self.assertEqual([root], list(root.iter('html'))) 409 self.assertEqual([root[0]], list(root.iter('head'))) 410 self.assertEqual([root[0][0]], list(root.iter('title'))) 411 self.assertEqual([root[1]], list(root.iter('body'))) 412 self.assertEqual([root[1][0]], list(root.iter('p')))

413

414 - def test_html_parser_target_tag(self):

415 assertFalse = self.assertFalse 416 events = [] 417 class Target(object): 418 def start(self, tag, attrib): 419 events.append(("start", tag)) 420 assertFalse(attrib)

421 def end(self, tag): 422 events.append(("end", tag))

Source Code for Module lxml.tests.test_htmlparser