lxml.cssselect

1 """CSS Selectors based on XPath. 2 3 This module supports selecting XML/HTML tags based on CSS selectors. 4 See the `CSSSelector` class for details. 5 """ 6 7 import re 8 from lxml import etree 9 10 __all__ = ['SelectorSyntaxError', 'ExpressionError', 11 'CSSSelector'] 12 13 try: 14 _basestring = basestring 15 except NameError: 16 _basestring = str 17

18 -class SelectorSyntaxError(SyntaxError):

19 pass

20

21 -class ExpressionError(RuntimeError):

22 pass

23

24 -class CSSSelector(etree.XPath):

25 """A CSS selector. 26 27 Usage:: 28 29 >>> from lxml import etree, cssselect 30 >>> select = cssselect.CSSSelector("a tag > child") 31 32 >>> root = etree.XML("<a><b><c/><tag><child>TEXT</child></tag></b></a>") 33 >>> [ el.tag for el in select(root) ] 34 ['child'] 35 """

36 - def __init__(self, css):

37 path = css_to_xpath(css) 38 etree.XPath.__init__(self, path) 39 self.css = css

40

41 - def __repr__(self):

42 return '<%s %s for %r>' % ( 43 self.__class__.__name__, 44 hex(abs(id(self)))[2:], 45 self.css)

46 47 ############################## 48 ## Token objects: 49 50 try: 51 _unicode = unicode 52 except NameError: 53 # Python 3 54 _unicode = str 55

56 -class _UniToken(_unicode):

57 - def __new__(cls, contents, pos):

58 obj = _unicode.__new__(cls, contents) 59 obj.pos = pos 60 return obj

61

62 - def __repr__(self):

63 return '%s(%s, %r)' % ( 64 self.__class__.__name__, 65 _unicode.__repr__(self), 66 self.pos)

67

68 -class Symbol(_UniToken):

69 pass

70

71 -class String(_UniToken):

72 pass

73

74 -class Token(_UniToken):

75 pass

76 77 ############################################################ 78 ## Parsing 79 ############################################################ 80 81 ############################## 82 ## Syntax objects: 83

84 -class Class(object):

85 """ 86 Represents selector.class_name 87 """ 88

89 - def __init__(self, selector, class_name):

90 self.selector = selector 91 self.class_name = class_name

92

93 - def __repr__(self):

94 return '%s[%r.%s]' % ( 95 self.__class__.__name__, 96 self.selector, 97 self.class_name)

98

99 - def xpath(self):

100 sel_xpath = self.selector.xpath() 101 sel_xpath.add_condition( 102 "contains(concat(' ', normalize-space(@class), ' '), %s)" % xpath_repr(' '+self.class_name+' ')) 103 return sel_xpath

104

105 -class Function(object):

106 """ 107 Represents selector:name(expr) 108 """ 109 110 unsupported = [ 111 'target', 'lang', 'enabled', 'disabled',] 112

113 - def __init__(self, selector, type, name, expr):

114 self.selector = selector 115 self.type = type 116 self.name = name 117 self.expr = expr

118

119 - def __repr__(self):

120 return '%s[%r%s%s(%r)]' % ( 121 self.__class__.__name__, 122 self.selector, 123 self.type, self.name, self.expr)

124

125 - def xpath(self):

126 sel_path = self.selector.xpath() 127 if self.name in self.unsupported: 128 raise ExpressionError( 129 "The psuedo-class %r is not supported" % self.name) 130 method = '_xpath_' + self.name.replace('-', '_') 131 if not hasattr(self, method): 132 raise ExpressionError( 133 "The psuedo-class %r is unknown" % self.name) 134 method = getattr(self, method) 135 return method(sel_path, self.expr)

136

137 - def _xpath_nth_child(self, xpath, expr, last=False, 138 add_name_test=True):

139 a, b = parse_series(expr) 140 if not a and not b and not last: 141 # a=0 means nothing is returned... 142 xpath.add_condition('false() and position() = 0') 143 return xpath 144 if add_name_test: 145 xpath.add_name_test() 146 xpath.add_star_prefix() 147 if a == 0: 148 if last: 149 b = 'last() - %s' % b 150 xpath.add_condition('position() = %s' % b) 151 return xpath 152 if last: 153 # FIXME: I'm not sure if this is right 154 a = -a 155 b = -b 156 if b > 0: 157 b_neg = str(-b) 158 else: 159 b_neg = '+%s' % (-b) 160 if a != 1: 161 expr = ['(position() %s) mod %s = 0' % (b_neg, a)] 162 else: 163 expr = [] 164 if b >= 0: 165 expr.append('position() >= %s' % b) 166 elif b < 0 and last: 167 expr.append('position() < (last() %s)' % b) 168 expr = ' and '.join(expr) 169 if expr: 170 xpath.add_condition(expr) 171 return xpath

172 # FIXME: handle an+b, odd, even 173 # an+b means every-a, plus b, e.g., 2n+1 means odd 174 # 0n+b means b 175 # n+0 means a=1, i.e., all elements 176 # an means every a elements, i.e., 2n means even 177 # -n means -1n 178 # -1n+6 means elements 6 and previous 179

180 - def _xpath_nth_last_child(self, xpath, expr):

181 return self._xpath_nth_child(xpath, expr, last=True)

182

183 - def _xpath_nth_of_type(self, xpath, expr):

184 if xpath.element == '*': 185 raise NotImplementedError( 186 "*:nth-of-type() is not implemented") 187 return self._xpath_nth_child(xpath, expr, add_name_test=False)

188

189 - def _xpath_nth_last_of_type(self, xpath, expr):

190 return self._xpath_nth_child(xpath, expr, last=True, add_name_test=False)

191

192 - def _xpath_contains(self, xpath, expr):

193 # text content, minus tags, must contain expr 194 if isinstance(expr, Element): 195 expr = expr._format_element() 196 xpath.add_condition('contains(css:lower-case(string(.)), %s)' 197 % xpath_repr(expr.lower())) 198 # FIXME: Currently case insensitive matching doesn't seem to be happening 199 return xpath

200

201 - def _xpath_not(self, xpath, expr):

202 # everything for which not expr applies 203 expr = expr.xpath() 204 cond = expr.condition 205 # FIXME: should I do something about element_path? 206 xpath.add_condition('not(%s)' % cond) 207 return xpath

208

209 -def _make_lower_case(context, s):

210 return s.lower()

211 212 ns = etree.FunctionNamespace('/css/') 213 ns.prefix = 'css' 214 ns['lower-case'] = _make_lower_case 215

216 -class Pseudo(object):

217 """ 218 Represents selector:ident 219 """ 220 221 unsupported = ['indeterminate', 'first-line', 'first-letter', 222 'selection', 'before', 'after', 'link', 'visited', 223 'active', 'focus', 'hover'] 224

225 - def __init__(self, element, type, ident):

226 self.element = element 227 assert type in (':', '::') 228 self.type = type 229 self.ident = ident

230

231 - def __repr__(self):

232 return '%s[%r%s%s]' % ( 233 self.__class__.__name__, 234 self.element, 235 self.type, self.ident)

236

237 - def xpath(self):

238 el_xpath = self.element.xpath() 239 if self.ident in self.unsupported: 240 raise ExpressionError( 241 "The psuedo-class %r is unsupported" % self.ident) 242 method = '_xpath_' + self.ident.replace('-', '_') 243 if not hasattr(self, method): 244 raise ExpressionError( 245 "The psuedo-class %r is unknown" % self.ident) 246 method = getattr(self, method) 247 el_xpath = method(el_xpath) 248 return el_xpath

249

250 - def _xpath_checked(self, xpath):

251 # FIXME: is this really all the elements? 252 xpath.add_condition("(@selected or @checked) and (name(.) = 'input' or name(.) = 'option')") 253 return xpath

254

255 - def _xpath_root(self, xpath):

256 # if this element is the root element 257 raise NotImplementedError

258

259 - def _xpath_first_child(self, xpath):

260 xpath.add_star_prefix() 261 xpath.add_name_test() 262 xpath.add_condition('position() = 1') 263 return xpath

264

265 - def _xpath_last_child(self, xpath):

266 xpath.add_star_prefix() 267 xpath.add_name_test() 268 xpath.add_condition('position() = last()') 269 return xpath

270

271 - def _xpath_first_of_type(self, xpath):

272 if xpath.element == '*': 273 raise NotImplementedError( 274 "*:first-of-type is not implemented") 275 xpath.add_star_prefix() 276 xpath.add_condition('position() = 1') 277 return xpath

278

279 - def _xpath_last_of_type(self, xpath):

280 if xpath.element == '*': 281 raise NotImplementedError( 282 "*:last-of-type is not implemented") 283 xpath.add_star_prefix() 284 xpath.add_condition('position() = last()') 285 return xpath

286

287 - def _xpath_only_child(self, xpath):

288 xpath.add_name_test() 289 xpath.add_star_prefix() 290 xpath.add_condition('last() = 1') 291 return xpath

292

293 - def _xpath_only_of_type(self, xpath):

294 if xpath.element == '*': 295 raise NotImplementedError( 296 "*:only-of-type is not implemented") 297 xpath.add_condition('last() = 1') 298 return xpath

299

300 - def _xpath_empty(self, xpath):

301 xpath.add_condition("not(*) and not(normalize-space())") 302 return xpath

303

304 -class Attrib(object):

305 """ 306 Represents selector[namespace|attrib operator value] 307 """ 308

309 - def __init__(self, selector, namespace, attrib, operator, value):

310 self.selector = selector 311 self.namespace = namespace 312 self.attrib = attrib 313 self.operator = operator 314 self.value = value

315

316 - def __repr__(self):

317 if self.operator == 'exists': 318 return '%s[%r[%s]]' % ( 319 self.__class__.__name__, 320 self.selector, 321 self._format_attrib()) 322 else: 323 return '%s[%r[%s %s %r]]' % ( 324 self.__class__.__name__, 325 self.selector, 326 self._format_attrib(), 327 self.operator, 328 self.value)

329

330 - def _format_attrib(self):

331 if self.namespace == '*': 332 return self.attrib 333 else: 334 return '%s|%s' % (self.namespace, self.attrib)

335

336 - def _xpath_attrib(self):

337 # FIXME: if attrib is *? 338 if self.namespace == '*': 339 return '@' + self.attrib 340 else: 341 return '@%s:%s' % (self.namespace, self.attrib)

342

343 - def xpath(self):

344 path = self.selector.xpath() 345 attrib = self._xpath_attrib() 346 value = self.value 347 if self.operator == 'exists': 348 assert not value 349 path.add_condition(attrib) 350 elif self.operator == '=': 351 path.add_condition('%s = %s' % (attrib, 352 xpath_repr(value))) 353 elif self.operator == '!=': 354 # FIXME: this seems like a weird hack... 355 if value: 356 path.add_condition('not(%s) or %s != %s' 357 % (attrib, attrib, xpath_repr(value))) 358 else: 359 path.add_condition('%s != %s' 360 % (attrib, xpath_repr(value))) 361 #path.add_condition('%s != %s' % (attrib, xpath_repr(value))) 362 elif self.operator == '~=': 363 path.add_condition("contains(concat(' ', normalize-space(%s), ' '), %s)" % (attrib, xpath_repr(' '+value+' '))) 364 elif self.operator == '|=': 365 # Weird, but true... 366 path.add_condition('%s = %s or starts-with(%s, %s)' % ( 367 attrib, xpath_repr(value), 368 attrib, xpath_repr(value + '-'))) 369 elif self.operator == '^=': 370 path.add_condition('starts-with(%s, %s)' % ( 371 attrib, xpath_repr(value))) 372 elif self.operator == '$=': 373 # Oddly there is a starts-with in XPath 1.0, but not ends-with 374 path.add_condition('substring(%s, string-length(%s)-%s) = %s' 375 % (attrib, attrib, len(value)-1, xpath_repr(value))) 376 elif self.operator == '*=': 377 # FIXME: case sensitive? 378 path.add_condition('contains(%s, %s)' % ( 379 attrib, xpath_repr(value))) 380 else: 381 assert 0, ("Unknown operator: %r" % self.operator) 382 return path

383

384 -class Element(object):

385 """ 386 Represents namespace|element 387 """ 388

389 - def __init__(self, namespace, element):

390 self.namespace = namespace 391 self.element = element

392

393 - def __repr__(self):

394 return '%s[%s]' % ( 395 self.__class__.__name__, 396 self._format_element())

397

398 - def _format_element(self):

399 if self.namespace == '*': 400 return self.element 401 else: 402 return '%s|%s' % (self.namespace, self.element)

403

404 - def xpath(self):

405 if self.namespace == '*': 406 el = self.element.lower() 407 else: 408 # FIXME: Should we lowercase here? 409 el = '%s:%s' % (self.namespace, self.element) 410 return XPathExpr(element=el)

411

412 -class Hash(object):

413 """ 414 Represents selector#id 415 """ 416

417 - def __init__(self, selector, id):

418 self.selector = selector 419 self.id = id

420

421 - def __repr__(self):

422 return '%s[%r#%s]' % ( 423 self.__class__.__name__, 424 self.selector, self.id)

425

426 - def xpath(self):

427 path = self.selector.xpath() 428 path.add_condition('@id = %s' % xpath_repr(self.id)) 429 return path

430

431 -class Or(object):

432

433 - def __init__(self, items):

434 self.items = items

435 - def __repr__(self):

436 return '%s(%r)' % ( 437 self.__class__.__name__, 438 self.items)

439

440 - def xpath(self):

441 paths = [item.xpath() for item in self.items] 442 return XPathExprOr(paths)

443

444 -class CombinedSelector(object):

445 446 _method_mapping = { 447 ' ': 'descendant', 448 '>': 'child', 449 '+': 'direct_adjacent', 450 '~': 'indirect_adjacent', 451 } 452

453 - def __init__(self, selector, combinator, subselector):

454 assert selector is not None 455 self.selector = selector 456 self.combinator = combinator 457 self.subselector = subselector

458

459 - def __repr__(self):

460 if self.combinator == ' ': 461 comb = '<followed>' 462 else: 463 comb = self.combinator 464 return '%s[%r %s %r]' % ( 465 self.__class__.__name__, 466 self.selector, 467 comb, 468 self.subselector)

469

470 - def xpath(self):

471 if self.combinator not in self._method_mapping: 472 raise ExpressionError( 473 "Unknown combinator: %r" % self.combinator) 474 method = '_xpath_' + self._method_mapping[self.combinator] 475 method = getattr(self, method) 476 path = self.selector.xpath() 477 return method(path, self.subselector)

478

479 - def _xpath_descendant(self, xpath, sub):

480 # when sub is a descendant in any way of xpath 481 xpath.join('/descendant::', sub.xpath()) 482 return xpath

483

484 - def _xpath_child(self, xpath, sub):

485 # when sub is an immediate child of xpath 486 xpath.join('/', sub.xpath()) 487 return xpath

488

489 - def _xpath_direct_adjacent(self, xpath, sub):

490 # when sub immediately follows xpath 491 xpath.join('/following-sibling::', sub.xpath()) 492 xpath.add_name_test() 493 xpath.add_condition('position() = 1') 494 return xpath

495

496 - def _xpath_indirect_adjacent(self, xpath, sub):

497 # when sub comes somewhere after xpath as a sibling 498 xpath.join('/following-sibling::', sub.xpath()) 499 return xpath

500 501 ############################## 502 ## XPathExpr objects: 503 504 _el_re = re.compile(r'^\w+\s*$') 505 _id_re = re.compile(r'^(\w*)#(\w+)\s*$') 506 _class_re = re.compile(r'^(\w*)\.(\w+)\s*$') 507

508 -def css_to_xpath(css_expr, prefix='descendant-or-self::'):

509 if isinstance(css_expr, _basestring): 510 match = _el_re.search(css_expr) 511 if match is not None: 512 return '%s%s' % (prefix, match.group(0).strip()) 513 match = _id_re.search(css_expr) 514 if match is not None: 515 return "%s%s[@id = '%s']" % ( 516 prefix, match.group(1) or '*', match.group(2)) 517 match = _class_re.search(css_expr) 518 if match is not None: 519 return "%s%s[contains(concat(' ', normalize-space(@class), ' '), ' %s ')]" % ( 520 prefix, match.group(1) or '*', match.group(2)) 521 css_expr = parse(css_expr) 522 expr = css_expr.xpath() 523 assert expr is not None, ( 524 "Got None for xpath expression from %s" % repr(css_expr)) 525 if prefix: 526 expr.add_prefix(prefix) 527 return str(expr)

528

529 -class XPathExpr(object):

530

531 - def __init__(self, prefix=None, path=None, element='*', condition=None, 532 star_prefix=False):

533 self.prefix = prefix 534 self.path = path 535 self.element = element 536 self.condition = condition 537 self.star_prefix = star_prefix

538

539 - def __str__(self):

540 path = '' 541 if self.prefix is not None: 542 path += str(self.prefix) 543 if self.path is not None: 544 path += str(self.path) 545 path += str(self.element) 546 if self.condition: 547 path += '[%s]' % self.condition 548 return path

549

550 - def __repr__(self):

551 return '%s[%s]' % ( 552 self.__class__.__name__, self)

553

554 - def add_condition(self, condition):

555 if self.condition: 556 self.condition = '%s and (%s)' % (self.condition, condition) 557 else: 558 self.condition = condition

559

560 - def add_path(self, part):

561 if self.path is None: 562 self.path = self.element 563 else: 564 self.path += self.element 565 self.element = part

566

567 - def add_prefix(self, prefix):

568 if self.prefix: 569 self.prefix = prefix + self.prefix 570 else: 571 self.prefix = prefix

572

573 - def add_name_test(self):

574 if self.element == '*': 575 # We weren't doing a test anyway 576 return 577 self.add_condition("name() = %s" % xpath_repr(self.element)) 578 self.element = '*'

579

580 - def add_star_prefix(self):

581 """ 582 Adds a /* prefix if there is no prefix. This is when you need 583 to keep context's constrained to a single parent. 584 """ 585 if self.path: 586 self.path += '*/' 587 else: 588 self.path = '*/' 589 self.star_prefix = True

590

591 - def join(self, combiner, other):

592 prefix = str(self) 593 prefix += combiner 594 path = (other.prefix or '') + (other.path or '') 595 # We don't need a star prefix if we are joining to this other 596 # prefix; so we'll get rid of it 597 if other.star_prefix and path == '*/': 598 path = '' 599 self.prefix = prefix 600 self.path = path 601 self.element = other.element 602 self.condition = other.condition

603

604 -class XPathExprOr(XPathExpr):

605 """ 606 Represents |'d expressions. Note that unfortunately it isn't 607 the union, it's the sum, so duplicate elements will appear. 608 """ 609

610 - def __init__(self, items, prefix=None):

611 for item in items: 612 assert item is not None 613 self.items = items 614 self.prefix = prefix

615

616 - def __str__(self):

617 prefix = self.prefix or '' 618 return ' | '.join([prefix + str(i) for i in self.items])

619

620 -def xpath_repr(s):

621 # FIXME: I don't think this is right, but lacking any reasonable 622 # specification on what XPath literals look like (which doesn't seem 623 # to be in the XPath specification) it is hard to do 'right' 624 if isinstance(s, Element): 625 # This is probably a symbol that looks like an expression... 626 s = s._format_element() 627 return repr(str(s))

628 629 ############################## 630 ## Parsing functions 631

632 -def parse(string):

633 stream = TokenStream(tokenize(string)) 634 stream.source = string 635 try: 636 return parse_selector_group(stream) 637 except SelectorSyntaxError: 638 import sys 639 e = sys.exc_info()[1] 640 e.args = tuple(["%s at %s -> %s" % ( 641 e, stream.used, list(stream))]) 642 raise

643

644 -def parse_selector_group(stream):

645 result = [] 646 while 1: 647 result.append(parse_selector(stream)) 648 if stream.peek() == ',': 649 stream.next() 650 else: 651 break 652 if len(result) == 1: 653 return result[0] 654 else: 655 return Or(result)

656

657 -def parse_selector(stream):

658 result = parse_simple_selector(stream) 659 while 1: 660 peek = stream.peek() 661 if peek == ',' or peek is None: 662 return result 663 elif peek in ('+', '>', '~'): 664 # A combinator 665 combinator = stream.next() 666 else: 667 combinator = ' ' 668 next_selector = parse_simple_selector(stream) 669 result = CombinedSelector(result, combinator, next_selector) 670 return result

671

672 -def parse_simple_selector(stream):

673 peek = stream.peek() 674 if peek != '*' and not isinstance(peek, Symbol): 675 element = namespace = '*' 676 else: 677 next = stream.next() 678 if next != '*' and not isinstance(next, Symbol): 679 raise SelectorSyntaxError( 680 "Expected symbol, got %r" % next) 681 if stream.peek() == '|': 682 namespace = next 683 stream.next() 684 element = stream.next() 685 if element != '*' and not isinstance(next, Symbol): 686 raise SelectorSyntaxError( 687 "Expected symbol, got %r" % next) 688 else: 689 namespace = '*' 690 element = next 691 result = Element(namespace, element) 692 has_hash = False 693 while 1: 694 peek = stream.peek() 695 if peek == '#': 696 if has_hash: 697 # You can't have two hashes 698 # (FIXME: is there some more general rule I'm missing?) 699 break 700 stream.next() 701 result = Hash(result, stream.next()) 702 has_hash = True 703 continue 704 elif peek == '.': 705 stream.next() 706 result = Class(result, stream.next()) 707 continue 708 elif peek == '[': 709 stream.next() 710 result = parse_attrib(result, stream) 711 next = stream.next() 712 if not next == ']': 713 raise SelectorSyntaxError( 714 "] expected, got %r" % next) 715 continue 716 elif peek == ':' or peek == '::': 717 type = stream.next() 718 ident = stream.next() 719 if not isinstance(ident, Symbol): 720 raise SelectorSyntaxError( 721 "Expected symbol, got %r" % ident) 722 if stream.peek() == '(': 723 stream.next() 724 peek = stream.peek() 725 if isinstance(peek, String): 726 selector = stream.next() 727 elif isinstance(peek, Symbol) and is_int(peek): 728 selector = int(stream.next()) 729 else: 730 # FIXME: parse_simple_selector, or selector, or...? 731 selector = parse_simple_selector(stream) 732 next = stream.next() 733 if not next == ')': 734 raise SelectorSyntaxError( 735 "Expected ), got %r and %r" 736 % (next, selector)) 737 result = Function(result, type, ident, selector) 738 else: 739 result = Pseudo(result, type, ident) 740 continue 741 else: 742 if peek == ' ': 743 stream.next() 744 break 745 # FIXME: not sure what "negation" is 746 return result

747

748 -def is_int(v):

749 try: 750 int(v) 751 except ValueError: 752 return False 753 else: 754 return True

755

756 -def parse_attrib(selector, stream):

757 attrib = stream.next() 758 if stream.peek() == '|': 759 namespace = attrib 760 stream.next() 761 attrib = stream.next() 762 else: 763 namespace = '*' 764 if stream.peek() == ']': 765 return Attrib(selector, namespace, attrib, 'exists', None) 766 op = stream.next() 767 if not op in ('^=', '$=', '*=', '=', '~=', '|=', '!='): 768 raise SelectorSyntaxError( 769 "Operator expected, got %r" % op) 770 value = stream.next() 771 if not isinstance(value, (Symbol, String)): 772 raise SelectorSyntaxError( 773 "Expected string or symbol, got %r" % value) 774 return Attrib(selector, namespace, attrib, op, value)

775

776 -def parse_series(s):

777 """ 778 Parses things like '1n+2', or 'an+b' generally, returning (a, b) 779 """ 780 if isinstance(s, Element): 781 s = s._format_element() 782 if not s or s == '*': 783 # Happens when there's nothing, which the CSS parser thinks of as * 784 return (0, 0) 785 if isinstance(s, int): 786 # Happens when you just get a number 787 return (0, s) 788 if s == 'odd': 789 return (2, 1) 790 elif s == 'even': 791 return (2, 0) 792 elif s == 'n': 793 return (1, 0) 794 if 'n' not in s: 795 # Just a b 796 return (0, int(s)) 797 a, b = s.split('n', 1) 798 if not a: 799 a = 1 800 elif a == '-' or a == '+': 801 a = int(a+'1') 802 else: 803 a = int(a) 804 if not b: 805 b = 0 806 elif b == '-' or b == '+': 807 b = int(b+'1') 808 else: 809 b = int(b) 810 return (a, b)

811 812 813 ############################################################ 814 ## Tokenizing 815 ############################################################ 816 817 _whitespace_re = re.compile(r'\s+') 818 819 _comment_re = re.compile(r'/\*.*?\*/', re.S) 820 821 _count_re = re.compile(r'[+-]?\d*n(?:[+-]\d+)?') 822

823 -def tokenize(s):

824 pos = 0 825 s = _comment_re.sub('', s) 826 while 1: 827 match = _whitespace_re.match(s, pos=pos) 828 if match: 829 preceding_whitespace_pos = pos 830 pos = match.end() 831 else: 832 preceding_whitespace_pos = 0 833 if pos >= len(s): 834 return 835 match = _count_re.match(s, pos=pos) 836 if match and match.group() != 'n': 837 sym = s[pos:match.end()] 838 yield Symbol(sym, pos) 839 pos = match.end() 840 continue 841 c = s[pos] 842 c2 = s[pos:pos+2] 843 if c2 in ('~=', '|=', '^=', '$=', '*=', '::', '!='): 844 yield Token(c2, pos) 845 pos += 2 846 continue 847 if c in '>+~,.*=[]()|:#': 848 if c in '.#' and preceding_whitespace_pos > 0: 849 yield Token(' ', preceding_whitespace_pos) 850 yield Token(c, pos) 851 pos += 1 852 continue 853 if c == '"' or c == "'": 854 # Quoted string 855 old_pos = pos 856 sym, pos = tokenize_escaped_string(s, pos) 857 yield String(sym, old_pos) 858 continue 859 old_pos = pos 860 sym, pos = tokenize_symbol(s, pos) 861 yield Symbol(sym, old_pos) 862 continue

863

864 -def tokenize_escaped_string(s, pos):

865 quote = s[pos] 866 assert quote in ('"', "'") 867 pos = pos+1 868 start = pos 869 while 1: 870 next = s.find(quote, pos) 871 if next == -1: 872 raise SelectorSyntaxError( 873 "Expected closing %s for string in: %r" 874 % (quote, s[start:])) 875 result = s[start:next] 876 try: 877 result = result.encode('ASCII', 'backslashreplace').decode('unicode_escape') 878 except UnicodeDecodeError: 879 # Probably a hanging \ 880 pos = next+1 881 else: 882 return result, next+1

883 884 _illegal_symbol = re.compile(r'[^\w\\-]', re.UNICODE) 885

886 -def tokenize_symbol(s, pos):

887 start = pos 888 match = _illegal_symbol.search(s, pos=pos) 889 if not match: 890 # Goes to end of s 891 return s[start:], len(s) 892 if match.start() == pos: 893 assert 0, ( 894 "Unexpected symbol: %r at %s" % (s[pos], pos)) 895 if not match: 896 result = s[start:] 897 pos = len(s) 898 else: 899 result = s[start:match.start()] 900 pos = match.start() 901 try: 902 result = result.encode('ASCII', 'backslashreplace').decode('unicode_escape') 903 except UnicodeDecodeError: 904 import sys 905 e = sys.exc_info()[1] 906 raise SelectorSyntaxError( 907 "Bad symbol %r: %s" % (result, e)) 908 return result, pos

909

910 -class TokenStream(object):

911

912 - def __init__(self, tokens, source=None):

913 self.used = [] 914 self.tokens = iter(tokens) 915 self.source = source 916 self.peeked = None 917 self._peeking = False 918 try: 919 self.next_token = self.tokens.next 920 except AttributeError: 921 # Python 3 922 self.next_token = self.tokens.__next__

923

924 - def next(self):

925 if self._peeking: 926 self._peeking = False 927 self.used.append(self.peeked) 928 return self.peeked 929 else: 930 try: 931 next = self.next_token() 932 self.used.append(next) 933 return next 934 except StopIteration: 935 return None

936

937 - def __iter__(self):

938 return iter(self.next, None)

939

940 - def peek(self):

941 if not self._peeking: 942 try: 943 self.peeked = self.next_token() 944 except StopIteration: 945 return None 946 self._peeking = True 947 return self.peeked

948

Source Code for Module lxml.cssselect