You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

html.py 7.3KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228
  1. """Compare two HTML documents."""
  2. import re
  3. from html.parser import HTMLParser
  4. WHITESPACE = re.compile(r'\s+')
  5. def normalize_whitespace(string):
  6. return WHITESPACE.sub(' ', string)
  7. class Element:
  8. def __init__(self, name, attributes):
  9. self.name = name
  10. self.attributes = sorted(attributes)
  11. self.children = []
  12. def append(self, element):
  13. if isinstance(element, str):
  14. element = normalize_whitespace(element)
  15. if self.children:
  16. if isinstance(self.children[-1], str):
  17. self.children[-1] += element
  18. self.children[-1] = normalize_whitespace(self.children[-1])
  19. return
  20. elif self.children:
  21. # removing last children if it is only whitespace
  22. # this can result in incorrect dom representations since
  23. # whitespace between inline tags like <span> is significant
  24. if isinstance(self.children[-1], str):
  25. if self.children[-1].isspace():
  26. self.children.pop()
  27. if element:
  28. self.children.append(element)
  29. def finalize(self):
  30. def rstrip_last_element(children):
  31. if children:
  32. if isinstance(children[-1], str):
  33. children[-1] = children[-1].rstrip()
  34. if not children[-1]:
  35. children.pop()
  36. children = rstrip_last_element(children)
  37. return children
  38. rstrip_last_element(self.children)
  39. for i, child in enumerate(self.children):
  40. if isinstance(child, str):
  41. self.children[i] = child.strip()
  42. elif hasattr(child, 'finalize'):
  43. child.finalize()
  44. def __eq__(self, element):
  45. if not hasattr(element, 'name') or self.name != element.name:
  46. return False
  47. if len(self.attributes) != len(element.attributes):
  48. return False
  49. if self.attributes != element.attributes:
  50. # attributes without a value is same as attribute with value that
  51. # equals the attributes name:
  52. # <input checked> == <input checked="checked">
  53. for i in range(len(self.attributes)):
  54. attr, value = self.attributes[i]
  55. other_attr, other_value = element.attributes[i]
  56. if value is None:
  57. value = attr
  58. if other_value is None:
  59. other_value = other_attr
  60. if attr != other_attr or value != other_value:
  61. return False
  62. return self.children == element.children
  63. def __hash__(self):
  64. return hash((self.name, *(a for a in self.attributes)))
  65. def _count(self, element, count=True):
  66. if not isinstance(element, str):
  67. if self == element:
  68. return 1
  69. if isinstance(element, RootElement):
  70. if self.children == element.children:
  71. return 1
  72. i = 0
  73. for child in self.children:
  74. # child is text content and element is also text content, then
  75. # make a simple "text" in "text"
  76. if isinstance(child, str):
  77. if isinstance(element, str):
  78. if count:
  79. i += child.count(element)
  80. elif element in child:
  81. return 1
  82. else:
  83. i += child._count(element, count=count)
  84. if not count and i:
  85. return i
  86. return i
  87. def __contains__(self, element):
  88. return self._count(element, count=False) > 0
  89. def count(self, element):
  90. return self._count(element, count=True)
  91. def __getitem__(self, key):
  92. return self.children[key]
  93. def __str__(self):
  94. output = '<%s' % self.name
  95. for key, value in self.attributes:
  96. if value:
  97. output += ' %s="%s"' % (key, value)
  98. else:
  99. output += ' %s' % key
  100. if self.children:
  101. output += '>\n'
  102. output += ''.join(str(c) for c in self.children)
  103. output += '\n</%s>' % self.name
  104. else:
  105. output += '>'
  106. return output
  107. def __repr__(self):
  108. return str(self)
  109. class RootElement(Element):
  110. def __init__(self):
  111. super().__init__(None, ())
  112. def __str__(self):
  113. return ''.join(str(c) for c in self.children)
  114. class HTMLParseError(Exception):
  115. pass
  116. class Parser(HTMLParser):
  117. SELF_CLOSING_TAGS = (
  118. 'br', 'hr', 'input', 'img', 'meta', 'spacer', 'link', 'frame', 'base',
  119. 'col',
  120. )
  121. def __init__(self):
  122. super().__init__(convert_charrefs=False)
  123. self.root = RootElement()
  124. self.open_tags = []
  125. self.element_positions = {}
  126. def error(self, msg):
  127. raise HTMLParseError(msg, self.getpos())
  128. def format_position(self, position=None, element=None):
  129. if not position and element:
  130. position = self.element_positions[element]
  131. if position is None:
  132. position = self.getpos()
  133. if hasattr(position, 'lineno'):
  134. position = position.lineno, position.offset
  135. return 'Line %d, Column %d' % position
  136. @property
  137. def current(self):
  138. if self.open_tags:
  139. return self.open_tags[-1]
  140. else:
  141. return self.root
  142. def handle_startendtag(self, tag, attrs):
  143. self.handle_starttag(tag, attrs)
  144. if tag not in self.SELF_CLOSING_TAGS:
  145. self.handle_endtag(tag)
  146. def handle_starttag(self, tag, attrs):
  147. # Special case handling of 'class' attribute, so that comparisons of DOM
  148. # instances are not sensitive to ordering of classes.
  149. attrs = [
  150. (name, " ".join(sorted(value.split(" "))))
  151. if name == "class"
  152. else (name, value)
  153. for name, value in attrs
  154. ]
  155. element = Element(tag, attrs)
  156. self.current.append(element)
  157. if tag not in self.SELF_CLOSING_TAGS:
  158. self.open_tags.append(element)
  159. self.element_positions[element] = self.getpos()
  160. def handle_endtag(self, tag):
  161. if not self.open_tags:
  162. self.error("Unexpected end tag `%s` (%s)" % (
  163. tag, self.format_position()))
  164. element = self.open_tags.pop()
  165. while element.name != tag:
  166. if not self.open_tags:
  167. self.error("Unexpected end tag `%s` (%s)" % (
  168. tag, self.format_position()))
  169. element = self.open_tags.pop()
  170. def handle_data(self, data):
  171. self.current.append(data)
  172. def handle_charref(self, name):
  173. self.current.append('&%s;' % name)
  174. def handle_entityref(self, name):
  175. self.current.append('&%s;' % name)
  176. def parse_html(html):
  177. """
  178. Take a string that contains *valid* HTML and turn it into a Python object
  179. structure that can be easily compared against other HTML on semantic
  180. equivalence. Syntactical differences like which quotation is used on
  181. arguments will be ignored.
  182. """
  183. parser = Parser()
  184. parser.feed(html)
  185. parser.close()
  186. document = parser.root
  187. document.finalize()
  188. # Removing ROOT element if it's not necessary
  189. if len(document.children) == 1:
  190. if not isinstance(document.children[0], str):
  191. document = document.children[0]
  192. return document