html.py 8.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272
  1. """Compare two HTML documents."""
  2. import html
  3. from html.parser import HTMLParser
  4. from django.utils.html import VOID_ELEMENTS
  5. from django.utils.regex_helper import _lazy_re_compile
  6. # ASCII whitespace is U+0009 TAB, U+000A LF, U+000C FF, U+000D CR, or U+0020
  7. # SPACE.
  8. # https://infra.spec.whatwg.org/#ascii-whitespace
  9. ASCII_WHITESPACE = _lazy_re_compile(r"[\t\n\f\r ]+")
  10. # https://html.spec.whatwg.org/#attributes-3
  11. BOOLEAN_ATTRIBUTES = {
  12. "allowfullscreen",
  13. "async",
  14. "autofocus",
  15. "autoplay",
  16. "checked",
  17. "controls",
  18. "default",
  19. "defer ",
  20. "disabled",
  21. "formnovalidate",
  22. "hidden",
  23. "ismap",
  24. "itemscope",
  25. "loop",
  26. "multiple",
  27. "muted",
  28. "nomodule",
  29. "novalidate",
  30. "open",
  31. "playsinline",
  32. "readonly",
  33. "required",
  34. "reversed",
  35. "selected",
  36. # Attributes for deprecated tags.
  37. "truespeed",
  38. }
  39. def normalize_whitespace(string):
  40. return ASCII_WHITESPACE.sub(" ", string)
  41. def normalize_attributes(attributes):
  42. normalized = []
  43. for name, value in attributes:
  44. if name == "class" and value:
  45. # Special case handling of 'class' attribute, so that comparisons
  46. # of DOM instances are not sensitive to ordering of classes.
  47. value = " ".join(
  48. sorted(value for value in ASCII_WHITESPACE.split(value) if value)
  49. )
  50. # Boolean attributes without a value is same as attribute with value
  51. # that equals the attributes name. For example:
  52. # <input checked> == <input checked="checked">
  53. if name in BOOLEAN_ATTRIBUTES:
  54. if not value or value == name:
  55. value = None
  56. elif value is None:
  57. value = ""
  58. normalized.append((name, value))
  59. return normalized
  60. class Element:
  61. def __init__(self, name, attributes):
  62. self.name = name
  63. self.attributes = sorted(attributes)
  64. self.children = []
  65. def append(self, element):
  66. if isinstance(element, str):
  67. element = normalize_whitespace(element)
  68. if self.children and isinstance(self.children[-1], str):
  69. self.children[-1] += element
  70. self.children[-1] = normalize_whitespace(self.children[-1])
  71. return
  72. elif self.children:
  73. # removing last children if it is only whitespace
  74. # this can result in incorrect dom representations since
  75. # whitespace between inline tags like <span> is significant
  76. if isinstance(self.children[-1], str) and self.children[-1].isspace():
  77. self.children.pop()
  78. if element:
  79. self.children.append(element)
  80. def finalize(self):
  81. def rstrip_last_element(children):
  82. if children and isinstance(children[-1], str):
  83. children[-1] = children[-1].rstrip()
  84. if not children[-1]:
  85. children.pop()
  86. children = rstrip_last_element(children)
  87. return children
  88. rstrip_last_element(self.children)
  89. for i, child in enumerate(self.children):
  90. if isinstance(child, str):
  91. self.children[i] = child.strip()
  92. elif hasattr(child, "finalize"):
  93. child.finalize()
  94. def __eq__(self, element):
  95. if not hasattr(element, "name") or self.name != element.name:
  96. return False
  97. if self.attributes != element.attributes:
  98. return False
  99. return self.children == element.children
  100. def __hash__(self):
  101. return hash((self.name, *self.attributes))
  102. def _count(self, element, count=True):
  103. if not isinstance(element, str) and self == element:
  104. return 1
  105. if isinstance(element, RootElement) and self.children == element.children:
  106. return 1
  107. i = 0
  108. elem_child_idx = 0
  109. for child in self.children:
  110. # child is text content and element is also text content, then
  111. # make a simple "text" in "text"
  112. if isinstance(child, str):
  113. if isinstance(element, str):
  114. if count:
  115. i += child.count(element)
  116. elif element in child:
  117. return 1
  118. else:
  119. # Look for element wholly within this child.
  120. i += child._count(element, count=count)
  121. if not count and i:
  122. return i
  123. # Also look for a sequence of element's children among self's
  124. # children. self.children == element.children is tested above,
  125. # but will fail if self has additional children. Ex: '<a/><b/>'
  126. # is contained in '<a/><b/><c/>'.
  127. if isinstance(element, RootElement) and element.children:
  128. elem_child = element.children[elem_child_idx]
  129. # Start or continue match, advance index.
  130. if elem_child == child:
  131. elem_child_idx += 1
  132. # Match found, reset index.
  133. if elem_child_idx == len(element.children):
  134. i += 1
  135. elem_child_idx = 0
  136. # No match, reset index.
  137. else:
  138. elem_child_idx = 0
  139. return i
  140. def __contains__(self, element):
  141. return self._count(element, count=False) > 0
  142. def count(self, element):
  143. return self._count(element, count=True)
  144. def __getitem__(self, key):
  145. return self.children[key]
  146. def __str__(self):
  147. output = "<%s" % self.name
  148. for key, value in self.attributes:
  149. if value is not None:
  150. output += ' %s="%s"' % (key, value)
  151. else:
  152. output += " %s" % key
  153. if self.children:
  154. output += ">\n"
  155. output += "".join(
  156. [
  157. html.escape(c) if isinstance(c, str) else str(c)
  158. for c in self.children
  159. ]
  160. )
  161. output += "\n</%s>" % self.name
  162. else:
  163. output += ">"
  164. return output
  165. def __repr__(self):
  166. return str(self)
  167. class RootElement(Element):
  168. def __init__(self):
  169. super().__init__(None, ())
  170. def __str__(self):
  171. return "".join(
  172. [html.escape(c) if isinstance(c, str) else str(c) for c in self.children]
  173. )
  174. class HTMLParseError(Exception):
  175. pass
  176. class Parser(HTMLParser):
  177. def __init__(self):
  178. super().__init__()
  179. self.root = RootElement()
  180. self.open_tags = []
  181. self.element_positions = {}
  182. def error(self, msg):
  183. raise HTMLParseError(msg, self.getpos())
  184. def format_position(self, position=None, element=None):
  185. if not position and element:
  186. position = self.element_positions[element]
  187. if position is None:
  188. position = self.getpos()
  189. if hasattr(position, "lineno"):
  190. position = position.lineno, position.offset
  191. return "Line %d, Column %d" % position
  192. @property
  193. def current(self):
  194. if self.open_tags:
  195. return self.open_tags[-1]
  196. else:
  197. return self.root
  198. def handle_startendtag(self, tag, attrs):
  199. self.handle_starttag(tag, attrs)
  200. if tag not in VOID_ELEMENTS:
  201. self.handle_endtag(tag)
  202. def handle_starttag(self, tag, attrs):
  203. attrs = normalize_attributes(attrs)
  204. element = Element(tag, attrs)
  205. self.current.append(element)
  206. if tag not in VOID_ELEMENTS:
  207. self.open_tags.append(element)
  208. self.element_positions[element] = self.getpos()
  209. def handle_endtag(self, tag):
  210. if not self.open_tags:
  211. self.error("Unexpected end tag `%s` (%s)" % (tag, self.format_position()))
  212. element = self.open_tags.pop()
  213. while element.name != tag:
  214. if not self.open_tags:
  215. self.error(
  216. "Unexpected end tag `%s` (%s)" % (tag, self.format_position())
  217. )
  218. element = self.open_tags.pop()
  219. def handle_data(self, data):
  220. self.current.append(data)
  221. def parse_html(html):
  222. """
  223. Take a string that contains HTML and turn it into a Python object structure
  224. that can be easily compared against other HTML on semantic equivalence.
  225. Syntactical differences like which quotation is used on arguments will be
  226. ignored.
  227. """
  228. parser = Parser()
  229. parser.feed(html)
  230. parser.close()
  231. document = parser.root
  232. document.finalize()
  233. # Removing ROOT element if it's not necessary
  234. if len(document.children) == 1 and not isinstance(document.children[0], str):
  235. document = document.children[0]
  236. return document