htmlement.py
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# The MIT License (MIT)
#
# Copyright (c) 2016 William Forde
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
# the Software, and to permit persons to whom the Software is furnished to do so,
# subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
"""
HTMLement
---------
Simple lightweight HTML parser with XPath support.
Github: https://github.com/willforde/python-htmlement
Documentation: https://python-htmlement.readthedocs.io/en/stable/?badge=stable
Testing: https://github.com/willforde/python-htmlement/actions
Coverage: https://codecov.io/gh/willforde/python-htmlement
Maintainability: https://codeclimate.com/github/willforde/python-htmlement/maintainability
"""
# Standard Lib
import xml.etree.ElementTree as Etree
import warnings
import re
# HTML Parser
from html.entities import name2codepoint
from html.parser import HTMLParser
__all__ = ["HTMLement", "fromstring", "fromstringlist", "parse"]
__version__ = "2.0.0"
# Add missing codepoints
# TODO: This may no longer be required
name2codepoint["apos"] = 0x0027
def fromstring(text, tag="", attrs=None, encoding=None):
"""
Parse's "HTML" document from a string into an element tree.
:param text: The "HTML" document to parse.
:type text: str or bytes
:param str tag: (optional) Name of "tag / element" which is used to filter down "the tree" to a required section.
:type tag: str
:param attrs: (optional) The attributes of the element, that will be used, when searchingfor the required section.
:type attrs: dict(str, str)
:param encoding: (optional) Encoding used, when decoding the source data before feeding it to the parser.
:type encoding: str
:return: The root element of the element tree.
:rtype: xml.etree.ElementTree.Element
:raises UnicodeDecodeError: If decoding of *text* fails.
"""
parser = HTMLement(tag, attrs, encoding)
parser.feed(text)
return parser.close()
def fromstringlist(sequence, tag="", attrs=None, encoding=None):
"""
Parses an "HTML document" from a sequence of "HTML sections" into an element tree.
:param sequence: A sequence of "HTML sections" to parse.
:type sequence: list(str or bytes)
:param str tag: (optional) Name of "tag / element" which is used to filter down "the tree" to a required section.
:type tag: str
:param attrs: (optional) The attributes of the element, that will be used, when searchingfor the required section.
:type attrs: dict(str, str)
:param encoding: (optional) Encoding used, when decoding the source data before feeding it to the parser.
:type encoding: str
:return: The root element of the element tree.
:rtype: xml.etree.ElementTree.Element
:raises UnicodeDecodeError: If decoding of a section within *sequence* fails.
"""
parser = HTMLement(tag, attrs, encoding)
for text in sequence:
parser.feed(text)
return parser.close()
def parse(source, tag="", attrs=None, encoding=None):
"""
Load an external "HTML document" into an element tree.
:param source: A filename or file like object containing HTML data.
:type source: str or io.TextIOBase
:param str tag: (optional) Name of "tag / element" which is used to filter down "the tree" to a required section.
:type tag: str
:param attrs: (optional) The attributes of the element, that will be used, when searchingfor the required section.
:type attrs: dict(str, str)
:param encoding: (optional) Encoding used, when decoding the source data before feeding it to the parser.
:type encoding: str
:return: The root element of the element tree.
:rtype: xml.etree.ElementTree.Element
:raises UnicodeDecodeError: If decoding of *source* fails.
"""
# Assume that source is a file pointer if no read methods is found
if not hasattr(source, "read"):
source = open(source, "r", encoding=encoding)
close_source = True
else:
close_source = False
try:
parser = HTMLement(tag, attrs, encoding)
while True:
# Read in 64k at a time
data = source.read(65536)
if not data:
break
# Feed the parser
parser.feed(data)
# Return the root element
return parser.close()
finally:
if close_source:
source.close()
class HTMLement(object):
"""
Python HTMLParser extension with ElementTree Parser support.
This HTML Parser extends :class:`html.parser.HTMLParser`, returning an :class:`xml.etree.ElementTree.Element`
instance. The returned root element natively supports the ElementTree API.
(e.g. you may use its limited support for `XPath expressions`__)
When a "tag" and "tag attributes" are given the parser will search for a required section. Only when the required
section is found, does the parser start parsing the "HTML document". The element that matches the search criteria
will then become the new "root element".
Attributes are given as a dict of {'name': 'value'}. Value can be the string to match, `True` or `False.`
`True` will match any attribute with given name and any value.
`False` will only give a match if given attribute does not exist in the element.
:param str tag: (optional) Name of "tag / element" which is used to filter down "the tree" to a required section.
:type tag: str
:param attrs: (optional) The attributes of the element, that will be used, when searchingfor the required section.
:type attrs: dict(str, str)
:param encoding: (optional) Encoding used, when decoding the source data before feeding it to the parser.
:type encoding: str
.. _Xpath: https://docs.python.org/3.6/library/xml.etree.elementtree.html#xpath-support
__ XPath_
"""
def __init__(self, tag="", attrs=None, encoding=None):
self._parser = ParseHTML(tag, attrs)
self.encoding = encoding
self._finished = False
def feed(self, data):
"""
Feeds data to the parser.
If *data*, is of type :class:`bytes` and where no encoding was specified, then the encoding
will be extracted from *data* using "meta tags", if available.
Otherwise encoding will default to "ISO-8859-1"
:param data: HTML data
:type data: str or bytes
:raises UnicodeDecodeError: If decoding of *data* fails.
"""
# Skip feeding data into parser if we already have what we want
if self._finished == 1:
return None
# Make sure that we have unicode before continuing
if isinstance(data, bytes):
if self.encoding:
data = data.decode(self.encoding)
else:
data = self._make_unicode(data)
# Parse the html document
try:
self._parser.feed(data)
except EOFError:
self._finished = True
self._parser.reset()
def close(self):
"""
Close the "tree builder" and return the "root element" of the "element tree".
:return: The "root element" of the "element tree".
:rtype: xml.etree.ElementTree.Element
:raises RuntimeError: If no element matching search criteria was found.
"""
return self._parser.close()
def _make_unicode(self, data):
"""
Convert *data* from type :class:`bytes` to type :class:`str`.
:param data: The html document.
:type data: bytes
:return: HTML data decoded.
:rtype: str
"""
# Atemp to find the encoding from the html source
end_head_tag = data.find(b"</head>")
if end_head_tag:
# Search for the charset attribute within the meta tags
charset_refind = b'<meta.+?charset=[\'"]*(.+?)["\'].*?>'
charset = re.search(charset_refind, data[:end_head_tag], re.IGNORECASE)
if charset:
self.encoding = encoding = charset.group(1).decode()
return data.decode(encoding)
# Decode the string into unicode using default encoding
warn_msg = "Unable to determine encoding, defaulting to iso-8859-1"
warnings.warn(warn_msg, UnicodeWarning, stacklevel=2)
self.encoding = "iso-8859-1"
return data.decode("iso-8859-1")
# noinspection PyAbstractClass
class ParseHTML(HTMLParser):
def __init__(self, tag="", attrs=None):
# Initiate HTMLParser
HTMLParser.__init__(self)
self.convert_charrefs = True
self._root = None # root element
self._data = [] # data collector
self._factory = Etree.Element
self.enabled = not tag
self._unw_attrs = []
self.tag = tag
# Split attributes into wanted and unwanted attributes
if attrs:
self.attrs = attrs
for key, value in attrs.copy().items():
if value == 0:
self._unw_attrs.append(key)
del attrs[key]
else:
self.attrs = {}
# Some tags in html do not require closing tags so thoes tags will need to be auto closed (Void elements)
# Refer to: https://www.w3.org/TR/html/syntax.html#void-elements
self._voids = frozenset(("area", "base", "br", "col", "hr", "img", "input", "link", "meta", "param",
# Only in HTML5
"embed", "keygen", "source", "track",
# Not supported in HTML5
"basefont", "frame", "isindex",
# SVG self closing tags
"rect", "circle", "ellipse", "line", "polyline", "polygon",
"path", "stop", "use", "image", "animatetransform"))
# Create temporary root element to protect from badly written sites that either
# have no html starting tag or multiple top level elements
elem = self._factory("html")
self._elem = [elem]
self._last = elem
self._tail = 0
def handle_starttag(self, tag, attrs):
self._handle_starttag(tag, attrs, self_closing=tag in self._voids)
def handle_startendtag(self, tag, attrs):
self._handle_starttag(tag, attrs, self_closing=True)
def _handle_starttag(self, tag, attrs, self_closing=False):
enabled = self.enabled
# Add tag element to tree if we have no filter or that the filter matches
if enabled or self._search(tag, attrs):
# Convert attrs to dictionary
attrs = {k: v or "" for k, v in attrs}
self._flush()
# Create the new element
elem = self._factory(tag, attrs)
self._elem[-1].append(elem)
self._last = elem
# Only append the element to the list of elements if it's not a self closing element
if self_closing:
self._tail = 1
else:
self._elem.append(elem)
self._tail = 0
# Set this element as the root element when the filter search matches
if not enabled:
self._root = elem
self.enabled = True
def handle_endtag(self, tag):
# Only process end tags when we have no filter or that the filter has been matched
if self.enabled and tag not in self._voids:
_elem = self._elem
_root = self._root
# Check that the closing tag is what's actualy expected
if _elem[-1].tag == tag:
self._flush()
self._tail = 1
self._last = elem = _elem.pop()
if elem is _root:
raise EOFError
# If a previous element is what we actually have then the expected element was not
# properly closed so we must close that before closing what we have now
elif len(_elem) >= 2 and any(_item.tag == tag for _item in _elem):
self._flush()
self._tail = 1
while True:
self._last = elem = _elem.pop()
if elem.tag == tag:
break
if elem is _root:
raise EOFError
else:
# Unable to match the tag to an element, ignoring it
return None
def handle_data(self, data):
if data.strip() and self.enabled:
self._data.append(data)
def handle_entityref(self, name):
if self.enabled:
try:
name = chr(name2codepoint[name])
except KeyError:
pass
self._data.append(name)
def handle_charref(self, name):
if self.enabled:
try:
if name[0].lower() == "x":
name = chr(int(name[1:], 16))
else:
name = chr(int(name))
except ValueError:
pass
self._data.append(name)
def handle_comment(self, data):
data = data.strip()
if data and self.enabled:
elem = Etree.Comment(data)
self._elem[-1].append(elem)
def close(self):
self._flush()
if self.enabled == 0:
msg = "Unable to find requested section with tag of '{}' and attributes of {}"
raise RuntimeError(msg.format(self.tag, self.attrs))
elif self._root is not None:
return self._root
else:
# Search the root element to find a proper html root element if one exists
tmp_root = self._elem[0]
proper_root = tmp_root.find("html")
if proper_root is None:
# Not proper root was found
return tmp_root
else:
# Proper root found
return proper_root
def _flush(self):
if self._data:
if self._last is not None:
text = "".join(self._data)
if self._tail:
self._last.tail = text
else:
self._last.text = text
self._data = []
def _search(self, tag, attrs):
# Only search when the tag matches
if tag == self.tag:
# If we have required attrs to match then search all attrs for wanted attrs
# And also check that we do not have any attrs that are unwanted
if self.attrs or self._unw_attrs:
if attrs:
wanted_attrs = self.attrs.copy()
unwanted_attrs = self._unw_attrs
for key, value in attrs:
# Check for unwanted attrs
if key in unwanted_attrs:
return False
# Check for wanted attrs
elif key in wanted_attrs:
c_value = wanted_attrs[key]
if c_value == value or c_value == 1:
# Remove this attribute from the wanted dict of attributes
# to indicate that this attribute has been found
del wanted_attrs[key]
# If wanted_attrs is now empty then all attributes must have been found
if not wanted_attrs:
return True
else:
# We only need to match tag
return True
# Unable to find required section
return False