# License: GPLv3 Copyright: 2008, Kovid Goyal <kovid at kovidgoyal.net>


import copy
import glob
import os
import re
import sys
import tempfile
from collections import deque
from itertools import chain
from math import ceil, floor

from calibre import __appname__, entity_regex, entity_to_unicode, fit_image, force_unicode, preferred_encoding
from calibre.constants import filesystem_encoding
from calibre.devices.interface import DevicePlugin as Device
from calibre.ebooks import ConversionError
from calibre.ebooks.BeautifulSoup import BeautifulSoup, Comment, Declaration, NavigableString, ProcessingInstruction, Tag
from calibre.ebooks.chardet import xml_to_unicode
from calibre.ebooks.lrf import Book
from calibre.ebooks.lrf.html.color_map import lrs_color
from calibre.ebooks.lrf.html.table import Table
from calibre.ebooks.lrf.pylrs.pylrs import (
    CR,
    BlockSpace,
    BookSetting,
    Canvas,
    CharButton,
    DropCaps,
    EmpLine,
    Image,
    ImageBlock,
    ImageStream,
    Italic,
    JumpButton,
    LrsError,
    Paragraph,
    Plot,
    RuledLine,
    Span,
    Sub,
    Sup,
    TextBlock,
)
from calibre.ptempfile import PersistentTemporaryFile
from polyglot.builtins import itervalues, string_or_bytes
from polyglot.urllib import unquote, urlparse

'''
Code to convert HTML ebooks into LRF ebooks.

I am indebted to esperanc for the initial CSS->Xylog Style conversion code
and to Falstaff for pylrs.
'''

from PIL import Image as PILImage


def update_css(ncss, ocss):
    for key in ncss.keys():
        if key in ocss:
            ocss[key].update(ncss[key])
        else:
            ocss[key] = ncss[key]


def munge_paths(basepath, url):
    purl = urlparse(unquote(url),)
    path, fragment = purl[2], purl[5]
    if path:
        path = path.replace('/', os.sep)
    if not path:
        path = basepath
    elif not os.path.isabs(path):
        dn = os.path.dirname(basepath)
        path = os.path.join(dn, path)
    return os.path.normpath(path), fragment


def strip_style_comments(match):
    src = match.group()
    while True:
        lindex = src.find('/*')
        if lindex < 0:
            break
        rindex = src.find('*/', lindex)
        if rindex < 0:
            src = src[:lindex]
            break
        src = src[:lindex] + src[rindex+2:]
    return src


def tag_regex(tagname):
    '''Return non-grouping regular expressions that match the opening and closing tags for tagname'''
    return dict(open=r'(?:<\s*{t}\s+[^<>]*?>|<\s*{t}\s*>)'.format(**dict(t=tagname)),
                close=r'</\s*{t}\s*>'.format(**dict(t=tagname)))


class HTMLConverter:
    SELECTOR_PAT   = re.compile(r'([A-Za-z0-9\-\_\:\.]+[A-Za-z0-9\-\_\:\.\s\,]*)\s*\{([^\}]*)\}')
    PAGE_BREAK_PAT = re.compile(r'page-break-(?:after|before)\s*:\s*(\w+)', re.IGNORECASE)
    IGNORED_TAGS   = (Comment, Declaration, ProcessingInstruction)

    MARKUP_MASSAGE   = [
                        # Close <a /> tags
                        (re.compile(r'<a(\s[^>]*)?/>', re.IGNORECASE),
                         lambda match: '<a'+match.group(1)+'></a>'),
                        # Strip comments from <style> tags. This is needed as
                        # sometimes there are unterminated comments
                        (re.compile(r'<\s*style.*?>(.*?)</\s*style\s*>', re.DOTALL|re.IGNORECASE),
                         lambda match: match.group().replace('<!--', '').replace('-->', '')),
                        # remove <p> tags from within <a href> tags
                        (re.compile(r'<\s*a\s+[^<>]*href\s*=[^<>]*>(.*?)<\s*/\s*a\s*>', re.DOTALL|re.IGNORECASE),
                         lambda match: re.compile(r'{open}|{close}'.format(**tag_regex('p')), re.IGNORECASE).sub('', match.group())),

                        # Replace common line break patterns with line breaks
                        (re.compile(r'<p>(&nbsp;|\s)*</p>', re.IGNORECASE), lambda m: '<br />'),

                        # Replace empty headers with line breaks
                        (re.compile(r'<h[0-5]?>(&nbsp;|\s)*</h[0-5]?>',
                                    re.IGNORECASE), lambda m: '<br />'),

                        # Replace entities
                        (entity_regex(), entity_to_unicode),
                        # Remove comments from within style tags as they can mess up BeatifulSoup
                        (re.compile(r'(<style.*?</style>)', re.IGNORECASE|re.DOTALL),
                         strip_style_comments),

                        # Remove self closing script tags as they also mess up BeautifulSoup
                        (re.compile(r'(?i)<script[^<>]+?/>'), lambda match: ''),

                        # BeautifulSoup treats self closing <div> tags as open <div> tags
                        (re.compile(r'(?i)<\s*div([^>]*)/\s*>'),
                         lambda match: f'<div{match.group(1)}></div>')

                        ]
    # Fix Baen markup
    BAEN = [
                     (re.compile(r'page-break-before:\s*\w+([\s;\}])', re.IGNORECASE),
                      lambda match: match.group(1)),
                     (re.compile(r'<p>\s*(<a id.*?>\s*</a>)\s*</p>', re.IGNORECASE),
                      lambda match: match.group(1)),
                     (re.compile(r'<\s*a\s+id="p[0-9]+"\s+name="p[0-9]+"\s*>\s*</a>', re.IGNORECASE),
                      lambda match: ''),
                     ]
    # Fix pdftohtml markup
    PDFTOHTML  = [
                  # Remove <hr> tags
                  (re.compile(r'<hr.*?>', re.IGNORECASE), lambda match: '<br />'),
                  # Remove page numbers
                  (re.compile(r'\d+<br>', re.IGNORECASE), lambda match: ''),
                  # Remove <br> and replace <br><br> with <p>
                  (re.compile(r'<br.*?>\s*<br.*?>', re.IGNORECASE), lambda match: '<p>'),
                  (re.compile(r'(.*)<br.*?>', re.IGNORECASE),
                   lambda match: match.group() if match.group(1).lstrip().startswith('<') or len(match.group(1)) < 40
                                else match.group(1)),
                  # Remove hyphenation
                  (re.compile(r'-\n\r?'), lambda match: ''),

                  ]

    # Fix Book Designer markup
    BOOK_DESIGNER = [
                     # HR
                     (re.compile(r'<hr>', re.IGNORECASE),
                      lambda match : '<span style="page-break-after:always"> </span>'),
                     # Create header tags
                     (re.compile(r'<h2[^><]*?id=BookTitle[^><]*?(align=)*(?(1)(\w+))*[^><]*?>[^><]*?</h2>', re.IGNORECASE),
                      lambda match : '<h1 id="BookTitle" align="{}">{}</h1>'.format(match.group(2) if match.group(2) else 'center', match.group(3))),
                     (re.compile(r'<h2[^><]*?id=BookAuthor[^><]*?(align=)*(?(1)(\w+))*[^><]*?>[^><]*?</h2>', re.IGNORECASE),
                      lambda match : '<h2 id="BookAuthor" align="{}">{}</h2>'.format(match.group(2) if match.group(2) else 'center', match.group(3))),
                     (re.compile(r'<span[^><]*?id=title[^><]*?>(.*?)</span>', re.IGNORECASE|re.DOTALL),
                      lambda match : f'<h2 class="title">{match.group(1)}</h2>'),
                     (re.compile(r'<span[^><]*?id=subtitle[^><]*?>(.*?)</span>', re.IGNORECASE|re.DOTALL),
                      lambda match : f'<h3 class="subtitle">{match.group(1)}</h3>'),
                     # Blank lines
                     (re.compile(r'<div[^><]*?>(&nbsp;){4}</div>', re.IGNORECASE),
                      lambda match : '<p></p>'),
                     ]

    def __hasattr__(self, attr):
        if hasattr(self.options, attr):
            return True
        return object.__hasattr__(self, attr)

    def __getattr__(self, attr):
        if hasattr(self.options, attr):
            return getattr(self.options, attr)
        return object.__getattribute__(self, attr)

    def __setattr__(self, attr, val):
        if hasattr(self.options, attr):
            setattr(self.options, attr, val)
        else:
            object.__setattr__(self, attr, val)

    CSS = {
            'h1'     : {'font-size'   : 'xx-large', 'font-weight':'bold', 'text-indent':'0pt'},
            'h2'     : {'font-size'   : 'x-large', 'font-weight':'bold', 'text-indent':'0pt'},
            'h3'     : {'font-size'   : 'large', 'font-weight':'bold', 'text-indent':'0pt'},
            'h4'     : {'font-size'   : 'large', 'text-indent':'0pt'},
            'h5'     : {'font-weight' : 'bold', 'text-indent':'0pt'},
            'b'      : {'font-weight' : 'bold'},
            'strong' : {'font-weight' : 'bold'},
            'i'      : {'font-style'  : 'italic'},
            'cite'   : {'font-style'  : 'italic'},
            'em'     : {'font-style'  : 'italic'},
            'small'  : {'font-size'   : 'small'},
            'pre'    : {'font-family' : 'monospace', 'white-space': 'pre'},
            'code'   : {'font-family' : 'monospace'},
            'tt'     : {'font-family' : 'monospace'},
            'center' : {'text-align'  : 'center'},
            'th'     : {'font-size'   : 'large', 'font-weight':'bold'},
            'big'    : {'font-size'   : 'large', 'font-weight':'bold'},
            '.libprs500_dropcaps' : {'font-size': 'xx-large'},
            'u'      : {'text-decoration': 'underline'},
            'sup'    : {'vertical-align': 'super', 'font-size': '60%'},
            'sub'    : {'vertical-align': 'sub', 'font-size': '60%'},
    }

    def __init__(self, book, fonts, options, logger, paths):
        '''
        Convert HTML files at C{paths} and add to C{book}. After creating
        the object, you must call L{self.writeto} to output the LRF/S file.

        @param book: The LRF book
        @type book:  L{lrf.pylrs.Book}
        @param fonts: dict specifying the font families to use
        '''
        # Defaults for various formatting tags
        object.__setattr__(self, 'options', options)
        self.log = logger
        self.fonts = fonts  # : dict specifying font families to use
        # Memory
        self.scaled_images    = {}    #: Temporary files with scaled version of images
        self.rotated_images   = {}    #: Temporary files with rotated version of images
        self.text_styles      = []    #: Keep track of already used textstyles
        self.block_styles     = []    #: Keep track of already used blockstyles
        self.images  = {}      #: Images referenced in the HTML document
        self.targets = {}      #: <a name=...> and id elements
        self.links   = deque()  # : <a href=...> elements
        self.processed_files = []
        self.extra_toc_entries = []  # : TOC entries gleaned from semantic information
        self.image_memory = []
        self.id_counter = 0
        self.unused_target_blocks = []  # : Used to remove extra TextBlocks
        self.link_level  = 0    #: Current link level
        self.memory = []        #: Used to ensure that duplicate CSS unhandled errors are not reported
        self.tops = {}          #: element representing the top of each HTML file in the LRF file
        self.previous_text = ''  # : Used to figure out when to lstrip
        self.stripped_space = ''
        self.preserve_block_style = False  # : Used so that <p> tags in <blockquote> elements are handled properly
        self.avoid_page_break = False
        self.current_page = book.create_page()

        # Styles
        self.blockquote_style = book.create_block_style(sidemargin=60,
                                                        topskip=20, footskip=20)
        self.unindented_style = book.create_text_style(parindent=0)

        self.in_table = False
        # List processing
        self.list_level = 0
        self.list_indent = 20
        self.list_counter = 1

        self.book = book                #: The Book object representing a BBeB book

        self.override_css = {}
        self.override_pcss = {}

        if self._override_css is not None:
            if os.access(self._override_css, os.R_OK):
                with open(self._override_css, 'rb') as f:
                    src = f.read()
            else:
                src = self._override_css
            if isinstance(src, bytes):
                src = src.decode('utf-8', 'replace')
            match = self.PAGE_BREAK_PAT.search(src)
            if match and not re.match(r'avoid', match.group(1), re.IGNORECASE):
                self.page_break_found = True
            ncss, npcss = self.parse_css(src)
            if ncss:
                update_css(ncss, self.override_css)
            if npcss:
                update_css(npcss, self.override_pcss)

        paths = [os.path.abspath(path) for path in paths]
        paths = [path.decode(sys.getfilesystemencoding()) if not isinstance(path, str) else path for path in paths]

        while len(paths) > 0 and self.link_level <= self.link_levels:
            for path in paths:
                if path in self.processed_files:
                    continue
                try:
                    self.add_file(path)
                except KeyboardInterrupt:
                    raise
                except:
                    if self.link_level == 0:  # Die on errors in the first level
                        raise
                    for link in self.links:
                        if link['path'] == path:
                            self.links.remove(link)
                            break
                    self.log.warn('Could not process '+path)
                    if self.verbose:
                        self.log.exception(' ')
            self.links = self.process_links()
            self.link_level += 1
            paths = [link['path'] for link in self.links]

        if self.current_page is not None and self.current_page.has_text():
            self.book.append(self.current_page)

        for text, tb in self.extra_toc_entries:
            self.book.addTocEntry(text, tb)

        if self.base_font_size > 0:
            self.log.info('\tRationalizing font sizes...')
            self.book.rationalize_font_sizes(self.base_font_size)

    def is_baen(self, soup):
        return bool(soup.find('meta', attrs={'name':'Publisher',
                        'content':re.compile(r'Baen', re.IGNORECASE)}))

    def is_book_designer(self, raw):
        return bool(re.search(r'<H2[^><]*id=BookTitle', raw))

    def preprocess(self, raw):
        nmassage = []
        nmassage.extend(HTMLConverter.MARKUP_MASSAGE)

        if not self.book_designer and self.is_book_designer(raw):
            self.book_designer = True
            self.log.info(_('\tBook Designer file detected.'))

        self.log.info(_('\tParsing HTML...'))

        if self.baen:
            nmassage.extend(HTMLConverter.BAEN)

        if self.pdftohtml:
            nmassage.extend(HTMLConverter.PDFTOHTML)
        if self.book_designer:
            nmassage.extend(HTMLConverter.BOOK_DESIGNER)
        if isinstance(raw, bytes):
            raw = xml_to_unicode(raw, replace_entities=True)[0]
        for pat, repl in nmassage:
            raw = pat.sub(repl, raw)
        soup = BeautifulSoup(raw)
        if not self.baen and self.is_baen(soup):
            self.baen = True
            self.log.info(_('\tBaen file detected. Re-parsing...'))
            return self.preprocess(raw)
        if self.book_designer:
            t = soup.find(id='BookTitle')
            if t:
                self.book.set_title(self.get_text(t))
            a = soup.find(id='BookAuthor')
            if a:
                self.book.set_author(self.get_text(a))
        if self.verbose:
            tdir = tempfile.gettempdir()
            if not os.path.exists(tdir):
                os.makedirs(tdir)
            try:
                with open(os.path.join(tdir, 'html2lrf-verbose.html'), 'wb') as f:
                    f.write(str(soup).encode('utf-8'))
                    self.log.info(_('Written preprocessed HTML to ')+f.name)
            except:
                pass

        return soup

    def add_file(self, path):
        self.css = HTMLConverter.CSS.copy()
        self.pseudo_css = self.override_pcss.copy()
        for selector in self.override_css:
            if selector in self.css:
                self.css[selector].update(self.override_css[selector])
            else:
                self.css[selector] = self.override_css[selector]

        self.file_name = os.path.basename(path)
        self.log.info(_('Processing %s')%(path if self.verbose else self.file_name))

        if not os.path.exists(path):
            path = path.replace('&', '%26')  # convertlit replaces & with %26 in file names
        with open(path, 'rb') as f:
            raw = f.read()
        if self.pdftohtml:  # Bug in pdftohtml that causes it to output invalid UTF-8 files
            raw = raw.decode('utf-8', 'ignore')
        elif self.encoding is not None:
            raw = raw.decode(self.encoding, 'ignore')
        else:
            raw = xml_to_unicode(raw, self.verbose)[0]
        soup = self.preprocess(raw)
        self.log.info(_('\tConverting to BBeB...'))
        self.current_style = {}
        self.page_break_found = False
        if not isinstance(path, str):
            path = path.decode(sys.getfilesystemencoding())
        self.target_prefix = path
        self.previous_text = '\n'
        self.tops[path] = self.parse_file(soup)
        self.processed_files.append(path)

    def parse_css(self, style):
        '''
        Parse the contents of a <style> tag or .css file.
        @param style: C{str(style)} should be the CSS to parse.
        @return: A dictionary with one entry per selector where the key is the
        selector name and the value is a dictionary of properties
        '''
        sdict, pdict = {}, {}
        style = re.sub(r'/\*.*?\*/', '', style)  # Remove /*...*/ comments
        for sel in re.findall(HTMLConverter.SELECTOR_PAT, style):
            for key in sel[0].split(','):
                val = self.parse_style_properties(sel[1])
                key = key.strip().lower()
                if '+' in key:
                    continue
                if ':' in key:
                    key, sep, pseudo = key.partition(':')
                    if key in pdict:
                        if pseudo in pdict[key]:
                            pdict[key][pseudo].update(val)
                        else:
                            pdict[key][pseudo] = val
                    else:
                        pdict[key] = {pseudo:val}
                else:
                    if key in sdict:
                        sdict[key].update(val)
                    else:
                        sdict[key] = val
        return sdict, pdict

    def parse_style_properties(self, props):
        '''
        Parses a style attribute. The code within a CSS selector block or in
        the style attribute of an HTML element.
        @return: A dictionary with one entry for each property where the key
                is the property name and the value is the property value.
        '''
        prop = {}
        for s in props.split(';'):
            l = s.split(':',1)
            if len(l)==2:
                key = l[0].strip().lower()
                val = l[1].strip()
                prop[key] = val
        return prop

    def tag_css(self, tag, parent_css={}):
        '''
        Return a dictionary of style properties applicable to Tag tag.
        '''
        def merge_parent_css(prop, pcss):
            # float should not be inherited according to the CSS spec
            # however we need to as we don't do alignment at a block level.
            # float is removed by the process_alignment function.
            inherited = ['text-align', 'float', 'white-space', 'color',
                         'line-height', 'vertical-align']
            temp = {}
            for key in pcss.keys():
                chk = key.lower()
                # float should not be inherited according to the CSS spec
                # however we need to as we don't do alignment at a block level.
                # float is removed by the process_alignment function.
                if chk.startswith('font') or chk in inherited:
                    temp[key] = pcss[key]
            prop.update(temp)

        prop, pprop = {}, {}
        tagname = tag.name.lower()
        if parent_css:
            merge_parent_css(prop, parent_css)
        if tag.has_attr('align'):
            al = tag['align'].lower()
            if al in ('left', 'right', 'center', 'justify'):
                prop['text-align'] = al
        if tagname in self.css:
            prop.update(self.css[tagname])
        if tagname in self.pseudo_css:
            pprop.update(self.pseudo_css[tagname])
        if tag.has_attr('class'):
            cls = tag['class']
            if isinstance(cls, list):
                cls = ' '.join(cls)
            cls = cls.lower()
            for cls in cls.split():
                for classname in ['.'+cls, tagname+'.'+cls]:
                    if classname in self.css:
                        prop.update(self.css[classname])
                    if classname in self.pseudo_css:
                        pprop.update(self.pseudo_css[classname])
        if tag.has_attr('id') and tag['id'] in self.css:
            prop.update(self.css[tag['id']])
        if tag.has_attr('style'):
            prop.update(self.parse_style_properties(tag['style']))
        return prop, pprop

    def parse_file(self, soup):
        def get_valid_block(page):
            for item in page.contents:
                if isinstance(item, (Canvas, TextBlock, ImageBlock, RuledLine)):
                    if isinstance(item, TextBlock) and not item.contents:
                        continue
                    return item
        if not self.current_page:
            self.current_page = self.book.create_page()
        self.current_block = self.book.create_text_block()
        self.current_para = Paragraph()
        if self.cover:
            self.add_image_page(self.cover)
            self.cover = None
        top = self.current_block
        self.current_block.must_append = True

        self.soup = soup
        self.process_children(soup, {}, {})
        self.soup = None

        if self.current_para and self.current_block:
            self.current_para.append_to(self.current_block)
        if self.current_block and self.current_page:
            self.current_block.append_to(self.current_page)
        if self.avoid_page_break:
            self.avoid_page_break = False
        elif self.current_page and self.current_page.has_text():
            self.book.append(self.current_page)
            self.current_page = None

        if top not in top.parent.contents:  # May have been removed for a cover image
            top = top.parent.contents[0]
        if not top.has_text() and top.parent.contents.index(top) == len(top.parent.contents)-1:
            # Empty block at the bottom of a page
            opage = top.parent
            top.parent.contents.remove(top)
            if self.book.last_page() is opage:
                if self.current_page and self.current_page.has_text():
                    for c in self.current_page.contents:
                        if isinstance(c, (TextBlock, ImageBlock)):
                            return c
                raise ConversionError(_('Could not parse file: %s')%self.file_name)
            else:
                try:
                    index = self.book.pages().index(opage)
                except ValueError:
                    self.log.warning(_('%s is an empty file')%self.file_name)
                    tb = self.book.create_text_block()
                    self.current_page.append(tb)
                    return tb
                for page in list(self.book.pages()[index+1:]):
                    for c in page.contents:
                        if isinstance(c, (TextBlock, ImageBlock, Canvas)):
                            return c
                raise ConversionError(_('Could not parse file: %s')%self.file_name)

        return top

    def create_link(self, children, tag):
        para = None
        for i in range(len(children)-1, -1, -1):
            if isinstance(children[i], (Span, EmpLine)):
                para = children[i]
                break
        if para is None:
            raise ConversionError(
                _('Failed to parse link %(tag)s %(children)s')%dict(
                    tag=tag, children=children))
        text = self.get_text(tag, 1000)
        if not text:
            text = 'Link'
            img = tag.find('img')
            if img:
                try:
                    text = img['alt']
                except KeyError:
                    pass

        path, fragment = munge_paths(self.target_prefix, tag['href'])
        return {'para':para, 'text':text, 'path':os.path.abspath(path),
                'fragment':fragment, 'in toc': (self.link_level == 0 and
                    not self.use_spine and not self.options.no_links_in_toc)}

    def get_text(self, tag, limit=None):
        css = self.tag_css(tag)[0]
        if ('display' in css and css['display'].lower() == 'none') or ('visibility' in css and css['visibility'].lower() == 'hidden'):
            return ''
        text, alt_text = '', ''
        for c in tag.contents:
            if limit is not None and len(text) > limit:
                break
            if isinstance(c, HTMLConverter.IGNORED_TAGS):
                continue
            if isinstance(c, NavigableString):
                text += str(c)
            elif isinstance(c, Tag):
                if c.name.lower() == 'img' and c.has_attr('alt'):
                    alt_text += c['alt']
                    continue
                text += self.get_text(c)
        return text if text.strip() else alt_text

    def process_links(self):
        def add_toc_entry(text, target):
            # TextBlocks in Canvases have a None parent or an Objects Parent
            if target.parent is not None and \
               hasattr(target.parent, 'objId'):
                self.book.addTocEntry(ascii_text, tb)
            else:
                self.log.debug(f'Cannot add link {ascii_text} to TOC')

        def get_target_block(fragment, targets):
            '''Return the correct block for the <a name> element'''
            bs = targets[fragment]
            if not isinstance(bs, BlockSpace):
                return bs
            ans, found, page = None, False, bs.parent
            for item in page.contents:
                if found:
                    if isinstance(item, (TextBlock, RuledLine, ImageBlock)):
                        ans = item
                        break
                if item == bs:
                    found = True
                    continue

            if not ans:
                for i in range(len(page.contents)-1, -1, -1):
                    if isinstance(page.contents[i], (TextBlock, RuledLine, ImageBlock)):
                        ans = page.contents[i]
                        break

            if not ans:
                ntb = self.book.create_text_block()
                ntb.Paragraph(' ')
                page.append(ntb)
                ans = ntb

            if found:
                targets[fragment] = ans
                page.contents.remove(bs)
            return ans

        outside_links = deque()
        while len(self.links) > 0:
            link = self.links.popleft()
            para, text, path, fragment = link['para'], link['text'], link['path'], link['fragment']
            ascii_text = text

            if not isinstance(path, str):
                path = path.decode(sys.getfilesystemencoding())
            if path in self.processed_files:
                if path+fragment in self.targets.keys():
                    tb = get_target_block(path+fragment, self.targets)
                else:
                    tb = self.tops[path]
                if link['in toc']:
                    add_toc_entry(ascii_text, tb)

                jb = JumpButton(tb)
                self.book.append(jb)
                cb = CharButton(jb, text=text)
                para.contents = []
                para.append(cb)
                try:
                    self.unused_target_blocks.remove(tb)
                except ValueError:
                    pass
            else:
                outside_links.append(link)

        return outside_links

    def create_toc(self, toc):
        for item in toc.top_level_items():
            ascii_text = item.text
            if not item.fragment and item.abspath in self.tops:
                self.book.addTocEntry(ascii_text, self.tops[item.abspath])
            elif item.abspath:
                url = item.abspath+(item.fragment if item.fragment else '')
                if url in self.targets:
                    self.book.addTocEntry(ascii_text, self.targets[url])

    def end_page(self):
        '''
        End the current page, ensuring that any further content is displayed
        on a new page.
        '''
        if self.current_para.has_text():
            self.current_para.append_to(self.current_block)
            self.current_para = Paragraph()
        if self.current_block.has_text() or self.current_block.must_append:
            self.current_block.append_to(self.current_page)
            self.current_block = self.book.create_text_block()
        if self.current_page.has_text():
            self.book.append(self.current_page)
            self.current_page = self.book.create_page()

    def add_image_page(self, path):
        if os.access(path, os.R_OK):
            self.end_page()
            pwidth, pheight = self.profile.screen_width, self.profile.screen_height - \
                              self.profile.fudge
            page = self.book.create_page(evensidemargin=0, oddsidemargin=0,
                                         topmargin=0, textwidth=pwidth,
                                         headheight=0, headsep=0, footspace=0,
                                         footheight=0,
                                         textheight=pheight)
            if path not in self.images:
                self.images[path] = ImageStream(path)
            im = PILImage.open(path)
            width, height = im.size
            canvas = Canvas(pwidth, pheight)
            ib = ImageBlock(self.images[path], x1=width,
                            y1=height, xsize=width, ysize=height,
                            blockwidth=width, blockheight=height)
            canvas.put_object(ib, int((pwidth-width)/2.), int((pheight-height)/2.))
            page.append(canvas)
            self.book.append(page)

    def process_children(self, ptag, pcss, ppcss={}):
        ''' Process the children of ptag '''
        # Need to make a copy of contents as when
        # extract is called on a child, it will
        # mess up the iteration.
        for c in copy.copy(ptag.contents):
            if isinstance(c, HTMLConverter.IGNORED_TAGS):
                continue
            elif isinstance(c, Tag):
                self.parse_tag(c, pcss)
            elif isinstance(c, NavigableString):
                self.add_text(c, pcss, ppcss)
        if not self.in_table:
            try:
                if self.minimize_memory_usage:
                    ptag.extract()
            except AttributeError:
                print(ptag, type(ptag))

    def get_alignment(self, css):
        val = css['text-align'].lower() if 'text-align' in css else None
        align = 'head'
        if val is not None:
            if val in ['right', 'foot']:
                align = 'foot'
            elif val == 'center':
                align = 'center'
        if 'float' in css:
            val = css['float'].lower()
            if val == 'left':
                align = 'head'
            if val == 'right':
                align = 'foot'
            css.pop('float')
        return align

    def process_alignment(self, css):
        '''
        Create a new TextBlock only if necessary as indicated by css
        @type css: dict
        '''
        align = self.get_alignment(css)
        if align != self.current_block.textStyle.attrs['align']:
            self.current_para.append_to(self.current_block)
            self.current_block.append_to(self.current_page)
            ts = self.book.create_text_style(**self.current_block.textStyle.attrs)
            ts.attrs['align'] = align
            try:
                index = self.text_styles.index(ts)
                ts = self.text_styles[index]
            except ValueError:
                self.text_styles.append(ts)
            self.current_block = self.book.create_text_block(
                                blockStyle=self.current_block.blockStyle,
                                textStyle=ts)
            self.current_para = Paragraph()
            return True
        return False

    def add_text(self, tag, css, pseudo_css, force_span_use=False):
        '''
        Add text to the current paragraph taking CSS into account.
        @param tag: Either a BeautifulSoup tag or a string
        @param css: A dict
        '''
        src = tag.string if hasattr(tag, 'string') else tag
        if len(src) > 32760:
            pos = 0
            while pos < len(src):
                self.add_text(src[pos:pos+32760], css, pseudo_css, force_span_use)
                pos += 32760
            return
        src = src.replace('\r\n', '\n').replace('\r', '\n')

        if 'first-letter' in pseudo_css and len(src) > 1:
            src = src.lstrip()
            f = src[0]
            next = 1
            if f in ("'", '"', '“', '‘', '”', '’'):
                if len(src) >= 2:
                    next = 2
                    f = src[:2]
            src = src[next:]
            ncss = css.copy()
            ncss.update(pseudo_css.pop('first-letter'))
            self.add_text(f, ncss, {}, force_span_use)

        collapse_whitespace = 'white-space' not in css or css['white-space'] != 'pre'
        if self.process_alignment(css) and collapse_whitespace:
            # Dont want leading blanks in a new paragraph
            src = src.lstrip()

        def append_text(src):
            fp, key, variant = self.font_properties(css)
            for x, y in [('\xad', ''), ('\xa0', ' '), ('ﬀ', 'ff'), ('ﬁ', 'fi'), ('ﬂ', 'fl'), ('ﬃ', 'ffi'), ('ﬄ', 'ffl')]:
                src = src.replace(x, y)

            def valigner(x):
                return x
            if 'vertical-align' in css:
                valign = css['vertical-align']
                if valign in ('sup', 'super', 'sub'):
                    fp['fontsize'] = int(fp['fontsize']) * 5 // 3
                    valigner = Sub if valign == 'sub' else Sup
            normal_font_size = int(fp['fontsize'])

            if variant == 'small-caps':
                dump = Span(fontsize=normal_font_size-30)
                temp = []
                for c in src:
                    if c.isupper():
                        if temp:
                            dump.append(valigner(''.join(temp)))
                            temp = []
                        dump.append(Span(valigner(c), fontsize=normal_font_size))
                    else:
                        temp.append(c.upper())
                src = dump
                if temp:
                    src.append(valigner(''.join(temp)))
            else:
                src = valigner(src)

            if key in ['italic', 'bi']:
                already_italic = False
                for fonts in self.fonts.values():
                    it = fonts['italic'][1] if 'italic' in fonts else ''
                    bi = fonts['bi'][1] if 'bi' in fonts else ''
                    if fp['fontfacename'] in (it, bi):
                        already_italic = True
                        break
                if not already_italic:
                    src = Italic(src)

            unneeded = []
            for prop in fp:
                if fp[prop] == self.current_block.textStyle.attrs[prop]:
                    unneeded.append(prop)
            for prop in unneeded:
                fp.pop(prop)
            attrs = {}
            if 'color' in css and not self.ignore_colors:
                attrs['textcolor'] = lrs_color(css['color'])
            attrs.update(fp)
            elem = Span(text=src, **attrs) if (attrs or force_span_use) else src
            if 'text-decoration' in css:
                dec = css['text-decoration'].lower()
                linepos = 'after' if dec == 'underline' else 'before' if dec == 'overline' else None
                if linepos is not None:
                    elem = EmpLine(elem, emplineposition=linepos)
            self.current_para.append(elem)

        if collapse_whitespace:
            src = re.sub(r'\s{1,}', ' ', src)
            if self.stripped_space and len(src) == len(src.lstrip(' \n\r\t')):
                src = self.stripped_space + src
            src, orig = src.rstrip(' \n\r\t'), src
            self.stripped_space = orig[len(src):]
            if len(self.previous_text) != len(self.previous_text.rstrip(' \n\r\t')):
                src = src.lstrip(' \n\r\t')
            if len(src):
                self.previous_text = src
                append_text(src)
        else:
            srcs = src.split('\n')
            for src in srcs[:-1]:
                append_text(src)
                self.line_break()
            last = srcs[-1]
            if len(last):
                append_text(last)

    def line_break(self):
        self.current_para.append(CR())
        self.previous_text = '\n'

    def end_current_para(self):
        '''
        End current paragraph with a paragraph break after it.
        '''
        if self.current_para.contents:
            self.current_block.append(self.current_para)
        self.current_block.append(CR())
        self.current_para = Paragraph()

    def end_current_block(self):
        '''
        End current TextBlock. Create new TextBlock with the same styles.
        '''
        if self.current_para.contents:
            self.current_block.append(self.current_para)
            self.current_para = Paragraph()
        if self.current_block.contents or self.current_block.must_append:
            self.current_page.append(self.current_block)
            self.current_block = self.book.create_text_block(textStyle=self.current_block.textStyle,
                                                         blockStyle=self.current_block.blockStyle)

    def process_image(self, path, tag_css, width=None, height=None,
                      dropcaps=False, rescale=False):
        def detect_encoding(im):
            fmt = im.format
            if fmt == 'JPG':
                fmt = 'JPEG'
            return fmt
        original_path = path
        if path in self.rotated_images:
            path = self.rotated_images[path].name
        if path in self.scaled_images:
            path = self.scaled_images[path].name

        try:
            im = PILImage.open(path)
        except OSError as err:
            self.log.warning(f'Unable to process image: {original_path}\n{err}')
            return
        encoding = detect_encoding(im)

        def scale_image(width, height):
            if width <= 0:
                width = 1
            if height <= 0:
                height = 1
            pt = PersistentTemporaryFile(suffix='_html2lrf_scaled_image_.'+encoding.lower())
            self.image_memory.append(pt)  # Necessary, trust me ;-)
            try:
                im.resize((int(width), int(height)), PILImage.Resampling.LANCZOS).save(pt, encoding)
                pt.close()
                self.scaled_images[path] = pt
                return pt.name
            except (OSError, SystemError) as err:  # PIL chokes on interlaced PNG images as well a some GIF images
                self.log.warning(
                    _('Unable to process image %(path)s. Error: %(err)s')%dict(
                        path=path, err=err))

        if width is None or height is None:
            width, height = im.size
        elif rescale and (width < im.size[0] or height < im.size[1]):
            path = scale_image(width, height)
            if not path:
                return

        factor = 720./self.profile.dpi
        pheight = int(self.current_page.pageStyle.attrs['textheight'])
        pwidth  = int(self.current_page.pageStyle.attrs['textwidth'])

        if dropcaps:
            scale = False
            if width > 0.75*pwidth:
                width = int(0.75*pwidth)
                scale = True
            if height > 0.75*pheight:
                height = int(0.75*pheight)
                scale = True
            if scale:
                path = scale_image(width, height)
            if path not in self.images:
                self.images[path] = ImageStream(path)
            im = Image(self.images[path], x0=0, y0=0, x1=width, y1=height,
                               xsize=width, ysize=height)
            line_height = (int(self.current_block.textStyle.attrs['baselineskip']) +
                            int(self.current_block.textStyle.attrs['linespace']))//10
            line_height *= self.profile.dpi/72
            lines = ceil(height/line_height)
            dc = DropCaps(lines)
            dc.append(Plot(im, xsize=ceil(width*factor), ysize=ceil(height*factor)))
            self.current_para.append(dc)
            return

        if self.autorotation and width > pwidth and width > height:
            pt = PersistentTemporaryFile(suffix='_html2lrf_rotated_image_.'+encoding.lower())
            try:
                im = im.rotate(90)
                im.save(pt, encoding)
                path = pt.name
                self.rotated_images[path] = pt
                width, height = im.size
            except OSError:  # PIL chokes on interlaced PNG files and since auto-rotation is not critical we ignore the error
                self.log.debug(_('Unable to process interlaced PNG %s')% original_path)
            finally:
                pt.close()

        scaled, width, height = fit_image(width, height, pwidth, pheight)
        if scaled:
            path = scale_image(width, height)

        if not path:
            return

        if path not in self.images:
            try:
                self.images[path] = ImageStream(path, encoding=encoding)
            except LrsError as err:
                self.log.warning(f'Could not process image: {original_path}\n{err}')
                return

        im = Image(self.images[path], x0=0, y0=0, x1=width, y1=height,
                               xsize=width, ysize=height)

        self.process_alignment(tag_css)

        if max(width, height) <= min(pwidth, pheight)/5:
            self.current_para.append(Plot(im, xsize=ceil(width*factor),
                                          ysize=ceil(height*factor)))
        elif height <= floor((2/3)*pheight):
            pb = self.current_block
            self.end_current_para()
            self.process_alignment(tag_css)
            self.current_para.append(Plot(im, xsize=width*factor,
                                          ysize=height*factor))
            self.current_block.append(self.current_para)
            self.current_page.append(self.current_block)
            self.current_block = self.book.create_text_block(
                                            textStyle=pb.textStyle,
                                            blockStyle=pb.blockStyle)
            self.current_para = Paragraph()
        else:
            self.end_page()
            if len(self.current_page.contents) == 1 and not self.current_page.has_text():
                self.current_page.contents[0:1] = []
            self.current_page.append(Canvas(width=pwidth,
                                            height=height))
            left = floor((pwidth - width)/2)
            self.current_page.contents[-1].put_object(
                            ImageBlock(self.images[path], xsize=width,
                                       ysize=height, x1=width, y1=height,
                                       blockwidth=width, blockheight=height),
                            left, 0)

    def process_page_breaks(self, tag, tagname, tag_css):
        if 'page-break-before' in tag_css.keys():
            if tag_css['page-break-before'].lower() != 'avoid':
                self.end_page()
            tag_css.pop('page-break-before')
        end_page = False
        if 'page-break-after' in tag_css.keys():
            if tag_css['page-break-after'].lower() == 'avoid':
                self.avoid_page_break = True
            else:
                end_page = True
            tag_css.pop('page-break-after')
        if (self.force_page_break_attr[0].match(tagname) and
           tag.has_attr(self.force_page_break_attr[1]) and
           self.force_page_break_attr[2].match(tag[self.force_page_break_attr[1]])) or \
           self.force_page_break.match(tagname):
            self.end_page()
            self.page_break_found = True
        if not self.page_break_found and self.page_break.match(tagname):
            number_of_paragraphs = sum(
                len([1 for i in block.contents if isinstance(i, Paragraph)])
                for block in self.current_page.contents if isinstance(block, TextBlock)
            )

            if number_of_paragraphs > 2:
                self.end_page()
                self.log.debug(f'Forcing page break at {tagname}')
        return end_page

    def block_properties(self, tag_css):

        def get(what):
            src = [None for i in range(4)]
            if what in tag_css:
                msrc = tag_css[what].split()
                for i in range(min(len(msrc), len(src))):
                    src[i] = msrc[i]
            for i, c in enumerate(('-top', '-right', '-bottom', '-left')):
                if what + c in tag_css:
                    src[i] = tag_css[what+c]
            return src

        s1, s2 = get('margin'), get('padding')

        bl = str(self.current_block.blockStyle.attrs['blockwidth'])+'px'

        def set(default, one, two):
            fval = None
            if one is not None:
                val = self.unit_convert(one, base_length='10pt' if 'em' in one else bl)
                if val is not None:
                    fval = val
            if two is not None:
                val = self.unit_convert(two, base_length='10pt' if 'em' in two else bl)
                if val is not None:
                    fval = val if fval is None else fval + val
            if fval is None:
                fval = default
            return fval

        ans = {}
        ans['topskip']    = set(self.book.defaultBlockStyle.attrs['topskip'], s1[0], s2[0])
        ans['footskip']   = set(self.book.defaultBlockStyle.attrs['footskip'], s1[2], s2[2])
        ans['sidemargin'] = set(self.book.defaultBlockStyle.attrs['sidemargin'], s1[3], s2[3])

        factor = 0.7
        if 2*int(ans['sidemargin']) >= factor*int(self.current_block.blockStyle.attrs['blockwidth']):
            # Try using (left + right)/2
            val = int(ans['sidemargin'])
            ans['sidemargin'] = set(self.book.defaultBlockStyle.attrs['sidemargin'], s1[1], s2[1])
            val += int(ans['sidemargin'])
            val /= 2.
            ans['sidemargin'] = int(val)
        if 2*int(ans['sidemargin']) >= factor*int(self.current_block.blockStyle.attrs['blockwidth']):
            ans['sidemargin'] = int((factor*int(self.current_block.blockStyle.attrs['blockwidth'])) / 2)

        for prop in ('topskip', 'footskip', 'sidemargin'):
            if isinstance(ans[prop], string_or_bytes):
                ans[prop] = int(ans[prop])
            if ans[prop] < 0:
                ans[prop] = 0

        return ans

    def font_properties(self, css):
        '''
        Convert the font propertiess in css to the Xylog equivalents. If the CSS
        does not contain a particular font property, the default from self.book.defaultTextSytle
        is used. Assumes 1em = 10pt
        @return: dict, key, variant. The dict contains the Xlog equivalents. key indicates
          the font type (i.e. bold, bi, normal) and variant is None or 'small-caps'
        '''
        t = {}
        for key in ('fontwidth', 'fontsize', 'wordspace', 'fontfacename', 'fontweight', 'baselineskip'):
            t[key] = self.book.defaultTextStyle.attrs[key]

        def font_weight(val):
            ans = 0
            m = re.search(r'([0-9]+)', val)
            if m:
                ans = int(m.group(1))
            elif val.find('bold') >= 0 or val.find('strong') >= 0:
                ans = 700
            return 'bold' if ans >= 700 else 'normal'

        def font_style(val):
            ans = 'normal'
            if 'italic' in val or 'oblique' in val:
                ans = 'italic'
            return ans

        def font_family(val):
            ans = 'serif'
            if max(val.find('courier'), val.find('mono'), val.find('fixed'), val.find('typewriter'))>=0:
                ans = 'mono'
            elif max(val.find('arial'), val.find('helvetica'), val.find('verdana'),
                 val.find('trebuchet'), val.find('sans')) >= 0:
                ans = 'sans'
            return ans

        def font_variant(val):
            ans = None
            if 'small-caps' in val.lower():
                ans = 'small-caps'
            return ans

        def font_key(family, style, weight):
            key = 'normal'
            if style == 'italic' and weight == 'normal':
                key = 'italic'
            elif style == 'normal' and weight == 'bold':
                key = 'bold'
            elif style == 'italic' and weight == 'bold':
                key = 'bi'
            return key

        def font_size(val):
            '''
            Assumes 1em=100%=10pt
            '''
            normal = 100
            ans = self.unit_convert(val, pts=True, base_length='10pt')

            if ans:
                if ans <= 0:
                    ans += normal
                    if ans == 0:  # Common case of using -1em to mean "smaller"
                        ans = int(font_size('smaller'))
                    if ans < 0:
                        ans = normal
            else:
                if ans == 0:
                    ans = int(font_size('smaller'))
                elif 'smaller' in val:
                    ans = normal - 20
                elif 'xx-small' in val:
                    ans = 40
                elif 'x-small' in val:
                    ans = 60
                elif 'small' in val:
                    ans = 80
                elif 'medium' in val:
                    ans = 100
                elif 'larger' in val:
                    ans = normal + 20
                elif 'xx-large' in val:
                    ans = 180
                elif 'x-large' in val:
                    ans = 140
                elif 'large' in val:
                    ans = 120
            if ans is not None:
                ans += int(self.font_delta * 20)
                ans = str(ans)
            return ans

        family, weight, style, variant = 'serif', 'normal', 'normal', None
        for key in css.keys():
            val = css[key].lower()
            if key == 'font':
                vals = val.split()
                for val in vals:
                    family = font_family(val)
                    if family != 'serif':
                        break
                for val in vals:
                    weight = font_weight(val)
                    if weight != 'normal':
                        break
                for val in vals:
                    style = font_style(val)
                    if style != 'normal':
                        break
                for val in vals:
                    sz = font_size(val)
                    if sz:
                        t['fontsize'] = sz
                        break
                for val in vals:
                    variant = font_variant(val)
                    if variant:
                        t['fontvariant'] = variant
                        break
            elif key in ['font-family', 'font-name']:
                family = font_family(val)
            elif key == 'font-size':
                ans = font_size(val)
                if ans:
                    t['fontsize'] = ans
            elif key == 'font-weight':
                weight = font_weight(val)
            elif key == 'font-style':
                style = font_style(val)
            elif key == 'font-variant':
                variant = font_variant(val)

        if variant:
            css['font-variant'] = variant

        key = font_key(family, style, weight)
        if key in self.fonts[family]:
            t['fontfacename'] = self.fonts[family][key][1]
        else:
            t['fontfacename'] = self.fonts[family]['normal'][1]
        if key in ['bold', 'bi']:
            t['fontweight'] = 700

        fs = int(t['fontsize'])
        if fs > 120:
            t['wordspace'] = fs // 4
        t['baselineskip'] = fs + 20
        return t, key, variant

    def unit_convert(self, val, pts=False, base_length='10pt'):
        '''
        Tries to convert html units in C{val} to pixels.
        @param pts: If True return 10*pts instead of pixels.
        @return: The number of pixels (an int) if successful. Otherwise, returns None.
        '''
        dpi = self.profile.dpi
        result = None
        try:
            result = int(val)
        except ValueError:
            pass
        m = re.search(r'\s*(-*[0-9]*\.?[0-9]*)\s*(%|em|px|mm|cm|in|dpt|pt|pc)', val)

        if m is not None and m.group(1):
            unit = float(m.group(1))
            if m.group(2) == '%':
                normal = self.unit_convert(base_length)
                result = (unit/100) * normal
            elif m.group(2) == 'px':
                result = unit
            elif m.group(2) == 'in':
                result = unit * dpi
            elif m.group(2) == 'pt':
                result = unit * dpi/72
            elif m.group(2) == 'dpt':
                result = unit * dpi/720
            elif m.group(2) == 'em':
                normal = self.unit_convert(base_length)
                result = unit * normal
            elif m.group(2) == 'pc':
                result = unit * (dpi/72) * 12
            elif m.group(2) == 'mm':
                result = unit * 0.04 * (dpi)
            elif m.group(2) == 'cm':
                result = unit * 0.4 * (dpi)
        if result is not None:
            if pts:
                result = round(result * (720/dpi))
            else:
                result = round(result)
        return result

    def text_properties(self, tag_css):
        indent = self.book.defaultTextStyle.attrs['parindent']
        if 'text-indent' in tag_css:
            bl = str(self.current_block.blockStyle.attrs['blockwidth'])+'px'
            if 'em' in tag_css['text-indent']:
                bl = '10pt'
            indent = self.unit_convert(str(tag_css['text-indent']), pts=True, base_length=bl)
            if not indent:
                indent = 0
            if indent > 0 and indent < 10 * self.minimum_indent:
                indent = int(10 * self.minimum_indent)

        fp = self.font_properties(tag_css)[0]
        fp['parindent'] = indent

        if 'line-height' in tag_css:
            bls, ls = int(self.book.defaultTextStyle.attrs['baselineskip']), \
                      int(self.book.defaultTextStyle.attrs['linespace'])
            try:  # See if line-height is a unitless number
                val = int(float(tag_css['line-height'].strip()) * (ls))
                fp['linespace'] = val
            except ValueError:
                val = self.unit_convert(tag_css['line-height'], pts=True, base_length='1pt')
            if val is not None:
                val -= bls
                if val >= 0:
                    fp['linespace'] = val

        return fp

    def process_block(self, tag, tag_css):
        ''' Ensure padding and text-indent properties are respected '''
        text_properties = self.text_properties(tag_css)
        block_properties = self.block_properties(tag_css)
        indent = (float(text_properties['parindent'])/10) * (self.profile.dpi/72)
        margin = float(block_properties['sidemargin'])
        # Since we're flattening the block structure, we need to ensure that text
        # doesn't go off the left edge of the screen
        if indent < 0 and margin + indent < 0:
            text_properties['parindent'] = int(-margin * (72/self.profile.dpi) * 10)

        align = self.get_alignment(tag_css)

        def fill_out_properties(props, default):
            for key in default.keys():
                if key not in props:
                    props[key] = default[key]

        fill_out_properties(block_properties, self.book.defaultBlockStyle.attrs)
        fill_out_properties(text_properties, self.book.defaultTextStyle.attrs)

        def properties_different(dict1, dict2):
            for key in dict1.keys():
                if dict1[key] != dict2[key]:
                    return True
            return False

        if properties_different(self.current_block.blockStyle.attrs, block_properties) or \
           properties_different(self.current_block.textStyle.attrs, text_properties) or\
           align != self.current_block.textStyle.attrs['align']:
            ts = self.current_block.textStyle.copy()
            ts.attrs.update(text_properties)
            ts.attrs['align'] = align
            bs = self.current_block.blockStyle.copy()
            if not self.preserve_block_style:
                bs.attrs.update(block_properties)
            self.current_block.append_to(self.current_page)
            try:
                index = self.text_styles.index(ts)
                ts = self.text_styles[index]
            except ValueError:
                self.text_styles.append(ts)
            try:
                index = self.block_styles.index(bs)
                bs = self.block_styles[index]
            except ValueError:
                self.block_styles.append(bs)
            self.current_block = self.book.create_text_block(blockStyle=bs,
                                                             textStyle=ts)
            return True
        return False

    def process_anchor(self, tag, tag_css, tag_pseudo_css):
        if not self.in_table:  # Anchors in tables are handled separately
            key = 'name' if tag.has_attr('name') else 'id'
            name = tag[key].replace('#', '')
            previous = self.current_block
            self.process_children(tag, tag_css, tag_pseudo_css)
            target = None

            if self.current_block == previous:
                self.current_block.must_append = True
                target = self.current_block
            else:
                found = False
                for item in self.current_page.contents:
                    if item == previous:
                        found = True
                        continue
                    if found:
                        target = item
                        break
                if target and not isinstance(target, (TextBlock, ImageBlock)):
                    if isinstance(target, RuledLine):
                        target = self.book.create_text_block(textStyle=self.current_block.textStyle,
                                                     blockStyle=self.current_block.blockStyle)
                        target.Paragraph(' ')
                        self.current_page.append(target)
                    else:
                        target = BlockSpace()
                        self.current_page.append(target)
                if target is None:
                    if self.current_block.has_text():
                        target = self.current_block
                    else:
                        target = self.current_block
                        self.current_block.must_append = True
            self.targets[self.target_prefix+name] = target
        else:
            self.process_children(tag, tag_css, tag_pseudo_css)

    def parse_tag(self, tag, parent_css):
        try:
            tagname = tag.name.lower()
        except AttributeError:
            if not isinstance(tag, HTMLConverter.IGNORED_TAGS):
                self.add_text(tag, parent_css, {})
            return
        tag_css, tag_pseudo_css = self.tag_css(tag, parent_css=parent_css)
        try:  # Skip element if its display attribute is set to none
            if tag_css['display'].lower() == 'none' or \
               tag_css['visibility'].lower() == 'hidden':
                return
        except KeyError:
            pass
        if not self.disable_chapter_detection and \
           (self.chapter_attr[0].match(tagname) and
            (self.chapter_attr[1].lower() == 'none' or
             (tag.has_attr(self.chapter_attr[1]) and
              self.chapter_attr[2].match(tag[self.chapter_attr[1]])))):
            self.log.debug(f'Detected chapter {tagname}')
            self.end_page()
            self.page_break_found = True

            if self.options.add_chapters_to_toc:
                self.current_block.must_append = True
                self.extra_toc_entries.append((self.get_text(tag,
                    limit=1000), self.current_block))

        end_page = self.process_page_breaks(tag, tagname, tag_css)
        try:
            if tagname in ['title', 'script', 'meta', 'del', 'frameset']:
                pass
            elif tagname == 'a' and self.link_levels >= 0:
                if tag.has_attr('href') and not self.link_exclude.match(tag['href']):
                    if urlparse(tag['href'])[0] not in ('', 'file'):
                        self.process_children(tag, tag_css, tag_pseudo_css)
                    else:
                        path = munge_paths(self.target_prefix, tag['href'])[0]
                        ext = os.path.splitext(path)[1]
                        if ext:
                            ext = ext[1:].lower()
                        if os.access(path, os.R_OK) and os.path.isfile(path):
                            if ext in ['png', 'jpg', 'bmp', 'jpeg']:
                                self.process_image(path, tag_css)
                            else:
                                text = self.get_text(tag, limit=1000)
                                if not text.strip():
                                    text = 'Link'
                                self.add_text(text, tag_css, {}, force_span_use=True)
                                self.links.append(self.create_link(self.current_para.contents, tag))
                                if tag.has_attr('id') or tag.has_attr('name'):
                                    key = 'name' if tag.has_attr('name') else 'id'
                                    self.targets[self.target_prefix+tag[key]] = self.current_block
                                    self.current_block.must_append = True
                        else:
                            self.log.debug('Could not follow link to '+tag['href'])
                            self.process_children(tag, tag_css, tag_pseudo_css)
                elif tag.has_attr('name') or tag.has_attr('id'):
                    self.process_anchor(tag, tag_css, tag_pseudo_css)
                else:
                    self.process_children(tag, tag_css, tag_pseudo_css)
            elif tagname == 'img':
                if tag.has_attr('src'):
                    path = munge_paths(self.target_prefix, tag['src'])[0]
                    if not os.path.exists(path):
                        path = path.replace('&', '%26')  # convertlit replaces & with %26
                    if os.access(path, os.R_OK) and os.path.isfile(path):
                        width, height = None, None
                        try:
                            width = int(tag['width'])
                            height = int(tag['height'])
                        except:
                            pass
                        dropcaps = tag.get('class') in ('libprs500_dropcaps', ['libprs500_dropcaps'])
                        self.process_image(path, tag_css, width, height,
                                           dropcaps=dropcaps, rescale=True)
                    elif not urlparse(tag['src'])[0]:
                        self.log.warn('Could not find image: '+tag['src'])
                else:
                    self.log.debug(f'Failed to process: {tag!s}')
            elif tagname in ['style', 'link']:
                ncss, npcss = {}, {}
                if tagname == 'style':
                    text = ''.join([str(i) for i in tag.findAll(text=True)])
                    css, pcss = self.parse_css(text)
                    ncss.update(css)
                    npcss.update(pcss)
                elif (tag.has_attr('type') and tag['type'] in ('text/css', 'text/x-oeb1-css') and tag.has_attr('href')):
                    path = munge_paths(self.target_prefix, tag['href'])[0]
                    try:
                        with open(path, 'rb') as f:
                            src = f.read().decode('utf-8', 'replace')
                        match = self.PAGE_BREAK_PAT.search(src)
                        if match and not re.match(r'avoid', match.group(1), re.IGNORECASE):
                            self.page_break_found = True
                        ncss, npcss = self.parse_css(src)
                    except OSError:
                        self.log.warn('Could not read stylesheet: '+tag['href'])
                if ncss:
                    update_css(ncss, self.css)
                    self.css.update(self.override_css)
                if npcss:
                    update_css(npcss, self.pseudo_css)
                    self.pseudo_css.update(self.override_pcss)
            elif tagname == 'pre':
                self.end_current_para()
                self.end_current_block()
                self.current_block = self.book.create_text_block()
                ts = self.current_block.textStyle.copy()
                self.current_block.textStyle = ts
                self.current_block.textStyle.attrs['parindent'] = '0'

                if tag.contents:
                    c = tag.contents[0]
                    if isinstance(c, NavigableString):
                        c = str(c).replace('\r\n', '\n').replace('\r', '\n')
                        if c.startswith('\n'):
                            c = c[1:]
                            tag.contents[0] = NavigableString(c)
                            tag.contents[0].setup(tag)
                self.process_children(tag, tag_css, tag_pseudo_css)
                self.end_current_block()
            elif tagname in ['ul', 'ol', 'dl']:
                self.list_level += 1
                if tagname == 'ol':
                    old_counter = self.list_counter
                    self.list_counter = 1
                    try:
                        self.list_counter = int(tag['start'])
                    except:
                        pass
                prev_bs = self.current_block.blockStyle
                self.end_current_block()
                attrs = self.current_block.blockStyle.attrs
                attrs = attrs.copy()
                attrs['sidemargin'] = self.list_indent*self.list_level
                bs = self.book.create_block_style(**attrs)
                self.current_block = self.book.create_text_block(
                                            blockStyle=bs,
                                            textStyle=self.unindented_style)
                self.process_children(tag, tag_css, tag_pseudo_css)
                self.end_current_block()
                self.current_block.blockStyle = prev_bs
                self.list_level -= 1
                if tagname == 'ol':
                    self.list_counter = old_counter
            elif tagname in ['li', 'dt', 'dd']:
                margin = self.list_indent*self.list_level
                if tagname == 'dd':
                    margin += 80
                if int(self.current_block.blockStyle.attrs['sidemargin']) != margin:
                    self.end_current_block()
                    attrs = self.current_block.blockStyle.attrs
                    attrs = attrs.copy()
                    attrs['sidemargin'] = margin
                    attrs['blockwidth'] = int(attrs['blockwidth']) + margin
                    bs = self.book.create_block_style(**attrs)
                    self.current_block = self.book.create_text_block(
                                            blockStyle=bs,
                                            textStyle=self.unindented_style)

                if self.current_para.has_text():
                    self.line_break()
                    self.current_block.append(self.current_para)
                self.current_para = Paragraph()
                self.previous_text = '\n'
                if tagname == 'li':
                    in_ol, parent = True, tag.parent
                    while parent:
                        if parent.name and parent.name.lower() in ['ul', 'ol']:
                            in_ol = parent.name.lower() == 'ol'
                            break
                        parent = parent.parent
                    prepend = str(self.list_counter)+'. ' if in_ol else '•' + ' '
                    self.current_para.append(Span(prepend))
                    self.process_children(tag, tag_css, tag_pseudo_css)
                    if in_ol:
                        self.list_counter += 1
                else:
                    self.process_children(tag, tag_css, tag_pseudo_css)
            elif tagname == 'blockquote':
                self.current_para.append_to(self.current_block)
                self.current_block.append_to(self.current_page)
                pb = self.current_block
                self.current_para = Paragraph()
                ts = self.book.create_text_style()
                ts.attrs['parindent'] = 0
                try:
                    index = self.text_styles.index(ts)
                    ts = self.text_styles[index]
                except ValueError:
                    self.text_styles.append(ts)
                bs = self.book.create_block_style()
                bs.attrs['sidemargin'], bs.attrs['topskip'], bs.attrs['footskip'] = \
                60, 20, 20
                try:
                    index = self.block_styles.index(bs)
                    bs = self.block_styles[index]
                except ValueError:
                    self.block_styles.append(bs)
                self.current_block = self.book.create_text_block(
                                        blockStyle=bs, textStyle=ts)
                self.previous_text = '\n'
                self.preserve_block_style = True
                self.process_children(tag, tag_css, tag_pseudo_css)
                self.preserve_block_style = False
                self.current_para.append_to(self.current_block)
                self.current_block.append_to(self.current_page)
                self.current_para = Paragraph()
                self.current_block = self.book.create_text_block(textStyle=pb.textStyle,
                                                                 blockStyle=pb.blockStyle)
            elif tagname in ['p', 'div', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6']:
                new_block = self.process_block(tag, tag_css)

                if (self.anchor_ids and tag.has_attr('id')) or (self.book_designer and tag.get('class') in ('title', ['title'])):
                    if not tag.has_attr('id'):
                        tag['id'] = __appname__+'_id_'+str(self.id_counter)
                        self.id_counter += 1

                    tkey = self.target_prefix+tag['id']
                    if not new_block:
                        self.end_current_block()
                    self.current_block.must_append = True
                    self.targets[tkey] = self.current_block
                    if (self.book_designer and tag.get('class') in ('title', ['title'])):
                        self.extra_toc_entries.append((self.get_text(tag, 100), self.current_block))

                src = self.get_text(tag, limit=1000)

                if not self.disable_chapter_detection and tagname.startswith('h'):
                    if self.chapter_regex.search(src):
                        self.log.debug(f'Detected chapter {src}')
                        self.end_page()
                        self.page_break_found = True

                        if self.options.add_chapters_to_toc:
                            self.current_block.must_append = True
                            self.extra_toc_entries.append((self.get_text(tag,
                                limit=1000), self.current_block))

                if self.current_para.has_text():
                    self.current_para.append_to(self.current_block)
                self.current_para = Paragraph()

                self.previous_text = '\n'

                if not tag.contents:
                    self.current_block.append(CR())
                    return

                if self.current_block.contents:
                    self.current_block.append(CR())

                self.process_children(tag, tag_css, tag_pseudo_css)

                if self.current_para.contents:
                    self.current_block.append(self.current_para)
                self.current_para = Paragraph()
                if tagname.startswith('h') or self.blank_after_para:
                    self.current_block.append(CR())
            elif tagname in ['b', 'strong', 'i', 'em', 'span', 'tt', 'big', 'code', 'cite', 'sup', 'sub']:
                self.process_children(tag, tag_css, tag_pseudo_css)
            elif tagname == 'font':
                if tag.has_attr('face'):
                    tag_css['font-family'] = tag['face']
                if tag.has_attr('color'):
                    tag_css['color'] = tag['color']
                self.process_children(tag, tag_css, tag_pseudo_css)
            elif tagname in ['br']:
                self.line_break()
                self.previous_text = '\n'
            elif tagname in ['hr', 'tr']:  # tr needed for nested tables
                self.end_current_block()
                if tagname == 'hr' and not tag_css.get('width', '').strip().startswith('0'):
                    self.current_page.RuledLine(linelength=int(self.current_page.pageStyle.attrs['textwidth']))
                self.previous_text = '\n'
                self.process_children(tag, tag_css, tag_pseudo_css)
            elif tagname == 'td':  # Needed for nested tables
                if not self.in_table:
                    self.current_para.append(' ')
                    self.previous_text = ' '
                self.process_children(tag, tag_css, tag_pseudo_css)
            elif tagname == 'table' and not self.ignore_tables and not self.in_table:
                tag_css = self.tag_css(tag)[0]  # Table should not inherit CSS
                try:
                    self.process_table(tag, tag_css)
                except Exception as err:
                    self.log.warning(_('An error occurred while processing a table: %s. Ignoring table markup.')%repr(err))
                    self.log.exception('')
                    self.log.debug(_('Bad table:\n%s')%str(tag)[:300])
                    self.in_table = False
                    self.process_children(tag, tag_css, tag_pseudo_css)
                finally:
                    if self.minimize_memory_usage:
                        tag.extract()
            else:
                self.process_children(tag, tag_css, tag_pseudo_css)
        finally:
            if end_page:
                self.end_page()

    def process_table(self, tag, tag_css):
        self.end_current_block()
        self.current_block = self.book.create_text_block()
        rowpad = 10
        table = Table(self, tag, tag_css, rowpad=rowpad, colpad=10)
        canvases = []
        ps = self.current_page.pageStyle.attrs
        for block, xpos, ypos, delta, targets in table.blocks(int(ps['textwidth']), int(ps['textheight'])):
            if not block:
                if ypos > int(ps['textheight']):
                    raise Exception(_('Table has cell that is too large'))
                canvases.append(Canvas(int(self.current_page.pageStyle.attrs['textwidth']), ypos+rowpad,
                        blockrule='block-fixed'))
                for name in targets:
                    self.targets[self.target_prefix+name] = canvases[-1]
            else:
                if xpos > 65535:
                    xpos = 65535
                canvases[-1].put_object(block, xpos + int(delta/2), ypos)

        for canvas in canvases:
            self.current_page.append(canvas)
        self.end_current_block()

    def remove_unused_target_blocks(self):
        for block in self.unused_target_blocks:
            block.parent.contents.remove(block)
            block.parent = None

    def writeto(self, path, lrs=False):
        self.remove_unused_target_blocks()
        self.book.renderLrs(path) if lrs else self.book.renderLrf(path)

    def cleanup(self):
        for _file in chain(itervalues(self.scaled_images), itervalues(self.rotated_images)):
            _file.__del__()


def process_file(path, options, logger):
    path = os.path.abspath(path)
    default_title = force_unicode(os.path.splitext(os.path.basename(path))[0], filesystem_encoding)
    dirpath = os.path.dirname(path)

    tpath = ''
    try_opf(path, options, logger)
    if getattr(options, 'cover', None):
        options.cover = os.path.expanduser(options.cover)
        if not os.path.isabs(options.cover):
            options.cover = os.path.join(dirpath, options.cover)
        if os.access(options.cover, os.R_OK):
            th = Device.THUMBNAIL_HEIGHT
            im = PILImage.open(options.cover)
            pwidth, pheight = options.profile.screen_width, \
                              options.profile.screen_height - options.profile.fudge
            width, height = im.size
            if width < pwidth:
                corrf = pwidth/width
                width, height = pwidth, int(corrf*height)

            scaled, width, height = fit_image(width, height, pwidth, pheight)
            try:
                cim = im.resize((width, height), PILImage.BICUBIC).convert('RGB') if \
                      scaled else im
                cf = PersistentTemporaryFile(prefix=__appname__+'_', suffix='.jpg')
                cf.close()
                cim.convert('RGB').save(cf.name)
                options.cover = cf.name

                tim = im.resize((int(0.75*th), th), PILImage.Resampling.LANCZOS).convert('RGB')
                tf = PersistentTemporaryFile(prefix=__appname__+'_', suffix='.jpg')
                tf.close()
                tim.save(tf.name)
                tpath = tf.name
            except OSError as err:  # PIL sometimes fails, for example on interlaced PNG files
                logger.warn(_('Could not read cover image: %s'), err)
                options.cover = None
        else:
            raise ConversionError(_('Cannot read from: %s')% (options.cover,))

    if not options.title:
        options.title = default_title

    for prop in ('author', 'author_sort', 'title', 'title_sort', 'publisher', 'freetext'):
        val = getattr(options, prop, None)
        if val and not isinstance(val, str):
            soup = BeautifulSoup(val)
            setattr(options, prop, str(soup))

    title = (options.title, options.title_sort)
    author = (options.author, options.author_sort)

    args = dict(font_delta=options.font_delta, title=title,
                author=author, sourceencoding='utf8',
                freetext=options.freetext, category=options.category,
                publisher=options.publisher,
                booksetting=BookSetting(dpi=10*options.profile.dpi,
                                        screenheight=options.profile.screen_height,
                                        screenwidth=options.profile.screen_width))
    if tpath:
        args['thumbnail'] = tpath
    header = None
    if options.header:
        header = Paragraph()
        fheader = options.headerformat
        if not options.title:
            options.title = _('Unknown')
        if not options.author:
            options.author = _('Unknown')
        if not fheader:
            fheader = '%t by %a'
        fheader = re.sub(r'(?<!%)%t', options.title, fheader)
        fheader = re.sub(r'(?<!%)%a', options.author, fheader)
        fheader = fheader.replace('%%a', '%a')
        fheader = fheader.replace('%%t', '%t')
        header.append(fheader + '  ')
    book, fonts = Book(options, logger, header=header, **args)
    le = re.compile(options.link_exclude) if options.link_exclude else \
         re.compile(r'$')
    pb = re.compile(options.page_break, re.IGNORECASE) if options.page_break else \
         re.compile(r'$')
    fpb = re.compile(options.force_page_break, re.IGNORECASE) if options.force_page_break else \
         re.compile(r'$')
    cq = options.chapter_attr.split(',')
    if len(cq) < 3:
        raise ValueError('The --chapter-attr setting must have 2 commas.')
    options.chapter_attr = [re.compile(cq[0], re.IGNORECASE), cq[1],
                            re.compile(cq[2], re.IGNORECASE)]
    options.force_page_break = fpb
    options.link_exclude = le
    options.page_break = pb
    if not isinstance(options.chapter_regex, str):
        options.chapter_regex = options.chapter_regex.decode(preferred_encoding)
    options.chapter_regex = re.compile(options.chapter_regex, re.IGNORECASE)
    fpba = options.force_page_break_attr.split(',')
    if len(fpba) != 3:
        fpba = ['$', '', '$']
    options.force_page_break_attr = [re.compile(fpba[0], re.IGNORECASE), fpba[1],
                                     re.compile(fpba[2], re.IGNORECASE)]
    if not hasattr(options, 'anchor_ids'):
        options.anchor_ids = True
    files = options.spine if (options.use_spine and hasattr(options, 'spine')) else [path]
    conv = HTMLConverter(book, fonts, options, logger, files)
    if options.use_spine and hasattr(options, 'toc') and options.toc is not None:
        conv.create_toc(options.toc)
    oname = options.output
    if not oname:
        suffix = '.lrs' if options.lrs else '.lrf'
        name = os.path.splitext(os.path.basename(path))[0] + suffix
        oname = os.path.join(os.getcwd(), name)
    oname = os.path.abspath(os.path.expanduser(oname))
    conv.writeto(oname, lrs=options.lrs)
    conv.cleanup()
    return oname


def try_opf(path, options, logger):
    if hasattr(options, 'opf'):
        opf = options.opf
    else:
        files = glob.glob(os.path.join(os.path.dirname(path),'*'))
        opf = None
        for f in files:
            ext = f.rpartition('.')[-1].lower()
            if ext == 'opf':
                opf = f
                break
    if opf is None:
        return

    dirpath = os.path.dirname(os.path.abspath(opf))
    from calibre.ebooks.metadata.opf2 import OPF as OPF2
    with open(opf, 'rb') as f:
        opf = OPF2(f, dirpath)
    try:
        title = opf.title
        if title and not getattr(options, 'title', None):
            options.title = title
        if getattr(options, 'author', 'Unknown') == 'Unknown':
            if opf.authors:
                options.author = ', '.join(opf.authors)
            if opf.author_sort:
                options.author_sort = opf.author_sort
        if options.publisher == 'Unknown':
            publisher = opf.publisher
            if publisher:
                options.publisher = publisher
        if not getattr(options, 'cover', None) or options.use_metadata_cover:
            orig_cover = getattr(options, 'cover', None)
            options.cover = None
            cover = opf.cover
            if cover:
                cover = cover.replace('/', os.sep)
                if not os.path.isabs(cover):
                    cover = os.path.join(dirpath, cover)
                if os.access(cover, os.R_OK):
                    try:
                        PILImage.open(cover)
                        options.cover = cover
                    except:
                        pass
            if not getattr(options, 'cover', None) and orig_cover is not None:
                options.cover = orig_cover
        if getattr(opf, 'spine', False):
            options.spine = [i.path for i in opf.spine if i.path]
        if not getattr(options, 'toc', None):
            options.toc   = opf.toc
    except Exception:
        logger.exception(_('Failed to process OPF file'))
