--- Here goes everything that does not belong anywhere else. Other parts are font handling, the command
--- list, page and grid setup, debugging and initialization. We start with the function publisher#dothings that
--- initializes some variables and starts processing (publisher#dispatch())
--
--  publisher.lua
--  speedata publisher
--
--  For a list of authors see `git blame'
--  See file COPYING in the root directory for license info.


file_start("publisher.lua")
barcodes = do_luafile("barcodes.lua")
local luxor = do_luafile("luxor.lua")
local xmlbuilder = require("xmlbuilder")
local spotcolors = require("spotcolors")

if os.getenv("SP_XMLPARSER") == "lxpath" then
    xpath = require("lxpath")
    xpath.stringmatch = unicode.utf8.match
    xpath.find_file = kpse.find_file
    xpath.parse_xml = splib.load_xmlfile
    xpath.ignoreNS = true
else
    xpath = do_luafile("xpath.lua")
end

hasharfbuzz, harfbuzz = pcall(require,'luaharfbuzz')
if not hasharfbuzz then
    warning("harfbuzz library not found")
end


local commands      = require("publisher.commands")
local page          = require("publisher.page")
local fontloader    = require("fonts.fontloader")
local fonts         = require("publisher.fonts")
local uuid          = require("uuid")
local html          = require("html")
local colors_module = require("publisher.colors")
local metadata      = require("publisher.metadata")
local links_module  = require("publisher.links")
par                 = require("par")
uuid.randomseed(tex.randomseed)

env_publisherversion = os.getenv("PUBLISHERVERSION")
module(...,package.seeall)

-- expose helpers from submodules
utf8_to_utf16_string_pdf = metadata.utf8_to_utf16_string_pdf


newxpath = false

if os.getenv("SP_XMLPARSER") == "lxpath" then
    newxpath = true
    do_luafile("layout_functions_lxpath.lua")
else
    do_luafile("layout_functions.lua")
end


-- so that node.copy_list copies the node properties
node.set_properties_mode(true)

--- One big point (DTP point, PostScript point) is approx. 65781 scaled points.
factor = 65781
-- factor = 65781.7

-- no more than this number of frames is allowed on a page
maxframes = 999

tenpoint_sp    = tex.sp("10pt")
twelvepoint_sp = tex.sp("12pt")
tenmm_sp       = tex.sp("10mm")
onemm_sp       = tex.sp("1mm")
onein_sp       = tex.sp("1in")
onept_sp       = tex.sp("1pt")
onepc_sp       = tex.sp("1pc")
onepp_sp       = tex.sp("1pp")
onedd_sp       = tex.sp("1dd")
onecc_sp       = tex.sp("1cc")
onecm_sp       = tenmm_sp

-- User has a pro plan
pro = false

has_pro_error = false

--- Attributes
--- ----------
--- Attributes are attached to nodes, so we can store information that are not present in the
--- nodes themselves or are evaluated later on (such as font selection - when generating glyph
--- nodes, we don't yet know what font the user will use).
---
--- Attributes may have any number, they just need to be constant across the whole source.
--- The attributes value must also be a number.

--- Instead of storing strings we store indexes to strings based on the attributes table.
--- Note: there are also properties in LuaTeX which are much more flexible, we use the old mechanism
--- because in disc nodes, the attributes are inherited (as far as I can see).
attributes = {
    ["background-color"] = true,
    ["bgpaddingbottom"] = true,
    ["bgpaddingtop"] = true,
    ["bordernumber"] = true,
    ["borderwd"] = true,
    ["borderht"] = true,
    ["borderdp"] = true,
    ["color"] = true,
    ["font-style"] = {"italic","oblique"},
    ["font-weight"] = {"normal","bold"},
    ["fontfamily"] = true,
    ["hyperlink"] = true,
    ["indent"] = true,
    ["margintop"] = true,
    ["marginbottom"] = true,
    ["newline"] = true,
    ["paddingtop"] = true,
    ["paddingbottom"] = true,
    ["rows"] = true,
    ["spaceglue"] = true,
    ["text-decoration-color"] = true,
    ["text-decoration-line"] = {"underline","overline","line-through"},
    ["text-decoration-style"] = {"solid","double","dotted","dashed","wavy"},
    ["transparency"] = true,
    ["underline_color"] = true,
    ["underline"] = true,
    ["vertical-align"] = {"baseline","top","middle","bottom","sub","super"},
}

attribute_name_number = {}
attribute_number_name = {}
do
    local c = 1
    local sorted_keys = {}
    -- attribute sorting is just for debugging purposes
    for k, _ in pairs(attributes) do
        sorted_keys[#sorted_keys + 1] = k
    end
    table.sort(sorted_keys)
    for _, k in ipairs(sorted_keys) do
        attribute_name_number[k] = c
        attribute_number_name[c] = k
        c = c + 1
    end
end

att_rows           = 98 -- see text formats for details

--- These attributes are for image shifting. The amount of shift up/left can
--- be negative and is counted in scaled points.
att_shift_left     = 100
att_shift_up       = 101

--- A tie glue (U+00A0) is a non-breaking space
att_tie_glue       = 201

--- These attributes are used in tabular material
att_space_prio     = 300
att_space_amount   = 301

att_break_below_forbidden = 400
att_break_above           = 401
att_omit_at_top           = 402
att_use_as_head           = 403
-- HTML tables should not be paragraph:format()ted
att_dont_format           = 404
att_margin_newcolumn      = 405
att_margin_top_boxstart   = 406
att_ignore_orphan_widowsetting = 407

att_margin_top = 450
att_margin_bottom = 451

att_break_before = 452

--- `att_is_table_row` is used in `tabular.lua` and if set to 1, it denotes
--- a regular table row, and not a spacer. Spacers must not appear
--- at the top or the bottom of a table, unless forced to.
att_is_table_row    = 500
att_tr_dynamic_data = 501

-- for border-collapse (vertical)
att_tr_shift_up     = 550

-- Force a hbox line height
att_lineheight = 600
att_dontadjustlineheight = 601

-- server-mode / line breaking (not used anymore?)
att_keep = 700

-- attributes for glue
att_leaderwd = 800
att_tablenewpage = 801

-- mknodes
att_newline = 900

-- PDF/UA - tagged PDF
att_role  = 1000


user_defined_addtolist = 1
user_defined_bookmark  = 2
user_defined_mark      = 3
user_defined_marker    = 4
user_defined_mark_append = 5


action_node    = node.id("action")
disc_node      = node.id("disc")
dir_node       = node.id("dir")
glue_node      = node.id("glue")
glue_spec_node = node.id("glue_spec")
glyph_node     = node.id("glyph")
hlist_node     = node.id("hlist")
kern_node      = node.id("kern")
penalty_node   = node.id("penalty")
rule_node      = node.id("rule")
vlist_node     = node.id("vlist")
whatsit_node   = node.id("whatsit")


for k,v in pairs(node.whatsits()) do
    if v == "user_defined" then
        -- for mark command
        user_defined_whatsit = k
    elseif v == "pdf_refximage" then
        pdf_refximage_whatsit = k
    elseif v == "pdf_action" then
        pdf_action_whatsit = k
    elseif v == "pdf_dest" then
        pdf_dest_whatsit = k
    elseif v == "pdf_start_link" then
        pdf_start_link_whatsit = k
    elseif v == "pdf_literal" then
        pdf_literal_whatsit = k
    end
end

-- sd:alternating
alternating = {}
alternating_value = {}


-- the return value for the LuaTeX process
errorcode = 0

-- sp --mode foo sets modes.foo = true
modes = {}

-- page numbers go from 1 to n. If reordering is necessary, we insert
-- a different index into the pagenum_tbl.
-- A value of {1,2,6,7,3,4,5} means place page 1 on position one, page 2 on
-- position two, page 6 on position three and so on
pagenum_tbl = {}
forward_pagestore = {}
total_inserted_pages = 0

-- pagelabel contains information about a page with the following structure (see shipout() and get_page_labels_str() )
-- pagelabels[pagenumber] = { pagenumber = pagenumber, matter = cp.matter }
pagelabels = {}

-- An array of strings - a mapping of real page numbers and user visible pagenumbers
visible_pagenumbers = {}


matters = { mainmatter = { label = "decimal", resetafter = false, resetbefore = true, prefix = "" },
            frontmatter = { label = "lowercase-romannumeral"}
}

default_areaname = "_page"
default_area     = "_page"

-- The name of the next requested page
nextpage = nil

-- The document language
defaultlanguage = 0

-- Start page
current_pagenumber = 1

pages = {}

-- page n shipped out to PDF?
pages_shippedout = {}


-- CSS properties. Use `:matches(tbl)` to find a matching rule. `tbl` has the following structure: `{element=..., id=..., class=... }`
css = do_luafile("css.lua"):new()

-- The defaults (set in the layout instructions file)
options = {
    resetmarks  = false,
    imagenotfounderror = true,
    gridwidth   = tenmm_sp,
    gridheight  = tenmm_sp,
    gridcells_x = 0,
    gridcells_y = 0,
    reportmissingglyphs = true,
    gridlocation = "background",
    fontloader = os.getenv("SP_FONTLOADER") or "harfbuzz",
    xmlparser = os.getenv("SP_XMLPARSER") or "lua",
    hyperlinkborderwidth = tex.sp("1pt"),
    namespaces = "lax",
    tablerulefix = false,
    markdownextensions = {},
}

current_layout_line = ""
current_layout_file = ""
current_data_line = ""

if newxpath then
    options.xmlparser = "go"
else
    options.xmlparser = "lua"
end

-- List of virtual areas. Key is the group name and value is
-- a hash with keys contents (a nodelist) and grid (grid).
groups    = {}

-- sometimes we want to save pages for later reuse. Keys are pagestore names
pagestore = {}

-- See commands.compatibility
compatibility = {
    movecursoronrightedge = true,
}

-- for external image conversion software
imagehandler = {}
resizehandler = {}

viewerpreferences = {}

-- Hyperlinks are stored in publisher.links (links_module) to be inserted later
-- in the pre shipout filter.
links_module.reset()

-- marker counter. Each mark will get its unique counter, so we can determine the
-- order in which markers appear.
markercount = 0
marker_min = {}
marker_max = {}
marker_id_value = {}

-- metapost graphics. Keys are name and values are "beginfig(1)...." texts.
metapostgraphics = {}
metapostcolors = {}
metapostvariables = {}
metapostcolorwarnings = {}

-- The current foreground color (used in underline)
current_fgcolor = nil

-- The color stack to use
defaultcolorstack = 0

data_dispatcher = {}
data_dispatcher_patterns = {}
user_defined_functions = { last = 0}

--- Compile a match pattern string into a test function and priority.
--- Simple element names (no special chars) return nil to signal the fast path.
--- Supported patterns:
---   "foo"           -> fast path (nil)
---   "*"             -> matches any element, priority -0.5
---   "foo[pred]"     -> self::foo[pred], priority 0.5
---   "*[pred]"       -> self::*[pred], priority 0.5
---   "parent/child"  -> self::child[parent::parent], priority 0.5
---   "anc//desc"     -> self::desc[ancestor::anc], priority 0.5
---@param pattern string
---@return function|nil matchfunc  nil means use fast path
---@return number priority
---@return string|nil elementname  for fast path only
function compile_match_pattern(pattern)
    -- Simple element name: no /, [, ], *
    if not string.find(pattern, "[/%[%]%*]") then
        return nil, 0, pattern
    end
    -- Wildcard: just "*"
    if pattern == "*" then
        return function(ctx, node)
            return xpath.is_element(node)
        end, -0.5, nil
    end
    -- Convert pattern to self-test XPath expression
    local selftest = convert_pattern_to_selftest(pattern)
    return function(ctx, node)
        local testctx = xpath.context:new({
            xmldoc = ctx.xmldoc,
            sequence = {node},
            namespaces = ctx.namespaces or {},
            vars = ctx.vars or {},
        })
        local seq, err = testctx:eval(selftest)
        if err then
            main.log("error","match pattern evaluation failed","pattern",pattern,"error",err)
            return false
        end
        return seq and #seq > 0
    end, 0.5, nil
end

--- Convert a match pattern to a self:: XPath expression.
--- "foo[pred]"    -> "self::foo[pred]"
--- "*[pred]"      -> "self::*[pred]"
--- "parent/child" -> "self::child[parent::parent]"
--- "anc//desc"    -> "self::desc[ancestor::anc]"
---@param pattern string
---@return string
function convert_pattern_to_selftest(pattern)
    -- Check for "//" (ancestor pattern)
    local ancestor, desc = string.match(pattern, "^([^/]+)//(.+)$")
    if ancestor and desc then
        local descname, descpred = string.match(desc, "^([^%[]+)(.*)")
        local ancname = string.match(ancestor, "^([^%[]+)")
        return "self::" .. descname .. "[ancestor::" .. ancname .. "]" .. descpred
    end
    -- Check for "/" (parent/child pattern)
    local parent, child = string.match(pattern, "^([^/]+)/([^/]+)$")
    if parent and child then
        local childname, childpred = string.match(child, "^([^%[]+)(.*)")
        local parentname = string.match(parent, "^([^%[]+)")
        return "self::" .. childname .. "[parent::" .. parentname .. "]" .. childpred
    end
    -- Name with predicate: "foo[...]" or "*[...]"
    local name, pred = string.match(pattern, "^([^%[]+)(.*)")
    if name then
        return "self::" .. name .. pred
    end
    return "self::" .. pattern
end

--- Find the best matching pattern-based record for a given mode and data node.
---@param mode string
---@param datanode table  the XML data element
---@param ctx table  the lxpath context (for xmldoc, namespaces, vars)
---@return table|nil layoutxml
function find_matching_pattern(mode, datanode, ctx)
    local patterns = data_dispatcher_patterns[mode]
    if not patterns then return nil end
    local best_match = nil
    local best_priority = -math.huge
    for _, entry in ipairs(patterns) do
        if entry.priority > best_priority then
            if entry.matchfunc(ctx, datanode) then
                best_match = entry.layoutxml
                best_priority = entry.priority
            end
        end
    end
    return best_match
end
markers = {}

-- PDF/UA - the /S /Document StructElem
local ktree = pdf.reserveobj()

-- This is a sample data structure in the structElements table:
-- ["doc"] = {
    -- ["added_tables"] = {
        -- ["P_2"] = "true"
    -- },
    -- ["obj"] = "2"
    -- ["role"] = "Document"
    -- [1] = {
        -- ["bbox"] = {
            -- [1] = "28.346"
            -- [2] = "733.369"
            -- [3] = "141.732"
            -- [4] = "813.54"
        -- },
        -- ["obj"] = "7"
        -- ["page"] = "6"
        -- ["role"] = "Figure"
        -- ["text"] = "A figure"
        -- [1] = "0"
    -- },
    -- [2] = {
        -- ["obj"] = "8"
        -- ["page"] = "6"
        -- ["role"] = "P"
        -- [1] = "1"
    -- },
structElements = {}

-- We will have to remember the current group and grid
current_group = nil
current_grid = nil

-- paragraph, table and textblock should set them
current_fontfamily = 0

fontaliases = {}

-- for HTML / CSS fontfamilies
fontgroup = {
     ["sans-serif"] = { regular={["local"] = "sans"}, bold={["local"]="sans-bold"}, italic={["local"]="sans-italic"}, bolditalic={["local"]="sans-bolditalic"} },
     ["serif"] = { regular={["local"] = "serif"}, bold={["local"]="serif-bold"}, italic={["local"]="serif-italic"}, bolditalic={["local"]="serif-bolditalic"} },
     ["monospace"] = { regular={["local"] = "monospace"}, bold={["local"]="monospace-bold"}, italic={["local"]="monospace-italic"}, bolditalic={["local"]="monospace-bolditalic"} },
}

-- Used when bookmarks are inserted in a non-text context
intextblockcontext = 0

-- The array 'masterpages' has tables similar to these:
-- { is_pagetype = test, res = tab, name = name_of_page_type }
-- where `is_pagetype` is an xpath expression to be evaluated,
-- `res` is a table with layoutxml instructions
-- `name` is a string.
masterpages = {}


-- if true, look for lowercase files
lowercase = false

--- Text formats is a hash with arbitrary names as keys and the values
--- are tables with alignment and indent. indent is the amount of
--- indentation in sp. alignment is one of "leftaligned", "rightaligned",
--- "centered", "justified", "start" and "end".
textformats = {

    text           = { indent = 0, alignment="justified",   rows = 1, orphan = 2, widow = 2, name = "text"},
    __centered     = { indent = 0, alignment="centered",    rows = 1, orphan = 2, widow = 2, name = "__centered"},
    __leftaligned  = { indent = 0, alignment="leftaligned", rows = 1, orphan = 2, widow = 2, name = "__leftaligned"},
    __rightaligned = { indent = 0, alignment="rightaligned",rows = 1, orphan = 2, widow = 2, name = "__rightaligned"},
    __justified    = { indent = 0, alignment="justified",   rows = 1, orphan = 2, widow = 2, name = "__justified"},
    justified      = { indent = 0, alignment="justified",   rows = 1, orphan = 2, widow = 2, name = "justified"},
    centered       = { indent = 0, alignment="centered",    rows = 1, orphan = 2, widow = 2, name = "centered"},
    left           = { indent = 0, alignment="leftaligned", rows = 1, orphan = 2, widow = 2, name = "left"},
    right          = { indent = 0, alignment="rightaligned",rows = 1, orphan = 2, widow = 2, name = "right"},
    __fivemm       = { indent = tex.sp("5mm"), alignment="justified",   rows = 1, orphan = 2, widow = 2},
}

function new_textformat(name, base,options )
    if name == "" then name = string_random(10) end
    local baseformat = textformats[base] or textformats.text
    options = options or {}
    local tf = {}
    for k,v in pairs(baseformat) do
        tf[k] = v
    end
    for k,v in pairs(options) do
        tf[k] = v
    end
    tf.name = name
    textformats[name] = tf
    return tf
end

--- The bookmarks table has the format
---
---     bookmarks = {
---       { --- first bookmark
---         name = "outline 1" destination = "..." open = true,
---          { name = "outline 1.1", destination = "..." },
---          { name = "outline 1.2", destination = "..." }
---       },
---       { -- second bookmark
---         name = "outline 2" destination = "..." open = false,
---          { name = "outline 2.1", destination = "..." },
---          { name = "outline 2.2", destination = "..." }
---
---       }
---     }
bookmarks = {}


--- We need the separator for writing files in a directory structure (image cache for now)
os_separator = "/"
if os.type == "windows" then
    os_separator = "\\"
end

-- A very large length
maxdimen = 1073741823


-- this should be 0x100000 (= 1048576), but this is easier to work with in the
-- layout. For example you want to insert a glyph id 467, then you can write
-- &#1100467; in the layout xml. Let me not make this public until I proof that
-- it works.
puastart = 1100000

-- It's convenient to just copy the stretching glue instead of writing
-- the stretch etc. over and over again.
glue_stretch2 = set_glue(nil, { stretch = 2^16, stretch_order = 2 })

-- For attached files. Each of this numbers should appear in the catalog
filespecnumbers = {}

local function lineinfo()
    if newxpath then
        return "line_layout", current_layout_line, "file", current_layout_file, "line_data" , current_data_line
    else
        return nil
    end
end

--- The dispatch table maps every element in the layout xml to a command in the `commands.lua` file.
local dispatch_table = {
    A                       = commands.a,
    Action                  = commands.action,
    AddToList               = commands.add_to_list,
    AddSearchpath           = commands.add_searchpath,
    AtPageCreation          = commands.atpagecreation,
    AtPageShipout           = commands.atpageshipout,
    Attribute               = commands.attribute,
    AttachFile              = commands.attachfile,
    B                       = commands.bold,
    Barcode                 = commands.barcode,
    Bookmark                = commands.bookmark,
    Box                     = commands.box,
    Br                      = commands.br,
    Circle                  = commands.circle,
    ClearPage               = commands.clearpage,
    Clip                    = commands.clip,
    Color                   = commands.color,
    Column                  = commands.column,
    Columns                 = commands.columns,
    Compatibility           = commands.compatibility,
    ["Copy-of"]             = commands.copy_of,
    DefineColor             = commands.define_color,
    DefineColorprofile      = commands.define_colorprofile,
    DefineFontfamily        = commands.define_fontfamily,
    DefineFontalias         = commands.define_fontalias,
    DefineGraphic           = commands.define_graphic,
    DefineTextformat        = commands.define_textformat,
    DefineMatter            = commands.definematter,
    Element                 = commands.element,
    EmptyLine               = commands.emptyline,
    Fontface                = commands.fontface,
    ForAll                  = commands.forall,
    Frame                   = commands.frame,
    Function                = commands.func,
    Grid                    = commands.grid,
    Group                   = commands.group,
    Groupcontents           = commands.groupcontents,
    HTML                    = commands.html,
    HSpace                  = commands.hspace,
    Hyphenation             = commands.hyphenation,
    I                       = commands.italic,
    Image                   = commands.image,
    Include                 = commands.include,
    Layout                  = commands.include,
    Initial                 = commands.initial,
    InsertPages             = commands.insert_pages,
    Li                      = commands.li,
    LoadDataset             = commands.load_dataset,
    LoadFontfile            = commands.load_fontfile,
    Loop                    = commands.loop,
    Makeindex               = commands.makeindex,
    Margin                  = commands.margin,
    Mark                    = commands.mark,
    Message                 = commands.message,
    NewPage                 = commands.new_page,
    NextFrame               = commands.next_frame,
    NextRow                 = commands.next_row,
    NoBreak                 = commands.nobreak,
    Ol                      = commands.ol,
    Options                 = commands.options,
    Output                  = commands.output,
    Overlay                 = commands.overlay,
    Pageformat              = commands.page_format,
    Pagetype                = commands.pagetype,
    Paragraph               = commands.paragraph,
    Param                   = commands.param,
    PDFOptions              = commands.pdfoptions,
    PlaceObject             = commands.place_object,
    Position                = commands.position,
    PositioningArea         = commands.positioning_area,
    PositioningFrame        = commands.positioning_frame,
    ProcessNode             = commands.process_node,
    ProcessRecord           = commands.process_node,
    Record                  = commands.record,
    Rule                    = commands.rule,
    SaveDataset             = commands.save_dataset,
    SavePages               = commands.save_pages,
    Section                 = commands.section,
    Sequence                = commands.sequence,
    SetGrid                 = commands.set_grid,
    SetVariable             = commands.setvariable,
    SortSequence            = commands.sort_sequence,
    Span                    = commands.span,
    StructureElement        = commands.structureelement,
    Stylesheet              = commands.stylesheet,
    Sub                     = commands.sub,
    Sup                     = commands.sup,
    Switch                  = commands.switch,
    Table                   = commands.table,
    TableNewPage            = commands.talbenewpage,
    Tablefoot               = commands.tablefoot,
    Tablehead               = commands.tablehead,
    Tablerule               = commands.tablerule,
    Td                      = commands.td,
    Textblock               = commands.textblock,
    Text                    = commands.text,
    Tr                      = commands.tr,
    Trace                   = commands.trace,
    Transformation          = commands.transformation,
    U                       = commands.underline,
    Ul                      = commands.ul,
    Until                   = commands.until_do,
    URL                     = commands.url,
    Value                   = commands.value,
    VSpace                  = commands.vspace,
    While                   = commands.while_do,
}

--- The returned table is an array with hashes. The keys of these
--- hashes are `elementname` and `contents`. For example:
---
---     {
---       [1] = {
---         ["elementname"] = "Paragraph"
---         ["contents"] = {
---           ["nodelist"] = "<node    nil <  58515 >    nil : glyph 1>"
---         },
---       },
---     },
function dispatch(layoutxml,dataxml,opts)
    local ret = {}
    local tmp
    if not layoutxml then
        assert(false,"No elements for dispatch, why?")
        return
    end
    for _,j in ipairs(layoutxml) do
        -- j a table, if it is an element in layoutxml
        if type(j)=="table" then
            local eltname = j[".__local_name"]
            if dispatch_table[eltname] ~= nil then
                if options.verbosity > 0 then
                    if newxpath then
                        main.log("debug","Call command","name",eltname,"line",j[".__line"])
                    else
                        main.log("debug","Call command","name",eltname)
                    end
                end
                if newxpath then
                    current_layout_line = j[".__line"]
                    current_layout_file = j[".__file"]
                    if dataxml.sequence and type(dataxml.sequence)  == "table" and dataxml.sequence[1] and type(dataxml.sequence[1]) == "table" and dataxml.sequence[1][".__line"] then
                        current_data_line = dataxml.sequence[1][".__line"]
                    else
                        current_data_line = "(unknown)"
                    end
                end

                tmp = dispatch_table[eltname](j,dataxml,opts)
                -- raw tables come from Function
                if type(tmp) == "table" and tmp.raw == true then
                    for i = 1, #tmp do
                        ret[#ret+1] = tmp[i]
                    end
                end
                -- Copy-of-elements can be resolved immediately
                if eltname == "Copy-of" or eltname == "Switch" or eltname == "ForAll" or eltname == "Loop" or eltname == "Transformation" or eltname == "Frame" or eltname == "Include" or eltname == "Layout" or eltname == "Clip" or eltname == "Section" then
                    if type(tmp)=="table" then
                        for i=1,#tmp do
                            if tmp[i].contents then
                                ret[#ret + 1] = { elementname = tmp[i].elementname, contents = tmp[i].contents }
                            else
                                ret[#ret + 1] = { elementname = "elementstructure" , contents = { tmp[i] } }
                            end
                        end
                    end
                else
                    ret[#ret + 1] = { elementname = eltname, contents = tmp}
                end
            else
                local prefix, localname = table.unpack( string.explode(j[".__name"],":"))
                if localname == nil then
                    -- no prefix given, string.explode is wrong about the components
                    -- Therefore we need to swap the both
                    prefix, localname = "", prefix
                end
                if j[".__ns"][prefix] == "urn:speedata.de:2009/publisher/en" then
                    err("Unknown element found in layoutfile: %q", j[".__local_name"] or "???")
                end
            end
        end
    end
    return ret
end

local function pdf_draw_pos(x,y)
    x = sp_to_bp(x)
    y = sp_to_bp(y)
    local wd = 0.1
    return string.format("q 0 g 0.2 w %g %g m %g %g l %g %g l %g %g l h f Q ",x - wd,y - wd,x - wd,y + wd,x + wd,y + wd,x + wd,y - wd)
end
local function pdf_circle_pos(x,y)
    return circle_pdfstring(x,y,10000,10000,"0G 0g","0G 0g",0)
end
local function pdf_circle_pos_big(x,y)
    return circle_pdfstring(x,y,100000,100000,"0G 0g","0G 0g",0)
end
local function pdf_curveto(x1,y1,x2,y2,x3,y3)
    x1 = sp_to_bp(x1)
    y1 = sp_to_bp(y1)
    x2 = sp_to_bp(x2)
    y2 = sp_to_bp(y2)
    x3 = sp_to_bp(x3)
    y3 = sp_to_bp(y3)
    return string.format("%g %g %g %g %g %g c",x1,y1,x2,y2,x3,y3)
end
local function pdf_moveto( x,y )
    x = sp_to_bp(x)
    y = sp_to_bp(y)
    return string.format("%g %g m",x,y)
end
local function pdf_lineto( x,y )
    x = sp_to_bp(x)
    y = sp_to_bp(y)
    return string.format("%g %g l",x,y)
end


local function pdf_draw_pos_bp(x,y)
    local wd = 0.1
    return string.format("q 0 g 0.2 w %g %g m %g %g l %g %g l %g %g l h f Q ",x - wd,y - wd,x - wd,y + wd,x + wd,y + wd,x + wd,y - wd)
end

local function pdf_circle_pos_bp(x,y)
    return circle_pdfstring_bp(x,y,0.5,0.5,"0G 0g","0G 0g",0)
end
local function pdf_circle_pos_big_bp(x,y)
    return circle_pdfstring_bp(x,y,3,3,"0G 0g","0G 0g",0)
end
local function pdf_curveto_bp(x1,y1,x2,y2,x3,y3)
    return string.format("%g %g %g %g %g %g c",x1,y1,x2,y2,x3,y3)
end
local function pdf_moveto_bp( x,y )
    return string.format("%g %g m",x,y)
end

local function pdf_lineto_bp( x,y )
    return string.format("%g %g l",x,y)
end

function transparentcolorstack()
    if defaultcolorstack == 0 then
        defaultcolorstack = pdf.newcolorstack("0 g 0 G/TRP1 gs","direct",true)
    end
end

--- Bookmarks are collected and later processed. This function (recursively)
--- creates TeX code from the generated tables.
function bookmarkstotex( tbl )
    local countstring
    local open_string
    if #tbl == 0 then
        countstring = ""
    else
        if tbl.open == "true" then
            open_string = ""
        else
            open_string = "-"
        end
        countstring = string.format("count %s%d",open_string,#tbl)
    end
    if tbl.destination then
        tex.sprint(string.format("\\pdfoutline goto num %s %s {%s}",tbl.destination, countstring ,utf8_to_utf16_string_pdf(tbl.name) ))
    end
    for i,v in ipairs(tbl) do
        bookmarkstotex(v)
    end
end

function page_initialized_p( pagenumber )
    return pages[pagenumber] ~= nil
end

-- action type is one of
-- 0 page
-- 1 goto
-- 2 thread
-- 3 user
-- return an action node or a whatsit with pdf_action subtype.
function get_action_node( action_type )
    local ai = node.new("whatsit",pdf_action_whatsit)
    ai.action_type = action_type
    return ai
end

roles_a = {
    "Art",
    "Artifact",
    "Div",
    "Document",
    "Figure",
    "H1",
    "H2",
    "H3",
    "H4",
    "H5",
    "H6",
    "Lbl",
    "Link",
    "P",
    "Part",
    "Sect",
    "Span",
    "Table",
    "TD",
    "TH",
    "TOC",
    "TOCI",
    "TR",
}
local roles = {}
for k,v in pairs(roles_a) do
    roles[v] = k
end

-- unique id for roles
rolecounter = 0

function get_rolenum( rolestring )
    if not rolestring then return nil end
    local ret = roles[rolestring]
    if ret then
        return ret
    end
    err("Unknown role %q",tostring(rolestring))
end

-- Get the page position of a structure element child for sorting.
-- Returns a page number from page_map for table children (struct elements),
-- or math.huge for number children (MCIDs, keep relative order).
local function get_struct_page_position(child, page_map)
    if type(child) == "number" then
        return math.huge
    end
    if child.page then
        return page_map[child.page] or math.huge
    end
    -- No page field: recurse into first child that has a page
    for i = 1, #child do
        local pos = get_struct_page_position(child[i], page_map)
        if pos ~= math.huge then
            return pos
        end
    end
    return math.huge
end

-- Sort the children of each structure element by page order (reading order).
-- Only reorders when pages were rearranged (InsertPages/SavePages).
local function sort_struct_tree_by_page_order(elem, page_ref_to_pagenumber)
    if type(elem) ~= "table" then return end
    -- First recurse into sub-elements
    for i = 1, #elem do
        if type(elem[i]) == "table" then
            sort_struct_tree_by_page_order(elem[i], page_ref_to_pagenumber)
        end
    end
    -- Now sort children of this element
    if #elem < 2 then return end
    -- Tag each child with its original index for stable sort
    local tagged = {}
    for i = 1, #elem do
        tagged[i] = { child = elem[i], orig = i, page_pos = get_struct_page_position(elem[i], page_ref_to_pagenumber) }
    end
    stable_sort(tagged, function(a, b)
        if a.page_pos ~= b.page_pos then
            return a.page_pos < b.page_pos
        end
        return a.orig < b.orig
    end)
    for i = 1, #elem do
        elem[i] = tagged[i].child
    end
end

local function struct_xml_escape(s)
    s = string.gsub(s, "&", "&amp;")
    s = string.gsub(s, "<", "&lt;")
    s = string.gsub(s, ">", "&gt;")
    s = string.gsub(s, '"', "&quot;")
    return s
end

local function dump_struct_tree_xml(elem, indent, page_ref_to_num)
    indent = indent or ""
    local lines = {}
    local role = elem.role or "Unknown"
    local attrs = ""
    if elem.page then
        local pagenum = page_ref_to_num and page_ref_to_num[tonumber(elem.page)] or elem.page
        attrs = attrs .. string.format(' page="%s"', tostring(pagenum))
    end
    if elem.actualtext then
        attrs = attrs .. string.format(' actualtext="%s"', struct_xml_escape(elem.actualtext))
    end
    if elem.alttext then
        attrs = attrs .. string.format(' alttext="%s"', struct_xml_escape(elem.alttext))
    end

    -- Collect MCIDs (integer children) and sub-elements (table children)
    local mcids = {}
    local children = {}
    for i = 1, #elem do
        local child = elem[i]
        if type(child) == "table" then
            children[#children + 1] = child
        else
            mcids[#mcids + 1] = tostring(child)
        end
    end

    if #children == 0 and #mcids == 0 then
        lines[#lines + 1] = indent .. "<" .. role .. attrs .. "/>"
    else
        lines[#lines + 1] = indent .. "<" .. role .. attrs .. ">"
        for _, mcid in ipairs(mcids) do
            lines[#lines + 1] = indent .. "  <MCID>" .. mcid .. "</MCID>"
        end
        for _, child in ipairs(children) do
            lines[#lines + 1] = dump_struct_tree_xml(child, indent .. "  ", page_ref_to_num)
        end
        lines[#lines + 1] = indent .. "</" .. role .. ">"
    end
    return table.concat(lines, "\n")
end

local function writeStructElements(itm,parentobjectnumber)
    local obj = itm.obj
    local objectnumbers = {}
    for i = 1, #itm do
        local thisitm = itm[i]
        if type(thisitm) == "table" then
            local onum = writeStructElements(thisitm,obj)
            objectnumbers[#objectnumbers+1] = onum
        else
            objectnumbers[#objectnumbers+1] = string.format("%d",thisitm)
        end
    end
    if itm.linkobjects then
        objectnumbers[#objectnumbers+1] = string.format("<</Type/OBJR /Obj %d 0 R /Pg %d 0 R >>",itm.linkobjects[1],itm.page)
    end

    local k
    if #objectnumbers > 1 then
        k = string.format("[ %s ]",table.concat(objectnumbers," "))
    else
        k = objectnumbers[1]
    end

    if k then
        if not itm.role then
            main.log("error","Unknown role in structure")
        end
        str = {
            "/Type /StructElem",
            "/S /" .. (itm.role or "???"),
            "/P " .. parentobjectnumber .. " 0 R",
            "/K "  .. k,
        }
        if itm.actualtext then
            str[#str + 1] = "/ActualText " .. utf8_to_utf16_string_pdf(itm.actualtext)
        end

        if itm.alttext then
            str[#str + 1] = "/Alt " .. utf8_to_utf16_string_pdf(itm.alttext)
        end
        if itm.bbox then
            local bbox = itm.bbox
            str[#str+1] = string.format("/A << /BBox [%s %s %s %s] /Placement /Block /O /Layout >>",bbox[1],bbox[2],bbox[3],bbox[4])
        end
        if itm.page then
            str[#str+1] = "/Pg " .. itm.page .. " 0 R"
        end
        pdf.obj({type = "raw", objnum = obj, immediate = true , string = "<<" ..  table.concat(str, " ") .. ">>" })
    end

    return string.format("%d 0 R",obj)

end

--- Start the processing (`dothings()`)
--- -------------------------------
--- This is the entry point of the processing. It is called from publisher.spinit#main_loop.
function dothings()
    log("Running LuaTeX version %s on %s",luatex_version,os.name)
    --- First we set some defaults.
    --- A4 paper is 210x297 mm
    local wd_sp = tex.sp("210mm")
    local ht_sp = tex.sp("297mm")
    set_pageformat(wd_sp,ht_sp)
    options.default_pagewidth = wd_sp
    options.default_pageheight = ht_sp

    get_languagecode(os.getenv("SP_MAINLANGUAGE") or "en_GB")

    lowercase = os.getenv("SP_IGNORECASE") == "1"
    local extra_parameter = { otfeatures = { kern = true, liga = false } }
    --- The free font family `TeXGyreHeros` is a Helvetica clone and is part of the
    --- [The TeX Gyre Collection of Fonts](http://www.gust.org.pl/projects/e-foundry/tex-gyre).
    --- We ship it in the distribution.
    fonts.load_fontfile("TeXGyreHeros-Regular",   "texgyreheros-regular.otf",extra_parameter)
    fonts.load_fontfile("TeXGyreHeros-Bold",      "texgyreheros-bold.otf",extra_parameter)
    fonts.load_fontfile("TeXGyreHeros-Italic",    "texgyreheros-italic.otf",extra_parameter)
    fonts.load_fontfile("TeXGyreHeros-BoldItalic","texgyreheros-bolditalic.otf",extra_parameter)

    -- These are used in HTML mode when the user switches to monospace or serif
    fonts.load_fontfile("CrimsonPro-Regular","CrimsonPro-Regular.ttf",extra_parameter)
    fonts.load_fontfile("CrimsonPro-Bold","CrimsonPro-Bold.ttf",extra_parameter)
    fonts.load_fontfile("CrimsonPro-Italic","CrimsonPro-Italic.ttf",extra_parameter)
    fonts.load_fontfile("CrimsonPro-BoldItalic","CrimsonPro-BoldItalic.ttf",extra_parameter)

    fonts.load_fontfile("CamingoCode-Regular","CamingoCode-Regular.ttf",extra_parameter)
    fonts.load_fontfile("CamingoCode-Bold","CamingoCode-Bold.ttf",extra_parameter)
    fonts.load_fontfile("CamingoCode-Italic","CamingoCode-Italic.ttf",extra_parameter)
    fonts.load_fontfile("CamingoCode-BoldItalic","CamingoCode-BoldItalic.ttf",extra_parameter)

    --- Define a basic font family with name `text`:
    define_default_fontfamily()

    local _sampler
    if os.getenv("SP_PROFILE") then
        _sampler = require "sampler"
        _sampler.start(tonumber(os.getenv("SP_PROFILE")) or 10000)
    end
    initialize_luatex_and_generate_pdf()
    if _sampler then
        _sampler.stop()
        _sampler.report("profile.txt")
    end
    -- The last thing is to put a stamp in the PDF
    if options.hidespinfo and options.hidespinfo == "true" or options.hidespinfo == "yes" then
        -- do nothing
        if not pro then
            err("Removing speedata info needs a pro plan")
            publisher.has_pro_error = true
            return nil
        end
    else
        pdf.obj({type="raw",string="(Created with the speedata Publisher - www.speedata.de)", immediate = true, objcompression = false})
    end
end

function get_extension(fn)
    return fn:match("^.+%.(.+)$")
end

function define_image_callback( extensionhandler )
    local extensions = {}
    local ext,handler
    if extensionhandler and extensionhandler ~= "" then
        for _,v in ipairs(string.explode(extensionhandler,";")) do
            _,_,ext,handler = string.find(v,"^(.*):(.*)$")
            extensions[ext] = handler
        end
    end

    local function find_image_file( asked_name )
        local file = kpse.find_file(asked_name)
        local ext = get_extension(asked_name)
        local handlername = extensions[ext]
        local handler = imagehandler[handlername or "*"]
        if handler then
            main.log("info","Convert image", "extension",ext, "handler",handlername or "*")
            file = splib.convertimage(file,handler)
        end
        return file
    end
    callback.register('find_image_file',find_image_file)
end

borderattributes = {}
do
    -- the idea of flatten_boxes is to return an array that only has
    -- par objects.
    -- The input of flatten_boxes is a mix of Box objects and Par objects.
    -- You can consider Box objects something similar to <div> blocks in HTML
    -- and Par objects like <p> that has acutal content in it (also: images and other stuff)
    -- Margin settings should go from <div> to the <p> (from Box to Par) so we can
    -- leave out the div stuff.
    local prependbox
    function flatten_boxes(box,parameter,ret)
        ret = ret or {}
        parameter = parameter or {}
        local indent = box.indent_amount or 0
        if indent and parameter.indent then
            indent = parameter.indent + indent
        end
        local new_parameter = {
            indent = indent
        }
        if box.prependbox and #box.prependbox > 0 then
            prependbox = prependbox or {}
            for i=1,#box.prependbox do
                table.insert(prependbox,box.prependbox[i])
            end
        end

        if box.padding_bottom and box.padding_bottom ~= 0 then
            box[1].padding_bottom = box.padding_bottom
        end
        if box.padding_top and box.padding_top ~= 0 then
            box[1].padding_top = box.padding_top
        end
        if box.draw_border then
            borderattributes[#borderattributes + 1] = box.border
            if #box > 1 then
                box[1].startborder = #borderattributes
            else
                box[1].startendborder = #borderattributes
            end
        end
        if box.startendborder then
            box[1].startendborder = box.startendborder
        end

        for i=1,#box do
            local thisbox = box[i]
            if not thisbox.min_width then
                -- a box with paragraphs inside
                -- Pass break_before to first child, break_after to last child
                if i == 1 and box.break_before then
                    thisbox.break_before = box.break_before
                end
                if i == #box and box.break_after then
                    thisbox.break_after = box.break_after
                end
                flatten_boxes(thisbox,new_parameter,ret)
                if thisbox.mode == "block" then ret.mode = "block" end
            else
                -- a regular paragraph
                if parameter.indent then
                    thisbox:indent(indent)
                end
                if box.width then
                    thisbox.width = box.width
                end
                if box.draw_border then
                    thisbox.draw_border = true
                    thisbox.border = box.border
                end
                if box.startborder then
                    thisbox.startborder = box.startborder
                end
                if prependbox then
                    for p=#prependbox,1,-1 do
                        thisbox:prepend(prependbox[p])
                    end
                    prependbox = nil
                end
                if i == 1 then
                    thisbox.firstbox = true
                    if box.border_top_width then
                        thisbox.border_top_width = box.border_top_width
                    end
                    if box.margintop then
                        thisbox.margin_top = box.margintop
                    end
                    if box.break_before then
                        thisbox.break_before = box.break_before
                    end
                end
                if i == #box then
                    thisbox.lastbox = true
                    if box.marginbottom then
                        thisbox.margin_bottom = box.marginbottom
                    end
                    if box.border_bottom_width then
                        thisbox.border_bottom_width = box.border_bottom_width
                    end
                    if box.break_after then
                        thisbox.break_after = box.break_after
                    end
                end

                ret[#ret + 1] = thisbox
            end
        end
        return ret
    end
end

-- When not in server mode, we initialize LuaTeX in such a way that
-- it has defaults, loads a layout file and a data file and
-- executes them both
function initialize_luatex_and_generate_pdf()
    if os.getenv("SP_VERBOSITY") == nil then
        options.verbosity = 0
    else
        options.verbosity = tonumber(os.getenv("SP_VERBOSITY"))
    end

    options.mpcolorwarning = true
    --- The default page type has 1cm margin
    masterpages[1] = { is_pagetype = "true()", res = { {elementname = "Margin", contents = function(_page) _page.grid:set_margin(tenmm_sp,tenmm_sp,tenmm_sp,tenmm_sp) end }}, name = "Default Page",ns={[""] = "urn:speedata.de:2009/publisher/en" } }

    --- The `vars` file hold a lua document holding table
    local vars
    local varsfun = loadfile(tex.jobname .. ".vars")
    if varsfun then vars = varsfun() else err("Could not load .vars file. Something strange is happening.") vars = {} end

    for i=4,#arg do
        local k,v = arg[i]:match("^(.+)=(.+)$")
        if k == "mode" then -- everything else handled after loading layout
            v = v:gsub("^\"(.*)\"$","%1")
            local _modes = string.explode(v,",")
            for _,m in ipairs(_modes) do
                modes[m] = true
            end
        elseif k == "pro" then
            pro = true
            main.log("info","speedata Publisher Pro")
        end
    end

    --- Both the data and the layout instructions are written in XML.
    local layoutxml = load_xml(arg[2],"layout instructions")
    if not layoutxml then
        err("Without a valid layout-XML file, I can't really do anything.")
        exit()
    end
    if newxpath then
        layoutxml = layoutxml[1] -- skip document
    end
    --- Used in `xpath.lua` to find out which language the function is in.
    local ns = layoutxml[".__namespace"]
    if not ns then
        err("Cannot find the namespace of the layout file. What should I do?")
        exit()
    end

    --- The currently active layout language. One of `de` or `en`.
    local current_layoutlanguage = string.gsub(ns,"urn:speedata.de:2009/publisher/","")
    if not (current_layoutlanguage=='de' or current_layoutlanguage=='en') then
        err("Cannot determine the language of the layout file.")
        exit()
    end
    if current_layoutlanguage == "de" then
        err("The German layout instructions have been removed\nin version 2.7 of the publisher.")
        exit()
    end
    local version
    local requirements
    if newxpath then
        local attr = layoutxml[".__attributes"]
        if attr and attr["version"] then
            version = attr["version"]
        end
        if attr and attr["require"] then
            requirements = attr["require"]
        end
    else
        version = layoutxml.version
        requirements = layoutxml.require
    end
    if version then
        local version_mismatch = false
        local publisher_version = string.explode(env_publisherversion,".")
        local requested_version = string.explode(version,".")

        if publisher_version[1] ~= requested_version[1] then
            if tonumber(publisher_version[1]) < tonumber(requested_version[1]) then
                version_mismatch = true
            end
        elseif tonumber(publisher_version[2]) < tonumber(requested_version[2]) then
            -- major number are same, minor are different
            version_mismatch = true
        elseif tonumber(requested_version[3]) and tonumber(publisher_version[3]) < tonumber(requested_version[3]) and tonumber(publisher_version[2]) == tonumber(requested_version[2]) then
            version_mismatch = true
        end
        if version_mismatch then
            err("Version mismatch. speedata Publisher is at version %s, requested version %s", env_publisherversion, version)
            exit()
        end
    end
    if requirements and type(requirements) == "string" then
        local r = string.explode(requirements,",")
        for _, req in ipairs(r) do
            if req == "lxpath" then
                if not newxpath then
                    main.log("error","failed to meet requirement", "requirement","lxpath","message","This layout requires the lxpath XML / XPath parser","help","see https://doc.speedata.de/publisher/en/lxpath/ how to activate")
                    exit(false)
                end
            elseif req == "luxor" then
                    if newxpath then
                        main.log("error","failed to meet requirement", "requirement","luxor","message","This layout requires the luxor XML / XPath parser","help","see https://doc.speedata.de/publisher/en/xpathfunctions/ how to activate")
                        exit(false)
                    end
            elseif req == "harfbuzz" then
                if options.fontloader ~= "harfbuzz" then
                    main.log("error","failed to meet requirement", "requirement","harfbuzz","message","This layout requires the harfbuzz font loader","help","see https://doc.speedata.de/publisher/en/configuration/ how to activate")
                    exit(false)
                end
            elseif req == "fontforge" then
                if options.fontloader ~= "fontforge" then
                    main.log("error","failed to meet requirement", "requirement","fontforge","message","This layout requires the fontforge font loader","help","see https://doc.speedata.de/publisher/en/configuration/ how to activate")
                    exit(false)
                end
            else
                err("This layout requires feature %q, but I don't know what it is.\nPerhaps I am too old?",req)
                exit(false)
            end
        end
    end
    if newxpath then
        tmp = os.getenv("SP_PREPEND_XML")
        if tmp and tmp ~= "" then
            main.log("error","--prepend-xml is not supported with the new XPath mode. Use xinclude instead.")
        end
        tmp = os.getenv("SP_EXTRA_XML")
        if tmp and tmp ~= "" then
            main.log("error","--extra-xml is not supported with the new XPath mode. Use xinclude instead.")
        end
    else
        tmp = os.getenv("SP_PREPEND_XML")
        if tmp and tmp ~= "" then
            for i,v in ipairs(string.explode(tmp,",")) do
                table.insert(layoutxml, i, luxor.parse_xml_file(v))
            end
        end
        tmp = os.getenv("SP_EXTRA_XML")
        if tmp and tmp ~= "" then
            for _,v in ipairs(string.explode(tmp,",")) do
                layoutxml[#layoutxml + 1] = luxor.parse_xml_file(v)
            end
        end
    end

    -- We allow the use of a dummy xml file for testing purpose
    local dataxml
    local datafilename = arg[3]
    if datafilename == "-dummy" then
        if newxpath then
            dataxml = splib.loadxmlstring("<data />")
        else
            dataxml = luxor.parse_xml("<data />")
        end
    elseif datafilename == "-" then
        log("Reading from stdin")
        dataxml = luxor.parse_xml(io.stdin:read("*a"),{htmlentities = true})
    else
        dataxml = load_xml(datafilename,"data file",{ htmlentities = true, ignoreeol = ( options.ignoreeol or false ) })
    end
    if not dataxml then
        main.log("error","Could not read data")
        exit()
    end
    if type(dataxml) ~= "table" then
        main.log("error","Something is wrong with the data: dataxml is not a table")
        exit()
    end

    if newxpath then
        local defaults = {
            _bleed = "0mm",
            _pageheight = "297mm",
            _pagewidth = "210mm",
            _jobname =  tex.jobname,
            _matter = "mainmatter",
            __maxwidth = tex.sp("190mm"),
            _lastpage = 1,
        }
        data = xpath.context:new()
        data.xmldoc = {dataxml}
        data.sequence = {dataxml}
        data.namespaces = layoutxml[".__ns"]

        for k, v in pairs(defaults) do
            data.vars[k] = v
        end

        -- from command line or publisher.cfg:
        for k, v in pairs(vars) do
            data.vars[k] = v
        end
        local mode_keys = {}
        for k, _ in pairs(modes) do
            mode_keys[#mode_keys+1] = k
        end
        table.sort(mode_keys)
        data.vars._mode = table.concat(mode_keys,",")

        seq, msg = data:execute("root()")
        if msg then
            main.log("error",msg)
        end
    else
        for k,v in pairs(vars) do
            xpath.set_variable(k,v)
        end
    end

    dispatch(layoutxml,data)
    if newxpath then
        -- for namespace mode == strict
        data.namespaces = dataxml[1][".__ns"]
    end

    -- options.ignoreeol is now set
    if newxpath then
        fixup_xmlfile(dataxml, options.ignoreeol or false)
    end

    -- We define two graphic states for overprinting on and off.
    GS_State_OP_On  = pdf.immediateobj([[<< /Type/ExtGState /OP true /OPM 1 >>]])
    GS_State_OP_Off = pdf.immediateobj([[<< /Type/ExtGState /OP false >>]])

    --- override options set in the `<Options>` element
    for i=4,#arg do
        local k,v = arg[i]:match("^(.+)=(.+)$")
        if k ~= "mode" then -- mode handled before loading layout
            v = v:gsub("^\"(.*)\"$","%1")
            options[k]=v
        end
    end

    if options.interaction == "false" then
        options.interaction = false
    elseif options.interaction == "true" then
        options.interaction = true
    end

    if options.showgrid == "false" then
        options.showgrid = false
    elseif options.showgrid == "true" then
        options.showgrid = true
    end

    if options.cutmarks == "true" then
        options.cutmarks = true
    elseif options.cutmarks == "false" then
        options.cutmarks = false
    end

    if options.trimmarks == "true" then
        options.trimmarks = true
    elseif options.trimmarks == "false" then
        options.trimmarks = false
    end

    if options.showgridallocation == "false" then
        options.showgridallocation = false
    elseif options.showgridallocation == "true" then
        options.showgridallocation = true
    end

    if options.reportmissingglyphs == "false" or options.reportmissingglyphs == "no" then
        options.reportmissingglyphs = false
    elseif options.reportmissingglyphs == "true" or options.reportmissingglyphs == "yes" then
        options.reportmissingglyphs = true
    elseif options.reportmissingglyphs == "warning" then
        options.reportmissingglyphs = "warning"
    end

    if options.imagehandler then
        string.gsub(options.imagehandler,"([a-zA-Z*]+):%((.-)%);?", function( imagetype,cmdline )
            imagehandler[imagetype] = cmdline
        end)
    end

    if options.resizehandler then
        string.gsub(options.resizehandler,"([a-zA-Z*]+):%((.-)%);?", function( imagetype,cmdline )
            resizehandler[imagetype] = cmdline
        end)
    end

    define_image_callback(options.extensionhandler or "")

    --- Set the starting page (which must be a number)
    if options.startpage then
        local num = options.startpage
        if num then
            current_pagenumber = tonumber(num)
            log("Set page number to %d",num)
        else
            main.log("error","Can't recognize starting page number", "startpage", options.startpage or "(not set)")
        end
    end

    if options.colorprofile then
        spotcolors.set_colorprofile_filename(options.colorprofile)
        warning("Options / colorprofile is obsolete. Use DefineColorprofile and PDFOptions / colorprofile instead.")
    end

    if options.format == "PDF/UA" and not structElements[".root"] then
        structElements[".root"] = {
            role = "Document",
            obj = pdf.reserveobj(),
        }
        structElements["doc"] = structElements[".root"]
    end


    local auxfilename = tex.jobname .. "-aux.xml"
    -- load help file if it exists
    if kpse.find_file(auxfilename) and options.resetmarks == false then
        local mark_tab = load_xml(auxfilename,"aux file",{ htmlentities = true, ignoreeol = true })
        if newxpath and mark_tab then
            mark_tab = mark_tab[1]
        end
        mark_tab = mark_tab or {}
        for i=1,#mark_tab do
            local mt = mark_tab[i]
            if type(mt) == "table" then
                local attributes
                if newxpath then
                    attributes = mt[".__attributes"]
                else
                    attributes = mt
                end
                if mt[".__local_name"] == "mark" then
                    markers[attributes.name] = { page = attributes.page}
                    local id = tonumber(attributes.id)
                    if id then
                        marker_id_value[id] = { page = attributes.page, name = attributes.name}

                        local pagenumber = tonumber(attributes.page)
                        if not marker_min[pagenumber] then
                            marker_min[pagenumber] = id
                        elseif marker_min[pagenumber] > id then
                            marker_min[pagenumber] = id
                        end
                        if not marker_max[pagenumber] then
                            marker_max[pagenumber] = id
                        elseif marker_max[pagenumber] < id then
                            marker_max[pagenumber] = id
                        end
                    end
                elseif mt[".__local_name"] == "pagelabel" then
                    visible_pagenumbers[tonumber(attributes.pagenumber)] = attributes.visible
                elseif mt[".__local_name"] == "lastpage" then
                    if newxpath then
                        data.vars["_lastpage"] = attributes.page
                    else
                        xpath.set_variable("_lastpage", attributes.page )
                    end
                end
            end
        end
    end


    if newxpath then
    else
        xpath.set_variable("_bleed", "0mm")
        xpath.set_variable("_pageheight", "297mm")
        xpath.set_variable("_pagewidth", "210mm")
        xpath.set_variable("_jobname", tex.jobname)
        xpath.set_variable("_matter","mainmatter")
        xpath.set_variable("__maxwidth", tex.sp("190mm"))
        if xpath.get_variable("_lastpage") == nil then
            xpath.set_variable("_lastpage", 1)
        end
    end

    -- The xml now looks like
    -- dataxml = {
    --     [1] = "\
    --       "
    --     [2] = {
    --       [1] = "mixed"
    --       [2] = {
    --         [".__ns"] = {
    --         },
    --         [".__parent"] = <foo>
    --         [".__name"] = "br"
    --         [".__type"] = "element"
    --         [".__local_name"] = "br"
    --       },
    --       [3] = "content"
    --       [".__ns"] = {
    --       },
    --       ["attr1"] = "value1"
    --       [".__parent"] = <data>
    --       [".__name"] = "foo"
    --       [".__type"] = "element"
    --       [".__local_name"] = "foo"
    --     },
    --     [3] = "\
    --   "
    --     [".__name"] = "data"
    --     [".__ns"] = {
    --     },
    --     [".__type"] = "element"
    --     [".__local_name"] = "data"
    --   },
    --
    -- That means the table entries are either strings or child elements.
    -- Attributes are table keys and metadata is stored as ".__" plus the metadata.

    --- Start data processing in the default mode (`""`)
    local name, tmp
    if newxpath then
        local seq, msg
        seq, msg = data:execute("root()")
        if msg then
            main.log("error",msg)
        end
        if options.namespaces == "strict" then
            seq, msg = data:eval("local-name()")
            if msg then
                main.log("error",msg)
            end
            name = xpath.string_value(seq)
            seq, msg = data:eval("namespace-uri()")
            if msg then
                main.log("error",msg)
            end
            local namespace_element = xpath.string_value(seq)
            name = "{" .. namespace_element .. "}" .. name
        else
            seq, msg = data:eval("local-name()")
            if msg then
                main.log("error",msg)
            end
            name = xpath.string_value(seq)
        end
    else
        name = dataxml[".__local_name"]
        xpath.set_variable("__position", 1)
    end

    --- The rare case that the user has not any `Record` commands in the layout file:
    if not data_dispatcher[""] and not data_dispatcher_patterns[""] then
        main.log("error","Can't find any “Record” commands in the layout file.")
        exit()
    end

    tmp = data_dispatcher[""] and data_dispatcher[""][name]
    -- Pattern matching fallback for root element (newxpath only)
    if not tmp and newxpath then
        local rootnode = data.sequence and data.sequence[1]
        if rootnode then
            tmp = find_matching_pattern("", rootnode, data)
        end
    end
    if tmp then
        if newxpath then
            -- For data:eval, the namespaces must be set the layout namespaces
            data.namespaces = layoutxml[".__ns"]
            dispatch(tmp,data)
        else
            dispatch(tmp,dataxml)
        end
    else
        name = name or ""
        local elt_ns, elt_localname = string.match(name,"{(.*)}(.*)")
        if elt_ns then
            main.log("error","Can't find “Record” command for the root node","namespace",elt_ns,"elementname",elt_localname)
        else
            main.log("error","Can't find “Record” command for the root node","elementname",name)
        end
        exit()
    end


    --- emit last page if necessary
    -- current_pagestore_name is set when in SavePages and nil otherwise
    if page_initialized_p(current_pagenumber) and current_pagestore_name == nil then
        dothingsbeforeoutput(pages[current_pagenumber],data)
        local n = node.vpack(pages[current_pagenumber].pagebox)
        shipout(n,current_pagenumber,dataxml)
    end
    local lastpage = current_pagenumber
    while not(page_initialized_p(lastpage)) and lastpage > 0 and current_pagestore_name == nil do
        lastpage = lastpage - 1
    end

    --- At this point, all pages are in the PDF
    --- We are not at the end of the processing. Let's write the PDF information and status files.
    local pdfcatalog = {}
    if sp_suppressinfo then
        pdf.settrailerid(" [ <FA052949448907805BA83C1E78896398> <FA052949448907805BA83C1E78896398> ]")
    end
    -- file attachment
    if #filespecnumbers > 0 then
        local afstring = {}
        for i = 1, #filespecnumbers do
            local filespecnum = filespecnumbers[i][1]
            afstring[#afstring+1] = string.format("%d 0 R",filespecnum)
        end
        local af = "[" .. table.concat(afstring," ") .. "]"

        local names = {}
        for i = 1, #filespecnumbers do
            local filespecnum = filespecnumbers[i][1]
            local filename = filespecnumbers[i][3]
            local conformancelevel = filespecnumbers[i][2]
            names[#names+1] = string.format([[%s %d 0 R]],utf8_to_utf16_string_pdf(filename),filespecnum)
        end
        pdfcatalog[#pdfcatalog + 1] = string.format([[ /Names << /EmbeddedFiles <<  /Names [%s] >> >> ]],table.concat(names," "))
        pdfcatalog[#pdfcatalog + 1] = string.format([[ /AF %s ]],af)
    end

    local str = get_page_labels_str()
    if str then
        pdfcatalog[#pdfcatalog + 1] = str
    end
    local langtbl = get_language(defaultlanguage)

    if langtbl and langtbl.locale then
        pdfcatalog[#pdfcatalog+1] = string.format(" /Lang (%s)",string.gsub(langtbl.locale,"^(%a+).*","%1"))
    end

    local vp = {}
    if viewerpreferences.numcopies and viewerpreferences.numcopies > 1 and viewerpreferences.numcopies <= 5 then
        vp[#vp + 1] = string.format("/NumCopies %d", viewerpreferences.numcopies)
    end
    if viewerpreferences.printscaling and viewerpreferences.printscaling ~= ""  then
        vp[#vp + 1] = string.format("/PrintScaling /%s", viewerpreferences.printscaling)
    end
    if viewerpreferences.picktray ~= nil  then
        vp[#vp + 1] = string.format("/PickTrayByPDFSize %s", viewerpreferences.picktray)
    end

    if viewerpreferences.duplex ~= nil and viewerpreferences.duplex ~= "" then
        vp[#vp + 1] = string.format("/Duplex /%s", viewerpreferences.duplex)
    end

    if options.pagelayout then
        pdfcatalog[#pdfcatalog + 1] = string.format("/PageLayout /%s", options.pagelayout)
    end

    if options.displaymode then
        pdfcatalog[#pdfcatalog + 1] = string.format("/PageMode /%s", options.displaymode)
    else
        pdfcatalog[#pdfcatalog + 1] = "/PageMode /UseNone"
    end


    -- Title   The document’s title.
    -- Author  The name of the person who created the document.
    -- Subject  The subject of the document.
    -- Keywords  Keywords associated with the document.

    -- Nothing set:
    -- Creator:         speedata Publisher 4.19.2, www.speedata.de
    -- Producer:        LuaTeX 1.15.0 (build 7509)
    --
    -- Creator set:
    -- Creator:         CREATOR
    -- Producer:        speedata Publisher 4.19.2 using LuaTeX

    -- suppressinfo:
    -- Creator:         speedata Publisher
    -- Producer:        LuaTeX

    -- suppressinfo / Creator set:
    -- Creator:         CREATOR
    -- Producer:        speedata Publisher using LuaTeX
    local infos = { string.format("/Creator %s /Producer %s",utf8_to_utf16_string_pdf(metadata.getcreator(options)), utf8_to_utf16_string_pdf(metadata.getproducer(options))) }
    if not sp_suppressinfo then
        infos[#infos+1] = "/Trapped /False"
    end
    if options.documenttitle and options.documenttitle ~= "" then
        infos[#infos + 1] = string.format("/Title %s",utf8_to_utf16_string_pdf(options.documenttitle))
    end
    if options.documentauthor and options.documentauthor ~= "" then
        infos[#infos + 1] = string.format("/Author %s", utf8_to_utf16_string_pdf(options.documentauthor))
    end
    if options.documentsubject and options.documentsubject ~= "" then
        infos[#infos + 1] = string.format("/Subject %s", utf8_to_utf16_string_pdf(options.documentsubject))
    end
    if options.documentkeywords and options.documentkeywords ~= "" then
        infos[#infos + 1] = string.format("/Keywords %s", utf8_to_utf16_string_pdf(options.documentkeywords))
    end

    if options.format then
        local metadataobjnum
        if options.format == "PDF/X-3:2002" or options.format == "PDF/X-4" then
            infos[#infos + 1] = string.format("/GTS_PDFXVersion (%s)",options.format)
            metadataobjnum = pdf.obj({ type="stream", string = metadata.getmetadata(filespecnumbers,options), immediate = true, attr = [[  /Subtype /XML /Type /Metadata ]],compresslevel = 0})
            local colorprofileobjnum = spotcolors.write_colorprofile()
            local cp = spotcolors.get_colorprofile()
            local outputintentsobjnum = pdf.obj({type = "raw",  immediate = true , string = string.format([[<<  /DestOutputProfile %d 0 R /Info %s /OutputCondition %s    /OutputConditionIdentifier %s   /RegistryName %s    /S /GTS_PDFX   /Type /OutputIntent  >>]],colorprofileobjnum,
 utf8_to_utf16_string_pdf(cp.info),
 utf8_to_utf16_string_pdf(cp.condition),
 utf8_to_utf16_string_pdf(cp.identifier),
 utf8_to_utf16_string_pdf(cp.registry))})
            local outputintentsarrayobjnum = pdf.obj({type="raw", string = string.format("[ %d 0 R ]",outputintentsobjnum), immediate = true })
            pdfcatalog[#pdfcatalog + 1] = string.format("/OutputIntents %d 0 R",outputintentsarrayobjnum )
        end
        if options.format == "PDF/A-3" then
            metadataobjnum = pdf.obj({ type="stream", string = metadata.getmetadata(filespecnumbers,options), immediate = true, attr = [[  /Subtype /XML /Type /Metadata ]],compresslevel = 0})
            pdf.setomitcidset(1)
            local colorprofileobjnum = spotcolors.write_colorprofile()
            local cp = spotcolors.get_colorprofile()
            local outputintentsobjnum = pdf.obj({type = "raw",  immediate = true , string = string.format([[<<  /DestOutputProfile %d 0 R /Info %s /OutputCondition %s    /OutputConditionIdentifier %s   /RegistryName %s    /S /GTS_PDFA1   /Type /OutputIntent  >>]],colorprofileobjnum,
 utf8_to_utf16_string_pdf(cp.info),
 utf8_to_utf16_string_pdf(cp.condition),
 utf8_to_utf16_string_pdf(cp.identifier),
 utf8_to_utf16_string_pdf(cp.registry))})
            local outputintentsarrayobjnum = pdf.obj({type="raw", string = string.format("[ %d 0 R ]",outputintentsobjnum), immediate = true })
            pdfcatalog[#pdfcatalog + 1] = string.format("/OutputIntents %d 0 R",outputintentsarrayobjnum )
        end
        if options.format == "PDF/UA" then
            pdfcatalog[#pdfcatalog + 1] = string.format(" /MarkInfo <<  /Marked true >> ")
            metadataobjnum = pdf.obj({ type="stream", string = metadata.getmetadata(filespecnumbers, options), immediate = true, attr = [[  /Subtype /XML /Type /Metadata ]],compresslevel = 0,})
            vp[#vp + 1] = "/DisplayDocTitle true"

            local parenttree = pdf.reserveobj()
            structTreeRootObjectNumber = pdf.reserveobj()
            -- Sort structure tree by reading order if pages were reordered (InsertPages/SavePages)
            local needs_reorder = false
            for i = 1, #pagenum_tbl do
                if pagenum_tbl[i] ~= i then
                    needs_reorder = true
                    break
                end
            end
            if needs_reorder then
                -- Map page object ref → output position (reading order).
                -- pagenum_tbl[k] = output position for internal page k (shipout order).
                local page_ref_to_num = {}
                for k = 1, #pagenum_tbl do
                    page_ref_to_num[pdf.getpageref(k)] = pagenum_tbl[k]
                end
                sort_struct_tree_by_page_order(structElements[".root"], page_ref_to_num)
            end
            if options.dumpstructtree then
                -- Build page object ref → logical page number mapping
                local pageref_to_num = {}
                for k = 1, #pagenum_tbl do
                    pageref_to_num[pdf.getpageref(k)] = pagenum_tbl[k]
                end
                local xmlstr = '<?xml version="1.0" encoding="UTF-8"?>\n' .. dump_struct_tree_xml(structElements[".root"], nil, pageref_to_num)
                local fn = tex.jobname .. "-struct.xml"
                local f = io.open(fn, "w")
                if f then
                    f:write(xmlstr)
                    f:write("\n")
                    f:close()
                    main.log("info","Structure tree written to " .. fn)
                else
                    main.log("error","Cannot open " .. fn .. " for writing")
                end
            end
            writeStructElements(structElements[".root"],structTreeRootObjectNumber)
            local strObjnum = pdf.obj({ type = "raw", objnum = structTreeRootObjectNumber,  string = string.format("<</Type /StructTreeRoot /K %d 0 R /ParentTree %d 0 R >>",structElements[".root"].obj,parenttree), immediate = true})
            local numentries = { "<< /Nums [" }
            for i = 1, #struct_root_numtree do
                numentries[#numentries+1] = tostring(i-1)
                numentries[#numentries+1] = tostring(struct_root_numtree[i])
            end
            numentries[#numentries + 1] = "] >>"
            pdf.obj({type = "raw", string = string.format(table.concat(numentries," ")), objnum = parenttree, immediate = true})

            pdfcatalog[#pdfcatalog + 1] = string.format("/StructTreeRoot %d 0 R",strObjnum)
        end

        if metadataobjnum then
            pdfcatalog[#pdfcatalog + 1] = string.format("/Metadata %d 0 R",metadataobjnum )
        end
    end

    if #vp > 0 then
        pdfcatalog[#pdfcatalog + 1] = "/ViewerPreferences <<" .. table.concat(vp," ") .. ">>"
    end

    local info = table.concat(infos, " ")

    local catalog = table.concat(pdfcatalog," ")

    if pdf.setinfo then
        pdf.setcatalog(catalog)
        pdf.setinfo(info)
    else
        pdf.catalog = catalog
        pdf.info = info
    end

    --- Now put the bookmarks in the pdf
    for _,v in ipairs(bookmarks) do
        bookmarkstotex(v)
    end
    local tab = {}
    for k,v in pairs(markers) do
        tab[#tab + 1] = string.format("  <mark name=%q page=%q id=%q />",xml_escape(tostring(k)),xml_escape(tostring(v.page)), tostring(v.count))
    end
    for i = 1,#visible_pagenumbers do
        tab[#tab + 1] = string.format("  <pagelabel pagenumber=%q visible=%q />",tostring(i),xml_escape(tostring(visible_pagenumbers[i])))
    end
    local file, errmsg = io.open(auxfilename,"wb")
    if file == nil then
        main.log("error","Could not open aux file for writing", "filename", auxfilename, "message", errmsg)
        return
    end
    file:write("<marker>\n")
    file:write(table.concat(tab,"\n"))
    file:write(string.format("\n <lastpage page='%d' />",lastpage))
    file:write("\n</marker>")
    file:close()
    if has_pro_error then
        log("*****************************************************")
        log("*                                                   *")
        log("* This layout uses features that require a Pro plan *")
        log("*                                                   *")
        log("* See                                               *")
        log("*  https://www.speedata.de/en/product/prices/       *")
        log("* for more information                              *")
        log("*                                                   *")
        log("*****************************************************")
    end
end

-- Create a PageLabels dictionary entry and update the visible_pagenumber
-- entry in the pagelabels table for referencing.
-- This is called at the end, when writing a dictionary
function get_page_labels_str()
    local labeltypes = {
        ["lowercase-romannumeral"] = "/r",
        ["uppercase-romannumeral"] = "/R",
        decimal = "/D",
        ["lowercase-letter"] = "/a",
        ["uppercase-letter"] = "/A"
    }
    local labelfunc = function(label,pagenumber)
        if label == "lowercase-romannumeral" then
            return tex.romannumeral(pagenumber)
        elseif label == "uppercase-romannumeral" then
            return string.upper(tex.romannumeral(pagenumber))
        elseif label == "lowercase-letter" then
            return string.char(96 + pagenumber)
        elseif label == "uppercase-letter" then
            return string.char(64 + pagenumber)
        else
            return pagenumber
        end
    end

    local prevmatter

    local tmp = {}
    local c = 0
    -- reset, there might be more pages in the previous run.
    visible_pagenumbers = {}
    for i = 1,#pagelabels do
        c = c + 1
        local p = pagelabels[i]
        if p then
            local mattername = p.matter
            local thismatter = matters[mattername]

            if prevmatter ~= mattername then
                local str = {}
                if thismatter.prefix and thismatter.prefix ~= "" then
                    str[#str + 1] = "/P " .. utf8_to_utf16_string_pdf(thismatter.prefix)
                end
                if thismatter.label then
                    str[#str + 1] = "/S " .. ( labeltypes[thismatter.label] or ("/D"))
                else
                    str[#str + 1] = "/S /D"
                end
                if prevmatter and matters[prevmatter].resetafter then
                    c = 1
                end
                if thismatter.resetbefore then
                    c = 1
                end
                if c > 1 then
                    str[#str + 1] = string.format("/St %d",c)
                end
                prevmatter = mattername
                tmp[#tmp + 1] = string.format("%d << %s >>", p.pagenumber - 1, table.concat(str," "))
            end
            visible_pagenumbers[i] = string.format("%s%s",thismatter.prefix or "",labelfunc(thismatter.label,c))
        end
    end
    local tmpstring = table.concat(tmp," ")
    if tmpstring == "" or tmpstring == "0 << /S /D >>" then return nil end
    return string.format("/PageLabels << /Nums [ %s ] >> ",tmpstring)
end

struct_root_numtree = {}

local ntmetafunctostring = function(tbl)
    local tmp = {}
    tmp[#tmp+1] = "["
    for i = 1, #tbl do
        local objnum = rawget(tbl,i)
        tmp[#tmp+1] = string.format("%d 0 R",objnum)
    end
    tmp[#tmp+1] = "]"
    return table.concat(tmp," ")
end

do
    local objcount
    local structelementobjects

    function find_role_attributes( nodelist,parenttree, page, curid )
        local head = nodelist
        while head do
            entry = nil
            if head.id == hlist_node or head.id == vlist_node then
                if head.list then
                    local r = node.has_attribute(head,att_role)
                    local parentid    = getprop(head,"parentid")

                    -- parentdid == "" is a maker for inheritance
                    if parentid == "" or parentid == nil then
                        parentid = curid
                    end
                    -- roleid is role, underscore, rolecounter, for example P_1
                    local roleid = getprop(head,"id")
                    if roleid then
                        local actualtext = getprop(head,"actualtext")
                        local alttext = getprop(head,"alttext")
                        local rolename = roles_a[r]
                        if rolename ~= "Artifact" then
                            local structpos = getprop(head,"structpos")
                            entry = {
                                obj = pdf.reserveobj(),
                                role = rolename,
                                page = page,
                                actualtext = actualtext,
                                alttext = alttext,
                            }
                            local parenttable = structElements[parentid]
                            if parenttable then
                                if structpos == "top" then
                                    table.insert(parenttable,1,entry)
                                elseif tonumber(structpos) then
                                    table.insert(parenttable,tonumber(structpos),entry)
                                else
                                    parenttable[#parenttable+1] = entry
                                end
                            end
                            structElements[roleid] = entry
                        end
                    end
                    find_role_attributes(head.list, parenttree, page, roleid or curid)
                end
            elseif node.has_attribute(head,att_role) then
                local r = node.has_attribute(head,att_role)
                local roleid      = getprop(head,"id")
                local parentid    = getprop(head,"parentid")
                if parentid == nil or parentid == "" or parentid == 0 or parentid == roleid then
                    parentid = curid
                end
                local actualtext  = getprop(head,"actualtext")
                local alttext     = getprop(head,"alttext")
                local bbox        = getprop(head,"bbox")
                -- role number to role name
                r = roles_a[r]

                local entry
                if structElements[roleid] then
                    entry = structElements[roleid]
                else
                    if r == "Link" then
                        local linkobjnum = getprop(head,"linkobjnum")
                        local structelemobjnum = getprop(head,"structelemobjnum")
                        local startlink = head.next
                        startlink.action.data = startlink.action.data .. "/F 2" .. string.format("/P %s 0 R ",page)
                        entry = {
                            obj = structelemobjnum,
                            role = r,
                            page = page,
                            actualtext = actualtext,
                            alttext = alttext,
                            bbox = bbox,
                            linkobjects = { linkobjnum }
                        }
                    else
                        entry = {
                                obj = pdf.reserveobj(),
                                role = r,
                                page = page,
                                actualtext = actualtext,
                                alttext = alttext,
                                bbox = bbox,
                            }
                    end
                    -- The parent needs links to the children, but only one for each. Therefore
                    -- the parent contains a table (added_tables) which records all roleids
                    -- that are already part of the structure.
                    -- Example:
                    -- Object 2 is: <</Type /StructElem /S /Document /P 16 0 R /K 8 0 R>>
                    -- Object 8 is: <</Type /StructElem /S /P /P 2 0 R /K [ 0 5 0 R ] /Pg 7 0 R>>
                    -- Object 7 is: << /Type /Page /Contents ... >>
                    -- Object 6 (not shown here): <</Type /StructElem /S /P /P 2 0 R /K 0 /Pg 5 0 R>>
                    -- Object 2 (the root) has one child: the P_1 (obj 6)
                    -- parentStructElem = {
                    --     ["added_tables"] = {
                    --         ["P_1"] = "true"
                    --     },
                    --     ["obj"] = "2"
                    --     ["role"] = "Document"
                    --     [1] = {
                    --          ["obj"] = "8"
                    --          ["page"] = "7"
                    --          ["role"] = "P"
                    --     },
                    -- },
                    if not parentid then
                        main.log("debug","Structure entry has no parent id","roleid",roleid or "(none)","role",r)
                    else
                        if r ~= "Artifact" then
                            local parenttable = structElements[parentid]
                            if parenttable then
                                parenttable[#parenttable+1] = entry
                            else
                                main.log("debug","Structure entry has no parent table","parentid",parentid or "(none)","role",r)
                            end
                        end
                    end
                end

                structelementobjects[#structelementobjects+1] = entry.obj
                local str
                if r == "Artifact" then
                    str = "/Artifact<<>>BDC"
                else
                    str = string.format("/%s<</MCID %d>>BDC", r,objcount)
                end
                head.data = str
                entry[#entry+1] = objcount
                objcount = objcount + 1
            end
            head = head.next
        end
    end

    -- called once for each page
    function insert_struct_elements( nodelist,page )
        -- structelementobjects contains struct tree object numbers for this page.
        structelementobjects = {}
        objcount = 0
        local parenttree = ktree
        -- Use shipout index (#pagenum_tbl) instead of logical page number,
        -- because pdf.getpageref uses internal page numbering (shipout order).
        local thispage = pdf.getpageref(#pagenum_tbl)
        find_role_attributes(nodelist,parenttree,thispage)

        page.structparents = #struct_root_numtree
        -- ntmetafunctostring returns object references in brackets for __tostring
        struct_root_numtree[#struct_root_numtree + 1] = setmetatable(structelementobjects,{__tostring = ntmetafunctostring })
     end
end

function shipout(nodelist, pagenumber,dataxml)
    pages_shippedout[pagenumber] = true
    pagenum_tbl[#pagenum_tbl+1] = pagenumber

    local cp = pages[pagenumber]
    local colorname = cp.defaultcolor
    if not matters[cp.matter] then
        local defaultmatter
        if newxpath then
            defaultmatter = xpath.string_value(dataxml.vars["_matter"])
        else
            defaultmatter = xpath.get_variable("_matter")
        end
        err("matter %q unknown, revert to %s",cp.matter or "-", defaultmatter )
        cp.matter = defaultmatter
    end
    pagelabels[pagenumber] = {
        pagenumber = pagenumber,
        matter = cp.matter,
    }
    if colorname then
        if not colors_module.colors[colorname] then
            main.log("error","Pagetype / defaultcolor: color is not defined yet.","name",colorname)
        else
            local colorindex = colors_module.colors[colorname].index
            nodelist = set_color_if_necessary(nodelist,colorindex)
            nodelist = node.vpack(nodelist)
        end
    end
    if options.format == "PDF/UA" then
        insert_struct_elements(nodelist,cp)
        -- second argument is extra page attributes
        cp.grid:trimbox(options.crop, string.format("/StructParents %d",cp.structparents))
    else
        cp.grid:trimbox(options.crop)
    end

    if options.showdebug then
        local visdebug = require("lua-visual-debug")
        visdebug.show_page_elements(nodelist)
    end
    nodelist = dothingsafteroutput(cp,nodelist)
    tex.box[666] = nodelist
    tex.shipout(666)
end

-- adds index metatble for namespace lookup to layout xml
function fixup_xmlfile(tbl,ignoreeol,parent)
    setmetatable(tbl,xml_stringvalue_mt)
    if parent then
        setmetatable(tbl[".__ns"],{__index = parent[".__ns"]})
    end
    for i = 1, #tbl do
        if type(tbl[i]) == "table" then
            fixup_xmlfile(tbl[i],ignoreeol,tbl)
        elseif ignoreeol and type(tbl[i]) == "string" then
            tbl[i] = string.gsub(tbl[i],"\n"," ")
        end
    end
end

--- Load an XML file from the hard drive. filename is without path but including extension,
--- filetype is a string representing the type of file read, such as "layout" or "data".
--- The return value is a lua table representing the XML file.
---
--- The XML file
---
---     <?xml version="1.0" encoding="UTF-8"?>
---     <data>
---       <element attribute="whatever">
---         <subelement>text in subelement</subelement>
---       </element>
---     </data>
---
--- is represented by this Lua table:
---
---     XML = {
---       [1] = " "
---       [2] = {
---         [1] = " "
---         [2] = {
---           [1] = "text in subelement"
---           [".__parent"] = (pointer to the "element" tree, which is
---                            the second entry in the top level)
---           [".__local_name"] = "subelement"
---         },
---         [3] = " "
---         [".__parent"] = (pointer to the root element)
---         [".__local_name"] = "element"
---         ["attribute"] = "whatever"
---       },
---       [3] = " "
---       [".__local_name"] = "data"
---     },
---@return table?
function load_xml(filename,filetype,parameter)
    if not filename or filename == "" then
        main.log("error","Load XML: no file name given")
        return
    end
    parameter = parameter or {}
    main.log("info", "Load XML", "type", filetype or "file", "filename", filename)
    if newxpath then
        local xmltable
        xmltable = splib.load_xmlfile(filename,filetype or "file")
        if not xmltable then
            return nil
        end
        return xmltable
    else
        if options.verbosity > 0 then
            log("Using old Lua based XML reader")
        end

        local path = kpse.find_file(filename)
        if not path then
            main.log("error","Can't find XML file. Abort","filename",filename or "?")
            return nil
        end
        if options.verbosity > 0 then
            calculate_md5sum(filename)
        end
        main.log("info","Load XML","type",filetype or "file","filename",path)
        local parsed_xml = luxor.parse_xml_file(path, parameter,kpse.find_file)
        -- if options.verbosity > 0 and filetype == "layout instructions" then
        --     printtable("parsed_xml",parsed_xml)
        -- end
        return parsed_xml
    end
end

function calculate_md5sum(filename)
    local p = kpse.find_file(filename)
    if p then
        local f,msg = io.open(p)
        if not f then
            main.log("error",msg)
            return nil
        end
        local str = f:read("*a")
        local sum = md5.sumhexa(str)
        f:close()
        log("filename %q, md5sum: %s",filename,sum)
    end
end

--- Place an object at a position given in scaled points (_x_ and _y_).
---
--- Parameter       | Description
--- ----------------|----------------------------------------------
--- nodelist        | The box to be placed
--- x               | The horizontal distance from the left edge in grid cells
--- y               | The vertical distance form the top edge in grid cells
--- clipatmargin    | Clip objects so they don't appear in the page margin
--- rotate          | Rotation counter clockwise in degrees (0-360).
--- origin_x        | Origin X for rotation. Left is 0 and right is 100
--- origin_y        | Origin Y for rotation. Top is 0 and bottom is 100
--- allocate        | Should the touched cells be allocated?
function output_absolute_position(param)
    local x = param.x
    local y = param.y
    local nodelist = param.nodelist
    local keepposition = param.keepposition
    local r = param.grid or current_grid
    --- We don't necessarily output things on a page, we can output them in a virtual page, called _group_.
    if current_group then
        -- Put the contents of the nodelist into the current group
        local group = groups[current_group]
        assert(group)

        local n = add_glue( nodelist ,"head",{ width = delta_x })
        n = node.hpack(n)
        n = add_glue(n, "head", {width = delta_y})
        n = node.vpack(n)

        if group.contents then
            -- There is already something in the group, we must add the new nodelist.
            -- The size of the new group: max(size of old group, size of new nodelist)
            local new_width, new_height
            new_width  = math.max(n.width, group.contents.width)
            new_height = math.max(n.height + n.depth, group.contents.height + group.contents.depth)

            group.contents.width  = 0
            group.contents.height = 0
            group.contents.depth  = 0

            local tail = node.tail(group.contents)
            tail.next = n
            n.prev = tail

            group.contents = node.vpack(group.contents)
            group.contents.width  = new_width
            group.contents.height = new_height
            group.contents.depth  = 0
        else
            -- group is empty
            group.contents = n
        end
        if allocate then
            r:allocate_cells({
                posx = x - shift_left,
                posy = y - shift_up,
                width_gridcells = width_gridcells,
                height_gridcells = height_gridcells,
                allocate_matrix = allocate_matrix,
            })
        end
    else
        if param.allocate then
            local additional_width,additional_height = 0,0

            local startcol_sp = x - current_grid.margin_left
            local startrow_sp = y - current_grid.margin_top

            if param.allocate_left then
              startcol_sp = startcol_sp - param.allocate_left
              additional_width = additional_width + param.allocate_left
            end
            if param.allocate_right then
              additional_width = additional_width + param.allocate_right
            end
            if param.allocate_top then
              startrow_sp = startrow_sp - param.allocate_top
              additional_height = additional_height + param.allocate_top
            end
            if param.allocate_bottom then
              additional_height = additional_height + param.allocate_bottom
            end

            local startcol  = math.floor(math.round( (startcol_sp - current_grid.extra_margin) / current_grid.gridwidth ,3)) + 1
            local delta_x = startcol_sp - current_grid:width_sp(startcol - 1)
            if delta_x < 100 then delta_x = 0 end

            local wd_grid = current_grid:width_in_gridcells_sp(nodelist.width + delta_x + additional_width - current_grid.extra_margin)
            local startrow  = math.floor(math.round( (startrow_sp - current_grid.extra_margin) / current_grid.gridheight ,3)) + 1
            local delta_y = startrow_sp - current_grid:height_sp(startrow - 1)
            if delta_y < 100 then delta_y = 0 end
            local ht_grid = current_grid:height_in_gridcells_sp(nodelist.height + delta_y + additional_height - current_grid.extra_margin)
            local _x,_y,_wd,_ht = startcol,startrow,wd_grid,ht_grid
            if _x < 1 then
                _wd = _wd + _x - 1
                _x = 1
            end
            if _y < 1 then
                _ht = _ht + _y - 1
                _y = 1
            end

            current_grid:allocate_cells({
                posx = _x,
                posy = _y,
                width_gridcells = _wd,
                height_gridcells = _ht,
                keepposition = keepposition,
                allocate_matrix = param.allocate_matrix})
        end


        if node.has_attribute(nodelist,att_shift_left) then
            x = x - ( node.has_attribute(nodelist,att_shift_left) or 0)
            y = y - ( node.has_attribute(nodelist,att_shift_up) or 0)
        end

        if param.rotate then
            nodelist = rotate(nodelist,param.rotate, param.origin_x or 0, param.origin_y or 0)
        end

        if param.clipatmargin then
            local wd = nodelist.width
            local ht = nodelist.height + nodelist.depth

            local clipleft = math.max(r.margin_left + r.extra_margin - x,0)
            local cliptop = math.max(r.extra_margin + r.margin_top - y, 0)
            local clipright = math.max(-1 *(tex.pagewidth - r.extra_margin - r.margin_right - x - wd),0)
            local pageframe = r.positioning_frames._page[1]
            local maxht = r.extra_margin + r.margin_top + pageframe.height * r.gridheight + (pageframe.height - 1) * r.grid_dy
            local clipbottom = math.max(-1 *( maxht - y - ht),0)

            nodelist = clip({
                box = nodelist,
                clip_top_sp = cliptop,
                clip_bottom_sp = clipbottom,
                clip_left_sp = clipleft,
                clip_right_sp = clipright,
                clip_width_sp = 0,
                clip_height_sp = 0,
                method = "frame",
            })
        end


        local n = add_glue( nodelist ,"head",{ width = x })
        n = node.hpack(n)
        n = add_glue(n, "head", {width = y})
        n = node.vpack(n)
        n.width  = 0
        n.height = 0
        n.depth  = 0
        local tail = node.tail(pages[current_pagenumber].pagebox)
        tail.next = n
        n.prev = tail
    end
end

-- annotate_nodelist is used for tooltips when debugging text formats.
do
    local annotcount = 0
    function annotate_nodelist(nodelist,text)
        text = text:gsub(" ","\\040")
        local annot = node.new(whatsit_node,"pdf_annot")
        local str = string.format([[ /Subtype /Widget /TU (%s) /T (tooltip zref@%d) /C [] /FT/Btn /F 768 /Ff 65536 /H/N /BS << /W 0 >>]],text,annotcount)
        annotcount = annotcount + 1
        annot.data = str
        annot.width = nodelist.width
        annot.height = nodelist.height
        annot.depth = nodelist.depth
        nodelist = node.insert_before(nodelist.head,nodelist.head,annot)
        return nodelist
    end
end

--- Put the object (nodelist) on grid cell (x,y). If `allocate`=`true` then
--- mark cells as occupied.
---
--- Parameter       | Description
--- ----------------|----------------------------------------------
--- nodelist        | The box to be placed
--- x               | The horizontal distance from the left edge in grid cells
--- y               | The vertical distance form the top edge in grid cells
--- allocate        | Mark these cells as 'occupied'
--- area            | The area on which the object should be placed. Defaults to the page area.
--- clipatmargin    | Clip objects so they don't appear in the page margin
--- valign          |
--- halign          |
--- allocate_matrix | For image-shapes
--- pagenumber      | The page the object should be placed
--- keepposition    | Move the local cursor?
--- grid            | The grid object. If not present, we use the default grid object
--- rotate          | Rotation counter clockwise in degrees (0-360).
--- origin_x        | Origin X for rotation. Left is 0 and right is 100
--- origin_y        | Origin Y for rotation. Top is 0 and bottom is 100
--- framewidth      | When frame=solid then this has the frame width
function output_at( param )
    local _wd, _ht, _dp = node.dimensions(param.nodelist)
    if param.framewidth then
        _wd = _wd + param.framewidth
        _ht = _ht + param.framewidth
    end

    -- current_grid is important here, because it can be a group
    local r = param.grid or current_grid


    local outputpage = current_pagenumber
    if param.pagenumber then
        outputpage = param.pagenumber
    end
    local nodelist = param.nodelist
    if options.showobjects then
        nodelist = boxit(nodelist)
    end
    local x = param.x
    local y = param.y

    local additional_width,additional_height = 0,0
    local shift_left,shift_up = 0,0

    if param.allocate_left and param.allocate_left > 100 then
        shift_left = r:width_in_gridcells_sp(param.allocate_left)
        additional_width = additional_width + r:width_in_gridcells_sp(param.allocate_left)
    end
    if param.allocate_right and param.allocate_right > 100 then
        additional_width = additional_width + r:width_in_gridcells_sp(param.allocate_right)
    end
    if param.allocate_top and param.allocate_top > 100 then
        shift_up = r:height_in_gridcells_sp(param.allocate_top)
        additional_height = additional_height + r:height_in_gridcells_sp(param.allocate_top)
    end
    if param.allocate_bottom and param.allocate_bottom > 100 then
        additional_height = additional_height + r:height_in_gridcells_sp(param.allocate_bottom)
    end


    local allocate = param.allocate
    local allocate_matrix = param.allocate_matrix
    local area = param.area or default_areaname
    local valign = param.valign
    local halign = param.halign
    local keepposition = param.keepposition

    local wd = nodelist.width
    local ht = nodelist.height + nodelist.depth

    -- For grid allocation
    local width_gridcells   = r:width_in_gridcells_sp(wd)
    if additional_width > 0 then
        width_gridcells = width_gridcells + additional_width
    end
    local height_gridcells  = r:height_in_gridcells_sp(ht,{floor = (param.vreference == "bottom") })
    if additional_height > 0 then
        height_gridcells = height_gridcells + additional_height
    end

    local delta_x, delta_y = r:position_grid_cell(x,y,area,wd,ht,valign,halign,width_gridcells,height_gridcells)

    if not delta_x then
        -- if delta_x is nil, delta_y has the error message
        main.log("error",delta_y)
        exit()
    end

    if node.has_attribute(nodelist,att_shift_left) then
        delta_x = delta_x - node.has_attribute(nodelist,att_shift_left)
        delta_y = delta_y - node.has_attribute(nodelist,att_shift_up)
    end
    if param.clipatmargin then
        local clipleft = math.max(r.margin_left + r.extra_margin - delta_x,0)
        local cliptop = math.max(r.extra_margin + r.margin_top - delta_y, 0)
        local clipright = math.max(-1 *(tex.pagewidth - r.extra_margin - r.margin_right - delta_x - wd),0)
        local pageframe = r.positioning_frames._page[1]
        local maxht = r.extra_margin + r.margin_top + pageframe.height * r.gridheight + (pageframe.height - 1) * r.grid_dy
        local clipbottom = math.max(-1 *( maxht - delta_y - ht),0)

        nodelist = clip({
            box = nodelist,
            clip_top_sp = cliptop,
            clip_bottom_sp = clipbottom,
            clip_left_sp = clipleft,
            clip_right_sp = clipright,
            clip_width_sp = 0,
            clip_height_sp = 0,
            method = "frame",
        })
    end


    local extra_crop = 0
    if param.framewidth then
        extra_crop = param.framewidth
    end

    -- set the crop area
    r:setarea(delta_x - extra_crop,delta_y - extra_crop, _wd + extra_crop, _ht + extra_crop + _dp)

    --- We don't necessarily output things on a page, we can output them in a virtual page, called _group_.
    if current_group then
        -- Put the contents of the nodelist into the current group
        local group = groups[current_group]
        assert(group)

        local n = add_glue( nodelist ,"head",{ width = delta_x })
        n = node.hpack(n)
        n = add_glue(n, "head", {width = delta_y})
        n = node.vpack(n)

        if group.contents then
            -- There is already something in the group, we must add the new nodelist.
            -- The size of the new group: max(size of old group, size of new nodelist)
            local new_width, new_height
            new_width  = math.max(n.width, group.contents.width)
            new_height = math.max(n.height + n.depth, group.contents.height + group.contents.depth)

            group.contents.width  = 0
            group.contents.height = 0
            group.contents.depth  = 0

            local tail = node.tail(group.contents)
            tail.next = n
            n.prev = tail

            group.contents = node.vpack(group.contents)
            group.contents.width  = new_width
            group.contents.height = new_height
            group.contents.depth  = 0
        else
            -- group is empty
            group.contents = n
        end
        if allocate then
            r:allocate_cells({
                posx = x - shift_left,
                posy = y - shift_up,
                width_gridcells = width_gridcells,
                height_gridcells = height_gridcells,
                allocate_matrix = allocate_matrix,
            })
        end
    else
        -- Put it on the current page
        if allocate then
            r:allocate_cells({
                posx = x - shift_left,
                posy = y - shift_up,
                width_gridcells = width_gridcells,
                height_gridcells = height_gridcells,
                allocate_matrix = allocate_matrix,
                area = area,
                keepposition = keepposition,
                objectwidth = _wd,
                objectheight = _ht + _dp
            })
        end
        if param.rotate then
            nodelist = rotate(nodelist,param.rotate, param.origin_x or 0, param.origin_y or 0)
        end

        place_at(pages[outputpage].pagebox,nodelist,delta_x,delta_y)
    end
end

function place_at(pagebox,nodelist,x_sp,y_sp)
    local tail = node.tail(pagebox)
    local n = add_glue( nodelist ,"head",{subtype = 1000, width = x_sp })
    n = node.hpack(n)
    n = add_glue(n, "head", { subtype = 1001, width = y_sp})
    n = node.vpack(n)
    n.width  = 0
    n.height = 0
    n.depth  = 0

    tail.next = n
    n.prev = tail
end

--- Return the XML structure that is stored at &lt;pagetype>. For every pagetype
--- in the table "masterpages" the function is_pagetype() gets called.
-- pagenumber is for debugging purpose
function detect_pagetype(pagenumber, data)
    -- ugly hack. file global variables are a bad idea.
    if not newxpath then
        xpath.push_state()
    end
    local cp = current_pagenumber
    current_pagenumber = pagenumber
    local ret = nil
    for i=#masterpages,1,-1 do
        local pagetype = masterpages[i]
        if nextpage then
            if pagetype.name == nextpage then
                main.log("info","Create page","type",pagetype.name or "(detect_pagetype)","pagenumber",pagenumber)
                nextpage = nil
                return pagetype.res
            end
        else
            if newxpath then
                assert(data,"detect_pagetype")
                local seq, msg = data:eval(pagetype.is_pagetype)
                if msg then
                    err(msg)
                end
                local ok
                ok, msg = xpath.boolean_value(seq)
                if msg then
                    err(msg)
                end
                if ok then
                    main.log("info","Create page","type",pagetype.name or "(detect_pagetype)","pagenumber",pagenumber)
                    ret = pagetype.res
                    current_pagenumber = cp
                    return ret
                end
            else
                if xpath.parse(data,pagetype.is_pagetype,pagetype.ns) == true then
                    main.log("info","Create page","type",pagetype.name or "(detect_pagetype)","pagenumber",pagenumber)
                    ret = pagetype.res
                    xpath.pop_state()
                    current_pagenumber = cp
                    return ret
                end
            end
        end
    end
    err("Can't find correct page type!")
    current_pagenumber = cp
    if not newxpath then
        xpath.pop_state()
    end
    return false
end

function initialize_page(pagenumber,data, from)
    local thispage

    if pagenumber then
        thispage = pagenumber
        if pages[pagenumber] ~= nil then
            current_grid=pages[pagenumber].grid
            return
        end
    else
        if page_initialized_p(current_pagenumber) then
            current_grid=pages[current_pagenumber].grid
            return
        end
    end

    if not pagenumber then
        thispage = current_pagenumber
    end

    local trim_amount = tex.sp(options.trim or 0)
    local extra_margin
    if options.cutmarks or options.trimmarks then
        if not pro then
            err("cutmarks need a pro plan")
            publisher.has_pro_error = true
        else
            extra_margin = tenmm_sp + trim_amount
        end
    elseif trim_amount > 0 then
        if not pro then
            err("page bleed needs a pro plan")
            publisher.has_pro_error = true
        else
            extra_margin = trim_amount
        end
    end
    local errorstring

    current_page, errorstring = page:new(options.default_pagewidth,options.default_pageheight, extra_margin, trim_amount,thispage)
    if not current_page then
        err("Can't create a new page. Is the page type (“PageType”) defined? %s",errorstring)
        exit()
    end
    current_grid = current_page.grid
    pages[thispage] = current_page

    local gridwidth, gridheight, nx, ny, dx, dy
    nx = options.gridcells_x
    ny = options.gridcells_y
    dx = options.gridcells_dx
    dy = options.gridcells_dy

    local pagetype = detect_pagetype(thispage,data)
    if pagetype == false then return false end
    if not pagetype then return false end
    if pagetype.width then
        current_page.width = tex.sp(pagetype.width)
    end
    if pagetype.height then
        current_page.height = tex.sp(pagetype.height)
    end
    if pagetype.width or pagetype.height then
        if newxpath then
            data.vars["_pagewidth"] =  pagetype.width
            data.vars["_pageheight"] = pagetype.height
        else
            xpath.set_variable("_pagewidth", pagetype.width)
            xpath.set_variable("_pageheight", pagetype.height)
        end
        set_pageformat(current_page.width,current_page.height)
    else
        -- 186467sp = 1mm
        local pagewd = current_page.width / 186467
        local pageht = current_page.height / 186467
        if newxpath then
            data.vars["_pagewidth"] = tostring(math.round(pagewd,0)) .. "mm"
            data.vars["_pageheight"] = tostring(math.round(pageht,0)) .. "mm"
        else
            xpath.set_variable("_pagewidth", tostring(math.round(pagewd,0)) .. "mm")
            xpath.set_variable("_pageheight", tostring(math.round(pageht,0)) .. "mm")
        end
    end

    local mattername
    if newxpath then
        mattername = pagetype.part or xpath.string_value(data.vars["_matter"])
    else
        mattername = pagetype.part or xpath.get_variable("_matter")
    end
    current_page.matter = mattername

    for _,j in ipairs(pagetype) do
        local eltname = elementname(j)
        local eltcontents = element_contents(j)
        if type(element_contents(j))=="function" and eltname=="Margin" then
            eltcontents(current_page)
        elseif eltname=="Grid" then
            local layoutxml = eltcontents.layoutxml
            local dataxml = eltcontents.dataxml
            local width  = read_attribute(layoutxml,dataxml,"width",  "length_sp")
            local height = read_attribute(layoutxml,dataxml,"height", "length_sp") -- shouldn't this be height_sp??? --PG
            local _nx     = read_attribute(layoutxml,dataxml,"nx",     "number")
            local _ny     = read_attribute(layoutxml,dataxml,"ny",     "number")
            local _dx     = read_attribute(layoutxml,dataxml,"dx",     "length_sp")
            local _dy     = read_attribute(layoutxml,dataxml,"dy",     "length_sp")

            gridwidth  = width
            gridheight = height
            nx = _nx
            ny = _ny
            dx = _dx
            dy = _dy
        end
    end

    if gridwidth == nil and options.gridwidth ~= 0 then
        gridwidth = options.gridwidth
    end

    if gridheight == nil and options.gridheight ~= 0 then
        gridheight = options.gridheight
    end

    current_page.grid:set_width_height({wd = gridwidth, ht = gridheight, nx = nx, ny = ny, dx = dx, dy = dy })

    -- The default color is applied during ship-out
    if newxpath then
        if pagetype.layoutxml and pagetype.layoutxml[".__attributes"].defaultcolor then
            current_page.defaultcolor = read_attribute(pagetype.layoutxml,nil,"defaultcolor","string")
        end
    else
        if pagetype.layoutxml and pagetype.layoutxml.defaultcolor then
            current_page.defaultcolor = read_attribute(pagetype.layoutxml,nil,"defaultcolor","string")
        end
    end
    current_page.graphic = pagetype.graphic
    current_page.backgroundcolor = pagetype.backgroundcolor
    local columnordering = pagetype.columnordering
    for _,j in ipairs(pagetype) do
        local eltname = elementname(j)
        if type(element_contents(j))=="function" and eltname=="Margin" then
            -- do nothing, done before
        elseif eltname=="Grid" then
            -- do nothing, done before
        elseif eltname=="AtPageCreation" then
            current_page.atpagecreation = element_contents(j)
        elseif eltname=="AtPageShipout" then
            current_page.AtPageShipout = element_contents(j)
        elseif eltname=="PositioningArea" then
            local name = element_contents(j).name
            current_grid.positioning_frames[name] = {}
            local current_positioning_area = current_grid.positioning_frames[name]
            -- we evaluate now, because the attributes in PositioningFrame can be page dependent.
            local d = element_contents(j).dataxml
            local l = element_contents(j).layoutxml
            local tab  = dispatch(l,d)
            local tmp = {}
            for i,k in ipairs(tab) do
                tmp[#tmp + 1] = element_contents(k)
                tmp[#tmp].order = i
            end
            if columnordering == "rtl" then
                table.sort(tmp,function(a,b)
                    if a.column == b.column then return a.order > b.order end
                    return a.column > b.column
                end)
            end

            for i=1,#tmp do
                table.insert(current_positioning_area,tmp[i])
            end
            local bgcolor = element_contents(j).bgcolor
            if bgcolor then
                for _, tbl in ipairs(tmp) do
                    local x = current_grid:posx_sp(tbl.column - 1) + current_grid.extra_margin + current_grid.margin_left
                    local y = current_grid:posy_sp(tbl.row - 1) + current_grid.extra_margin + current_grid.margin_top
                    local wd = current_grid:posx_sp(tbl.width)
                    local ht = current_grid:posy_sp(tbl.height)

                    local nl = box(wd,ht,bgcolor)

                    output_absolute_position({
                        nodelist = nl,
                        x = x,
                        y = y,
                        allocate = false,
                    })
                end
            end
            current_positioning_area.colorname = element_contents(j).colorname
        else
            err("Element name %q unknown (setup_page())",eltname or "<create_page>")
        end
    end

    local cp = current_page
    current_page = pages[thispage]
    if current_page.atpagecreation then
        pagebreak_impossible = true
        local cpn = current_pagenumber
        current_pagenumber = thispage
        current_grid = pages[thispage].grid
        dispatch(current_page.atpagecreation,data)
        current_pagenumber = cpn
        pagebreak_impossible = false
        local graphic
        if newxpath then
            local attrs = current_page.atpagecreation[".__attributes"]
            if attrs then graphic = attrs.graphic end
        else
            graphic = current_page.atpagecreation.graphic
        end
        if graphic then
            local _,whatsit, _ = metapost.prepareboxgraphic(current_page.width,current_page.height,graphic,metapost.extra_page_parameter(current_page))
            place_at(current_page.pagebox,whatsit, current_page.grid.extra_margin,current_page.height+current_page.grid.extra_margin)
        end
    end

    local css_rules
    local cg = current_page.grid

    for k,v in pairs(cg.positioning_frames) do
        css_rules = css:matches({element = 'area', class=class,id=k}) or {}
        if css_rules["border-width"] then
            for i,frame in ipairs(v) do
                frame.draw = { color = "green", width = css_rules["border-width"] }
            end
        end
    end
    current_page = cp

end

-- skippages are set in commands.new_page if openon="..."
skippages = nil
--- _Must_ be called before something can be put on the page. Looks for hooks to be run before page creation.
function setup_page(pagenumber,fromwhere,dataxml)
    if current_group then return end
    if skippages then
        local tmp = skippages
        skippages = nil
        if tmp.doubleopen then
            new_page("setup_page - skippages doubleopen",dataxml)
            nextpage = tmp.skippagetype
        end
        new_page("setup_page - skippages 2",dataxml)
        nextpage = tmp.pagetype
    end

     initialize_page(pagenumber,dataxml,fromwhere)
end

--- Switch to the next frame in the given area.
function next_area( areaname, grid, dataxml,origin )
    grid = grid or current_grid
    local current_framenumber = grid:framenumber(areaname)
    if not current_framenumber then
        err("Cannot determine current area number (areaname=%q)",areaname or "(undefined)")
        return
    end
    if current_framenumber >= grid:number_of_frames(areaname) then
        new_page("next_area",dataxml)
    else
        grid:set_framenumber(areaname, current_framenumber + 1)
    end
    grid:set_current_row(1,areaname,"next_area")
    grid:set_current_column(1,areaname,"next_area")
end

--- Switch to a new page and ship out the current page.
--- This new page is only created if something is typeset on it.
function new_page(from,dataxml)
    -- w("new page from %s",from or "-")
    if pagebreak_impossible then
        return
    end
    local thispage = pages[current_pagenumber]
    if not thispage then
        -- new_page() is called without anything on the page yet
        setup_page(nil,"new_page",dataxml)
        thispage = current_page
    end

    dothingsbeforeoutput(thispage,data)

    local n = node.vpack(pages[current_pagenumber].pagebox)
    if current_pagestore_name then
        local thispagestore = pagestore[current_pagestore_name]
        thispagestore[#thispagestore + 1] = n
    else
        shipout(n,current_pagenumber,dataxml)
    end
    current_pagenumber = current_pagenumber + 1
end

function clearpage(options)
    local thispage = pages[current_pagenumber]

    if thispage then
        dothingsbeforeoutput(thispage,data)
        local n = node.vpack(pages[current_pagenumber].pagebox)
        if current_pagestore_name then
            local thispagestore = pagestore[current_pagestore_name]
            thispagestore[#thispagestore + 1] = n
        else
            shipout(n,current_pagenumber,options.dataxml)
        end

        current_pagenumber = current_pagenumber + 1
    else
        if options.force then
            initialize_page(nil,options.dataxml)
            local tmp = pages[current_pagenumber]
            dothingsbeforeoutput(tmp,data)
            local n = node.vpack(pages[current_pagenumber].pagebox)
            shipout(n,current_pagenumber,options.dataxml)
            current_pagenumber = current_pagenumber + 1
        end
    end

    local doubleopen = false
    if ( options.openon == "right" and math.fmod(current_pagenumber,2) == 0 ) or ( options.openon == "left" and math.fmod(current_pagenumber,2) == 1 ) then
        doubleopen = true
    end

    if doubleopen then
        -- shipout dummy page
        if options.skippagetype then
            nextpage = options.skippagetype
        end
        initialize_page(nil,options.dataxml)
        local tmp = pages[current_pagenumber]
        dothingsbeforeoutput(tmp,data)
        local n = node.vpack(pages[current_pagenumber].pagebox)
        shipout(n,current_pagenumber,options.dataxml)
        current_pagenumber = current_pagenumber + 1
    end

    if options.matter then
        if newxpath then
            options.dataxml.vars["_matter"] = options.matter
        else
            xpath.set_variable("_matter",options.matter)
        end
    end
    if options.pagetype then
        nextpage = options.pagetype
    end
end

-- a,b are both arrays of 6 numbers
function concat_transformation( a, b )
    local c = {}
    c[1] = a[1] * b[1] + a[2] * b[3]
    c[2] = a[1] * b[2] + a[2] * b[4]
    c[3] = a[3] * b[1] + a[4] * b[3]
    c[4] = a[3] * b[2] + a[4] * b[4]
    c[5] = a[5] * b[1] + a[6] * b[3] + b[5]
    c[6] = a[5] * b[2] + a[6] * b[4] + b[6]
    return c
end

-- Place a text in the background
function bgtext( box, textstring, angle, colorname, fontfamily, bgsize)
    local colorindex = colors_module.colors[colorname].index
    local boxheight, boxwidth = box.height, box.width
    local angle_rad = -1 * math.rad(angle)
    local sin = math.sin(angle_rad)
    local cos = math.cos(angle_rad)

    a = par:new(nil,"bgtext")
    a:append(textstring, {fontfamily = fontfamily,color = colorindex})
    a:mknodelist(data)
    local textbox = node.hpack(a.objects[1])
    local rotated_height = sin * textbox.width  + cos * textbox.height
    local scale
    local shift_up = 0
    if bgsize == "contain" then
        scale = boxheight  / rotated_height
    else
        scale = 1
        shift_up = sp_to_bp((boxheight - rotated_height) / 2)
    end
    local rotated_width  = sin * textbox.height + cos * textbox.width
    local shift_right = sp_to_bp( (boxwidth - rotated_width * scale ) / 2)

    -- rotate: [cos θ sin θ −sin θ cos θ 0 0 ]
    local rotate_matrix = {   cos, sin, -1 * sin,   cos,           0, 0        }
    local scale_matrix  = { scale,   0,        0, scale,           0, 0        }
    local shift_matrix  = {     1,   0,        0,     1, shift_right, shift_up }
    local result_matrix
    result_matrix = concat_transformation(rotate_matrix,scale_matrix)
    result_matrix = concat_transformation(result_matrix,shift_matrix)
    local matrixstring = string.format("%g %g %g %g %d %g",math.round(result_matrix[1],3),math.round(result_matrix[2],3),math.round(result_matrix[3],3),math.round(result_matrix[4],3),math.round(result_matrix[5],3),math.round(result_matrix[6],3))
    local x = matrix( textbox, matrixstring,0,0 )
    x = node.hpack(x)
    x.width = 0
    x.height = 0
    box = node.insert_before(box,box,x)
    box = node.hpack(box)
    return box
end

--- Draw a background behind the rectangular (box) object.
function background( box, colorname,origin )
    -- color '-' means 'no color'
    if colorname == "-" then return box end
    if not colors_module.colors[colorname] then
        if origin then
            main.log("warn","Background: color is not defined","name",colorname,"from",origin)
        else
            main.log("warn","Background: color is not defined","name",colorname)
        end
        return box
    end
    local colentry = colors_module.colors[colorname]

    local pdfcolorstring = colentry.pdfstring
    local wd, ht, dp = sp_to_bp(box.width),sp_to_bp(box.height),sp_to_bp(box.depth)
    n = node.new("whatsit","pdf_literal")
    setprop(n,"origin","background")
    setprop(n,"role",get_rolenum("Artifact"))
    n.data = string.format("q %s 0 -%g %g %g re f Q",pdfcolorstring,dp,wd,ht + dp)
    n.mode = 0
    if colentry.alpha then
        setprop(n,"opacity",colentry.alpha)
    end
    if node.type(box.id) == "hlist" then
        -- pdfliteral does not use up any space, so we can add it to the already packed box.
        n.next = box.list
        box.list.prev = n
        box.list = n
        return box
    else
        n.next = box
        box.prev = n
        n = node.hpack(n)
        return n
    end
end

--- Draw a colored frame around a given TeX box
--- The control points of the frame are
--- ![control points](img/roundedcorners.svg)
-- obj is a table with the keys
--   box: the TeX box
--   colorname: the name of a color (defaults to "black")
--   width: the width of the border (defaults to 0)
--   clip: should the outside be clipped?
--   b_x_y_radius, where x = b or t and y = r or l: the radius of the corners
-- The function returns a hbox
function frame(obj)
    local  box, width
    box          = obj.box
    width        = obj.rulewidth or 0
    local b_b_r_radius = sp_to_bp(obj.b_b_r_radius)
    local b_t_r_radius = sp_to_bp(obj.b_t_r_radius)
    local b_t_l_radius = sp_to_bp(obj.b_t_l_radius)
    local b_b_l_radius = sp_to_bp(obj.b_b_l_radius)

    local b_b_r_radius_inner = math.round(math.max(sp_to_bp(obj.b_b_r_radius) - width / factor,0),3)
    local b_t_r_radius_inner = math.round(math.max(sp_to_bp(obj.b_t_r_radius) - width / factor,0),3)
    local b_t_l_radius_inner = math.round(math.max(sp_to_bp(obj.b_t_l_radius) - width / factor,0),3)
    local b_b_l_radius_inner = math.round(math.max(sp_to_bp(obj.b_b_l_radius) - width / factor,0),3)

    -- See https://en.wikipedia.org/wiki/File:Circle_and_cubic_bezier.svg
    -- https://en.wikipedia.org/wiki/Composite_B%C3%A9zier_curve
    -- 0.5522847498
    -- http://spencermortensen.com/articles/bezier-circle/
    -- 0.551915024494
    local circle_bezier = 0.551915024494

    local colentry = colors_module.get_colentry_from_name(obj.colorname,"black")
    if not colentry then
        main.log("error","Color is not defined","name",tostring(obj.colorname))
        colentry = colors_module.colors["black"]
    end

    local pdfcolorstring = colentry.pdfstring
    local wd, ht, dp = sp_to_bp(box.width),sp_to_bp(box.height),sp_to_bp(box.depth)
    local rw = math.round(width / factor,3) -- width of stroke
    if obj.colorname == "-" then
        rw = 0
    end

    -- outer boundary
    local x1, y1   = -rw + b_b_l_radius                     , -rw - dp
    local x2, y2   =  rw + wd - b_b_r_radius                , -rw - dp
    local x3, y3   =  rw + wd - circle_bezier * b_b_r_radius, -rw - dp
    local x4, y4   =  rw + wd                               , -rw - dp + circle_bezier * b_b_r_radius
    local x5, y5   =  rw + wd                               , -rw + b_b_r_radius
    local x6, y6   =  rw + wd                               ,  rw + ht - b_t_r_radius
    local x7, y7   =  rw + wd                               ,  rw + ht - circle_bezier * b_t_r_radius
    local x8, y8   =  rw + wd - circle_bezier * b_t_r_radius,  rw + ht
    local x9, y9   =  rw + wd - b_t_r_radius                ,  rw + ht
    local x10, y10 = -rw + b_t_l_radius                     ,  rw + ht
    local x11, y11 = -rw + circle_bezier * b_t_l_radius     ,  rw + ht
    local x12, y12 = -rw                                    ,  rw + ht - circle_bezier * b_t_l_radius
    local x13, y13 = -rw                                    ,  rw + ht - b_t_l_radius
    local x14, y14 = -rw                                    , -rw - dp + b_b_l_radius
    local x15, y15 = -rw                                    , -rw -dp + circle_bezier * b_b_l_radius
    local x16, y16 = -rw + circle_bezier * b_b_l_radius     , -rw - dp

    x1,  y1  = math.round(x1,3),  math.round(y1,3)
    x2,  y2  = math.round(x2,3),  math.round(y2,3)
    x3,  y3  = math.round(x3,3),  math.round(y3,3)
    x4,  y4  = math.round(x4,3),  math.round(y4,3)
    x5,  y5  = math.round(x5,3),  math.round(y5,3)
    x6,  y6  = math.round(x6,3),  math.round(y6,3)
    x7,  y7  = math.round(x7,3),  math.round(y7,3)
    x8,  y8  = math.round(x8,3),  math.round(y8,3)
    x9,  y9  = math.round(x9,3),  math.round(y9,3)
    x10, y10 = math.round(x10,3), math.round(y10,3)
    x11, y11 = math.round(x11,3), math.round(y11,3)
    x12, y12 = math.round(x12,3), math.round(y12,3)
    x13, y13 = math.round(x13,3), math.round(y13,3)
    x14, y14 = math.round(x14,3), math.round(y14,3)
    x15, y15 = math.round(x15,3), math.round(y15,3)
    x16, y16 = math.round(x16,3), math.round(y16,3)


    -- inner boundary
    local xx1, yy1   =   b_b_l_radius_inner                      , -dp
    local xx2, yy2   =   wd - b_b_r_radius_inner                 , -dp
    local xx3, yy3   =   wd - circle_bezier * b_b_r_radius_inner , -dp
    local xx4, yy4   =   wd                                      , -dp + circle_bezier * b_b_r_radius_inner
    local xx5, yy5   =   wd                                      , b_b_r_radius_inner
    local xx6, yy6   =   wd                                      , ht - b_t_r_radius_inner
    local xx7, yy7   =   wd                                      , ht - circle_bezier * b_t_r_radius_inner
    local xx8, yy8   =   wd - circle_bezier * b_t_r_radius_inner , ht
    local xx9, yy9   =   wd - b_t_r_radius_inner                 , ht
    local xx10, yy10 =   b_t_l_radius_inner                      , ht
    local xx11, yy11 =   circle_bezier * b_t_l_radius_inner      , ht
    local xx12, yy12 =   0                                       , ht - circle_bezier * b_t_l_radius_inner
    local xx13, yy13 =   0                                       , ht - b_t_l_radius_inner
    local xx14, yy14 =   0                                       , -dp + b_b_l_radius_inner
    local xx15, yy15 =   0                                       , -dp + circle_bezier * b_b_l_radius_inner
    local xx16, yy16 =   circle_bezier * b_b_l_radius_inner      , -dp

    xx1,  yy1  = math.round(xx1,3),  math.round(yy1,3)
    xx2,  yy2  = math.round(xx2,3),  math.round(yy2,3)
    xx3,  yy3  = math.round(xx3,3),  math.round(yy3,3)
    xx4,  yy4  = math.round(xx4,3),  math.round(yy4,3)
    xx5,  yy5  = math.round(xx5,3),  math.round(yy5,3)
    xx6,  yy6  = math.round(xx6,3),  math.round(yy6,3)
    xx7,  yy7  = math.round(xx7,3),  math.round(yy7,3)
    xx8,  yy8  = math.round(xx8,3),  math.round(yy8,3)
    xx9,  yy9  = math.round(xx9,3),  math.round(yy9,3)
    xx10, yy10 = math.round(xx10,3), math.round(yy10,3)
    xx11, yy11 = math.round(xx11,3), math.round(yy11,3)
    xx12, yy12 = math.round(xx12,3), math.round(yy12,3)
    xx13, yy13 = math.round(xx13,3), math.round(yy13,3)
    xx14, yy14 = math.round(xx14,3), math.round(yy14,3)
    xx15, yy15 = math.round(xx15,3), math.round(yy15,3)
    xx16, yy16 = math.round(xx16,3), math.round(yy16,3)

    local n_clip, rule_clip

    if obj.clip then
        n_clip = node.new("whatsit","pdf_literal")
        setprop(n_clip,"origin","obj.clip")
        rule_clip = {}
        rule_clip[#rule_clip + 1] = string.format("%g %g m",xx1,yy1)
        rule_clip[#rule_clip + 1] = string.format("%g %g l",xx2,yy2)
        rule_clip[#rule_clip + 1] = string.format("%g %g %g %g %g %g c", xx3,yy3, xx4,yy4, xx5, yy5 )
        rule_clip[#rule_clip + 1] = string.format("%g %g l",xx6, yy6)
        rule_clip[#rule_clip + 1] = string.format("%g %g %g %g %g %g c", xx7,yy7,xx8,yy8, xx9,yy9  )
        rule_clip[#rule_clip + 1] = string.format("%g %g l",xx10, yy10)
        rule_clip[#rule_clip + 1] = string.format("%g %g %g %g %g %g c", xx11,yy11,xx12,yy12, xx13,yy13  )
        rule_clip[#rule_clip + 1] = string.format("%g %g l",xx14,yy14 )
        rule_clip[#rule_clip + 1] = string.format("%g %g %g %g %g %g c W n ", xx15,yy15,xx16,yy16, xx1,yy1  )
    end

    local n = node.new("whatsit","pdf_literal")
    setprop(n,"origin","publisher.frame")
    local rule = {}

    -- We need to add q .. Q because the color would leak into the inner objects (#55)
    rule[#rule + 1] = string.format("q %s",pdfcolorstring)
    rule[#rule + 1] = string.format("%g w",rw)           -- rule width

    rule[#rule + 1] = string.format("%g %g m",xx1,yy1)
    rule[#rule + 1] = string.format("%g %g l",xx2,yy2)
    rule[#rule + 1] = string.format("%g %g %g %g %g %g c", xx3,yy3, xx4,yy4, xx5, yy5 )
    rule[#rule + 1] = string.format("%g %g l",xx6, yy6)
    rule[#rule + 1] = string.format("%g %g %g %g %g %g c", xx7,yy7,xx8,yy8, xx9,yy9  )
    rule[#rule + 1] = string.format("%g %g l",xx10, yy10)
    rule[#rule + 1] = string.format("%g %g %g %g %g %g c", xx11,yy11,xx12,yy12, xx13,yy13  )
    rule[#rule + 1] = string.format("%g %g l",xx14,yy14 )
    rule[#rule + 1] = string.format("%g %g %g %g %g %g c ", xx15,yy15,xx16,yy16, xx1,yy1  )

    rule[#rule + 1] = string.format("%g %g m",x1,y1)
    rule[#rule + 1] = string.format("%g %g l",x2,y2)
    rule[#rule + 1] = string.format("%g %g %g %g %g %g c", x3,y3, x4,y4, x5, y5 )
    rule[#rule + 1] = string.format("%g %g l",x6, y6)
    rule[#rule + 1] = string.format("%g %g %g %g %g %g c", x7,y7,x8,y8, x9,y9  )
    rule[#rule + 1] = string.format("%g %g l",x10, y10)
    rule[#rule + 1] = string.format("%g %g %g %g %g %g c", x11,y11,x12,y12, x13,y13  )
    rule[#rule + 1] = string.format("%g %g l",x14,y14 )
    rule[#rule + 1] = string.format("%g %g %g %g %g %g c", x15,y15,x16,y16, x1,y1  )
    if rw == 0 then
        rule[#rule + 1] = "n"
    else
        rule[#rule + 1] = "f*"
    end
    rule[#rule + 1] = "Q"

    n.data = table.concat(rule," ")
    if colentry.alpha then
        setprop(n,"opacity",colentry.alpha)
    end

    setprop(n,"origin","frame")
    setprop(n,"role",get_rolenum("Artifact"))


    if obj.clip then
        n_clip.data = table.concat(rule_clip, " ")
        node.setproperty(n_clip,{origin = "frame/clip"})
    end

    if n_clip and node.is_node(n_clip) then
        setprop(n_clip,"origin","frame")
        setprop(n_clip,"role",get_rolenum("Artifact"))
    end


    local pdf_save    = node.new("whatsit","pdf_save")
    local pdf_restore = node.new("whatsit","pdf_restore")

    node.insert_after(pdf_save,pdf_save,n)
    if obj.clip then
        node.insert_after(n,n,n_clip)
        node.insert_after(n_clip,n_clip,box)
    else
        node.insert_after(n,n,box)
    end

    local hvbox = node.hpack(pdf_save)
    local savedp = hvbox.depth
    hvbox.depth = 0
    node.insert_after(hvbox,node.tail(hvbox),pdf_restore)
    hvbox = node.vpack(hvbox)
    hvbox = node.vpack(hvbox)
    hvbox.depth = savedp
    return hvbox
end

function clip(obj)
    local box = obj.box
    local wd, ht, dp = sp_to_bp(box.width),sp_to_bp(box.height),sp_to_bp(box.depth)

    local kern_left = node.new("kern")
    local kern_top = node.new("kern")

    if obj.clip_width_sp ~= 0 then
        if obj.clip_left_sp ~= 0 or obj.clip_right_sp == 0 then
            obj.clip_right_sp = box.width - obj.clip_width_sp - obj.clip_left_sp
        else
            obj.clip_left_sp = box.width - obj.clip_width_sp - obj.clip_right_sp
        end
    end

    if obj.clip_height_sp ~= 0 then
        if obj.clip_top_sp ~= 0 or obj.clip_bottom_sp == 0  then
            obj.clip_bottom_sp = box.height - obj.clip_height_sp - obj.clip_top_sp
        else
            obj.clip_top_sp = box.height - obj.clip_height_sp - obj.clip_bottom_sp
        end
    end



    local clip_top_bp = sp_to_bp(obj.clip_top_sp)
    local clip_bottom_bp = sp_to_bp(obj.clip_bottom_sp)
    local clip_left_bp = sp_to_bp(obj.clip_left_sp)
    local clip_right_bp = sp_to_bp(obj.clip_right_sp)

    if obj.method == "clip" then
        kern_left.kern = -1 * obj.clip_left_sp
        kern_top.kern = -1 * obj.clip_top_sp
    end

    node.insert_after(kern_left,kern_left,box)
    box = node.hpack(kern_left)
    node.insert_after(kern_top,kern_top,box)
    box = node.vpack(kern_top)

    n_clip = node.new("whatsit","pdf_literal")
    setprop(n_clip,"origin","obj.clip")
    local n_clip, rule_clip
    rule_clip = {}
    if obj.method == "clip" then
        rule_clip[#rule_clip + 1] = string.format(" %g %g %g %g re W n ", 0,  -1 * dp + clip_bottom_bp, wd - clip_right_bp - clip_left_bp,ht+dp - clip_bottom_bp - clip_top_bp )
    elseif obj.method == "frame" then
        rule_clip[#rule_clip + 1] = string.format(" %g %g %g %g re W n ", clip_left_bp,  -1 * dp + clip_bottom_bp, wd - clip_right_bp - clip_left_bp,ht+dp - clip_bottom_bp - clip_top_bp )
    else
        err("Clip: method %s not implemented",obj.method)
    end

    n_clip = node.new("whatsit","pdf_literal")
    n_clip.data = table.concat(rule_clip, " ")
    node.setproperty(n_clip,{origin = "frame/clip (2)"})

    local pdf_save    = node.new("whatsit","pdf_save")
    local pdf_restore = node.new("whatsit","pdf_restore")

    node.insert_after(pdf_save,pdf_save,n_clip)
    node.insert_after(n_clip,n_clip,box)

    local hvbox = node.hpack(pdf_save)
    local savedp = hvbox.depth
    hvbox.depth = 0
    node.insert_after(hvbox,node.tail(hvbox),pdf_restore)
    hvbox = node.vpack(hvbox)
    if obj.method == "clip" then
        hvbox.width = hvbox.width - obj.clip_right_sp
        hvbox.height = hvbox.height - obj.clip_bottom_sp
    end
    hvbox.depth = savedp

    return hvbox
end

-- Set the PDF page-resources for the current page.
function setpageresources(thispage)
    if next(thispage.transparenttext) ~= nil and defaultcolorstack == 0 then
        transparentcolorstack()
    end
    -- thispage.transparenttext is something like { 40 = true, 20 = true}
    -- but only if we use alpha values for color
    local transparenttextresources = ""
    if defaultcolorstack ~= 0 then
        local tmp = {"/TRP1 << /CA 1 /ca 1 >>"}
        for k,_ in pairs(thispage.transparenttext) do
            tmp[#tmp + 1] = string.format("/TRP%s << /CA %g /ca %g >>",k,k/100,k/100)
        end
        transparenttextresources = table.concat(tmp,"")
    end
    local gstateresource = string.format(" /ExtGState << %s/GS0 %d 0 R /GS1 %d 0 R >>", transparenttextresources, GS_State_OP_On, GS_State_OP_Off)
    if table.not_empty(colors_module.used_spotcolors) then
        pdf.setpageresources("/ColorSpace << " .. spotcolors.getresource(colors_module.used_spotcolors) .. " >>" .. gstateresource )
    else
        pdf.setpageresources(gstateresource)
    end
end

--- Get PDF string for circle
---
--- ![Control points in the circle](img/circlepoints.svg)
---
function circle_pdfstring(center_x, center_y, radiusx_sp, radiusy_sp, stroke_colorstring, fill_colorstring, rulewidth_sp )
    local circle_bezier = 0.551915024494

    local shift_dn, shift_rt = -radiusy_sp + center_y, -radiusx_sp + center_x
    local dx = radiusx_sp * (1 - circle_bezier)
    local dy = radiusy_sp * (1 - circle_bezier)

    local x1 = shift_rt
    local y1 = shift_dn + radiusy_sp
    local x2 = x1
    local y2 = shift_dn + radiusy_sp * 2 - dy
    local x3 = shift_rt + dx
    local y3 = shift_dn + radiusy_sp * 2
    local x4 = shift_rt + radiusx_sp
    local y4 = shift_dn + radiusy_sp * 2
    local x5 = shift_rt + radiusx_sp * 2 - dx
    local y5 = y3
    local x6 = shift_rt + radiusx_sp * 2
    local y6 = y2
    local x7 = x6
    local y7 = y1
    local x8 = x6
    local y8 = shift_dn + dy
    local x9 = x5
    local y9 = shift_dn
    local x10 = x4
    local y10 = y9
    local x11 = x3
    local y11 = y9
    local x12 = x1
    local y12 = y8
    local circle = {}
    circle[#circle + 1] = "q"
    circle[#circle + 1] = string.format("%g w %s %s", sp_to_bp(rulewidth_sp), stroke_colorstring, fill_colorstring)
    circle[#circle + 1] = pdf_moveto(x1,y1)
    circle[#circle + 1] = pdf_curveto(x2,y2,x3,y3,x4,y4)
    circle[#circle + 1] = pdf_curveto(x5,y5,x6,y6,x7,y7)
    circle[#circle + 1] = pdf_curveto(x8,y8,x9,y9,x10,y10)
    circle[#circle + 1] = pdf_curveto(x11,y11,x12,y12,x1,y1)
    if fill_colorstring == "" then
        circle[#circle + 1] = "s"
    else
        circle[#circle + 1] = "b"
    end
    circle[#circle + 1] = "Q"
    return table.concat(circle, " ")
end

function circle_pdfstring_bp(center_x, center_y, radiusx_bp, radiusy_bp, stroke_colorstring, fill_colorstring, rulewidth_bp )
    local circle_bezier = 0.551915024494

    local shift_dn, shift_rt = -radiusy_bp + center_y, -radiusx_bp + center_x
    local dx = radiusx_bp * (1 - circle_bezier)
    local dy = radiusy_bp * (1 - circle_bezier)

    local x1 = shift_rt
    local y1 = shift_dn + radiusy_bp
    local x2 = x1
    local y2 = shift_dn + radiusy_bp * 2 - dy
    local x3 = shift_rt + dx
    local y3 = shift_dn + radiusy_bp * 2
    local x4 = shift_rt + radiusx_bp
    local y4 = shift_dn + radiusy_bp * 2
    local x5 = shift_rt + radiusx_bp * 2 - dx
    local y5 = y3
    local x6 = shift_rt + radiusx_bp * 2
    local y6 = y2
    local x7 = x6
    local y7 = y1
    local x8 = x6
    local y8 = shift_dn + dy
    local x9 = x5
    local y9 = shift_dn
    local x10 = x4
    local y10 = y9
    local x11 = x3
    local y11 = y9
    local x12 = x1
    local y12 = y8
    local circle = {}
    circle[#circle + 1] = "q"
    circle[#circle + 1] = string.format("%g w %s %s", rulewidth_bp, stroke_colorstring, fill_colorstring)
    circle[#circle + 1] = pdf_moveto_bp(x1,y1)
    circle[#circle + 1] = pdf_curveto_bp(x2,y2,x3,y3,x4,y4)
    circle[#circle + 1] = pdf_curveto_bp(x5,y5,x6,y6,x7,y7)
    circle[#circle + 1] = pdf_curveto_bp(x8,y8,x9,y9,x10,y10)
    circle[#circle + 1] = pdf_curveto_bp(x11,y11,x12,y12,x1,y1)
    if fill_colorstring == "" then
        circle[#circle + 1] = "s"
    else
        circle[#circle + 1] = "b"
    end
    circle[#circle + 1] = "Q"
    return table.concat(circle, " ")
end

--- Draw a circle
function circle( radiusx_sp, radiusy_sp, colorname,framecolorname,rulewidth_sp)
    if rulewidth_sp < 5 then
        framecolorname = colorname
    end
    local colentry = colors_module.get_colentry_from_name(colorname)
    if not colentry then
        err("Color %q unknown, reverting to black",colorname or "(no color name given)")
        colentry = colors_module.colors["black"]
    end
    local framecolentry = colors_module.get_colentry_from_name(framecolorname)

    if not framecolentry then
        err("Color %q unknown, reverting to black",framecolorname or "(no color name given)")
        framecolentry = colors_module.colors["black"]
    end
    local fillcolor   = colentry.pdfstring_fill
    local bordercolor = framecolentry.pdfstring_stroking

    local paint = node.new("whatsit","pdf_literal")
    paint.data = circle_pdfstring(0,0,radiusx_sp, radiusy_sp, bordercolor, fillcolor, rulewidth_sp)
    if colentry.alpha then
        setprop(paint,"opacity",colentry.alpha)
    end
    local v = node.vpack(paint)
    return v
end

function do_metapostimage(dataxml,txt,width,height,clip)
    metapostgraphics["_image"] = txt
    local cp = publisher.current_page
    local width_sp, height_sp
    if tonumber(width) then
        width_sp = cp.grid:width_sp(tonumber(width))
    else
        width_sp = tex.sp(width)
    end
    if tonumber(height) then
        height_sp = cp.grid:height_sp(tonumber(height))
    else
        height_sp = tex.sp(height)
    end

    local box, bbox = metapost.boxgraphic(width_sp,height_sp,"_image",{},{})
    if not box then
        main.log("error","Could not create metapost image")
        return
    end
    local image = {
        xsize = publisher.bp_to_sp(bbox[3] - bbox[1]),
        ysize = publisher.bp_to_sp(bbox[4] - bbox[2])
    }

    image.width = image.xsize
    image.height = image.ysize

    height    = set_image_length(dataxml,height,   "height") or image.height
    width     = set_image_length(dataxml,width,    "width" ) or image.width
    minheight = set_image_length(dataxml,minheight,"height") or 0
    minwidth  = set_image_length(dataxml,minwidth, "width" ) or 0
    maxheight = set_image_length(dataxml,maxheight,"height") or maxdimen
    maxwidth  = set_image_length(dataxml,maxwidth, "width" ) or maxdimen

    if not clip then
        width, height = calculate_image_width_height( image, width,height,minwidth,minheight,maxwidth, maxheight,stretch)

        box = node.hpack(box)
        box.width = width
        box.height = height
        box = node.vpack(box)

        local scaler      = node.new("whatsit","pdf_literal")
        local pdf_save    = node.new("whatsit","pdf_save")
        local pdf_restore = node.new("whatsit","pdf_restore")

        local scaleX = math.round(width / image.xsize,3)
        local scaleY = math.round(height / image.ysize,3)
        local shiftX = -bbox[1] * scaleX
        local shiftY = -bbox[2] * scaleY + sp_to_bp(height) * ( scaleY - 1 )
        local scalerdata = string.format("%g 0 0 %g %g %g cm ", scaleX,scaleY,shiftX,shiftY )
        scaler.data = scalerdata

        box.head = node.insert_before(box.head,box.head, scaler)
        box.head = node.insert_before(box.head,box.head,pdf_save)
        box.height = 0
        box.depth = 0
        box = node.vpack(box)
        node.insert_after(box.head,box.head,pdf_restore)
        box.height = height
    end

    return {box,nil}
end

function mpbox(parameter,width,height)
    local width_sp = width
    local height_sp = height
    local extra_parameter = {}

    extra_parameter.bordertopwidth = sp_to_bp(parameter.border_top_width) .. "bp"
    extra_parameter.borderbottomwidth = sp_to_bp(parameter.border_bottom_width) .. "bp"
    extra_parameter.borderleftwidth = sp_to_bp(parameter.border_left_width) .. "bp"
    extra_parameter.borderrightwidth = sp_to_bp(parameter.border_right_width) .. "bp"
    -- no width if style is none
    if parameter.border_top_style == "none" then
        extra_parameter.bordertopwidth = "0bp"
    end
    if parameter.border_bottom_style == "none" then
        extra_parameter.borderbottomwidth = "0bp"
    end
    if parameter.border_left_style == "none" then
        extra_parameter.borderleftwidth = "0bp"
    end
    if parameter.border_right_style == "none" then
        extra_parameter.borderrightwidth = "0bp"
    end

    extra_parameter.paddingtop = sp_to_bp(parameter.padding_top or 0) .. "bp"
    extra_parameter.paddingbottom = sp_to_bp(parameter.padding_bottom or 0) .. "bp"
    extra_parameter.paddingleft = sp_to_bp(parameter.padding_left or 0) .. "bp"
    extra_parameter.paddingright = sp_to_bp(parameter.padding_right or 0) .. "bp"

    extra_parameter.colors = {
        bordertopcolor = parameter.border_top_color,
        borderbottomcolor = parameter.border_bottom_color,
        borderleftcolor = parameter.border_left_color,
        borderrightcolor = parameter.border_right_color
    }
    extra_parameter.strings = {
        bordertopstyle = parameter.border_top_style,
        borderbottomstyle = parameter.border_bottom_style,
        borderleftstyle = parameter.border_left_style,
        borderrightstyle = parameter.border_right_style
    }
    extra_parameter.boolean = {
        debug = parameter.debug == true,
    }

    local mptext = [[
        wd = box.width  ;
        ht = box.height - bordertopwidth - borderbottomwidth ;
        if ht < 0: ht := 0; fi;
        if wd < 0: wd := 0; fi;
        z1 = (0,-borderbottomwidth - paddingbottom);
        x2 = borderleftwidth;
        x3 = x2 + wd + paddingleft + paddingright;
        x4 = x3 + borderrightwidth;

        y2 = y1 + borderbottomwidth;
        y3 = y2 + ht + borderbottomwidth + paddingtop + paddingbottom;
        y4 = y3 + bordertopwidth;

        % draw z1 -- (x4,y1) -- z4 -- (x1,y4) -- cycle;
        % draw z2 -- (x3,y2) -- z3 -- (x2,y3) -- cycle;

        picture border; border = nullpicture;

        path clip_top, clip_bottom, clip_left, clip_right;
        clip_top = (x1,y4) -- (x2,y3) -- z3 -- z4 -- cycle;
        clip_bottom = (x1,y1) -- (x2,y2) -- (x3,y2) -- (x4,y1) -- cycle;
        clip_left = (x1,y1) -- (x2,y2) -- (x2,y3) -- (x1,y4) -- cycle;
        clip_right = (x4,y1) -- (x3,y2) -- (x3,y3) -- (x4,y4) -- cycle;
        def isdarkcolor(expr c) =
            (redpart c < 0.2) and (greenpart c < 0.2) and (bluepart c < 0.2 )
        enddef;

        def drawborder(expr bordercolor, bwd, style, a, b, clippath, pos) =
            begingroup;
            interim linecap := butt;

            color col; col = bordercolor;
            string str;
            str = "withcolor col withpen pencircle scaled " & decimal bwd ;
            if style = "dashed":
                str := str & " dashed dashpattern(on 4bp off 5bp)"
            elseif ( style = "inset" )  and  (  (pos = "top" )  or (pos = "left") ):
                if isdarkcolor(col):
                    str := str & " withcolor 0.2[col, white] ";
                else:
                    str := str & " withcolor 0.5[col, black] ";
                fi;
            elseif ( style = "inset" ) and isdarkcolor(col) and ( (pos = "bottom" ) or (pos = "right") ):
                str := str & " withcolor 0.5[col, white] ";
            elseif ( style = "outset" )  and  (  (pos = "right" ) or (pos = "bottom") ):
                str := str & " withcolor 0.5[col, black] ";
            fi;
            drawoptions(scantokens(str));

            draw a -- b ;
            clip currentpicture to clippath ;
            addto border also currentpicture ;
            endgroup;
        enddef;

        y34 = 0.5[y3,y4];
        y12 = 0.5[y1,y2];
        x12 = 0.5[x1,x2];
        x34 = 0.5[x3,x4];

        drawborder(bordertopcolor,bordertopwidth, bordertopstyle, (x1,y34),(x4,y34), clip_top, "top" );
        drawborder(borderbottomcolor,borderbottomwidth,borderbottomstyle, (x1,y12),(x4,y12), clip_bottom, "bottom");
        drawborder(borderleftcolor,borderleftwidth, borderleftstyle,(x12,y1),(x12,y4), clip_left, "left" );
        drawborder(borderrightcolor,borderrightwidth, borderrightstyle,(x34,y1),(x34,y4), clip_right, "right" );

        currentpicture := border;
        if debug:
            drawoptions(scaled 3bp withcolor black );
            drawdot(z1) withcolor blue;
            drawdot(z2) withcolor red;
            drawdot(z3) withcolor red;
            drawdot(z4) withcolor blue;
        fi;
    ]]
    metapostgraphics.__htmlbox = mptext

    local instr,bbox = metapost.boxgraphic(width_sp,height_sp,"__htmlbox",extra_parameter)
    if not instr then
        main.log("error","Could not create metapost image")
        return
    end

    local ret = node.hpack(instr,width,"exactly")
    ret.height = height
    if parameter.shiftdown then
        ret.height = ret.height + parameter.shiftdown
    end
    ret = node.vpack(ret)

    node.set_attribute(ret,att_dontadjustlineheight,1)
    ret.height = 0
    ret.depth = 0
    ret.shift = parameter.margin_left
    return ret
end

--- Create a colored area. width and height are in scaled points.
--- Optional border_color (color name) and border_width (in sp) draw a stroke
--- rectangle at the outer edge; the fill area shrinks inward by border_width.
function box( width_sp,height_sp,colorname,border_color,border_width_sp )
    local h,v
    local _width   = sp_to_bp(width_sp)
    local _height  = sp_to_bp(height_sp)
    local pdfcmds = {}

    -- Border stroke (full size, centered on edge → offset inward by half line width)
    if border_color and border_width_sp and border_width_sp > 0 then
        local bw = sp_to_bp(border_width_sp)
        local half = bw / 2
        local border_colentry = colors_module.colors[border_color]
        if not border_colentry then
            err("Color %q unknown, reverting to black",border_color or "(no color name given)")
            border_colentry = colors_module.colors["black"]
        end
        pdfcmds[#pdfcmds+1] = string.format("q %s %g w %g %g %g %g re S Q",
            border_colentry.pdfstring, bw,
            half, -half, _width - bw, -(_height - bw))
    end

    -- Fill (shrunk by border width on each side)
    if colorname ~= "-" then
        local colentry = colors_module.colors[colorname]
        if not colentry then
            err("Color %q unknown, reverting to black",colorname or "(no color name given)")
            colentry = colors_module.colors["black"]
        end
        local bw = border_width_sp and sp_to_bp(border_width_sp) or 0
        local fill_x = bw
        local fill_y = -bw
        local fill_w = _width - 2 * bw
        local fill_h = _height - 2 * bw
        if fill_w > 0 and fill_h > 0 then
            pdfcmds[#pdfcmds+1] = string.format("q %s 1 0 0 1 0 0 cm %g %g %g %g re f Q",
                colentry.pdfstring, fill_x, fill_y, fill_w, -fill_h)
        end
    end

    if #pdfcmds > 0 then
        local paint = node.new("whatsit","pdf_literal")
        setprop(paint,"role",get_rolenum("Artifact"))
        paint.data = table.concat(pdfcmds, " ")
        paint.mode = 0
        if colorname ~= "-" then
            local colentry = colors_module.colors[colorname]
            if colentry and colentry.alpha then
                set_attribute(paint,"color",colors_module.get_colorindex_from_name(colorname))
            end
        end
        local hglue = set_glue(nil,{width = 0, stretch = 2^16, stretch_order = 3 })
        h = node.insert_after(paint,paint,hglue)
        h = node.hpack(h,width_sp,"exactly")
    else
        h = create_empty_hbox_with_width(width_sp)
    end

    local vglue = set_glue(nil,{width = 0, stretch = 2^16, stretch_order = 3 })
    v = node.insert_after(h,h,vglue)
    v = node.vpack(h,height_sp,"exactly")
    return v
end


-- Draw a box with HTML properties given at head
function htmlbox(dirmode, head, width_sp, height_sp, depth_sp)
    local debug_htmlbox = 0
    local properties = node.getproperty(head)
    if not properties then
        err("Internal error: htmlbox() - no properties given")
        return
    end
    local rules = {}
    rules[#rules + 1] = "q"
    --- We start with 4 trapezoids (1 for each border). Later on clip paths are added.
    ---
    ---      4    4------------------------------3   3  y0
    ---      |\    \                            /   /|
    ---      | \    \                          /   / |
    ---      |  \    \                        /   /  |
    ---      |   \    \                      /   /   |
    ---      |    \    \                    /   /    |
    ---      |     3    1------------------2   4     |  y1
    ---      |     |                           |     |
    ---      |     |                           |     |
    ---      |     |                           |     |
    ---      |     |                           |     |
    ---      |     |                           |     |
    ---      |    2    4--------------------3   1    |  y2
    ---      |   /    /                      \   \   |
    ---      |  /    /                        \   \  |
    ---      | /    /                          \   \ |
    ---      |/    /                            \   \|
    ---      1    /                              \   2  y3
    ---          1--------------------------------2
    ---      x0      x1                       x2     x3
    ---
    --- Baseline is at 0
    --- depth is negative downwards
    --- height is positive upwards
    local colorstring

    local function get_rule(x1, y1, x2, y2, x3, y3, x4, y4)
        local _x1, _y1 = sp_to_bp(x1), sp_to_bp(y1)
        local _x2, _y2 = sp_to_bp(x2), sp_to_bp(y2)
        local _x3, _y3 = sp_to_bp(x3), sp_to_bp(y3)
        local _x4, _y4 = sp_to_bp(x4), sp_to_bp(y4)
        local ret = string.format("%s 0 w %g %g m %g %g l %g %g l %g %g l b", colorstring, _x1, _y1, _x2, _y2, _x3, _y3, _x4, _y4)
        return ret
    end

    local b_b_r_radius = properties.border_bottom_right_radius
    local b_b_l_radius = properties.border_bottom_left_radius
    local b_t_r_radius = properties.border_top_right_radius
    local b_t_l_radius = properties.border_top_left_radius

    local border_top_width    = properties.border_top_width
    local border_right_width  = properties.border_right_width
    local border_bottom_width = properties.border_bottom_width
    local border_left_width   = properties.border_left_width

    local padding_top    = properties.padding_top
    local padding_bottom = properties.padding_bottom
    local padding_left   = properties.padding_left

    local margin_left   = properties.margin_left or 0
    -- ht == y3, wd == x3
    depth_sp = math.max(depth_sp,properties.depth or 0)
    height_sp = properties.lineheight - depth_sp
    local sp_x0, sp_x1, sp_x2, wd
    local sp_y0, sp_y1, sp_y2, ht

    if dirmode == "horizontal" then
        local content_top    = height_sp  + (properties.shiftdown or 0)
        local content_bottom = -depth_sp  + (properties.shiftdown or 0)

        sp_y0 = content_top + padding_top + border_top_width
        sp_y1 = content_top + padding_top
        ht    = content_bottom - padding_bottom - border_bottom_width
        sp_y2 = content_bottom - padding_bottom

        sp_x0 = -1 * (padding_left + border_left_width)
        sp_x1 = sp_x0 + border_left_width
        sp_x2 = sp_x1 + width_sp
        wd    = sp_x2 + border_right_width
    else
        local content_top    = height_sp
        local content_bottom = -depth_sp

        -- vertical
        sp_y0 = content_top + padding_top + border_top_width
        sp_y1 = content_top + padding_top
        ht    = content_bottom - padding_bottom - border_bottom_width
        sp_y2 = content_bottom - padding_bottom

        -- horizontal
        local content_left  = properties.shiftright or 0
        local content_right = width_sp + (properties.shiftright or 0) + margin_left

        sp_x0 = content_left - padding_left - border_left_width
        sp_x1 = content_left - padding_left
        sp_x2 = content_right + padding_left
        wd    = sp_x2 + border_right_width
    end

    --- The trapezoids must extend closer to the center of the border, because if the border
    --- radius is larger than the border width, the border goes "into" the surrounding object.
    -- 3 might not be correct. TODO: what is the correct factor? Should depend on the radius
    local extend_top = 0
    local extend_right = 0
    local extend_bottom = 0
    local extend_left = 0
    if b_t_l_radius > 0 or b_t_r_radius > 0 then
        extend_top = 3
    end
    if b_t_r_radius > 0 or b_b_r_radius > 0 then
        extend_right = 3
    end
    if b_b_l_radius > 0 or b_b_r_radius > 0 then
        extend_bottom = 3
    end
    if b_t_l_radius > 0 or b_b_l_radius > 0 then
        extend_left = 3
    end
    local inner_top = sp_y1 - extend_top *  border_top_width
    local inner_right = sp_x2  - extend_right * border_right_width
    local inner_bottom = sp_y2 + extend_bottom * border_bottom_width
    local inner_left = sp_x1 + extend_left * border_left_width

    if properties.border_top_style ~= "none" and border_top_width > 0 then
        colorstring = colors_module.colors[properties.border_top_color].pdfstring
        rules[#rules + 1] = get_rule(inner_left,inner_top, inner_right, inner_top, wd, sp_y0, sp_x0, sp_y0)
    end
    if properties.border_right_style ~= "none" and border_right_width > 0 then
        colorstring = colors_module.colors[properties.border_right_color].pdfstring
        rules[#rules + 1] = get_rule(inner_right,inner_bottom, wd, ht, wd, sp_y0, inner_right,inner_top)
    end
    if properties.border_bottom_style ~= "none" and border_bottom_width > 0 then
        colorstring = colors_module.colors[properties.border_bottom_color].pdfstring
        rules[#rules + 1] = get_rule(sp_x0, ht, wd, ht,inner_right,inner_bottom , inner_left,inner_bottom)
    end
    if properties.border_left_style ~= "none" and border_left_width > 0 then
        colorstring = colors_module.colors[properties.border_left_color].pdfstring
        rules[#rules + 1] = get_rule(sp_x0, ht, inner_left, inner_bottom, inner_left, inner_top, sp_x0, sp_y0)
    end
    rules[#rules + 1] = "Q"

    -- Let's calculate the outer boundary first.
    local circle_bezier = 0.551915024494

    -- xn, yn = outer path, xin, yin = inner path used for clipping
    local x0, y0   = sp_x0                              , ht + b_b_l_radius
    local x1, y1   = sp_x0 + b_b_l_radius   , ht
    local x2, y2   = wd - b_b_r_radius                , y1
    local x3, y3   = x2 + circle_bezier * b_b_r_radius, y1
    local x5, y5   = wd                               , ht + b_b_r_radius
    local x4, y4   = wd                               , y5 - circle_bezier * b_b_r_radius
    local x6, y6   = wd                               , sp_y0 - b_t_r_radius
    local x7, y7   = wd                               , y6 + b_t_r_radius * circle_bezier
    local x9, y9   = wd - b_t_r_radius                , sp_y0
    local x8, y8   = x9 + circle_bezier * b_t_r_radius, y9
    local x10, y10 = sp_x0 + b_t_l_radius   , y9
    local x11, y11 = x10 - circle_bezier * b_t_l_radius, y9
    local x13, y13 = sp_x0                              ,y9 - b_t_l_radius
    local x12, y12 = sp_x0                             ,y13 + circle_bezier * b_t_l_radius
    local x14, y14 = sp_x0                             , y1 + b_b_l_radius
    local x15, y15 = sp_x0                             , y14 - b_b_l_radius * circle_bezier
    local x16, y16 = x1 - circle_bezier * b_b_l_radius, y1

    local b_b_r_inner_radius_x = math.max(0, b_b_r_radius - border_right_width )
    local b_b_r_inner_radius_y = math.max(0, b_b_r_radius - border_bottom_width )
    local b_b_l_inner_radius_x = math.max(0, b_b_l_radius - border_left_width )
    local b_b_l_inner_radius_y = math.max(0, b_b_l_radius - border_bottom_width )
    local b_t_r_inner_radius_x = math.max(0, b_t_r_radius - border_right_width )
    local b_t_r_inner_radius_y = math.max(0, b_t_r_radius - border_top_width )
    local b_t_l_inner_radius_x = math.max(0, b_t_l_radius - border_left_width)
    local b_t_l_inner_radius_y = math.max(0, b_t_l_radius - border_top_width)

    -- bottom left
    local xi14, yi14 = sp_x0 + border_left_width             ,math.max(ht + border_bottom_width, y14)
    local xi1, yi1   = math.max(x1,sp_x0 + border_left_width),ht + border_bottom_width

    -- bottom right
    local xi2, yi2   = math.min(x2,wd - border_right_width), yi1
    local xi5, yi5   = wd - border_right_width             ,  math.max(y5,ht + border_bottom_width)

    -- top right
    local xi6, yi6   = xi5,math.min(y6, sp_y0 - border_top_width)
    local xi9, yi9   = math.min(x9,wd - border_right_width), math.min(y9, sp_y0 - border_top_width)

    -- top left
    local xi10, yi10 = math.max(sp_x0 + border_left_width,x10),math.min(sp_y0 - border_top_width,y10)
    local xi13, yi13 = math.max(sp_x0 + border_left_width,x13),math.min(sp_y0 - border_top_width,y13 )

    -- control points
    -- bottom left
    local xi16, yi16 = xi1 - circle_bezier * b_b_l_inner_radius_x,yi1
    local xi15, yi15 = xi14, yi14 - circle_bezier * b_b_l_inner_radius_y

    -- bottom right
    local xi3, yi3   = xi2 + circle_bezier * b_b_r_inner_radius_x ,  yi2
    local xi4, yi4   = xi5, yi5 - circle_bezier * b_b_r_inner_radius_y

    -- top right
    local xi7, yi7   = xi6,yi6 + circle_bezier * b_t_r_inner_radius_y
    local xi8, yi8   = xi9 + circle_bezier * b_t_r_inner_radius_x ,yi9

    -- top left
    local xi11, yi11 = xi10 - circle_bezier * b_t_l_inner_radius_x ,yi10
    local xi12, yi12 = xi13 ,yi13 + circle_bezier *  b_t_l_inner_radius_y

    if debug_htmlbox > 1 then
        rules[#rules + 1] = circle_pdfstring(x1 ,y14,b_b_l_radius,b_b_l_radius,"0 G ","",1000 )
        rules[#rules + 1] = circle_pdfstring(x2 ,y5,b_b_r_radius,b_b_r_radius,"0 G ","",1000 )
        rules[#rules + 1] = circle_pdfstring(x9 ,y6,b_t_r_radius,b_t_r_radius,"0 G ","",1000 )
        rules[#rules + 1] = circle_pdfstring(x10 ,y13,b_t_l_radius,b_t_l_radius,"0 G ","",1000 )

        rules[#rules + 1] = circle_pdfstring(xi1 ,yi14,b_b_l_inner_radius_x,b_b_l_inner_radius_y,"0 G ","",1000 )
        rules[#rules + 1] = circle_pdfstring(xi2 ,yi5,b_b_r_inner_radius_x,b_b_r_inner_radius_y,"0 G ","",1000 )
        rules[#rules + 1] = circle_pdfstring(xi9 ,yi6,b_t_r_inner_radius_x,b_t_r_inner_radius_y,"0 G ","",1000 )
        rules[#rules + 1] = circle_pdfstring(xi10 ,yi13,b_t_l_inner_radius_x,b_t_l_inner_radius_y,"0 G ","",1000 )
    end


    local rules_clip = {}

    rules_clip[#rules_clip + 1] = pdf_moveto(x1,y1)
    rules_clip[#rules_clip + 1] = pdf_lineto(x2,y2)
    rules_clip[#rules_clip + 1] = pdf_curveto(x3,y3,x4,y4,x5,y5)
    rules_clip[#rules_clip + 1] = pdf_lineto(x6,y6)
    rules_clip[#rules_clip + 1] = pdf_curveto(x7,y7,x8,y8,x9,y9)
    rules_clip[#rules_clip + 1] = pdf_lineto(x10,y10)
    rules_clip[#rules_clip + 1] = pdf_curveto(x11,y11,x12,y12,x13,y13)
    rules_clip[#rules_clip + 1] = pdf_lineto(x14,y14)
    rules_clip[#rules_clip + 1] = pdf_curveto(x15,y15,x16,y16,x1,y1)

    rules_clip[#rules_clip + 1] = pdf_moveto(xi1,yi1)
    rules_clip[#rules_clip + 1] = pdf_lineto(xi2,yi2)
    rules_clip[#rules_clip + 1] = pdf_curveto(xi3,yi3,xi4,yi4,xi5,yi5)
    rules_clip[#rules_clip + 1] = pdf_lineto(xi6,yi6)
    rules_clip[#rules_clip + 1] = pdf_curveto(xi7,yi7,xi8,yi8,xi9,yi9)
    rules_clip[#rules_clip + 1] = pdf_lineto(xi10,yi10)
    rules_clip[#rules_clip + 1] = pdf_curveto(xi11,yi11,xi12,yi12,xi13,yi13)
    rules_clip[#rules_clip + 1] = pdf_lineto(xi14,yi14)
    rules_clip[#rules_clip + 1] = pdf_curveto(xi15,yi15,xi16,yi16,xi1,yi1)

    if debug_htmlbox > 0 then
        rules[#rules + 1] = "q 0.3 w"
        rules[#rules + 1] = pdf_moveto(x0,0)
        rules[#rules + 1] = pdf_lineto(wd,0)
        rules[#rules + 1] = "S"
        rules[#rules + 1] = pdf_moveto(x0,-depth_sp)
        rules[#rules + 1] = pdf_lineto(wd,-depth_sp)
        rules[#rules + 1] = "S"
        rules[#rules + 1] = pdf_moveto(x0,height_sp)
        rules[#rules + 1] = pdf_lineto(wd,height_sp)
        rules[#rules + 1] = "S"
        rules[#rules + 1] = "Q"
    end

    if debug_htmlbox > 1 then
        rules[#rules + 1] = "q 0.3 w"
        rules[#rules + 1] = pdf_moveto(x1,y1)
        rules[#rules + 1] = pdf_lineto(x2,y2)
        rules[#rules + 1] = pdf_curveto(x3,y3,x4,y4,x5,y5)
        rules[#rules + 1] = pdf_lineto(x6,y6)
        rules[#rules + 1] = pdf_curveto(x7,y7,x8,y8,x9,y9)
        rules[#rules + 1] = pdf_lineto(x10,y10)
        rules[#rules + 1] = pdf_curveto(x11,y11,x12,y12,x13,y13)
        rules[#rules + 1] = pdf_lineto(x14,y14)
        rules[#rules + 1] = pdf_curveto(x15,y15,x16,y16,x1,y1)
        rules[#rules + 1] = "S"

        rules[#rules + 1] = pdf_moveto(xi1,yi1)
        rules[#rules + 1] = pdf_lineto(xi2,yi2)
        rules[#rules + 1] = pdf_curveto(xi3,yi3,xi4,yi4,xi5,yi5)
        rules[#rules + 1] = pdf_lineto(xi6,yi6)
        rules[#rules + 1] = pdf_curveto(xi7,yi7,xi8,yi8,xi9,yi9)
        rules[#rules + 1] = pdf_lineto(xi10,yi10)
        rules[#rules + 1] = pdf_curveto(xi11,yi11,xi12,yi12,xi13,yi13)
        rules[#rules + 1] = pdf_lineto(xi14,yi14)
        rules[#rules + 1] = pdf_curveto(xi15,yi15,xi16,yi16,xi1,yi1)
        rules[#rules + 1] = "S Q"
    end

    rules_clip[#rules_clip + 1] = "h W* n"

    local n_clip = node.new("whatsit","pdf_literal")
    setprop(n_clip,"origin","htmlbox.clip")
    local n_clip_data = table.concat(rules_clip," ")
    local concat_rules = table.concat(rules, " ")
    n_clip_data = n_clip_data .. " " .. concat_rules
    n_clip.data = n_clip_data

    local pdf_save    = node.new("whatsit","pdf_save")
    local pdf_restore = node.new("whatsit","pdf_restore")

    node.insert_after(pdf_save,pdf_save,n_clip)

    local hvbox = node.hpack(pdf_save)
    hvbox.depth = 0
    node.insert_after(hvbox,node.tail(hvbox),pdf_restore)
    hvbox = node.vpack(hvbox)
    node.setproperty(hvbox,{origin="hvbox"})

    if dirmode == "horizontal" then
        return hvbox
    end
    local vbox = node.vpack(hvbox)
    local shiftdown = properties.shiftdown or 0
    local g = set_glue(nil, { width = shiftdown})
    vbox.head = node.insert_before(vbox.head,vbox.head,g)
    vbox.height = 0
    vbox.depth = 0
    return vbox
end

--- After everything is ready for page ship-out, we add debug output and crop marks if necessary
function dothingsbeforeoutput( thispage,data )
    local cg = thispage.grid

    if thispage and thispage.AtPageShipout then
        pagebreak_impossible = true
        dispatch(thispage.AtPageShipout,data)
        pagebreak_impossible = false
        local graphic = thispage.AtPageShipout.graphic
        if graphic then
            local _,whatsit = metapost.prepareboxgraphic(thispage.width,thispage.height,graphic,metapost.extra_page_parameter(thispage))
            place_at(thispage.pagebox,whatsit,thispage.grid.extra_margin,thispage.height+thispage.grid.extra_margin)
        end
    end

    local nodelist = thispage.pagebox
    local rules = insert_nonmoving_whatsits(nodelist,nil,"vertical",0,0,thispage.width,thispage.height)
    for _, rule in pairs(rules) do
        -- @class whatsit_node
        local wr = node.new("whatsit","pdf_literal")
        wr.data = rule[3]
        wr.mode = 0
        setprop(wr,"origin","tr-later")
        output_absolute_position({x = rule[1], y = rule[2], nodelist = wr})
    end

    local firstbox

    -- for spot colors, if necessary
    setpageresources(thispage)

    -- White background on page. Todo: Make color customizable and background optional.
    local wd = sp_to_bp(current_page.width)
    local ht = sp_to_bp(current_page.height)

    local x = 0 + current_page.grid.extra_margin
    local y = 0 + current_page.grid.extra_margin

    if options.trim then
        local trim_bp = sp_to_bp(options.trim)
        wd = wd + trim_bp * 2
        ht = ht + trim_bp * 2
        x = x - options.trim
        y = y - options.trim
    end

    -- White background
    if ( options.background and options.background ~= '-' ) or ( thispage.backgroundcolor and thispage.backgroundcolor ~= "-" ) then
        local col = thispage.backgroundcolor or options.background
        if col == "-" then
            goto skipbgcolor
        end
        local colentry = colors_module.get_colentry_from_name(col ,"white")
        if not colentry then
            main.log("error","Color is not defined","name",tostring(options.backgroun))
            colentry = colors_module.colors["white"]
        end
        local pdfcolorstring = colentry.pdfstring

        firstbox = node.new("whatsit","pdf_literal")
        firstbox.data = string.format("q %s 1 0 0 1 0 0 cm %g %g %g %g re f Q",pdfcolorstring, sp_to_bp(x), sp_to_bp(y),wd ,ht)
        firstbox.mode = 1
    end
    ::skipbgcolor::

    if options.showgridallocation then
        local lit = node.new("whatsit","pdf_literal")
        lit.mode = 1
        lit.data = cg:draw_gridallocation()

        if firstbox then
            local tail = node.tail(firstbox)
            tail.next = lit
            lit.prev = tail
        else
            firstbox = lit
        end
    end

    for framename,v in pairs(cg.positioning_frames) do
        for _,frame in ipairs(v) do
            if frame.draw then
                local lit = node.new("whatsit","pdf_literal")
                lit.mode = 1
                lit.data = cg:draw_frame(frame,tex.sp(frame.draw.width))
                if firstbox then
                    local tail = node.tail(firstbox)
                    tail.next = lit
                    lit.prev = tail
                else
                    firstbox = lit
                end
            end
        end
    end

    if options.showgrid and options.gridlocation == "background" then
        local lit = node.new("whatsit","pdf_literal")
        lit.mode = 1
        lit.data = cg:draw_grid()
        if firstbox then
            local tail = node.tail(firstbox)
            tail.next = lit
            lit.prev = tail
        else
            firstbox = lit
        end
    end

    if options.cutmarks then
        if not pro then
            err("cutmarks need a pro plan")
            publisher.has_pro_error = true
        else
            local lit = node.new("whatsit","pdf_literal")
            lit.mode = 1
            lit.data = cg:cutmarks()
            if firstbox then
                local tail = node.tail(firstbox)
                tail.next = lit
                lit.prev = tail
            else
                firstbox = lit
            end
        end
    end

    if options.trimmarks then
        local lit = node.new("whatsit","pdf_literal")
        lit.mode = 1
        lit.data = cg:trimmarks()
        if firstbox then
            local tail = node.tail(firstbox)
            tail.next = lit
            lit.prev = tail
        else
            firstbox = lit
        end
    end

    if firstbox then
        local list_start = nodelist
        thispage.pagebox = firstbox
        node.tail(firstbox).next = list_start
        list_start.prev = node.tail(firstbox)
    end
end

function dothingsafteroutput(thispage,nodelist)
    if options.showgrid and options.gridlocation == "foreground" then
        local cg = thispage.grid
        local lit = node.new("whatsit","pdf_literal")
        lit.mode = 1
        lit.data = cg:draw_grid()
        node.insert_after(nodelist,nodelist,lit)
        nodelist = node.vpack(nodelist)
    end
    return nodelist
end

--- Read the contents of the attribute `attname`. `typ` is one of
--- `string`, `number`, `length` and `boolean`.
--- `default` gives something that is to be returned if no attribute with this name is present.
function read_attribute( layoutxml,dataxml,attname,typ,default,context)
    local namespaces = layoutxml[".__ns"]

    local attr

    if newxpath then
        local attributes = layoutxml[".__attributes"]
        if not attributes then
            return default
        end
        if not attributes[attname] then
            return default
        end
        attr = attributes[attname]
    else
        if not layoutxml[attname] then
            return default -- can be nil
        end
        attr = layoutxml[attname]
    end

    local val,num,ret
    if typ ~= "xpath" and typ ~= "xpathraw" and typ ~= "rawstring" then
        val = string.gsub(attr,"{(.-)}", function (x)
            if newxpath then
                local copysequence = dataxml.sequence
                local seq, msg = dataxml:eval(x)
                if msg then
                    err(msg)
                    return nil
                end
                local txt
                txt, msg = xpath.string_value(seq)
                if msg then
                    err(msg)
                    return nil
                end
                dataxml.sequence = copysequence
                return txt
            else
                local ok, xp = xpath.parse_raw(dataxml,x,namespaces)
                if not ok then
                    err(xp)
                    return nil
                end
                return xpath.textvalue(xp[1])
            end
        end)
    else
        val = attr
    end

    if val == "nil" then val = nil end
    if typ=="xpath" then
        if newxpath then
            local seq, msg = dataxml:eval(val)
            if msg then
                err(msg)
                return nil
            end
            return xpath.string_value(seq)
        else
            return xpath.textvalue(xpath.parse(dataxml,val,namespaces))
        end
    elseif typ=="xpathraw" then
        if newxpath then
            local seq, msg = dataxml:eval(val)
            if msg then
                err(msg)
                return nil
            end
            return seq
        else
            local ok,tmp = xpath.parse_raw(dataxml,val,namespaces)
            if not ok then err(tmp)
                return nil
            else
                return tmp
            end
        end
    elseif typ=="string" or typ == "rawstring" then
        return tostring(val or default)
    elseif typ=="number" then
        return tonumber(val)
        -- something like "3pt"
    elseif typ=="length" then
        return val
        -- same as before, just changed to scaled points
    elseif typ=="length_sp" then
        num = tonumber(val or default)
        if num == 0 then return 0 end
        if num then -- most likely really a number, we need to multiply with grid width
            ret = current_grid:width_sp(num)
        else
            ret = val
        end
        if not ret then return end
        return tex.sp(ret)
    elseif typ=="height_sp" then
        num = tonumber(val or default)
        if num then -- most likely really a number, we need to multiply with grid height
            setup_page(nil,"read_attribute height_sp",dataxml)
            ret = current_page.grid.gridheight * num
        else
            ret = val
        end
        if not ret then return end
        return tex.sp(ret)
    elseif typ=="width_sp" then
        num = tonumber(val or default)
        if num then -- most likely really a number, we need to multiply with grid width
            setup_page(nil,"read_attribute width_sp",dataxml)
            ret = current_page.grid:width_sp(num)
        else
            ret = val
        end
        if not ret then return end
        return tex.sp(ret)
    elseif typ=="boolean" then
        val = val or default
        if val=="yes" then
            return true
        elseif val=="no" then
            return false
        end
        return nil
    elseif typ=="booleanornumber" then
        val = val or default
        if val=="yes" then
            return true
        elseif val=="no" then
            return false
        else
            return tonumber(val)
        end
    elseif typ=="booleanorlength" then
        val = val or default
        if val=="yes" then
            return true
        elseif val=="no" then
            return false
        else
            return tex.sp(val)
        end
    else
        warning("read_attribute (2): unknown type: %s",type(val))
    end
    return val
end

-- Return the element name of the given element (elt)
function elementname(elt)
    if not elt then
        main.log("error","Could not get element name",lineinfo())
        return nil
    end
    return elt.elementname
end

--- Return the contents of an entry from the `dispatch()` function call.
function element_contents( elt )
    return elt.contents
end

-- To split the textblock in pieces
local marker
marker = node.new("whatsit","user_defined")
marker.user_id = user_defined_marker
marker.type = 100  -- type 100: "value is a number"
marker.value = 1

--- Convert `<b>`, `<u>` and `<i>` in text to publisher recognized elements.
-- The 'new' HTML parse is in the file html.lua
function parse_html(elt, parameter, data)
    parameter = parameter or {}

    if elt.typ == "csshtmltree" then
        -- Provide the runtime environment for the modular parser
        local env = {
            publisher = publisher,
            tex       = tex,
            node      = node,
            img       = img,
            xpath     = xpath,
            math      = math,
            err       = err,
        }
        -- New modular entry point (same call site for you)
        return html.parse_html_new(elt, parameter, data, env)
    else
        err("This should not happen (parse_html)")
    end
end

-- node.direct API locals for hot loops
local d              = node.direct
local d_todirect     = d.todirect
local d_tonode       = d.tonode
local d_getnext      = d.getnext
local d_getid        = d.getid
local d_new          = d.new
local d_setkern      = d.setkern
local d_insert_before = d.insert_before
local d_has_attribute = d.has_attribute
local d_set_attribute = d.set_attribute
local d_getproperty  = d.getproperty
local d_setproperty  = d.setproperty
local d_setfield     = d.setfield
local d_getlist      = d.getlist
local d_dimensions   = d.dimensions

-- Hot-path locals to avoid repeated global lookups
local attribute_name_number_map = attribute_name_number
local attributes_map = attributes

-- Pre-resolved attribute numbers for hot loops (setstyles, hbglyphlist, hbkern)
local att_fontfamily    = attribute_name_number["fontfamily"]
local att_color         = attribute_name_number["color"]
local att_bgcolor       = attribute_name_number["background-color"]
local att_bgpadtop      = attribute_name_number["bgpaddingtop"]
local att_bgpadbot      = attribute_name_number["bgpaddingbottom"]
local att_hyperlink     = attribute_name_number["hyperlink"]
local att_spaceglue     = attribute_name_number["spaceglue"]
local att_newline       = attribute_name_number["newline"]
local att_fontstyle     = attribute_name_number["font-style"]
local att_fontweight    = attribute_name_number["font-weight"]
local att_td_line       = attribute_name_number["text-decoration-line"]
local att_td_style      = attribute_name_number["text-decoration-style"]
local att_td_color      = attribute_name_number["text-decoration-color"]
local att_verticalalign = attribute_name_number["vertical-align"]

-- Pre-resolved table-valued attribute indices (reverse lookup: string -> index)
local attval_fontstyle = {}
for i,v in ipairs(attributes["font-style"]) do attval_fontstyle[v] = i end
local attval_fontweight = {}
for i,v in ipairs(attributes["font-weight"]) do attval_fontweight[v] = i end
local attval_td_line = {}
for i,v in ipairs(attributes["text-decoration-line"]) do attval_td_line[v] = i end
local attval_td_style = {}
for i,v in ipairs(attributes["text-decoration-style"]) do attval_td_style[v] = i end
local attval_verticalalign = {}
for i,v in ipairs(attributes["vertical-align"]) do attval_verticalalign[v] = i end

function get_attributes(nodelist, reuse_table)
    local attributes = reuse_table or {}
    if reuse_table then
        for k in pairs(reuse_table) do
            reuse_table[k] = nil
        end
    end
    local n = nodelist and nodelist.attr
    while n do
        local num = n.number
        if num then
            attributes[num] = n.value
        end
        n = n.next
    end
    return attributes
end

-- Get an attribute value. If the attribute table entry has a table, return the
-- string value of the attribute
function get_attribute(nodelist,attribute_name)
    if not nodelist then return nil end
    local att_number = attribute_name_number_map[attribute_name]
    if not att_number then return nil end
    local entry = attributes_map[attribute_name]
    local val = node.has_attribute(nodelist, att_number)
    if val == nil then return nil end
    if type(entry) == "table" then
        return entry[val]
    end
    return val
end

-- set_attribute sets an attribute for this node. The attribute name must be
-- present in the global attribute list or a number from the list attribute_name_number.
-- The value must be a number or a string value
function set_attribute(nodelist,attribute_name,value)
    local att_number = attribute_name_number_map[attribute_name]
    if not att_number then err("Internal error: attribute %s unknown",attribute_name or "?") return end
    local entry = attributes_map[attribute_name]
    local att_value
    if type(entry) == "table" then
        for k,v in ipairs(entry) do
            if v == value then att_value = k break end
        end
    else
        att_value = value
    end
    if att_value == nil then
        node.unset_attribute(nodelist,att_number)
    else
        node.set_attribute(nodelist,att_number,att_value)
    end
end

function clear_attribute(nodelist,attribute_name)
    local att_number = attribute_name_number_map[attribute_name]
    if not att_number then err("Internal error: attribute %s unknown",attribute_name or "?") return end
    node.unset_attribute(nodelist,att_number)
end


-- list of attributes { key = val, key = val }
function set_attributes(nodelist,att_tbl)
    if att_tbl == nil then return end
    for k, v in pairs(att_tbl) do
        if k and v then
            local num = k
            if type(k) == "number" then k = attribute_number_name[k] end
            if not k then w("attribute name %d not found",num) end
            set_attribute(nodelist,k,v)
        end
    end
end

--- Look for `user_defined` at end of page (ship-out) and runs actions encoded
--  in them. This is the fixed page layout, do not change it. Inserting nodes is
--  okay, as long as the page structure is kept intact.
function insert_nonmoving_whatsits( head, parent, blockinline,curx, cury, pagewidth, pageheight )
    local get_attr = get_attribute
    local insert_nm = insert_nonmoving_whatsits
    local setp = setprop
    local getp = getprop
    local set_attrs = set_attributes
    local default_stack = defaultcolorstack
    local opt_format = options.format
    local links_get = links_module.get
    local node_new = node.new
    local node_insert_before = node.insert_before
    local node_insert_after = node.insert_after
    local node_set_attr = node.set_attribute
    local attr_num_color = attribute_name_number_map["color"]
    local attr_num_hyperlink = attribute_name_number_map["hyperlink"]
    local roles_lookup = roles_a
    local pdf_reserveobj = pdf.reserveobj
    local spbp = sp_to_bp
    local pages_tbl = pages
    local attr_table_reuse = {}
    local borderattrs = borderattributes
    local colors = colors_module.colors
    local colorname_table = colors_module.colortable
    local attr_num_bordernumber = attribute_name_number_map["bordernumber"]
    local attr_num_borderwd = attribute_name_number_map["borderwd"]
    local attr_num_borderht = attribute_name_number_map["borderht"]
    local node_has_attribute = node.has_attribute
    local node_getproperty = node.getproperty
    local firsthead = head
    if not head then return {} end
    local fun
    local prev_hyperlink, prev_fgcolor, prev_role
    local nonextnode = false
    local linklevel = 0
    local currentfont = 1
    local rules = {}
    while head do
        local head_props = node_getproperty(head)
        local head_props_t = head_props and type(head_props) == "table" and head_props or nil
        -- what is subtype 1?
        if head.id==hlist_node and head.subtype == 1 then
            local r = insert_nm(head.list,head,"horizontal",curx,cury,pagewidth, pageheight)
            for _, rule in ipairs(r) do
                rules[#rules+1] = rule
            end
            local bordernumber = node_has_attribute(head, attr_num_bordernumber)
            if bordernumber then
                local ba = borderattrs[bordernumber]
                local wd = node_has_attribute(head, attr_num_borderwd)
                local ht = node_has_attribute(head, attr_num_borderht)
                wd = wd - ba.border_left_width - ba.margin_right + ba.padding_left
                setp(head,"border_bottom_color",ba.border_bottom_color)
                setp(head,"border_bottom_left_radius",ba.border_bottom_left_radius)
                setp(head,"border_bottom_right_radius",ba.border_bottom_right_radius)
                setp(head,"border_bottom_style",ba.border_bottom_style)
                setp(head,"border_bottom_width",ba.border_bottom_width)
                setp(head,"border_left_color",ba.border_left_color)
                setp(head,"border_left_style",ba.border_left_style)
                setp(head,"border_left_width",ba.border_left_width)
                setp(head,"border_right_color",ba.border_right_color)
                setp(head,"border_right_style",ba.border_right_style)
                setp(head,"border_right_width",ba.border_right_width)
                setp(head,"border_top_color",ba.border_top_color)
                setp(head,"border_top_left_radius",ba.border_top_left_radius)
                setp(head,"border_top_right_radius",ba.border_top_right_radius)
                setp(head,"border_top_style",ba.border_top_style)
                setp(head,"border_top_width",ba.border_top_width)
                setp(head,"borderstart","true")
                setp(head,"debug","false")
                setp(head,"margin_bottom",ba.margin_bottom)
                setp(head,"margin_left",ba.margin_left)
                setp(head,"margin_right",ba.margin_right)
                setp(head,"margin_top",ba.margin_top)
                setp(head,"padding_bottom",ba.padding_bottom)
                setp(head,"padding_left",ba.padding_left)
                setp(head,"padding_right",ba.padding_right)
                setp(head,"padding_top",ba.padding_top)
                setp(head,"lineheight",ht)
                setp(head,"depth", parent.depth)
                local boxnode = htmlbox("horizontal", head, wd, parent.height, parent.depth)
                local hbox = node.hpack(boxnode)
                local g
                g = set_glue(nil, { width = ba.border_left_width + ba.margin_left + ba.padding_left })
                hbox.head = node.insert_before(hbox.head,boxnode,g)
                g = set_glue(nil, { width = parent.height })
                local vl = node.vpack(g)
                node.insert_after(vl,g,hbox)
                g = set_glue(nil, { width = -parent.height })
                node.insert_after(vl,hbox,g)
                vl.height = 0
                parent.head = node.insert_before(parent.head,head,vl)
            end
            if blockinline == "vertical" then cury = cury + head.height + head.depth else curx = curx + head.width end
        elseif head.id==hlist_node or head.id == vlist_node then
            -- Check if this is a paragraph vlist from par:format with no features that need processing
            local skip_descent = false
            if head.id == vlist_node then
                if head_props_t and head_props_t.origin == "par:format"
                    and not head_props_t.has_color
                    and not head_props_t.has_hyperlink
                    and not head_props_t.has_role
                    and not head_props_t.has_special_nodes then
                    skip_descent = true
                end
            end
            if not skip_descent then
                local bordernumber = node_has_attribute(head, attr_num_bordernumber)
                if bordernumber then
                    local boxnode = mpbox(borderattrs[bordernumber],head.width,head.height + head.depth)
                    parent.head = node_insert_before(parent.head,head,boxnode)
                end
                if head.id == hlist_node then
                    local r = insert_nm(head.list,head,"horizontal",curx,cury,pagewidth, pageheight)
                    for _, rule in ipairs(r) do
                        rules[#rules+1] = rule
                    end
                else
                    local r = insert_nm(head.list,head,"vertical",curx,cury,pagewidth, pageheight)
                    for _, rule in ipairs(r) do
                        rules[#rules+1] = rule
                    end
                end
            end
            if blockinline == "vertical" then cury = cury + head.height + head.depth else curx = curx + head.width end
        else
            if head.id == glue_node then
                if blockinline == "horizontal" then
                    curx = curx + head.width
                else
                    cury = cury + head.width
                end
            elseif head.id == rule_node then
                if head.subtype == 2 then
                    local left = curx
                    local bot = pageheight - cury - head.height
                    local right = head.width + curx
                    local top = pageheight - cury
                setp(head,"bbox",{ sp_to_bp(left),sp_to_bp(bot),sp_to_bp(right),sp_to_bp(top )})
            end
            if blockinline == "vertical" then cury = cury + head.height + head.depth else curx = curx + head.width end
        end

            local fgcolor = node_has_attribute(head, attr_num_color)
            local bordernumber = node_has_attribute(head, attr_num_bordernumber)

            if bordernumber then
                local ba = borderattrs[bordernumber]
                local wd, ht
                wd = node_has_attribute(head, attr_num_borderwd)
                ht = node_has_attribute(head, attr_num_borderht)
                setp(head,"border_bottom_color",ba.border_bottom_color)
                setp(head,"border_bottom_left_radius",ba.border_bottom_left_radius)
                setp(head,"border_bottom_right_radius",ba.border_bottom_right_radius)
                setp(head,"border_bottom_style",ba.border_bottom_style)
                setp(head,"border_bottom_width",ba.border_bottom_width)
                setp(head,"border_left_color",ba.border_left_color)
                setp(head,"border_left_style",ba.border_left_style)
                setp(head,"border_left_width",ba.border_left_width)
                setp(head,"border_right_color",ba.border_right_color)
                setp(head,"border_right_style",ba.border_right_style)
                setp(head,"border_right_width",ba.border_right_width)
                setp(head,"border_top_color",ba.border_top_color)
                setp(head,"border_top_left_radius",ba.border_top_left_radius)
                setp(head,"border_top_right_radius",ba.border_top_right_radius)
                setp(head,"border_top_style",ba.border_top_style)
                setp(head,"border_top_width",ba.border_top_width)
                setp(head,"borderstart","true")
                setp(head,"debug","false")
                setp(head,"margin_bottom",ba.margin_bottom)
                setp(head,"margin_left",ba.margin_left)
                setp(head,"margin_right",ba.margin_right)
                setp(head,"margin_top",ba.margin_top)
                setp(head,"padding_bottom",ba.padding_bottom)
                setp(head,"padding_left",ba.padding_left)
                setp(head,"padding_right",ba.padding_right)
                setp(head,"padding_top",ba.padding_top)
                setp(head,"lineheight",ht)
                if blockinline == "vertical" then
                    wd = wd - ba.border_left_width - ba.margin_left - ba.padding_left - ba.margin_right
                    setp(head,"shiftdown",ht + ba.padding_top + ba.border_top_width + ba.margin_top)
                    setp(head,"shiftright",ba.border_left_width + ba.margin_left + ba.padding_left)
                else
                    ht = parent.height + parent.depth
                    setp(head,"lineheight",ht)
                    wd = parent.width - ba.border_left_width - ba.margin_left - ba.border_right_width - ba.margin_right
                    ht = head.height + head.depth
                end

                local boxnode = htmlbox(blockinline, head, wd, parent.height,parent.depth)
                parent.head = node_insert_before(parent.head,head,boxnode)
                setp(head,"borderstart",false)
            end
            -- Re-read properties if border/rule code may have created them
            if not head_props_t then
                head_props = node_getproperty(head)
                head_props_t = head_props and type(head_props) == "table" and head_props or nil
            end
            local transparency  = head_props_t and head_props_t.opacity
            local bbox          = head_props_t and head_props_t.bbox
            local hl            = node_has_attribute(head, attr_num_hyperlink) or (head_props_t and head_props_t.hyperlink)
            local role, structelemobjnum, id, parentid, rc, actualtext, alttext
            if opt_format == "PDF/UA" and head_props_t then
                actualtext       = head_props_t.actualtext
                alttext          = head_props_t.alttext
                role             = head_props_t.role
                structelemobjnum = head_props_t.structelemobjnum
                id               = head_props_t.id
                parentid         = head_props_t.parent
                rc               = head_props_t.rolecounter
            end
            if head.id == glyph_node then
                currentfont = head.font
                if role and head.next and head.next.id == disc_node then
                    setprop(head.next,"role",role)
                    setprop(head.next,"parent",parentid)
                    setprop(head.next,"rolecounter",rc)
                end
            end

            -- Fast path: skip annotation work when no colors/links/roles/opacity are present
            local needs_annotations = fgcolor or prev_fgcolor or hl or prev_hyperlink or role or prev_role or transparency
            if needs_annotations then
                local insert_startlink = false
                local insert_endlink = false
                local insert_startcolor = false
                local insert_endcolor = false
                local insert_startrole = false
                local insert_endrole = false

                if fgcolor and head.next == nil then
                    -- at end insert endcolor if in color mode
                    if prev_fgcolor == nil  then
                        insert_startcolor = true
                    end
                    insert_endcolor = true
                    prev_fgcolor = nil
                elseif fgcolor ~= prev_fgcolor then
                    -- 1: fgcolor nil and prev_fgcolor != nil
                    -- 2: fgcolor val and prev_fgcolor diff val
                    -- 3: fgcolor val and prev_fgcolor nil
                    if fgcolor == nil and prev_fgcolor then
                        -- 1
                        insert_endcolor = true
                    elseif fgcolor and prev_fgcolor then
                        -- 2
                        insert_endcolor = true
                        insert_startcolor = true
                    else
                        -- 3
                        insert_startcolor = true
                    end
                    prev_fgcolor = fgcolor
                end

                if role and ( head.next == nil or head.next.id == vlist_node or head.next.id == hlist_node )  then
                    -- at end insert endrole if in role mode
                    if prev_role == nil  then
                        insert_startrole = true
                    end
                    insert_endrole = true
                    nonextnode = true
                elseif role ~= prev_role then
                    -- 1: role nil and prev_role != nil
                    -- 2: role val and prev_role diff val
                    -- 3: role val and prev_role nil
                    if role == nil and prev_role then
                        -- 1
                        insert_endrole = true
                    elseif role and prev_role then
                        -- 2
                        insert_endrole = true
                        insert_startrole = true
                    else
                        -- 3
                        insert_startrole = true
                    end
                end
                -- case 1: link ends at the end of the list
                --         this is due to a (line-) broken link
                --         => end link
                --  case 2: hyperlink value of the node changes
                --         either insert a start link or an end link marker
                if hl and head.next == nil and linklevel > 0 then
                    insert_endlink = true
                    prev_hyperlink = nil
                elseif hl ~= prev_hyperlink then
                    if hl ~= nil then
                        insert_startlink = true
                        prev_hyperlink = hl
                    else
                        insert_endlink = true
                        prev_hyperlink = nil
                    end
                end
                if head.next == nil then
                    insert_startlink = false
                end
                if insert_endlink then
                    linklevel = linklevel - 1
                    local enl = node_new("whatsit","pdf_end_link")
                    parent.head = node_insert_before(parent.head,head,enl)
                    if insert_endrole then
                        local emc = node_new("whatsit","pdf_literal")
                        emc.data = "EMC"
                        emc.mode = 1
                        setp(emc,"origin","insert_endrole")
                        node_insert_after(parent.head,enl,emc)
                        insert_endrole = false
                    end
                end
                if insert_startlink then
                    linklevel = linklevel + 1
                    -- 3 = user
                    local ai = get_action_node(3)
                    ai.data = tostring(links_get(hl))
                    local stl = node_new("whatsit","pdf_start_link")
                    stl.action = ai
                    stl.width = -1073741824
                    stl.height = -1073741824
                    stl.depth = -1073741824
                    stl.objnum = pdf_reserveobj()
                    parent.head = node_insert_before(parent.head,head,stl)
                    if insert_endrole then
                        local emc = node_new("whatsit","pdf_literal")
                        emc.data = "EMC"
                        emc.mode = 1
                        setp(emc,"origin","insert_endrole")
                        parent.head = node_insert_before(parent.head,stl,emc)
                        insert_endrole = false
                    end

                    if insert_startrole then
                        local bdc = node_new("whatsit","pdf_literal")
                        node_set_attr(bdc,att_role, role)
                        setp(bdc,"parentid", parentid)
                        setp(bdc,"rolecounter", rc)
                        setp(bdc,"id",id)
                        setp(bdc,"bbox",bbox)
                        setp(bdc,"actualtext",actualtext)
                        setp(bdc,"origin","insert_startrole")
                        setp(bdc,"structelemobjnum",structelemobjnum)
                        setp(bdc,"linkobjnum",stl.objnum)
                        bdc.data = ""
                        bdc.mode = 1
                        parent.head = node_insert_before(parent.head,stl,bdc)
                        head = head.next
                        insert_startrole = false
                    end

                end

                -- Lazy: only read full attribute table when color stacks need it
                local attr_table
                if insert_endcolor or insert_startcolor then
                    attr_table = get_attributes(head, attr_table_reuse)
                    attr_table_reuse = attr_table
                end
                -- Save original head for startcolor insertion, because
                -- endcolor processing may advance head past the colored node.
                local startcolor_before = head
                if insert_endcolor then
                    local colstop = node_new("whatsit","pdf_colorstack")
                    set_attrs(colstop,attr_table)
                    colstop.data = ""
                    colstop.command = 2
                    colstop.stack = default_stack
                    setp(colstop,"origin","setcolor")
                    if fgcolor and not prev_fgcolor then
                        parent.head = node_insert_after(parent.head,head,colstop)
                        head = head.next
                    else
                        parent.head = node_insert_before(parent.head,head,colstop)
                    end
                end
                if insert_startcolor then
                    local colstart = node_new("whatsit","pdf_colorstack")
                    set_attrs(colstart,attr_table)
                    local colorname = colorname_table[fgcolor]
                    local colorentry = colors[colorname]
                    local col = colorentry.pdfstring
                    local alpha = colorentry.alpha
                    if alpha then
                        local thispage = pages[current_pagenumber]
                        thispage.transparenttext = thispage.transparenttext or {}
                        thispage.transparenttext[alpha] = true
                        col = col .. string.format("/TRP%d gs",alpha )
                    end
                    colstart.data = col
                    colstart.command = 1
                    colstart.stack = default_stack

                    setp(colstart,"origin","setcolor")
                    parent.head = node_insert_before(parent.head,startcolor_before,colstart)
                end
                if insert_endrole then
                    local emc = node_new("whatsit","pdf_literal")
                    emc.data = "EMC"
                    emc.mode = 1
                    setp(emc,"origin","insert_endrole")
                    if nonextnode and prev_role then
                        parent.head = node_insert_after(parent.head,head,emc)
                        head = head.next
                    elseif prev_role then
                        parent.head = node_insert_before(parent.head,head,emc)
                    else
                        -- single item?
                        local bdc = node_new("whatsit","pdf_literal")
                        node_set_attr(bdc,att_role, role)
                        setp(bdc,"parentid", parentid)
                        setp(bdc,"rolecounter", rc)
                        setp(bdc,"bbox",bbox)
                        setp(bdc,"id",id)
                        setp(bdc,"actualtext",actualtext)
                        setp(bdc,"alttext",alttext)
                        setp(bdc,"rolename",tostring(roles_lookup[role]))
                        setp(bdc,"origin","insert_startrole")
                        bdc.data = ""
                        bdc.mode = 1
                        parent.head = node_insert_before(parent.head,head,bdc)

                        parent.head = node_insert_after(parent.head,head,emc)
                        head = emc
                        insert_startrole = false
                    end
                end
                if insert_startrole then
                    local bdc = node_new("whatsit","pdf_literal")
                    node_set_attr(bdc,att_role, role)
                    setp(bdc,"parentid", parentid)
                    setp(bdc,"rolecounter", rc)
                    setp(bdc,"bbox",bbox)
                    setp(bdc,"id",id)
                    setp(bdc,"actualtext",actualtext)
                    setp(bdc,"alttext",alttext)
                    setp(bdc,"rolename",tostring(roles_lookup[role]))
                    setp(bdc,"origin","insert_startrole")
                    bdc.data = ""
                    bdc.mode = 1
                    parent.head = node_insert_before(parent.head,head,bdc)
                end
                prev_role = role

                if transparency then
                    local colstart = node_new("whatsit","pdf_colorstack")
                    colstart.data = string.format("/TRP%d gs",transparency )
                    colstart.command = 1
                    colstart.stack = default_stack
                    current_page.transparenttext[transparency] = true

                    local colend = node_new("whatsit","pdf_colorstack")
                    colend.command = 2
                    colend.stack = default_stack
                    parent.head = node_insert_before(parent.head,head,colstart)
                    node_insert_after(parent.head,head,colend)
                    head = head.next
                end
            end

            -- HTML inline border
            if head_props_t then
                if head_props_t.borderstart then
                    local cur = head
                    while cur do
                        local cur_properties = node.getproperty(cur)
                        if cur_properties and cur_properties.borderend then
                            if cur.next then cur = cur.next end
                            break
                        end
                        cur = cur.next
                    end
                    local wd,ht,dp = node.dimensions(head,cur)
                    local boxnode = htmlbox(blockinline,head,wd,ht,dp)
                    parent.head = node.insert_before(parent.head,head,boxnode)
                end
            end
            -- Now let's look at user defined whatsits, that are ment
            -- for markers, bookmarks etc.
            if head.id==whatsit_node then
                if head.subtype == user_defined_whatsit then
                    -- action
                    if head.user_id == user_defined_addtolist then
                        -- this part is obsolete (2.9.3)
                        -- the value is the index of the hash of user_defined_functions
                        fun = user_defined_functions[head.value]
                        fun()
                        -- use and forget
                        user_defined_functions[head.value] = nil
                    -- bookmark
                    elseif head.user_id == user_defined_bookmark then
                        local level,openclose,dest,str =  string.match(head.value,"([^+]*)+([^+]*)+([^+]*)+(.*)")
                        level = tonumber(level)
                        local open_p
                        if openclose == "1" then
                            open_p = true
                        else
                            open_p = false
                        end
                        local i = 1
                        local current_bookmark_table = bookmarks -- level 1 == top level
                        -- create levels if necessary
                        while i < level do
                            if #current_bookmark_table == 0 then
                                current_bookmark_table[1] = {}
                                err("No bookmark given for this level (%d)!",level)
                            end
                            current_bookmark_table = current_bookmark_table[#current_bookmark_table]
                            i = i + 1
                        end
                        current_bookmark_table[#current_bookmark_table + 1] = {name = str, destination = dest, open = open_p}
                    elseif head.user_id == user_defined_mark then
                        local marker = head.value
                        markercount = markercount + 1
                        markers[marker] = { page = current_pagenumber, count = markercount }
                    elseif head.user_id == user_defined_mark_append then
                        local marker = head.value
                        if markers[marker] == nil then
                            markers[marker] = { page = tostring(current_pagenumber) }
                        else
                            markers[marker]["page"] = tostring(markers[marker]["page"]) .. "," ..  tostring(current_pagenumber)
                        end
                    end
                elseif head.subtype == pdf_literal_whatsit then
                    local data = getprop(head,"data")
                    if data then
                        head.data = ""
                        rules[#rules+1] = { curx, cury, data }
                    end
                end
            elseif options.format == "PDF/UA" and head.id == glue_node and node.has_attribute(head,att_spaceglue) == 1 then
                -- a space in PDF/UA should be a real glyph
                -- subtype >= 1000 is assigend to glue for arranging pages
                local g = node.new("glyph")
                g.subtype = 1
                g.font = currentfont
                g.char = 32
                g.width = head.width
                parent.head = node.insert_before(parent.head,head,g)
                head.width = 0
            end
        end
        head = head.next
    end
    return rules
end

--- Node(list) creation
--- -------------------


rightskip = node.new("glue_spec")
rightskip.width = 0
rightskip.stretch = 1 * 2^16
rightskip.stretch_order = 3

leftskip = node.new("glue_spec")
leftskip.width = 0
leftskip.stretch = 1 * 2^16
leftskip.stretch_order = 3

--- Return the larger glue(spec) values
function bigger_glue_spec( a,b )
    if a.stretch_order > b.stretch_order then return a end
    if b.stretch_order > a.stretch_order then return b end
    if a.stretch > b.stretch then return a end
    if b.stretch > a.stretch then return b end
    if a.width > b.width then return a else return b end
end

-- used for br
function short_newline(fam)
    local strutheight = fonts.lookup_fontfamily_number_instance[fam].baselineskip
    local dummypenalty
    dummypenalty = node.new("penalty")
    dummypenalty.penalty = 10000
    node.set_attribute(dummypenalty,att_newline,1)

    local list, cur
    add_rule(list,"tail",{height = 0.75 * strutheight, depth = 0.25 * strutheight,width = 0}, "short_newline")
    g = set_glue(nil,{},"short_newline")
    list,cur = node.insert_after(list,node.tail(list),g)
    return list, cur
end

-- newline returns a nodelist that behaves as a new line in TeX
function newline(fam)
    local strutheight = fonts.lookup_fontfamily_number_instance[fam].baselineskip
    local dummypenalty
    dummypenalty = node.new("penalty")
    dummypenalty.penalty = 10000
    node.set_attribute(dummypenalty,att_newline,1)

    local list, cur
    list, cur = dummypenalty,dummypenalty

    local strut = node.new(rule_node)
    -- set to 60000 for example for debugging
    strut.width=0
    strut.height = strutheight*0.75
    strut.depth = strutheight*0.25
    list,cur = node.insert_after(list,cur,strut)

    local p1,g,p2
    p1 = node.new("penalty")
    p1.penalty = 10000
    g = set_glue(nil,{stretch = 2^16, stretch_order = 2})
    p2 = node.new("penalty")
    p2.penalty = -10000
    node.set_attribute(p1,att_newline,1)
    node.set_attribute(p2,att_newline,1)
    node.set_attribute(g,att_newline,1)
    -- important for empty lines (adjustlineheight)
    node.set_attribute(p1,att_fontfamily,fam)
    list,cur = node.insert_after(list,cur,p1)
    list,cur = node.insert_after(list,cur,g)
    list,cur = node.insert_after(list,cur,p2)
    -- add glue so next word can hyphenate (#274)
    g = set_glue(nil,{})
    list,cur = node.insert_after(list,cur,g)
    return list, cur
end

-- Used to set the line height within nobreak.
function addstrut(nodelist,where,origin)
    local strutheight = 0
    local head = nodelist
    while head do
        if head.id == hlist_node then
            strutheight = head.height
            if node.has_attribute(head,att_dont_format) then
                -- 0.25 is the depth of the line, and hopefully
                -- this is the highest thing in the line
                head.shift = 0.25 * head.height
            end
        end
        head = head.next

    end
    head = nodelist
    while head do
        if node.has_attribute(head,att_fontfamily) then
            break
        end
        head = head.next
    end
    local fontfamily

    if head == nil then
        fontfamily = nil
    else
        fontfamily = node.has_attribute(head,att_fontfamily)
    end
    if fontfamily == nil or fontfamily == 0 then
        fontfamily = fonts.lookup_fontfamily_name_number["text"]
    end

    local fi = fonts.lookup_fontfamily_number_instance[fontfamily]
    strutheight = math.max(fi.baselineskip, strutheight)
    -- for debugging purposes set width to 20000:
    local strut = add_rule(nodelist,"head",{height = 0.75 * strutheight, depth = 0.25 * strutheight, width = 0 },origin or "addstrut")
    return strut
end


-- Remove the first \n in a paragraph value table. See #132
function remove_first_whitespace ( tbl )
    if newxpath and xpath.is_attribute(tbl) then
        tbl.value = string.gsub(tbl.value,"^[\n\t]*(.-)$","%1")
        return true
    end
    for i=1,#tbl do
        if type(tbl[i]) == "string" then
            tbl[i] = string.gsub(tbl[i],"^[\n\t]*(.-)$","%1")
            return true
        end
        if type(tbl[i]) == "table" then
            local ret
            if tbl[i].contents and type(tbl[i].contents) == "table" then
                ret = remove_first_whitespace(tbl[i].contents)
            else
                ret = remove_first_whitespace(tbl[i])
            end
            if ret then return true end
        end
    end
end

-- Remove the final \n in a paragraph value table. See #132
function remove_last_whitespace ( tbl )
    for i=#tbl,1,-1 do
        if type(tbl[i]) == "string" then
            if string.match(tbl[i],"^%s*$") then
                table.remove( tbl,i )
            else
                tbl[i] = string.gsub(tbl[i],"^(.-)[\n\t]*$","%1")
            end
            return true
        end
        if type(tbl[i]) == "table" then
            local ret
            if tbl[i].contents and type(tbl[i].contents) == "table" then
                ret = remove_last_whitespace(tbl[i].contents)
            else
                local tic = tbl[i].contents
                -- the last contents could be an image for example. See #342
                if type(tic) == "userdata" then ret = true else
                    ret = remove_last_whitespace(tbl[i])
                end
            end
            return ret
        end
    end
end

function setprop(n, prop, value)
    local props = node.getproperty(n)
    if not props then
      props = {}
      node.setproperty(n, props)
    end
    props[prop] = value
end

function getprop( n, prop )
    local props = node.getproperty(n)
    if not props then return nil end
    if type(props) == "table" then return props[prop] end
    return nil
end

function clearprop(n, prop)
    local props = node.getproperty(n)
    if not props then return nil end
    if type(props) == "table" then
        local ret = props[prop]
        props[prop] = nil
        return ret
    end
    return nil
end


local function setstyles(n,parameter)
    if parameter.bold == 1 then
        node.set_attribute(n,att_fontweight,attval_fontweight["bold"])
        setprop(n,"font-weight","bold")
    end
    if parameter.italic == 1 then
        node.set_attribute(n,att_fontstyle,attval_fontstyle["italic"])
    end
    if parameter.textdecorationline then
        node.set_attribute(n,att_td_line,attval_td_line[parameter.textdecorationline])
        node.set_attribute(n,att_td_style,attval_td_style[parameter.textdecorationstyle])
        node.set_attribute(n,att_td_color,current_fgcolor)
    end
    if parameter.color and parameter.color ~= 1 then
        node.set_attribute(n,att_td_color,parameter.color)
        node.set_attribute(n,att_color,parameter.color)
    end
    if parameter.hyperlink then
        node.set_attribute(n,att_hyperlink,parameter.hyperlink)
    end
    if parameter.languagecode and n.id == glyph_node then
        n.lang = parameter.languagecode
    end
    if parameter.backgroundcolor then
        node.set_attribute(n,att_bgcolor,parameter.backgroundcolor)
        local bg_padding_top = tex.sp(parameter.bg_padding_top or 0)
        local bg_padding_bottom = tex.sp(parameter.bg_padding_bottom or 0)
        node.set_attribute(n,att_bgpadtop,bg_padding_top)
        node.set_attribute(n,att_bgpadbot,bg_padding_bottom)
    end
    if parameter.verticalalign then
        node.set_attribute(n,att_verticalalign,attval_verticalalign[parameter.verticalalign])
    end
    if parameter.indent then
        setprop(n,"indent",parameter.indent)
    end
    if parameter.role then
        setprop(n,"role",parameter.role)
    end
    if parameter.structelemobjnum then
        setprop(n,"structelemobjnum",parameter.structelemobjnum)
    end
    if parameter.actualtext then
        setprop(n,"actualtext", parameter.actualtext)
    end
    if parameter.alttext then
        setprop(n,"alttext", parameter.alttext)
    end
    if parameter.parent then
        if parameter.parent == "" then
            -- ignore
        else
            setprop(n,"parent",parameter.parent)
        end
    end
    if parameter.id then
        setprop(n,"id",parameter.id)
    end
    if parameter.rolecounter then
        setprop(n,"rolecounter",parameter.rolecounter)
    end
end

function getfallbacks(cluster,glyphslist,fallback_fontdefinitions)
    local fontnum
    local ret = {}
    local fallback = fallback_fontdefinitions[1]
    fontnum = fallback.fontnum
    local tbl = fallback
    local i = 1

    while i <= #glyphslist do
        local buf = harfbuzz.Buffer.new()
        local pos = glyphslist[i][1]
        local thisglyph = glyphslist[i][2]
        ret[#ret+1] = {
            pos = pos,
            fonttable = tbl,
            fontnumber = fontnum,
        }
        local curret = ret[#ret]
        -- the cluster may have more than one unicode character, so I
        -- create a list
        local uclist = {}
        local uc = cluster[thisglyph.cluster]
        uclist[#uclist+1] = uc
        buf:add_utf8(unicode.utf8.char(uc))
        while #glyphslist > i and glyphslist[i+1][1] == pos + 1 do
            i = i + 1
            pos = pos + 1
            thisglyph = glyphslist[i][2]
            uc = cluster[thisglyph.cluster]
            uclist[#uclist+1] = uc
            buf:add_utf8(unicode.utf8.char(uc))
        end
        buf:set_cluster_level(buf.CLUSTER_LEVEL_MONOTONE_CHARACTERS)
        buf:set_flags(harfbuzz.Buffer.FLAG_REMOVE_DEFAULT_IGNORABLES)
        buf:guess_segment_properties()
        shape(tbl,buf)
        local nglyphs = buf:get_glyphs()
        curret.remove_len = #uclist
        for j = 1, #nglyphs do
            curret[#curret + 1] = {
                glyph = nglyphs[j],
                codepoint = nglyphs[j].codepoint,
            }
            local jcp = nglyphs[j].codepoint
            if jcp == 0 then
                curret[#curret].uc = uclist[j]
            else
                curret[#curret].uc = tbl.backmap[jcp]
            end
        end
        i = i + 1
    end
    return ret
end

function hbglyphlist(arguments)
    local tbl = arguments.tbl
    local glyphs = arguments.glyphs
    local cluster = arguments.cluster
    local parameter = arguments.parameter
    local allowbreak = arguments.allowbreak
    local newlines_at = arguments.newlines_at
    local fontfamily = parameter.fontfamily
    local direction = arguments.direction
    local thislang = arguments.thislang
    local fontnumber = arguments.fontnumber
    local is_cjk = arguments.is_cjk
    local decoration_active = parameter.textdecorationline ~= nil
    local background_active = parameter.backgroundcolor ~= nil

    local thisfont = fonts.used_fonts[fontnumber]
    local reportmissingglyphs = options.reportmissingglyphs
    local lastitemwasglyph
    local space   = tbl.parameters.space
    local shrink  = tbl.parameters.space_shrink
    local stretch = tbl.parameters.space_stretch
    local list, cur
    local n
    local preserve_whitespace = parameter.whitespace == "pre"
    local zeroglyphs = {}

    for i=1,#glyphs do
        local thisglyph = glyphs[i]
        if thisglyph.codepoint == 0 then
            zeroglyphs[#zeroglyphs+1] = {i,thisglyph}
        end
    end

    local fallbacks
    -- fallacks: remove all glyph entries that are covered by fallbacks, then
    -- re-insert a more complex table at these positions.
    -- detect the other table type in the loop below (for i=1,#glyphs do .. end)
    if #zeroglyphs > 0 and #tbl.fallback_fontdefinitions > 0 then
        fallbacks = getfallbacks(cluster,zeroglyphs,tbl.fallback_fontdefinitions)
        for i = #fallbacks, 1, -1 do
            local fb = fallbacks[i]
            for j = 1, fb.remove_len do
                table.remove(glyphs, fb.pos)
            end
            for j = #fb, 1, -1 do
                table.insert(glyphs, fb.pos, {pos = fb.pos,
                fonttable = fb.fonttable,
                  fontnumber = fb.fontnumber,
                  glyph = fb[j].glyph,
                  codepoint = fb[j].codepoint,
                  uc = fb[j].uc
                })
            end
          end
    end

    local thistbl = tbl

    -- Pre-compute style attributes (parameter is constant throughout the loop)
    local style_attrs = {}
    local style_attrs_len = 0
    local style_props_tmpl = nil
    local style_langcode = parameter.languagecode
    local node_set_attribute = node.set_attribute

    if parameter.bold == 1 then
        style_attrs_len = style_attrs_len + 2
        style_attrs[style_attrs_len - 1] = att_fontweight
        style_attrs[style_attrs_len] = attval_fontweight["bold"]
        style_props_tmpl = {}
        style_props_tmpl["font-weight"] = "bold"
    end
    if parameter.italic == 1 then
        style_attrs_len = style_attrs_len + 2
        style_attrs[style_attrs_len - 1] = att_fontstyle
        style_attrs[style_attrs_len] = attval_fontstyle["italic"]
    end
    if parameter.textdecorationline then
        style_attrs_len = style_attrs_len + 2
        style_attrs[style_attrs_len - 1] = att_td_line
        style_attrs[style_attrs_len] = attval_td_line[parameter.textdecorationline]
        style_attrs_len = style_attrs_len + 2
        style_attrs[style_attrs_len - 1] = att_td_style
        style_attrs[style_attrs_len] = attval_td_style[parameter.textdecorationstyle]
        style_attrs_len = style_attrs_len + 2
        style_attrs[style_attrs_len - 1] = att_td_color
        style_attrs[style_attrs_len] = current_fgcolor
    end
    if parameter.color and parameter.color ~= 1 then
        style_attrs_len = style_attrs_len + 2
        style_attrs[style_attrs_len - 1] = att_td_color
        style_attrs[style_attrs_len] = parameter.color
        style_attrs_len = style_attrs_len + 2
        style_attrs[style_attrs_len - 1] = att_color
        style_attrs[style_attrs_len] = parameter.color
    end
    if parameter.hyperlink then
        style_attrs_len = style_attrs_len + 2
        style_attrs[style_attrs_len - 1] = att_hyperlink
        style_attrs[style_attrs_len] = parameter.hyperlink
    end
    if parameter.backgroundcolor then
        style_attrs_len = style_attrs_len + 2
        style_attrs[style_attrs_len - 1] = att_bgcolor
        style_attrs[style_attrs_len] = parameter.backgroundcolor
        style_attrs_len = style_attrs_len + 2
        style_attrs[style_attrs_len - 1] = att_bgpadtop
        style_attrs[style_attrs_len] = tex.sp(parameter.bg_padding_top or 0)
        style_attrs_len = style_attrs_len + 2
        style_attrs[style_attrs_len - 1] = att_bgpadbot
        style_attrs[style_attrs_len] = tex.sp(parameter.bg_padding_bottom or 0)
    end
    if parameter.verticalalign then
        style_attrs_len = style_attrs_len + 2
        style_attrs[style_attrs_len - 1] = att_verticalalign
        style_attrs[style_attrs_len] = attval_verticalalign[parameter.verticalalign]
    end
    if parameter.indent then
        style_props_tmpl = style_props_tmpl or {}
        style_props_tmpl["indent"] = parameter.indent
    end
    if parameter.role then
        style_props_tmpl = style_props_tmpl or {}
        style_props_tmpl["role"] = parameter.role
    end
    if parameter.structelemobjnum then
        style_props_tmpl = style_props_tmpl or {}
        style_props_tmpl["structelemobjnum"] = parameter.structelemobjnum
    end
    if parameter.actualtext then
        style_props_tmpl = style_props_tmpl or {}
        style_props_tmpl["actualtext"] = parameter.actualtext
    end
    if parameter.alttext then
        style_props_tmpl = style_props_tmpl or {}
        style_props_tmpl["alttext"] = parameter.alttext
    end
    if parameter.parent and parameter.parent ~= "" then
        style_props_tmpl = style_props_tmpl or {}
        style_props_tmpl["parent"] = parameter.parent
    end
    if parameter.id then
        style_props_tmpl = style_props_tmpl or {}
        style_props_tmpl["id"] = parameter.id
    end
    if parameter.rolecounter then
        style_props_tmpl = style_props_tmpl or {}
        style_props_tmpl["rolecounter"] = parameter.rolecounter
    end

    -- Pre-compute glyph creation values
    local glyph_left = parameter.left or tex.lefthyphenmin
    local glyph_right = parameter.right or tex.righthyphenmin
    local famtab = fonts.lookup_fontfamily_number_instance[fontfamily]
    local glyph_yoffset_base = 0
    if parameter.verticalalign == "sub" then
        glyph_yoffset_base = -famtab.subshift
    elseif parameter.verticalalign == "super" then
        glyph_yoffset_base = famtab.supershift
    end
    local xoffset_dir = 1
    if direction == "rtl" then xoffset_dir = -1 end

    -- Pre-build allowbreak lookup set
    local allowbreak_set = {}
    for _, c in utf8.codes(allowbreak) do
        allowbreak_set[c] = true
    end

    -- Apply pre-computed styles to a node (replaces setstyles in the loop)
    local function apply_styles(n)
        for j = 1, style_attrs_len, 2 do
            node_set_attribute(n, style_attrs[j], style_attrs[j+1])
        end
        if style_langcode and n.id == glyph_node then n.lang = style_langcode end
        if style_props_tmpl then
            local p = node.getproperty(n)
            if not p then p = {}; node.setproperty(n, p) end
            for k, v in next, style_props_tmpl do p[k] = v end
        end
    end

    for i=1,#glyphs do
        local thisfontnumber = fontnumber
        local thisglyph = glyphs[i]
        local cp = thisglyph.codepoint
        local cpcluster = cluster[thisglyph.cluster]
        if cpcluster and cp == 0 and cpcluster > puastart then
            cp = cpcluster - puastart
        end
        local uc
        if thisglyph.pos then
            -- a special table from fallback
            thisfontnumber = thisglyph.fontnumber
            thistbl = thisglyph.fonttable
            uc = thisglyph.uc
            cp = thisglyph.codepoint
            thisglyph = thisglyph.glyph
        else
            thistbl = tbl
            uc = thistbl.backmap[cp] or cp
        end

        -- FIXME cp == 0 doesn't look right
        local tabregularspace = ( cp == 0 and cluster[thisglyph.cluster] == 9 and parameter.tab ~= "hspace")
        -- skip double space
        if i > 1 and ( uc == 32 or tabregularspace) and cp == glyphs[i-1].codepoint and cluster[thisglyph.cluster] ~= 160 and not preserve_whitespace then
            goto continue
        end
        if false then
            -- just for simple adding at the beginning
        elseif uc == 127 then
            -- ignore DEL
        elseif uc == 160 and #glyphs == 1 then
            -- ignore
        elseif uc == 32 or tabregularspace then
            local thiscluster = thisglyph.cluster
            local thisclustervalue = cluster[thiscluster]
            if thisclustervalue == 160 then -- no break space
                n = node.new(penalty_node)
                n.penalty = 10000
                list,cur = node.insert_after(list,cur,n)
                n = set_glue(nil,{width = space, shrink = shrink, stretch = stretch},"uc=32,160")
                node_set_attribute(n,att_tie_glue,1)
                list,cur = node.insert_after(list,cur,n)
            elseif thisclustervalue == 8194 then -- en space
                n = set_glue(nil,{width = thistbl.parameters.enspace },"uc=8194")
                -- prevent from stretching with ragged shape
                n.subtype = 1
                list,cur = node.insert_after(list,cur,n)
            elseif thisclustervalue == 8195 then -- em space
                n = set_glue(nil,{width = thistbl.parameters.emspace },"uc=8195")
                -- prevent from stretching with ragged shape
                n.subtype = 1
                list,cur = node.insert_after(list,cur,n)
            elseif thisclustervalue == 8196 then -- three per em space
                n = set_glue(nil,{width = thistbl.parameters.thirdspace },"uc=8196")
                -- prevent from stretching with ragged shape
                n.subtype = 1
                list,cur = node.insert_after(list,cur,n)
            elseif thisclustervalue == 8197 then -- four per em space
                n = set_glue(nil,{width = thistbl.parameters.quarterspace },"uc=8197")
                -- prevent from stretching with ragged shape
                n.subtype = 1
                list,cur = node.insert_after(list,cur,n)
            elseif thisclustervalue == 8198 then -- six per em space
                n = set_glue(nil,{width = thistbl.parameters.sixthspace },"uc=8198")
                -- prevent from stretching with ragged shape
                n.subtype = 1
                list,cur = node.insert_after(list,cur,n)
            elseif thisclustervalue == 8201 then -- thin space
                n = set_glue(nil,{width = thistbl.parameters.thinspace },"uc=8201")
                -- prevent from stretching with ragged shape
                n.subtype = 1
                list,cur = node.insert_after(list,cur,n)
            elseif thisclustervalue == 8202 then -- hair space
                n = set_glue(nil,{width = thistbl.parameters.hairspace},"uc=8202")
                -- prevent from stretching with ragged shape
                n.subtype = 1
                list,cur = node.insert_after(list,cur,n)
            elseif thisclustervalue == 8203 then
                -- U+200B ZERO WIDTH SPACE
                p = node.new(penalty_node)
                p.penalty = -10
                list,cur = node.insert_after(list,cur,p)
            elseif thisclustervalue == 8205 then
                -- U+200D ZERO WIDTH JOINER
                -- ignore
            elseif preserve_whitespace then
                local n = add_rule(nil,"head",{height = 0 * factor, depth = 0, width = thistbl.zerowidth },"preserve_whitespace")
                list,cur = node.insert_after(list,cur,n)
            else
                n = set_glue(nil,{width = space,shrink = shrink, stretch = stretch},"uc=32")
                apply_styles(n)
                node_set_attribute(n,att_spaceglue,1)
                list,cur = node.insert_after(list,cur,n)
                if parameter.letterspacing then
                    n.width = n.width + parameter.letterspacing
                end
            end
            if decoration_active then
                node_set_attribute(n,att_td_line,attval_td_line[parameter.textdecorationline])
                node_set_attribute(n,att_td_style,attval_td_style[parameter.textdecorationstyle])
                node_set_attribute(n,att_td_color,current_fgcolor)
            end

            if background_active then
                node_set_attribute(n,att_bgcolor,parameter.backgroundcolor)
                if parameter.bg_padding_top then
                    node_set_attribute(n,att_bgpadtop,parameter.bg_padding_top)
                end
                if parameter.bg_pading_bottom then
                    node_set_attribute(n,att_bgpadbot,parameter.bg_padding_bottom)
                end
            end
            if n then
                node_set_attribute(n,att_fontfamily,fontfamily)
            end
        elseif ( cp == 0 or uc == 10 ) and newlines_at[thisglyph.cluster] then
            local dummypenalty
            dummypenalty = node.new(penalty_node)
            dummypenalty.penalty = 10000
            node_set_attribute(dummypenalty,att_newline,1)
            list,cur = node.insert_after(list,cur,dummypenalty)

            local ht = famtab.size
            local strut = add_rule(nil,"head",{height = ht * 0.75, depth = 0.25 * ht, width = 0 }, "newline")
            node_set_attribute(strut,att_newline,1)
            setprop(strut,"origin","strut newline hb")
            list,cur = node.insert_after(list,cur,strut)

            local p1,g,p2
            p1 = node.new(penalty_node)
            p1.penalty = 10000

            g = set_glue(nil,{stretch = 2^16, stretch_order = 2})

            p2 = node.new(penalty_node)
            p2.penalty = -10000
            node_set_attribute(p1,att_newline,1)
            node_set_attribute(p2,att_newline,1)
            node_set_attribute(g,att_newline,1)


            -- important for empty lines (adjustlineheight)
            node_set_attribute(p1,att_fontfamily,fontfamily)

            list,cur = node.insert_after(list,cur,p1)
            list,cur = node.insert_after(list,cur,g)
            list,cur = node.insert_after(list,cur,p2)

            -- add glue so next word can hyphenate (#274)
            g = set_glue(nil,{})
            list,cur = node.insert_after(list,cur,g)
        elseif cp == 0 then
            local code = cluster[thisglyph.cluster]
            if code == 9 then
                if parameter.tab == 'hspace' then
                    local n = set_glue(nil,{width = 0, stretch = 2^16, stretch_order = 3})
                    list, cur = node.insert_after(list,cur,n)
                else
                    -- a regular space, handled above (tabregularspace)
                end
            else
                if reportmissingglyphs then
                    if reportmissingglyphs == "warning" then
                        main.log("warn","Glyph is missing from the font","font",thisfont.name,"glyph_hex",string.format("%04x",code),"loc","hbglyphlist")
                    else
                        main.log("error","Glyph is missing from the font","font",thisfont.name,"glyph_hex",string.format("%04x",code),"loc","hbglyphlist")
                    end
                end
            end
        else
            n = node.new(glyph_node, 1)
            n.font = thisfontnumber
            n.char = uc
            n.uchyph = 1
            n.left = glyph_left
            n.right = glyph_right
            if glyph_yoffset_base ~= 0 then
                n.yoffset = glyph_yoffset_base
            end

            if thisglyph.x_offset ~= 0 then
                n.xoffset = xoffset_dir * thisglyph.x_offset * thistbl.mag
            end
            if thisglyph.y_offset ~= 0 then
                n.yoffset = thisglyph.y_offset * thistbl.mag
            end
            node_set_attribute(n,att_fontfamily,fontfamily)
            apply_styles(n)
            list,cur = node.insert_after(list,cur,n)

            if cur and cur.prev and cur.prev.id == glyph_node then
                lastitemwasglyph = true
            end

            -- CJK
            if is_cjk and i < #glyphs and uc > 12032 then
                -- don't break within non-cjk words
                if prohibited_at_end[thislang][unicode.utf8.char(uc)] then
                    -- ignore
                else
                    -- add breaking point between this glyph and next glyph unless prohibited
                    if i < #glyphs then
                        local nextchar = glyphs[i+1].codepoint
                        local nextuc = thistbl.backmap[nextchar] or nextchar
                        if not prohibited_at_beginning[thislang][unicode.utf8.char(nextuc)] then
                            local pen = node.new(penalty_node)
                            pen.penalty = 0
                            if parameter.textformat.alignment == "justified" then
                                local g = set_glue(nil,{stretch = 2^16, stretch_order = 0})
                                list,cur = node.insert_after(list,cur,g)
                            end
                            list,cur = node.insert_after(list,cur,pen)
                        end
                    end
                end
            end
            -- simplified chinese
            -- characters that must not appear at the beginning of a line
            -- !%),.:;?]}¢°·'""†‡›℃∶、。〃〆〕〗〞﹚﹜！＂％＇），．：；？！］｝～
            -- characters that must not appear at the end of a line
            -- $(£¥·'"〈《「『【〔〖〝﹙﹛＄（．［｛￡￥

            -- glyph ids inserted by sd:symbol()
            if  uc > puastart and thisglyph.codepoint == 0 then
                thisglyph.x_advance = thistbl.characters[uc].hadvance
            end
            local diff = thisglyph.x_advance - thistbl.characters[uc].hadvance
            local kernvalue = diff * tbl.mag
            if parameter.letterspacing then
                kernvalue = kernvalue + parameter.letterspacing
            end
            if kernvalue ~= 0 then
                local property = "kernafter"
                if direction == "rtl" then property = "kernbefore" end
                publisher.setprop(cur,property, kernvalue)
            end
            if uc == -1 then
            elseif uc > puastart then
                -- ignore
            elseif ( uc == 45 or uc == 8211) and lastitemwasglyph and allowbreak_set[45] then
                -- only break if allowbreak contains the hyphen char
                local pen = node.new(penalty_node)
                pen.penalty = 10000
                list = node.insert_before(list,cur,pen)
                local disc = node.new(disc_node)
                list,cur = node.insert_after(list,cur,disc)
                local g = set_glue(nil)
                apply_styles(pen)
                apply_styles(disc)
                apply_styles(g)
                list,cur = node.insert_after(list,cur,g)
            elseif allowbreak_set[uc] then
                -- allowbreak lists characters where the publisher may break lines
                local pen = node.new(penalty_node)
                pen.penalty = 0
                list,cur = node.insert_after(list,cur,pen)
            end
        end
        ::continue::
    end

    if not list then
        -- This should never happen.
        warning("No head found")
        return node.new("hlist")
    end
    local aa = parameter.add_attributes or {}
    for i=1,#aa do
        set_attribute_recurse(list,aa[i][1],aa[i][2])
    end
    return list
end

-- return a glue node
local function ffgetglue(s,tbl)
    local n
    if s == 8194 then
        n = set_glue(nil,{width = tbl.parameters.enspace },"uc=8194")
    elseif s == 8195 then -- em space
        n = set_glue(nil,{width = tbl.parameters.emspace },"uc=8195")
    elseif s == 8196 then -- three per em space
        n = set_glue(nil,{width = tbl.parameters.thirdspace },"uc=8196")
    elseif s == 8197 then -- four per em space
        n = set_glue(nil,{width = tbl.parameters.quarterspace },"uc=8197")
    elseif s == 8198 then -- six per em space
        n = set_glue(nil,{width = tbl.parameters.sixthspace },"uc=8198")
    elseif s == 8201 then -- thin space
        n = set_glue(nil,{width = tbl.parameters.thinspace },"uc=8201")
    elseif s == 8202 then -- hair space
        n = set_glue(nil,{width = tbl.parameters.hairspace },"uc=8202")
    elseif s == 8203 then
        n = set_glue(nil,{width = 0 },"uc=8203")
    end
    return n
end

local function ffglyphlist(arguments)
    local tbl = arguments.tbl
    local str = arguments.str

    local parameter = arguments.parameter
    local allowbreak = arguments.allowbreak
    local fontfamily = parameter.fontfamily
    local fontnumber = arguments.fontnumber
    local languagecode = arguments.languagecode
    local space   = tbl.parameters.space
    local shrink  = tbl.parameters.space_shrink
    local stretch = tbl.parameters.space_stretch

    local match = unicode.utf8.match
    local allow_newline = true
    if options.htmlignoreeol then
        allow_newline = false
    end

    local head, last, n

    local lastitemwasglyph
    local newline = 10
    local breakatspace = true
    if not string.find(allowbreak, " ") then
        breakatspace = false
    end
    local preserve_whitespace = parameter.whitespace == "pre"
    -- There is a string with UTF-8 chars
    for s in string.utfvalues(str) do
        local char = unicode.utf8.char(s)
        -- If the next char is a newline (&#x0A;) a \\ is inserted
        if s == newline and allow_newline then
            -- This is to enable hyphenation again. When we add a rule right after a word
            -- hyphenation is disabled. So we insert a penalty of 10k which should not do
            -- harm. Perhaps there is a better solution, but this seems to work OK.
            local dummypenalty
            dummypenalty = node.new("penalty")
            dummypenalty.penalty = 10000
            node.set_attribute(dummypenalty,att_newline,1)
            head,last = node.insert_after(head,last,dummypenalty)
            if fontfamily == nil then
                err( "ffglyphlist: fontfamily is nil")
                return
            end
            local ff = fonts.lookup_fontfamily_number_instance[fontfamily]
            if ff == nil then
                err( string.format("Could not find instance of family %s",fontfamily))
                return
            end
            local ht = ff.size
            local strut = add_rule(nil,"head",{height = ht * 0.75, depth = ht * 0.25, width = 0 }, "newline")
            node.set_attribute(strut,att_newline,1)
            head,last = node.insert_after(head,last,strut)

            local p1,g,p2
            p1 = node.new("penalty")
            p1.penalty = 10000

            g = set_glue(nil,{stretch = 2^16, stretch_order = 2})

            p2 = node.new("penalty")
            p2.penalty = -10000

            node.set_attribute(p1,att_newline,1)
            node.set_attribute(p2,att_newline,1)
            node.set_attribute(g,att_newline,1)
            local attr = { fontfamily = fontfamily}
            set_attributes(p1,attr)
            set_attributes(p2,attr)
            set_attributes(g,attr)

            head,last = node.insert_after(head,last,p1)
            head,last = node.insert_after(head,last,g)
            head,last = node.insert_after(head,last,p2)

            -- add glue so next word can hyphenate (#274)
            g = set_glue(nil,{})
            head,last = node.insert_after(head,last,g)
        elseif preserve_whitespace and match(char,"^%s$") then
            local tbl = fonts.used_fonts[fontnumber]
            local strut = add_rule(nil,"head",{height = 0 * factor, depth = 0, width = tbl.zerowidth }, "preserve_whitespace")
            head,last = node.insert_after(head,last,strut)
        elseif match(char,"^%s$") and last and last.id == glue_node and not node.has_attribute(last,att_tie_glue,1) then
            -- double space, use the bigger glue
            local tmp
            if s >= 8194 and s <= 8198 or s >= 8201 and s <= 8203 then
                tmp = ffgetglue(s,tbl)
            else
                tmp = set_glue(nil, {width = space, shrink = shrink, stretch = stretch},"double glue")
            end
            local tmp2 = bigger_glue_spec(last,tmp)
            last.width = tmp2.width
            last.stretch = tmp2.stretch
            last.shrink = tmp2.shrink
            last.stretch_order = tmp2.stretch_order
            last.shrink_order = tmp2.shrink_order
        elseif s == 160 then -- non breaking space U+00A0
            n = node.new("penalty")
            n.penalty = 10000

            head,last = node.insert_after(head,last,n)
            n = set_glue(nil,{width = space, shrink = shrink, stretch = stretch})
            node.set_attribute(n,att_tie_glue,1)

            head,last = node.insert_after(head,last,n)

            if parameter.textdecorationline then
                set_attribute(n,"text-decoration-line",parameter.textdecorationline)
                set_attribute(n,"text-decoration-style",parameter.textdecorationstyle)
                set_attribute(n,"text-decoration-color",current_fgcolor)
            end

            if parameter.backgroundcolor then
                set_attribute(n,"background-color",parameter.backgroundcolor)
                set_attribute(n,"bgpaddingtop",parameter.bg_padding_top)
                set_attribute(n,"bgpaddingbottom",parameter.bg_padding_bottom)
            end
            set_attribute(n,"fontfamily",fontfamily)
        elseif s == 173 then -- soft hyphen
            -- The soft hyphen is used in server-mode /v0/format
            n = node.new(penalty_node)
            n.penalty = 10000
            head, last = node.insert_after(head,last,n)

            n = node.new(disc_node)
            node.set_attribute(n,att_keep,1)
            head, last = node.insert_after(head,last,n)

            n = node.new(penalty_node)
            n.penalty = 10000
            head, last = node.insert_after(head,last,n)

            n = set_glue(nil)
            head, last = node.insert_after(head,last,n)
        elseif s == 9 and parameter.tab == 'hspace' then
            local n = set_glue(nil,{width = 0, stretch = 2^16, stretch_order = 3})
            head, last = node.insert_after(head,last,n)
        elseif s >= 8194 and s <= 8198 or s >= 8201 and s <= 8203 then
            local n = ffgetglue(s,tbl)
            -- prevent from stretching with ragged shape
            n.subtype = 1
            head, last = node.insert_after(head,last,n)
        -- anchor is necessary. Otherwise à (C3A0) would match A0 - %s
        elseif match(char,"^%s$") then -- Space
            if breakatspace == false then
                n = node.new("penalty")
                n.penalty = 10000

                head,last = node.insert_after(head,last,n)

            end
            -- ; and : should have the possibility to break easily if a space follows
            if last and last.id == glyph_node and ( last.char == 58 or last.char == 59) then
                n = node.new("penalty")
                n.penalty = 0
                head,last = node.insert_after(head,last,n)
            end

            n = set_glue(nil,{width = space,shrink = shrink, stretch = stretch},"space")
            setstyles(n,parameter)
            if breakatspace == false then
                node.set_attribute(n,att_tie_glue,1)
            end

            if parameter.textdecorationline then
                set_attribute(n,"text-decoration-line",parameter.textdecorationline)
                set_attribute(n,"text-decoration-color",current_fgcolor)
            end
            if parameter.backgroundcolor then
                set_attribute(n,"background-color",parameter.backgroundcolor)
                if parameter.bg_padding_top then
                    set_attribute(n,"bgpaddingtop",parameter.bg_padding_top)
                end
                if parameter.bg_padding_bottom then
                    set_attribute(n,"bgpaddingbottom",parameter.bg_padding_bottom)
                end
            end
            set_attribute(n,"fontfamily",fontfamily)
            head,last = node.insert_after(head,last,n)
            if parameter.letterspacing then
                n.width = n.width + parameter.letterspacing
            end
        else
            -- A regular character?!?
            n = node.new("glyph")
            n.font = fontnumber
            n.subtype = 1
            n.char = s
            n.lang = languagecode
            n.uchyph = 1
            n.left = parameter.left or tex.lefthyphenmin
            n.right = parameter.right or tex.righthyphenmin
            setstyles(n,parameter)
            set_attribute(n,"fontfamily",fontfamily)

            local famtab = fonts.lookup_fontfamily_number_instance[fontfamily]
            if parameter.verticalalign == "sub" then
                n.yoffset = -famtab.subshift
            elseif parameter.verticalalign == "super" then
                n.yoffset = famtab.supershift
            end

            if parameter.letterspacing then
                local k = node.new("kern")
                k.kern = parameter.letterspacing
                setstyles(k,parameter)
                head,last = node.insert_after(head,last,k)
                lastitemwasglyph = true
            end
            if last and last.id == glyph_node then
                lastitemwasglyph = true
            end

            head,last = node.insert_after(head,last,n)
            -- CJK
            if s >= 12032 then
                local pen = node.new("penalty")
                pen.penalty = 0
                head,last = node.insert_after(head,last,pen)
            end

            -- Some characters must be treated in a special way.
            -- Hyphens must be separated from words:
            if n.char == 8209 then -- non breaking hyphen U+2011
                n.char = 45
                local pen = node.new("penalty")
                pen.penalty = 10000
                head,last = node.insert_after(head,last,pen)
            elseif ( n.char == 45 or n.char == 8211) and lastitemwasglyph and string.find(allowbreak, "-",1,true) then
                -- only break if allowbreak contains the hyphen char
                local pen = node.new("penalty")
                pen.penalty = 10000
                head = node.insert_before(head,last,pen)
                local disc = node.new("disc")
                head,last = node.insert_after(head,last,disc)
                local g = set_glue(nil)
                setstyles(pen,parameter)
                setstyles(disc,parameter)
                setstyles(g,parameter)
                head,last = node.insert_after(head,last,g)
            elseif string.find(allowbreak,char,1,true) then
                -- allowbreak lists characters where the publisher may break lines
                local pen = node.new("penalty")
                pen.penalty = 0
                head,last = node.insert_after(head,last,pen)
            end
        end
    end

    if not head then
        -- This should never happen.
        warning("No head found")
        return node.new("hlist")
    end
    local aa = parameter.add_attributes or {}
    for i=1,#aa do
        set_attribute_recurse(head,aa[i][1],aa[i][2])
    end
    return head
end

function getinstancename(parameter)
    local instancename
    if parameter.bold == 1 then
        if parameter.italic == 1 then
            instancename = "bolditalic"
        else
            instancename = "bold"
        end
    elseif parameter.italic == 1 then
        instancename = "italic"
    else
        instancename = "normal"
    end
    if parameter.fontsize == "small" then
        instancename = instancename .. "script"
    end
    return instancename
end

-- Return a list of nodes
function mknodes(str,parameter,origin)
    -- if it's an empty string, we make a zero-width rule
    if not str or string.len(str) == 0 then
        -- a space char can have a width, so we return a zero width something
        local strut = add_rule(nil,"head",{height = 1 * factor, depth = 0, width = 0 }, "pardir")
        setprop(strut, "pardir", parameter.direction)
        return strut, parameter.direction
    end
    parameter = parameter or {}
    local languagecode = parameter.languagecode or defaultlanguage

    local fontfamily = parameter.fontfamily
    if parameter.monospace then
        fontfamily = fonts.lookup_fontfamily_name_number.monospace
    end
    local fontnumber = fonts.get_fontinstance(fontfamily,getinstancename(parameter))
    local tbl = fonts.used_fonts[fontnumber]

    -- Convert letterspacing_em (in 1/1000 em) to absolute letterspacing based on font size
    if parameter.letterspacing_em and not parameter.letterspacing then
        local fam_instance = fonts.lookup_fontfamily_number_instance[fontfamily]
        if fam_instance and fam_instance.size then
            parameter.letterspacing = parameter.letterspacing_em * fam_instance.size / 1000
        end
    end

    local maindirection
    local segments
    if parameter.bidi and not ( str == "\n" ) then
        local dir = 0
        if parameter.direction == "ltr" then
            dir = 1
        elseif parameter.direction == "rtl" then
            dir = 2
        end
        segments = splib.segmentize_text(str,dir)
        if segments[1][1] == 0 then
            segments.maindirection = "ltr"
        else
            segments.maindirection = "rtl"
        end
    else
        local dir = nil
        segments = {}
        if parameter.direction == "rtl" then
            dir = 1
            segments.maindirection = "rtl"
        elseif parameter.direction == "ltr" then
            segments.maindirection = "ltr"
        end
        segments[1] = {dir,str}
    end
    maindirection = parameter.direction or segments.maindirection
    local nodelistsegments

    for i=1,#segments do
        str = segments[i][2]
        local direction = segments[i][1]
        local thislang = "en"
        if languagecode and languages_id_lang[languagecode].locale then
            thislang = languages_id_lang[languagecode].locale
        end
        local thissegment
        if tbl.face then
            local script = nil
            if thislang == "--" then
                thislang = nil
                script = nil
            elseif thislang == "zh" then
                script = "Hans"
            end

            local newlines_at = {}

            local cluster = {}
            local pos = 0
            for c in unicode.utf8.gmatch(str,".") do
                cluster[pos] = unicode.utf8.byte(c)
                if c == "\n" then
                    newlines_at[pos] = true
                end
                pos = pos + #c
            end
            local buf = harfbuzz.Buffer.new()
            buf:add_utf8(str)
            if direction == 0 then direction = "ltr" elseif direction == 1 then direction = "rtl" end
            -- shape returns the guessed script and direction from the buffer
            script, direction = shape(tbl,buf, { language = thislang, script = script, direction = direction } )
            local is_cjk = false
            if script == "Hans" or script == "Hira" or script == "Hant" or script == "Hani" then
                is_cjk = true
                -- script can be guessed from buffer and thislang could be empty, so
                -- lang must be set again.
                thislang = "zh"
            elseif script == "Kana" then
                is_cjk = true
                thislang = "ja"
            end

            local glyphs = buf:get_glyphs()
            if #glyphs == 0 then
                goto nextsegment
            end
            thissegment = hbglyphlist({
                glyphs = glyphs,
                tbl = tbl,
                cluster = cluster,
                parameter = parameter,
                allowbreak = parameter.allowbreak or " -",
                newlines_at = newlines_at,
                script = script,
                direction = direction or maindirection,
                thislang = thislang,
                fontnumber = fontnumber,
                is_cjk = is_cjk,
            })
        else
            -- old fontforge code
            thissegment = ffglyphlist({
                str = str,
                tbl = tbl,
                parameter = parameter,
                allowbreak = parameter.allowbreak or " -",
                fontfamily = fontfamily,
                direction = maindirection,
                thislang = thislang,
                fontnumber = fontnumber,
                languagecode = languagecode,
            })
        end
        direction = direction or 0
        thissegment = setsegmentdir(thissegment,direction,maindirection)

        if nodelistsegments then
            local tail = node.tail(nodelistsegments)
            tail.next = thissegment
            thissegment.prev = tail
        else
            nodelistsegments = thissegment
        end
        ::nextsegment::
    end
    if maindirection then
        setprop(nodelistsegments,"pardir",maindirection)
    end
    if not nodelistsegments then
        local strut = add_rule(nil,"head",{height = 1 * factor, depth = 0, width = 0 }, "pardir")
        return strut, parameter.direction
    end
    return nodelistsegments, maindirection
end

-- direction is ltr or rtl or 0 or 1
function setsegmentdir(nodelist,direction, maindirection)
    local dirstring
    if direction == 0 or direction == "ltr" then
        dirstring = "TLT"
    elseif direction == 1 or direction == "rtl" then
        dirstring = "TRT"
    end

    -- don't do anything if this segment goes in the paragraph direction
    if ( maindirection == nil or maindirection == "ltr" ) and dirstring == "TLT" then
        return nodelist
    elseif maindirection == "rtl" and dirstring == "TRT" then
        return nodelist
    end

    local dirstart = node.new(dir_node)
    local dirend = node.new(dir_node)
    dirstart.dir = "+" .. dirstring
    dirend.dir = "-" .. dirstring
    node.setproperty(dirstart,node.getproperty(nodelist))
    local ff = get_attribute(nodelist,"fontfamily")
    set_attribute(dirstart,"fontfamily",ff)
    nodelist = node.insert_before(nodelist,nodelist,dirstart)

    local tail = node.tail(nodelist)
    local ff = get_attribute(tail,"fontfamily")
    set_attribute(dirend,"fontfamily",ff)
    node.setproperty(dirend,node.getproperty(tail))
    node.insert_after(nodelist,tail,dirend)
    return nodelist
end

-- head_or_tail = "head" oder "tail" (default: tail). Return new head (perhaps same as nodelist)
function add_rule( nodelist,head_or_tail,parameter,origin)
    parameter = parameter or {}

    local n=node.new(rule_node)
    n.width  = parameter.width
    n.height = parameter.height
    n.depth  = parameter.depth
    if origin then setprop(n,"origin",origin) end
    if not nodelist then return n end

    if head_or_tail=="head" then
        nodelist = node.insert_before(nodelist,nodelist,n)
        return nodelist
    else
        local last=node.slide(nodelist)
        last.next = n
        n.prev = last
        return nodelist,n
    end
    assert(false,"never reached")
end

--- Return a hbox with width `labelwidth`
function bullet_hbox( labelwidth,parameter )
    local bullet, pre_glue, post_glue
    bullet = mknodes("•",parameter)
    pre_glue = set_glue(nil,{stretch = 2^16, stretch_order = 3})
    pre_glue.next = bullet

    post_glue = set_glue(nil,{width = 4 * 2^16})
    post_glue.prev = bullet
    bullet.next = post_glue
    local bullet_hbox = node.hpack(pre_glue,labelwidth,"exactly")

    if options.showobjects then
        boxit(bullet_hbox)
    end
    set_attribute(bullet_hbox,"indent",labelwidth)
    node.set_attribute(bullet_hbox,att_rows,-1)
    return bullet_hbox
end

--- Return a hbox with width `labelwidth`
function number_hbox( num, labelwidth,parameter )
    local pre_glue, post_glue
    local digits = mknodes( tostring(num) .. ".",parameter)
    pre_glue = set_glue(nil,{stretch = 2^16, stretch_order = 3})
    pre_glue.next = digits

    post_glue = set_glue(nil,{width = 4 * 2^16})
    post_glue.prev = node.tail(digits)
    node.tail(digits).next = post_glue
    local digit_hbox = node.hpack(pre_glue,labelwidth,"exactly")

    if options.showobjects then
        boxit(digit_hbox)
    end
    set_attribute(digit_hbox,"indent",labelwidth)
    node.set_attribute(digit_hbox,att_rows,-1)
    return digit_hbox
end

-- Create a hbox for a label
function whatever_hbox( label,labelwidth,options,labelsep_wd,labelalign )
    local fam = options.fontfamily
    labelsep_wd = labelsep_wd or fonts.lookup_fontfamily_number_instance[fam].size / 2
    labelalign = labelalign or "right"
    local shrink_glue = set_glue(nil,{shrink = 2^16, shrink_order = 3,width =  labelwidth})
    local label_sep = set_glue(nil,{width = labelsep_wd})

    local label_hbox
    if labelalign == "right" then
        shrink_glue.next = label
        local t = node.slide(label)
        t.next = label_sep
        label_hbox = node.hpack(shrink_glue,labelwidth,"exactly")
    else
        local t = node.slide(label)
        t.next = label_sep
        label_sep.next = shrink_glue
        label_hbox = node.hpack(label,labelwidth,"exactly")
    end
    set_attribute(label_hbox.head,"fontfamily",fam)
    label_hbox.head = addstrut(label_hbox.head,"head","whatever_hbox/strut")

    return label_hbox
end


-- just the plain size, not included the stretch or shrink
function get_glue_size( n )
    local spec

    if node.has_field(n,"spec") then
        spec = n.spec
    else
        spec = n
    end
    return spec.width
end


-- Add a glue to the front or tail of the given nodelist. `head_or_tail` is
-- either the string `head` or `tail`. `parameter` is a table with the keys
-- `width`, `stretch` and `stretch_order`. If the nodelist is nil, a simple
-- node list consisting of a glue will be created.
function add_glue( nodelist,head_or_tail,parameter,origin)
    parameter = parameter or {}

    local n = set_glue(nil, parameter)
    n.subtype = parameter.subtype or 0
    if origin then setprop(n,"origin",origin) end
    if nodelist == nil then return n end

    if head_or_tail=="head" then
        n.next = nodelist
        nodelist.prev = n
        return n
    else
        local last=node.slide(nodelist)
        last.next = n
        n.prev = last
        return nodelist,n
    end
    assert(false,"never reached")
end

-- 0pt plus 1fil minus 1fil
function hss_glue()
    return make_glue({stretch = 2^16, stretch_order = 2, shrink=2^16, shrink_order = 2})
end

function make_glue( parameter )
    return set_glue(nil, parameter)
end

local function add_properties_to_discnodes(head)
    while head do
        if head.id == glyph_node and head.next and head.next.id == disc_node then
            local role = getprop(head,"role")
            local parent = getprop(head,"parent")
            local rolecounter = getprop(head,"rolecounter")
            local id = getprop(head,"id")
            if head.next.pre then
                setprop(head.next.pre,"role",role)
                setprop(head.next.pre,"parent",parent)
                setprop(head.next.pre,"rolecounter",rolecounter)
                setprop(head.next.pre,"id",id)
            end
            if head.next.post then
                setprop(head.next.post,"role",role)
                setprop(head.next.post,"parent",parent)
                setprop(head.next.post,"rolecounter",rolecounter)
                setprop(head.next.post,"id",id)
            end
            if head.next.replace then
                setprop(head.next.replace,"role",role)
                setprop(head.next.replace,"parent",parent)
                setprop(head.next.replace,"rolecounter",rolecounter)
                setprop(head.next.replace,"id",id)
            end

        end
        head = head.next
    end
end

function finish_par( nodelist,hsize,parameters )
    assert(nodelist)
    node.slide(nodelist)

    if not parameters.disable_hyphenation then
        lang.hyphenate(nodelist)

        if options.format == "PDF/UA" then
            add_properties_to_discnodes(nodelist)
        end
    end

    local n = node.new("penalty")
    setprop(n,"origin","finishpar")
    n.penalty = 10000
    local last = node.slide(nodelist)

    last.next = n
    n.prev = last
    last = n

    -- mode harfbuzz sets haskerns, different kind of kerning
    n = node.kerning(nodelist)
    n = hbkern(n)
    -- 15 is a parfillskip
    n,last = add_glue(n,"tail",{ subtype = 15, width = 0, stretch = 2^16, stretch_order = 2})
end

function hbkern(nodelist)
    local head = d_todirect(nodelist)
    local start = head
    local curkern = 0
    while head do
        local id = d_getid(head)
        if id == glyph_node then
            local props = d_getproperty(head)
            if props then
                local k = props.kernbefore
                if k then
                    props.kernbefore = nil
                    if k ~= 0 then curkern = k end
                end
            end

            if curkern ~= 0 then
                local kern = d_new(kern_node)
                d_setkern(kern, curkern)
                start = d_insert_before(start, head, kern)
                local ul = d_has_attribute(head, att_td_line)
                if ul then d_set_attribute(kern, att_td_line, ul) end
                local uccolor = d_has_attribute(head, att_td_color)
                if uccolor then d_set_attribute(kern, att_td_color, uccolor) end
                local bgcolor = d_has_attribute(head, att_bgcolor)
                if bgcolor then d_set_attribute(kern, att_bgcolor, bgcolor) end
                d_setproperty(kern, d_getproperty(head))
                local hl = d_has_attribute(head, att_hyperlink)
                if hl then d_set_attribute(kern, att_hyperlink, hl) end
                curkern = 0
            end
            if props then
                local k = props.kernafter
                if k and k ~= 0 then
                    curkern = k
                end
            end
        elseif id == disc_node then
            if curkern ~= 0 then
                local kern = d_new(kern_node)
                d_setkern(kern, curkern)
                d_setfield(head, "replace", kern)
                local ul = d_has_attribute(head, att_td_line)
                local uccolor = d_has_attribute(head, att_td_color)
                local bgcolor = d_has_attribute(head, att_bgcolor)
                local hyperlink = d_has_attribute(head, att_hyperlink)
                if ul then d_set_attribute(kern, att_td_line, ul) end
                if uccolor then d_set_attribute(kern, att_td_color, uccolor) end
                if bgcolor then d_set_attribute(kern, att_bgcolor, bgcolor) end
                if hyperlink then d_set_attribute(kern, att_hyperlink, hyperlink) end
                d_setproperty(kern, d_getproperty(head))
                curkern = 0
            end
        else
            curkern = 0
        end
        head = d_getnext(head)
    end
    return d_tonode(start)
end

function fix_justification(nodelist,alignment,parent,direction)
    if alignment == "start" then
        if direction == "rtl" then
            alignment = "rightaligned"
        else
            alignment = "leftaligned"
        end
    elseif alignment == "end" then
        if direction == "rtl" then
            alignment = "leftaligned"
        else
            alignment = "rightaligned"
        end
    end
    local curalignment = alignment
    if direction == "rtl" then
        if alignment == "rightaligned" then
            curalignment = "leftaligned"
        elseif alignment == "leftaligned" then
            curalignment = "rightaligned"
        end
    end

    local head = nodelist
    while head do
        if head.id == 0 then -- hlist

            -- we are on a line now. We assume that the spacing needs correction.
            -- The goal depends on the current line (par shape!)
            local goal
            if head.width == 1 then
                goal, _, _ = node.dimensions(head.glue_set, head.glue_sign, head.glue_order, head.head)
            else
                goal = head.width
            end
            local font_before_glue

            -- The following code is problematic, in tabular material. This is my older comment
            -- There was code here (39826d4c5 and before) that changed
            -- the glue depending on the font before that glue. That
            -- was problematic, because LuaTeX does not copy the
            -- altered glue_spec node on copy_list (in paragraph:format())
            -- which, when reformatted, gets a complaint by LuaTeX about
            -- infinite shrinkage in a paragraph

            -- a new glue spec node - we must not(!) alter the current glue spec
            -- because this list is copied in paragraph:format()
            local spec_new

            for n in node.traverse(head.head) do
                if n.id == glyph_node then
                    font_before_glue = n.font
                elseif n.id == glue_node then
                    if n.subtype==0 and font_before_glue and get_glue_value(n,"width") > 0 and head.glue_sign == 1 then
                        local fonttable = font.fonts[font_before_glue]
                        if not fonttable then fonttable = font.fonts[1] err("Some font not found") end
                        set_glue_values(n,{width = fonttable.parameters.space, shrink_order = head.glue_order, stretch = 0, stretch_order = 0})
                    end
                end
            end

            if curalignment == "rightaligned" or curalignment == "centered" then

                local list_start = head.head
                local rightskip_node = node.tail(head.head)
                local parfillskip

                -- first we remove everything between the rightskip and the
                -- last non-glue/non-penalty item
                -- the glues might contain "plus 1 fill" and the penalties are not
                -- useful
                local tmp = rightskip_node.prev
                while tmp and ( tmp.id == glue_node or tmp.id == penalty_node or tmp.id == dir_node ) do
                    tmp = tmp.prev
                    if tmp == nil then break end
                    head.head = node.remove(head.head,tmp.next)
                end

                local wd = node.dimensions(head.glue_set, head.glue_sign, head.glue_order,head.head)

                local leftskip_node
                if curalignment == "rightaligned" then
                    leftskip_node = set_glue(nil,{width = goal - wd})
                else
                    leftskip_node = set_glue(nil,{width = ( goal - wd ) / 2 })
                end
                head.head = node.insert_before(head.head,head.head,leftskip_node)
            end
        elseif head.id == 1 then -- vlist
            fix_justification(head.head,alignment,head,direction)
        end
        head = head.next
    end
    return nodelist
end

local function check_if_a_line_exeeds(nodelist,wd)
    local head = nodelist
    while head do
        if head.id == vlist_node then
            return check_if_a_line_exeeds(head.head,wd)
        elseif head.id == hlist_node then
            local width = node.dimensions(head.glue_set,head.glue_sign,head.glue_order,head.head)
            if width > wd + 400 then
                return true
            end
        end
        head = head.next
    end
    return false
end

function do_linebreak( nodelist,hsize,parameters )
    if nodelist == nil then
        err("No nodelist found for line breaking.")
        return box(tenmm_sp,tenmm_sp,"black")
    end

    parameters = parameters or {}
    finish_par(nodelist,hsize,parameters)

    local pdfignoreddimen
    pdfignoreddimen = -65536000


    local default_parameters = {
        hsize = hsize,
        emergencystretch = 0.1 * hsize,
        hyphenpenalty = 0,
        linepenalty = 10,
        pretolerance = 0,
        tolerance = 2000,
        doublehyphendemerits = 1000,
        pdfeachlineheight = pdfignoreddimen,
        pdfeachlinedepth  = pdfignoreddimen,
        pdflastlinedepth  = pdfignoreddimen,
        pdfignoreddimen   = pdfignoreddimen,
    }

    -- This could be done with a meta table, but somehow LuaTeX 104 doesn't like it
    for k,v in pairs(parameters) do
        default_parameters[k] = v
    end

    -- Try to break the paragraph until there is no line
    -- longer than expected
    local j
    local c = 0
    local line_exceeds_right_margin = true
    while true do
        j = tex.linebreak(node.copy_list(nodelist),default_parameters)
        if not check_if_a_line_exeeds(j,hsize) then
            line_exceeds_right_margin = false
            break
        end
        default_parameters.emergencystretch = default_parameters.emergencystretch + 0.1 * hsize
        c = c + 1
        if c > 9 then
            break
        end
        node.flush_list(j)
    end

    if line_exceeds_right_margin and options.overfulllineerror ~= nil then
        if options.overfulllineerror then
            main.log("error","Overfull line found","page",current_pagenumber,lineinfo())
        else
            main.log("warn","Overfull line found","page",current_pagenumber,lineinfo())
        end
    end
    node.flush_list(nodelist)

    -- Adjust line heights. Always take the largest font in a row.
    -- Use direct node API for speed in this tight loop.
    local dhead = d_todirect(j)
    local maxlineheight
    local fam
    local _h, _d
    local lookup_fam = fonts.lookup_fontfamily_number_instance
    local cur = dhead
    while cur do
        if d_getid(cur) == hlist_node then
            local lineheight
            maxlineheight = 0
            local head_list = d_getlist(cur)
            local adjustlineheight = true
            while head_list do
                if not lineheight then
                    lineheight = d_has_attribute(head_list, att_lineheight)
                end
                if d_has_attribute(head_list, att_dontadjustlineheight) then
                    adjustlineheight = false
                end
                local hlid = d_getid(head_list)
                if hlid == hlist_node or hlid == vlist_node then
                    local sublist = d_getlist(head_list)
                    if sublist then
                        _, _h, _d = d_dimensions(sublist)
                        local total = _h + _d
                        if total > maxlineheight then maxlineheight = total end
                    end
                else
                    fam = d_has_attribute(head_list, att_fontfamily)
                    if fam and fam > 0 then
                        local bls = lookup_fam[fam].baselineskip
                        if bls > maxlineheight then maxlineheight = bls end
                    end
                end
                head_list = d_getnext(head_list)
            end
            if adjustlineheight then
                if lineheight and lineheight > 0.75 * maxlineheight then
                    d_setfield(cur, "height", lineheight)
                    d_setfield(cur, "depth", 0.25 * maxlineheight)
                else
                    d_setfield(cur, "height", 0.75 * maxlineheight)
                    d_setfield(cur, "depth", 0.25 * maxlineheight)
                end
            end
        end
        cur = d_getnext(cur)
    end
    local ret = node.vpack(j)
    setprop(ret,"origin","do_linebreak")
    return ret
end

function create_empty_vbox_width_width_height(wd,ht)
    local hb = create_empty_hbox_with_width(wd)
    local n = set_glue(nil,{width = 0, stretch = 2^16, stretch_order = 3})
    node.insert_after(hb,hb,n)
    n = node.vpack(n,ht,"exactly")
    node.set_attribute(n,att_dontadjustlineheight,1)
    return n
end

function create_empty_hbox_with_width( wd )
    local n = set_glue(nil,{width = 0, stretch = 2^16, stretch_order = 3})
    n = node.hpack(n,wd,"exactly")
    return n
end

do
    local destcounter = 0
    -- Create a pdf anchor (dest object). It returns a whatsit node and the
    -- number of the anchor, so it can be used in a pdf link or an outline.
    function mknumdest()
        destcounter = destcounter + 1
        local d = node.new("whatsit","pdf_dest")
        d.named_id = 0
        d.dest_id = destcounter
        d.dest_type = 0
        return d, destcounter
    end
end

-- See PDF v1.7 spec 8.2 Document-Level Navigation
-- dest_type:
-- xyz   = 0  goto the current position
-- fit   = 1  fit the page in the window
-- fith  = 2  fit the width of the page
-- fitv  = 3  fit the height of the page
-- fitb  = 4  fit the ‘Bounding Box’ of the page
-- fitbh = 5  fit the width of ‘Bounding Box’ of the page
-- fitbv = 6  fit the height of ‘Bounding Box’ of the page
-- fitr  = 7 ?


function mkstringdest(name)
    local d = node.new("whatsit","pdf_dest")
    d.named_id = 1
    d.dest_id = utf8_to_utf16_string_pdf(name)
    d.dest_type = 0
    return d
end

-- Generate a hlist with necessary nodes for the bookmarks. To be inserted into
-- a vlist that gets shipped out
function mkbookmarknodes(level,open_p,title,data)
    -- The bookmarks need three values, the level, the name and if it is
    -- open or closed
    setup_page(nil, "mkbookmarknodes",data)
    local openclosed
    if open_p then openclosed = 1 else openclosed = 2 end
    level = level or 1
    title = title or "no title for bookmark given"

    local n,counter = mknumdest()
    local udw = node.new("whatsit","user_defined")
    udw.user_id = user_defined_bookmark
    udw.type = 115 -- a string
    udw.value = string.format("%d+%d+%d+%s",level,openclosed,counter,title)
    n.next = udw
    udw.prev = n
    -- this hlist sometimes gets reused, for example with Td/sethead=yes.
    -- therefore we need to find it in tabular#remove_bookmark_nodes()
    -- and remove them.
    local hlist = node.hpack(n)
    return hlist
end


-- blue rule below the hbox for debugging purpose
function addhrule(hbox)
    local n = node.new("whatsit","pdf_literal")
    n.data = string.format("q 0.3 w [2 1] 0 d 0 0 1 RG 0 %g  m %g %g l S Q",  sp_to_bp(hbox.height),  -sp_to_bp(hbox.width) ,  sp_to_bp(hbox.height) )
    local tail = node.tail(hbox)
    hbox = node.insert_after(hbox,tail,n)
    hbox = node.hpack(hbox)
    return hbox
end

function boxit( box )
    local box = node.hpack(box)

    local rule_width = 0.1
    local wd = box.width                 / factor - rule_width
    local ht = (box.height + box.depth)  / factor - rule_width
    local dp = box.depth                 / factor - rule_width / 2

    local wbox = node.new("whatsit","pdf_literal")
    wbox.data = string.format("q 0.1 G %g w %g %g %g %g re s Q", rule_width, rule_width / 2, -dp, -wd, ht)
    wbox.mode = 0
    -- Draw box at the end so its contents gets "below" it.
    local tmp = node.tail(box.list)
    tmp.next = wbox
    return box
end

-- color is an integer
function set_color_if_necessary( nodelist,color )
    local dontformat = node.has_attribute(nodelist,att_dont_format)

    if not color then return nodelist end

    local colorname
    if color == -1 then
        colorname = "black"
    else
        colorname = colors_module.colortable[color]
    end
    -- When we uncomment the if .. end here, the typesetting
    -- process is much slower. See #143
    if colorname == "black" then return nodelist end
    local colstart, colstop
    colstart = node.new("whatsit","pdf_colorstack")
    colstop  = node.new("whatsit","pdf_colorstack")
    colstart.data = colors_module.colors[colorname].pdfstring
    colstop.data  = ""
    colstart.command = 1
    colstop.command  = 2
    colstart.stack = defaultcolorstack
    colstop.stack  = defaultcolorstack

    if dontformat then
        node.set_attribute(colstart,att_dont_format,dontformat)
    end

    nodelist = node.insert_before(nodelist,nodelist,colstart)
    local last = node.tail(nodelist)
    nodelist = node.insert_after(nodelist,tail,colstop)

    setprop(colstart,"origin","setcolorifnecessary")
    setprop(colstop,"origin","setcolorifnecessary")
    return nodelist
end

-- Set an attribute to the list and all sublists.
function set_attribute_recurse(nodelist,attribute,value)
    while nodelist do
        if nodelist.id==vlist_node or nodelist.id==hlist_node  then
            set_attribute_recurse(nodelist.list,attribute,value)
        else
            set_attribute(nodelist,attribute,value)
        end
        nodelist=nodelist.next
    end
end

function set_fontfamily_if_necessary(nodelist,fontfamily)
    local fam
    while nodelist do
        if nodelist.id==vlist_node or nodelist.id==hlist_node  then
            fam = set_fontfamily_if_necessary(nodelist.list,fontfamily)
        elseif nodelist.id == glue_node and nodelist.subtype == 100  then
            fam = set_fontfamily_if_necessary(nodelist.leader,fontfamily)
        else
            fam = get_attribute(nodelist,"fontfamily")
            -- See #242, #235 and referenced bugs (and change 5af208f)
            if fam == 0 or ( fam == nil and nodelist.id == rule_node and get_attribute(nodelist,"publisher") == 1 )  then
                set_attribute(nodelist,"fontfamily",fontfamily)
                fam = fontfamily
            end
        end
        nodelist=nodelist.next
    end
    return fam
end

function break_url( nodelist )
    local p

    local slash = string.byte("/")
    for n in node.traverse_id(glyph_node,nodelist) do
        p = node.new("penalty")

        if n.char == slash then
            p.penalty=-10
        else
            p.penalty=-5
        end
        set_attribute(p,"hyperlink",get_attribute(n,"hyperlink"))
        p.next = n.next
        if n.next and n.next.prev then
            n.next.prev = p
        end
        n.next = p
        p.prev = n
    end
    return nodelist
end

function colorbar( wd,ht,dp,color,origin,orientation)
    local colorname = color
    if color == "-" then
        -- ok, ignore
    else
        if not colorname or colorname == "" then
            colorname = "black"
        end
        if not colors_module.colors[colorname] then
            err("Color %q not found",color)
            colorname = "black"
        end
    end

    local rule_start = node.new("whatsit","pdf_literal")
    setprop(rule_start,"origin","colorbar")
    if colorname ~= "-" then
        local ht_bp = sp_to_bp(ht)
        local wd_bp = sp_to_bp(wd)
        rule_start.mode = 0
        local data = "q ".. colors_module.colors[colorname].pdfstring
        local rule_width_bp, rule_length_bp

        if not options.tablerulefix then
            if orientation == "horizontal" then
                -- draw horizontal line
                rule_width_bp = ht_bp
                rule_length_bp = wd_bp
                data = data .. string.format(" %g w 0 %g m %g %g l s Q ", rule_width_bp, rule_width_bp / 2, rule_length_bp, rule_width_bp / 2)
            else
                rule_width_bp = wd_bp
                rule_length_bp = ht_bp
                data = data .. string.format(" %g w %g %g m %g %g l s Q ", rule_width_bp, rule_width_bp /2, rule_length_bp, rule_width_bp / 2, 0)
            end
        else
            if orientation == "horizontal" then
                rule_width_bp = ht_bp
                rule_length_bp = wd_bp
                -- draw horiztontal line a bit lower
                data = data .. string.format(" %g w %g %g m %g %g l s Q ", rule_width_bp, 0 , rule_width_bp / -2, rule_length_bp, rule_width_bp / -2)
            else
                rule_width_bp = wd_bp
                rule_length_bp = ht_bp
                -- draw the vertical borders in negative direction
                data = data .. string.format(" %g w %g %g m %g %g l s Q ", rule_width_bp, rule_width_bp / 2 , -1 * rule_length_bp, rule_width_bp / 2 , 0)
            end
        end
        rule_start.data = data
        if options.tablerulefix then
            setprop(rule_start,"data", data)
        end
        setprop(rule_start,"role",get_rolenum("Artifact"))
    end
    local h = node.hpack(rule_start)
    h.width = wd
    h.depth = dp
    h.height = ht
    origin = origin or "origin_colorbar"
    setprop(h,"origin",origin)
    return h
end

local explode = function(s,p)
   local t = { }
   for s in string.gmatch(s,p) do
       if s ~= "" then
           t[#t+1] = s
       end
   end
   return t
end

function montage( nodelist_background,nodelist_foreground, origin_x, origin_y )
    local wd_bg = nodelist_background.width
    local ht_bg = nodelist_background.height + nodelist_background.depth
    local dp_bg = nodelist_background.depth
    local wd_fg = nodelist_foreground.width
    local ht_fg = nodelist_foreground.height + nodelist_foreground.depth
    local wd = wd_bg - wd_fg
    local ht = ht_bg - ht_fg
    origin_x = 100 - origin_x
    origin_y = 100 - origin_y
    local x = math.round(  sp_to_bp(wd - (wd * origin_x) / 100  ), 3 )
    local y = math.round(  sp_to_bp(ht - (ht * origin_y) / 100  ), 3 )

    local pdf_literal_q = node.new("whatsit","pdf_literal")
    pdf_literal_q.data = string.format("1 0 0 1 %g %g cm ",x, y)

    local pdf_literal_Q = node.new("whatsit","pdf_literal")
    pdf_literal_Q.data = string.format("1 0 0 1 %g 0 cm ",-1 * math.round(sp_to_bp(wd_bg),3))

    local pdf_save    = node.new("whatsit","pdf_save")
    local pdf_restore = node.new("whatsit","pdf_restore")
    local hbox

    hbox = node.insert_before(nodelist_background,nodelist_background, pdf_save)
    hbox = node.insert_after(hbox,node.tail(hbox),pdf_literal_Q)
    hbox = node.insert_after(hbox,node.tail(hbox),pdf_literal_q)
    hbox = node.insert_after(hbox,node.tail(hbox),nodelist_foreground)

    hbox = node.hpack(hbox)
    hbox.depth = 0
    hbox = node.insert_after(hbox,node.tail(hbox),pdf_restore)

    hbox = node.vpack(hbox)
    hbox.width = wd_bg
    hbox.height = ht_bg - dp_bg
    hbox.depth = dp_bg
    return hbox
end

--- Apply transformation matrix to object given at _nodelist_. Called from commands#transformation.
function matrix( nodelist,matrix,origin_x,origin_y )
    local wd,ht = nodelist.width, nodelist.height
    local tbl = explode(matrix,"[^\t ]+")

    origin_x = 100 - origin_x
    origin_y = 100 - origin_y
    local x = math.round(  sp_to_bp(wd - (wd * origin_x) / 100  )  , 3 )
    local y = math.round(  sp_to_bp(ht - (ht * origin_y) / 100  )  , 3 )

    local pdf_literal_q = node.new("whatsit","pdf_literal")
    local pdf_literal_Q = node.new("whatsit","pdf_literal")

    pdf_literal_q.data = string.format("q 1 0 0 1 %g %g cm q %g %g %g %g %g %g cm q 1 0 0 1 %g %g cm ",x,-1 * y,tbl[1],tbl[2],tbl[3],tbl[4],tbl[5],tbl[6],x * -1,y )
    pdf_literal_Q.data = "Q Q Q"

    local pdf_save    = node.new("whatsit","pdf_save")
    local pdf_restore = node.new("whatsit","pdf_restore")

    local hbox
    hbox = node.insert_before(nodelist,nodelist,pdf_literal_q)
    node.insert_after(nodelist,nodelist,pdf_literal_Q)
    hbox = node.insert_before(hbox,pdf_literal_q,pdf_save)
    hbox = node.hpack(hbox)

    hbox.depth = 0
    node.insert_after(hbox,node.tail(hbox),pdf_restore)

    local newbox = node.vpack(hbox)
    return newbox
end

--- Rotate an object clockwise with a given angle (in degrees).
---
--- First rotate the object at the top left corner (default)
--- If the origin is not top left, we need to shift the object
function rotate( nodelist,angle,origin_x,origin_y )
    local wd,ht = nodelist.width, nodelist.height + nodelist.depth
    nodelist.width = 0
    nodelist.height = 0
    nodelist.depth = 0

    -- positive would be counter clockwise, but CSS is clockwise. So we multiply by -1
    local angle_rad = -1 * math.rad(angle)
    local sin = math.round(math.sin(angle_rad),3)
    local cos = math.round(math.cos(angle_rad),3)
    local q = node.new("whatsit","pdf_literal")
    q.mode = 0

    origin_x = 100 - origin_x
    origin_y = 100 - origin_y
    local x = math.round(  sp_to_bp(wd - (wd * origin_x) / 100  )  , 3 )
    local y = math.round(  sp_to_bp(ht - (ht * origin_y) / 100  )  , 3 )
    q.data = string.format("q 1 0 0 1 %g -%g cm  q %g %g %g %g 0 0 cm q 1 0 0 1 -%g %g cm ",x,y,cos,sin, -1 * sin,cos,x,y )
    q.next = nodelist
    local tail = node.tail(nodelist)
    local Q = node.new("whatsit","pdf_literal")
    Q.data = "Q Q Q"
    tail.next = Q
    local tmp = node.vpack(q)
    tmp.width = 0
    tmp.height = 0
    tmp.depth = 0
    return tmp
end

--- Rotate a table cell clockwise with a given angle (in degrees).
-- This is a simple and very basic implementation which needs to be extended in the future.
function rotateTd( nodelist,angle, width_sp)
    if angle % 360 == 0 then return nodelist end

    -- positive would be counter clockwise, but CSS is clockwise. So we multiply by -1
    local angle_rad = -1 * math.rad(angle)

    -- With multi paragraph table cells it is easier if we have only one node to deal with.
    if nodelist.next then
        nodelist = node.vpack(nodelist)
    end

    -- When text is rotated, it needs to get shifted to the right and to the bottom
    local _wd,_ht,_dp = nodelist.width, nodelist.height,nodelist.depth
    local ht = _ht + _dp

    nodelist.width = 0
    nodelist.height = 0
    nodelist.depth = 0

    local sin = math.round(math.sin(angle_rad),3)
    local cos = math.round(math.cos(angle_rad),3)

    local q = node.new("whatsit","pdf_literal")
    q.mode = 0

    local shift_x, shift_y

    local shift_x_wd = cos * _wd
    local shift_x_ht = sin * ht
    if shift_x_wd > 0 then shift_x_wd = 0 end
    if shift_x_ht > 0 then shift_x_ht = 0 end

    local shift_y_wd = -1 * sin * _wd
    local shift_y_ht = cos * ht
    if shift_y_wd > 0 then shift_y_wd = 0 end
    if shift_y_ht > 0 then shift_y_ht = 0 end

    shift_x =  sp_to_bp( shift_x_ht + shift_x_wd) * -1
    shift_y =  sp_to_bp( shift_y_ht + shift_y_wd)

    q.data = string.format("q %g %g %g %g %g %g cm ",cos,sin, -1 * sin,cos,shift_x,shift_y)
    local Q = node.new("whatsit","pdf_literal")
    Q.data = "Q"

    _,q = node.insert_before(nodelist,nodelist,q)
    _,Q = node.insert_after(q,nodelist,Q)
    q = node.vpack(q)

    q.width  = math.abs(_wd * cos) + math.abs(ht * sin)
    q.height = math.abs(ht * cos) + math.abs(_wd * sin)
    q.depth  = 0

    return q
end

--- Rotate a text on a given angle (`angle` on textblock).
function rotate_textblock( nodelist,angle )
    local wd,ht = nodelist.width, nodelist.height + nodelist.depth
    nodelist.width = 0
    nodelist.height = 0
    nodelist.depth = 0
    local angle_rad = math.rad(angle)
    local sin = math.round(math.sin(angle_rad),3)
    local cos = math.round(math.cos(angle_rad),3)
    local q = node.new("whatsit","pdf_literal")
    q.mode = 0
    local shift_x = math.round(math.min(0,math.sin(angle_rad) * sp_to_bp(ht)) + math.min(0,     math.cos(angle_rad) * sp_to_bp(wd)),3)
    local shift_y = math.round(math.max(0,math.sin(angle_rad) * sp_to_bp(wd)) + math.max(0,-1 * math.cos(angle_rad) * sp_to_bp(ht)),3)
    q.data = string.format("q %g %g %g %g %g %g cm",cos,sin, -1 * sin,cos, -1 * shift_x ,-1 * shift_y )
    q.next = nodelist
    local tail = node.tail(nodelist)
    local Q = node.new("whatsit","pdf_literal")
    Q.data = "Q"
    tail.next = Q
    local tmp = node.vpack(q)
    tmp.width  = math.abs(wd * cos) + math.abs(ht * math.cos(math.rad(90 - angle)))
    tmp.height = math.abs(ht * math.sin(math.rad(90 - angle))) + math.abs(wd * sin)
    tmp.depth = 0
    return tmp
end

--- Make a string XML safe
function xml_escape( str )
    if type(str) == "table" then
        str = table.concat(str)
    end
    if not str then return "" end
    local replace = {
        [">"] = "&gt;",
        ["<"] = "&lt;",
        ["\""] = "&quot;",
        ["&"] = "&amp;",
    }
    -- FIXME, str can be bool
    local ret = string.gsub(str,".",replace)
    return ret
end


--- See commands#save_dataset() for  documentation on the data structure for `xml_element`.
function xml_to_string_newxpath( xml_element, level,namespace_written)
    local new_namespaces = copy_table_from_defaults(namespace_written or {})
    local str = ""
    if type(xml_element) == "string" then
        return xml_escape(xml_element)
    end
    if type(xml_element) ~= "table" then
        err("xml_to_string is not a table, but a %s %q",type(xml_element),tostring(xml_element))
        return "error in publisher run"
    end
    level = level or 0
    local eltname = xml_element[".__name"] or xml_element[".__local_name"] or ""
    if level == 0 and eltname == "" then eltname = "undefined" end
    if eltname ~= "" then
        str = str ..  "<" .. eltname
        if type(xml_element[".__attributes"]) == "table" then
            for k,v in pairs(xml_element[".__attributes"]) do
                str = str .. string.format(" %s=%q", k,xml_escape(v))
            end
        end
        if xml_element[".__ns"] then
            for k,v in pairs(xml_element[".__ns"]) do
                local key = k
                if new_namespaces[k] == nil then
                    if type(k) == "string" then
                        if k == "" then
                            k = "xmlns"
                        else
                            k = "xmlns:" .. k
                        end
                        str = str .. string.format(" %s=%q", k,xml_escape(v))
                    end
                end
                new_namespaces[key] = true
            end
        end
        str = str .. ">"
    end
    for i,v in ipairs(xml_element) do
        if type(v) == "string" and v == "" then
            -- ok, nothing do do
        else
            str = str .. xml_to_string_newxpath(v,level + 1,new_namespaces)
        end
    end
    if eltname ~= "" then
        str = str ..  "</" .. eltname .. ">"
    end
    return str
end


--- See commands#save_dataset() for  documentation on the data structure for `xml_element`.
function xml_to_string( xml_element, level )
    local str = ""
    if type(xml_element) == "string" then
        return xml_escape(xml_element)
    end
    if type(xml_element) ~= "table" then
        err("xml_to_string is not a table, but a %s %q",type(xml_element),tostring(xml_element))
        return "error in publisher run"
    end
    level = level or 0
    local eltname = xml_element[".__name"] or xml_element[".__local_name"] or ""
    if level == 0 and eltname == "" then eltname = "undefined" end
    if eltname ~= "" then
        str = str ..  "<" .. eltname
        for k,v in pairs(xml_element) do
            if type(k) == "string" and not k:match("^%.") then
                str = str .. string.format(" %s=%q", k,xml_escape(v))
            end
        end
        if xml_element[".__ns"] then
            for k,v in pairs(xml_element[".__ns"]) do
                if type(k) == "string" then
                    if k == "" then
                        k = "xmlns"
                    else
                        k = "xmlns:" .. k
                    end
                    str = str .. string.format(" %s=%q", k,xml_escape(v))
                end
            end
        end
        str = str .. ">"
    end
    for i,v in ipairs(xml_element) do
        if type(v) == "string" and v == "" then
            -- ok, nothing do do
        else
            str = str .. xml_to_string(v,level + 1)
        end
    end
    if eltname ~= "" then
        str = str ..  "</" .. eltname .. ">"
    end
    return str
end

function xml_stringvalue( self )
    if type(self) == "string" then return self end
    local ret = {}
    for i=1,#self do
        local val = self[i]
        if type(val) == "table" then
            ret[#ret + 1] = xml_stringvalue(val)
        else
            ret[#ret + 1] = tostring(val)
        end
    end
    return table.concat(ret)
end

xml_stringvalue_mt = {
    __tostring = xml_stringvalue
}



--- Hyphenation and language handling
--- ---------------------------------

--- We map from symbolic names to (part of) file names. The hyphenation pattern files are
--- in the format `hyph-XYZ.pat.txt` and we need to find out that `XYZ` part.
language_mapping = {
    ["Ancient Greek"]                = "grc",
    ["Armenian"]                     = "hy",
    ["Bahasa Indonesia"]             = "id",
    ["Basque"]                       = "eu",
    ["Bulgarian"]                    = "bg",
    ["Catalan"]                      = "ca",
    ["Chinese"]                      = "zh",
    ["Croatian"]                     = "hr",
    ["Czech"]                        = "cs",
    ["Danish"]                       = "da",
    ["Dutch"]                        = "nl",
    ["English"]                      = "en_GB",
    ["English (Great Britain)"]      = "en_GB",
    ["English (USA)"]                = "en_US",
    ["Esperanto"]                    = "eo",
    ["Estonian"]                     = "et",
    ["Finnish"]                      = "fi",
    ["French"]                       = "fr",
    ["Galician"]                     = "gl",
    ["German"]                       = "de",
    ["Greek"]                        = "el",
    ["Gujarati"]                     = "gu",
    ["Hindi"]                        = "hi",
    ["Hungarian"]                    = "hu",
    ["Icelandic"]                    = "is",
    ["Irish"]                        = "ga",
    ["Italian"]                      = "it",
    ["Kannada"]                      = "kn",
    ["Kurmanji"]                     = "ku",
    ["Latvian"]                      = "lv",
    ["Lithuanian"]                   = "lt",
    ["Malayalam"]                    = "ml",
    ["Norwegian Bokmål"]             = "nb",
    ["Norwegian Nynorsk"]            = "nn",
    ["Other"]                        = "--",
    ["Polish"]                       = "pl",
    ["Portuguese"]                   = "pt",
    ["Romanian"]                     = "ro",
    ["Russian"]                      = "ru",
    ["Sanskrit"]                     = "sa",
    ["Serbian"]                      = "sr",
    ["Serbian (cyrillic)"]           = "sc",
    ["Slovak"]                       = "sk",
    ["Slovenian"]                    = "sl",
    ["Spanish"]                      = "es",
    ["Swedish"]                      = "sv",
    ["Turkish"]                      = "tr",
    ["Ukrainian"]                    = "uk",
    ["Welsh"]                        = "cy",
}


language_filename = {
    ["bg"]    = "bg",
    ["ca"]    = "ca",
    ["cs"]    = "cs",
    ["cy"]    = "cy",
    ["da"]    = "da",
    ["de"]    = "de-1996",
    ["el"]    = "el-monoton",
    ["en"]    = "en-gb",
    ["en_gb"] = "en-gb",
    ["en_us"] = "en-us",
    ["eo"]    = "eo",
    ["es"]    = "es",
    ["et"]    = "et",
    ["eu"]    = "eu",
    ["fi"]    = "fi",
    ["fr"]    = "fr",
    ["ga"]    = "ga",
    ["gl"]    = "gl",
    ["grc"]   = "grc",
    ["gu"]    = "gu",
    ["hi"]    = "hi",
    ["hr"]    = "hr",
    ["hu"]    = "hu",
    ["hy"]    = "hy",
    ["id"]    = "id",
    ["is"]    = "is",
    ["it"]    = "it",
    ["ku"]    = "kmr",
    ["kn"]    = "kn",
    ["lt"]    = "lt",
    ["ml"]    = "ml",
    ["lv"]    = "lv",
    ["nb"]    = "nb",
    ["nl"]    = "nl",
    ["nn"]    = "nn",
    ["no"]    = "nb",
    ["pl"]    = "pl",
    ["pt"]    = "pt",
    ["ro"]    = "ro",
    ["ru"]    = "ru",
    ["sa"]    = "sa",
    ["sk"]    = "sk",
    ["sl"]    = "sl",
    ["sr"]    = "sr",
    ["sc"]    = "sr-cyrl",
    ["sv"]    = "sv",
    ["tr"]    = "tr",
    ["uk"]    = "uk",
    ["zh"]    = "",
    ["--"]    = "",
}

--- Once a hyphenation pattern file is loaded, we only need the _id_ of it. This is stored in the
--- `languages` table. Key is the filename part (such as `de-1996`) and the value is the internal
--- language id.
languages = {}
languages_id_lang = {}

--- Return a lang object
function get_language(id_or_locale_or_name)
    local orig_id_or_locale_or_name = id_or_locale_or_name
    local num = tonumber(id_or_locale_or_name)
    if num then
        return languages_id_lang[num]
    end
    local locale = string.lower(id_or_locale_or_name)

    if language_mapping[id_or_locale_or_name] then
        locale = language_mapping[id_or_locale_or_name]
    end
    locale = string.lower(locale)
    if languages[locale] then
        return languages[locale]
    end

    local filename_part
    if language_filename[locale] then
        filename_part = language_filename[locale]
    else
        local sep = "_"
        if string.match( locale ,"%-" ) then
            sep = "-"
        end
        local langcode, _ = table.unpack(string.explode(locale,sep))
        if language_filename[langcode] then
            filename_part = language_filename[langcode]
        end
    end

    local l = lang.new()

    if filename_part == "" then
        -- ignore this
        -- probably cjk or another language without hyphenation patterns
    elseif not filename_part then
        warning("Can't find hyphenation patterns for language %s",tostring(orig_id_or_locale_or_name))
        return 0
    else
        local filename = string.format("hyph-%s.pat.txt",filename_part)
        main.log("debug","Loading hyphenation pattern","filename",filename)
        local path = kpse.find_file(filename)
        local pattern_file = io.open(path)
        local pattern = pattern_file:read("*all")
        pattern_file:close()

        l:patterns(pattern)
    end

    local id = l:id()
    main.log("debug","Language ID","id",id)
    local ret = { id = id, l = l, locale = locale }
    languages_id_lang[id] = ret
    languages[locale] = ret
    return ret
end

--- The language name is something like `German` or a locale.
function get_languagecode( locale_or_name )
    local tmp = get_language(locale_or_name)
    if type(tmp) ~= "table" then
        return 0
    end
    return tmp.id
end

function set_mainlanguage( mainlanguage )
    main.log("info","Setting default language","lang",mainlanguage or "?")
    defaultlanguage = get_languagecode(mainlanguage)
end


--- Return the language numbers used in this nodelist.
--- Used before `do_linebreak()` to change pre-hyphenchar
--- temporarily.
function get_languages_used( nodelist )
    local langs = {}
    for n in node.traverse_id(glyph_node,nodelist) do
        langs[n.lang] = true
    end
    local ret = {}
    for k,_ in pairs(langs) do
        ret[#ret + 1] = k
    end
    return ret
end


--- Misc
--- --------
function set_pageformat( wd,ht )
    options.pagewidth   = wd
    options.pageheight  = ht
    tex.pagewidth =  wd
    tex.pageheight = ht
end

-- Return remaining height (sp), first row, last row
function get_remaining_height(area,allocate)
    local cols = current_grid:number_of_columns(area)
    local startcol = 1
    local row,firstrow,lastrow,maxrows
    firstrow = current_grid:current_row(area)
    if not firstrow then
        err("get remaining height: no current row")
        firstrow = 1
    end
    maxrows = current_grid:number_of_rows(area)
    if allocate == "auto" then
        while firstrow <= maxrows and (not current_grid:row_has_some_space(firstrow,area)) do
            firstrow = firstrow + 1
        end

        local row = firstrow + 1
        while row <= maxrows and current_grid:row_has_some_space(row,area) do
            row = row + 1
        end

        if row > maxrows then
            return ( row - firstrow ) * current_grid.gridheight, firstrow
        end
        local lastrow = row
        while not current_grid:fits_in_row_area(startcol,cols,lastrow,area) and lastrow <= maxrows do

            lastrow = lastrow + 1
        end
        lastrow = lastrow - 1
        if lastrow == firstrow then lastrow = nil end
        if lastrow >= maxrows then lastrow = nil end
        return (row - firstrow) * current_grid.gridheight, firstrow,lastrow
    end
    if not tonumber(maxrows) then
        main.log("error","maxrows not set, why?")
        return
    end
    if not current_grid:fits_in_row_area(startcol,cols,firstrow,area) then
        while firstrow <= maxrows do
            if current_grid:fits_in_row_area(startcol,cols,firstrow,area) then
                break
            end
            firstrow = firstrow + 1
        end
    end

    row = firstrow

    while current_grid:fits_in_row_area(startcol,cols,row,area) and row <= maxrows do
        row = row + 1
    end

    lastrow = row

    while row <= maxrows do
        if current_grid:fits_in_row_area(startcol,cols,row,area) then
            return ( lastrow - firstrow)  * current_grid.gridheight, firstrow, firstrow
        end
        row = row + 1
    end
    return ( lastrow - firstrow)  * current_grid.gridheight, firstrow, nil

end

function next_row(rownumber,areaname,rows,dataxml)
    local grid = current_grid

    if rownumber then
        grid:set_current_row(rownumber,areaname,"next_row")
        return
    end

    local current_row
    local noc = grid:number_of_columns(areaname)
    if noc == nil then
        main.log("error","number of columns is not set for area","area",areaname)
        return
    end
    current_row = grid:find_suitable_row(1,noc,rows,areaname)
    if not current_row then
        next_area(areaname,nil, dataxml,"next_row")
        setup_page(nil,"next_row",dataxml)
        grid = current_page.grid
        grid:set_current_row(1,areaname,"cannot find suitable row")
    else
        -- Version 2.7.3 and before had the problem that the cursor is past the right
        -- edge. See bug #105 (https://github.com/speedata/publisher/issues/105) for
        -- a description.
        -- A <NextRow rows="1" /> would go to the next free row, which could be the current
        -- row.
        -- <NextRow rows="1" /> should instead go to the beginning of the next row. So a
        -- <NextRow rows="1" /> directly after <PlaceObject>...</PlaceObject> width the right
        -- edge at the right margin will leave one blank line.
        -- The old behavior is to decrease 1 from the movement, which makes no sense these days.
        local dec = 0
        if grid:current_column(areaname) > 1 then
            dec = 1
        end
        local grid_number_of_rows = grid:number_of_rows(areaname)
        if current_row + rows - dec > grid_number_of_rows then
            next_area(areaname,nil, dataxml,"next_row 2")
            setup_page(nil,"next_row",dataxml)
            grid = current_page.grid
            grid:set_current_row(1,areaname,"next_row -> next_area")
            return
        end
        grid:set_current_row(current_row + rows - dec,areaname,"next_row has_current_row")
        grid:set_current_column(1,areaname)
    end
end

function empty_block()
    local r = node.new("hlist")
    r.width = 0
    r.height = 0
    r.depth = 0
    local v = node.vpack(r)
    return v
end


function emergency_block()
    local r = node.new(rule_node)
    r.width = 5 * 2^16
    r.height = 5 * 2^16
    r.depth = 0
    local v = node.vpack(r)
    return v
end


-- resolve all font aliases
function get_fontname(fontname)
    if not fontname then return nil end
    local result = fontname
    while true do
        if fontaliases[result] then
            result = fontaliases[result]
        else
            break
        end
    end
    return result
end


--- Defaults
--- --------

--- This function is only called once from `dothings()` during startup phase. We define
--- a family with regular, bold, italic and bolditalic font with size 10pt (we always
--- measure font size in dtp points)
function define_default_fontfamily()
    define_fontfamily(
        "TeXGyreHeros-Regular",
        "TeXGyreHeros-Bold",
        "TeXGyreHeros-Italic",
        "TeXGyreHeros-BoldItalic",
        "text",
        tenpoint_sp,
        twelvepoint_sp
    )

    fontaliases["sans"] = "TeXGyreHeros-Regular"
    fontaliases["sans-bold"] = "TeXGyreHeros-Bold"
    fontaliases["sans-italic"] = "TeXGyreHeros-Italic"
    fontaliases["sans-bolditalic"] = "TeXGyreHeros-BoldItalic"

    fontaliases["serif"] = "CrimsonPro-Regular"
    fontaliases["serif-bold"] = "CrimsonPro-Bold"
    fontaliases["serif-italic"] = "CrimsonPro-Italic"
    fontaliases["serif-bolditalic"] = "CrimsonPro-BoldItalic"

    fontaliases["monospace"] = "CamingoCode-Regular"
    fontaliases["monospace-bold"] = "CamingoCode-Bold"
    fontaliases["monospace-italic"] = "CamingoCode-Italic"
    fontaliases["monospace-bolditalic"] = "CamingoCode-BoldItalic"
end

function define_fontfamily( regular,bold,italic,bolditalic, name, size, baselineskip,scriptsize,supershift,subshift)
    if not size then
        main.log("error","DefineFontfamily needs size value")
        return
    end
    if not scriptsize then
        scriptsize = math.round(size * 0.8,0)
    end
    if not supershift then
        supershift = math.round(size * 0.3,0)
    end
    if not subshift then
        subshift = math.round(size * 0.3,0)
    end
    if not tonumber(size) then
        main.log("error","DefineFontfamily needs size value")
        return
    end
    local fam={
        size         = size,
        baselineskip = baselineskip,
        scriptsize   = scriptsize,
        supershift   = supershift,
        subshift     = subshift,
        name = name
    }
    local ok,tmp
    if regular then
        ok,tmp = fonts.make_font_instance(regular,fam.size)
        if not ok then return false, tmp end
        fam.normal = tmp
        fam.fontfaceregular = regular
        ok,tmp = fonts.make_font_instance(regular,fam.scriptsize)
        if not ok then return false, tmp end
        fam.normalscript = tmp
    end

    if bold then
        ok,tmp = fonts.make_font_instance(bold,fam.size)
        if not ok then return false, tmp end
        fam.bold = tmp
        fam.fontfacebold = bold
        ok,tmp = fonts.make_font_instance(bold,fam.scriptsize)
        if not ok then return false, tmp end
        fam.boldscript = tmp
    end

    if italic then
        ok,tmp = fonts.make_font_instance(italic,fam.size)
        if not ok then return false, tmp end
        fam.italic = tmp
        fam.fontfaceitalic = italic
        ok,tmp = fonts.make_font_instance(italic,fam.scriptsize)
        if not ok then return false, tmp end
        fam.italicscript = tmp
    end

    if bolditalic then
        ok,tmp = fonts.make_font_instance(bolditalic,fam.size)
        if not ok then return false, tmp end
        fam.bolditalic = tmp
        fam.fontfacebolditalic = bolditalic
        ok,tmp = fonts.make_font_instance(bolditalic,fam.scriptsize)
        if not ok then return false, tmp end
        fam.bolditalicscript = tmp
    end

    fonts.lookup_fontfamily_number_instance[#fonts.lookup_fontfamily_number_instance + 1] = fam
    local fontnumber = #fonts.lookup_fontfamily_number_instance
    fonts.lookup_fontfamily_name_number[name] = fontnumber
    main.log("info","Define font family","name",name,"size",math.round(size / factor, 3),"leading",math.round(baselineskip / factor,3), "id",fontnumber)
    if regular then main.log("debug", "Instance created", "instance", "regular", "name", regular,"id",fam.normal) end
    if bold then main.log("debug", "Instance created", "instance", "bold", "name", bold,"id",fam.bold) end
    if italic then main.log("debug", "Instance created", "instance", "italic", "name", italic,"id",fam.italic) end
    if bolditalic then main.log("debug", "Instance created", "instance", "bolditalic", "name", bolditalic,"id",fam.bolditalic) end
    return fontnumber
end


-- deepcopy is for <Copy-of>
function deepcopy(t)
    local typ = type(t)
    if typ ~= 'table' then return t end
    local mt = getmetatable(t)
    local res = {}
    for k,v in pairs(t) do
        typ = type(v)
        if typ == 'table' then
            if k ~= ".__parent" and k ~= ".__context" then
                v = deepcopy(v)
            end
        else
            if node.is_node(v) then
                v = node.copy_list(v)
            end
        end
        res[k] = v
    end
    setmetatable(res,mt)
    return res
end


function copy_table_from_defaults( defaults )
    if type(defaults) ~= "table" then
        return defaults
    end
    local newtbl = {}
    for key,value in next,defaults,nil do
        newtbl[key] = value
    end
    return newtbl
end

-- Return the height of the page given by the relative page number
-- (starting from the current_pagenumber).
-- This is used in tables to get the height of a page in a multi
-- page table. Called from tabular.lua / set in commands.lua (#table)
function getheight( relative_framenumber,dataxml )
    local grid = current_grid
    local cp, cg, cpn, cfn -- current page, current grid, current page number, current frame number
    cp = current_page
    cg = current_grid
    cpn = current_pagenumber
    local areaname
    if newxpath then
        areaname = dataxml.vars["__currentarea"]
    else
        areaname = xpath.get_variable("__currentarea")
    end
    areaname = areaname or default_areaname
    local current_framenumber = grid:framenumber(areaname)
    cfn = current_framenumber

    local thispagenumber = current_pagenumber
    local thispage
    c = 1
    while c < relative_framenumber do
        if grid:number_of_frames(areaname) == current_framenumber then
            thispagenumber = thispagenumber + 1
            thispage = pages[thispagenumber]
            -- be aware that setup_page(..,) calls setup_page() but without
            -- parameter. Therefore the current_pagenumber has to be set
            current_pagenumber = thispagenumber
            if not thispage then
                setup_page(thispagenumber,"getheight",dataxml)
            end
            current_framenumber = 1
        else
            current_framenumber = current_framenumber + 1
        end
        current_page = pages[thispagenumber]
        current_pagenumber = thispagenumber
        current_grid = current_page.grid
        c = c + 1
    end
    local firstrow = current_grid:first_free_row(areaname,current_framenumber)
    local remaining_height = current_grid:remaining_height_sp(firstrow,areaname,current_framenumber)
    current_pagenumber = cpn
    current_grid = cg
    current_page = cp
    current_grid:set_framenumber(areaname,cfn)

    return remaining_height
end


-- Return true iff the paragraph has at lines or less text
-- lines left over and is not at the last line.
function less_or_equal_than_n_lines( nodelist, lines )
    if lines == 0 then return false end
    for i=1,lines - 1 do
        if nodelist.id == publisher.hlist_node and nodelist.next then
            nodelist = nodelist.next
        else
            if i == 1 then
                return false
            end
        end
    end
    return nodelist.next == nil
end

function join_table_to_box(objects,from)
    for i=1,#objects - 1 do
        objects[i].next = objects[i+1]
    end
    if objects[1] == nil then
        return nil
    end
    node.slide(objects[1])
    local vbox = node.vpack(objects[1])
    setprop(vbox,"origin","join_table_hbox " .. (from or "") )
    return vbox
end


--- vsplit
--- ======
--- The idea of vsplit is to take a long paragraph and break it into small pieces of text
--- ![Idea of vsplit](img/vsplit.png)
--- Of course its not without things to take care of.
---
---  1. Orphans and widows
---  1. The size of the destination area
---
--- Input
--- -----
--- The table `objects_t` is an array of vboxes, containing material for the current frame of height
--- `frameheight`. It is not defined if the height of the vboxes is larger than the height of the frame.
--- Therefore we dissect all the paragraphs and place them into one large list, the `hlist`.
---
--- Output
--- ------
--- The return value is a vbox that should be placed in the PDF and has a height <= frameheight. If there
--- is material left over for a next area, the `objects_t` table is changed and vsplit gets called again.
--- Making `objects_t` empty is a signal for the function calling vsplit (commands/text) that all
--- text has been put into the PDF.
function vsplit( objects_t, parameter )
    --- Step 1: collect all the objects in one big table.
    --- ------------------------------------------------
    --- The objects that are not allowed to break are temporarily
    --- collected in a special vertical list that gets vpacked to
    --- disallow an "area" break.
    ---
    --- ![Step 1](img/vsplit2.png)
    --- (assuming that there is a `break-below="no"` for the text format of the header).
    local balance = parameter.balance
    local valignlast = parameter.valignlast
    local frameheight = parameter.maxheight
    local lastpaddingbottommax = parameter.lastpaddingbottommax


    local hlist = {}
    -- We need the height for the decision to balance the text
    local ht_hlist = 0


    -- a list for hboxes with break_below = true
    local tmplist = {}
    local count_lists = #objects_t
    local vlist = table.remove(objects_t,1)
    local i = 1
    local margin_newcolumn
    while vlist do
        local head = vlist.head
        while head do
            local bordernumber = get_attribute(head,"bordernumber")
            if bordernumber then
                -- move bordernumber to vlist
                set_attribute(vlist,"bordernumber",bordernumber)
                clear_attribute(vlist,"bordernumber")
            end

            local tmp_margin_newcolumn = node.has_attribute(head, publisher.att_margin_newcolumn)

            if tmp_margin_newcolumn then
                margin_newcolumn = tmp_margin_newcolumn
            end
            node.set_attribute(head,publisher.att_margin_newcolumn,margin_newcolumn)

            if i == count_lists and head.next == nil then
                -- the last object must not be in the tmplist
                -- But don't reset extend_to_next_block - we need it to persist for break-after: avoid
                node.unset_attribute(head,publisher.att_break_below_forbidden)
            end
            head.prev = nil
            local break_below_forbidden = node.has_attribute(head,publisher.att_break_below_forbidden)
            local break_before = node.has_attribute(head,publisher.att_break_before)
            if break_below_forbidden and not break_before then
                node.unset_attribute(head,publisher.att_margin_newcolumn)
                tmplist[#tmplist + 1] = head
                local tmp = head.next
                head.next = nil
                head = tmp
            else
                -- break allowed
                -- if there is anything in the tmplist, we vpack it and add it to the current hlist.
                if #tmplist > 0 then
                    tmplist[#tmplist + 1] = head

                    local tmp = head.next
                    head.next = nil
                    head = tmp

                    local margin_newcolumn_tmplist = node.has_attribute(tmplist[1], publisher.att_margin_newcolumn)
                    local vbox = join_table_to_box(tmplist,"break allowed")
                    node.set_attribute(vbox,publisher.att_margin_newcolumn,margin_newcolumn_tmplist)

                    hlist[#hlist + 1] = vbox
                    ht_hlist = ht_hlist + vbox.height + vbox.depth
                    tmplist = {}
                else
                    hlist[#hlist + 1] = head
                    if head.id == publisher.glue_node then
                        ht_hlist = publisher.get_glue_size(head)
                    else
                        ht_hlist = ht_hlist + ( head.height or 0 ) + ( head.depth or 0 )
                    end
                    local tmp = head.next
                    head.next = nil
                    head = tmp
                end
            end
        end
        vlist = table.remove(objects_t,1)
        i = i + 1
    end
    -- the hlist now has lot's of rows. Widows/orphans are packed together in a vbox with n hboxes.

    if balance > 1 and ht_hlist < balance * frameheight then
        -- TODO: splitpos should be based on the actual height
        local splitpos = math.ceil(#hlist / balance)

        local margin_newcolumn_obj1 = node.has_attribute(hlist[1], publisher.att_margin_newcolumn)
        if margin_newcolumn_obj1 and margin_newcolumn_obj1 > 0 then
            table.insert(hlist,1,publisher.add_glue(nil,"head",{width=margin_newcolumn_obj1}))
            splitpos = splitpos + 1
        end
        local obj1 = join_table_to_box({table.unpack(hlist,1,splitpos)},"balance > 1 obj1")
        if hlist[splitpos + 1] then
            local margin_newcolumn_obj2 = node.has_attribute(hlist[splitpos + 1], publisher.att_margin_newcolumn)
            if margin_newcolumn_obj2 and margin_newcolumn_obj2 > 0 then
                table.insert(hlist,splitpos + 1,publisher.add_glue(nil,"head",{width=margin_newcolumn_obj2}))
            end
            local obj2 = join_table_to_box({table.unpack(hlist,splitpos + 1)},"balance > 1 obj2")
            if valignlast == "bottom" then
                local remaining_height = frameheight - math.max(obj1.height, obj2.height)

                if remaining_height > lastpaddingbottommax then
                    remaining_height = remaining_height - lastpaddingbottommax
                end
                obj1.head = publisher.add_glue(obj1.head,"head",{width = remaining_height} )
                obj2.head = publisher.add_glue(obj2.head,"head",{width = remaining_height} )
            end
            return obj1, obj2
        else
            if valignlast == "bottom" then
                local remaining_height = frameheight - obj1.height
                if remaining_height > lastpaddingbottommax then
                    remaining_height = remaining_height - lastpaddingbottommax
                end
                obj1.head = publisher.add_glue(obj1.head,"head",{width = remaining_height} )
            end
            return obj1
        end
    end
    --- Step 2: Fill vbox (the return value)
    --- ------------------------------------
    --- Two cases: the objects have enough material to fill up the area (a)
    --- or we have no objects left for the area and return the final vbox for this area. (b)
    --- The task is to go though collection of h/vboxes (the hlist) and create one big vbox.
    --- This is done by filling the table `thisarea`.
    ---
    --- ![final step for area](img/vsplit3.png)
    local goal = frameheight
    local accumulated_height = 0
    local thisarea = {}
    local remaining_objects = {}
    local area_filled = false
    local lineheight = 0
    while not area_filled do
        for i=1,#hlist do
            local hbox = table.remove(hlist,1)
            local break_before = node.has_attribute(hbox,publisher.att_break_before)
            node.set_attribute(hbox,publisher.att_break_before,nil)
            if #thisarea == 0 then
                -- This is for a different margin-top at the beginning of a new column.
                if hbox.id == publisher.vlist_node then
                    local vbox = hbox
                    if vbox.list and vbox.list.id == publisher.glue_node then
                        local margin_top_boxstart = node.has_attribute(vbox.list, publisher.att_margin_top_boxstart)
                        vbox.list.width = margin_top_boxstart
                        hbox = node.vpack(vbox.list)
                    end
                end
            end

            if #thisarea == 0 and node.has_attribute(hbox, publisher.att_omit_at_top) then
                -- When the margin-below appears at the top of the new frame, we just ignore
                -- it. Too bad Lua doesn't have a 'next' in for-loops
            else
                local margin_newcolumn = node.has_attribute(hbox, publisher.att_margin_newcolumn)
                if margin_newcolumn and margin_newcolumn > 0 and #thisarea == 0 then
                    thisarea[#thisarea + 1] = publisher.add_glue(nil,"head",{width=margin_newcolumn})
                    lineheight = margin_newcolumn
                end

                if hbox.id == publisher.hlist_node or hbox.id == publisher.vlist_node then
                    lineheight = lineheight +  hbox.height + hbox.depth
                elseif hbox.id == publisher.glue_node then
                    lineheight = lineheight + get_glue_value(hbox,"width")
                elseif hbox.id == publisher.rule_node then
                    lineheight = lineheight + hbox.height + hbox.depth
                elseif hbox.id == publisher.whatsit_node then
                    -- ignore
                else
                    w("unknown node 1: %d",hbox.id)
                end
                -- 20 is some rounding error
                -- break_before: 1 = "page" (only if not at top), 2 = "always" (force break)
                -- Use page_has_content from parameter if available, otherwise fall back to #thisarea
                local page_has_content = parameter.page_has_content or (#thisarea > 0)
                local force_break = (break_before == 2) or (break_before == 1 and page_has_content)
                if accumulated_height + lineheight <= goal + 20 and not force_break then
                    thisarea[#thisarea + 1] = hbox
                    accumulated_height = accumulated_height + lineheight
                    lineheight = 0
                else
                    -- objects > goal
                    -- This is case (a)
                    remaining_objects[1] = hbox
                    area_filled = true
                    break
                end
            end
        end
        area_filled = true
    end

    if #hlist > 0 then
        for i=1,#hlist do
            remaining_objects[#remaining_objects + 1] = hlist[i]
        end
    end
    -- Sometimes there is a single glue (margin-bottom) left, we should ignore it
    if #remaining_objects == 1 and node.has_attribute(remaining_objects[1], publisher.att_omit_at_top)  then
        -- ignore!?
    else
        objects_t[1] = join_table_to_box(remaining_objects,"remaining objects != 1")
    end

    --- It's a common situation where there is a single free row but the next material is
    --- too high for the row. So we return an empty list and hope that the calling function
    --- is clever enough to detect this case. (Well, it's not too difficult to detect, as
    --- the `objects_t` table is not empty yet.)

    return join_table_to_box(thisarea,"return") or publisher.empty_block()
end

--- Image handling
--- --------------

function set_image_length(dataxml,len,width_or_height)
    if len == nil or len == "auto" then
        return nil
    elseif len == "100%" and width_or_height == "width" then
        if newxpath then
            return dataxml.vars["__maxwidth"]
        else
            return xpath.get_variable("__maxwidth")
        end
    elseif tonumber(len) then
        if width_or_height == "width" then
            return current_grid:width_sp(len)
        else
            return len * current_grid.gridheight
        end
    else
        return tex.sp(len)
    end
end


-- Calculate the image width and height
-- stretch: grow to maxwidth,maxheight if needed
function calculate_image_width_height( image, width, height, minwidth, minheight, maxwidth, maxheight,stretch )
    -- from https://www.w3.org/TR/CSS2/visudet.html#min-max-widths:
    --
    -- Constraint Violation                                                           Resolved Width                      Resolved Height
    -- ===================================================================================================================================================
    --  1 none                                                                        w                                   h
    --  2 w > max-width                                                               max-width                           max(max-width * h/w, min-height)
    --  3 w < min-width                                                               min-width                           min(min-width * h/w, max-height)
    --  4 h > max-height                                                              max(max-height * w/h, min-width)    max-height
    --  5 h < min-height                                                              min(min-height * w/h, max-width)    min-height
    --  6 (w > max-width) and (h > max-height), where (max-width/w ≤ max-height/h)    max-width                           max(min-height, max-width * h/w)
    --  7 (w > max-width) and (h > max-height), where (max-width/w > max-height/h)    max(min-width, max-height * w/h)    max-height
    --  8 (w < min-width) and (h < min-height), where (min-width/w ≤ min-height/h)    min(max-width, min-height * w/h)    min-height
    --  9 (w < min-width) and (h < min-height), where (min-width/w > min-height/h)    min-width                           min(max-height, min-width * h/w)
    -- 10 (w < min-width) and (h > max-height)                                        min-width                           max-height
    -- 11 (w > max-width) and (h < min-height)                                        max-width                           min-height

    -- if stretch and max{height,width} then the image should grow as needed
    if stretch and maxheight < maxdimen and maxwidth < maxdimen then
        local stretchamount = math.min(maxwidth / image.xsize , maxheight / image.ysize )
        if stretchamount > 1 then
            return image.xsize * stretchamount, image.ysize * stretchamount
        end
    end

    if width < minwidth and height > maxheight then
        -- w("10")
        width = minwidth
        height = maxheight
    elseif width > maxwidth and height < minheight then
        -- w("11")
        width = maxwidth
        height = minheight
    elseif width > maxwidth and height > maxheight and maxwidth / width <= maxheight / height then
        -- w("6")
        height = math.max(minheight, maxwidth * height/width)
        width = maxwidth
    elseif width > maxwidth and height > maxheight and maxwidth / width > maxheight / height then
        -- w("7")
        width = math.max(minwidth,maxheight * width / height)
        height = maxheight
    elseif width < minwidth and height < minheight and minwidth / width <= minheight / height then
        -- w("8")
        width = math.min(maxwidth,minheight * width / height)
        height = minheight
    elseif width < minwidth and height < minheight and minwidth / width > minheight / height then
        -- w("9")
        width = minwidth
        height = math.min(maxheight,minwidth * height / width)
    elseif width > maxwidth then
        -- w("2")
        height = math.max(maxwidth * height / width, minheight )
        width = maxwidth
    elseif width < minwidth then
        -- w("3")
        height = math.min(minwidth * height / width, maxheight)
        width = minwidth
    elseif height > maxheight then
        -- w("4")
        width = math.max(maxheight * width / height, minwidth)
        height = maxheight
    elseif height < minheight then
        -- w("5")
        width = math.min(minheight * width / height, maxwidth)
        height = minheight
    end

    -- If one of height or width is given, the other one should
    -- be adjusted to keep the aspect ratio
    if height == image.height then
        if width ~= image.width then
            height = height * width / image.width
        end
    elseif width == image.width then
        if height ~= image.height then
            width = width *  height / image.height
        end
    end
    return width, height
end


local images = {}

function reload_image(filename,typ,width,height)
    main.log("info","Reload image","width",tostring(width),"height",tostring(height),"filename",filename)
    local filename_extension = get_extension(filename)
    local handlername_for_extension
    if options.extensionhandler and options.extensionhandler ~= "" then
        for _,v in ipairs(string.explode(options.extensionhandler,";")) do
            local _,_,ext,handler = string.find(v,"^(.*):(.*)$")
            if ext == filename_extension then
                handlername_for_extension = handler
                break
            end
        end
    end
    local rh = resizehandler[handlername_for_extension or "*"]
    local fn
    fn = splib.reloadimage({ filename = filename, imagetype = typ,width = width,height = height, resizehandler = rh})
    return fn
end

function new_image(filename,page,box,fallback,imageshape)
    return imageinfo(filename,page,box,fallback,imageshape)
end

function validateimagetype(filename)
    local localfilename = kpse.find_file(filename)
    local f,errmsg = io.open(localfilename)
    if not f then
        err(errmsg)
        return nil
    end
    local whatever = f:read(5)
    if string.match(whatever,"<svg") then
        localfilename = splib.convert_svg_image(localfilename)
    end
    f:close()
    return localfilename
end

function get_fallback_image_name( filename, missingfilename )
    if filename then
        main.log("info","Using fallback","fallback",filename or "(filename)", "requested",missingfilename or "(empty)")
        if not kpse.find_file(filename) then
            err("fallback image %q not found",filename or "<filename>")
            return "filenotfound.pdf"
        end
        return filename
    else
        return "filenotfound.pdf"
    end
end

-- Box is none, media, crop, bleed, trim, art
function imageinfo( filename,page,box,fallback,imageshape )
    page = page or 1
    box = box or "crop"
    -- there is no filename, we should fail or throw an error
    if not filename then
        err("No filename given for image")
        filename = get_fallback_image_name(fallback)
    end
    if type(filename) ~= "string" then
        err("something is wrong with the filename for the image, not a string")
        filename = get_fallback_image_name(fallback)
    end

    local new_name = filename .. tostring(page) .. tostring(box)

    if images[new_name] then
        return images[new_name]
    end
    main.log("info","Searching for image","filename",tostring(filename))
    if not kpse.find_file(filename) then
        if options.imagenotfounderror then
            main.log("error","Image not found","filename", filename or "???", lineinfo())
        else
            main.log("warn","Image not found","filename", filename or "???", lineinfo())
        end
        filename = get_fallback_image_name(fallback,filename)
        page = 1
    end
    main.log("info","Load image","filename",tostring(filename))
    -- example is wrong: one based index
    -- <?xml version="1.0" ?>
    -- <imageinfo>
    --    <cells_x>30</cells_x>
    --    <cells_y>21</cells_y>
    --    <segment x1='13' y1='0' x2='16' y2='0' />
    --    <segment x1='13' y1='1' x2='16' y2='1' />
    --    <segment x1='11' y1='2' x2='18' y2='2' />
    --    <segment x1='10' y1='3' x2='18' y2='3' />
    --    <segment x1='10' y1='4' x2='18' y2='4' />
    --    <segment x1='9' y1='5' x2='20' y2='5' />
    --    <segment x1='8' y1='6' x2='20' y2='6' />
    --    <segment x1='8' y1='7' x2='20' y2='7' />
    --    <segment x1='7' y1='8' x2='21' y2='8' />
    --    <segment x1='6' y1='9' x2='21' y2='9' />
    --    <segment x1='5' y1='10' x2='24' y2='10' />
    --    <segment x1='5' y1='11' x2='24' y2='11' />
    --    <segment x1='4' y1='12' x2='25' y2='12' />
    --    <segment x1='3' y1='13' x2='25' y2='13' />
    --    <segment x1='3' y1='14' x2='27' y2='14' />
    --    <segment x1='2' y1='15' x2='27' y2='15' />
    --    <segment x1='1' y1='16' x2='28' y2='16' />
    --  </imageinfo>
    local mt
    -- don't request XML shape file for http locations
    if imageshape and not string.match(filename, "^https?://") then
        local xmlfilename = string.gsub(filename,"(%..*)$","") .. ".xml"

        if kpse.find_file(xmlfilename) then
            local xmltab,msg = load_xml(xmlfilename,"Imageinfo")
            if not xmltab then
                err(msg)
            else
                if newxpath then
                    -- document node, need the root node.
                    -- should be more complex like root(),
                    -- -> needts to be fixed in the future
                    xmltab = xmltab[1]
                end
                mt = {}
                local segments = {}
                local cells_x,cells_y
                for _,v in ipairs(xmltab) do
                    if v[".__local_name"] == "cells_x" then
                        cells_x = tonumber(v[1])
                    elseif v[".__local_name"] == "cells_y" then
                        cells_y = tonumber(v[1])
                    elseif v[".__local_name"] == "segment" then
                        -- 0 based segments
                        if newxpath then
                            local attrs = v[".__attributes"]
                            segments[#segments + 1] = {tonumber(attrs.x1),tonumber(attrs.y1),tonumber(attrs.x2),tonumber(attrs.y2)}
                        else
                            segments[#segments + 1] = {tonumber(v.x1),tonumber(v.y1),tonumber(v.x2),tonumber(v.y2)}
                        end
                    end
                end
                -- we have parsed the file, let's build a beautiful 2dim array
                mt.max_x = cells_x
                mt.max_y = cells_y
                for i=1,cells_y do
                    mt[i] = {}
                    for j=1,cells_x do
                        mt[i][j] = 0
                    end
                end
                for i,v in ipairs(segments) do
                    for x=v[1],v[3] do
                        for y=v[2],v[4] do
                            mt[y][x] = 1
                        end
                    end
                end
            end
        end
    end

    if not images[new_name] then
        if string.match(filename, ".svg$") then
            filename = splib.convert_svg_image(filename)
            if filename == nil or filename == "" then filename = "filenotfound.pdf" else log("Using converted file %q instead",filename) end

        end
        filename = validateimagetype(filename)
        local image_info = img.scan{filename = filename, pagebox = box, page=page,keepopen=true }
        if image_info.orientation == 0 then
            -- good, no transformation
        elseif image_info.orientation == 1 then
            image_info.transform = 0
        elseif image_info.orientation == 2 then
            image_info.transform = 4
        elseif image_info.orientation == 3 then
            image_info.transform = 2
        elseif image_info.orientation == 4 then
            image_info.transform = 6
        elseif image_info.orientation == 5 then
            image_info.transform = 5
        elseif image_info.orientation == 6 then
            image_info.transform = 3
        elseif image_info.orientation == 7 then
            image_info.transform = 7
        elseif image_info.orientation == 8 then
            image_info.transform = 1
        end
        images[new_name] = { img = image_info, allocate = mt }
    end
    return images[new_name]
end

--- Sorting
--- -------
--- The sorting code is currently used for index generation (commands#makeindex)

-- see http://lua.2524044.n2.nabble.com/A-stable-sort-td7648892.html
-- public domain or cc0

local max_chunk_size = 12

function insertion_sort( array, first, last, goes_before )
  for i = first + 1, last do
    local k = first
    local v = array[i]
    for j = i, first + 1, -1 do
      if goes_before( v, array[j-1] ) then
        array[j] = array[j-1]
      else
        k = j
        break
      end
    end
    array[k] = v
  end
end

function merge( array, workspace, low, middle, high, goes_before )
  local i, j, k
  i = 1
  -- Copy first half of array to auxiliary array
  for j = low, middle do
    workspace[ i ] = array[ j ]
    i = i + 1
  end
  i = 1
  j = middle + 1
  k = low
  while true do
    if (k >= j) or (j > high) then
      break
    end
    if goes_before( array[ j ], workspace[ i ] )  then
      array[ k ] = array[ j ]
      j = j + 1
    else
      array[ k ] = workspace[ i ]
      i = i + 1
    end
    k = k + 1
  end
  -- Copy back any remaining elements of first half
  for k = k, j-1 do
    array[ k ] = workspace[ i ]
    i = i + 1
  end
end


function merge_sort( array, workspace, low, high, goes_before )
  if high - low < max_chunk_size then
    insertion_sort( array, low, high, goes_before )
  else
    local middle = math.floor((low + high)/2)
    merge_sort( array, workspace, low, middle, goes_before )
    merge_sort( array, workspace, middle + 1, high, goes_before )
    merge( array, workspace, low, middle, high, goes_before )
  end
end

function stable_sort( array, goes_before )
    local n = #array
    if n < 2 then return array end
    goes_before = goes_before or function (a, b)  return a < b  end
    local workspace = {}
    --  Allocate some room.
    workspace[ math.floor( (n+1)/2 ) ] = array[1]
    if goes_before(array[1],array[1]) then
        error"invalid order function for sorting"
    end
    merge_sort( array, workspace, 1, n, goes_before )
    return array
end
-- end of stable sorting function


--- Garbage Collection
--- -------------------
--- This is somewhat experimental. The idea is to remove all nodes from the
--- contents of the old value of that variable. Hopefully this has no
--- evil side effects. We'll find out....
function flush_table(tbl)
    for k,v in pairs(tbl) do
        if k == ".__context" or k == ".__parent" then
            -- nothing, to prevent infinite loops
        elseif type(v) == "table" then
            flush_table(v)
        elseif type(v) == "userdata" then
            node.flush_list(v)
        else
            k = nil
        end
    end
end

-- clear variable
function flush_variable( varname )
    local x
    if newxpath then
        x = data.vars[varname]
    else
        x = xpath.get_variable(varname)
    end
    if type(x) == "table" then
        flush_table(x)
    end
end


-- random string https://gist.github.com/haggen/2fd643ea9a261fea2094

local charset = {}

-- qwertyuiopasdfghjklzxcvbnmQWERTYUIOPASDFGHJKLZXCVBNM1234567890
for i = 48,  57 do table.insert(charset, string.char(i)) end
for i = 65,  90 do table.insert(charset, string.char(i)) end
for i = 97, 122 do table.insert(charset, string.char(i)) end

function string_random(length)
  if length > 0 then
    return string_random(length - 1) .. charset[math.random(1, #charset)]
  else
    return ""
  end
end


shape = function(tbl, buf, options)
    local font = tbl.font
    options = options or { }
    local hblang, script, dir

    if options.language then
        hblang = harfbuzz.Language.new(options.language)
        buf:set_language(hblang)
    end
    if options.script then
        script = harfbuzz.Script.new(options.script)
        buf:set_script(script)
    end
    if options.direction then
        dir = harfbuzz.Direction.new(options.direction)
        buf:set_direction(dir)
    end
    buf:set_cluster_level(buf.CLUSTER_LEVEL_MONOTONE_CHARACTERS)
    buf:set_flags(harfbuzz.Buffer.FLAG_REMOVE_DEFAULT_IGNORABLES)
    buf:guess_segment_properties()

    local bufdir = tostring(buf:get_direction())
    local bufscript = tostring(buf:get_script())
    harfbuzz.shape_full(font, buf, tbl.otfeatures, {"ot","graphite2","fallback"})
    if bufdir == "rtl" then
        buf:reverse()
    end
    return bufscript,bufdir
end



file_end("publisher.lua")
