-- lx-bundle:true

-- This file is part of Lx, created by Liqube <www.liqube.com>
-- Copyright Liqube Audio. All rights reserved.
-- Lx Interim License. See LICENSE for details.

local common = require("core.common")

local format = {}
format.items = {}
format.REMOVE = {}

-- If this format module is present, it will override plain_text_format,
-- which is the default for plain text files
local plain_text_override_module = "format.plain"

-- Default format for plain text files
local plain_text_format = {
	name = "Plain Text",
	shortname = "Plain Text",
	kind = "text",
	patterns = {},
	symbols = {},
	post_symbols = {},
}

-- Default format for binary files
local plain_binary_format = {
	name = "Binary Data",
	shortname = "Binary Data",
	kind = "binary",
	patterns = {},
	symbols = {},
	post_symbols = {},
}

-- Lazy format registry/cache state
local registry_loaded = false
local registry_entries = {}
local registry_name_index = {}
local loaded_modules = {}
local cache_disabled = false
local session_no_cache = false
local loading_module = nil
local reported_module_errors = {}
local reported_layout_conflicts = {}
local reported_bad_inherit_refs = {}
local extensions_by_module = {}
local extension_owner = nil
local format_override_indexes = {
	extensions = {},
	filenames = {},
}
local format_override_indexes_ready = false

-- Forward declaration used by format.register before helper definition
local cache_file_spec_lookups
-- Forward declaration used while format modules receive their stable module ids
local apply_registered_extensions

-- Keep detector presence cheap to query from loaded format tables and cached entries
local function has_detect_flag(target)
	return target and target.has_detect == true
end

-- Keep detected candidates ahead of generic fallbacks when scores tie
local function candidate_has_detect(target)
	return (type(target) == "table" and type(target.detect) == "function") or has_detect_flag(target)
end

-- Sort candidates by specificity first, then by detector priority, then by recency
local function sort_candidates(matches)
	table.sort(matches, function(a, b)
		if a.score ~= b.score then
			return a.score > b.score
		end

		local a_detect = candidate_has_detect(a.target)
		local b_detect = candidate_has_detect(b.target)

		if a_detect ~= b_detect then
			return a_detect
		end

		return a.order > b.order
	end)
end

-- Run a callback next frame when coroutine scheduling is available
local function run_next_frame(core_mod, fn)
	if type(core_mod) == "table" then
		core_mod.add_coroutine(function()
			coroutine.yield(0)
			fn()
		end)
		return
	end

	fn()
end

-- Trigger visible error attention with best-effort fallbacks
local function flash_error_attention(core_mod, msg)
	local function pulse_once()
		if type(core_mod) == "table" and core_mod.status_view then
			core_mod.status_view:show_error(msg)
		end

		local command = package.loaded["core.command"]

		if command and type(command.perform) == "function" then
			pcall(command.perform, "frameflash:error")
		end
	end

	pulse_once()

	run_next_frame(core_mod, pulse_once)
end

-- Bring startup focus to the latest error entry in the log view
local function focus_last_error_log(core_mod)
	run_next_frame(core_mod, function()
		local command = package.loaded["core.command"]

		if command and type(command.perform) == "function" then
			pcall(command.perform, "log:open-last-error")
		end
	end)
end

local function report_format_module_error(modname, err)
	if reported_module_errors[modname] then
		return
	end
	reported_module_errors[modname] = true
	local short_msg = string.format('Failed to load format module "%s"', tostring(modname))
	local detail_msg = tostring(err)
	local msg = string.format("%s: %s", short_msg, detail_msg)

	local core_mod = package.loaded["core"]
	local item = nil
	if core_mod then
		item = core_mod.error("%s", short_msg)
		if item and detail_msg ~= "" then
			item.info = detail_msg
		end
	else
		print("[format] " .. msg)
	end

	if item and core_mod then
		item.startup_error = true
		item.startup_error_source = core_mod._startup_error_source
		core_mod.startup_error_log_id = item.id
	end

	flash_error_attention(core_mod, short_msg)

	focus_last_error_log(core_mod)

	-- Avoid duplicate diagnostics: only persist to error.log when app log isn't available
	if not (core_mod and core_mod.error) then
		-- Keep fallback format errors with the user's other writable Lx files
		local path = system.join_path(USER_DIR, "error.log")
		local ok, fp = pcall(io.open, path, "ab")
		if ok and fp then
			fp:write("[format] ", msg, "\n")
			fp:close()
		end
	end
end

-- Warn once when both flat and nested format entry files exist for one module name
local function report_layout_conflict(name, flat_path, nested_path)
	if reported_layout_conflicts[name] then
		return
	end

	reported_layout_conflicts[name] = true

	local msg = string.format(
		'Format module "%s" has both flat and nested entries; using flat "%s", ignoring "%s"',
		tostring(name),
		tostring(flat_path),
		tostring(nested_path)
	)

	local core_mod = package.loaded["core"]
	if core_mod then
		core_mod.warning("%s", msg)
	else
		print("[format] " .. msg)
	end
end

-- Write one format-cache status line to the log view when core logging is available
local function log_format_cache_status(message, detail)
	local core_mod = package.loaded["core"]
	if not core_mod or type(core_mod.log_quiet) ~= "function" then
		return
	end

	if type(detail) == "string" and detail ~= "" then
		core_mod.log_quiet("%s: %s", message, detail)
		return
	end

	core_mod.log_quiet("%s", message)
end

-- Return nested fallback module id for canonical 'format.<name>' when flat file is absent
local function get_nested_fallback_module(modname)
	if type(modname) ~= "string" then
		return nil
	end

	local base = modname:match("^format%.([%w_]+)$")
	if not base then
		return nil
	end

	if type(FORMATS_DIR) ~= "string" then
		return modname .. "." .. base
	end

	local flat_path = system.join_path(FORMATS_DIR, base .. ".lua")
	if system.is_file(flat_path) == true then
		return nil
	end

	local dir_path = system.join_path(FORMATS_DIR, base)
	local nested_path = system.join_path(dir_path, base .. ".lua")
	if system.is_file(nested_path) == true then
		return modname .. "." .. base
	end

	return nil
end

-- Require format module by canonical id, with nested-file fallback when flat entry is missing
local function require_format_module(modname)
	loading_module = modname
	local ok, err = pcall(require, modname)
	loading_module = nil
	if ok then
		return true
	end

	local fallback = get_nested_fallback_module(modname)
	if not fallback then
		return false, err
	end

	loading_module = modname
	local ok_fallback, err_fallback = pcall(require, fallback)
	loading_module = nil
	if ok_fallback then
		return true
	end

	return false, err_fallback
end

-- Registry cache path outside 'data/' and temp-write path
local function get_registry_paths()
	return CACHE_DIR,
		CACHE_DIR .. "formats$.lua",
		CACHE_DIR .. "formats$.tmp"
end

-- Read CLI flags defensively from raw ARGS (supports sparse/non-1-based layouts)
local function has_cli_flag(flag)
	if type(ARGS) ~= "table" then
		return false
	end
	for _, v in pairs(ARGS) do
		if v == flag then
			return true
		end
	end
	return false
end

-- Activate process-wide no-cache mode if CLI flag is present
local function apply_no_cache_from_cli()
	if not session_no_cache and has_cli_flag("--no-cache") then
		session_no_cache = true
	end
	if session_no_cache then
		cache_disabled = true
	end
	return session_no_cache
end

-- Enumerate all format module names from 'data/formats'
local function get_format_modules()
	local bundled = rawget(_G, "BUNDLED_FORMATS")
	if type(bundled) == "table" and #bundled > 0 then
		local modules = {}
		local seen = {}

		local function add_bundled_module(modname)
			if type(modname) ~= "string" or not modname:match("^format%.") or modname == "format._" then
				return
			end

			local canonical = modname
			local base = modname:match("^format%.([%w_]+)%.%1$")
			if base then
				canonical = "format." .. base
			end

			if seen[canonical] then
				return
			end

			seen[canonical] = true
			modules[#modules + 1] = canonical
		end

		for _, modname in ipairs(bundled) do
			add_bundled_module(modname)
		end
		table.sort(modules, function(a, b)
			local an = a:gsub("^format%.", "")
			local bn = b:gsub("^format%.", "")
			if an ~= bn then
				if an:match("^" .. bn:gsub("([^%w])", "%%%1") .. "_") then
					return true
				end
				if bn:match("^" .. an:gsub("([^%w])", "%%%1") .. "_") then
					return false
				end
			end
			return an < bn
		end)
		return modules
	end

	if type(FORMATS_DIR) ~= "string" then
		return {}
	end

	local files = system.list_dir(FORMATS_DIR) or {}

	local module_names = {}
	local seen_modules = {}

	local function add_module_name(name)
		if type(name) ~= "string" or name == "" then
			return
		end

		if seen_modules[name] then
			return
		end

		seen_modules[name] = true
		module_names[#module_names + 1] = name
	end

	local flat_entries = {}

	-- Discover canonical format.<name> from data/formats/<name>.lua
	for _, filename in ipairs(files) do
		if filename ~= "_.lua" and filename:match("%.lua$") then
			local name = filename:gsub("%.lua$", "")
			flat_entries[name] = true
			add_module_name(name)
		end
	end

	-- Discover canonical format.<name> from data/formats/<name>/<name>.lua
	for _, entry in ipairs(files) do
		local dir_path = system.join_path(FORMATS_DIR, entry)
		local info = system.get_file_info(dir_path)

		if info and info.type == "dir" then
			local nested = system.join_path(dir_path, entry .. ".lua")

			if system.is_file(nested) == true then
				if flat_entries[entry] then
					local flat_path = system.join_path(FORMATS_DIR, entry .. ".lua")
					report_layout_conflict(entry, flat_path, nested)
				else
					add_module_name(entry)
				end
			end
		end
	end

	-- Order specific module variants before prefix aggregators
	table.sort(module_names, function(a, b)
		local an = a
		local bn = b

		if an ~= bn then
			if an:match("^" .. bn:gsub("([^%w])", "%%%1") .. "_") then
				return true
			end

			if bn:match("^" .. an:gsub("([^%w])", "%%%1") .. "_") then
				return false
			end
		end

		return an < bn
	end)

	local modules = {}

	for _, name in ipairs(module_names) do
		modules[#modules + 1] = "format." .. name
	end

	return modules
end

-- Require format modules and tag newly registered formats with their module id
local function load_format_modules(modules)
	for _, mod in ipairs(modules) do
		if loaded_modules[mod] == nil then
			local before = #format.items
			local ok, err = require_format_module(mod)
			loaded_modules[mod] = ok and true or false
			if ok then
				for i = before + 1, #format.items do
					local item = format.items[i]
					if type(item) == "table" and (type(item._module_name) ~= "string" or item._module_name == "") then
						item._module_name = mod
					end

					if type(item) == "table" then
						apply_registered_extensions(item)
					end
				end
			else
				report_format_module_error(mod, err)
			end
		end
	end
end

-- Derive a stable module label from format module/source id
local function get_module_label(module_name)
	if type(module_name) ~= "string" or module_name == "" then
		return "format"
	end
	local label = module_name:gsub("\\", "/"):match("([^/]+)$") or module_name
	label = label:gsub("%.lua$", "")
	label = label:match("([^%.]+)$") or label
	if label == "" then
		return "format"
	end
	return label
end

--- Collapse nested self-name source paths back to canonical module ids
-- @param prefix string module namespace prefix
-- @param path string normalized source path
-- @return string|nil canonical module id
local function canonicalize_nested_self_name_source(prefix, path)
	local rel = path:match("data/" .. prefix .. "/(.+)%.lua$")
	if not rel then
		return nil
	end

	local parts = {}
	for part in rel:gmatch("[^/]+") do
		parts[#parts + 1] = part
	end

	if #parts == 2 and parts[1] == parts[2] then
		return prefix .. "." .. parts[1]
	end

	return prefix .. "." .. rel:gsub("/", ".")
end

-- Derive fallback label from format file patterns when module id is generic
local function get_label_from_files(files)
	local list = type(files) == "string" and { files } or files
	if type(list) ~= "table" then
		return nil
	end
	for _, pat in ipairs(list) do
		if type(pat) == "string" then
			local ext = pat:match("^%%%.([%w_]+)%$$")
			if ext and ext ~= "" then
				return ext
			end
		end
	end
	return nil
end

-- Apply/repair format display names using module-derived fallback labels
local function apply_format_names(t, module_hint)
	local module_name = module_hint or t._module_name
	local label = get_module_label(module_name)
	if label == "format" then
		label = get_label_from_files(t.files) or label
	end
	local fallback_name = "[" .. label .. "]"

	if not t.name and not t.shortname then
		t.name = fallback_name
		t.shortname = fallback_name
	elseif t.name and not t.shortname then
		t.shortname = t.name
	elseif t.shortname and not t.name then
		t.name = t.shortname
	end

	-- Repair placeholder values when a concrete module label is known
	if label ~= "format" then
		if t.name == "[format]" then
			t.name = fallback_name
		end
		if t.shortname == "[format]" then
			t.shortname = fallback_name
		end
	end
end

-- Deep-copy helper used by syntax inheritance
local function deep_copy(v, seen)
	if type(v) ~= "table" then
		return v
	end
	if v == format.REMOVE then
		return format.REMOVE
	end
	seen = seen or {}
	if seen[v] then
		return seen[v]
	end
	local out = {}
	seen[v] = out
	for k, vv in pairs(v) do
		out[deep_copy(k, seen)] = deep_copy(vv, seen)
	end
	local mt = getmetatable(v)
	if mt then
		setmetatable(out, mt)
	end
	return out
end

-- Deep-merge 'src' onto 'dst'; supports tombstone deletes via 'format.REMOVE'
local function deep_merge(dst, src)
	for k, v in pairs(src) do
		if v == format.REMOVE then
			dst[k] = nil
		else
			local cur = dst[k]
			if type(cur) == "table" and type(v) == "table" then
				deep_merge(cur, v)
			else
				dst[k] = deep_copy(v)
			end
		end
	end
end

--- Attach every registered plugin extension to one resolved format
-- Extensions stay grouped by owner so plugins cannot replace each other through load order
-- @param target table resolved format definition
apply_registered_extensions = function(target)
	local module_name = target and target._module_name
	local registered = type(module_name) == "string" and extensions_by_module[module_name] or nil

	if not registered then
		return
	end

	target._extensions = target._extensions or {}

	for owner_id, extension in pairs(registered) do
		target._extensions[owner_id] = extension
	end
end

--- Run one module load with a stable format-extension owner
-- The previous owner is restored even when the loaded module fails
-- @param owner_id string stable contributing module id
-- @param fn function module loader
-- @param ... any loader arguments
-- @return ... loader results
function format.with_extension_owner(owner_id, fn, ...)
	assert(type(owner_id) == "string" and owner_id ~= "", "format extension owner must be a non-empty string")
	assert(type(fn) == "function", "format extension loader must be a function")

	local previous_owner = extension_owner
	extension_owner = owner_id
	local results = table.pack(pcall(fn, ...))
	extension_owner = previous_owner

	if not results[1] then
		error(results[2], 0)
	end

	return table.unpack(results, 2, results.n)
end

--- Add one owner-scoped format extension to a format module
-- The target may remain lazy; format.register applies queued extensions when the module resolves
-- @param module_name string stable format module id
-- @param extension table capabilities supplied by the active owner
function format.extend(module_name, extension)
	assert(type(module_name) == "string" and module_name ~= "", "format.extend: module name must be a non-empty string")
	assert(type(extension) == "table", "format.extend: extension table expected")
	assert(extension.id == nil, "format.extend: extension owner is supplied by the module loader")

	local owner_id = extension_owner
	assert(type(owner_id) == "string" and owner_id ~= "", "format.extend: no extension owner is active")

	local owned_extension = {}
	local has_capability = false

	for key, value in pairs(extension) do
		owned_extension[key] = value
		has_capability = true
	end

	assert(has_capability, "format.extend: extension requires at least one capability")

	local registered = extensions_by_module[module_name]
	if not registered then
		registered = {}
		extensions_by_module[module_name] = registered
	end

	assert(registered[owner_id] == nil, string.format(
		"format.extend: duplicate extension %q for %q",
		owner_id,
		module_name
	))

	registered[owner_id] = owned_extension

	-- Apply immediately when the target is already loaded without forcing a lazy module load
	for index = 1, #format.items do
		local target = format.items[index]

		if type(target) == "table" and target._module_name == module_name then
			apply_registered_extensions(target)
		end
	end
end

--- Return owner-scoped format extensions exposed by one resolved format
-- @param target table resolved format definition
-- @param capability string|nil optional capability name such as 'views'
-- @return table extensions sorted by owner id
function format.get_extensions(target, capability)
	assert(type(target) == "table", "format.get_extensions: resolved format table expected")
	assert(capability == nil or (type(capability) == "string" and capability ~= ""),
		"format.get_extensions: capability must be a non-empty string")

	local out = {}

	for owner_id, extension in pairs(target._extensions or {}) do
		local value = capability == nil and extension or extension[capability]

		if value ~= nil then
			out[#out + 1] = {
				id = owner_id,
				value = value,
			}
		end
	end

	table.sort(out, function(left, right)
		return left.id < right.id
	end)

	return out
end

--- Return the preferred open mode declared directly by one format
-- @param target table resolved format definition
-- @return string|nil mode declared mode
function format.get_default_mode(target)
	assert(type(target) == "table", "format.get_default_mode: resolved format table expected")

	return target.default_mode
end

--- Return the general document kind handled by one format
-- Ordinary syntax formats are text unless they clearly say otherwise
-- @param target table resolved format definition
-- @return string kind
function format.get_kind(target)
	assert(type(target) == "table", "format.get_kind: resolved format table expected")

	local kind = target.kind

	if kind == nil then
		return "text"
	end
	if type(kind) ~= "string" or kind == "" then
		error("format kind must be a non-empty string", 2)
	end

	return kind
end

--- Report one compact symbol-spec validation error
-- Uses the shared core error channel when available
-- @param message string human-readable error
-- @return nil
local function report_symbol_spec_error(message)
	local core_mod = package.loaded["core"]
	if core_mod and type(core_mod.error) == "function" then
		core_mod.error("%s", tostring(message))
	else
		print("[format] " .. tostring(message))
	end
end

--- Expand one symbol name into the normalized lookup map
-- Applies raw/lower/upper/capital variants once at syntax load time
-- @param out table normalized symbol map
-- @param symbol_name string source symbol text
-- @param symbol_type string token type
-- @param flags string expansion flags
-- @return nil
local function expand_symbol_entry(out, symbol_name, symbol_type, flags)
	if flags:find("r", 1, true) then
		out[symbol_name] = symbol_type
	end
	if flags:find("l", 1, true) then
		out[symbol_name:lower()] = symbol_type
	end
	if flags:find("u", 1, true) then
		out[symbol_name:upper()] = symbol_type
	end
	if flags:find("c", 1, true) then
		out[symbol_name:sub(1, 1):upper() .. symbol_name:sub(2):lower()] = symbol_type
	end
end

--- Validate compact symbol expansion flags
-- Unknown flags are rejected so syntax typos do not silently change matching
-- @param owner string syntax name used in diagnostics
-- @param symbol_name string symbol key or '_'
-- @param flags string|any compact expansion flags
-- @return boolean valid
local function validate_symbol_flags(owner, symbol_name, flags)
	if type(flags) ~= "string" or flags == "" then
		report_symbol_spec_error(string.format(
			"Invalid symbol flags in %s[%s]: expected non-empty string, got %s",
			tostring(owner or "syntax"),
			tostring(symbol_name),
			type(flags)
		))
		return false
	end

	for i = 1, #flags do
		local flag = flags:sub(i, i)
		if flag ~= "r" and flag ~= "l" and flag ~= "u" and flag ~= "c" then
			report_symbol_spec_error(string.format(
				"Invalid symbol flag '%s' in %s[%s]",
				tostring(flag),
				tostring(owner or "syntax"),
				tostring(symbol_name)
			))
			return false
		end
	end

	return true
end

--- Normalize author-facing symbol specs into a flat lookup table
-- Supports '_expansion_flags' defaults and optional per-symbol override flags
-- @param owner string syntax name used in diagnostics
-- @param raw_symbols table|any source symbol spec
-- @return table normalized symbol lookup table
local function normalize_symbol_map(owner, raw_symbols)
	if type(raw_symbols) ~= "table" then
		return {}
	end

	local normalized = {}
	local default_flags = "r"
	local configured_default = raw_symbols._expansion_flags
	if configured_default ~= nil then
		if validate_symbol_flags(owner, "_expansion_flags", configured_default) then
			default_flags = configured_default
		end
	end

	-- Expand each author-facing entry once so token lookup stays a simple hash-table read
	for symbol_name, symbol_spec in pairs(raw_symbols) do
		if symbol_name ~= "_expansion_flags" then
			if symbol_spec == format.REMOVE then
				normalized[symbol_name] = format.REMOVE
			elseif type(symbol_spec) == "string" then
				expand_symbol_entry(normalized, symbol_name, symbol_spec, default_flags)
			elseif type(symbol_spec) == "table" then
				local symbol_type = symbol_spec[1]
				local symbol_flags = symbol_spec[2] or default_flags
				if type(symbol_type) ~= "string" or symbol_type == "" then
					report_symbol_spec_error(string.format(
						"Invalid symbol spec in %s[%s]: expected type string at index 1",
						tostring(owner or "syntax"),
						tostring(symbol_name)
					))
				elseif validate_symbol_flags(owner, symbol_name, symbol_flags) then
					expand_symbol_entry(normalized, symbol_name, symbol_type, symbol_flags)
				else
					expand_symbol_entry(normalized, symbol_name, symbol_type, default_flags)
				end
			else
				report_symbol_spec_error(string.format(
					"Invalid symbol spec in %s[%s]: expected string or table, got %s",
					tostring(owner or "syntax"),
					tostring(symbol_name),
					type(symbol_spec)
				))
			end
		end
	end

	return normalized
end

local valid_surround_behaviors = {
	stop = true,
	cycle = true,
	unwrap = true,
}

--- Normalize syntax-owned surrounding pair definitions
-- String entries use the typed key as the opening marker and the value as the closing marker
-- @param owner string syntax name used in diagnostics
-- @param raw_pairs table|any author-facing surrounding pair definitions
-- @return table normalized surrounding pair lookup
local function normalize_surrounding_pairs(owner, raw_pairs)
	if raw_pairs == nil then
		return {}
	end
	if type(raw_pairs) ~= "table" then
		error(string.format("Invalid surrounding_pairs in %s: expected table", tostring(owner)))
	end

	local normalized = {}
	for typed, raw_pair in pairs(raw_pairs) do
		if type(typed) ~= "string" or typed == "" then
			error(string.format("Invalid surrounding pair key in %s: expected non-empty string", tostring(owner)))
		end

		local pair = type(raw_pair) == "string" and { close = raw_pair } or raw_pair
		if type(pair) ~= "table" then
			error(string.format("Invalid surrounding pair for '%s' in %s: expected string or table", typed, tostring(owner)))
		end

		local open_marker = pair.open or typed
		if type(open_marker) ~= "string" or open_marker == "" then
			error(string.format("Invalid opening marker for '%s' in %s", typed, tostring(owner)))
		end
		if type(pair.close) ~= "string" or pair.close == "" then
			error(string.format("Invalid closing marker for '%s' in %s", typed, tostring(owner)))
		end
		if pair.min_repeat ~= nil and (type(pair.min_repeat) ~= "number"
			or pair.min_repeat < 1 or pair.min_repeat % 1 ~= 0) then
			error(string.format("Invalid minimum surround repeat for '%s' in %s", typed, tostring(owner)))
		end
		if pair.max_repeat ~= nil and (type(pair.max_repeat) ~= "number"
			or pair.max_repeat < 1 or pair.max_repeat % 1 ~= 0) then
			error(string.format("Invalid maximum surround repeat for '%s' in %s", typed, tostring(owner)))
		end
		local min_repeat = pair.min_repeat or 1
		if pair.max_repeat ~= nil and pair.max_repeat < min_repeat then
			error(string.format("Maximum surround repeat for '%s' in %s must reach min_repeat", typed, tostring(owner)))
		end
		if pair.behavior ~= nil and not valid_surround_behaviors[pair.behavior] then
			error(string.format("Invalid surround behavior for '%s' in %s", typed, tostring(owner)))
		end
		if pair.behavior ~= nil and pair.max_repeat == nil then
			error(string.format("Surround behavior for '%s' in %s requires max_repeat", typed, tostring(owner)))
		end
		if pair.resolve ~= nil and type(pair.resolve) ~= "function" then
			error(string.format("Invalid surrounding pair resolver for '%s' in %s", typed, tostring(owner)))
		end

		normalized[typed] = {
			open = open_marker,
			close = pair.close,
			min_repeat = min_repeat,
			max_repeat = pair.max_repeat,
			behavior = pair.behavior,
			resolve = pair.resolve,
		}
	end

	return normalized
end

-- Flattens pattern helper output so one helper can return one rule or many rules
local function flatten_pattern_rules(items)
	local out = {}

	local function append_item(item)
		if type(item) ~= "table" then
			return
		end
		if item.pattern ~= nil then
			out[#out + 1] = item
			return
		end
		for i = 1, #item do
			append_item(item[i])
		end
	end

	for i = 1, #items do
		append_item(items[i])
	end

	return out
end

-- Resolve inherit source by module id, then by extension alias
local function resolve_inherit_source(syntax_ref)
	if type(syntax_ref) ~= "string" or syntax_ref == "" then
		return nil
	end

	local function warn_bad_inherit_ref_once(ref)
		local key = tostring(ref)
		if reported_bad_inherit_refs[key] then
			return
		end
		reported_bad_inherit_refs[key] = true

		local msg = string.format(
			"Invalid inherit.syntax value, expected 'module:<name>' or '.<ext>': %q",
			tostring(ref)
		)
		local core_mod = package.loaded["core"]
		if core_mod then
			core_mod.error("%s", msg)
		else
			print("[format] " .. msg)
		end
	end

	local module_name = nil
	local ext_query = nil
	if syntax_ref:sub(1, 7) == "module:" then
		local tail = syntax_ref:sub(8)
		if tail == "" then
			warn_bad_inherit_ref_once(syntax_ref)
			return nil
		end
		module_name = "format." .. tail
	elseif syntax_ref:sub(1, 1) == "." then
		ext_query = syntax_ref:sub(2):lower()
		if ext_query == "" then
			warn_bad_inherit_ref_once(syntax_ref)
			return nil
		end
	else
		warn_bad_inherit_ref_once(syntax_ref)
		return nil
	end

	local function find_loaded_extension_match()
		if not ext_query or ext_query == "" then
			return nil
		end
		for i = #format.items, 1, -1 do
			local item = format.items[i]
			if type(item) == "table" then
				local exts = item.extensions
				if type(exts) == "string" then
					if exts:lower() == ext_query then
						return item
					end
				elseif type(exts) == "table" then
					for _, ext in ipairs(exts) do
						if type(ext) == "string" and ext:lower() == ext_query then
							return item
						end
					end
				end
			end
		end
		return nil
	end

	if module_name then
		-- Preserve module context so inherited syntax is tagged with its real module id
		local prev_loading_module = loading_module
		require_format_module(module_name)
		loading_module = prev_loading_module
		for i = #format.items, 1, -1 do
			local item = format.items[i]
			if type(item) == "table" and item._module_name == module_name then
				return item
			end
		end
		return nil
	end

	local ext_match = find_loaded_extension_match()
	if ext_match then
		return ext_match
	end

	-- Extension fallback can require modules not loaded yet
	load_format_modules(get_format_modules())
	return find_loaded_extension_match()
end

-- Applies optional inline inheritance spec:
--   inherit = { syntax = "xml", keys = { "patterns", "comment" } }
-- or:
--   inherit = {
--     { syntax = "xml", keys = { "patterns" } },
--     { syntax = "c", keys = { "symbols" } },
--   }
local function apply_inherit(t)
	if type(t) ~= "table" then
		return t
	end
	local inherit = t.inherit
	if type(inherit) ~= "table" then
		return t
	end

	local specs = {}
	if inherit.syntax ~= nil or inherit.keys ~= nil then
		specs[1] = inherit
	else
		for _, spec in ipairs(inherit) do
			if type(spec) == "table" then
				specs[#specs + 1] = spec
			end
		end
	end
	if #specs == 0 then
		return t
	end

	local out = {}
	local inherited_any = false
	for _, spec in ipairs(specs) do
		local keys = spec.keys
		if type(keys) == "table" then
			local selected = {}
			local selected_count = 0
			for _, k in ipairs(keys) do
				if type(k) == "string" and not selected[k] then
					selected[k] = true
					selected_count = selected_count + 1
				end
			end
			for k, v in pairs(keys) do
				if type(k) == "string" and v and not selected[k] then
					selected[k] = true
					selected_count = selected_count + 1
				end
			end

			if selected_count > 0 then
				local base = resolve_inherit_source(spec.syntax)
				if type(base) == "table" then
					for k in pairs(selected) do
						if base[k] ~= nil then
							out[k] = deep_copy(base[k])
							inherited_any = true
						end
					end
				end
			end
		end
	end
	if not inherited_any then
		return t
	end

	for k, v in pairs(t) do
		if k ~= "inherit" then
			if v == format.REMOVE then
				out[k] = nil
			else
				local cur = out[k]
				if type(cur) == "table" and type(v) == "table" then
					-- Keep child pattern rules first, then inherited fallback rules
					if k == "patterns" or k == "outline_patterns" then
						local merged = deep_copy(v)
						for i = 1, #cur do
							merged[#merged + 1] = deep_copy(cur[i])
						end
						out[k] = merged
					else
						deep_merge(cur, v)
					end
				else
					out[k] = deep_copy(v)
				end
			end
		end
	end
	return out
end

--- Register one format definition and normalize its optional fields
function format.register(t)
	-- Normalize compact symbol specs before inheritance so inherited maps stay plain and predictable
	t.symbols = normalize_symbol_map(t.shortname or t.name or t._module_name or "syntax", t.symbols)

	t = apply_inherit(t)
	local syntax_owner = t.shortname or t.name or t._module_name or "syntax"

	-- Normalize optional syntax fields so partial definitions cannot crash consumers
	-- This allows format modules to omit syntax fields such as 'patterns' and 'symbols' safely
	t.patterns = flatten_pattern_rules(type(t.patterns) == "table" and t.patterns or {})
	t.outline_patterns = flatten_pattern_rules(type(t.outline_patterns) == "table" and t.outline_patterns or {})
	t.symbols = type(t.symbols) == "table" and t.symbols or {}
	t.post_symbols = type(t.post_symbols) == "table" and t.post_symbols or {}
	-- Keep syntax-owned generic format handlers predictable for command forwarding
	t.format_commands = type(t.format_commands) == "table" and t.format_commands or {}
	-- Keep text input pair handling independent from author-facing shorthand
	t.surrounding_pairs = normalize_surrounding_pairs(syntax_owner, t.surrounding_pairs)

	-- Keep header matcher compatibility: 'headers' may be a string or a pattern list
	if type(t.headers) ~= "string" and type(t.headers) ~= "table" then
		t.headers = {}
	end
	-- Keep header recognition dependent on a filename match when requested
	t.require_file_match = t.require_file_match == true
	-- Internal formats can be exact redirect targets without appearing as public choices
	t.internal = t.internal == true
	-- This forces a re-check of all headers to find a possibly better
	-- match across all available syntax files
	-- Useful if a subformat syntax of a format may be recognized via header,
	-- for example xml schema, or comment
	t.force_header_check = t.force_header_check == true

	-- Prefer explicit loader context for stable module attribution
	if not t._module_name and loading_module then
		t._module_name = loading_module
	end

	-- Support 'caseless = true' on syntax patterns and outline_patterns
	for _, group in ipairs({ t.patterns or {}, t.outline_patterns or {} }) do
		for _, p in ipairs(group) do
			if p.caseless and p.pattern then
				if type(p.pattern) == "table" then
					p.pattern[1] = common.caseless_pattern(p.pattern[1])
					p.pattern[2] = common.caseless_pattern(p.pattern[2])
				else
					p.pattern = common.caseless_pattern(p.pattern)
				end
			end
		end
	end

	-- File-reference patterns use the same one-time caseless expansion as syntax patterns
	for _, rule in ipairs(t.file_references or {}) do
		if rule.caseless then
			if type(rule.prefix) == "string" then
				rule.prefix = common.caseless_pattern(rule.prefix)
			end
			if type(rule.match) == "string" then
				rule.match = common.caseless_pattern(rule.match)
			end
		end
	end

	-- Capture source/module hint for diagnostics (e.g. doc:show-syntax)
	if not t._module_name then
		local ok, info = pcall(debug.getinfo, 2, "S")
		if ok and info and type(info.source) == "string" then
			local src = info.source
			local path = src:gsub("^@", ""):gsub("\\", "/")
			local mod = canonicalize_nested_self_name_source("plugins", path)
			if mod then
				t._module_name = mod
			else
				mod = canonicalize_nested_self_name_source("format", path)
				if mod then
					t._module_name = mod
				else
					t._module_name = path:match("([^/]+)%.lua$") or path
				end
			end
		end
	end

	apply_format_names(t)
	apply_registered_extensions(t)
	cache_file_spec_lookups(t)

	table.insert(format.items, t)
end

-- Find the newest loaded format whose named pattern list matches the string
local function find(string, field)
	for i = #format.items, 1, -1 do
		local t = format.items[i]
		if common.match_pattern(string, t[field] or {}) then
			return t
		end
	end
end

-- Score header match strength so stronger header hits can override extension-only matches
local function get_header_match_score(target, header)
	local s, e = common.match_pattern(header, target.headers or {})
	if not s then
		return nil
	end
	local score = (e - s + 1)
	if s == 1 then
		score = score + 1000000
	end
	if s == 1 and e == #header then
		score = score + 1000000
	end
	return score
end

-- Return the strongest loaded format header match
local function find_best_loaded_header_match(header)
	local best, best_score
	for i = #format.items, 1, -1 do
		local t = format.items[i]
		local score = not t.require_file_match and get_header_match_score(t, header)
		if score and (not best or score > best_score) then
			best, best_score = t, score
		end
	end
	return best, best_score
end

-- Build extension candidates (max two parts): "sql", "sqlite.sql"
local function build_extension_candidates(base_lower)
	local out = {}
	local short = base_lower:match("%.([^%.]+)$")
	if short then
		out[#out + 1] = short
	end
	local long = base_lower:match("%.([^%.]+%.[^%.]+)$")
	if long and long ~= short then
		-- Put longer candidate first so it wins inside extension-set matching
		table.insert(out, 1, long)
	end
	return out
end

-- Extract basename and lowercase extension candidates for format matching
local function parse_file_parts(filename)
	-- Normalize basename and extension once per syntax lookup
	local path = tostring(filename or "")
	local base = path:match("([^/\\]+)$") or path
	local base_lower = base:lower()
	local ext_candidates = build_extension_candidates(base_lower)
	return base, base_lower, ext_candidates
end

--- Build one case-insensitive format-override index
-- Conflicting keys are rejected because Lua table order must not decide which format wins
-- @param source table configured override map
-- @param setting_name string setting path used in errors
-- @return table index format names keyed by lowercase filename or extension
local function build_format_override_index(source, setting_name)
	local index = {}

	if source == nil then
		return index
	end
	if type(source) ~= "table" then
		error(setting_name .. " must be a table", 3)
	end

	for configured_key, configured_format in pairs(source) do
		if type(configured_key) ~= "string" or configured_key == "" then
			error(setting_name .. " keys must be non-empty strings", 3)
		end
		if type(configured_format) ~= "string" or configured_format == "" then
			error(setting_name .. " values must be non-empty format names", 3)
		end
		if setting_name == "config.extension_overrides" and configured_key:sub(1, 1) == "." then
			error(setting_name .. " keys must omit the leading dot", 3)
		end
		if setting_name == "config.filename_overrides"
			and (configured_key:find("/", 1, true) or configured_key:find("\\", 1, true))
		then
			error(setting_name .. " keys must be final filenames without a path", 3)
		end

		local key = configured_key:lower()
		local previous_format = index[key]

		if previous_format and previous_format:lower() ~= configured_format:lower() then
			error(string.format(
				"%s contains conflicting case-insensitive key %q",
				setting_name,
				configured_key
			), 3)
		end

		index[key] = configured_format
	end

	return index
end

--- Rebuild direct case-insensitive lookups from configured format overrides
-- Call this after user or project settings change
function format.refresh_overrides()
	local config = require("core.config")

	local extension_names = build_format_override_index(
		config.extension_overrides,
		"config.extension_overrides"
	)
	local filename_names = build_format_override_index(
		config.filename_overrides,
		"config.filename_overrides"
	)
	local resolved_formats = {}

	-- Resolve each named format once so opening a file only performs map lookups
	local function resolve_index(names, setting_name)
		local index = {}

		for key, format_name in pairs(names) do
			local name_key = format_name:lower()
			local resolved = resolved_formats[name_key]

			if not resolved then
				resolved = format.get_by_name(format_name)

				if not resolved then
					error(string.format("%s[%q] names unknown format %q", setting_name, key, format_name), 2)
				end

				resolved_formats[name_key] = resolved
			end

			index[key] = resolved
		end

		return index
	end

	format_override_indexes = {
		extensions = resolve_index(extension_names, "config.extension_overrides"),
		filenames = resolve_index(filename_names, "config.filename_overrides"),
	}
	format_override_indexes_ready = true
end

--- Return an exact configured filename or extension override
-- Filename overrides are more specific and therefore win over extension overrides
-- @param filename string candidate file path
-- @return table|nil format_definition
local function get_format_override(filename)
	if not format_override_indexes_ready then
		format.refresh_overrides()
	end

	local _, base_lower = parse_file_parts(filename)
	local filename_override = format_override_indexes.filenames[base_lower]

	if filename_override then
		return filename_override
	end

	-- Pattern breakdown: the final dot starts the extension and the capture excludes further dots
	local extension = base_lower:match("%.([^%.]+)$")

	return extension and format_override_indexes.extensions[extension] or nil
end

-- Copy only string values from a string-or-list input
local function copy_string_list(list_like)
	if type(list_like) == "string" then
		return { list_like }
	end
	local out = {}
	if type(list_like) ~= "table" then
		return out
	end
	for _, v in ipairs(list_like) do
		if type(v) == "string" then
			out[#out + 1] = v
		end
	end
	return out
end

-- Normalize extension candidates to lowercase without leading dots
local function normalize_extensions(list_like)
	local out = {}
	for _, v in ipairs(copy_string_list(list_like)) do
		local ext = v:gsub("^%.+", ""):lower()
		if ext ~= "" then
			out[#out + 1] = ext
		end
	end
	return out
end

-- Normalize filename candidates to lowercase
local function normalize_filenames(list_like)
	local out = {}
	for _, v in ipairs(copy_string_list(list_like)) do
		out[#out + 1] = v:lower()
	end
	return out
end

--- Normalize case-insensitive patterns that match only the final filename
-- @param list_like string|table|nil configured filename patterns
-- @return table normalized_patterns
local function normalize_filename_patterns(list_like)
	local out = {}
	for _, value in ipairs(copy_string_list(list_like)) do
		out[#out + 1] = value:lower()
	end
	return out
end

--- Normalize format matching into path, filename, and extension lists
-- @param target table|nil format definition or cached registry entry
-- @return table file_spec
local function normalize_file_spec(target)
	local spec = { files = {}, filename_patterns = {}, filenames = {}, extensions = {} }

	-- Keep list entries unique while preserving insertion order
	local function add_unique(dst, src)
		local seen = {}
		for _, v in ipairs(dst) do
			seen[v] = true
		end
		for _, v in ipairs(src) do
			if not seen[v] then
				dst[#dst + 1] = v
				seen[v] = true
			end
		end
	end

	local files = target and target.files

	if type(files) == "string" then
		spec.files = { files }
	elseif type(files) == "table" then
		-- Backward-compatible read for temporary nested file-spec shape
		if files.patterns ~= nil or files.exact ~= nil or files.extensions ~= nil then
			add_unique(spec.files, copy_string_list(files.patterns))
			add_unique(spec.filenames, normalize_filenames(files.exact))
			add_unique(spec.extensions, normalize_extensions(files.extensions))
			for _, v in ipairs(files) do
				if type(v) == "string" then
					spec.files[#spec.files + 1] = v
				end
			end
		else
			spec.files = copy_string_list(files)
		end
	end

	add_unique(spec.filenames, normalize_filenames(target and target.filenames))
	add_unique(spec.filename_patterns, normalize_filename_patterns(target and target.filename_patterns))
	add_unique(spec.extensions, normalize_extensions(target and target.extensions))
	return spec
end

-- Convert a list into a key-lookup set for O(1) membership checks
local function build_lookup_set(list)
	local out = {}
	for _, v in ipairs(list or {}) do
		out[v] = true
	end
	return out
end

-- Materialize file-match lookup fields on formats and cache records
cache_file_spec_lookups = function(target)
	-- Build exact and extension lookup sets for fast file matching
	local spec = normalize_file_spec(target)
	target.files = spec.files
	target.filename_patterns = spec.filename_patterns
	target.filenames = spec.filenames
	target.extensions = spec.extensions
	target._files_name_set = build_lookup_set(spec.filenames)
	target._files_ext_set = build_lookup_set(spec.extensions)
end

--- Return a numeric file-match score, with more specific matches ranked higher
-- @param target table format definition or cached registry entry
-- @param filename string complete candidate path
-- @param base_lower string lowercase final filename
-- @param ext_candidates table lowercase file endings
-- @return number|nil score
local function match_file_spec(target, filename, base_lower, ext_candidates)
	local names = target._files_name_set
	if names and names[base_lower] then
		return 3000000
	end
	-- A specific filename pattern must beat a generic extension such as '.txt'
	if common.match_pattern(base_lower, target.filename_patterns or {}) ~= false then
		return 2500000
	end
	local exts = target._files_ext_set
	if exts and ext_candidates then
		for _, ext in ipairs(ext_candidates) do
			if exts[ext] then
				-- Prefer longer compound extensions like "sqlite.sql" over "sql"
				return 2000000 + #ext
			end
		end
	end
	if common.match_pattern(filename, target.files or {}) ~= false then
		return 1000000
	end
end

-- Copies string values from a list-like field into a normalized array
local function copy_list(t)
	if type(t) == "string" then
		return { t }
	end
	local out = {}
	if type(t) ~= "table" then
		return out
	end
	for _, v in ipairs(t) do
		if type(v) == "string" then
			out[#out + 1] = v
		end
	end
	return out
end

--- Copy normalized file matching lists for format registry serialization
-- @param target table format definition
-- @return table file_spec
local function copy_file_spec_fields(target)
	local spec = normalize_file_spec(target)
	return {
		files = copy_list(spec.files),
		filename_patterns = copy_list(spec.filename_patterns),
		filenames = copy_list(spec.filenames),
		extensions = copy_list(spec.extensions),
	}
end

-- Derive stable short format name used by CLI lookup
local function derive_entry_name(entry)
	if type(entry) ~= "table" then
		return nil
	end
	if type(entry.name) == "string" and entry.name ~= "" then
		return entry.name:lower()
	end
	if type(entry.extensions) == "table" and type(entry.extensions[1]) == "string" and entry.extensions[1] ~= "" then
		return entry.extensions[1]:lower()
	end
	if type(entry.module) == "string" then
		local tail = entry.module:match("^format%.(.+)$")
		if type(tail) == "string" and tail ~= "" then
			return tail:lower()
		end
	end
	return nil
end

-- Rebuild fast format-name index from current registry entries
local function rebuild_registry_name_index()
	registry_name_index = {}

	for i = 1, #registry_entries do
		local entry = registry_entries[i]
		local name = derive_entry_name(entry)

		if type(name) == "string" and name ~= "" then
			registry_name_index[name] = entry
		end

		-- The module tail is the stable name used by '--format' and user overrides
		-- Pattern breakdown: the fixed 'format.' prefix is removed while the remaining name is captured
		local module_name = type(entry.module) == "string"
			and entry.module:match("^format%.(.+)$")
			or nil

		if module_name then
			registry_name_index[module_name:lower()] = entry
		end

		-- Common extension aliases may name the same format, such as 'jpg' and 'jpeg'
		for _, extension in ipairs(entry.extensions or {}) do
			if type(extension) == "string" and extension ~= "" then
				registry_name_index[extension:lower()] = entry
			end
		end
	end
end

-- Merge list values while preserving insertion order
local function append_unique(dst, src)
	local seen = {}
	for _, v in ipairs(dst) do
		seen[v] = true
	end
	for _, v in ipairs(src) do
		if not seen[v] then
			dst[#dst + 1] = v
			seen[v] = true
		end
	end
end

-- Serialize Lua string arrays for generated registry output
local function serialize_list(list)
	local parts = {}
	for i, v in ipairs(list) do
		parts[i] = string.format("%q", v)
	end
	-- Emit compact empty arrays in cache output
	if #parts == 0 then
		return "{}"
	end
	return "{ " .. table.concat(parts, ", ") .. " }"
end

--- Serialize registry entries into one complete cache payload
-- @param entries table[] registry entries
-- @return string data cache payload
local function serialize_registry(entries)
	local parts = {
		"-- Auto-generated format registry cache\n",
		"return {\n",
	}

	for _, e in ipairs(entries) do
		parts[#parts + 1] = table.concat({
			"  { module = ", string.format("%q", e.module), ", ",
			"name = ", string.format("%q", e.name or derive_entry_name(e) or ""), ", ",
			"display_name = ", string.format("%q", e.display_name or e.name or derive_entry_name(e) or ""), ", ",
			"files = ", serialize_list(e.files), ", ",
			"filename_patterns = ", serialize_list(e.filename_patterns), ", ",
			"filenames = ", serialize_list(e.filenames), ", ",
			"extensions = ", serialize_list(e.extensions), ", ",
			"headers = ", serialize_list(e.headers), ", ",
			"has_detect = ", e.has_detect and "true" or "false", ", ",
			"require_header_match = ", e.require_header_match and "true" or "false", ", ",
			"require_file_match = ", e.require_file_match and "true" or "false", ", ",
			"force_header_check = ", e.force_header_check and "true" or "false", " },\n",
		})
	end

	parts[#parts + 1] = "}\n"

	return table.concat(parts)
end

--- Validate and atomically replace the format registry cache
-- @param entries table[] registry entries
-- @return boolean|nil written
-- @return string|nil error_message
local function write_registry(entries)
	if cache_disabled or session_no_cache then
		return nil, "cache disabled"
	end

	local dir, path = get_registry_paths()
	local data = serialize_registry(entries)
	local chunk, validation_error = load(data, "@formats$.lua", "t", {})

	if not chunk then
		return nil, validation_error or "generated cache is invalid"
	end

	local valid, result = pcall(chunk)

	if not valid or type(result) ~= "table" or #result ~= #entries then
		return nil, valid and "generated cache has an invalid result" or result
	end

	if not system.is_dir(dir) then
		os.execute(string.format('mkdir "%s"', dir))
	end

	return system.write_file_atomic(path, data)
end

--- Generate and cache the registry from current format modules
-- @return table[] entries generated cache entries, or empty on write failure
local function generate_registry()
	local modules = get_format_modules()

	local old_items = format.items
	local old_loaded = loaded_modules
	format.items = {}
	loaded_modules = {}

	-- Require each format module and force module attribution for new entries
	for _, mod in ipairs(modules) do
		package.loaded[mod] = nil
		local fallback = get_nested_fallback_module(mod)
		if fallback then
			package.loaded[fallback] = nil
		end
	end
	load_format_modules(modules)

	local by_module = {}
	local entries = {}
	for _, t in ipairs(format.items) do
		local mod = t._module_name
		if type(mod) == "string" and mod:match("^format%.") and mod ~= "format._" and not t.internal then
			local entry = by_module[mod]
			if not entry then
				entry = {
					module = mod,
					name = nil,
					display_name = nil,
					files = {},
					filename_patterns = {},
					filenames = {},
					extensions = {},
					headers = {},
					has_detect = false,
					require_header_match = false,
					require_file_match = false,
					force_header_check = false,
				}
				by_module[mod] = entry
				entries[#entries + 1] = entry
			end
			local spec = copy_file_spec_fields(t)
			if type(entry.display_name) ~= "string" or entry.display_name == "" then
				entry.display_name = t.name or t.shortname
			end
			append_unique(entry.files, spec.files)
			append_unique(entry.filename_patterns, spec.filename_patterns)
			append_unique(entry.filenames, spec.filenames)
			append_unique(entry.extensions, spec.extensions)
			append_unique(entry.headers, copy_list(t.headers))
			if type(t.detect) == "function" then
				entry.has_detect = true
			end
			if t.require_header_match then
				entry.require_header_match = true
			end
			if t.require_file_match then
				entry.require_file_match = true
			end
			if t.force_header_check then
				entry.force_header_check = true
			end
		end
	end

	local filtered = {}
	for _, e in ipairs(entries) do
		e.name = derive_entry_name(e)
		filtered[#filtered + 1] = {
			module = e.module,
			name = e.name,
			display_name = e.display_name,
			files = e.files,
			filename_patterns = e.filename_patterns,
			filenames = e.filenames,
			extensions = e.extensions,
			headers = e.headers,
			has_detect = e.has_detect,
			require_header_match = e.require_header_match,
			require_file_match = e.require_file_match,
			force_header_check = e.force_header_check,
		}
	end

	table.sort(filtered, function(a, b)
		return a.module < b.module
	end)

	local ok, write_error = write_registry(filtered)
	-- Restore the live registry before handling any cache outcome
	format.items = old_items
	loaded_modules = old_loaded
	for _, mod in ipairs(modules) do
		package.loaded[mod] = nil
		local fallback = get_nested_fallback_module(mod)
		if fallback then
			package.loaded[fallback] = nil
		end
	end

	if not ok then
		log_format_cache_status("Format cache update failed", write_error)
		return {}
	end

	return filtered
end

-- Detect stale cache when format files exist without registry entries
local function is_registry_stale(entries)
	if type(entries) ~= "table" then
		return true, "cache payload is invalid"
	end

	local formats_dir = type(FORMATS_DIR) == "string" and FORMATS_DIR or ""
	local entry_modules = {}

	-- Track every module name currently recorded in the cache snapshot
	for _, e in ipairs(entries) do
		if type(e) == "table" and type(e.module) == "string" then
			-- The detector flag is required so older cache layouts rebuild instead of losing detection
			if type(e.has_detect) ~= "boolean" then
				return true, "cache detector metadata is outdated"
			end

			entry_modules[e.module] = true
		end
	end

	local files = system.list_dir(formats_dir) or {}
	local discovered_modules = {}

	-- Rebuild the module set from disk so cache-only deleted entries can be detected
	for _, filename in ipairs(files) do
		if filename ~= "_.lua" and filename:match("%.lua$") then
			local mod = "format." .. filename:gsub("%.lua$", "")
			discovered_modules[mod] = true
			if not entry_modules[mod] then
				return true, string.format("new format module found: %s", mod)
			end
		end

		local dir_path = system.join_path(formats_dir, filename)
		local info = system.get_file_info(dir_path)
		if info and info.type == "dir" then
			local nested = system.join_path(dir_path, filename .. ".lua")
			if system.is_file(nested) == true then
				local mod = "format." .. filename
				discovered_modules[mod] = true
				if not entry_modules[mod] then
					return true, string.format("new format module found: %s", mod)
				end
			end
		end
	end

	-- Any cached module missing from the current on-disk scan makes the registry stale
	for module_name in pairs(entry_modules) do
		if not discovered_modules[module_name] then
			return true, string.format("cached format module removed: %s", module_name)
		end
	end

	return false, nil
end

-- Detect stale cache when any format source file is newer than cache file
local function is_registry_older_than_format_files()
	local _, cache_path = get_registry_paths()
	local cache_info = system.get_file_info(cache_path)

	if not cache_info or type(cache_info.modified) ~= "number" then
		return true, "cache timestamp is unavailable"
	end

	local formats_dir = type(FORMATS_DIR) == "string" and FORMATS_DIR or ""
	local files = system.list_dir(formats_dir) or {}

	for _, filename in ipairs(files) do
		if filename ~= "_.lua" and filename:match("%.lua$") then
			local full = formats_dir .. filename
			local info = system.get_file_info(full)

			if info and type(info.modified) == "number" and info.modified > cache_info.modified then
				return true, string.format("source changed: %s", full)
			end
		end

		local dir_path = system.join_path(formats_dir, filename)
		local dir_info = system.get_file_info(dir_path)
		if dir_info and dir_info.type == "dir" then
			local nested = system.join_path(dir_path, filename .. ".lua")
			local info = system.get_file_info(nested)
			if info and type(info.modified) == "number" and info.modified > cache_info.modified then
				return true, string.format("source changed: %s", nested)
			end
		end
	end

	return false, nil
end

-- Load cached registry; auto-generate if missing or invalid
local function load_registry()
	-- Honor no-cache mode even if syntax is queried before init_registry()
	if apply_no_cache_from_cli() then
		registry_loaded = true
		if #format.items == 0 then
			loaded_modules = {}
			local modules = get_format_modules()
			for _, mod in ipairs(modules) do
				package.loaded[mod] = nil
			end
			load_format_modules(modules)
		end
		return
	end
	if cache_disabled or session_no_cache then
		return
	end
	if registry_loaded then
		return
	end

	-- Empty/stale/missing caches are rebuilt automatically
	local _, path, temp = get_registry_paths()
	-- Remove stale temp cache from interrupted prior writes
	os.remove(temp)
	local entries
	local refresh_reason = nil
	local fn = loadfile(path)
	if fn then
		local ok, res = pcall(fn)
		if ok and type(res) == "table" then
			entries = res
		else
			refresh_reason = "cache file could not be read"
		end
	else
		refresh_reason = "cache file is missing"
	end

	if type(entries) == "table" and #entries == 0 then
		refresh_reason = "cache is empty"
	end

	if type(entries) == "table" and refresh_reason == nil then
		local stale, stale_reason = is_registry_stale(entries)
		if stale then
			refresh_reason = stale_reason or "cache contents changed"
		end
	end

	if type(entries) == "table" and refresh_reason == nil then
		local outdated, outdated_reason = is_registry_older_than_format_files()
		if outdated then
			refresh_reason = outdated_reason or "format sources are newer than cache"
		end
	end

	if refresh_reason ~= nil then
		-- Tell the user why the cache will be refreshed before the rebuild starts
		log_format_cache_status("Format cache needs updating", refresh_reason)
		log_format_cache_status("Rebuilding format cache", path)
		entries = generate_registry()
	else
		log_format_cache_status("Using format cache", string.format("%d entries from %s", #entries, path))
	end

	registry_entries = type(entries) == "table" and entries or {}
	for _, e in ipairs(registry_entries) do
		e.name = derive_entry_name(e)
		e.has_detect = e.has_detect == true
		e.force_header_check = e.force_header_check == true
		cache_file_spec_lookups(e)
	end
	rebuild_registry_name_index()
	registry_loaded = true
end

-- Lazy-load a format module once and memoize status
local function ensure_module_loaded(modname)
	if loaded_modules[modname] ~= nil then
		return loaded_modules[modname]
	end

	-- Preserve module context for format.register during lazy loads
	local before = #format.items
	local ok, err = require_format_module(modname)

	if ok then
		for i = before + 1, #format.items do
			local item = format.items[i]

			-- Bind missing/placeholder module names to the concrete loaded module
			if type(item._module_name) ~= "string" or item._module_name == "" or item._module_name == "format" then
				item._module_name = modname
			end

			-- Ensure fallback names reflect resolved module labels
			apply_format_names(item, modname)
			apply_registered_extensions(item)
		end
	else
		report_format_module_error(modname, err)
	end

	loaded_modules[modname] = ok and true or false
	return loaded_modules[modname]
end

-- Return newest loaded format table attributed to a module name
local function find_loaded_by_module(modname)
	if type(modname) ~= "string" or modname == "" then
		return nil
	end

	for i = #format.items, 1, -1 do
		local item = format.items[i]

		if type(item) == "table" and item._module_name == modname then
			return item
		end
	end
	return nil
end

--- Find one cached registry entry by module name
-- Keeps plain text override lookup on the existing cache path
-- @param modname string|nil module name
-- @return table|nil registry entry
local function find_registry_entry_by_module(modname)
	if type(modname) ~= "string" or modname == "" then
		return nil
	end

	for i = #registry_entries, 1, -1 do
		local entry = registry_entries[i]

		if type(entry) == "table" and entry.module == modname then
			return entry
		end
	end

	return nil
end

--- Validate that every format extension names a loaded or registered format module
-- This runs after startup plugins load so valid extensions do not force lazy format modules open
-- @return boolean valid true when every target resolves
function format.validate_extensions()
	load_registry()
	format.refresh_overrides()

	local unresolved = {}

	for module_name, registered in pairs(extensions_by_module) do
		if not find_loaded_by_module(module_name) and not find_registry_entry_by_module(module_name) then
			local owners = {}

			for owner_id in pairs(registered) do
				owners[#owners + 1] = owner_id
			end

			table.sort(owners)
			unresolved[#unresolved + 1] = string.format("%s [%s]", module_name, table.concat(owners, ", "))
		end
	end

	table.sort(unresolved)

	if #unresolved > 0 then
		error("format.extend: unresolved target module(s): " .. table.concat(unresolved, "; "), 2)
	end

	return true
end

--- Resolve the plain text fallback format
-- Loads the optional 'format.plain' override through the shared registry loader
-- @return table plain text format
local function get_plain_text_format()
	local loaded_override = find_loaded_by_module(plain_text_override_module)

	if loaded_override then
		return loaded_override
	end

	load_registry()

	local entry = find_registry_entry_by_module(plain_text_override_module)

	if entry and ensure_module_loaded(entry.module) then
		return find_loaded_by_module(entry.module) or plain_text_format
	end

	return plain_text_format
end

-- Collect loaded header candidates in score order so detectors can validate ambiguous headers
local function find_loaded_header_matches(header)
	local matches = {}

	for i = #format.items, 1, -1 do
		local item = format.items[i]
		local score = not item.require_file_match and get_header_match_score(item, header)

		if score then
			matches[#matches + 1] = { target = item, score = score, order = i }
		end
	end

	sort_candidates(matches)

	return matches
end

-- Collect cached header candidates in score order so registry-backed detectors can reject safely
local function find_registry_header_matches(header)
	local matches = {}

	for i = #registry_entries, 1, -1 do
		local entry = registry_entries[i]
		local score = not entry.require_file_match and get_header_match_score(entry, header)

		if score then
			matches[#matches + 1] = { target = entry, score = score, order = i }
		end
	end

	sort_candidates(matches)

	return matches
end

-- Collect loaded file candidates in match order so detectors can reject weak extension hits
local function find_loaded_file_matches(filename, header)
	local _, base_lower, ext_candidates = parse_file_parts(filename)
	local matches = {}

	for i = #format.items, 1, -1 do
		local item = format.items[i]
		local score = match_file_spec(item, filename, base_lower, ext_candidates)

		if score then
			if item.require_header_match then
				if common.match_pattern(header, item.headers or {}) then
					matches[#matches + 1] = { target = item, score = score, order = i }
				end
			else
				matches[#matches + 1] = { target = item, score = score, order = i }
			end
		end
	end

	sort_candidates(matches)

	return matches
end

-- Collect cached file candidates in match order so the registry can drive lazy detector checks
local function find_registry_file_matches(filename, header)
	local _, base_lower, ext_candidates = parse_file_parts(filename)
	local matches = {}

	for i = #registry_entries, 1, -1 do
		local entry = registry_entries[i]
		local score = match_file_spec(entry, filename, base_lower, ext_candidates)

		if score then
			if entry.require_header_match then
				if common.match_pattern(header, entry.headers or {}) then
					matches[#matches + 1] = { target = entry, score = score, order = i }
				end
			else
				matches[#matches + 1] = { target = entry, score = score, order = i }
			end
		end
	end

	sort_candidates(matches)

	return matches
end

--- Resolve one detector redirect without running another detector
-- Supports a format module, a file extension, or Lx's built-in binary format
-- @param format_ref string redirect target
-- @param ctx table detection input
-- @return table|nil format_definition
local function resolve_detect_redirect(format_ref, ctx)
	if format_ref == "binary" then
		return plain_binary_format
	end

	if type(format_ref) ~= "string" or format_ref == "" then
		return nil
	end

	if format_ref:sub(1, 7) == "module:" then
		local module_name = format_ref:sub(8)

		if module_name ~= "" then
			module_name = "format." .. module_name

			if ensure_module_loaded(module_name) then
				return find_loaded_by_module(module_name)
			end
		end

		return nil
	end

	if format_ref:sub(1, 1) ~= "." or #format_ref == 1 then
		return nil
	end

	-- A synthetic name resolves the declared extension without re-running detectors
	local filename = "detect-redirect" .. format_ref
	local header = tostring(ctx and ctx.data or "")
	local loaded_matches = find_loaded_file_matches(filename, header)

	if loaded_matches[1] then
		return loaded_matches[1].target
	end

	load_registry()

	for _, match in ipairs(find_registry_file_matches(filename, header)) do
		local entry = match.target

		if type(entry) == "table"
			and type(entry.module) == "string"
			and ensure_module_loaded(entry.module) then
			return find_loaded_by_module(entry.module)
		end
	end

	return nil
end

--- Merge public detect metadata without overwriting an earlier accepted hint
-- @param dst table accumulated public metadata
-- @param src table normalized detector result
local function merge_detect_meta(dst, src)
	if type(dst) ~= "table" or type(src) ~= "table" then
		return
	end

	for key, value in pairs(src) do
		-- Hook control fields guide detection and must not leak into document metadata
		if key ~= "action" and key ~= "format" and dst[key] == nil and value ~= nil then
			dst[key] = value
		end
	end
end

--- Return one completed detection result without an empty metadata container
-- @param primary_format table selected primary format
-- @param detect_state table accumulated detection state
-- @return table primary_format
-- @return table|nil metadata
local function finish_detection(primary_format, detect_state)
	local metadata = next(detect_state.meta) and detect_state.meta or nil

	return primary_format, metadata
end

-- Normalize detector calls so format lookup and load-time codepage detection share one contract
local function run_detect(target, ctx)
	local detect = target and target.detect

	if type(detect) ~= "function" then
		return nil
	end

	local ok, result = pcall(detect, ctx)

	if not ok then
		return nil
	end

	if result == nil or result == false then
		return nil
	end

	if result == true then
		return { action = "accept" }
	end

	if type(result) ~= "table" then
		return nil
	end

	local action = result.action

	if action == "accept" or action == "continue" or action == "redirect" then
		return result
	end

	return nil
end

--- Apply one normalized detector result
-- @param target table format whose detector ran
-- @param result table|nil normalized detector result
-- @param ctx table detection input
-- @param detect_state table accumulated detection output
-- @return table|nil resolved_format
local function apply_detect_result(target, result, ctx, detect_state)
	if not result then
		return nil
	end

	if result.action == "accept" then
		merge_detect_meta(detect_state.meta, result)

		return target
	end

	if result.action == "redirect" then
		local redirected = resolve_detect_redirect(result.format, ctx)

		if redirected and redirected ~= target then
			merge_detect_meta(detect_state.meta, result)

			return redirected
		end
	end

	if result.action == "continue" then
		merge_detect_meta(detect_state.meta, result)
	end

	return nil
end

--- Resolve already-loaded candidates without extra module work when no detector is present
-- @param candidates table ordered loaded format candidates
-- @param ctx table detection input
-- @param detect_state table accumulated detection output
-- @return table|nil resolved_format
local function resolve_loaded_candidates(candidates, ctx, detect_state)
	for i = 1, #candidates do
		local target = candidates[i].target

		if type(target) == "table" then
			if type(target.detect) == "function" then
				local result = run_detect(target, ctx)
				local resolved = apply_detect_result(target, result, ctx, detect_state)

				if resolved then
					return resolved
				end
			else
				return target
			end
		end
	end
end

--- Resolve cached candidates while only loading modules that must be inspected
-- @param candidates table ordered cached format candidates
-- @param ctx table detection input
-- @param detect_state table accumulated detection output
-- @return table|nil resolved_format
local function resolve_registry_candidates(candidates, ctx, detect_state)
	for i = 1, #candidates do
		local entry = candidates[i].target

		if type(entry) == "table" and type(entry.module) == "string" and entry.module ~= "" then
			if ensure_module_loaded(entry.module) then
				local loaded = find_loaded_by_module(entry.module)

				if loaded then
					if has_detect_flag(entry) then
						local result = run_detect(loaded, ctx)
						local resolved = apply_detect_result(loaded, result, ctx, detect_state)

						if resolved then
							return resolved
						end
					else
						return loaded
					end
				end
			end
		end
	end
end

--- Resolve a format and public metadata produced during detection
-- @param filename string document filename
-- @param header string opening document data
-- @param is_binary boolean whether binary mode was forced
-- @param probe table|nil native byte findings
-- @return table primary_format
-- @return table|nil metadata
function format.get(filename, header, is_binary, probe)
	if is_binary then
		return plain_binary_format, nil
	end

	local configured_override = get_format_override(filename)

	if configured_override then
		return configured_override, nil, nil
	end

	-- Let detectors validate extension-based candidates before plain fallback wins
	local detect_ctx = {
		filename = filename,
		header = header,
		data = header,
		is_binary = false,
		probe = probe,
	}
	local detect_state = {
		meta = {},
	}
	local found_by_file = resolve_loaded_candidates(find_loaded_file_matches(filename, header), detect_ctx, detect_state)

	if found_by_file and found_by_file.force_header_check then
		local file_header_score = get_header_match_score(found_by_file, header)
		local best_header, best_header_score = find_best_loaded_header_match(header)

		if best_header and best_header ~= found_by_file and (not file_header_score or best_header_score > file_header_score) then
			return finish_detection(best_header, detect_state)
		end
	end

	local found = found_by_file or resolve_loaded_candidates(find_loaded_header_matches(header), detect_ctx, detect_state)
	if found then
		return finish_detection(found, detect_state)
	end

	load_registry()

	-- Reuse the cache for candidate ordering, then lazy-load only the modules we need to inspect
	local entry = resolve_registry_candidates(find_registry_file_matches(filename, header), detect_ctx, detect_state)

	if not entry then
		entry = resolve_registry_candidates(find_registry_header_matches(header), detect_ctx, detect_state)
	end

	if entry and entry.force_header_check then
		local file_header_score = get_header_match_score(entry, header)
		local best_header, best_header_score = find_best_loaded_header_match(header)

		if best_header and best_header ~= entry and (not file_header_score or best_header_score > file_header_score) then
			entry = best_header
		end
	end

	if entry then
		local loaded_by_file = resolve_loaded_candidates(find_loaded_file_matches(filename, header), detect_ctx, detect_state)

		if loaded_by_file and loaded_by_file.force_header_check then
			local file_header_score = get_header_match_score(loaded_by_file, header)
			local best_header, best_header_score = find_best_loaded_header_match(header)

			if best_header and best_header ~= loaded_by_file and (not file_header_score or best_header_score > file_header_score) then
				return finish_detection(best_header, detect_state)
			end
		end
		local primary_format = loaded_by_file
			or resolve_loaded_candidates(find_loaded_header_matches(header), detect_ctx, detect_state)
			or get_plain_text_format()

		return finish_detection(primary_format, detect_state)
	end

	return finish_detection(get_plain_text_format(), detect_state)
end

-- Resolve a format by short name, for example 'md', 'lua', or 'json'
function format.get_by_name(name)
	if type(name) ~= "string" or name == "" then
		return nil
	end

	local q = name:lower()
	load_registry()

	local entry = registry_name_index[q]

	if entry and ensure_module_loaded(entry.module) then
		return find_loaded_by_module(entry.module)
	end
	return nil
end

-- Resolve a format by owning module name, for example 'format.lua'
function format.get_by_module(module_name)
	if type(module_name) ~= "string" or module_name == "" then
		return nil
	end

	load_registry()

	if ensure_module_loaded(module_name) then
		return find_loaded_by_module(module_name)
	end
	return nil
end

--- Resolve one declared syntax target used by embedded-language consumers
-- Accepts either '.extension' or 'module:name' references
-- @param syntax_ref string declared syntax target
-- @return table|nil format_definition resolved non-plain syntax
function format.resolve_syntax_reference(syntax_ref)
	if type(syntax_ref) ~= "string" or syntax_ref == "" then
		return nil
	end

	if syntax_ref:sub(1, 7) == "module:" then
		local module_name = syntax_ref:sub(8)

		if module_name == "" then
			return nil
		end

		return format.get_by_module("format." .. module_name)
	end

	if syntax_ref:sub(1, 1) == "." then
		local extension = syntax_ref:sub(2)

		if extension == "" then
			return nil
		end

		-- A synthetic filename keeps this lookup limited to the declared extension
		local format_definition = format.get("syntax-reference." .. extension, "", false)

		return format_definition ~= get_plain_text_format() and format_definition or nil
	end

	return nil
end

-- Resolve default extension for one format module
function format.get_default_extension_by_module(module_name)
	if type(module_name) ~= "string" or module_name == "" then
		return nil
	end

	local function normalize_extension(ext)
		if type(ext) ~= "string" then
			return nil
		end

		ext = ext:match("^%s*(.-)%s*$") or ""
		ext = ext:gsub("^%.*", "")

		if ext == "" then
			return nil
		end
		return ext
	end

	local function syntax_default_extension(syn)
		if type(syn) ~= "table" then
			return nil
		end

		local ext = normalize_extension(syn.default_extension)

		if ext then
			return ext
		end

		if type(syn.extensions) == "table" then
			return normalize_extension(syn.extensions[1])
		end

		return normalize_extension(syn.extensions)
	end

	load_registry()

	if ensure_module_loaded(module_name) then
		local loaded = find_loaded_by_module(module_name)
		local ext = syntax_default_extension(loaded)

		if ext then
			return ext
		end
	end

	for i = #registry_entries, 1, -1 do
		local entry = registry_entries[i]

		if type(entry) == "table" and entry.module == module_name then
			if type(entry.extensions) == "table" then
				return normalize_extension(entry.extensions[1])
			end
			return nil
		end
	end

	return nil
end

-- Return a display-ready list of formats without eagerly loading their modules
function format.get_available()
	load_registry()
	local out = {}

	if #registry_entries > 0 then
		for i = 1, #registry_entries do
			local entry = registry_entries[i]
			local name = entry and entry.name

			if type(name) == "string" and name ~= "" then
				out[#out + 1] = {
					name = name,
					module = entry.module,
				}
			end
		end
		return out
	end

	for i = 1, #format.items do
		local syn = format.items[i]

		if type(syn) == "table" and not syn.internal then
			local name = syn.shortname or syn.name

			if type(name) == "string" and name ~= "" then
				out[#out + 1] = {
					name = name:lower(),
					module = syn._module_name,
				}
			end
		end
	end
	return out
end

-- Return format registry entries with file association metadata.
-- The returned tables are copies so callers cannot mutate the registry cache.
function format.get_file_association_entries()
	load_registry()
	local out = {}
	local source = #registry_entries > 0 and registry_entries or format.items

	for i = 1, #source do
		local entry = source[i]
		if type(entry) == "table" and not entry.internal then
			local extensions = copy_list(entry.extensions)
			if #extensions > 0 then
				out[#out + 1] = {
					module = entry.module or entry._module_name,
					name = entry.display_name or entry.name or derive_entry_name(entry) or "",
					extensions = extensions,
				}
			end
		end
	end

	return out
end

-- Ensure the format registry is available during startup without eager format loads
function format.init_registry(opts)
	opts = opts or {}

	-- Honor CLI flags even if caller omitted options
	if opts.no_cache == nil then
		opts.no_cache = has_cli_flag("--no-cache")
	end

	if opts.force_update == nil then
		opts.force_update = has_cli_flag("--update-cache")
	end

	-- Respect raw CLI no-cache even when option is omitted/false
	if apply_no_cache_from_cli() then
		opts.no_cache = true
	end

	-- Keep no-cache active for the entire process once requested
	if opts.no_cache then
		session_no_cache = true
	end

	if session_no_cache then
		if registry_loaded and cache_disabled and #format.items > 0 then
			return #format.items
		end

		-- Bypass cache file use/creation and preload all format modules
		cache_disabled = true
		registry_loaded = true
		registry_entries = {}
		registry_name_index = {}

		-- Force fresh module registration when cache is disabled
		loaded_modules = {}
		local modules = get_format_modules()

		for _, mod in ipairs(modules) do
			package.loaded[mod] = nil
		end

		load_format_modules(modules)

		return #format.items
	end

	cache_disabled = false

	if opts.force_update then
		-- Force cache regeneration regardless of existing cache file
		local _, path = get_registry_paths()

		log_format_cache_status("Format cache needs updating", "forced by --update-cache")
		log_format_cache_status("Rebuilding format cache", path)

		registry_entries = generate_registry()

		for _, e in ipairs(registry_entries) do
			e.name = derive_entry_name(e)
			cache_file_spec_lookups(e)
		end

		rebuild_registry_name_index()
		registry_loaded = true
	else
		load_registry()
	end

	return #registry_entries
end

return format
