ispconfig-py/tools/gen_methods.py

"""Generate Python wrappers from the ISPConfig method inventory.

Reads ``tools/method_inventory.json`` and emits/updates one module per
functional area under ``src/ispconfig/``. Each module has two clearly-marked
sections:

    # ---- AUTO-GENERATED START (do not hand-edit above this line) ----
    ... wrappers ...
    # ---- AUTO-GENERATED END ----
    # ---- HAND-EDIT ONLY BELOW ----
    ... helpers, convenience methods, etc ...

Re-running the generator replaces only the auto block; hand-edits below
the delimiter are preserved. Hand-audited helpers already in the tree
(enable_php, a_add with fix_type_bug, user_get with filter-dict
normalization, etc.) live below the delimiter and are untouched.

If an auto-generated method name collides with a hand-audited one
(detected by scanning the hand-edit block for ``def <name>(``), the
generator skips the auto wrapper — the hand version wins.

Re-run flow::

    python3 tools/extract_inventory.py <php-src> tools/method_inventory.json
    python3 tools/gen_methods.py
    ruff format src/ tools/
    git diff --stat
"""

from __future__ import annotations

import json
import re
import sys
from pathlib import Path
from typing import Any

ROOT = Path(__file__).resolve().parent.parent
INVENTORY = ROOT / "tools" / "method_inventory.json"
SRC = ROOT / "src" / "ispconfig"

AUTO_START = "# ---- AUTO-GENERATED START (do not hand-edit above this line) ----"
AUTO_END = "# ---- AUTO-GENERATED END ----"
HAND_MARKER = "# ---- HAND-EDIT ONLY BELOW ----"

# Routing: method-name prefix -> (module filename, class name, method-name
# stripping rule). The stripping rule strips a prefix from the Python method
# name when we can keep it; we don't, for auto-generated wrappers, so the
# generator emits ``method_foo_bar`` as-is. (Hand-audited helpers strip the
# prefix for ergonomics — kept below the delimiter.)
#
# Order matters: first match wins. Longer prefixes first.
_ROUTES: list[tuple[str, str, str]] = [
    # sites.*
    ("sites_web_vhost_aliasdomain_", "sites.py", "SitesModule"),
    ("sites_web_vhost_subdomain_", "sites.py", "SitesModule"),
    ("sites_web_aliasdomain_", "sites.py", "SitesModule"),
    ("sites_web_subdomain_", "sites.py", "SitesModule"),
    ("sites_web_domain_backup", "backups.py", "BackupsModule"),
    ("sites_web_domain_", "sites.py", "SitesModule"),
    ("sites_web_folder_user_", "sites.py", "SitesModule"),
    ("sites_web_folder_", "sites.py", "SitesModule"),
    ("sites_database_user_", "databases.py", "DatabasesModule"),
    ("sites_database_", "databases.py", "DatabasesModule"),
    ("sites_ftp_user_", "ftp.py", "FtpModule"),
    ("sites_shell_user_", "shell.py", "ShellModule"),
    ("sites_cron_", "cron.py", "CronModule"),
    ("sites_webdav_user_", "webdav.py", "WebdavModule"),
    ("sites_aps_", "aps.py", "ApsModule"),
    ("client_", "clients.py", "ClientsModule"),
    ("mail_", "mail.py", "MailModule"),
    ("mailquota_", "mail.py", "MailModule"),
    ("dns_", "dns.py", "DnsModule"),
    ("server_", "server.py", "ServerModule"),
    ("monitor_", "monitor.py", "MonitorModule"),
    ("domains_", "domains.py", "DomainsModule"),
    ("openvz_", "openvz.py", "OpenvzModule"),
    # admin / misc buckets
    ("sys_datalog_", "admin.py", "AdminModule"),
    ("system_config_", "admin.py", "AdminModule"),
    ("config_value_", "admin.py", "AdminModule"),
    ("update_record_permissions", "admin.py", "AdminModule"),
    # quotas and misc helpers from sites.inc.php that don't match the
    # `sites_*` prefix
    ("quota_get_by_user", "misc.py", "MiscModule"),
    ("databasequota_get_by_user", "misc.py", "MiscModule"),
    ("ftptrafficquota_data", "misc.py", "MiscModule"),
    ("trafficquota_get_by_user", "misc.py", "MiscModule"),
    ("client_get_sites_by_user", "misc.py", "MiscModule"),
]

# Modules that already exist in the tree with a hand-audited class. For
# these, we append the auto block to an existing class body rather than
# writing a fresh module.
_PREEXISTING = {
    "sites.py": "SitesModule",
    "dns.py": "DnsModule",
    "mail.py": "MailModule",
    "databases.py": "DatabasesModule",
    "clients.py": "ClientsModule",
}

# Methods we skip at generation time: ``login`` and ``logout`` are already
# on the top-level client, and ``get_function_list`` is a no-arg
# introspection call we expose via ``ISPConfigClient.list_functions()``.
_SKIP_METHODS = {"login", "logout", "get_function_list"}


def route(method: str) -> tuple[str, str] | None:
    for prefix, fname, cls in _ROUTES:
        if method == prefix or method.startswith(prefix):
            return fname, cls
    return None


def _py_method_name(method: str, prefix: str) -> str:
    """Strip the routed prefix from a method name if it's safe to.

    For the auto-generated wrappers we keep the full PHP method name as
    the Python method name. This is verbose but unambiguous — and matches
    ``raw_call`` semantics so you can grep for the exact PHP string.

    Hand-audited helpers may provide shorter names; they live below the
    delimiter and take precedence.
    """
    return method  # keep the full PHP name; stripping invites collisions


def _docstring(rec: dict[str, Any]) -> list[str]:
    """Return docstring lines, indented with 8 spaces (method-body level)."""
    indent = "        "
    lines = [f'{indent}"""']
    summary = rec["doc"].get("summary") or f"Auto-generated wrapper for ``{rec['method']}``."
    # Escape embedded triple quotes defensively.
    summary = summary.replace('"""', "'''")
    lines.append(f"{indent}{summary}")
    lines.append("")
    lines.append(f"{indent}Source: ``{rec['file']}`` line {rec['line']}.")
    sig_params = [p["name"] for p in rec["params"] if p["name"] != "session_id"]
    if sig_params:
        lines.append(f"{indent}PHP signature: ``{rec['raw_signature']}``.")
    if rec["doc"]["params"]:
        lines.append("")
        lines.append(f"{indent}Params (from PHPDoc):")
        for p in rec["doc"]["params"]:
            pname = p.get("name") or "?"
            if pname == "session_id":
                continue
            ptype = p.get("type", "?")
            desc = (p.get("desc") or "").replace('"""', "'''")
            if desc:
                lines.append(f"{indent}    {pname} ({ptype}): {desc}")
            else:
                lines.append(f"{indent}    {pname} ({ptype})")
    if rec["doc"].get("return"):
        lines.append("")
        ret_type = rec["doc"]["return"].get("type", "Any")
        ret_desc = (rec["doc"]["return"].get("desc") or "").replace('"""', "'''")
        suffix = f" - {ret_desc}" if ret_desc else ""
        lines.append(f"{indent}Returns: {ret_type}{suffix}")
    lines.append("")
    lines.append(f"{indent}AUTO-GENERATED - param shapes may need verification against your")
    lines.append(f"{indent}ISPConfig version. File issues at Sulkta-Coop/ispconfig-py.")
    lines.append(f'{indent}"""')
    return lines


_PHP_TYPE_MAP = {
    "int": "int",
    "integer": "int",
    "string": "str",
    "bool": "bool",
    "boolean": "bool",
    "array": "dict[str, Any] | list[Any]",
    "mixed": "Any",
    "float": "float",
    "double": "float",
}


def _py_param_type(phpdoc_params: list[dict[str, str]], name: str) -> str:
    for p in phpdoc_params:
        if p.get("name") == name:
            t = (p.get("type") or "").strip().lower().split("|")[0]
            return _PHP_TYPE_MAP.get(t, "Any")
    return "Any"


def _emit_method(rec: dict[str, Any]) -> list[str]:
    method = rec["method"]
    # Parameters after session_id.
    params = [p for p in rec["params"] if p["name"] != "session_id"]
    sig_parts = ["self"]
    arg_encodes: list[str] = []
    for p in params:
        py_type = _py_param_type(rec["doc"]["params"], p["name"])
        default = p["default"]
        if default is not None:
            # PHP defaults are best-effort translated.
            default_py = _translate_php_default(default)
            # PEP 484: a None default needs ``| None`` on the annotation.
            # Any already includes None, so no change needed for Any.
            if default_py == "None" and py_type not in ("Any", "dict[str, Any] | list[Any]"):
                py_type = f"{py_type} | None"
            sig_parts.append(f"{p['name']}: {py_type} = {default_py}")
        else:
            sig_parts.append(f"{p['name']}: {py_type}")
        arg_encodes.append(f'("{p["name"]}", {p["name"]})')
    signature = ", ".join(sig_parts)
    lines: list[str] = []
    lines.append(f"    def {method}({signature}) -> Any:")
    lines.extend(_docstring(rec))
    if arg_encodes:
        args_str = ", ".join(arg_encodes)
        lines.append(f'        return self._c._call("{method}", {args_str})')
    else:
        lines.append(f'        return self._c._call("{method}")')
    lines.append("")
    return lines


def _translate_php_default(value: str) -> str:
    v = value.strip()
    low = v.lower()
    if low in ("null",):
        return "None"
    if low == "true":
        return "True"
    if low == "false":
        return "False"
    if low in ("array()", "[]"):
        return "None"  # PHP empty-array default -> None in Python signature
    # Numeric literals pass through.
    if re.match(r"^-?\d+(\.\d+)?$", v):
        return v
    # Quoted strings.
    if (v.startswith("'") and v.endswith("'")) or (v.startswith('"') and v.endswith('"')):
        inner = v[1:-1]
        return '"' + inner.replace("\\", "\\\\").replace('"', '\\"') + '"'
    # Unknown → fall back to None (safest default for optional params).
    return "None"


# -----------------------------------------------------------------------
# Module file handling
# -----------------------------------------------------------------------


def _module_header(fname: str, cls: str) -> str:
    """Stock module header for NEW module files (not preexisting)."""
    stem = fname.replace(".py", "")
    mod_title = stem.replace("_", " ").title()
    return (
        f'"""``{stem}.*`` — auto-generated ISPConfig remote-API wrappers.\n\n'
        f"This module is produced by ``tools/gen_methods.py`` from the\n"
        f"``tools/method_inventory.json`` catalog. Hand-edits go below the\n"
        f"``{HAND_MARKER.strip('# ')}`` marker — they survive regeneration.\n"
        f'"""\n\n'
        f"from __future__ import annotations\n\n"
        f"from typing import TYPE_CHECKING, Any\n\n"
        f"if TYPE_CHECKING:\n"
        f"    from .client import ISPConfigClient\n\n\n"
        f"class {cls}:\n"
        f'    """Auto-generated module: {mod_title}.\n\n'
        f"    All methods below the ``AUTO-GENERATED START`` marker are produced\n"
        f"    by ``tools/gen_methods.py``. Do not hand-edit that block — changes\n"
        f"    will be overwritten on the next regeneration. Add helpers and\n"
        f"    overrides below the ``HAND-EDIT ONLY BELOW`` marker instead.\n"
        f'    """\n\n'
        f"    def __init__(self, client: ISPConfigClient) -> None:\n"
        f"        self._c = client\n\n"
    )


def _find_existing_auto_block(text: str) -> tuple[int, int] | None:
    """Return (start_line_idx, end_line_idx) of the existing auto block, if any."""
    lines = text.splitlines()
    start = end = None
    for i, line in enumerate(lines):
        if AUTO_START in line:
            start = i
        if AUTO_END in line and start is not None:
            end = i
            break
    if start is None or end is None:
        return None
    return start, end


def _find_class_body_insertion(text: str, cls: str) -> int | None:
    """Find the line index of the first blank line AFTER the class's __init__
    (a sensible spot to inject the auto block in a preexisting module).
    """
    lines = text.splitlines()
    in_class = False
    saw_init = False
    for i, line in enumerate(lines):
        if re.match(rf"class\s+{cls}\b", line):
            in_class = True
            continue
        if in_class and "def __init__" in line:
            saw_init = True
        if saw_init and line.strip() == "":
            return i + 1
    return None


def _extract_hand_method_names(text: str, auto_end: int | None) -> set[str]:
    """Scan everything AFTER the auto block for ``def <name>(`` so the
    generator can skip methods a hand-audited helper already claims.
    """
    lines = text.splitlines()
    start = (auto_end + 1) if auto_end is not None else 0
    names: set[str] = set()
    for line in lines[start:]:
        m = re.match(r"\s+def\s+(\w+)\s*\(", line)
        if m:
            names.add(m.group(1))
    return names


def _render_auto_block(records: list[dict[str, Any]], hand_names: set[str]) -> list[str]:
    lines: list[str] = []
    lines.append(f"    {AUTO_START}")
    lines.append("    # Regenerate with: python3 tools/gen_methods.py")
    lines.append("")
    emitted_names: set[str] = set()
    for rec in sorted(records, key=lambda r: r["method"]):
        if rec["method"] in hand_names:
            lines.append(f"    # skipped {rec['method']}: hand-audited helper below takes precedence")
            lines.append("")
            continue
        if rec["method"] in emitted_names:
            continue
        emitted_names.add(rec["method"])
        lines.extend(_emit_method(rec))
    lines.append(f"    {AUTO_END}")
    lines.append("")
    lines.append(f"    {HAND_MARKER}")
    lines.append("")
    return lines


def _ensure_any_imported(text: str) -> str:
    """Make sure ``Any`` is imported from ``typing`` — auto-generated wrappers
    always need it. Leaves other imports untouched; idempotent.
    """
    if re.search(r"^from\s+typing\s+import[^\n]*\bAny\b", text, re.MULTILINE):
        return text
    m = re.search(r"^(from\s+typing\s+import\s+)([^\n]+)$", text, re.MULTILINE)
    if m:
        current = m.group(2)
        # Append ``Any``; keep symbols sorted lexically for stable diffs.
        symbols = sorted({s.strip() for s in current.split(",")} | {"Any"})
        new_line = m.group(1) + ", ".join(symbols)
        return text[: m.start()] + new_line + text[m.end() :]
    # No ``from typing import`` at all — add one after ``from __future__``.
    fut = re.search(r"^from\s+__future__\s+import[^\n]*$", text, re.MULTILINE)
    if fut:
        insertion = fut.end()
        return text[:insertion] + "\n\nfrom typing import Any" + text[insertion:]
    # Last resort: prepend.
    return "from typing import Any\n\n" + text


def _update_existing_module(path: Path, cls: str, records: list[dict[str, Any]]) -> None:
    text = path.read_text(encoding="utf-8")
    text = _ensure_any_imported(text)
    block = _find_existing_auto_block(text)
    if block is None:
        # First-time injection: find where to insert.
        lines = text.splitlines()
        insertion = _find_class_body_insertion(text, cls)
        if insertion is None:
            print(f"WARN: couldn't find insertion point in {path}", file=sys.stderr)
            return
        hand_names = _extract_hand_method_names(text, auto_end=None)
        auto_lines = _render_auto_block(records, hand_names)
        new_lines = lines[:insertion] + auto_lines + lines[insertion:]
        path.write_text("\n".join(new_lines).rstrip() + "\n", encoding="utf-8")
        return
    start, end = block
    lines = text.splitlines()
    hand_names = _extract_hand_method_names(text, auto_end=end)
    auto_lines = _render_auto_block(records, hand_names)
    # Replace start..end+possible trailing hand-marker line; we also re-emit the
    # hand marker so we own its exact placement. Consume any blank lines plus
    # a HAND_MARKER (with optional trailing blank) between AUTO_END and the
    # first real code below — we'll re-emit the marker ourselves at the end
    # of ``auto_lines``.
    after = end + 1
    while after < len(lines) and lines[after].strip() == "":
        after += 1
    if after < len(lines) and HAND_MARKER in lines[after]:
        after += 1
        while after < len(lines) and lines[after].strip() == "":
            after += 1
    new_lines = lines[:start] + auto_lines + lines[after:]
    path.write_text("\n".join(new_lines).rstrip() + "\n", encoding="utf-8")


def _write_new_module(path: Path, cls: str, records: list[dict[str, Any]]) -> None:
    header = _module_header(path.name, cls)
    auto_lines = _render_auto_block(records, hand_names=set())
    body = header + "\n".join(auto_lines).rstrip() + "\n"
    path.write_text(body, encoding="utf-8")


def main() -> None:
    records = json.loads(INVENTORY.read_text(encoding="utf-8"))
    SRC.mkdir(parents=True, exist_ok=True)

    by_module: dict[tuple[str, str], list[dict[str, Any]]] = {}
    unrouted: list[str] = []
    for rec in records:
        if rec["method"] in _SKIP_METHODS:
            continue
        routed = route(rec["method"])
        if routed is None:
            unrouted.append(rec["method"])
            # Default bucket = misc.py
            routed = ("misc.py", "MiscModule")
        key = routed
        by_module.setdefault(key, []).append(rec)

    for (fname, cls), recs in sorted(by_module.items()):
        path = SRC / fname
        if path.exists() and fname in _PREEXISTING:
            _update_existing_module(path, cls, recs)
            action = "updated"
        elif path.exists():
            _update_existing_module(path, cls, recs)
            action = "refreshed"
        else:
            _write_new_module(path, cls, recs)
            action = "created"
        print(f"{action:<10} {fname:<16} ({len(recs)} methods)")

    if unrouted:
        print(f"\nUNROUTED ({len(unrouted)}) — filed under misc.py:", file=sys.stderr)
        for m in unrouted:
            print(f"  {m}", file=sys.stderr)


if __name__ == "__main__":
    main()