view mrjunejune/inference/public_knowledge.py @ 279:b3b547563ec7

Add Google connector service and agent wiki Implement the C/Seobeo Google Drive and Gmail connector with encrypted OAuth storage, Zenbu authentication, browser testing, AI tool discovery, chunked HTTP decoding, and Bazel coverage. Consolidate repository guidance into progressive wiki documentation and enforce arena-first allocation for new first-party C code. Co-authored-by: Copilot <[email protected]> Copilot-Session: 84c338fd-0939-4bb3-b7f3-1062eb213e5d
author MrJuneJune <me@mrjunejune.com>
date Mon, 17 Aug 2026 22:22:36 -0700
parents 056790c4fb0d
children
line wrap: on
line source

"""
Stdlib-only deterministic loader and compiler for the assistant knowledge base.

Loads prompt files from mrjunejune/assistant/ and compiles them with verified
public facts into a hashed, deterministic system prompt. Fails closed on any
malformed input.

Public API:
    compile_prompt(profile, assistant_dir=None) -> dict
        Returns {"content": str, "version": int, "hash": str}.
        Raises ValueError for unknown profile or any validation failure.
"""

import hashlib
import json
import pathlib
import re
from typing import Optional
from urllib.parse import urlsplit

# ---------------------------------------------------------------------------
# Constants
# ---------------------------------------------------------------------------

_KNOWN_PROFILES: frozenset = frozenset({"public_visitor", "invited_friend", "june_admin"})

_REQUIRED_TOP_KEYS: frozenset = frozenset({"version", "facts"})
_REQUIRED_FACT_KEYS: frozenset = frozenset(
    {"id", "topic", "text", "sourceLabel", "sourceUrl", "visibility", "status"}
)

_EXPECTED_VERSION: int = 1
_MAX_PROMPT_FILE_BYTES: int = 8192   # 8 KB per prompt file
_MAX_COMPILED_BYTES: int = 16384     # 16 KB total compiled ceiling

_MAX_ID_LEN: int = 64
_MAX_TOPIC_LEN: int = 64
_MAX_TEXT_LEN: int = 500
_MAX_SOURCE_LABEL_LEN: int = 128

_SLUG_RE = re.compile(r"^[a-z0-9]([a-z0-9-]*[a-z0-9])?$")

# Contact patterns: email and North-American-style phone numbers
_CONTACT_PATTERNS: list = [
    re.compile(r"[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}"),
    re.compile(r"(\+?1[-.\s]?)?\(?\d{3}\)?[-.\s]\d{3}[-.\s]\d{4}"),
]
_SECRET_VALUE_PATTERNS: list = [
    re.compile(
        r"\b(password|passwd|secret|token|api[_\-]?key|private[_\-]?key)"
        r"\s*[:=]\s*\S+",
        re.IGNORECASE,
    ),
    re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----"),
]
_CONTROL_PATTERN = re.compile(r"[\x00-\x1f\x7f]")
_PROMPT_CONTROL_PATTERN = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]")
_RESERVED_CORPUS_PATTERNS: tuple[str, ...] = (
    "--- knowledge corpus ---",
    "system:",
    "assistant:",
    "user:",
    "<system",
    "ignore previous instructions",
    "ignore prior instructions",
    "follow these instructions",
)

# Credential / secret keyword patterns
_SECRET_PATTERNS: list = [
    re.compile(
        r"\b(password|passwd|secret|token|api[_\-]?key|private[_\-]?key)\b",
        re.IGNORECASE,
    ),
]

_FACTS_DELIMITER: str = "--- KNOWLEDGE CORPUS ---"


# ---------------------------------------------------------------------------
# Internal helpers
# ---------------------------------------------------------------------------


def _default_assistant_dir() -> pathlib.Path:
    return pathlib.Path(__file__).parent.parent / "assistant"


def _validate_source_url(url: str, fact_id: str) -> None:
    if not isinstance(url, str) or not url:
        raise ValueError(f"Fact '{fact_id}': sourceUrl is empty")
    if _CONTROL_PATTERN.search(url) or "\\" in url:
        raise ValueError(f"Fact '{fact_id}': sourceUrl contains unsafe characters")
    if url.startswith("/"):
        if url.startswith("//") or any(
            segment in (".", "..") for segment in url.split("/")
        ):
            raise ValueError(
                f"Fact '{fact_id}': sourceUrl must be a safe local path"
            )
        return
    parsed = urlsplit(url)
    if (
        parsed.scheme != "https"
        or not parsed.hostname
        or parsed.username is not None
        or parsed.password is not None
    ):
        raise ValueError(
            f"Fact '{fact_id}': sourceUrl must be HTTPS or a safe local path"
        )
    try:
        parsed.port
    except ValueError as exc:
        raise ValueError(f"Fact '{fact_id}': sourceUrl has an invalid port") from exc


def _check_no_contact_or_secret(text: str, fact_id: str, field: str) -> None:
    for pat in _CONTACT_PATTERNS:
        if pat.search(text):
            raise ValueError(
                f"Fact '{fact_id}': {field} contains contact data (email or phone number)"
            )
    for pat in _SECRET_PATTERNS:
        if pat.search(text):
            raise ValueError(
                f"Fact '{fact_id}': {field} contains a secret or credential pattern"
            )

def _validate_corpus_text(text: str, fact_id: str, field: str) -> None:
    if _CONTROL_PATTERN.search(text):
        raise ValueError(f"Fact '{fact_id}': {field} contains control characters")
    lowered = text.lower()
    for reserved in _RESERVED_CORPUS_PATTERNS:
        if reserved in lowered:
            raise ValueError(
                f"Fact '{fact_id}': {field} contains reserved prompt instructions"
            )
    _check_no_contact_or_secret(text, fact_id, field)


def _load_and_validate_facts(assistant_dir: pathlib.Path) -> list:
    facts_path = assistant_dir / "knowledge" / "public_facts.json"
    try:
        raw = facts_path.read_text(encoding="utf-8")
    except OSError as exc:
        raise ValueError(f"Cannot read facts file: {exc}") from exc

    try:
        data = json.loads(raw)
    except json.JSONDecodeError as exc:
        raise ValueError(f"Facts file is not valid JSON: {exc}") from exc

    if not isinstance(data, dict):
        raise ValueError("Facts file top level must be a JSON object")

    extra_keys = set(data.keys()) - _REQUIRED_TOP_KEYS
    missing_keys = _REQUIRED_TOP_KEYS - set(data.keys())
    if extra_keys or missing_keys:
        raise ValueError(
            f"Facts file top-level keys must be exactly {sorted(_REQUIRED_TOP_KEYS)}; "
            f"extra={sorted(extra_keys)}, missing={sorted(missing_keys)}"
        )

    if (
        not isinstance(data["version"], int)
        or isinstance(data["version"], bool)
        or data["version"] != _EXPECTED_VERSION
    ):
        raise ValueError(
            f"Facts file version must be {_EXPECTED_VERSION}; got {data['version']!r}"
        )

    raw_facts = data["facts"]
    if not isinstance(raw_facts, list) or len(raw_facts) == 0:
        raise ValueError("Facts file 'facts' must be a non-empty list")

    seen_ids: set = set()
    validated: list = []

    for i, fact in enumerate(raw_facts):
        if not isinstance(fact, dict):
            raise ValueError(f"Fact at index {i} must be a JSON object")

        extra = set(fact.keys()) - _REQUIRED_FACT_KEYS
        missing = _REQUIRED_FACT_KEYS - set(fact.keys())
        if extra or missing:
            raise ValueError(
                f"Fact at index {i}: keys must be exactly "
                f"{sorted(_REQUIRED_FACT_KEYS)}; "
                f"extra={sorted(extra)}, missing={sorted(missing)}"
            )

        fid = fact["id"]
        if not isinstance(fid, str) or not fid:
            raise ValueError(f"Fact at index {i}: 'id' must be a non-empty string")
        if len(fid) > _MAX_ID_LEN:
            raise ValueError(f"Fact '{fid}': id exceeds {_MAX_ID_LEN} characters")
        if not _SLUG_RE.match(fid):
            raise ValueError(
                f"Fact '{fid}': id must be lowercase alphanumeric with hyphens"
            )
        if fid in seen_ids:
            raise ValueError(f"Duplicate fact id: {fid!r}")
        seen_ids.add(fid)

        topic = fact["topic"]
        if not isinstance(topic, str) or not topic:
            raise ValueError(f"Fact '{fid}': 'topic' must be a non-empty string")
        if len(topic) > _MAX_TOPIC_LEN:
            raise ValueError(f"Fact '{fid}': topic exceeds {_MAX_TOPIC_LEN} characters")
        if not _SLUG_RE.match(topic):
            raise ValueError(
                f"Fact '{fid}': topic must be lowercase alphanumeric with hyphens"
            )

        text = fact["text"]
        if not isinstance(text, str) or len(text) < 10:
            raise ValueError(
                f"Fact '{fid}': 'text' must be a string of at least 10 characters"
            )
        if len(text) > _MAX_TEXT_LEN:
            raise ValueError(
                f"Fact '{fid}': text exceeds {_MAX_TEXT_LEN} characters"
            )
        _validate_corpus_text(text, fid, "text")

        source_label = fact["sourceLabel"]
        if not isinstance(source_label, str) or not source_label:
            raise ValueError(f"Fact '{fid}': 'sourceLabel' must be a non-empty string")
        if len(source_label) > _MAX_SOURCE_LABEL_LEN:
            raise ValueError(
                f"Fact '{fid}': sourceLabel exceeds {_MAX_SOURCE_LABEL_LEN} characters"
            )
        _validate_corpus_text(source_label, fid, "sourceLabel")

        _validate_source_url(fact["sourceUrl"], fid)

        if not isinstance(fact["visibility"], str) or fact["visibility"] != "public":
            raise ValueError(
                f"Fact '{fid}': visibility must be 'public'; got {fact['visibility']!r}"
            )
        if not isinstance(fact["status"], str) or fact["status"] != "verified":
            raise ValueError(
                f"Fact '{fid}': status must be 'verified'; got {fact['status']!r}"
            )

        validated.append(fact)

    validated.sort(key=lambda f: f["id"])
    return validated


def _load_prompt_file(path: pathlib.Path) -> str:
    try:
        raw_bytes = path.read_bytes()
    except OSError as exc:
        raise ValueError(f"Cannot read prompt file {path.name}: {exc}") from exc
    if len(raw_bytes) > _MAX_PROMPT_FILE_BYTES:
        raise ValueError(
            f"Prompt file {path.name} exceeds {_MAX_PROMPT_FILE_BYTES} bytes "
            f"(got {len(raw_bytes)})"
        )
    try:
        content = raw_bytes.decode("utf-8")
    except UnicodeDecodeError as exc:
        raise ValueError(f"Prompt file {path.name} is not valid UTF-8") from exc
    if _PROMPT_CONTROL_PATTERN.search(content):
        raise ValueError(f"Prompt file {path.name} contains control characters")
    for pattern in _CONTACT_PATTERNS + _SECRET_VALUE_PATTERNS:
        if pattern.search(content):
            raise ValueError(
                f"Prompt file {path.name} contains contact data or a secret value"
            )
    return content


def _format_facts(facts: list) -> str:
    lines: list = []
    for fact in facts:
        lines.append(
            json.dumps(
                {
                    "fact": fact["text"],
                    "id": fact["id"],
                    "sourceLabel": fact["sourceLabel"],
                    "sourceUrl": fact["sourceUrl"],
                    "topic": fact["topic"],
                },
                ensure_ascii=False,
                separators=(",", ":"),
                sort_keys=True,
            )
        )
    return "\n".join(lines)


# ---------------------------------------------------------------------------
# Public API
# ---------------------------------------------------------------------------


def compile_prompt(
    profile: str,
    assistant_dir: Optional[pathlib.Path] = None,
) -> dict:
    """
    Compile a deterministic system prompt for *profile*.

    Parameters
    ----------
    profile:
        One of ``"public_visitor"``, ``"invited_friend"``, or ``"june_admin"``.
    assistant_dir:
        Path to the ``mrjunejune/assistant/`` directory.  Defaults to the
        sibling of the ``inference/`` package at runtime.

    Returns
    -------
    dict with keys:
        - ``"content"`` (str): compiled prompt text
        - ``"version"`` (int): schema version (always 1)
        - ``"hash"`` (str): lowercase hex SHA-256 of ``content`` encoded UTF-8

    Raises
    ------
    ValueError
        On unknown profile, malformed files, validation failures, or size
        limit exceeded.
    """
    if profile not in _KNOWN_PROFILES:
        raise ValueError(
            f"Unknown profile {profile!r}; "
            f"known profiles: {sorted(_KNOWN_PROFILES)}"
        )

    base = pathlib.Path(assistant_dir) if assistant_dir is not None else _default_assistant_dir()

    common = _load_prompt_file(base / "common.md")
    profile_section = _load_prompt_file(base / f"{profile}.md")
    facts = _load_and_validate_facts(base)
    facts_block = _format_facts(facts)

    content = (
        common.rstrip()
        + "\n\n"
        + profile_section.rstrip()
        + "\n\n"
        + _FACTS_DELIMITER
        + "\n\n"
        + facts_block
    )

    encoded = content.encode("utf-8")
    if len(encoded) > _MAX_COMPILED_BYTES:
        raise ValueError(
            f"Compiled prompt exceeds {_MAX_COMPILED_BYTES} bytes "
            f"(got {len(encoded)})"
        )

    return {
        "content": content,
        "version": _EXPECTED_VERSION,
        "hash": hashlib.sha256(encoded).hexdigest(),
    }