view mrjunejune/inference/litellm_proxy.py @ 279:b3b547563ec7

Add Google connector service and agent wiki Implement the C/Seobeo Google Drive and Gmail connector with encrypted OAuth storage, Zenbu authentication, browser testing, AI tool discovery, chunked HTTP decoding, and Bazel coverage. Consolidate repository guidance into progressive wiki documentation and enforce arena-first allocation for new first-party C code. Co-authored-by: Copilot <[email protected]> Copilot-Session: 84c338fd-0939-4bb3-b7f3-1062eb213e5d
author MrJuneJune <me@mrjunejune.com>
date Mon, 17 Aug 2026 22:22:36 -0700
parents 1f9877b637e9
children
line wrap: on
line source

from __future__ import annotations

import argparse
import asyncio
import json
import os
import pathlib
from collections.abc import AsyncIterator, Awaitable, Callable
from typing import Any

import litellm
import uvicorn
import yaml
from fastapi import FastAPI, HTTPException, Request
from fastapi.responses import JSONResponse, StreamingResponse

Completion = Callable[..., Awaitable[Any]]


def resolve_token_directory(explicit: str | None = None) -> pathlib.Path:
    if explicit:
        return pathlib.Path(explicit).expanduser().resolve()
    configured = os.environ.get("GITHUB_COPILOT_TOKEN_DIR")
    if configured:
        return pathlib.Path(configured).expanduser().resolve()
    state_root = os.environ.get("MRJUNEJUNE_INFERENCE_STATE")
    if state_root:
        return (
            pathlib.Path(state_root).expanduser().resolve()
            / "litellm-copilot"
        )
    xdg_state = os.environ.get("XDG_STATE_HOME")
    root = (
        pathlib.Path(xdg_state).expanduser()
        if xdg_state
        else pathlib.Path.home() / ".local" / "state"
    )
    return (root / "mrjunejune" / "inference" / "litellm-copilot").resolve()


def _serialize(value: Any) -> dict[str, Any]:
    if hasattr(value, "model_dump"):
        return value.model_dump(exclude_none=True)
    if hasattr(value, "dict"):
        return value.dict()
    if isinstance(value, dict):
        return value
    raise TypeError("LiteLLM returned an unsupported response")


def create_app(
    completion: Completion = litellm.acompletion,
    *,
    model_alias: str = "jrpg-copilot",
    upstream_model: str = "github_copilot/gpt-4",
) -> FastAPI:
    app = FastAPI(docs_url=None, redoc_url=None, openapi_url=None)

    @app.get("/health/liveliness")
    async def health() -> dict[str, str]:
        return {"status": "ready"}

    @app.get("/health/readiness")
    async def readiness() -> dict[str, str]:
        from litellm.llms.github_copilot.authenticator import Authenticator

        try:
            await asyncio.to_thread(Authenticator().get_api_key)
        except Exception as error:
            raise HTTPException(
                status_code=503,
                detail="GitHub Copilot authentication unavailable",
            ) from error
        return {"status": "ready"}

    @app.post("/v1/chat/completions")
    async def chat(request: Request):
        master_key = os.environ.get("LITELLM_MASTER_KEY", "")
        authorization = request.headers.get("Authorization", "")
        if not master_key or authorization != f"Bearer {master_key}":
            raise HTTPException(status_code=401, detail="Invalid gateway key")

        payload = await request.json()
        if not isinstance(payload, dict) or payload.get("model") != model_alias:
            raise HTTPException(status_code=400, detail="Unknown model alias")
        payload = dict(payload)
        payload["model"] = upstream_model
        configured_max = int(os.environ.get("LITELLM_MAX_OUTPUT_TOKENS", "1024"))
        requested_max = payload.get("max_tokens")
        if not isinstance(requested_max, int) or requested_max > configured_max:
            payload["max_tokens"] = configured_max

        try:
            response = await completion(**payload)
        except Exception as error:
            raise HTTPException(
                status_code=502,
                detail="GitHub Copilot provider request failed",
            ) from error

        if payload.get("stream"):
            async def events() -> AsyncIterator[str]:
                async for chunk in response:
                    yield (
                        "data: "
                        + json.dumps(_serialize(chunk), separators=(",", ":"))
                        + "\n\n"
                    )
                yield "data: [DONE]\n\n"

            return StreamingResponse(events(), media_type="text/event-stream")
        return JSONResponse(_serialize(response))

    return app


def _load_model(config_path: str) -> tuple[str, str]:
    with open(config_path, encoding="utf-8") as config_file:
        config = yaml.safe_load(config_file)
    models = config.get("model_list", [])
    if len(models) != 1:
        raise ValueError("LiteLLM config must contain exactly one model")
    model = models[0]
    return model["model_name"], model["litellm_params"]["model"]


def main() -> None:
    parser = argparse.ArgumentParser(description="Minimal LiteLLM gateway")
    parser.add_argument("--host", default="127.0.0.1")
    parser.add_argument("--port", type=int, default=4000)
    parser.add_argument("--config")
    parser.add_argument("--authenticate", action="store_true")
    parser.add_argument(
        "--token-dir",
        help=(
            "Persistent GitHub Copilot token directory. Defaults to "
            "GITHUB_COPILOT_TOKEN_DIR, MRJUNEJUNE_INFERENCE_STATE, or "
            "~/.local/state/mrjunejune/inference/litellm-copilot."
        ),
    )
    args = parser.parse_args()
    if args.authenticate:
        from litellm.llms.github_copilot.authenticator import Authenticator

        token_directory = resolve_token_directory(args.token_dir)
        token_directory.mkdir(parents=True, exist_ok=True, mode=0o700)
        token_directory.chmod(0o700)
        os.environ["GITHUB_COPILOT_TOKEN_DIR"] = str(token_directory)
        authenticator = Authenticator()
        authenticator.get_access_token()
        authenticator.get_api_key()
        print(f"GitHub Copilot authentication stored in {token_directory}")
        return
    if not args.config:
        parser.error("--config is required unless --authenticate is used")
    model_alias, upstream_model = _load_model(args.config)
    uvicorn.run(
        create_app(model_alias=model_alias, upstream_model=upstream_model),
        host=args.host,
        port=args.port,
        access_log=False,
    )


if __name__ == "__main__":
    main()