Mercurial

comparison mrjunejune/inference_stack.sh @ 260:1f9877b637e9

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Add Copilot-powered cyberpunk JRPG chat Integrate the production JRPG chat with Seobeo streaming, Deita persistence, and a Bazel-managed Copilot SDK and LiteLLM inference stack. Co-authored-by: Copilot <[email protected]>
author MrJuneJune <mrjunejune@users.noreply.github.com>
date Wed, 05 Aug 2026 09:19:41 -0700
parents
children b401627fc49e
comparison
equal deleted inserted replaced
259:667156fcd3e3 260:1f9877b637e9
1 #!/usr/bin/env bash
2 set -euo pipefail
3
4 server="$(realpath "$1")"
5 sidecar_launcher="$(realpath "$2")"
6 sidecar_zip="$(realpath "$3")"
7 copilot_cli="$(realpath "$4")"
8 litellm_zip="$(realpath "$5")"
9 litellm_config="$(realpath "$6")"
10 python="$(realpath "$7")"
11
12 : "${LITELLM_MASTER_KEY:?LITELLM_MASTER_KEY must be set}"
13
14 litellm_host="${LITELLM_HOST:-127.0.0.1}"
15 litellm_port="${LITELLM_PORT:-4000}"
16 default_state_root="${XDG_STATE_HOME:-$HOME/.local/state}/mrjunejune/inference"
17 state_root="${MRJUNEJUNE_INFERENCE_STATE:-$default_state_root}"
18 token_dir="${GITHUB_COPILOT_TOKEN_DIR:-$state_root/litellm-copilot}"
19 mkdir -p "$state_root/copilot" "$token_dir"
20 chmod 700 "$state_root" "$state_root/copilot" "$token_dir"
21
22 export GITHUB_COPILOT_TOKEN_DIR="$token_dir"
23 if [[ ! -s "$GITHUB_COPILOT_TOKEN_DIR/access-token" ]]; then
24 echo "GitHub Copilot is not authenticated for LiteLLM." >&2
25 echo "Run: bazel run //mrjunejune/inference:litellm_proxy -- --authenticate --token-dir \"$GITHUB_COPILOT_TOKEN_DIR\"" >&2
26 exit 1
27 fi
28
29 litellm_pid=
30 server_pid=
31
32 cleanup() {
33 for pid in "$server_pid" "$litellm_pid"; do
34 if [[ -n "$pid" ]]; then
35 kill -- "-$pid" 2>/dev/null || true
36 fi
37 done
38 for pid in "$server_pid" "$litellm_pid"; do
39 [[ -n "$pid" ]] || continue
40 for _ in $(seq 1 30); do
41 if ! kill -0 -- "-$pid" 2>/dev/null; then
42 break
43 fi
44 sleep 0.1
45 done
46 if kill -0 -- "-$pid" 2>/dev/null; then
47 kill -KILL -- "-$pid" 2>/dev/null || true
48 fi
49 wait "$pid" 2>/dev/null || true
50 done
51 }
52 trap cleanup EXIT INT TERM
53
54 setsid "$python" "$litellm_zip" \
55 --config "$litellm_config" \
56 --host "$litellm_host" \
57 --port "$litellm_port" &
58 litellm_pid=$!
59
60 for _ in $(seq 1 150); do
61 if ! kill -0 "$litellm_pid" 2>/dev/null; then
62 echo "LiteLLM exited before becoming ready" >&2
63 exit 1
64 fi
65 if curl --fail --silent \
66 --connect-timeout 1 \
67 --max-time 2 \
68 "http://${litellm_host}:${litellm_port}/health/readiness" \
69 >/dev/null; then
70 break
71 fi
72 sleep 0.1
73 done
74
75 if ! curl --fail --silent \
76 --connect-timeout 1 \
77 --max-time 2 \
78 "http://${litellm_host}:${litellm_port}/health/readiness" \
79 >/dev/null; then
80 echo "LiteLLM readiness timed out" >&2
81 exit 1
82 fi
83
84 export MRJUNEJUNE_PYTHON_PATH="$python"
85 export MRJUNEJUNE_SIDECAR_ZIP="$sidecar_zip"
86 export MRJUNEJUNE_INFERENCE_SIDECAR_PATH="$sidecar_launcher"
87 export MRJUNEJUNE_COPILOT_CLI_PATH="$copilot_cli"
88 export COPILOT_SIDECAR_HOME="$state_root/copilot"
89 export LITELLM_BASE_URL="http://${litellm_host}:${litellm_port}/v1"
90 export LITELLM_MODEL="${LITELLM_MODEL:-jrpg-copilot}"
91 export LITELLM_WIRE_API="${LITELLM_WIRE_API:-completions}"
92 export LITELLM_API_KEY="$LITELLM_MASTER_KEY"
93 export MRJUNEJUNE_ALLOW_ANONYMOUS_INFERENCE="${MRJUNEJUNE_ALLOW_ANONYMOUS_INFERENCE:-0}"
94
95 setsid "$server" &
96 server_pid=$!
97 while true; do
98 if ! kill -0 "$server_pid" 2>/dev/null; then
99 set +e
100 wait "$server_pid"
101 server_status=$?
102 set -e
103 exit "$server_status"
104 fi
105 if ! kill -0 "$litellm_pid" 2>/dev/null; then
106 wait "$litellm_pid" 2>/dev/null || true
107 echo "LiteLLM exited while the public server was running" >&2
108 exit 1
109 fi
110 sleep 0.2
111 done