Mercurial

comparison mrjunejune/inference_stack.sh @ 261:b401627fc49e

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Add JRPG mock flows and interactive previews Add scripted mock SSE commands, custom event forwarding, animated chat turns, full-height message navigation, and a cyberpunk resume dossier. Co-authored-by: Copilot <[email protected]>
author MrJuneJune <mrjunejune@users.noreply.github.com>
date Wed, 05 Aug 2026 20:38:32 -0700
parents 1f9877b637e9
children 04fee26ecce0
comparison
equal deleted inserted replaced
260:1f9877b637e9 261:b401627fc49e
6 sidecar_zip="$(realpath "$3")" 6 sidecar_zip="$(realpath "$3")"
7 copilot_cli="$(realpath "$4")" 7 copilot_cli="$(realpath "$4")"
8 litellm_zip="$(realpath "$5")" 8 litellm_zip="$(realpath "$5")"
9 litellm_config="$(realpath "$6")" 9 litellm_config="$(realpath "$6")"
10 python="$(realpath "$7")" 10 python="$(realpath "$7")"
11 mock_sidecar="${8:-}"
12 if (( $# >= 8 )); then
13 shift 8
14 else
15 shift 7
16 fi
11 17
12 : "${LITELLM_MASTER_KEY:?LITELLM_MASTER_KEY must be set}" 18 mode=live
19 if [[ "${1:-}" == "--mock" ]]; then
20 mode=mock
21 shift
22 fi
23 if (( $# != 0 )); then
24 echo "Usage: bazel run //mrjunejune:run_inference_stack -- [--mock]" >&2
25 exit 2
26 fi
13 27
14 litellm_host="${LITELLM_HOST:-127.0.0.1}" 28 litellm_host="${LITELLM_HOST:-127.0.0.1}"
15 litellm_port="${LITELLM_PORT:-4000}" 29 litellm_port="${LITELLM_PORT:-4000}"
16 default_state_root="${XDG_STATE_HOME:-$HOME/.local/state}/mrjunejune/inference" 30 default_state_root="${XDG_STATE_HOME:-$HOME/.local/state}/mrjunejune/inference"
17 state_root="${MRJUNEJUNE_INFERENCE_STATE:-$default_state_root}" 31 state_root="${MRJUNEJUNE_INFERENCE_STATE:-$default_state_root}"
18 token_dir="${GITHUB_COPILOT_TOKEN_DIR:-$state_root/litellm-copilot}"
19 mkdir -p "$state_root/copilot" "$token_dir"
20 chmod 700 "$state_root" "$state_root/copilot" "$token_dir"
21
22 export GITHUB_COPILOT_TOKEN_DIR="$token_dir"
23 if [[ ! -s "$GITHUB_COPILOT_TOKEN_DIR/access-token" ]]; then
24 echo "GitHub Copilot is not authenticated for LiteLLM." >&2
25 echo "Run: bazel run //mrjunejune/inference:litellm_proxy -- --authenticate --token-dir \"$GITHUB_COPILOT_TOKEN_DIR\"" >&2
26 exit 1
27 fi
28
29 litellm_pid= 32 litellm_pid=
30 server_pid= 33 server_pid=
31 34
32 cleanup() { 35 cleanup() {
33 for pid in "$server_pid" "$litellm_pid"; do 36 for pid in "$server_pid" "$litellm_pid"; do
49 wait "$pid" 2>/dev/null || true 52 wait "$pid" 2>/dev/null || true
50 done 53 done
51 } 54 }
52 trap cleanup EXIT INT TERM 55 trap cleanup EXIT INT TERM
53 56
54 setsid "$python" "$litellm_zip" \ 57 if [[ "$mode" == "mock" ]]; then
55 --config "$litellm_config" \ 58 if [[ -z "$mock_sidecar" ]]; then
56 --host "$litellm_host" \ 59 echo "Mock sidecar is unavailable in this runtime" >&2
57 --port "$litellm_port" &
58 litellm_pid=$!
59
60 for _ in $(seq 1 150); do
61 if ! kill -0 "$litellm_pid" 2>/dev/null; then
62 echo "LiteLLM exited before becoming ready" >&2
63 exit 1 60 exit 1
64 fi 61 fi
65 if curl --fail --silent \ 62 mock_state="${MRJUNEJUNE_MOCK_STATE:-$state_root/mock}"
63 mkdir -p "$mock_state"
64 chmod 700 "$mock_state"
65 export MRJUNEJUNE_DB_PATH="${MRJUNEJUNE_DB_PATH:-$mock_state/conversations.db}"
66 if [[ "$mock_sidecar" == /* ]]; then
67 export MRJUNEJUNE_INFERENCE_SIDECAR_PATH="$mock_sidecar"
68 else
69 export MRJUNEJUNE_INFERENCE_SIDECAR_PATH="$PWD/$mock_sidecar"
70 fi
71 export MRJUNEJUNE_COPILOT_CLI_PATH="$MRJUNEJUNE_INFERENCE_SIDECAR_PATH"
72 export MRJUNEJUNE_ALLOW_ANONYMOUS_INFERENCE=1
73 echo "Starting mock inference at http://127.0.0.1:${MRJUNEJUNE_PORT:-6969}/jrpg"
74 else
75 : "${LITELLM_MASTER_KEY:?LITELLM_MASTER_KEY must be set}"
76 token_dir="${GITHUB_COPILOT_TOKEN_DIR:-$state_root/litellm-copilot}"
77 mkdir -p "$state_root/copilot" "$token_dir"
78 chmod 700 "$state_root" "$state_root/copilot" "$token_dir"
79
80 export GITHUB_COPILOT_TOKEN_DIR="$token_dir"
81 if [[ ! -s "$GITHUB_COPILOT_TOKEN_DIR/access-token" ]]; then
82 echo "GitHub Copilot is not authenticated for LiteLLM." >&2
83 echo "Run: bazel run //mrjunejune/inference:litellm_proxy -- --authenticate --token-dir \"$GITHUB_COPILOT_TOKEN_DIR\"" >&2
84 exit 1
85 fi
86
87 setsid "$python" "$litellm_zip" \
88 --config "$litellm_config" \
89 --host "$litellm_host" \
90 --port "$litellm_port" &
91 litellm_pid=$!
92
93 for _ in $(seq 1 150); do
94 if ! kill -0 "$litellm_pid" 2>/dev/null; then
95 echo "LiteLLM exited before becoming ready" >&2
96 exit 1
97 fi
98 if curl --fail --silent \
99 --connect-timeout 1 \
100 --max-time 2 \
101 "http://${litellm_host}:${litellm_port}/health/readiness" \
102 >/dev/null; then
103 break
104 fi
105 sleep 0.1
106 done
107
108 if ! curl --fail --silent \
66 --connect-timeout 1 \ 109 --connect-timeout 1 \
67 --max-time 2 \ 110 --max-time 2 \
68 "http://${litellm_host}:${litellm_port}/health/readiness" \ 111 "http://${litellm_host}:${litellm_port}/health/readiness" \
69 >/dev/null; then 112 >/dev/null; then
70 break 113 echo "LiteLLM readiness timed out" >&2
114 exit 1
71 fi 115 fi
72 sleep 0.1
73 done
74 116
75 if ! curl --fail --silent \ 117 export MRJUNEJUNE_PYTHON_PATH="$python"
76 --connect-timeout 1 \ 118 export MRJUNEJUNE_SIDECAR_ZIP="$sidecar_zip"
77 --max-time 2 \ 119 export MRJUNEJUNE_INFERENCE_SIDECAR_PATH="$sidecar_launcher"
78 "http://${litellm_host}:${litellm_port}/health/readiness" \ 120 export MRJUNEJUNE_COPILOT_CLI_PATH="$copilot_cli"
79 >/dev/null; then 121 export COPILOT_SIDECAR_HOME="$state_root/copilot"
80 echo "LiteLLM readiness timed out" >&2 122 export LITELLM_BASE_URL="http://${litellm_host}:${litellm_port}/v1"
81 exit 1 123 export LITELLM_MODEL="${LITELLM_MODEL:-jrpg-copilot}"
124 export LITELLM_WIRE_API="${LITELLM_WIRE_API:-completions}"
125 export LITELLM_API_KEY="$LITELLM_MASTER_KEY"
126 export MRJUNEJUNE_ALLOW_ANONYMOUS_INFERENCE="${MRJUNEJUNE_ALLOW_ANONYMOUS_INFERENCE:-0}"
82 fi 127 fi
83
84 export MRJUNEJUNE_PYTHON_PATH="$python"
85 export MRJUNEJUNE_SIDECAR_ZIP="$sidecar_zip"
86 export MRJUNEJUNE_INFERENCE_SIDECAR_PATH="$sidecar_launcher"
87 export MRJUNEJUNE_COPILOT_CLI_PATH="$copilot_cli"
88 export COPILOT_SIDECAR_HOME="$state_root/copilot"
89 export LITELLM_BASE_URL="http://${litellm_host}:${litellm_port}/v1"
90 export LITELLM_MODEL="${LITELLM_MODEL:-jrpg-copilot}"
91 export LITELLM_WIRE_API="${LITELLM_WIRE_API:-completions}"
92 export LITELLM_API_KEY="$LITELLM_MASTER_KEY"
93 export MRJUNEJUNE_ALLOW_ANONYMOUS_INFERENCE="${MRJUNEJUNE_ALLOW_ANONYMOUS_INFERENCE:-0}"
94 128
95 setsid "$server" & 129 setsid "$server" &
96 server_pid=$! 130 server_pid=$!
97 while true; do 131 while true; do
98 if ! kill -0 "$server_pid" 2>/dev/null; then 132 if ! kill -0 "$server_pid" 2>/dev/null; then
100 wait "$server_pid" 134 wait "$server_pid"
101 server_status=$? 135 server_status=$?
102 set -e 136 set -e
103 exit "$server_status" 137 exit "$server_status"
104 fi 138 fi
105 if ! kill -0 "$litellm_pid" 2>/dev/null; then 139 if [[ -n "$litellm_pid" ]] && ! kill -0 "$litellm_pid" 2>/dev/null; then
106 wait "$litellm_pid" 2>/dev/null || true 140 wait "$litellm_pid" 2>/dev/null || true
107 echo "LiteLLM exited while the public server was running" >&2 141 echo "LiteLLM exited while the public server was running" >&2
108 exit 1 142 exit 1
109 fi 143 fi
110 sleep 0.2 144 sleep 0.2