Mercurial

comparison mrjunejune/inference_stack.sh @ 265:056790c4fb0d

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
add role-aware Epi assistant prompts Add verified June knowledge, guest/member/admin Copilot profiles, profile-isolated session recovery, animated Epi greetings, and a single authoritative runtime config workflow for inference. Co-authored-by: Copilot <[email protected]>
author MrJuneJune <me@mrjunejune.com>
date Fri, 07 Aug 2026 10:50:30 -0700
parents 04fee26ecce0
children 13d61401c57d
comparison
equal deleted inserted replaced
264:04fee26ecce0 265:056790c4fb0d
17 17
18 mode=live 18 mode=live
19 if [[ "${1:-}" == "--mock" ]]; then 19 if [[ "${1:-}" == "--mock" ]]; then
20 mode=mock 20 mode=mock
21 shift 21 shift
22 elif [[ "${1:-}" == "--authenticate" ]]; then
23 mode=authenticate
24 shift
25 elif [[ "${1:-}" == "--check-config" ]]; then
26 mode=check-config
27 shift
22 fi 28 fi
23 if (( $# != 0 )); then 29 if (( $# != 0 )); then
24 echo "Usage: bazel run //mrjunejune:run_inference_stack -- [--mock]" >&2 30 echo "Usage: bazel run //mrjunejune:run_inference_stack -- [--mock|--authenticate|--check-config]" >&2
25 exit 2 31 exit 2
26 fi 32 fi
33
34 find_config() {
35 local candidate
36 for candidate in \
37 "${BUILD_WORKSPACE_DIRECTORY:-}/mrjunejune/.config" \
38 "$PWD/mrjunejune/.config" \
39 "/etc/mrjunejune/.config" \
40 "${HOME:-}/.config/mrjunejune/.config"; do
41 [[ "$candidate" != /mrjunejune/.config ]] || continue
42 [[ "$candidate" != /.config/mrjunejune/.config ]] || continue
43 if [[ -f "$candidate" ]]; then
44 realpath "$candidate"
45 return 0
46 fi
47 done
48 return 1
49 }
50
51 is_supported_config_key() {
52 case "$1" in
53 UPLOAD_AUTH_TOKEN|S3_REGION|S3_BUCKET|S3_URL_EXPIRES|S3_CLOUDFRONT_URL|\
54 DB_PATH|MRJUNEJUNE_DB_PATH|AWS_MRJUNEJUNE_ACCESS_KEY|\
55 AWS_MRJUNEJUNE_SECRET_ACCESS_KEY|AUTH_COOKIE_SECRET|\
56 AUTH_BOOTSTRAP_USERNAME|AUTH_BOOTSTRAP_PASSWORD_HASH|AUTH_TRUSTED_PROXY|\
57 AUTH_SESSION_IDLE_TTL|AUTH_SESSION_ABS_TTL|AUTH_GUEST_TTL|\
58 AUTH_DEV_INSECURE_COOKIE|AUTH_GUEST_DAILY_TURNS|\
59 AUTH_GUEST_DAILY_OUTPUT_TOKENS|AUTH_GUEST_REQUEST_OUTPUT_TOKENS|\
60 SERVER_HOST|MRJUNEJUNE_PORT|MRJUNEJUNE_ALLOW_GUEST_INFERENCE|\
61 MRJUNEJUNE_INFERENCE_STATE|MRJUNEJUNE_MOCK_STATE|\
62 GITHUB_COPILOT_TOKEN_DIR|LITELLM_MASTER_KEY|LITELLM_HOST|LITELLM_PORT|\
63 LITELLM_MODEL|LITELLM_WIRE_API|COPILOT_SESSION_IDLE_SECONDS|\
64 COPILOT_MAX_SESSIONS)
65 return 0
66 ;;
67 esac
68 return 1
69 }
70
71 load_config() {
72 local config_path="$1"
73 local line key value
74 while IFS= read -r line || [[ -n "$line" ]]; do
75 line="${line%$'\r'}"
76 [[ "$line" =~ ^[[:space:]]*$ ]] && continue
77 [[ "$line" =~ ^[[:space:]]*# ]] && continue
78 if [[ "$line" != *=* ]]; then
79 echo "Invalid config line in $config_path" >&2
80 exit 1
81 fi
82 key="${line%%=*}"
83 value="${line#*=}"
84 key="${key#"${key%%[![:space:]]*}"}"
85 key="${key%"${key##*[![:space:]]}"}"
86 if ! [[ "$key" =~ ^[A-Z][A-Z0-9_]*$ ]] ||
87 ! is_supported_config_key "$key"; then
88 echo "Unsupported config key in $config_path: $key" >&2
89 exit 1
90 fi
91 printf -v "$key" '%s' "$value"
92 export -n "$key" 2>/dev/null || true
93 done < "$config_path"
94 }
95
96 config_file="$(find_config || true)"
97 if [[ -z "$config_file" ]]; then
98 echo "Missing mrjunejune/.config." >&2
99 echo "Copy mrjunejune/.config.development to mrjunejune/.config." >&2
100 exit 1
101 fi
102 load_config "$config_file"
103 : "${MRJUNEJUNE_ALLOW_GUEST_INFERENCE:?MRJUNEJUNE_ALLOW_GUEST_INFERENCE must be set in $config_file}"
27 104
28 litellm_host="${LITELLM_HOST:-127.0.0.1}" 105 litellm_host="${LITELLM_HOST:-127.0.0.1}"
29 litellm_port="${LITELLM_PORT:-4000}" 106 litellm_port="${LITELLM_PORT:-4000}"
30 default_state_root="${XDG_STATE_HOME:-$HOME/.local/state}/mrjunejune/inference" 107 default_state_root="${XDG_STATE_HOME:-$HOME/.local/state}/mrjunejune/inference"
31 state_root="${MRJUNEJUNE_INFERENCE_STATE:-$default_state_root}" 108 state_root="${MRJUNEJUNE_INFERENCE_STATE:-$default_state_root}"
32 litellm_pid= 109 litellm_pid=
33 server_pid= 110 server_pid=
111
112 if [[ "$mode" == "check-config" ]]; then
113 printf 'config=%s\n' "$config_file"
114 if [[ -n "${LITELLM_MASTER_KEY:-}" ]]; then
115 echo 'litellm_master_key=set'
116 else
117 echo 'litellm_master_key=missing (generated on first local live run)'
118 fi
119 printf 'copilot_token_dir=%s\n' \
120 "${GITHUB_COPILOT_TOKEN_DIR:-$state_root/litellm-copilot}"
121 printf 'guest_inference=%s\n' \
122 "${MRJUNEJUNE_ALLOW_GUEST_INFERENCE:-false}"
123 exit 0
124 fi
125
126 if [[ "$mode" == "authenticate" ]]; then
127 token_dir="${GITHUB_COPILOT_TOKEN_DIR:-$state_root/litellm-copilot}"
128 mkdir -p "$token_dir"
129 chmod 700 "$token_dir"
130 exec "$python" "$litellm_zip" --authenticate --token-dir "$token_dir"
131 fi
34 132
35 cleanup() { 133 cleanup() {
36 for pid in "$server_pid" "$litellm_pid"; do 134 for pid in "$server_pid" "$litellm_pid"; do
37 if [[ -n "$pid" ]]; then 135 if [[ -n "$pid" ]]; then
38 kill -- "-$pid" 2>/dev/null || true 136 kill -- "-$pid" 2>/dev/null || true
67 export MRJUNEJUNE_INFERENCE_SIDECAR_PATH="$mock_sidecar" 165 export MRJUNEJUNE_INFERENCE_SIDECAR_PATH="$mock_sidecar"
68 else 166 else
69 export MRJUNEJUNE_INFERENCE_SIDECAR_PATH="$PWD/$mock_sidecar" 167 export MRJUNEJUNE_INFERENCE_SIDECAR_PATH="$PWD/$mock_sidecar"
70 fi 168 fi
71 export MRJUNEJUNE_COPILOT_CLI_PATH="$MRJUNEJUNE_INFERENCE_SIDECAR_PATH" 169 export MRJUNEJUNE_COPILOT_CLI_PATH="$MRJUNEJUNE_INFERENCE_SIDECAR_PATH"
72 export MRJUNEJUNE_ALLOW_ANONYMOUS_INFERENCE=1 170 export MRJUNEJUNE_ALLOW_GUEST_INFERENCE
73 # Bind to loopback for local dev; insecure cookies are only allowed here. 171 # Bind to loopback for local dev; insecure cookies are only allowed here.
74 export SERVER_HOST="${SERVER_HOST:-127.0.0.1}" 172 export SERVER_HOST="${SERVER_HOST:-127.0.0.1}"
75 export AUTH_DEV_INSECURE_COOKIE="${AUTH_DEV_INSECURE_COOKIE:-true}" 173 export AUTH_DEV_INSECURE_COOKIE="${AUTH_DEV_INSECURE_COOKIE:-true}"
76 echo "Starting mock inference at http://${SERVER_HOST}:${MRJUNEJUNE_PORT:-6969}/jrpg" 174 echo "Starting mock inference at http://${SERVER_HOST}:${MRJUNEJUNE_PORT:-6969}/jrpg"
77 else 175 else
78 : "${LITELLM_MASTER_KEY:?LITELLM_MASTER_KEY must be set}" 176 if [[ -z "${LITELLM_MASTER_KEY:-}" ]]; then
177 if [[ -n "${BUILD_WORKSPACE_DIRECTORY:-}" &&
178 "$config_file" == "$BUILD_WORKSPACE_DIRECTORY/"* &&
179 -w "$config_file" ]]; then
180 umask 077
181 LITELLM_MASTER_KEY="sk-$(
182 "$python" -c 'import secrets; print(secrets.token_hex(24))'
183 )"
184 printf '\n# Generated once for the local LiteLLM proxy.\nLITELLM_MASTER_KEY=%s\n' \
185 "$LITELLM_MASTER_KEY" >> "$config_file"
186 chmod 600 "$config_file"
187 export LITELLM_MASTER_KEY
188 echo "Generated LITELLM_MASTER_KEY in $config_file"
189 else
190 echo "LITELLM_MASTER_KEY is missing from $config_file" >&2
191 exit 1
192 fi
193 fi
194 export LITELLM_MASTER_KEY
79 token_dir="${GITHUB_COPILOT_TOKEN_DIR:-$state_root/litellm-copilot}" 195 token_dir="${GITHUB_COPILOT_TOKEN_DIR:-$state_root/litellm-copilot}"
80 mkdir -p "$state_root/copilot" "$token_dir" 196 mkdir -p "$state_root/copilot" "$token_dir"
81 chmod 700 "$state_root" "$state_root/copilot" "$token_dir" 197 chmod 700 "$state_root" "$state_root/copilot" "$token_dir"
82 198
83 export GITHUB_COPILOT_TOKEN_DIR="$token_dir" 199 export GITHUB_COPILOT_TOKEN_DIR="$token_dir"
84 if [[ ! -s "$GITHUB_COPILOT_TOKEN_DIR/access-token" ]]; then 200 if [[ ! -s "$GITHUB_COPILOT_TOKEN_DIR/access-token" ]]; then
85 echo "GitHub Copilot is not authenticated for LiteLLM." >&2 201 echo "GitHub Copilot is not authenticated for LiteLLM." >&2
86 echo "Run: bazel run //mrjunejune/inference:litellm_proxy -- --authenticate --token-dir \"$GITHUB_COPILOT_TOKEN_DIR\"" >&2 202 echo "Run: bazel run //mrjunejune:run_inference_stack -- --authenticate" >&2
87 exit 1 203 exit 1
88 fi 204 fi
89 205
90 setsid "$python" "$litellm_zip" \ 206 setsid "$python" "$litellm_zip" \
91 --config "$litellm_config" \ 207 --config "$litellm_config" \
124 export COPILOT_SIDECAR_HOME="$state_root/copilot" 240 export COPILOT_SIDECAR_HOME="$state_root/copilot"
125 export LITELLM_BASE_URL="http://${litellm_host}:${litellm_port}/v1" 241 export LITELLM_BASE_URL="http://${litellm_host}:${litellm_port}/v1"
126 export LITELLM_MODEL="${LITELLM_MODEL:-jrpg-copilot}" 242 export LITELLM_MODEL="${LITELLM_MODEL:-jrpg-copilot}"
127 export LITELLM_WIRE_API="${LITELLM_WIRE_API:-completions}" 243 export LITELLM_WIRE_API="${LITELLM_WIRE_API:-completions}"
128 export LITELLM_API_KEY="$LITELLM_MASTER_KEY" 244 export LITELLM_API_KEY="$LITELLM_MASTER_KEY"
129 export MRJUNEJUNE_ALLOW_ANONYMOUS_INFERENCE="${MRJUNEJUNE_ALLOW_ANONYMOUS_INFERENCE:-0}" 245 export MRJUNEJUNE_ALLOW_GUEST_INFERENCE
130 # run_inference_stack is a local dev command; bind to loopback. 246 # run_inference_stack is a local dev command; bind to loopback.
131 export SERVER_HOST="${SERVER_HOST:-127.0.0.1}" 247 export SERVER_HOST="${SERVER_HOST:-127.0.0.1}"
132 export AUTH_DEV_INSECURE_COOKIE="${AUTH_DEV_INSECURE_COOKIE:-true}" 248 export AUTH_DEV_INSECURE_COOKIE="${AUTH_DEV_INSECURE_COOKIE:-true}"
133 fi 249 fi
134 250
251 export MRJUNEJUNE_CONFIG_PATH="$config_file"
135 setsid "$server" & 252 setsid "$server" &
136 server_pid=$! 253 server_pid=$!
137 while true; do 254 while true; do
138 if ! kill -0 "$server_pid" 2>/dev/null; then 255 if ! kill -0 "$server_pid" 2>/dev/null; then
139 set +e 256 set +e