Mercurial
view s3/s3_uploader.c @ 279:b3b547563ec7
Add Google connector service and agent wiki
Implement the C/Seobeo Google Drive and Gmail connector with encrypted OAuth storage, Zenbu authentication, browser testing, AI tool discovery, chunked HTTP decoding, and Bazel coverage. Consolidate repository guidance into progressive wiki documentation and enforce arena-first allocation for new first-party C code.
Co-authored-by: Copilot <[email protected]>
Copilot-Session: 84c338fd-0939-4bb3-b7f3-1062eb213e5d
| author | MrJuneJune <me@mrjunejune.com> |
|---|---|
| date | Mon, 17 Aug 2026 22:22:36 -0700 |
| parents | 5ec271d612ae |
| children |
line wrap: on
line source
#include "s3_uploader.h" #include "seobeo/seobeo.h" #include <stdio.h> #include <stdlib.h> #include <string.h> #include <time.h> #include <ctype.h> #include <openssl/hmac.h> #include <openssl/sha.h> #define S3_ARENA_SIZE (10 * ONE_MEGA_BYTE) #define S3_RESULT_ARENA_SIZE (64 * 1024) #define S3_SERVICE_NAME "s3" #define S3_AWS4_REQUEST "aws4_request" #define S3_ALGORITHM "AWS4-HMAC-SHA256" // --- Internal Structures --- // typedef struct { char date[9]; // YYYYMMDD char datetime[17]; // YYYYMMDDTHHMMSSZ } S3_Timestamp; // --- Forward Declarations --- // static void s3__get_timestamp(S3_Timestamp *p_ts); static void s3__sha256_hex(const uint8 *data, size_t len, char *out); static void s3__hmac_sha256(const uint8 *key, size_t key_len, const uint8 *data, size_t data_len, uint8 *out); static void s3__hex_encode(const uint8 *data, size_t len, char *out); static char *s3__uri_encode(const char *str, Dowa_Arena *p_arena); static char *s3__build_canonical_request(const char *method, const char *uri, const char *query, const char *headers, const char *signed_headers, const char *payload_hash, Dowa_Arena *p_arena); static char *s3__build_string_to_sign(const char *datetime, const char *date, const char *region, const char *canonical_request, Dowa_Arena *p_arena); static void s3__calculate_signing_key(const char *secret_key, const char *date, const char *region, uint8 *out); static char *s3__build_authorization_header(const char *access_key, const char *date, const char *region, const char *signed_headers, const uint8 *signing_key, const char *string_to_sign, Dowa_Arena *p_arena); static uint8 *s3__load_file(const char *path, size_t *p_size); // --- Public API Implementation --- // S3_Result S3_Upload_File(const S3_Config *p_config, const char *local_path, const char *s3_key) { const char *content_type = S3_Guess_Content_Type(local_path); return S3_Upload_File_With_Content_Type(p_config, local_path, s3_key, content_type); } S3_Result S3_Upload_File_With_Content_Type(const S3_Config *p_config, const char *local_path, const char *s3_key, const char *content_type) { S3_Result result = {0}; size_t file_size = 0; uint8 *file_data = s3__load_file(local_path, &file_size); if (!file_data) { result.p_arena = Dowa_Arena_Create(S3_RESULT_ARENA_SIZE); result.success = FALSE; result.status_code = 0; result.error_message = Dowa_String_Copy_Arena("Failed to read file", result.p_arena); return result; } result = S3_Upload_Data(p_config, file_data, file_size, s3_key, content_type); Dowa_Free(file_data); return result; } S3_Result S3_Upload_Data(const S3_Config *p_config, const uint8 *data, size_t data_length, const char *s3_key, const char *content_type) { S3_Result result = {0}; result.p_arena = Dowa_Arena_Create(S3_RESULT_ARENA_SIZE); if (!p_config || !data || !s3_key) { result.success = FALSE; result.error_message = Dowa_String_Copy_Arena("Invalid parameters", result.p_arena); return result; } Dowa_Arena *p_arena = Dowa_Arena_Create(S3_ARENA_SIZE); // Get timestamp S3_Timestamp ts; s3__get_timestamp(&ts); // Calculate payload hash char payload_hash[65]; s3__sha256_hex(data, data_length, payload_hash); // Build host char *host; if (p_config->endpoint) { host = Dowa_String_Copy_Arena((char *)p_config->endpoint, p_arena); } else if (p_config->use_path_style) { size_t host_len = strlen("s3.") + strlen(p_config->region) + strlen(".amazonaws.com") + 1; host = Dowa_Arena_Allocate(p_arena, host_len); snprintf(host, host_len, "s3.%s.amazonaws.com", p_config->region); } else { size_t host_len = strlen(p_config->bucket) + strlen(".s3.") + strlen(p_config->region) + strlen(".amazonaws.com") + 1; host = Dowa_Arena_Allocate(p_arena, host_len); snprintf(host, host_len, "%s.s3.%s.amazonaws.com", p_config->bucket, p_config->region); } // Build URI path char *uri_path; char *encoded_key = s3__uri_encode(s3_key, p_arena); if (p_config->use_path_style) { size_t uri_len = strlen("/") + strlen(p_config->bucket) + strlen("/") + strlen(encoded_key) + 1; uri_path = Dowa_Arena_Allocate(p_arena, uri_len); snprintf(uri_path, uri_len, "/%s/%s", p_config->bucket, encoded_key); } else { size_t uri_len = strlen("/") + strlen(encoded_key) + 1; uri_path = Dowa_Arena_Allocate(p_arena, uri_len); snprintf(uri_path, uri_len, "/%s", encoded_key); } // Build canonical headers (must be sorted alphabetically) // Note: Content-Length is NOT signed for S3 PUT requests (it's sent but not in signature) size_t headers_len = 512 + strlen(host) + strlen(content_type); char *canonical_headers = Dowa_Arena_Allocate(p_arena, headers_len); snprintf(canonical_headers, headers_len, "content-type:%s\n" "host:%s\n" "x-amz-content-sha256:%s\n" "x-amz-date:%s\n", content_type, host, payload_hash, ts.datetime); const char *signed_headers = "content-type;host;x-amz-content-sha256;x-amz-date"; // Build canonical request char *canonical_request = s3__build_canonical_request("PUT", uri_path, "", // No query string canonical_headers, signed_headers, payload_hash, p_arena); // Build string to sign char *string_to_sign = s3__build_string_to_sign(ts.datetime, ts.date, p_config->region, canonical_request, p_arena); // Calculate signing key uint8 signing_key[32]; s3__calculate_signing_key(p_config->secret_access_key, ts.date, p_config->region, signing_key); // Build authorization header char *auth_header = s3__build_authorization_header(p_config->access_key_id, ts.date, p_config->region, signed_headers, signing_key, string_to_sign, p_arena); // Build URL size_t url_len = strlen("https://") + strlen(host) + strlen(uri_path) + 1; char *url = Dowa_Arena_Allocate(p_arena, url_len); snprintf(url, url_len, "https://%s%s", host, uri_path); // Execute request using seobeo Seobeo_Client_Request *p_req = Seobeo_Client_Request_Create(url); Seobeo_Client_Request_Set_Method(p_req, "PUT"); Seobeo_Client_Request_Set_Body(p_req, (const char *)data, data_length); Seobeo_Client_Request_Add_Header_Map(p_req, "Connection", "keep-alive"); Seobeo_Client_Request_Add_Header_Map(p_req, "Content-Type", content_type); Seobeo_Client_Request_Add_Header_Map(p_req, "x-amz-date", ts.datetime); Seobeo_Client_Request_Add_Header_Map(p_req, "x-amz-content-sha256", payload_hash); Seobeo_Client_Request_Add_Header_Map(p_req, "Authorization", auth_header); Seobeo_Log(SEOBEO_DEBUG, "[S3] Uploading %zu bytes to %s\n", data_length, url); Seobeo_Log(SEOBEO_DEBUG, "[S3] Content-Type: %s\n", content_type); Seobeo_Log(SEOBEO_DEBUG, "[S3] x-amz-date: %s\n", ts.datetime); Seobeo_Log(SEOBEO_DEBUG, "[S3] x-amz-content-sha256: %s\n", payload_hash); Seobeo_Log(SEOBEO_DEBUG, "[S3] Authorization: %.80s...\n", auth_header); Seobeo_Client_Response *p_resp = Seobeo_Client_Request_Execute(p_req); if (p_resp) { result.status_code = p_resp->status_code; if (p_resp->status_code >= 200 && p_resp->status_code < 300) { result.success = TRUE; // Extract ETag from response headers if (p_resp->headers) { char *etag = Dowa_HashMap_Get(p_resp->headers, "etag"); if (!etag) etag = Dowa_HashMap_Get(p_resp->headers, "ETag"); if (etag) { result.etag = Dowa_String_Copy_Arena(etag, result.p_arena); } } } else { result.success = FALSE; if (p_resp->body && p_resp->body_length > 0) { result.error_message = Dowa_String_Copy_Arena(p_resp->body, result.p_arena); } else { result.error_message = Dowa_String_Copy_Arena("Upload failed", result.p_arena); } } Seobeo_Client_Response_Destroy(p_resp); } else { result.success = FALSE; result.error_message = Dowa_String_Copy_Arena("Failed to execute request", result.p_arena); } Seobeo_Client_Request_Destroy(p_req); Dowa_Arena_Free(p_arena); return result; } void S3_Result_Destroy(S3_Result *p_result) { if (p_result && p_result->p_arena) { Dowa_Arena_Free(p_result->p_arena); p_result->p_arena = NULL; p_result->error_message = NULL; p_result->etag = NULL; } } // --- Presigned URL Implementation --- // static char *s3__uri_encode_strict(const char *str, Dowa_Arena *p_arena) { // Stricter encoding for query string values (no forward slash allowed) if (!str) return NULL; size_t len = strlen(str); size_t max_len = len * 3 + 1; char *out = Dowa_Arena_Allocate(p_arena, max_len); char *p = out; for (size_t i = 0; i < len; i++) { char c = str[i]; if ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '-' || c == '_' || c == '.' || c == '~') { *p++ = c; } else { sprintf(p, "%%%02X", (unsigned char)c); p += 3; } } *p = '\0'; return out; } static S3_Presigned_URL s3__presign_url(const S3_Config *p_config, const char *s3_key, const char *method, const char *content_type, int32 expires_seconds) { S3_Presigned_URL result = {0}; result.p_arena = Dowa_Arena_Create(S3_RESULT_ARENA_SIZE); if (!p_config || !s3_key) { result.success = FALSE; result.error_message = Dowa_String_Copy_Arena("Invalid parameters", result.p_arena); return result; } Dowa_Arena *p_arena = Dowa_Arena_Create(S3_ARENA_SIZE); // Get timestamp S3_Timestamp ts; s3__get_timestamp(&ts); // Build host char *host; if (p_config->endpoint) { host = Dowa_String_Copy_Arena((char *)p_config->endpoint, p_arena); } else if (p_config->use_path_style) { size_t host_len = strlen("s3.") + strlen(p_config->region) + strlen(".amazonaws.com") + 1; host = Dowa_Arena_Allocate(p_arena, host_len); snprintf(host, host_len, "s3.%s.amazonaws.com", p_config->region); } else { size_t host_len = strlen(p_config->bucket) + strlen(".s3.") + strlen(p_config->region) + strlen(".amazonaws.com") + 1; host = Dowa_Arena_Allocate(p_arena, host_len); snprintf(host, host_len, "%s.s3.%s.amazonaws.com", p_config->bucket, p_config->region); } // Build URI path char *uri_path; char *encoded_key = s3__uri_encode(s3_key, p_arena); if (p_config->use_path_style) { size_t uri_len = strlen("/") + strlen(p_config->bucket) + strlen("/") + strlen(encoded_key) + 1; uri_path = Dowa_Arena_Allocate(p_arena, uri_len); snprintf(uri_path, uri_len, "/%s/%s", p_config->bucket, encoded_key); } else { size_t uri_len = strlen("/") + strlen(encoded_key) + 1; uri_path = Dowa_Arena_Allocate(p_arena, uri_len); snprintf(uri_path, uri_len, "/%s", encoded_key); } // Build credential scope size_t scope_len = strlen(p_config->access_key_id) + strlen(ts.date) + strlen(p_config->region) + strlen(S3_SERVICE_NAME) + strlen(S3_AWS4_REQUEST) + 10; char *credential = Dowa_Arena_Allocate(p_arena, scope_len); snprintf(credential, scope_len, "%s/%s/%s/%s/%s", p_config->access_key_id, ts.date, p_config->region, S3_SERVICE_NAME, S3_AWS4_REQUEST); char *encoded_credential = s3__uri_encode_strict(credential, p_arena); // Build signed headers (host is required, content-type for PUT) const char *signed_headers; if (content_type && strcmp(method, "PUT") == 0) { signed_headers = "content-type;host"; } else { signed_headers = "host"; } // Build canonical query string (must be sorted alphabetically) char expires_str[16]; snprintf(expires_str, sizeof(expires_str), "%d", expires_seconds); // URL-encode signed headers (semicolon becomes %3B) char *encoded_signed_headers = s3__uri_encode_strict(signed_headers, p_arena); size_t query_len = 1024 + strlen(encoded_credential); char *canonical_query = Dowa_Arena_Allocate(p_arena, query_len); snprintf(canonical_query, query_len, "X-Amz-Algorithm=%s" "&X-Amz-Credential=%s" "&X-Amz-Date=%s" "&X-Amz-Expires=%s" "&X-Amz-SignedHeaders=%s", S3_ALGORITHM, encoded_credential, ts.datetime, expires_str, encoded_signed_headers); // Build canonical headers size_t headers_len = 256 + strlen(host) + (content_type ? strlen(content_type) : 0); char *canonical_headers = Dowa_Arena_Allocate(p_arena, headers_len); if (content_type && strcmp(method, "PUT") == 0) { snprintf(canonical_headers, headers_len, "content-type:%s\nhost:%s\n", content_type, host); } else { snprintf(canonical_headers, headers_len, "host:%s\n", host); } // For presigned URLs, payload is UNSIGNED-PAYLOAD const char *payload_hash = "UNSIGNED-PAYLOAD"; // Build canonical request char *canonical_request = s3__build_canonical_request(method, uri_path, canonical_query, canonical_headers, signed_headers, payload_hash, p_arena); // Build string to sign char *string_to_sign = s3__build_string_to_sign(ts.datetime, ts.date, p_config->region, canonical_request, p_arena); // Calculate signing key uint8 signing_key[32]; s3__calculate_signing_key(p_config->secret_access_key, ts.date, p_config->region, signing_key); // Calculate signature uint8 signature[32]; s3__hmac_sha256(signing_key, 32, (const uint8 *)string_to_sign, strlen(string_to_sign), signature); char signature_hex[65]; s3__hex_encode(signature, 32, signature_hex); // Build final presigned URL size_t url_len = strlen("https://") + strlen(host) + strlen(uri_path) + 1 + strlen(canonical_query) + strlen("&X-Amz-Signature=") + 64 + 1; char *url = Dowa_Arena_Allocate(result.p_arena, url_len); snprintf(url, url_len, "https://%s%s?%s&X-Amz-Signature=%s", host, uri_path, canonical_query, signature_hex); result.success = TRUE; result.url = url; Dowa_Arena_Free(p_arena); return result; } S3_Presigned_URL S3_Presign_Put(const S3_Config *p_config, const char *s3_key, const char *content_type, int32 expires_seconds) { return s3__presign_url(p_config, s3_key, "PUT", content_type, expires_seconds); } S3_Presigned_URL S3_Presign_Get(const S3_Config *p_config, const char *s3_key, int32 expires_seconds) { return s3__presign_url(p_config, s3_key, "GET", NULL, expires_seconds); } void S3_Presigned_URL_Destroy(S3_Presigned_URL *p_url) { if (p_url && p_url->p_arena) { Dowa_Arena_Free(p_url->p_arena); p_url->p_arena = NULL; p_url->url = NULL; p_url->error_message = NULL; } } const char *S3_Guess_Content_Type(const char *filename) { if (!filename) return "application/octet-stream"; const char *dot = strrchr(filename, '.'); if (!dot) return "application/octet-stream"; dot++; // Skip the dot // Common content types if (strcasecmp(dot, "html") == 0 || strcasecmp(dot, "htm") == 0) return "text/html"; if (strcasecmp(dot, "css") == 0) return "text/css"; if (strcasecmp(dot, "js") == 0) return "application/javascript"; if (strcasecmp(dot, "json") == 0) return "application/json"; if (strcasecmp(dot, "xml") == 0) return "application/xml"; if (strcasecmp(dot, "txt") == 0) return "text/plain"; if (strcasecmp(dot, "csv") == 0) return "text/csv"; // Images if (strcasecmp(dot, "png") == 0) return "image/png"; if (strcasecmp(dot, "jpg") == 0 || strcasecmp(dot, "jpeg") == 0) return "image/jpeg"; if (strcasecmp(dot, "gif") == 0) return "image/gif"; if (strcasecmp(dot, "svg") == 0) return "image/svg+xml"; if (strcasecmp(dot, "webp") == 0) return "image/webp"; if (strcasecmp(dot, "ico") == 0) return "image/x-icon"; // Audio/Video if (strcasecmp(dot, "mp3") == 0) return "audio/mpeg"; if (strcasecmp(dot, "mp4") == 0) return "video/mp4"; if (strcasecmp(dot, "webm") == 0) return "video/webm"; if (strcasecmp(dot, "ogg") == 0) return "audio/ogg"; if (strcasecmp(dot, "wav") == 0) return "audio/wav"; // Documents if (strcasecmp(dot, "pdf") == 0) return "application/pdf"; if (strcasecmp(dot, "zip") == 0) return "application/zip"; if (strcasecmp(dot, "gz") == 0 || strcasecmp(dot, "gzip") == 0) return "application/gzip"; if (strcasecmp(dot, "tar") == 0) return "application/x-tar"; // Fonts if (strcasecmp(dot, "woff") == 0) return "font/woff"; if (strcasecmp(dot, "woff2") == 0) return "font/woff2"; if (strcasecmp(dot, "ttf") == 0) return "font/ttf"; if (strcasecmp(dot, "otf") == 0) return "font/otf"; return "application/octet-stream"; } // --- Internal Implementation --- // static void s3__get_timestamp(S3_Timestamp *p_ts) { time_t now = time(NULL); struct tm *utc = gmtime(&now); strftime(p_ts->date, sizeof(p_ts->date), "%Y%m%d", utc); strftime(p_ts->datetime, sizeof(p_ts->datetime), "%Y%m%dT%H%M%SZ", utc); } static void s3__hex_encode(const uint8 *data, size_t len, char *out) { static const char hex[] = "0123456789abcdef"; for (size_t i = 0; i < len; i++) { out[i * 2] = hex[(data[i] >> 4) & 0x0F]; out[i * 2 + 1] = hex[data[i] & 0x0F]; } out[len * 2] = '\0'; } static void s3__sha256_hex(const uint8 *data, size_t len, char *out) { uint8 hash[SHA256_DIGEST_LENGTH]; SHA256(data, len, hash); s3__hex_encode(hash, SHA256_DIGEST_LENGTH, out); } static void s3__hmac_sha256(const uint8 *key, size_t key_len, const uint8 *data, size_t data_len, uint8 *out) { unsigned int out_len = 0; HMAC(EVP_sha256(), key, (int)key_len, data, data_len, out, &out_len); } static char *s3__uri_encode(const char *str, Dowa_Arena *p_arena) { if (!str) return NULL; size_t len = strlen(str); // Worst case: every char becomes %XX (3 chars) size_t max_len = len * 3 + 1; char *out = Dowa_Arena_Allocate(p_arena, max_len); char *p = out; for (size_t i = 0; i < len; i++) { char c = str[i]; // Unreserved characters per RFC 3986 if ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '-' || c == '_' || c == '.' || c == '~' || c == '/') { *p++ = c; } else { sprintf(p, "%%%02X", (unsigned char)c); p += 3; } } *p = '\0'; return out; } static char *s3__build_canonical_request(const char *method, const char *uri, const char *query, const char *headers, const char *signed_headers, const char *payload_hash, Dowa_Arena *p_arena) { size_t len = strlen(method) + strlen(uri) + strlen(query) + strlen(headers) + strlen(signed_headers) + strlen(payload_hash) + 10; char *out = Dowa_Arena_Allocate(p_arena, len); snprintf(out, len, "%s\n%s\n%s\n%s\n%s\n%s", method, uri, query, headers, signed_headers, payload_hash); return out; } static char *s3__build_string_to_sign(const char *datetime, const char *date, const char *region, const char *canonical_request, Dowa_Arena *p_arena) { // Hash the canonical request char canonical_hash[65]; s3__sha256_hex((const uint8 *)canonical_request, strlen(canonical_request), canonical_hash); // Build credential scope size_t scope_len = strlen(date) + strlen(region) + strlen(S3_SERVICE_NAME) + strlen(S3_AWS4_REQUEST) + 4; char *scope = Dowa_Arena_Allocate(p_arena, scope_len); snprintf(scope, scope_len, "%s/%s/%s/%s", date, region, S3_SERVICE_NAME, S3_AWS4_REQUEST); // Build string to sign size_t len = strlen(S3_ALGORITHM) + strlen(datetime) + strlen(scope) + strlen(canonical_hash) + 10; char *out = Dowa_Arena_Allocate(p_arena, len); snprintf(out, len, "%s\n%s\n%s\n%s", S3_ALGORITHM, datetime, scope, canonical_hash); return out; } static void s3__calculate_signing_key(const char *secret_key, const char *date, const char *region, uint8 *out) { // AWS4 + SecretAccessKey size_t key_len = 4 + strlen(secret_key) + 1; char *k_secret = malloc(key_len); snprintf(k_secret, key_len, "AWS4%s", secret_key); uint8 k_date[32]; uint8 k_region[32]; uint8 k_service[32]; s3__hmac_sha256((const uint8 *)k_secret, strlen(k_secret), (const uint8 *)date, strlen(date), k_date); s3__hmac_sha256(k_date, 32, (const uint8 *)region, strlen(region), k_region); s3__hmac_sha256(k_region, 32, (const uint8 *)S3_SERVICE_NAME, strlen(S3_SERVICE_NAME), k_service); s3__hmac_sha256(k_service, 32, (const uint8 *)S3_AWS4_REQUEST, strlen(S3_AWS4_REQUEST), out); Dowa_Free(k_secret); } static char *s3__build_authorization_header(const char *access_key, const char *date, const char *region, const char *signed_headers, const uint8 *signing_key, const char *string_to_sign, Dowa_Arena *p_arena) { // Calculate signature uint8 signature[32]; s3__hmac_sha256(signing_key, 32, (const uint8 *)string_to_sign, strlen(string_to_sign), signature); char signature_hex[65]; s3__hex_encode(signature, 32, signature_hex); // Build credential size_t cred_len = strlen(access_key) + strlen(date) + strlen(region) + strlen(S3_SERVICE_NAME) + strlen(S3_AWS4_REQUEST) + 10; char *credential = Dowa_Arena_Allocate(p_arena, cred_len); snprintf(credential, cred_len, "%s/%s/%s/%s/%s", access_key, date, region, S3_SERVICE_NAME, S3_AWS4_REQUEST); // Build full authorization header size_t auth_len = strlen(S3_ALGORITHM) + strlen(credential) + strlen(signed_headers) + strlen(signature_hex) + 64; char *auth = Dowa_Arena_Allocate(p_arena, auth_len); snprintf(auth, auth_len, "%s Credential=%s, SignedHeaders=%s, Signature=%s", S3_ALGORITHM, credential, signed_headers, signature_hex); return auth; } static uint8 *s3__load_file(const char *path, size_t *p_size) { FILE *f = fopen(path, "rb"); if (!f) { *p_size = 0; return NULL; } fseek(f, 0, SEEK_END); long size = ftell(f); fseek(f, 0, SEEK_SET); if (size <= 0) { fclose(f); *p_size = 0; return NULL; } uint8 *data = malloc((size_t)size); if (!data) { fclose(f); *p_size = 0; return NULL; } size_t read = fread(data, 1, (size_t)size, f); fclose(f); if (read != (size_t)size) { Dowa_Free(data); *p_size = 0; return NULL; } *p_size = (size_t)size; return data; }