Mercurial
view s3/s3_uploader.c @ 207:58d9b64d8dca
Updated deployment script to include sqlite3
| author | MrJuneJune <me@mrjunejune.com> |
|---|---|
| date | Sun, 15 Feb 2026 12:25:50 -0800 |
| parents | 240337164a80 |
| children |
line wrap: on
line source
#include "s3_uploader.h" #include "seobeo/seobeo.h" #include <stdio.h> #include <stdlib.h> #include <string.h> #include <time.h> #include <ctype.h> #include <openssl/hmac.h> #include <openssl/sha.h> #define S3_ARENA_SIZE (10 * ONE_MEGA_BYTE) #define S3_RESULT_ARENA_SIZE (64 * 1024) #define S3_SERVICE_NAME "s3" #define S3_AWS4_REQUEST "aws4_request" #define S3_ALGORITHM "AWS4-HMAC-SHA256" // --- Internal Structures --- // typedef struct { char date[9]; // YYYYMMDD char datetime[17]; // YYYYMMDDTHHMMSSZ } S3_Timestamp; // --- Forward Declarations --- // static void s3__get_timestamp(S3_Timestamp *p_ts); static void s3__sha256_hex(const uint8 *data, size_t len, char *out); static void s3__hmac_sha256(const uint8 *key, size_t key_len, const uint8 *data, size_t data_len, uint8 *out); static void s3__hex_encode(const uint8 *data, size_t len, char *out); static char *s3__uri_encode(const char *str, Dowa_Arena *p_arena); static char *s3__build_canonical_request(const char *method, const char *uri, const char *query, const char *headers, const char *signed_headers, const char *payload_hash, Dowa_Arena *p_arena); static char *s3__build_string_to_sign(const char *datetime, const char *date, const char *region, const char *canonical_request, Dowa_Arena *p_arena); static void s3__calculate_signing_key(const char *secret_key, const char *date, const char *region, uint8 *out); static char *s3__build_authorization_header(const char *access_key, const char *date, const char *region, const char *signed_headers, const uint8 *signing_key, const char *string_to_sign, Dowa_Arena *p_arena); static uint8 *s3__load_file(const char *path, size_t *p_size); // --- Public API Implementation --- // S3_Result S3_Upload_File(const S3_Config *p_config, const char *local_path, const char *s3_key) { const char *content_type = S3_Guess_Content_Type(local_path); return S3_Upload_File_With_Content_Type(p_config, local_path, s3_key, content_type); } S3_Result S3_Upload_File_With_Content_Type(const S3_Config *p_config, const char *local_path, const char *s3_key, const char *content_type) { S3_Result result = {0}; result.p_arena = Dowa_Arena_Create(S3_RESULT_ARENA_SIZE); size_t file_size = 0; uint8 *file_data = s3__load_file(local_path, &file_size); if (!file_data) { result.success = FALSE; result.status_code = 0; result.error_message = Dowa_String_Copy_Arena("Failed to read file", result.p_arena); return result; } result = S3_Upload_Data(p_config, file_data, file_size, s3_key, content_type); free(file_data); return result; } S3_Result S3_Upload_Data(const S3_Config *p_config, const uint8 *data, size_t data_length, const char *s3_key, const char *content_type) { S3_Result result = {0}; result.p_arena = Dowa_Arena_Create(S3_RESULT_ARENA_SIZE); if (!p_config || !data || !s3_key) { result.success = FALSE; result.error_message = Dowa_String_Copy_Arena("Invalid parameters", result.p_arena); return result; } Dowa_Arena *p_arena = Dowa_Arena_Create(S3_ARENA_SIZE); // Get timestamp S3_Timestamp ts; s3__get_timestamp(&ts); // Calculate payload hash char payload_hash[65]; s3__sha256_hex(data, data_length, payload_hash); // Build host char *host; if (p_config->endpoint) { host = Dowa_String_Copy_Arena((char *)p_config->endpoint, p_arena); } else if (p_config->use_path_style) { size_t host_len = strlen("s3.") + strlen(p_config->region) + strlen(".amazonaws.com") + 1; host = Dowa_Arena_Allocate(p_arena, host_len); snprintf(host, host_len, "s3.%s.amazonaws.com", p_config->region); } else { size_t host_len = strlen(p_config->bucket) + strlen(".s3.") + strlen(p_config->region) + strlen(".amazonaws.com") + 1; host = Dowa_Arena_Allocate(p_arena, host_len); snprintf(host, host_len, "%s.s3.%s.amazonaws.com", p_config->bucket, p_config->region); } // Build URI path char *uri_path; char *encoded_key = s3__uri_encode(s3_key, p_arena); if (p_config->use_path_style) { size_t uri_len = strlen("/") + strlen(p_config->bucket) + strlen("/") + strlen(encoded_key) + 1; uri_path = Dowa_Arena_Allocate(p_arena, uri_len); snprintf(uri_path, uri_len, "/%s/%s", p_config->bucket, encoded_key); } else { size_t uri_len = strlen("/") + strlen(encoded_key) + 1; uri_path = Dowa_Arena_Allocate(p_arena, uri_len); snprintf(uri_path, uri_len, "/%s", encoded_key); } // Build canonical headers (must be sorted alphabetically) // Note: Content-Length is NOT signed for S3 PUT requests (it's sent but not in signature) size_t headers_len = 512 + strlen(host) + strlen(content_type); char *canonical_headers = Dowa_Arena_Allocate(p_arena, headers_len); snprintf(canonical_headers, headers_len, "content-type:%s\n" "host:%s\n" "x-amz-content-sha256:%s\n" "x-amz-date:%s\n", content_type, host, payload_hash, ts.datetime); const char *signed_headers = "content-type;host;x-amz-content-sha256;x-amz-date"; // Build canonical request char *canonical_request = s3__build_canonical_request("PUT", uri_path, "", // No query string canonical_headers, signed_headers, payload_hash, p_arena); // Build string to sign char *string_to_sign = s3__build_string_to_sign(ts.datetime, ts.date, p_config->region, canonical_request, p_arena); // Calculate signing key uint8 signing_key[32]; s3__calculate_signing_key(p_config->secret_access_key, ts.date, p_config->region, signing_key); // Build authorization header char *auth_header = s3__build_authorization_header(p_config->access_key_id, ts.date, p_config->region, signed_headers, signing_key, string_to_sign, p_arena); // Build URL size_t url_len = strlen("https://") + strlen(host) + strlen(uri_path) + 1; char *url = Dowa_Arena_Allocate(p_arena, url_len); snprintf(url, url_len, "https://%s%s", host, uri_path); // Execute request using seobeo Seobeo_Client_Request *p_req = Seobeo_Client_Request_Create(url); Seobeo_Client_Request_Set_Method(p_req, "PUT"); Seobeo_Client_Request_Set_Body(p_req, (const char *)data, data_length); Seobeo_Client_Request_Add_Header_Map(p_req, "Connection", "keep-alive"); Seobeo_Client_Request_Add_Header_Map(p_req, "Content-Type", content_type); Seobeo_Client_Request_Add_Header_Map(p_req, "x-amz-date", ts.datetime); Seobeo_Client_Request_Add_Header_Map(p_req, "x-amz-content-sha256", payload_hash); Seobeo_Client_Request_Add_Header_Map(p_req, "Authorization", auth_header); Seobeo_Log(SEOBEO_DEBUG, "[S3] Uploading %zu bytes to %s\n", data_length, url); Seobeo_Log(SEOBEO_DEBUG, "[S3] Content-Type: %s\n", content_type); Seobeo_Log(SEOBEO_DEBUG, "[S3] x-amz-date: %s\n", ts.datetime); Seobeo_Log(SEOBEO_DEBUG, "[S3] x-amz-content-sha256: %s\n", payload_hash); Seobeo_Log(SEOBEO_DEBUG, "[S3] Authorization: %.80s...\n", auth_header); Seobeo_Client_Response *p_resp = Seobeo_Client_Request_Execute(p_req); if (p_resp) { result.status_code = p_resp->status_code; if (p_resp->status_code >= 200 && p_resp->status_code < 300) { result.success = TRUE; // Extract ETag from response headers if (p_resp->headers) { char *etag = Dowa_HashMap_Get(p_resp->headers, "etag"); if (!etag) etag = Dowa_HashMap_Get(p_resp->headers, "ETag"); if (etag) { result.etag = Dowa_String_Copy_Arena(etag, result.p_arena); } } } else { result.success = FALSE; if (p_resp->body && p_resp->body_length > 0) { result.error_message = Dowa_String_Copy_Arena(p_resp->body, result.p_arena); } else { result.error_message = Dowa_String_Copy_Arena("Upload failed", result.p_arena); } } Seobeo_Client_Response_Destroy(p_resp); } else { result.success = FALSE; result.error_message = Dowa_String_Copy_Arena("Failed to execute request", result.p_arena); } Seobeo_Client_Request_Destroy(p_req); Dowa_Arena_Free(p_arena); return result; } void S3_Result_Destroy(S3_Result *p_result) { if (p_result && p_result->p_arena) { Dowa_Arena_Free(p_result->p_arena); p_result->p_arena = NULL; p_result->error_message = NULL; p_result->etag = NULL; } } // --- Presigned URL Implementation --- // static char *s3__uri_encode_strict(const char *str, Dowa_Arena *p_arena) { // Stricter encoding for query string values (no forward slash allowed) if (!str) return NULL; size_t len = strlen(str); size_t max_len = len * 3 + 1; char *out = Dowa_Arena_Allocate(p_arena, max_len); char *p = out; for (size_t i = 0; i < len; i++) { char c = str[i]; if ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '-' || c == '_' || c == '.' || c == '~') { *p++ = c; } else { sprintf(p, "%%%02X", (unsigned char)c); p += 3; } } *p = '\0'; return out; } static S3_Presigned_URL s3__presign_url(const S3_Config *p_config, const char *s3_key, const char *method, const char *content_type, int32 expires_seconds) { S3_Presigned_URL result = {0}; result.p_arena = Dowa_Arena_Create(S3_RESULT_ARENA_SIZE); if (!p_config || !s3_key) { result.success = FALSE; result.error_message = Dowa_String_Copy_Arena("Invalid parameters", result.p_arena); return result; } Dowa_Arena *p_arena = Dowa_Arena_Create(S3_ARENA_SIZE); // Get timestamp S3_Timestamp ts; s3__get_timestamp(&ts); // Build host char *host; if (p_config->endpoint) { host = Dowa_String_Copy_Arena((char *)p_config->endpoint, p_arena); } else if (p_config->use_path_style) { size_t host_len = strlen("s3.") + strlen(p_config->region) + strlen(".amazonaws.com") + 1; host = Dowa_Arena_Allocate(p_arena, host_len); snprintf(host, host_len, "s3.%s.amazonaws.com", p_config->region); } else { size_t host_len = strlen(p_config->bucket) + strlen(".s3.") + strlen(p_config->region) + strlen(".amazonaws.com") + 1; host = Dowa_Arena_Allocate(p_arena, host_len); snprintf(host, host_len, "%s.s3.%s.amazonaws.com", p_config->bucket, p_config->region); } // Build URI path char *uri_path; char *encoded_key = s3__uri_encode(s3_key, p_arena); if (p_config->use_path_style) { size_t uri_len = strlen("/") + strlen(p_config->bucket) + strlen("/") + strlen(encoded_key) + 1; uri_path = Dowa_Arena_Allocate(p_arena, uri_len); snprintf(uri_path, uri_len, "/%s/%s", p_config->bucket, encoded_key); } else { size_t uri_len = strlen("/") + strlen(encoded_key) + 1; uri_path = Dowa_Arena_Allocate(p_arena, uri_len); snprintf(uri_path, uri_len, "/%s", encoded_key); } // Build credential scope size_t scope_len = strlen(p_config->access_key_id) + strlen(ts.date) + strlen(p_config->region) + strlen(S3_SERVICE_NAME) + strlen(S3_AWS4_REQUEST) + 10; char *credential = Dowa_Arena_Allocate(p_arena, scope_len); snprintf(credential, scope_len, "%s/%s/%s/%s/%s", p_config->access_key_id, ts.date, p_config->region, S3_SERVICE_NAME, S3_AWS4_REQUEST); char *encoded_credential = s3__uri_encode_strict(credential, p_arena); // Build signed headers (host is required, content-type for PUT) const char *signed_headers; if (content_type && strcmp(method, "PUT") == 0) { signed_headers = "content-type;host"; } else { signed_headers = "host"; } // Build canonical query string (must be sorted alphabetically) char expires_str[16]; snprintf(expires_str, sizeof(expires_str), "%d", expires_seconds); // URL-encode signed headers (semicolon becomes %3B) char *encoded_signed_headers = s3__uri_encode_strict(signed_headers, p_arena); size_t query_len = 1024 + strlen(encoded_credential); char *canonical_query = Dowa_Arena_Allocate(p_arena, query_len); snprintf(canonical_query, query_len, "X-Amz-Algorithm=%s" "&X-Amz-Credential=%s" "&X-Amz-Date=%s" "&X-Amz-Expires=%s" "&X-Amz-SignedHeaders=%s", S3_ALGORITHM, encoded_credential, ts.datetime, expires_str, encoded_signed_headers); // Build canonical headers size_t headers_len = 256 + strlen(host) + (content_type ? strlen(content_type) : 0); char *canonical_headers = Dowa_Arena_Allocate(p_arena, headers_len); if (content_type && strcmp(method, "PUT") == 0) { snprintf(canonical_headers, headers_len, "content-type:%s\nhost:%s\n", content_type, host); } else { snprintf(canonical_headers, headers_len, "host:%s\n", host); } // For presigned URLs, payload is UNSIGNED-PAYLOAD const char *payload_hash = "UNSIGNED-PAYLOAD"; // Build canonical request char *canonical_request = s3__build_canonical_request(method, uri_path, canonical_query, canonical_headers, signed_headers, payload_hash, p_arena); // Build string to sign char *string_to_sign = s3__build_string_to_sign(ts.datetime, ts.date, p_config->region, canonical_request, p_arena); // Calculate signing key uint8 signing_key[32]; s3__calculate_signing_key(p_config->secret_access_key, ts.date, p_config->region, signing_key); // Calculate signature uint8 signature[32]; s3__hmac_sha256(signing_key, 32, (const uint8 *)string_to_sign, strlen(string_to_sign), signature); char signature_hex[65]; s3__hex_encode(signature, 32, signature_hex); // Build final presigned URL size_t url_len = strlen("https://") + strlen(host) + strlen(uri_path) + 1 + strlen(canonical_query) + strlen("&X-Amz-Signature=") + 64 + 1; char *url = Dowa_Arena_Allocate(result.p_arena, url_len); snprintf(url, url_len, "https://%s%s?%s&X-Amz-Signature=%s", host, uri_path, canonical_query, signature_hex); result.success = TRUE; result.url = url; Dowa_Arena_Free(p_arena); return result; } S3_Presigned_URL S3_Presign_Put(const S3_Config *p_config, const char *s3_key, const char *content_type, int32 expires_seconds) { return s3__presign_url(p_config, s3_key, "PUT", content_type, expires_seconds); } S3_Presigned_URL S3_Presign_Get(const S3_Config *p_config, const char *s3_key, int32 expires_seconds) { return s3__presign_url(p_config, s3_key, "GET", NULL, expires_seconds); } void S3_Presigned_URL_Destroy(S3_Presigned_URL *p_url) { if (p_url && p_url->p_arena) { Dowa_Arena_Free(p_url->p_arena); p_url->p_arena = NULL; p_url->url = NULL; p_url->error_message = NULL; } } const char *S3_Guess_Content_Type(const char *filename) { if (!filename) return "application/octet-stream"; const char *dot = strrchr(filename, '.'); if (!dot) return "application/octet-stream"; dot++; // Skip the dot // Common content types if (strcasecmp(dot, "html") == 0 || strcasecmp(dot, "htm") == 0) return "text/html"; if (strcasecmp(dot, "css") == 0) return "text/css"; if (strcasecmp(dot, "js") == 0) return "application/javascript"; if (strcasecmp(dot, "json") == 0) return "application/json"; if (strcasecmp(dot, "xml") == 0) return "application/xml"; if (strcasecmp(dot, "txt") == 0) return "text/plain"; if (strcasecmp(dot, "csv") == 0) return "text/csv"; // Images if (strcasecmp(dot, "png") == 0) return "image/png"; if (strcasecmp(dot, "jpg") == 0 || strcasecmp(dot, "jpeg") == 0) return "image/jpeg"; if (strcasecmp(dot, "gif") == 0) return "image/gif"; if (strcasecmp(dot, "svg") == 0) return "image/svg+xml"; if (strcasecmp(dot, "webp") == 0) return "image/webp"; if (strcasecmp(dot, "ico") == 0) return "image/x-icon"; // Audio/Video if (strcasecmp(dot, "mp3") == 0) return "audio/mpeg"; if (strcasecmp(dot, "mp4") == 0) return "video/mp4"; if (strcasecmp(dot, "webm") == 0) return "video/webm"; if (strcasecmp(dot, "ogg") == 0) return "audio/ogg"; if (strcasecmp(dot, "wav") == 0) return "audio/wav"; // Documents if (strcasecmp(dot, "pdf") == 0) return "application/pdf"; if (strcasecmp(dot, "zip") == 0) return "application/zip"; if (strcasecmp(dot, "gz") == 0 || strcasecmp(dot, "gzip") == 0) return "application/gzip"; if (strcasecmp(dot, "tar") == 0) return "application/x-tar"; // Fonts if (strcasecmp(dot, "woff") == 0) return "font/woff"; if (strcasecmp(dot, "woff2") == 0) return "font/woff2"; if (strcasecmp(dot, "ttf") == 0) return "font/ttf"; if (strcasecmp(dot, "otf") == 0) return "font/otf"; return "application/octet-stream"; } // --- Internal Implementation --- // static void s3__get_timestamp(S3_Timestamp *p_ts) { time_t now = time(NULL); struct tm *utc = gmtime(&now); strftime(p_ts->date, sizeof(p_ts->date), "%Y%m%d", utc); strftime(p_ts->datetime, sizeof(p_ts->datetime), "%Y%m%dT%H%M%SZ", utc); } static void s3__hex_encode(const uint8 *data, size_t len, char *out) { static const char hex[] = "0123456789abcdef"; for (size_t i = 0; i < len; i++) { out[i * 2] = hex[(data[i] >> 4) & 0x0F]; out[i * 2 + 1] = hex[data[i] & 0x0F]; } out[len * 2] = '\0'; } static void s3__sha256_hex(const uint8 *data, size_t len, char *out) { uint8 hash[SHA256_DIGEST_LENGTH]; SHA256(data, len, hash); s3__hex_encode(hash, SHA256_DIGEST_LENGTH, out); } static void s3__hmac_sha256(const uint8 *key, size_t key_len, const uint8 *data, size_t data_len, uint8 *out) { unsigned int out_len = 0; HMAC(EVP_sha256(), key, (int)key_len, data, data_len, out, &out_len); } static char *s3__uri_encode(const char *str, Dowa_Arena *p_arena) { if (!str) return NULL; size_t len = strlen(str); // Worst case: every char becomes %XX (3 chars) size_t max_len = len * 3 + 1; char *out = Dowa_Arena_Allocate(p_arena, max_len); char *p = out; for (size_t i = 0; i < len; i++) { char c = str[i]; // Unreserved characters per RFC 3986 if ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '-' || c == '_' || c == '.' || c == '~' || c == '/') { *p++ = c; } else { sprintf(p, "%%%02X", (unsigned char)c); p += 3; } } *p = '\0'; return out; } static char *s3__build_canonical_request(const char *method, const char *uri, const char *query, const char *headers, const char *signed_headers, const char *payload_hash, Dowa_Arena *p_arena) { size_t len = strlen(method) + strlen(uri) + strlen(query) + strlen(headers) + strlen(signed_headers) + strlen(payload_hash) + 10; char *out = Dowa_Arena_Allocate(p_arena, len); snprintf(out, len, "%s\n%s\n%s\n%s\n%s\n%s", method, uri, query, headers, signed_headers, payload_hash); return out; } static char *s3__build_string_to_sign(const char *datetime, const char *date, const char *region, const char *canonical_request, Dowa_Arena *p_arena) { // Hash the canonical request char canonical_hash[65]; s3__sha256_hex((const uint8 *)canonical_request, strlen(canonical_request), canonical_hash); // Build credential scope size_t scope_len = strlen(date) + strlen(region) + strlen(S3_SERVICE_NAME) + strlen(S3_AWS4_REQUEST) + 4; char *scope = Dowa_Arena_Allocate(p_arena, scope_len); snprintf(scope, scope_len, "%s/%s/%s/%s", date, region, S3_SERVICE_NAME, S3_AWS4_REQUEST); // Build string to sign size_t len = strlen(S3_ALGORITHM) + strlen(datetime) + strlen(scope) + strlen(canonical_hash) + 10; char *out = Dowa_Arena_Allocate(p_arena, len); snprintf(out, len, "%s\n%s\n%s\n%s", S3_ALGORITHM, datetime, scope, canonical_hash); return out; } static void s3__calculate_signing_key(const char *secret_key, const char *date, const char *region, uint8 *out) { // AWS4 + SecretAccessKey size_t key_len = 4 + strlen(secret_key) + 1; char *k_secret = malloc(key_len); snprintf(k_secret, key_len, "AWS4%s", secret_key); uint8 k_date[32]; uint8 k_region[32]; uint8 k_service[32]; s3__hmac_sha256((const uint8 *)k_secret, strlen(k_secret), (const uint8 *)date, strlen(date), k_date); s3__hmac_sha256(k_date, 32, (const uint8 *)region, strlen(region), k_region); s3__hmac_sha256(k_region, 32, (const uint8 *)S3_SERVICE_NAME, strlen(S3_SERVICE_NAME), k_service); s3__hmac_sha256(k_service, 32, (const uint8 *)S3_AWS4_REQUEST, strlen(S3_AWS4_REQUEST), out); free(k_secret); } static char *s3__build_authorization_header(const char *access_key, const char *date, const char *region, const char *signed_headers, const uint8 *signing_key, const char *string_to_sign, Dowa_Arena *p_arena) { // Calculate signature uint8 signature[32]; s3__hmac_sha256(signing_key, 32, (const uint8 *)string_to_sign, strlen(string_to_sign), signature); char signature_hex[65]; s3__hex_encode(signature, 32, signature_hex); // Build credential size_t cred_len = strlen(access_key) + strlen(date) + strlen(region) + strlen(S3_SERVICE_NAME) + strlen(S3_AWS4_REQUEST) + 10; char *credential = Dowa_Arena_Allocate(p_arena, cred_len); snprintf(credential, cred_len, "%s/%s/%s/%s/%s", access_key, date, region, S3_SERVICE_NAME, S3_AWS4_REQUEST); // Build full authorization header size_t auth_len = strlen(S3_ALGORITHM) + strlen(credential) + strlen(signed_headers) + strlen(signature_hex) + 64; char *auth = Dowa_Arena_Allocate(p_arena, auth_len); snprintf(auth, auth_len, "%s Credential=%s, SignedHeaders=%s, Signature=%s", S3_ALGORITHM, credential, signed_headers, signature_hex); return auth; } static uint8 *s3__load_file(const char *path, size_t *p_size) { FILE *f = fopen(path, "rb"); if (!f) { *p_size = 0; return NULL; } fseek(f, 0, SEEK_END); long size = ftell(f); fseek(f, 0, SEEK_SET); if (size <= 0) { fclose(f); *p_size = 0; return NULL; } uint8 *data = malloc((size_t)size); if (!data) { fclose(f); *p_size = 0; return NULL; } size_t read = fread(data, 1, (size_t)size, f); fclose(f); if (read != (size_t)size) { free(data); *p_size = 0; return NULL; } *p_size = (size_t)size; return data; }