From dbb2ea9070b8923a6dbfa8d43bd5446c256966ad Mon Sep 17 00:00:00 2001 From: Yazhuo Zhang Date: Wed, 16 Oct 2024 17:45:54 +0200 Subject: [PATCH 01/19] Implemented middleware app as a hybrid function (mix compute and i/o) --- functions/CMakeLists.txt | 3 +- functions/example_app_hybrid/CMakeLists.txt | 13 ++ functions/example_app_hybrid/app_hybrid.c | 220 ++++++++++++++++++++ 3 files changed, 235 insertions(+), 1 deletion(-) create mode 100644 functions/example_app_hybrid/CMakeLists.txt create mode 100644 functions/example_app_hybrid/app_hybrid.c diff --git a/functions/CMakeLists.txt b/functions/CMakeLists.txt index d8c5259..403261a 100644 --- a/functions/CMakeLists.txt +++ b/functions/CMakeLists.txt @@ -2,7 +2,7 @@ set(CMAKE_VERSION 3.20) cmake_minimum_required(VERSION ${CMAKE_VERSION}) # option to include libc-dependent examples -option(USE_LIBC "Use libc" OFF) +option(USE_LIBC "Use libc" ON) # prevent in-source builds if(${CMAKE_SOURCE_DIR} STREQUAL ${CMAKE_BINARY_DIR}) @@ -89,4 +89,5 @@ if(USE_LIBC) add_subdirectory(example_app) add_subdirectory(files) add_subdirectory(stdio) + add_subdirectory(example_app_hybrid) endif() diff --git a/functions/example_app_hybrid/CMakeLists.txt b/functions/example_app_hybrid/CMakeLists.txt new file mode 100644 index 0000000..3e76115 --- /dev/null +++ b/functions/example_app_hybrid/CMakeLists.txt @@ -0,0 +1,13 @@ +cmake_minimum_required(VERSION ${CMAKE_VERSION}) + +set(EXECUTABLE "app_bybrid") + +add_executable(${EXECUTABLE} +app_hybrid.c +) + +target_compile_options(${EXECUTABLE} PRIVATE -static -O0) +target_link_options(${EXECUTABLE} PRIVATE -static) +target_link_libraries(${EXECUTABLE} PRIVATE + dlibc +) \ No newline at end of file diff --git a/functions/example_app_hybrid/app_hybrid.c b/functions/example_app_hybrid/app_hybrid.c new file mode 100644 index 0000000..6c76214 --- /dev/null +++ b/functions/example_app_hybrid/app_hybrid.c @@ -0,0 +1,220 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "unistd.h" + +#define BUFFER_SIZE 4096 +#define SERVER_NUMBER 10 +#define AUTH_SERVER_PORT 80 + +// Structure to store log event details +typedef struct log_node { + struct log_node *next; + char timestamp[100]; + char server_id[100]; + char type[100]; + char details[100]; +} log_node; + +char event_template[] = +"%*[^{]" +"\"details\":\"%100[^\"]," +"\"event_type\":\"%100[^\"]," +"\"server_id\":\"%100[^\"]," +"\"timestamp\":\"%100[^\"]" +"}"; + +int connect_to_server(const char *hostname, int port) { + struct sockaddr_in server_addr; + struct hostent *host = gethostbyname(hostname); + if (!host) { + perror("Failed to resolve hostname"); + return -1; + } + + int sockfd = socket(AF_INET, SOCK_STREAM, 0); + if (sockfd < 0) { + perror("Socket creation failed"); + return -1; + } + + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(port); + memcpy(&server_addr.sin_addr.s_addr, host->h_addr_list[0], host->h_length); + + if (connect(sockfd, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) { + perror("Socket connection failed"); + close(sockfd); + return -1; + } + + return sockfd; +} + + +int send_http_request(int sockfd, const char *request, char *response, size_t response_size) { + if (send(sockfd, request, strlen(request), 0) < 0) { + perror("Send failed"); + return -1; + } + + ssize_t total_received = 0, bytes; + while ((bytes = recv(sockfd, response + total_received, response_size - total_received - 1, 0)) > 0) { + total_received += bytes; + if (total_received >= response_size - 1) break; + } + response[total_received] = '\0'; + return total_received > 0 ? 0 : -1; +} + + +void parse_auth_response(const char *response, char *token) { + const char *start = strstr(response, "\"token\":"); + if (start) { + sscanf(start, "\"token\": \"%[^\"]", token); + } +} + +void parse_log_events(const char *response, log_node **log_root) { + log_node *current = NULL; + + const char *start = strstr(response, "\"events\":"); + if (start) { + start += 10; + while (1) { + current = malloc(sizeof(log_node)); + if (sscanf(start, + event_template, + current->timestamp, + current->server_id, + current->type, + current->details) != 4) { + free(current); + break; + } + + current->next = *log_root; + *log_root = current; + + start = strstr(start, "},{"); + if (!start) break; + start += 3; + } + } +} + +// Render logs to an HTML file +void render_logs_to_html(log_node *log_root) { + FILE *log_file = fopen("log.html", "w+"); + if (!log_file) { + perror("Failed to open HTML file"); + return; + } + + // write preamble + fprintf(log_file, "\n"); + fprintf(log_file, "\n"); + fprintf(log_file, "\n"); + fprintf(log_file, " \n"); + fprintf(log_file, " Logs\n"); + fprintf(log_file, "\n"); + fprintf(log_file, "\n"); + fprintf(log_file, " \n"); + fprintf(log_file, " \n"); + fprintf(log_file, " \n"); + fprintf(log_file, " \n"); + fprintf(log_file, " \n"); + fprintf(log_file, " \n"); + fprintf(log_file, " \n"); + + // write log information + for(log_node* current = log_root; current != NULL; current = current->next){ + fprintf(log_file, " \n"); + fprintf(log_file, " \n", current->timestamp); + fprintf(log_file, " \n", current->server_id); + fprintf(log_file, " \n", current->type); + fprintf(log_file, " \n", current->details); + fprintf(log_file, " \n"); + } + + // write ending + fprintf(log_file, "
TimestampServer IDEvent TypeDetails
%s%s%s%s
\n"); + fprintf(log_file, "\n"); + + fclose(log_file); +} + +// Main function to handle authorization, log fetching, and rendering +int main() { + // Get the authorization server address from the environment variable + const char *auth_server = getenv("STORAGE_HOST"); + if (!auth_server) { + fprintf(stderr, "Storage host must be provided with environment variable STORAGE_HOST\n"); + return 1; + } + + const char *auth_token = "fapw84ypf3984viuhsvpoi843ypoghvejkfld"; + + // handle + int auth_sock = connect_to_server(auth_server, AUTH_SERVER_PORT); + if (auth_sock < 0) return 1; + + char auth_request[BUFFER_SIZE]; + snprintf(auth_request, sizeof(auth_request), + "POST /authorize HTTP/1.1\r\n" + "Host: %s\r\n" + "Content-Type: application/json\r\n" + "Content-Length: %zu\r\n\r\n" + "{\"token\": \"%s\"}", auth_server, strlen(auth_token) + 12, auth_token); + + char auth_response[BUFFER_SIZE]; + if (send_http_request(auth_sock, auth_request, auth_response, BUFFER_SIZE) < 0) { + close(auth_sock); + return 1; + } + close(auth_sock); + + char token[256]; + parse_auth_response(auth_response, token); + + // fan_out + log_node *log_root = NULL; + char log_request[BUFFER_SIZE]; + char log_response[BUFFER_SIZE]; + + for (int i = 0; i < SERVER_NUMBER; ++i) { + int log_sock = connect_to_server(auth_server, AUTH_SERVER_PORT); + if (log_sock < 0) continue; + + snprintf(log_request, sizeof(log_request), + "GET /logs/%d HTTP/1.1\r\n" + "Host: %s\r\n" + "Authorization: Bearer %s\r\n\r\n", i, auth_server, token); + + if (send_http_request(log_sock, log_request, log_response, BUFFER_SIZE) == 0) { + parse_log_events(log_response, &log_root); + } + close(log_sock); + } + + // render + render_logs_to_html(log_root); + + // Free log nodes + log_node *current = log_root; + while (current) { + log_node *next = current->next; + free(current); + current = next; + } + + return 0; +} \ No newline at end of file From f855136d2f4f393a249c9d96d06a0f579f4812e8 Mon Sep 17 00:00:00 2001 From: Yazhuo Zhang Date: Wed, 16 Oct 2024 17:45:54 +0200 Subject: [PATCH 02/19] Implemented middleware app as a hybrid function (mix compute and i/o) --- functions/CMakeLists.txt | 15 ++ functions/example_app_hybrid/CMakeLists.txt | 13 ++ functions/example_app_hybrid/app_hybrid.c | 220 ++++++++++++++++++++ 3 files changed, 248 insertions(+) create mode 100644 functions/example_app_hybrid/CMakeLists.txt create mode 100644 functions/example_app_hybrid/app_hybrid.c diff --git a/functions/CMakeLists.txt b/functions/CMakeLists.txt index 2f85b72..1a3e219 100644 --- a/functions/CMakeLists.txt +++ b/functions/CMakeLists.txt @@ -6,9 +6,13 @@ project(function_examples LANGUAGES C CXX ASM) include(FetchContent) # option to include libc-dependent examples +<<<<<<< HEAD set(ARCHITECTURE ${CMAKE_SYSTEM_PROCESSOR} CACHE STRING "the architecture to build for") set(PLATFORM "debug" CACHE STRING "backend to build for") message(STATUS "Building for ${PLATFORM} on ${ARCHITECTURE}") +======= +option(USE_LIBC "Use libc" ON) +>>>>>>> dbb2ea9 (Implemented middleware app as a hybrid function (mix compute and i/o)) # prevent in-source builds if(${CMAKE_SOURCE_DIR} STREQUAL ${CMAKE_BINARY_DIR}) @@ -97,8 +101,19 @@ add_subdirectory(matmul) add_subdirectory(example_app_nolibc) # dependent on dlibc +<<<<<<< HEAD add_subdirectory(dirigent_busy) add_subdirectory(example_app) add_subdirectory(files) add_subdirectory(stdio) add_subdirectory(stdio_cxx) +======= +if(USE_LIBC) + # add_subdirectory(image_processing) + add_subdirectory(dirigent_busy) + add_subdirectory(example_app) + add_subdirectory(files) + add_subdirectory(stdio) + add_subdirectory(example_app_hybrid) +endif() +>>>>>>> dbb2ea9 (Implemented middleware app as a hybrid function (mix compute and i/o)) diff --git a/functions/example_app_hybrid/CMakeLists.txt b/functions/example_app_hybrid/CMakeLists.txt new file mode 100644 index 0000000..3e76115 --- /dev/null +++ b/functions/example_app_hybrid/CMakeLists.txt @@ -0,0 +1,13 @@ +cmake_minimum_required(VERSION ${CMAKE_VERSION}) + +set(EXECUTABLE "app_bybrid") + +add_executable(${EXECUTABLE} +app_hybrid.c +) + +target_compile_options(${EXECUTABLE} PRIVATE -static -O0) +target_link_options(${EXECUTABLE} PRIVATE -static) +target_link_libraries(${EXECUTABLE} PRIVATE + dlibc +) \ No newline at end of file diff --git a/functions/example_app_hybrid/app_hybrid.c b/functions/example_app_hybrid/app_hybrid.c new file mode 100644 index 0000000..6c76214 --- /dev/null +++ b/functions/example_app_hybrid/app_hybrid.c @@ -0,0 +1,220 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "unistd.h" + +#define BUFFER_SIZE 4096 +#define SERVER_NUMBER 10 +#define AUTH_SERVER_PORT 80 + +// Structure to store log event details +typedef struct log_node { + struct log_node *next; + char timestamp[100]; + char server_id[100]; + char type[100]; + char details[100]; +} log_node; + +char event_template[] = +"%*[^{]" +"\"details\":\"%100[^\"]," +"\"event_type\":\"%100[^\"]," +"\"server_id\":\"%100[^\"]," +"\"timestamp\":\"%100[^\"]" +"}"; + +int connect_to_server(const char *hostname, int port) { + struct sockaddr_in server_addr; + struct hostent *host = gethostbyname(hostname); + if (!host) { + perror("Failed to resolve hostname"); + return -1; + } + + int sockfd = socket(AF_INET, SOCK_STREAM, 0); + if (sockfd < 0) { + perror("Socket creation failed"); + return -1; + } + + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(port); + memcpy(&server_addr.sin_addr.s_addr, host->h_addr_list[0], host->h_length); + + if (connect(sockfd, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) { + perror("Socket connection failed"); + close(sockfd); + return -1; + } + + return sockfd; +} + + +int send_http_request(int sockfd, const char *request, char *response, size_t response_size) { + if (send(sockfd, request, strlen(request), 0) < 0) { + perror("Send failed"); + return -1; + } + + ssize_t total_received = 0, bytes; + while ((bytes = recv(sockfd, response + total_received, response_size - total_received - 1, 0)) > 0) { + total_received += bytes; + if (total_received >= response_size - 1) break; + } + response[total_received] = '\0'; + return total_received > 0 ? 0 : -1; +} + + +void parse_auth_response(const char *response, char *token) { + const char *start = strstr(response, "\"token\":"); + if (start) { + sscanf(start, "\"token\": \"%[^\"]", token); + } +} + +void parse_log_events(const char *response, log_node **log_root) { + log_node *current = NULL; + + const char *start = strstr(response, "\"events\":"); + if (start) { + start += 10; + while (1) { + current = malloc(sizeof(log_node)); + if (sscanf(start, + event_template, + current->timestamp, + current->server_id, + current->type, + current->details) != 4) { + free(current); + break; + } + + current->next = *log_root; + *log_root = current; + + start = strstr(start, "},{"); + if (!start) break; + start += 3; + } + } +} + +// Render logs to an HTML file +void render_logs_to_html(log_node *log_root) { + FILE *log_file = fopen("log.html", "w+"); + if (!log_file) { + perror("Failed to open HTML file"); + return; + } + + // write preamble + fprintf(log_file, "\n"); + fprintf(log_file, "\n"); + fprintf(log_file, "\n"); + fprintf(log_file, " \n"); + fprintf(log_file, " Logs\n"); + fprintf(log_file, "\n"); + fprintf(log_file, "\n"); + fprintf(log_file, " \n"); + fprintf(log_file, " \n"); + fprintf(log_file, " \n"); + fprintf(log_file, " \n"); + fprintf(log_file, " \n"); + fprintf(log_file, " \n"); + fprintf(log_file, " \n"); + + // write log information + for(log_node* current = log_root; current != NULL; current = current->next){ + fprintf(log_file, " \n"); + fprintf(log_file, " \n", current->timestamp); + fprintf(log_file, " \n", current->server_id); + fprintf(log_file, " \n", current->type); + fprintf(log_file, " \n", current->details); + fprintf(log_file, " \n"); + } + + // write ending + fprintf(log_file, "
TimestampServer IDEvent TypeDetails
%s%s%s%s
\n"); + fprintf(log_file, "\n"); + + fclose(log_file); +} + +// Main function to handle authorization, log fetching, and rendering +int main() { + // Get the authorization server address from the environment variable + const char *auth_server = getenv("STORAGE_HOST"); + if (!auth_server) { + fprintf(stderr, "Storage host must be provided with environment variable STORAGE_HOST\n"); + return 1; + } + + const char *auth_token = "fapw84ypf3984viuhsvpoi843ypoghvejkfld"; + + // handle + int auth_sock = connect_to_server(auth_server, AUTH_SERVER_PORT); + if (auth_sock < 0) return 1; + + char auth_request[BUFFER_SIZE]; + snprintf(auth_request, sizeof(auth_request), + "POST /authorize HTTP/1.1\r\n" + "Host: %s\r\n" + "Content-Type: application/json\r\n" + "Content-Length: %zu\r\n\r\n" + "{\"token\": \"%s\"}", auth_server, strlen(auth_token) + 12, auth_token); + + char auth_response[BUFFER_SIZE]; + if (send_http_request(auth_sock, auth_request, auth_response, BUFFER_SIZE) < 0) { + close(auth_sock); + return 1; + } + close(auth_sock); + + char token[256]; + parse_auth_response(auth_response, token); + + // fan_out + log_node *log_root = NULL; + char log_request[BUFFER_SIZE]; + char log_response[BUFFER_SIZE]; + + for (int i = 0; i < SERVER_NUMBER; ++i) { + int log_sock = connect_to_server(auth_server, AUTH_SERVER_PORT); + if (log_sock < 0) continue; + + snprintf(log_request, sizeof(log_request), + "GET /logs/%d HTTP/1.1\r\n" + "Host: %s\r\n" + "Authorization: Bearer %s\r\n\r\n", i, auth_server, token); + + if (send_http_request(log_sock, log_request, log_response, BUFFER_SIZE) == 0) { + parse_log_events(log_response, &log_root); + } + close(log_sock); + } + + // render + render_logs_to_html(log_root); + + // Free log nodes + log_node *current = log_root; + while (current) { + log_node *next = current->next; + free(current); + current = next; + } + + return 0; +} \ No newline at end of file From 63d7d6cde5c1981536c100aa1043221663372228 Mon Sep 17 00:00:00 2001 From: Tom Kuchler Date: Fri, 18 Oct 2024 14:31:31 +0200 Subject: [PATCH 03/19] Add syscalls --- functions/CMakeLists.txt | 10 --- functions/example_app_hybrid/CMakeLists.txt | 4 + functions/example_app_hybrid/app_hybrid.c | 16 ++-- .../linux_syscalls/CMakeLists.txt | 10 +++ .../arch/aarch64/syscall_arch.h | 79 +++++++++++++++++ .../linux_syscalls/arch/x86_64/syscall_arch.h | 87 +++++++++++++++++++ .../linux_syscalls/syscall.h | 78 +++++++++++++++++ 7 files changed, 265 insertions(+), 19 deletions(-) create mode 100644 functions/example_app_hybrid/linux_syscalls/CMakeLists.txt create mode 100644 functions/example_app_hybrid/linux_syscalls/arch/aarch64/syscall_arch.h create mode 100644 functions/example_app_hybrid/linux_syscalls/arch/x86_64/syscall_arch.h create mode 100644 functions/example_app_hybrid/linux_syscalls/syscall.h diff --git a/functions/CMakeLists.txt b/functions/CMakeLists.txt index 465e258..f155007 100644 --- a/functions/CMakeLists.txt +++ b/functions/CMakeLists.txt @@ -6,17 +6,9 @@ project(function_examples LANGUAGES C CXX ASM) include(FetchContent) # option to include libc-dependent examples -<<<<<<< HEAD -<<<<<<< HEAD set(ARCHITECTURE ${CMAKE_SYSTEM_PROCESSOR} CACHE STRING "the architecture to build for") set(PLATFORM "debug" CACHE STRING "backend to build for") message(STATUS "Building for ${PLATFORM} on ${ARCHITECTURE}") -======= -option(USE_LIBC "Use libc" ON) ->>>>>>> dbb2ea9 (Implemented middleware app as a hybrid function (mix compute and i/o)) -======= -option(USE_LIBC "Use libc" ON) ->>>>>>> dbb2ea9070b8923a6dbfa8d43bd5446c256966ad # prevent in-source builds if(${CMAKE_SOURCE_DIR} STREQUAL ${CMAKE_BINARY_DIR}) @@ -42,8 +34,6 @@ execute_process( COMMAND_ERROR_IS_FATAL ANY ) -message(WARNING "${COMPILER_RUNTIME_INCLUDE}") - add_library(dandelion_runtime INTERFACE IMPORTED) target_include_directories(dandelion_runtime INTERFACE "${CMAKE_CURRENT_BINARY_DIR}/dandelion_sdk/include" diff --git a/functions/example_app_hybrid/CMakeLists.txt b/functions/example_app_hybrid/CMakeLists.txt index 3e76115..bec4f34 100644 --- a/functions/example_app_hybrid/CMakeLists.txt +++ b/functions/example_app_hybrid/CMakeLists.txt @@ -1,5 +1,7 @@ cmake_minimum_required(VERSION ${CMAKE_VERSION}) +add_subdirectory(linux_syscalls) + set(EXECUTABLE "app_bybrid") add_executable(${EXECUTABLE} @@ -9,5 +11,7 @@ app_hybrid.c target_compile_options(${EXECUTABLE} PRIVATE -static -O0) target_link_options(${EXECUTABLE} PRIVATE -static) target_link_libraries(${EXECUTABLE} PRIVATE + linux_syscalls dlibc + dandelion_runtime ) \ No newline at end of file diff --git a/functions/example_app_hybrid/app_hybrid.c b/functions/example_app_hybrid/app_hybrid.c index 6c76214..440bc92 100644 --- a/functions/example_app_hybrid/app_hybrid.c +++ b/functions/example_app_hybrid/app_hybrid.c @@ -5,12 +5,10 @@ #include #include #include -#include -#include -#include - #include "unistd.h" +#include "syscall.h" + #define BUFFER_SIZE 4096 #define SERVER_NUMBER 10 #define AUTH_SERVER_PORT 80 @@ -40,17 +38,17 @@ int connect_to_server(const char *hostname, int port) { return -1; } - int sockfd = socket(AF_INET, SOCK_STREAM, 0); + int sockfd = linux_socket(LINUX_AF_INET, LINUX_SOCK_STREAM, 0); if (sockfd < 0) { perror("Socket creation failed"); return -1; } - server_addr.sin_family = AF_INET; + server_addr.sin_family = LINUX_AF_INET; server_addr.sin_port = htons(port); memcpy(&server_addr.sin_addr.s_addr, host->h_addr_list[0], host->h_length); - if (connect(sockfd, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) { + if (linux_connect(sockfd, &server_addr, sizeof(server_addr)) < 0) { perror("Socket connection failed"); close(sockfd); return -1; @@ -61,13 +59,13 @@ int connect_to_server(const char *hostname, int port) { int send_http_request(int sockfd, const char *request, char *response, size_t response_size) { - if (send(sockfd, request, strlen(request), 0) < 0) { + if (linux_send(sockfd, request, strlen(request), 0) < 0) { perror("Send failed"); return -1; } ssize_t total_received = 0, bytes; - while ((bytes = recv(sockfd, response + total_received, response_size - total_received - 1, 0)) > 0) { + while ((bytes = linux_recv(sockfd, response + total_received, response_size - total_received - 1, 0)) > 0) { total_received += bytes; if (total_received >= response_size - 1) break; } diff --git a/functions/example_app_hybrid/linux_syscalls/CMakeLists.txt b/functions/example_app_hybrid/linux_syscalls/CMakeLists.txt new file mode 100644 index 0000000..d65c5d7 --- /dev/null +++ b/functions/example_app_hybrid/linux_syscalls/CMakeLists.txt @@ -0,0 +1,10 @@ +add_library(linux_syscalls INTERFACE) + +target_include_directories(linux_syscalls INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}) +if(ARCHITECTURE MATCHES "aarch64|arm64|ARM64") + target_include_directories(linux_syscalls INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/arch/aarch64) +elseif(ARCHITECTURE MATCHES "x86_64|amd64|AMD64") + target_include_directories(linux_syscalls INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/arch/x86_64) +else() + message(FATAL_ERROR "Unsupported architecture: ${CMAKE_SYSTEM_PROCESSOR}") +endif() diff --git a/functions/example_app_hybrid/linux_syscalls/arch/aarch64/syscall_arch.h b/functions/example_app_hybrid/linux_syscalls/arch/aarch64/syscall_arch.h new file mode 100644 index 0000000..b621c98 --- /dev/null +++ b/functions/example_app_hybrid/linux_syscalls/arch/aarch64/syscall_arch.h @@ -0,0 +1,79 @@ +#define __SYSCALL_LL_E(x) (x) +#define __SYSCALL_LL_O(x) (x) + +#define SYS_openat 56 +#define SYS_close 57 +#define SYS_getdents64 61 +#define SYS_lseek 62 +#define SYS_read 63 +#define SYS_write 64 +#define SYS_exit_group 94 +#define SYS_socket 198 +#define SYS_connect 203 +#define SYS_sendto 206 +#define SYS_recvfrom 207 +#define SYS_mmap 222 + +#define __asm_syscall(...) \ + do { \ + __asm__ __volatile__("svc 0" : "=r"(x0) : __VA_ARGS__ : "memory", "cc"); \ + return x0; \ + } while (0) + +static inline long __syscall0(long n) { + register long x8 __asm__("x8") = n; + register long x0 __asm__("x0"); + __asm_syscall("r"(x8)); +} + +static inline long __syscall1(long n, long a) { + register long x8 __asm__("x8") = n; + register long x0 __asm__("x0") = a; + __asm_syscall("r"(x8), "0"(x0)); +} + +static inline long __syscall2(long n, long a, long b) { + register long x8 __asm__("x8") = n; + register long x0 __asm__("x0") = a; + register long x1 __asm__("x1") = b; + __asm_syscall("r"(x8), "0"(x0), "r"(x1)); +} + +static inline long __syscall3(long n, long a, long b, long c) { + register long x8 __asm__("x8") = n; + register long x0 __asm__("x0") = a; + register long x1 __asm__("x1") = b; + register long x2 __asm__("x2") = c; + __asm_syscall("r"(x8), "0"(x0), "r"(x1), "r"(x2)); +} + +static inline long __syscall4(long n, long a, long b, long c, long d) { + register long x8 __asm__("x8") = n; + register long x0 __asm__("x0") = a; + register long x1 __asm__("x1") = b; + register long x2 __asm__("x2") = c; + register long x3 __asm__("x3") = d; + __asm_syscall("r"(x8), "0"(x0), "r"(x1), "r"(x2), "r"(x3)); +} + +static inline long __syscall5(long n, long a, long b, long c, long d, long e) { + register long x8 __asm__("x8") = n; + register long x0 __asm__("x0") = a; + register long x1 __asm__("x1") = b; + register long x2 __asm__("x2") = c; + register long x3 __asm__("x3") = d; + register long x4 __asm__("x4") = e; + __asm_syscall("r"(x8), "0"(x0), "r"(x1), "r"(x2), "r"(x3), "r"(x4)); +} + +static inline long __syscall6(long n, long a, long b, long c, long d, long e, + long f) { + register long x8 __asm__("x8") = n; + register long x0 __asm__("x0") = a; + register long x1 __asm__("x1") = b; + register long x2 __asm__("x2") = c; + register long x3 __asm__("x3") = d; + register long x4 __asm__("x4") = e; + register long x5 __asm__("x5") = f; + __asm_syscall("r"(x8), "0"(x0), "r"(x1), "r"(x2), "r"(x3), "r"(x4), "r"(x5)); +} diff --git a/functions/example_app_hybrid/linux_syscalls/arch/x86_64/syscall_arch.h b/functions/example_app_hybrid/linux_syscalls/arch/x86_64/syscall_arch.h new file mode 100644 index 0000000..d7f5687 --- /dev/null +++ b/functions/example_app_hybrid/linux_syscalls/arch/x86_64/syscall_arch.h @@ -0,0 +1,87 @@ +#define __SYSCALL_LL_E(x) (x) +#define __SYSCALL_LL_O(x) (x) + +#define SYS_read 0 +#define SYS_write 1 +#define SYS_close 3 +#define SYS_lseek 8 +#define SYS_mmap 9 +#define SYS_socket 41 +#define SYS_connect 42 +#define SYS_sendto 44 +#define SYS_recvfrom 45 +#define SYS_arch_prctl 158 +#define SYS_exit_group 231 +#define SYS_openat 257 +#define SYS_getdents64 217 + +#define ARCH_SET_FS 0x1002 + +static __inline long __syscall0(long n) { + unsigned long ret; + __asm__ __volatile__("syscall" : "=a"(ret) : "a"(n) : "rcx", "r11", "memory"); + return ret; +} + +static __inline long __syscall1(long n, long a1) { + unsigned long ret; + __asm__ __volatile__("syscall" + : "=a"(ret) + : "a"(n), "D"(a1) + : "rcx", "r11", "memory"); + return ret; +} + +static __inline long __syscall2(long n, long a1, long a2) { + unsigned long ret; + __asm__ __volatile__("syscall" + : "=a"(ret) + : "a"(n), "D"(a1), "S"(a2) + : "rcx", "r11", "memory"); + return ret; +} + +static __inline long __syscall3(long n, long a1, long a2, long a3) { + unsigned long ret; + __asm__ __volatile__("syscall" + : "=a"(ret) + : "a"(n), "D"(a1), "S"(a2), "d"(a3) + : "rcx", "r11", "memory"); + return ret; +} + +static __inline long __syscall4(long n, long a1, long a2, long a3, long a4) { + unsigned long ret; + register long r10 __asm__("r10") = a4; + __asm__ __volatile__("syscall" + : "=a"(ret) + : "a"(n), "D"(a1), "S"(a2), "d"(a3), "r"(r10) + : "rcx", "r11", "memory"); + return ret; +} + +static __inline long __syscall5(long n, long a1, long a2, long a3, long a4, + long a5) { + unsigned long ret; + register long r10 __asm__("r10") = a4; + register long r8 __asm__("r8") = a5; + __asm__ __volatile__("syscall" + : "=a"(ret) + : "a"(n), "D"(a1), "S"(a2), "d"(a3), "r"(r10), "r"(r8) + : "rcx", "r11", "memory"); + return ret; +} + +static __inline long __syscall6(long n, long a1, long a2, long a3, long a4, + long a5, long a6) { + unsigned long ret; + register long r10 __asm__("r10") = a4; + register long r8 __asm__("r8") = a5; + register long r9 __asm__("r9") = a6; + __asm__ __volatile__("syscall" + : "=a"(ret) + : "a"(n), "D"(a1), "S"(a2), "d"(a3), "r"(r10), "r"(r8), + "r"(r9) + : "rcx", "r11", "memory"); + return ret; +} diff --git a/functions/example_app_hybrid/linux_syscalls/syscall.h b/functions/example_app_hybrid/linux_syscalls/syscall.h new file mode 100644 index 0000000..1c33bfa --- /dev/null +++ b/functions/example_app_hybrid/linux_syscalls/syscall.h @@ -0,0 +1,78 @@ +#ifndef _LINUX_SYSCALL_H +#define _LINUX_SYSCALL_H + +#include "syscall_arch.h" + +#define LINUX_PROT_READ 0x1 +#define LINUX_PROT_WRITE 0x2 +#define LINUX_PROT_EXEC 0x4 + +#define LINUX_MAP_ANONYMOUS 0x20 +#define LINUX_MAP_PRIVATE 0x2 + +#define LINUX_O_RDONLY 00 +#define LINUX_AT_FDCWD -100 + +// networking defines +#define LINUX_AF_INET 2 +#define LINUX_SOCK_STREAM 1 + +#ifndef __scc +#define __scc(X) ((long)(X)) +typedef long syscall_arg_t; +#endif + +#define __syscall1(n, a) __syscall1(n, __scc(a)) +#define __syscall2(n, a, b) __syscall2(n, __scc(a), __scc(b)) +#define __syscall3(n, a, b, c) __syscall3(n, __scc(a), __scc(b), __scc(c)) +#define __syscall4(n, a, b, c, d) \ + __syscall4(n, __scc(a), __scc(b), __scc(c), __scc(d)) +#define __syscall5(n, a, b, c, d, e) \ + __syscall5(n, __scc(a), __scc(b), __scc(c), __scc(d), __scc(e)) +#define __syscall6(n, a, b, c, d, e, f) \ + __syscall6(n, __scc(a), __scc(b), __scc(c), __scc(d), __scc(e), __scc(f)) +#define __syscall7(n, a, b, c, d, e, f, g) \ + __syscall7(n, __scc(a), __scc(b), __scc(c), __scc(d), __scc(e), __scc(f), \ + __scc(g)) + +#define __SYSCALL_NARGS_X(a, b, c, d, e, f, g, h, n, ...) n +#define __SYSCALL_NARGS(...) \ + __SYSCALL_NARGS_X(__VA_ARGS__, 7, 6, 5, 4, 3, 2, 1, 0, ) +#define __SYSCALL_CONCAT_X(a, b) a##b +#define __SYSCALL_CONCAT(a, b) __SYSCALL_CONCAT_X(a, b) +#define __SYSCALL_DISP(b, ...) \ + __SYSCALL_CONCAT(b, __SYSCALL_NARGS(__VA_ARGS__))(__VA_ARGS__) + +#define __syscall(...) __SYSCALL_DISP(__syscall, __VA_ARGS__) + +static inline int linux_socket(int domain, int type, int protocol) { + return __syscall(SYS_socket, domain, type, protocol); +} + +struct in_addr { + uint32_t s_addr; +}; + +struct sockaddr_in { + unsigned short sin_family; + uint16_t sin_port; + struct in_addr sin_addr; +}; + +static inline int linux_connect(int socketfd, const struct sockaddr_in* addr, int addrlen){ + return __syscall(SYS_connect, socketfd, addr, addrlen); +} + +static inline ptrdiff_t linux_send(int socketfd, const void* buf, size_t len, int flags){ + return __syscall(SYS_sendto, socketfd, buf, len, flags, NULL, 0); +} + +static inline ptrdiff_t linux_recv(int socketfd, void* buf, size_t len, int flags){ + return __syscall(SYS_recvfrom, socketfd, buf, len, flags, NULL, NULL); +} + +static inline int linux_close(int fd){ + return __syscall(SYS_close, fd); +} + +#endif From d93266638b1b9021553670611039360e6eac07ad Mon Sep 17 00:00:00 2001 From: Yazhuo Zhang Date: Fri, 18 Oct 2024 17:01:42 +0200 Subject: [PATCH 04/19] Add custom functions to replace htons and gethostbyname --- functions/example_app_hybrid/CMakeLists.txt | 2 +- functions/example_app_hybrid/app_hybrid.c | 42 +++++++++++---------- 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/functions/example_app_hybrid/CMakeLists.txt b/functions/example_app_hybrid/CMakeLists.txt index bec4f34..017bdd8 100644 --- a/functions/example_app_hybrid/CMakeLists.txt +++ b/functions/example_app_hybrid/CMakeLists.txt @@ -8,7 +8,7 @@ add_executable(${EXECUTABLE} app_hybrid.c ) -target_compile_options(${EXECUTABLE} PRIVATE -static -O0) +target_compile_options(${EXECUTABLE} PRIVATE -static -O3) target_link_options(${EXECUTABLE} PRIVATE -static) target_link_libraries(${EXECUTABLE} PRIVATE linux_syscalls diff --git a/functions/example_app_hybrid/app_hybrid.c b/functions/example_app_hybrid/app_hybrid.c index 440bc92..e2b034c 100644 --- a/functions/example_app_hybrid/app_hybrid.c +++ b/functions/example_app_hybrid/app_hybrid.c @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include "unistd.h" @@ -30,13 +31,20 @@ char event_template[] = "\"timestamp\":\"%100[^\"]" "}"; -int connect_to_server(const char *hostname, int port) { + +unsigned short my_htons(unsigned short port) { + return (port >> 8) | (port << 8); +} + +int my_inet_pton(const char *src, uint32_t *dst) { + unsigned int b1, b2, b3, b4; + if (sscanf(src, "%u.%u.%u.%u", &b1, &b2, &b3, &b4) != 4) return 0; + *dst = (b1 << 24) | (b2 << 16) | (b3 << 8) | b4; + return 1; +} + +int connect_to_server(const char *ip, int port) { struct sockaddr_in server_addr; - struct hostent *host = gethostbyname(hostname); - if (!host) { - perror("Failed to resolve hostname"); - return -1; - } int sockfd = linux_socket(LINUX_AF_INET, LINUX_SOCK_STREAM, 0); if (sockfd < 0) { @@ -45,8 +53,9 @@ int connect_to_server(const char *hostname, int port) { } server_addr.sin_family = LINUX_AF_INET; - server_addr.sin_port = htons(port); - memcpy(&server_addr.sin_addr.s_addr, host->h_addr_list[0], host->h_length); + server_addr.sin_port = my_htons(port); + + my_inet_pton(ip, &server_addr.sin_addr.s_addr); if (linux_connect(sockfd, &server_addr, sizeof(server_addr)) < 0) { perror("Socket connection failed"); @@ -152,17 +161,12 @@ void render_logs_to_html(log_node *log_root) { // Main function to handle authorization, log fetching, and rendering int main() { - // Get the authorization server address from the environment variable - const char *auth_server = getenv("STORAGE_HOST"); - if (!auth_server) { - fprintf(stderr, "Storage host must be provided with environment variable STORAGE_HOST\n"); - return 1; - } - + + const char *auth_server_ip = "10.233.0.12"; const char *auth_token = "fapw84ypf3984viuhsvpoi843ypoghvejkfld"; // handle - int auth_sock = connect_to_server(auth_server, AUTH_SERVER_PORT); + int auth_sock = connect_to_server(auth_server_ip, AUTH_SERVER_PORT); if (auth_sock < 0) return 1; char auth_request[BUFFER_SIZE]; @@ -171,7 +175,7 @@ int main() { "Host: %s\r\n" "Content-Type: application/json\r\n" "Content-Length: %zu\r\n\r\n" - "{\"token\": \"%s\"}", auth_server, strlen(auth_token) + 12, auth_token); + "{\"token\": \"%s\"}", auth_server_ip, strlen(auth_token) + 12, auth_token); char auth_response[BUFFER_SIZE]; if (send_http_request(auth_sock, auth_request, auth_response, BUFFER_SIZE) < 0) { @@ -189,13 +193,13 @@ int main() { char log_response[BUFFER_SIZE]; for (int i = 0; i < SERVER_NUMBER; ++i) { - int log_sock = connect_to_server(auth_server, AUTH_SERVER_PORT); + int log_sock = connect_to_server(auth_server_ip, AUTH_SERVER_PORT); if (log_sock < 0) continue; snprintf(log_request, sizeof(log_request), "GET /logs/%d HTTP/1.1\r\n" "Host: %s\r\n" - "Authorization: Bearer %s\r\n\r\n", i, auth_server, token); + "Authorization: Bearer %s\r\n\r\n", i, auth_server_ip, token); if (send_http_request(log_sock, log_request, log_response, BUFFER_SIZE) == 0) { parse_log_events(log_response, &log_root); From 9a3e7c090ea7b78bfdb5fa01ec6e6fe9cee5db5d Mon Sep 17 00:00:00 2001 From: Tom Kuchler Date: Mon, 21 Oct 2024 11:21:11 +0200 Subject: [PATCH 05/19] Fix mmu build --- functions/CMakeLists.txt | 7 ++++++- functions/README.md | 12 +++--------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/functions/CMakeLists.txt b/functions/CMakeLists.txt index f155007..39a06be 100644 --- a/functions/CMakeLists.txt +++ b/functions/CMakeLists.txt @@ -8,7 +8,12 @@ include(FetchContent) # option to include libc-dependent examples set(ARCHITECTURE ${CMAKE_SYSTEM_PROCESSOR} CACHE STRING "the architecture to build for") set(PLATFORM "debug" CACHE STRING "backend to build for") -message(STATUS "Building for ${PLATFORM} on ${ARCHITECTURE}") + +if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE "Debug") +endif() + +message(STATUS "Building ${CMAKE_BUILD_TYPE} for ${PLATFORM} on ${ARCHITECTURE}") # prevent in-source builds if(${CMAKE_SOURCE_DIR} STREQUAL ${CMAKE_BINARY_DIR}) diff --git a/functions/README.md b/functions/README.md index 2d5db2f..ed46ccf 100644 --- a/functions/README.md +++ b/functions/README.md @@ -6,17 +6,11 @@ Some functions depend on the function interface directly while others use it through mlibc. # Building the C functions -The cmake build process will build the function interface, libc and the functions. -To do so cmake needs to know that it needs to use the cross compile file dandelion. -Additionally, cmake and needs to know the platform to build for (cheri, mmu_freebsd, mmu_linux). +The cmake build process will build the functions based on the latest version of the dandelion SDK. +For this cmake meeds to know the platform to build for (cheri, mmu_freebsd, mmu_linux, debug) and architecture. From the top folder functions can be built with: ``` mkdir build cd build -cmake -DCMAKE_TOOLCHAIN_FILE=../functions/dandelion.cmake -DDANDELION_PLATFORM= -DTARGET_CPU= ../functions +cmake -DPLATFORM= -DARCHITECTURE= ../functions ``` -The mlibc build currently only seems to work with clang. -To set this up set CC=clang and CXX=clang++ additionally set LDFLAGS=-fuse-ld=lld - -## Building dlibc -To turn on building of libc enable the cmake option with `-DUSE_LIBC=on` \ No newline at end of file From eabe1861530d0be718a481018b9bac1bbc2f00d2 Mon Sep 17 00:00:00 2001 From: Yazhuo Zhang Date: Mon, 21 Oct 2024 13:29:27 +0200 Subject: [PATCH 06/19] Updated my_inet_pton and my_htons (not working at linux_connect) --- functions/example_app_hybrid/app_hybrid.c | 55 +++++++++++++++++------ 1 file changed, 42 insertions(+), 13 deletions(-) diff --git a/functions/example_app_hybrid/app_hybrid.c b/functions/example_app_hybrid/app_hybrid.c index e2b034c..277fada 100644 --- a/functions/example_app_hybrid/app_hybrid.c +++ b/functions/example_app_hybrid/app_hybrid.c @@ -12,7 +12,7 @@ #define BUFFER_SIZE 4096 #define SERVER_NUMBER 10 -#define AUTH_SERVER_PORT 80 +const uint16_t AUTH_SERVER_PORT = 8080; // Structure to store log event details typedef struct log_node { @@ -32,18 +32,36 @@ char event_template[] = "}"; -unsigned short my_htons(unsigned short port) { - return (port >> 8) | (port << 8); +uint16_t my_htons(uint16_t port) { + return ((port >> 8) & 0x00FF) | ((port << 8) & 0xFF00); } -int my_inet_pton(const char *src, uint32_t *dst) { - unsigned int b1, b2, b3, b4; - if (sscanf(src, "%u.%u.%u.%u", &b1, &b2, &b3, &b4) != 4) return 0; - *dst = (b1 << 24) | (b2 << 16) | (b3 << 8) | b4; - return 1; +int my_inet_pton(int af, const char *src, void *dst) { + if (af == LINUX_AF_INET) { + uint8_t array[4] = {}; + for (int i = 0; i < 4; i++) { + char *end; + long value = strtol(src, &end, 10); + if (value < 0 || value > 255) { + return 0; + } + + if (*end != '\0' && *end != '.' && i < 3) { + return 0; + } + src = end + 1; + array[i] = (uint8_t)value; + } + + struct in_addr *addr = (struct in_addr *)dst; + memcpy(&addr->s_addr, array, 4); // Copy the 4 bytes into s_addr + + return 1; // Success + } + return -1; // Unsupported address family } -int connect_to_server(const char *ip, int port) { +int connect_to_server(const char *ip, uint16_t port) { struct sockaddr_in server_addr; int sockfd = linux_socket(LINUX_AF_INET, LINUX_SOCK_STREAM, 0); @@ -55,14 +73,25 @@ int connect_to_server(const char *ip, int port) { server_addr.sin_family = LINUX_AF_INET; server_addr.sin_port = my_htons(port); - my_inet_pton(ip, &server_addr.sin_addr.s_addr); + if (my_inet_pton(LINUX_AF_INET, ip, &server_addr.sin_addr) != 1) { + perror("Invalid IP address"); + close(sockfd); + return -1; + } - if (linux_connect(sockfd, &server_addr, sizeof(server_addr)) < 0) { + printf("s_addr: 0x%08X\n", server_addr.sin_addr.s_addr); + + int err = linux_connect(sockfd, (struct sockaddr *)&server_addr, sizeof(server_addr)); + if (err < 0) { + printf("Error: %d\n", err); + printf("size of server_addr: %ld\n", sizeof(server_addr)); perror("Socket connection failed"); close(sockfd); return -1; } + printf("Socket connected\n"); + return sockfd; } @@ -161,8 +190,7 @@ void render_logs_to_html(log_node *log_root) { // Main function to handle authorization, log fetching, and rendering int main() { - - const char *auth_server_ip = "10.233.0.12"; + const char *auth_server_ip = "127.0.0.1"; const char *auth_token = "fapw84ypf3984viuhsvpoi843ypoghvejkfld"; // handle @@ -182,6 +210,7 @@ int main() { close(auth_sock); return 1; } + close(auth_sock); char token[256]; From aee917bb6603ac8616e980156d1f259fe70869ad Mon Sep 17 00:00:00 2001 From: Tom Kuchler Date: Mon, 21 Oct 2024 18:47:45 +0200 Subject: [PATCH 07/19] Add sorting and fix socket connect --- functions/example_app_hybrid/app_hybrid.c | 122 ++++++++++++++---- .../linux_syscalls/syscall.h | 9 ++ 2 files changed, 105 insertions(+), 26 deletions(-) diff --git a/functions/example_app_hybrid/app_hybrid.c b/functions/example_app_hybrid/app_hybrid.c index 277fada..75bfaab 100644 --- a/functions/example_app_hybrid/app_hybrid.c +++ b/functions/example_app_hybrid/app_hybrid.c @@ -12,7 +12,7 @@ #define BUFFER_SIZE 4096 #define SERVER_NUMBER 10 -const uint16_t AUTH_SERVER_PORT = 8080; +const uint16_t AUTH_SERVER_PORT = 8000; // Structure to store log event details typedef struct log_node { @@ -24,12 +24,12 @@ typedef struct log_node { } log_node; char event_template[] = -"%*[^{]" -"\"details\":\"%100[^\"]," -"\"event_type\":\"%100[^\"]," -"\"server_id\":\"%100[^\"]," -"\"timestamp\":\"%100[^\"]" -"}"; +"%*[^{]{" +"\"details\":\"%100[^\"]\"," +"\"event_type\":\"%100[^\"]\"," +"\"server_id\":\"%100[^\"]\"," +"\"timestamp\":\"%100[^\"]\"" +"%}"; uint16_t my_htons(uint16_t port) { @@ -79,19 +79,15 @@ int connect_to_server(const char *ip, uint16_t port) { return -1; } - printf("s_addr: 0x%08X\n", server_addr.sin_addr.s_addr); - - int err = linux_connect(sockfd, (struct sockaddr *)&server_addr, sizeof(server_addr)); + int err = linux_connect(sockfd, &server_addr, sizeof(server_addr)); if (err < 0) { - printf("Error: %d\n", err); + printf("Error: %s\n", strerror(-err)); printf("size of server_addr: %ld\n", sizeof(server_addr)); perror("Socket connection failed"); close(sockfd); return -1; } - printf("Socket connected\n"); - return sockfd; } @@ -103,12 +99,17 @@ int send_http_request(int sockfd, const char *request, char *response, size_t re } ssize_t total_received = 0, bytes; - while ((bytes = linux_recv(sockfd, response + total_received, response_size - total_received - 1, 0)) > 0) { - total_received += bytes; - if (total_received >= response_size - 1) break; + // while ((bytes = linux_recv(sockfd, response + total_received, response_size - total_received - 1, 0)) > 0) { + // total_received += bytes; + // if (total_received >= response_size - 1) break; + // } + // blocks and waits forever if there is no next packate, could validate HTTP sice on the packages to see if we need more, this works for now + total_received = linux_recv(sockfd, response + total_received, response_size - total_received - 1, 0); + if(total_received < 0){ + return -1; } response[total_received] = '\0'; - return total_received > 0 ? 0 : -1; + return 0; } @@ -124,15 +125,16 @@ void parse_log_events(const char *response, log_node **log_root) { const char *start = strstr(response, "\"events\":"); if (start) { - start += 10; + start += 9; while (1) { current = malloc(sizeof(log_node)); - if (sscanf(start, + int found = sscanf(start, event_template, - current->timestamp, - current->server_id, + current->details, current->type, - current->details) != 4) { + current->server_id, + current->timestamp); + if (found != 4) { free(current); break; } @@ -142,19 +144,43 @@ void parse_log_events(const char *response, log_node **log_root) { start = strstr(start, "},{"); if (!start) break; - start += 3; + start += 1; } } } // Render logs to an HTML file void render_logs_to_html(log_node *log_root) { - FILE *log_file = fopen("log.html", "w+"); + FILE *log_file = fopen("/requests/log.html", "w+"); if (!log_file) { perror("Failed to open HTML file"); return; } + // sort by date + // get number + size_t num_entries = 0; + log_node* current = log_root; + while(current){ + num_entries++; + current = current->next; + } + log_node** log_heap = malloc(sizeof(log_node*) * num_entries); + current = log_root; + for(size_t index = 0; index < num_entries; index++){ + log_heap[index] = current; + current = current->next; + for(size_t insert = index; insert > 0; insert = insert / 2){ + if(strcmp(log_heap[insert/2]->timestamp, log_heap[insert]->timestamp) <= 0){ + break; + } else { + log_node* tmp = log_heap[insert]; + log_heap[insert] = log_heap[insert/2]; + log_heap[insert/2] = tmp; + } + } + } + // write preamble fprintf(log_file, "\n"); fprintf(log_file, "\n"); @@ -172,13 +198,57 @@ void render_logs_to_html(log_node *log_root) { fprintf(log_file, " \n"); // write log information - for(log_node* current = log_root; current != NULL; current = current->next){ + for(; num_entries > 0; num_entries--){ + current = log_heap[0]; fprintf(log_file, " \n"); fprintf(log_file, " %s\n", current->timestamp); fprintf(log_file, " %s\n", current->server_id); fprintf(log_file, " %s\n", current->type); fprintf(log_file, " %s\n", current->details); fprintf(log_file, " \n"); + // remove current root from heap + size_t index = 0; + log_heap[0] = log_heap[num_entries-1]; + // trade down until no more children + while(index*2 + 1 < num_entries){ + // compare with first child + int is_bigger = strcmp(log_heap[index]->timestamp, log_heap[index*2+1]->timestamp); + // check if second child exists + if(index*2 + 2 >= num_entries){ + // second child does not exist, if is bigger than child swap + if(is_bigger > 0){ + log_node* tmp = log_heap[index]; + log_heap[index] = log_heap[index*2+1]; + log_heap[index*2+1] = tmp; + } + break; + } + // know fist child is smaller, if second is smaller than first swap with second otherwise with first + if(is_bigger > 0){ + size_t swap_index; + if(strcmp(log_heap[index*2+1]->timestamp, log_heap[index*2+2]->timestamp) <= 0){ + // first is smaller + swap_index = index*2+1; + } else { + swap_index = index*2+2; + } + log_node* tmp = log_heap[index]; + log_heap[index] = log_heap[swap_index]; + log_heap[swap_index] = tmp; + index = swap_index; + } else { + // first is not smaller so need to check second if swap is neccessary + if(strcmp(log_heap[index]->timestamp, log_heap[index*2+2]->timestamp) > 0){ + // is bigger so need to swap + log_node* tmp = log_heap[index]; + log_heap[index] = log_heap[index*2+2]; + log_heap[index*2+2] = tmp; + index = index*2+2; + } else { + break; + } + } + } } // write ending @@ -203,7 +273,7 @@ int main() { "Host: %s\r\n" "Content-Type: application/json\r\n" "Content-Length: %zu\r\n\r\n" - "{\"token\": \"%s\"}", auth_server_ip, strlen(auth_token) + 12, auth_token); + "{\"token\": \"%s\"}", auth_server_ip, strlen(auth_token) + 13, auth_token); char auth_response[BUFFER_SIZE]; if (send_http_request(auth_sock, auth_request, auth_response, BUFFER_SIZE) < 0) { diff --git a/functions/example_app_hybrid/linux_syscalls/syscall.h b/functions/example_app_hybrid/linux_syscalls/syscall.h index 1c33bfa..2ec02e6 100644 --- a/functions/example_app_hybrid/linux_syscalls/syscall.h +++ b/functions/example_app_hybrid/linux_syscalls/syscall.h @@ -49,6 +49,11 @@ static inline int linux_socket(int domain, int type, int protocol) { return __syscall(SYS_socket, domain, type, protocol); } +struct sockaddr { + unsigned short sa_family; + char sa_data[14]; +}; + struct in_addr { uint32_t s_addr; }; @@ -57,6 +62,10 @@ struct sockaddr_in { unsigned short sin_family; uint16_t sin_port; struct in_addr sin_addr; + unsigned char sin_zero[sizeof (struct sockaddr) + - sizeof(unsigned short) + - sizeof(uint16_t) + - sizeof(struct in_addr)]; }; static inline int linux_connect(int socketfd, const struct sockaddr_in* addr, int addrlen){ From a6507393a09ebe05f870e656fef9f8ca7439c0e1 Mon Sep 17 00:00:00 2001 From: Tom Kuchler Date: Sun, 27 Oct 2024 21:19:57 +0100 Subject: [PATCH 08/19] Intermediate state --- functions/CMakeLists.txt | 37 +++++++++++++++++++++--------------- functions/cfg_template.txt | 18 ++++++++++++++++++ functions/cross_template.txt | 17 ----------------- functions/dandelion.cmake | 11 ----------- 4 files changed, 40 insertions(+), 43 deletions(-) create mode 100644 functions/cfg_template.txt delete mode 100644 functions/cross_template.txt delete mode 100644 functions/dandelion.cmake diff --git a/functions/CMakeLists.txt b/functions/CMakeLists.txt index 39a06be..7299a8c 100644 --- a/functions/CMakeLists.txt +++ b/functions/CMakeLists.txt @@ -28,7 +28,7 @@ endif() FetchContent_Declare( dandelion_sdk DOWNLOAD_COMMAND gh release download latest -R eth-easl/dandelionFunctionInterface -p "*${CMAKE_BUILD_TYPE}_${PLATFORM}_${ARCHITECTURE}.tar.gz" - COMMAND tar -xzf "dandelion_sdk_${CMAKE_BUILD_TYPE}_${PLATFORM}_${ARCHITECTURE}.tar.gz" -C ${CMAKE_CURRENT_BINARY_DIR} + COMMAND tar -xzf "dandelion_sdk_${CMAKE_BUILD_TYPE}_${PLATFORM}_${ARCHITECTURE}.tar.gz" -C ${CMAKE_BINARY_DIR} ) FetchContent_MakeAvailable(dandelion_sdk) @@ -41,34 +41,41 @@ execute_process( add_library(dandelion_runtime INTERFACE IMPORTED) target_include_directories(dandelion_runtime INTERFACE - "${CMAKE_CURRENT_BINARY_DIR}/dandelion_sdk/include" + "${CMAKE_BINARY_DIR}/dandelion_sdk/include" ) target_compile_options(dandelion_runtime INTERFACE -nostdinc -idirafter${COMPILER_RUNTIME_INCLUDE}) target_link_libraries(dandelion_runtime INTERFACE - "${CMAKE_CURRENT_BINARY_DIR}/dandelion_sdk/lib/libdandelion_runtime.a" - "${CMAKE_CURRENT_BINARY_DIR}/dandelion_sdk/lib/libdandelion_system.a" + "${CMAKE_BINARY_DIR}/dandelion_sdk/lib/libdandelion_runtime.a" + "${CMAKE_BINARY_DIR}/dandelion_sdk/lib/libdandelion_system.a" ) target_link_options(dandelion_runtime INTERFACE -static -nostdlib) add_library(dlibc INTERFACE IMPORTED) target_include_directories(dlibc INTERFACE - "${CMAKE_CURRENT_BINARY_DIR}/dandelion_sdk/include" - "${CMAKE_CURRENT_BINARY_DIR}/dandelion_sdk/include/sys" + "${CMAKE_BINARY_DIR}/dandelion_sdk/include" + "${CMAKE_BINARY_DIR}/dandelion_sdk/include/sys" ) target_link_libraries(dlibc INTERFACE - "${CMAKE_CURRENT_BINARY_DIR}/dandelion_sdk/lib/libc.a" - "${CMAKE_CURRENT_BINARY_DIR}/dandelion_sdk/lib/libg.a" - "${CMAKE_CURRENT_BINARY_DIR}/dandelion_sdk/lib/libm.a" - "${CMAKE_CURRENT_BINARY_DIR}/dandelion_sdk/lib/libdandelion_file_system.a" + "${CMAKE_BINARY_DIR}/dandelion_sdk/lib/libc.a" + "${CMAKE_BINARY_DIR}/dandelion_sdk/lib/libg.a" + "${CMAKE_BINARY_DIR}/dandelion_sdk/lib/libm.a" + "${CMAKE_BINARY_DIR}/dandelion_sdk/lib/libdandelion_file_system.a" ) -target_link_options(dlibc INTERFACE -T${CMAKE_CURRENT_BINARY_DIR}/dandelion_sdk/linker.ld) +target_link_options(dlibc INTERFACE -T${CMAKE_BINARY_DIR}/dandelion_sdk/linker.ld) add_library(dlibcxx INTERFACE IMPORTED) -target_include_directories(dlibcxx INTERFACE "${CMAKE_CURRENT_BINARY_DIR}/dandelion_sdk/include/c++/v1") +target_include_directories(dlibcxx INTERFACE "${CMAKE_BINARY_DIR}/dandelion_sdk/include/c++/v1") target_link_libraries(dlibcxx INTERFACE - "${CMAKE_CURRENT_BINARY_DIR}/dandelion_sdk/lib/libc++.a" - "${CMAKE_CURRENT_BINARY_DIR}/dandelion_sdk/lib/libc++abi.a" - "${CMAKE_CURRENT_BINARY_DIR}/dandelion_sdk/lib/libunwind.a" + "${CMAKE_BINARY_DIR}/dandelion_sdk/lib/libc++.a" + "${CMAKE_BINARY_DIR}/dandelion_sdk/lib/libc++abi.a" + "${CMAKE_BINARY_DIR}/dandelion_sdk/lib/libunwind.a" +) + +# create compiler wrapper for use to compile non integrated programs +configure_file( + ${CMAKE_CURRENT_SOURCE_DIR}/cfg_template.txt + ${CMAKE_BINARY_DIR}/dandelion.cfg + FILE_PERMISSIONS OWNER_READ OWNER_EXECUTE ) if(CMAKE_BUILD_TYPE MATCHES "Debug") diff --git a/functions/cfg_template.txt b/functions/cfg_template.txt new file mode 100644 index 0000000..25f6b86 --- /dev/null +++ b/functions/cfg_template.txt @@ -0,0 +1,18 @@ +# compilation target +--target=@aRCHITECTURE@-linux-elf + +# include flags +-nostdinc +-idirafter@CMAKE_BINARY_DIR@/dandelion_sdk/include +-idirafter@CMAKE_BINARY_DIR@/dandelion_sdk/include/sys +-idirafter@COMPILER_RUNTIME_INCLUDE@ + +# linker flags +-nostdlib +-L@CMAKE_BINARY_DIR@/dandelion_sdk/lib +-lm +-lc +-lg +-ldandelion_file_system +-ldandelion_runtime +-ldandelion_system \ No newline at end of file diff --git a/functions/cross_template.txt b/functions/cross_template.txt deleted file mode 100644 index 48c3102..0000000 --- a/functions/cross_template.txt +++ /dev/null @@ -1,17 +0,0 @@ -[binaries] -c = '${CMAKE_C_COMPILER}' -cpp = '${CMAKE_CXX_COMPILER}' -ld = '${CMAKE_LINKER}' -ar = '${CMAKE_AR}' - -[host_machine] -system = 'dandelion' -endian = 'little' -cpu_family = '${TARGET_CPU}' -cpu = '${TARGET_CPU}' - -[built-in options] -default_library = 'static' - -[cmake] -DANDELION_PLATFORM = '${DANDELION_PLATFORM}' \ No newline at end of file diff --git a/functions/dandelion.cmake b/functions/dandelion.cmake deleted file mode 100644 index f499c67..0000000 --- a/functions/dandelion.cmake +++ /dev/null @@ -1,11 +0,0 @@ -# set name of target OS -set(CMAKE_SYSTEM_NAME Generic) -set(CMAKE_SYSTEM_PROCESSOR "${TARGET_CPU}") -# make sure no libraries from the host platform are used -set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) -set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) -set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) -set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) - -# set copile target for compiler check to cross compile -set(CMAKE_TRY_COMPILE_TARGET_TYPE "STATIC_LIBRARY") \ No newline at end of file From 7b7818621cd131a50af564894d5dc3ce74128647 Mon Sep 17 00:00:00 2001 From: Yazhuo Zhang Date: Tue, 29 Oct 2024 17:51:20 +0100 Subject: [PATCH 09/19] Debug: Hardcoded auth_server_ip --- .gitignore | 2 +- functions/example_app_hybrid/app_hybrid.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 24e14e6..9ca27e0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ *.swp .vscode -build +build* target *.check_cache Cargo.lock diff --git a/functions/example_app_hybrid/app_hybrid.c b/functions/example_app_hybrid/app_hybrid.c index 75bfaab..633fa4b 100644 --- a/functions/example_app_hybrid/app_hybrid.c +++ b/functions/example_app_hybrid/app_hybrid.c @@ -260,7 +260,7 @@ void render_logs_to_html(log_node *log_root) { // Main function to handle authorization, log fetching, and rendering int main() { - const char *auth_server_ip = "127.0.0.1"; + const char *auth_server_ip = "10.233.0.13"; const char *auth_token = "fapw84ypf3984viuhsvpoi843ypoghvejkfld"; // handle From b04487fc3b8af553ccd6865d9fa62f4bd7f2bdc0 Mon Sep 17 00:00:00 2001 From: Tom Kuchler Date: Thu, 7 Nov 2024 10:27:50 +0100 Subject: [PATCH 10/19] Add wrapper for c++ --- functions/CMakeLists.txt | 39 +++++++++++++++++++++++++++++++++++- functions/cfg++_template.txt | 9 +++++++++ functions/cfg_template.txt | 6 +++++- 3 files changed, 52 insertions(+), 2 deletions(-) create mode 100644 functions/cfg++_template.txt diff --git a/functions/CMakeLists.txt b/functions/CMakeLists.txt index 7299a8c..9e09de6 100644 --- a/functions/CMakeLists.txt +++ b/functions/CMakeLists.txt @@ -75,7 +75,14 @@ target_link_libraries(dlibcxx INTERFACE configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/cfg_template.txt ${CMAKE_BINARY_DIR}/dandelion.cfg - FILE_PERMISSIONS OWNER_READ OWNER_EXECUTE + FILE_PERMISSIONS OWNER_READ +) + +# create compiler wrapper for use to compile non integrated programs +configure_file( + ${CMAKE_CURRENT_SOURCE_DIR}/cfg++_template.txt + ${CMAKE_BINARY_DIR}/dandelion++.cfg + FILE_PERMISSIONS OWNER_READ ) if(CMAKE_BUILD_TYPE MATCHES "Debug") @@ -113,3 +120,33 @@ add_subdirectory(files) add_subdirectory(stdio) add_subdirectory(stdio_cxx) add_subdirectory(example_app_hybrid) + +set(CONFIG_FLAG "--config ${CMAKE_BINARY_DIR}/dandelion.cfg") +set(CONFIG_FLAGXX "--config ${CMAKE_BINARY_DIR}/dandelion++.cfg") + +include(ExternalProject) +ExternalProject_Add( + llvmproject + PREFIX llvmproject + GIT_REPOSITORY https://github.com/llvm/llvm-project + GIT_TAG release/18.x + SOURCE_SUBDIR llvm + UPDATE_DISCONNECTED 1 + INSTALL_DIR ${CMAKE_BINARY_DIR}/llvm + LIST_SEPARATOR "," + CMAKE_ARGS + -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DCMAKE_TRY_COMPILE_TARGET_TYPE=STATIC_LIBRARY + -DCMAKE_SYSTEM_NAME=Generic + -DCMAKE_INSTALL_PREFIX= + -DCMAKE_C_FLAGS:STRING=${CONFIG_FLAG} + -DCMAKE_CXX_FLAGS:STRING=${CONFIG_FLAGXX} + -DLLVM_ENABLE_PROJECTS=clang,lld + -DLLVM_TARGETS_TO_BUILD=AArch64,X86 + -DBUILD_SHARED_LIBS=OFF + -DLLVM_ENABLE_PIC=OFF + -DLLVM_ENABLE_THREADS=OFF + EXCLUDE_FROM_ALL TRUE +) \ No newline at end of file diff --git a/functions/cfg++_template.txt b/functions/cfg++_template.txt new file mode 100644 index 0000000..d6e53a5 --- /dev/null +++ b/functions/cfg++_template.txt @@ -0,0 +1,9 @@ +# additional compiler flags +-idirafter${CMAKE_BINARY_DIR}/dandelion_sdk/include/c++/v1 + +# additional linker flags +-lc++ +-lc++abi +-lunwind + +@dandelion.cfg \ No newline at end of file diff --git a/functions/cfg_template.txt b/functions/cfg_template.txt index 25f6b86..8205833 100644 --- a/functions/cfg_template.txt +++ b/functions/cfg_template.txt @@ -1,5 +1,6 @@ # compilation target ---target=@aRCHITECTURE@-linux-elf +--target=@ARCHITECTURE@-unknown-elf +-D_GNU_SOURCE=1 # include flags -nostdinc @@ -8,6 +9,9 @@ -idirafter@COMPILER_RUNTIME_INCLUDE@ # linker flags +-T@CMAKE_BINARY_DIR@/dandelion_sdk/linker.ld +-fuse-ld=lld +-static -nostdlib -L@CMAKE_BINARY_DIR@/dandelion_sdk/lib -lm From d50eb3ab95ba27bf829c09cef970852a4d9e59e0 Mon Sep 17 00:00:00 2001 From: Tom Kuchler Date: Fri, 29 Nov 2024 15:16:25 +0100 Subject: [PATCH 11/19] Fix example apps, parallelize requests to hybrid --- functions/example_app/CMakeLists.txt | 12 +- functions/example_app/fan_out.c | 2 +- functions/example_app/template.c | 12 +- functions/example_app_hybrid/CMakeLists.txt | 6 +- functions/example_app_hybrid/app_hybrid.c | 130 ++++++++++++++---- .../arch/aarch64/syscall_arch.h | 1 + .../linux_syscalls/arch/x86_64/syscall_arch.h | 1 + .../linux_syscalls/syscall.h | 18 +++ 8 files changed, 140 insertions(+), 42 deletions(-) diff --git a/functions/example_app/CMakeLists.txt b/functions/example_app/CMakeLists.txt index 29407c3..f98a5a8 100644 --- a/functions/example_app/CMakeLists.txt +++ b/functions/example_app/CMakeLists.txt @@ -14,22 +14,22 @@ add_executable(${TEMPLATE} template.c ) -target_compile_options(${HANDLE} PRIVATE -static -O3) -target_link_options(${HANDLE} PRIVATE -static) +target_compile_options(${HANDLE} PRIVATE -static -Os -flto) +target_link_options(${HANDLE} PRIVATE -static -Os -flto) target_link_libraries(${HANDLE} PRIVATE dlibc dandelion_runtime ) -target_compile_options(${FAN_OUT} PRIVATE -static -O3) -target_link_options(${FAN_OUT} PRIVATE -static) +target_compile_options(${FAN_OUT} PRIVATE -static -Os -flto) +target_link_options(${FAN_OUT} PRIVATE -static -Os -flto) target_link_libraries(${FAN_OUT} PRIVATE dlibc dandelion_runtime ) -target_compile_options(${TEMPLATE} PRIVATE -static -O3) -target_link_options(${TEMPLATE} PRIVATE -static) +target_compile_options(${TEMPLATE} PRIVATE -static -Os -flto) +target_link_options(${TEMPLATE} PRIVATE -static -Os -flto) target_link_libraries(${TEMPLATE} PRIVATE dlibc dandelion_runtime diff --git a/functions/example_app/fan_out.c b/functions/example_app/fan_out.c index 43fe560..e32c13e 100644 --- a/functions/example_app/fan_out.c +++ b/functions/example_app/fan_out.c @@ -34,7 +34,7 @@ int main(int argc, char const *argv[]) { // get the json with the athorization char username[21] = {0}; char token[257] = {0}; - if (fscanf(auth_file, "{\n authorized: %20s,\n token: %256s\n}", username, token) < 0) { + if (fscanf(auth_file, "%*[^\"]\"authorized\":\"%20[^\"]\",\"token\":\"%256[^\"]\"}", username, token) < 0) { perror("Failed to parse line from token file"); return -1; } diff --git a/functions/example_app/template.c b/functions/example_app/template.c index 41fd62e..1b495cd 100644 --- a/functions/example_app/template.c +++ b/functions/example_app/template.c @@ -17,12 +17,12 @@ typedef struct log_node{ } log_node; char event_template[] = -"%*[^{]" -"\"details\":\"%100[^\"]," -"\"event_type\":\"%100[^\"]," -"\"server_id\":\"%100[^\"]," -"\"timestamp\":\"%100[^\"]" -"}"; +"{" +"\"details\":\"%100[^\"]\"," +"\"event_type\":\"%100[^\"]\"," +"\"server_id\":\"%100[^\"]\"," +"\"timestamp\":\"%100[^\"]\"" +"%*[^{]"; int main(int argc, char const *argv[]) { diff --git a/functions/example_app_hybrid/CMakeLists.txt b/functions/example_app_hybrid/CMakeLists.txt index 017bdd8..c460643 100644 --- a/functions/example_app_hybrid/CMakeLists.txt +++ b/functions/example_app_hybrid/CMakeLists.txt @@ -2,14 +2,14 @@ cmake_minimum_required(VERSION ${CMAKE_VERSION}) add_subdirectory(linux_syscalls) -set(EXECUTABLE "app_bybrid") +set(EXECUTABLE "app_hybrid") add_executable(${EXECUTABLE} app_hybrid.c ) -target_compile_options(${EXECUTABLE} PRIVATE -static -O3) -target_link_options(${EXECUTABLE} PRIVATE -static) +target_compile_options(${EXECUTABLE} PRIVATE -static -Os -flto) +target_link_options(${EXECUTABLE} PRIVATE -static -Os -flto) target_link_libraries(${EXECUTABLE} PRIVATE linux_syscalls dlibc diff --git a/functions/example_app_hybrid/app_hybrid.c b/functions/example_app_hybrid/app_hybrid.c index 633fa4b..69279bc 100644 --- a/functions/example_app_hybrid/app_hybrid.c +++ b/functions/example_app_hybrid/app_hybrid.c @@ -92,8 +92,8 @@ int connect_to_server(const char *ip, uint16_t port) { } -int send_http_request(int sockfd, const char *request, char *response, size_t response_size) { - if (linux_send(sockfd, request, strlen(request), 0) < 0) { +int send_http_request(int sockfd, const char *request, size_t request_size, char *response, size_t response_size) { + if (linux_send(sockfd, request, request_size, 0) < 0) { perror("Send failed"); return -1; } @@ -260,23 +260,53 @@ void render_logs_to_html(log_node *log_root) { // Main function to handle authorization, log fetching, and rendering int main() { - const char *auth_server_ip = "10.233.0.13"; - const char *auth_token = "fapw84ypf3984viuhsvpoi843ypoghvejkfld"; + // read authentification server ip from file + FILE *auth_server = fopen("/servers/server.txt", "r"); + if (auth_server == NULL) { + perror("Failed to open file to get auth server"); + return -1; + } + char *auth_server_ip = NULL; + size_t server_line_len = 0; + size_t read_chars = __getline(&auth_server_ip, &server_line_len, auth_server); + if (read_chars < 0) { + perror("Fauled to read line from auth server file\n"); + return -1; + } + // strip possible endln from the getline + if(auth_server_ip[read_chars-1] == '\n'){ + auth_server_ip[read_chars-1] = '\0'; + } + + // read token from incomming file + FILE *token_file = fopen("/responses/Authorization", "r"); + if (token_file == NULL) { + perror("Failed to open file with token"); + return -1; + } + char auth_token[257] = {0}; + if (fscanf(token_file, "Bearer %256s", auth_token) < 0) { + perror("Failed to parse line from token file"); + return -1; + } // handle int auth_sock = connect_to_server(auth_server_ip, AUTH_SERVER_PORT); if (auth_sock < 0) return 1; char auth_request[BUFFER_SIZE]; - snprintf(auth_request, sizeof(auth_request), - "POST /authorize HTTP/1.1\r\n" - "Host: %s\r\n" - "Content-Type: application/json\r\n" - "Content-Length: %zu\r\n\r\n" + int written = snprintf(auth_request, sizeof(auth_request), + "POST /authorize HTTP/1.1\n" + "Host: %s\n" + "Content-Type: application/json\n" + "Content-Length: %zu\n\n" "{\"token\": \"%s\"}", auth_server_ip, strlen(auth_token) + 13, auth_token); - + if(written < 0 || written > BUFFER_SIZE){ + perror("Failed to format auth request"); + return -1; + } char auth_response[BUFFER_SIZE]; - if (send_http_request(auth_sock, auth_request, auth_response, BUFFER_SIZE) < 0) { + if (send_http_request(auth_sock, auth_request, written, auth_response, BUFFER_SIZE) < 0) { close(auth_sock); return 1; } @@ -288,22 +318,70 @@ int main() { // fan_out log_node *log_root = NULL; - char log_request[BUFFER_SIZE]; - char log_response[BUFFER_SIZE]; - - for (int i = 0; i < SERVER_NUMBER; ++i) { - int log_sock = connect_to_server(auth_server_ip, AUTH_SERVER_PORT); - if (log_sock < 0) continue; - - snprintf(log_request, sizeof(log_request), - "GET /logs/%d HTTP/1.1\r\n" - "Host: %s\r\n" - "Authorization: Bearer %s\r\n\r\n", i, auth_server_ip, token); - - if (send_http_request(log_sock, log_request, log_response, BUFFER_SIZE) == 0) { - parse_log_events(log_response, &log_root); + char log_request[SERVER_NUMBER][BUFFER_SIZE]; + int request_length[SERVER_NUMBER]; + char log_response[SERVER_NUMBER][BUFFER_SIZE]; + size_t response_read[SERVER_NUMBER]; + struct pollfd poll_fds[SERVER_NUMBER]; + + int log_sockets[SERVER_NUMBER]; + // open sockets + for(int server = 0; server < SERVER_NUMBER; server++){ + log_sockets[server] = connect_to_server(auth_server_ip, AUTH_SERVER_PORT); + if(log_sockets[server] < 0){ + perror("Failed to open log socket"); + return -1; + } + } + // format requests + for (int server = 0; server < SERVER_NUMBER; server++) { + request_length[server] = snprintf(log_request[server], sizeof(log_request[server]), + "GET /logs/%d HTTP/1.1\n" + "Host: %s\n" + "Authorization: Bearer %s\n\n", server, auth_server_ip, token); + if(request_length < 0){ + perror("Could not format server log request"); + return -1; + } + } + // issue all requests + for (int server = 0; server < SERVER_NUMBER; server++){ + if (linux_send(log_sockets[server], log_request[server], request_length[server], 0) < 0) { + perror("Send failed"); + return -1; } - close(log_sock); + response_read[server] = 0; + poll_fds[server].fd = log_sockets[server]; + poll_fds[server].events = POLLIN; + poll_fds[server].revents = 0; + } + unsigned int received_responses = 0; + while(received_responses < SERVER_NUMBER){ + int poll_result = linux_ppoll(poll_fds, SERVER_NUMBER); + if(poll_result < 0) { + perror("Polling failed"); + return -1; + } + // check all the file descriptors for arrived data + for(int server = 0; server < SERVER_NUMBER; server++){ + if(poll_fds[server].fd > 0 && (poll_fds[server].revents & (POLLIN | POLLHUP))){ + received_responses++; + // disable further polling on this socket + poll_fds[server].fd = -1; + int new_data = linux_recv(log_sockets[server], log_response[server] + response_read[server], BUFFER_SIZE - response_read[server] - 1, 0); + if(new_data < 0) { + perror("Failed to read from socket"); + return -1; + } + response_read[server] += new_data; + log_response[server][new_data] = '\0'; + parse_log_events(log_response[server], &log_root); + } + } + } + // close sockets + for(int server = 0; server < SERVER_NUMBER; server++){ + close(log_sockets[server]); } // render diff --git a/functions/example_app_hybrid/linux_syscalls/arch/aarch64/syscall_arch.h b/functions/example_app_hybrid/linux_syscalls/arch/aarch64/syscall_arch.h index b621c98..0e5a982 100644 --- a/functions/example_app_hybrid/linux_syscalls/arch/aarch64/syscall_arch.h +++ b/functions/example_app_hybrid/linux_syscalls/arch/aarch64/syscall_arch.h @@ -7,6 +7,7 @@ #define SYS_lseek 62 #define SYS_read 63 #define SYS_write 64 +#define SYS_ppoll 73 #define SYS_exit_group 94 #define SYS_socket 198 #define SYS_connect 203 diff --git a/functions/example_app_hybrid/linux_syscalls/arch/x86_64/syscall_arch.h b/functions/example_app_hybrid/linux_syscalls/arch/x86_64/syscall_arch.h index d7f5687..e6292bb 100644 --- a/functions/example_app_hybrid/linux_syscalls/arch/x86_64/syscall_arch.h +++ b/functions/example_app_hybrid/linux_syscalls/arch/x86_64/syscall_arch.h @@ -14,6 +14,7 @@ #define SYS_exit_group 231 #define SYS_openat 257 #define SYS_getdents64 217 +#define SYS_ppoll 271 #define ARCH_SET_FS 0x1002 diff --git a/functions/example_app_hybrid/linux_syscalls/syscall.h b/functions/example_app_hybrid/linux_syscalls/syscall.h index 2ec02e6..17c166a 100644 --- a/functions/example_app_hybrid/linux_syscalls/syscall.h +++ b/functions/example_app_hybrid/linux_syscalls/syscall.h @@ -17,6 +17,14 @@ #define LINUX_AF_INET 2 #define LINUX_SOCK_STREAM 1 +// poll defines +#define POLLIN 0x0001 +#define POLLPRI 0x0002 +#define POLLOUT 0x0004 +#define POLLERR 0x0008 +#define POLLHUP 0x0010 +#define POLLNVAL 0x0020 + #ifndef __scc #define __scc(X) ((long)(X)) typedef long syscall_arg_t; @@ -84,4 +92,14 @@ static inline int linux_close(int fd){ return __syscall(SYS_close, fd); } +struct pollfd { + int fd; + short events; + short revents; +}; + +static inline int linux_ppoll(struct pollfd* fds, unsigned int nfds) { + return __syscall(SYS_ppoll, fds, nfds, NULL, NULL); +} + #endif From ce3fda9e352d85dac932771c16cd28b2ef9047e4 Mon Sep 17 00:00:00 2001 From: Tom Kuchler Date: Fri, 6 Dec 2024 13:49:01 +0100 Subject: [PATCH 12/19] Update busy function --- functions/README.md | 5 +- functions/busy/busy.c | 106 +++++++++++++++++++++++------------------- 2 files changed, 62 insertions(+), 49 deletions(-) diff --git a/functions/README.md b/functions/README.md index ed46ccf..32f422e 100644 --- a/functions/README.md +++ b/functions/README.md @@ -7,10 +7,11 @@ through mlibc. # Building the C functions The cmake build process will build the functions based on the latest version of the dandelion SDK. -For this cmake meeds to know the platform to build for (cheri, mmu_freebsd, mmu_linux, debug) and architecture. +For this cmake meeds to know the platform to build for (cheri, mmu_freebsd, mmu_linux, debug) and architecture (x86_64, aarch64) +The build type can be Release or Debug. From the top folder functions can be built with: ``` mkdir build cd build -cmake -DPLATFORM= -DARCHITECTURE= ../functions +cmake -DPLATFORM= -DARCHITECTURE= -DCMAKE_BUILD_TYPE- ../functions ``` diff --git a/functions/busy/busy.c b/functions/busy/busy.c index d61c5fd..7c52d33 100644 --- a/functions/busy/busy.c +++ b/functions/busy/busy.c @@ -8,20 +8,26 @@ #include "dandelion/crt.h" #include "dandelion/runtime.h" -void* local_memcpy(void* dest, const void* src, size_t n) { +#define ARRAY_ITEMS 16 + +typedef struct data_item { + int8_t value[ARRAY_ITEMS]; +} data_item; + +void *local_memcpy(void *dest, const void *src, size_t n) { // copy using size_t steps if available size_t pointer_size = sizeof(size_t); size_t copied = 0; size_t to_copy = pointer_size * (n / pointer_size); - size_t* to = dest; - const size_t* from = src; + size_t *to = dest; + const size_t *from = src; for (; copied < to_copy; copied += pointer_size) { *to = *from; from++; to++; } - uint8_t* to8 = (uint8_t*)to; - const uint8_t* from8 = (uint8_t*)from; + uint8_t *to8 = (uint8_t *)to; + const uint8_t *from8 = (uint8_t *)from; for (; copied < n; copied++) { *to8 = *from8; from8++; @@ -31,60 +37,66 @@ void* local_memcpy(void* dest, const void* src, size_t n) { } int busy() { // we expect two sets, one with the response data we work on - // another one with the the response preable to attach the body to + // another one with the the response preable to attach the body to size_t input_set_count = dandelion_input_set_count(); - if (input_set_count != 2) return -1; + if (input_set_count != 2) + return -1; size_t data_set_size = dandelion_input_buffer_count(0); - if (data_set_size != 1) return -2; + if (data_set_size != 1) + return -2; size_t preamble_set_size = dandelion_input_buffer_count(1); - if (preamble_set_size != 1) return -3; + if (preamble_set_size != 1) + return -3; - IoBuffer* data_buffer = dandelion_get_input(0, 0); - IoBuffer* preamble_buffer = dandelion_get_input(1, 0); + IoBuffer *data_buffer = dandelion_get_input(0, 0); + IoBuffer *preamble_buffer = dandelion_get_input(1, 0); // start reading the http response that is the input - char* data = (char*) data_buffer->data; + char *data = (char *)data_buffer->data; size_t data_len = data_buffer->data_len; size_t preamble_len = preamble_buffer->data_len; - if (data_len < 8) return -4; - if ((size_t) data % 8 != 0) return -5; + if (data_len < 8) + return -4; + if ((size_t)data % 8 != 0) + return -5; - uint64_t iterations = *(uint64_t*)data; - - size_t output_len = data_len + preamble_len; - char* output_buffer = dandelion_alloc(output_len, 64); - local_memcpy(output_buffer, preamble_buffer->data, preamble_len); - // not copying the input data buffer fully, so the runtime is actually decoupled from the input since + uint64_t iterations = *(uint64_t *)data; + data_item *data_items = (data_item *)(data + 8); -#if defined(__x86_64__) - uint64_t counter = 0; - __asm__ volatile( - "1:\n" - "addq $1, %[counter]\n" - "cmp %[counter], %[iterations]\n" - "jg 1b\n" - : [counter] "+r"(counter) - : [iterations] "r"(iterations)); + // the igreggated statistics we want to collect + // sum, average, variance, min, max + int64_t sum = 0; + int64_t min = 256; + int64_t max = -256; -#elif defined(__aarch64__) - uint64_t counter = 0; - __asm__ volatile( - "1:\n" - "add %[counter], %[counter], #1\n" - "cmp %[iterations], %[counter]\n" - "b.gt 1b\n" - : [counter] "+r"(counter) - : [iterations] "r"(iterations)); -#elif defined(__wasm__) - volatile uint64_t counter = 0; - while (counter < iterations) { - counter += 1; + size_t struct_index = 0; + size_t value_index = 0; + size_t max_index = (data_len - 8) / sizeof(data_item); + for (size_t iteration = 0; iteration < iterations; iteration++) { + for (size_t array_index = 0; array_index < ARRAY_ITEMS; array_index++) { + int8_t value = data_items[struct_index].value[array_index]; + if (value > max) + max = value; + if (value < min) + min = value; + sum += value; + } + // using size_t which has no sign, guarantees that the index is not negative + // when computing the remainder + struct_index = + (struct_index + data_items[struct_index].value[value_index]) % + max_index; + value_index = (value_index + 1) % ARRAY_ITEMS; } -#else -#error "Unimplemented architecture" -#endif - *((uint64_t*)(output_buffer + preamble_len)) = counter; + size_t output_len = 3 * sizeof(int64_t) + preamble_len; + char *output_buffer = dandelion_alloc(output_len, 64); + local_memcpy(output_buffer, preamble_buffer->data, preamble_len); + local_memcpy(output_buffer + preamble_len, &sum, sizeof(int64_t)); + local_memcpy(output_buffer + preamble_len + sizeof(int64_t), &min, + sizeof(int64_t)); + local_memcpy(output_buffer + preamble_len + 2 * sizeof(int64_t), &max, + sizeof(int64_t)); // keep ident and key from data buffer IoBuffer request_buffer = *data_buffer; @@ -98,5 +110,5 @@ int busy() { // takes two inputs, first one is an blob of data to hand through the function // that also contains a number that tells the function to loop a X times. // The second input is a preamble to format the output into a http request with -// the previous data as body +// the previous data as body DANDELION_ENTRY(busy) From fc15bee53c8eb6ec9902463958e118bc1d274b9e Mon Sep 17 00:00:00 2001 From: Tom Kuchler Date: Sat, 7 Dec 2024 11:50:58 +0100 Subject: [PATCH 13/19] Add compression function --- functions/CMakeLists.txt | 5 +- functions/compression/CMakeLists.txt | 11 + functions/compression/compression.c | 74 + functions/compression/qoi.h | 649 +++++++ functions/compression/stb_image_write.h | 2055 +++++++++++++++++++++++ 5 files changed, 2792 insertions(+), 2 deletions(-) create mode 100644 functions/compression/CMakeLists.txt create mode 100644 functions/compression/compression.c create mode 100644 functions/compression/qoi.h create mode 100644 functions/compression/stb_image_write.h diff --git a/functions/CMakeLists.txt b/functions/CMakeLists.txt index 9e09de6..43b3f82 100644 --- a/functions/CMakeLists.txt +++ b/functions/CMakeLists.txt @@ -90,7 +90,7 @@ if(CMAKE_BUILD_TYPE MATCHES "Debug") endif() # cheri compile flags -if("${DANDELION_PLATFORM}" STREQUAL "cheri") +if("${PLATFORM}" STREQUAL "cheri") add_compile_options(-march=morello -mabi=aapcs) add_link_options(-march=morello -mabi=aapcs -fuse-ld=lld -Xlinker --image-base=0x10000) get_property(CROSS_COMPILE_OPTIONS DIRECTORY PROPERTY COMPILE_OPTIONS) @@ -98,7 +98,7 @@ if("${DANDELION_PLATFORM}" STREQUAL "cheri") get_property(CROSS_LINK_OPTIONS DIRECTORY PROPERTY LINK_OPTIONS) string(REPLACE ";" " " CROSS_LINK_OPTIONS "${CROSS_LINK_OPTIONS}") # WebAssembly flags -elseif("${DANDELION_PLATFORM}" STREQUAL "wasm") +elseif("${PLATFORM}" STREQUAL "wasm") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --verbose -target wasm32-unknown-unknown -Wl,--export-all -Wl,--import-undefined -Wl,--import-memory") else() # currently still want to set the base image lower for all other images, if some don't want make special case @@ -120,6 +120,7 @@ add_subdirectory(files) add_subdirectory(stdio) add_subdirectory(stdio_cxx) add_subdirectory(example_app_hybrid) +add_subdirectory(compression) set(CONFIG_FLAG "--config ${CMAKE_BINARY_DIR}/dandelion.cfg") set(CONFIG_FLAGXX "--config ${CMAKE_BINARY_DIR}/dandelion++.cfg") diff --git a/functions/compression/CMakeLists.txt b/functions/compression/CMakeLists.txt new file mode 100644 index 0000000..08416d6 --- /dev/null +++ b/functions/compression/CMakeLists.txt @@ -0,0 +1,11 @@ +cmake_minimum_required(VERSION ${CMAKE_VERSION}) + +set(EXECUTABLE "compression") + +add_executable(${EXECUTABLE} + compression.c +) + +target_compile_options(${EXECUTABLE} PRIVATE -fno-stack-protector -O3) +target_link_options(${EXECUTABLE} PRIVATE -static -O3) +target_link_libraries(${EXECUTABLE} PRIVATE dandelion_runtime) diff --git a/functions/compression/compression.c b/functions/compression/compression.c new file mode 100644 index 0000000..3bb9f24 --- /dev/null +++ b/functions/compression/compression.c @@ -0,0 +1,74 @@ +// System Headers +#include "dandelion/crt.h" +#include "dandelion/runtime.h" +#include "dandelion/system/system.h" +#include +// Standard Libraries + +// Project External Libraries +#define QOI_NO_STDIO +#define QOI_MALLOC(size) dandelion_alloc(size, 8) +#define QOI_FREE(ptr) dandelion_free(ptr) +#define QOI_IMPLEMENTATION +#include "qoi.h" + +void *realloc_sized(void *old, size_t old_size, size_t new_size) { + // if new size is smaller just pretend we removed the bigger part + if (new_size < old_size) { + return old; + } + void *new = dandelion_alloc(new_size, 8); + if (new == NULL) { + return NULL; + } + memcpy(new, old, old_size); + // dandelion_free(old); + return new; +} + +#define STBI_WRITE_NO_STDIO +#define STBIW_MALLOC(size) dandelion_alloc(size, 8) +#define STBIW_FREE(ptr) dandelion_free(ptr) +#define STBIW_REALLOC_SIZED(ptr, old_size, new_size) \ + realloc_sized(ptr, old_size, new_size) +#define PNG_ONLY +#define STB_IMAGE_WRITE_IMPLEMENTATION +#include "stb_image_write.h" + +int main() { + // check we get expected inputs + size_t input_set_count = dandelion_input_set_count(); + if (input_set_count != 1) + return -1; + size_t qoi_set_size = dandelion_input_buffer_count(0); + if (qoi_set_size != 1) + return -2; + + IoBuffer *qoi_buffer = dandelion_get_input(0, 0); + + qoi_desc in_descriptor = {}; + int qoi_size = qoi_buffer->data_len; + void *pixels = qoi_decode(qoi_buffer->data, qoi_size, &in_descriptor, 0); + + if (pixels == NULL) { + return -3; + } + + int png_len; + void *png = stbi_write_png_to_mem(pixels, 0, in_descriptor.width, + in_descriptor.height, + in_descriptor.channels, &png_len); + if (png == NULL) { + return -4; + } + + IoBuffer png_buffer = {0}; + png_buffer.data = png; + png_buffer.key = qoi_buffer->key; + png_buffer.data_len = png_len; + + dandelion_add_output(0, png_buffer); + return 0; +} + +DANDELION_ENTRY(main); \ No newline at end of file diff --git a/functions/compression/qoi.h b/functions/compression/qoi.h new file mode 100644 index 0000000..f2800b0 --- /dev/null +++ b/functions/compression/qoi.h @@ -0,0 +1,649 @@ +/* + +Copyright (c) 2021, Dominic Szablewski - https://phoboslab.org +SPDX-License-Identifier: MIT + + +QOI - The "Quite OK Image" format for fast, lossless image compression + +-- About + +QOI encodes and decodes images in a lossless format. Compared to stb_image and +stb_image_write QOI offers 20x-50x faster encoding, 3x-4x faster decoding and +20% better compression. + + +-- Synopsis + +// Define `QOI_IMPLEMENTATION` in *one* C/C++ file before including this +// library to create the implementation. + +#define QOI_IMPLEMENTATION +#include "qoi.h" + +// Encode and store an RGBA buffer to the file system. The qoi_desc describes +// the input pixel data. +qoi_write("image_new.qoi", rgba_pixels, &(qoi_desc){ + .width = 1920, + .height = 1080, + .channels = 4, + .colorspace = QOI_SRGB +}); + +// Load and decode a QOI image from the file system into a 32bbp RGBA buffer. +// The qoi_desc struct will be filled with the width, height, number of channels +// and colorspace read from the file header. +qoi_desc desc; +void *rgba_pixels = qoi_read("image.qoi", &desc, 4); + + + +-- Documentation + +This library provides the following functions; +- qoi_read -- read and decode a QOI file +- qoi_decode -- decode the raw bytes of a QOI image from memory +- qoi_write -- encode and write a QOI file +- qoi_encode -- encode an rgba buffer into a QOI image in memory + +See the function declaration below for the signature and more information. + +If you don't want/need the qoi_read and qoi_write functions, you can define +QOI_NO_STDIO before including this library. + +This library uses malloc() and free(). To supply your own malloc implementation +you can define QOI_MALLOC and QOI_FREE before including this library. + +This library uses memset() to zero-initialize the index. To supply your own +implementation you can define QOI_ZEROARR before including this library. + + +-- Data Format + +A QOI file has a 14 byte header, followed by any number of data "chunks" and an +8-byte end marker. + +struct qoi_header_t { + char magic[4]; // magic bytes "qoif" + uint32_t width; // image width in pixels (BE) + uint32_t height; // image height in pixels (BE) + uint8_t channels; // 3 = RGB, 4 = RGBA + uint8_t colorspace; // 0 = sRGB with linear alpha, 1 = all channels linear +}; + +Images are encoded row by row, left to right, top to bottom. The decoder and +encoder start with {r: 0, g: 0, b: 0, a: 255} as the previous pixel value. An +image is complete when all pixels specified by width * height have been covered. + +Pixels are encoded as + - a run of the previous pixel + - an index into an array of previously seen pixels + - a difference to the previous pixel value in r,g,b + - full r,g,b or r,g,b,a values + +The color channels are assumed to not be premultiplied with the alpha channel +("un-premultiplied alpha"). + +A running array[64] (zero-initialized) of previously seen pixel values is +maintained by the encoder and decoder. Each pixel that is seen by the encoder +and decoder is put into this array at the position formed by a hash function of +the color value. In the encoder, if the pixel value at the index matches the +current pixel, this index position is written to the stream as QOI_OP_INDEX. +The hash function for the index is: + + index_position = (r * 3 + g * 5 + b * 7 + a * 11) % 64 + +Each chunk starts with a 2- or 8-bit tag, followed by a number of data bits. The +bit length of chunks is divisible by 8 - i.e. all chunks are byte aligned. All +values encoded in these data bits have the most significant bit on the left. + +The 8-bit tags have precedence over the 2-bit tags. A decoder must check for the +presence of an 8-bit tag first. + +The byte stream's end is marked with 7 0x00 bytes followed a single 0x01 byte. + + +The possible chunks are: + + +.- QOI_OP_INDEX ----------. +| Byte[0] | +| 7 6 5 4 3 2 1 0 | +|-------+-----------------| +| 0 0 | index | +`-------------------------` +2-bit tag b00 +6-bit index into the color index array: 0..63 + +A valid encoder must not issue 2 or more consecutive QOI_OP_INDEX chunks to the +same index. QOI_OP_RUN should be used instead. + + +.- QOI_OP_DIFF -----------. +| Byte[0] | +| 7 6 5 4 3 2 1 0 | +|-------+-----+-----+-----| +| 0 1 | dr | dg | db | +`-------------------------` +2-bit tag b01 +2-bit red channel difference from the previous pixel between -2..1 +2-bit green channel difference from the previous pixel between -2..1 +2-bit blue channel difference from the previous pixel between -2..1 + +The difference to the current channel values are using a wraparound operation, +so "1 - 2" will result in 255, while "255 + 1" will result in 0. + +Values are stored as unsigned integers with a bias of 2. E.g. -2 is stored as +0 (b00). 1 is stored as 3 (b11). + +The alpha value remains unchanged from the previous pixel. + + +.- QOI_OP_LUMA -------------------------------------. +| Byte[0] | Byte[1] | +| 7 6 5 4 3 2 1 0 | 7 6 5 4 3 2 1 0 | +|-------+-----------------+-------------+-----------| +| 1 0 | green diff | dr - dg | db - dg | +`---------------------------------------------------` +2-bit tag b10 +6-bit green channel difference from the previous pixel -32..31 +4-bit red channel difference minus green channel difference -8..7 +4-bit blue channel difference minus green channel difference -8..7 + +The green channel is used to indicate the general direction of change and is +encoded in 6 bits. The red and blue channels (dr and db) base their diffs off +of the green channel difference and are encoded in 4 bits. I.e.: + dr_dg = (cur_px.r - prev_px.r) - (cur_px.g - prev_px.g) + db_dg = (cur_px.b - prev_px.b) - (cur_px.g - prev_px.g) + +The difference to the current channel values are using a wraparound operation, +so "10 - 13" will result in 253, while "250 + 7" will result in 1. + +Values are stored as unsigned integers with a bias of 32 for the green channel +and a bias of 8 for the red and blue channel. + +The alpha value remains unchanged from the previous pixel. + + +.- QOI_OP_RUN ------------. +| Byte[0] | +| 7 6 5 4 3 2 1 0 | +|-------+-----------------| +| 1 1 | run | +`-------------------------` +2-bit tag b11 +6-bit run-length repeating the previous pixel: 1..62 + +The run-length is stored with a bias of -1. Note that the run-lengths 63 and 64 +(b111110 and b111111) are illegal as they are occupied by the QOI_OP_RGB and +QOI_OP_RGBA tags. + + +.- QOI_OP_RGB ------------------------------------------. +| Byte[0] | Byte[1] | Byte[2] | Byte[3] | +| 7 6 5 4 3 2 1 0 | 7 .. 0 | 7 .. 0 | 7 .. 0 | +|-------------------------+---------+---------+---------| +| 1 1 1 1 1 1 1 0 | red | green | blue | +`-------------------------------------------------------` +8-bit tag b11111110 +8-bit red channel value +8-bit green channel value +8-bit blue channel value + +The alpha value remains unchanged from the previous pixel. + + +.- QOI_OP_RGBA ---------------------------------------------------. +| Byte[0] | Byte[1] | Byte[2] | Byte[3] | Byte[4] | +| 7 6 5 4 3 2 1 0 | 7 .. 0 | 7 .. 0 | 7 .. 0 | 7 .. 0 | +|-------------------------+---------+---------+---------+---------| +| 1 1 1 1 1 1 1 1 | red | green | blue | alpha | +`-----------------------------------------------------------------` +8-bit tag b11111111 +8-bit red channel value +8-bit green channel value +8-bit blue channel value +8-bit alpha channel value + +*/ + + +/* ----------------------------------------------------------------------------- +Header - Public functions */ + +#ifndef QOI_H +#define QOI_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* A pointer to a qoi_desc struct has to be supplied to all of qoi's functions. +It describes either the input format (for qoi_write and qoi_encode), or is +filled with the description read from the file header (for qoi_read and +qoi_decode). + +The colorspace in this qoi_desc is an enum where + 0 = sRGB, i.e. gamma scaled RGB channels and a linear alpha channel + 1 = all channels are linear +You may use the constants QOI_SRGB or QOI_LINEAR. The colorspace is purely +informative. It will be saved to the file header, but does not affect +how chunks are en-/decoded. */ + +#define QOI_SRGB 0 +#define QOI_LINEAR 1 + +typedef struct { + unsigned int width; + unsigned int height; + unsigned char channels; + unsigned char colorspace; +} qoi_desc; + +#ifndef QOI_NO_STDIO + +/* Encode raw RGB or RGBA pixels into a QOI image and write it to the file +system. The qoi_desc struct must be filled with the image width, height, +number of channels (3 = RGB, 4 = RGBA) and the colorspace. + +The function returns 0 on failure (invalid parameters, or fopen or malloc +failed) or the number of bytes written on success. */ + +int qoi_write(const char *filename, const void *data, const qoi_desc *desc); + + +/* Read and decode a QOI image from the file system. If channels is 0, the +number of channels from the file header is used. If channels is 3 or 4 the +output format will be forced into this number of channels. + +The function either returns NULL on failure (invalid data, or malloc or fopen +failed) or a pointer to the decoded pixels. On success, the qoi_desc struct +will be filled with the description from the file header. + +The returned pixel data should be free()d after use. */ + +void *qoi_read(const char *filename, qoi_desc *desc, int channels); + +#endif /* QOI_NO_STDIO */ + + +/* Encode raw RGB or RGBA pixels into a QOI image in memory. + +The function either returns NULL on failure (invalid parameters or malloc +failed) or a pointer to the encoded data on success. On success the out_len +is set to the size in bytes of the encoded data. + +The returned qoi data should be free()d after use. */ + +void *qoi_encode(const void *data, const qoi_desc *desc, int *out_len); + + +/* Decode a QOI image from memory. + +The function either returns NULL on failure (invalid parameters or malloc +failed) or a pointer to the decoded pixels. On success, the qoi_desc struct +is filled with the description from the file header. + +The returned pixel data should be free()d after use. */ + +void *qoi_decode(const void *data, int size, qoi_desc *desc, int channels); + + +#ifdef __cplusplus +} +#endif +#endif /* QOI_H */ + + +/* ----------------------------------------------------------------------------- +Implementation */ + +#ifdef QOI_IMPLEMENTATION +#include +#include + +#ifndef QOI_MALLOC + #define QOI_MALLOC(sz) malloc(sz) + #define QOI_FREE(p) free(p) +#endif +#ifndef QOI_ZEROARR + #define QOI_ZEROARR(a) memset((a),0,sizeof(a)) +#endif + +#define QOI_OP_INDEX 0x00 /* 00xxxxxx */ +#define QOI_OP_DIFF 0x40 /* 01xxxxxx */ +#define QOI_OP_LUMA 0x80 /* 10xxxxxx */ +#define QOI_OP_RUN 0xc0 /* 11xxxxxx */ +#define QOI_OP_RGB 0xfe /* 11111110 */ +#define QOI_OP_RGBA 0xff /* 11111111 */ + +#define QOI_MASK_2 0xc0 /* 11000000 */ + +#define QOI_COLOR_HASH(C) (C.rgba.r*3 + C.rgba.g*5 + C.rgba.b*7 + C.rgba.a*11) +#define QOI_MAGIC \ + (((unsigned int)'q') << 24 | ((unsigned int)'o') << 16 | \ + ((unsigned int)'i') << 8 | ((unsigned int)'f')) +#define QOI_HEADER_SIZE 14 + +/* 2GB is the max file size that this implementation can safely handle. We guard +against anything larger than that, assuming the worst case with 5 bytes per +pixel, rounded down to a nice clean value. 400 million pixels ought to be +enough for anybody. */ +#define QOI_PIXELS_MAX ((unsigned int)400000000) + +typedef union { + struct { unsigned char r, g, b, a; } rgba; + unsigned int v; +} qoi_rgba_t; + +static const unsigned char qoi_padding[8] = {0,0,0,0,0,0,0,1}; + +static void qoi_write_32(unsigned char *bytes, int *p, unsigned int v) { + bytes[(*p)++] = (0xff000000 & v) >> 24; + bytes[(*p)++] = (0x00ff0000 & v) >> 16; + bytes[(*p)++] = (0x0000ff00 & v) >> 8; + bytes[(*p)++] = (0x000000ff & v); +} + +static unsigned int qoi_read_32(const unsigned char *bytes, int *p) { + unsigned int a = bytes[(*p)++]; + unsigned int b = bytes[(*p)++]; + unsigned int c = bytes[(*p)++]; + unsigned int d = bytes[(*p)++]; + return a << 24 | b << 16 | c << 8 | d; +} + +void *qoi_encode(const void *data, const qoi_desc *desc, int *out_len) { + int i, max_size, p, run; + int px_len, px_end, px_pos, channels; + unsigned char *bytes; + const unsigned char *pixels; + qoi_rgba_t index[64]; + qoi_rgba_t px, px_prev; + + if ( + data == NULL || out_len == NULL || desc == NULL || + desc->width == 0 || desc->height == 0 || + desc->channels < 3 || desc->channels > 4 || + desc->colorspace > 1 || + desc->height >= QOI_PIXELS_MAX / desc->width + ) { + return NULL; + } + + max_size = + desc->width * desc->height * (desc->channels + 1) + + QOI_HEADER_SIZE + sizeof(qoi_padding); + + p = 0; + bytes = (unsigned char *) QOI_MALLOC(max_size); + if (!bytes) { + return NULL; + } + + qoi_write_32(bytes, &p, QOI_MAGIC); + qoi_write_32(bytes, &p, desc->width); + qoi_write_32(bytes, &p, desc->height); + bytes[p++] = desc->channels; + bytes[p++] = desc->colorspace; + + + pixels = (const unsigned char *)data; + + QOI_ZEROARR(index); + + run = 0; + px_prev.rgba.r = 0; + px_prev.rgba.g = 0; + px_prev.rgba.b = 0; + px_prev.rgba.a = 255; + px = px_prev; + + px_len = desc->width * desc->height * desc->channels; + px_end = px_len - desc->channels; + channels = desc->channels; + + for (px_pos = 0; px_pos < px_len; px_pos += channels) { + px.rgba.r = pixels[px_pos + 0]; + px.rgba.g = pixels[px_pos + 1]; + px.rgba.b = pixels[px_pos + 2]; + + if (channels == 4) { + px.rgba.a = pixels[px_pos + 3]; + } + + if (px.v == px_prev.v) { + run++; + if (run == 62 || px_pos == px_end) { + bytes[p++] = QOI_OP_RUN | (run - 1); + run = 0; + } + } + else { + int index_pos; + + if (run > 0) { + bytes[p++] = QOI_OP_RUN | (run - 1); + run = 0; + } + + index_pos = QOI_COLOR_HASH(px) % 64; + + if (index[index_pos].v == px.v) { + bytes[p++] = QOI_OP_INDEX | index_pos; + } + else { + index[index_pos] = px; + + if (px.rgba.a == px_prev.rgba.a) { + signed char vr = px.rgba.r - px_prev.rgba.r; + signed char vg = px.rgba.g - px_prev.rgba.g; + signed char vb = px.rgba.b - px_prev.rgba.b; + + signed char vg_r = vr - vg; + signed char vg_b = vb - vg; + + if ( + vr > -3 && vr < 2 && + vg > -3 && vg < 2 && + vb > -3 && vb < 2 + ) { + bytes[p++] = QOI_OP_DIFF | (vr + 2) << 4 | (vg + 2) << 2 | (vb + 2); + } + else if ( + vg_r > -9 && vg_r < 8 && + vg > -33 && vg < 32 && + vg_b > -9 && vg_b < 8 + ) { + bytes[p++] = QOI_OP_LUMA | (vg + 32); + bytes[p++] = (vg_r + 8) << 4 | (vg_b + 8); + } + else { + bytes[p++] = QOI_OP_RGB; + bytes[p++] = px.rgba.r; + bytes[p++] = px.rgba.g; + bytes[p++] = px.rgba.b; + } + } + else { + bytes[p++] = QOI_OP_RGBA; + bytes[p++] = px.rgba.r; + bytes[p++] = px.rgba.g; + bytes[p++] = px.rgba.b; + bytes[p++] = px.rgba.a; + } + } + } + px_prev = px; + } + + for (i = 0; i < (int)sizeof(qoi_padding); i++) { + bytes[p++] = qoi_padding[i]; + } + + *out_len = p; + return bytes; +} + +void *qoi_decode(const void *data, int size, qoi_desc *desc, int channels) { + const unsigned char *bytes; + unsigned int header_magic; + unsigned char *pixels; + qoi_rgba_t index[64]; + qoi_rgba_t px; + int px_len, chunks_len, px_pos; + int p = 0, run = 0; + + if ( + data == NULL || desc == NULL || + (channels != 0 && channels != 3 && channels != 4) || + size < QOI_HEADER_SIZE + (int)sizeof(qoi_padding) + ) { + return NULL; + } + + bytes = (const unsigned char *)data; + + header_magic = qoi_read_32(bytes, &p); + desc->width = qoi_read_32(bytes, &p); + desc->height = qoi_read_32(bytes, &p); + desc->channels = bytes[p++]; + desc->colorspace = bytes[p++]; + + if ( + desc->width == 0 || desc->height == 0 || + desc->channels < 3 || desc->channels > 4 || + desc->colorspace > 1 || + header_magic != QOI_MAGIC || + desc->height >= QOI_PIXELS_MAX / desc->width + ) { + return NULL; + } + + if (channels == 0) { + channels = desc->channels; + } + + px_len = desc->width * desc->height * channels; + pixels = (unsigned char *) QOI_MALLOC(px_len); + if (!pixels) { + return NULL; + } + + QOI_ZEROARR(index); + px.rgba.r = 0; + px.rgba.g = 0; + px.rgba.b = 0; + px.rgba.a = 255; + + chunks_len = size - (int)sizeof(qoi_padding); + for (px_pos = 0; px_pos < px_len; px_pos += channels) { + if (run > 0) { + run--; + } + else if (p < chunks_len) { + int b1 = bytes[p++]; + + if (b1 == QOI_OP_RGB) { + px.rgba.r = bytes[p++]; + px.rgba.g = bytes[p++]; + px.rgba.b = bytes[p++]; + } + else if (b1 == QOI_OP_RGBA) { + px.rgba.r = bytes[p++]; + px.rgba.g = bytes[p++]; + px.rgba.b = bytes[p++]; + px.rgba.a = bytes[p++]; + } + else if ((b1 & QOI_MASK_2) == QOI_OP_INDEX) { + px = index[b1]; + } + else if ((b1 & QOI_MASK_2) == QOI_OP_DIFF) { + px.rgba.r += ((b1 >> 4) & 0x03) - 2; + px.rgba.g += ((b1 >> 2) & 0x03) - 2; + px.rgba.b += ( b1 & 0x03) - 2; + } + else if ((b1 & QOI_MASK_2) == QOI_OP_LUMA) { + int b2 = bytes[p++]; + int vg = (b1 & 0x3f) - 32; + px.rgba.r += vg - 8 + ((b2 >> 4) & 0x0f); + px.rgba.g += vg; + px.rgba.b += vg - 8 + (b2 & 0x0f); + } + else if ((b1 & QOI_MASK_2) == QOI_OP_RUN) { + run = (b1 & 0x3f); + } + + index[QOI_COLOR_HASH(px) % 64] = px; + } + + pixels[px_pos + 0] = px.rgba.r; + pixels[px_pos + 1] = px.rgba.g; + pixels[px_pos + 2] = px.rgba.b; + + if (channels == 4) { + pixels[px_pos + 3] = px.rgba.a; + } + } + + return pixels; +} + +#ifndef QOI_NO_STDIO +#include + +int qoi_write(const char *filename, const void *data, const qoi_desc *desc) { + FILE *f = fopen(filename, "wb"); + int size, err; + void *encoded; + + if (!f) { + return 0; + } + + encoded = qoi_encode(data, desc, &size); + if (!encoded) { + fclose(f); + return 0; + } + + fwrite(encoded, 1, size, f); + fflush(f); + err = ferror(f); + fclose(f); + + QOI_FREE(encoded); + return err ? 0 : size; +} + +void *qoi_read(const char *filename, qoi_desc *desc, int channels) { + FILE *f = fopen(filename, "rb"); + int size, bytes_read; + void *pixels, *data; + + if (!f) { + return NULL; + } + + fseek(f, 0, SEEK_END); + size = ftell(f); + if (size <= 0 || fseek(f, 0, SEEK_SET) != 0) { + fclose(f); + return NULL; + } + + data = QOI_MALLOC(size); + if (!data) { + fclose(f); + return NULL; + } + + bytes_read = fread(data, 1, size, f); + fclose(f); + pixels = (bytes_read != size) ? NULL : qoi_decode(data, bytes_read, desc, channels); + QOI_FREE(data); + return pixels; +} + +#endif /* QOI_NO_STDIO */ +#endif /* QOI_IMPLEMENTATION */ diff --git a/functions/compression/stb_image_write.h b/functions/compression/stb_image_write.h new file mode 100644 index 0000000..738e3c2 --- /dev/null +++ b/functions/compression/stb_image_write.h @@ -0,0 +1,2055 @@ +/* stb_image_write - v1.16 - public domain - http://nothings.org/stb + writes out PNG/BMP/TGA/JPEG/HDR images to C stdio - Sean Barrett 2010-2015 + no warranty implied; use at your own risk + + Before #including, + + #define STB_IMAGE_WRITE_IMPLEMENTATION + + in the file that you want to have the implementation. + + Will probably not work correctly with strict-aliasing optimizations. + +ABOUT: + + This header file is a library for writing images to C stdio or a callback. + + The PNG output is not optimal; it is 20-50% larger than the file + written by a decent optimizing implementation; though providing a custom + zlib compress function (see STBIW_ZLIB_COMPRESS) can mitigate that. + This library is designed for source code compactness and simplicity, + not optimal image file size or run-time performance. + +BUILDING: + + You can #define STBIW_ASSERT(x) before the #include to avoid using assert.h. + You can #define STBIW_MALLOC(), STBIW_REALLOC(), and STBIW_FREE() to replace + malloc,realloc,free. + You can #define STBIW_MEMMOVE() to replace memmove() + You can #define STBIW_ZLIB_COMPRESS to use a custom zlib-style compress +function for PNG compression (instead of the builtin one), it must have the +following signature: unsigned char * my_compress(unsigned char *data, int +data_len, int *out_len, int quality); The returned data will be freed with +STBIW_FREE() (free() by default), so it must be heap allocated with +STBIW_MALLOC() (malloc() by default), + +UNICODE: + + If compiling for Windows and you wish to use Unicode filenames, compile + with + #define STBIW_WINDOWS_UTF8 + and pass utf8-encoded filenames. Call stbiw_convert_wchar_to_utf8 to convert + Windows wchar_t filenames to utf8. + +USAGE: + + There are five functions, one for each image file format: + + int stbi_write_png(char const *filename, int w, int h, int comp, const void +*data, int stride_in_bytes); int stbi_write_bmp(char const *filename, int w, int +h, int comp, const void *data); int stbi_write_tga(char const *filename, int w, +int h, int comp, const void *data); int stbi_write_jpg(char const *filename, int +w, int h, int comp, const void *data, int quality); int stbi_write_hdr(char +const *filename, int w, int h, int comp, const float *data); + + void stbi_flip_vertically_on_write(int flag); // flag is non-zero to flip +data vertically + + There are also five equivalent functions that use an arbitrary write +function. You are expected to open/close your file-equivalent before and after +calling these: + + int stbi_write_png_to_func(stbi_write_func *func, void *context, int w, int +h, int comp, const void *data, int stride_in_bytes); int +stbi_write_bmp_to_func(stbi_write_func *func, void *context, int w, int h, int +comp, const void *data); int stbi_write_tga_to_func(stbi_write_func *func, void +*context, int w, int h, int comp, const void *data); int +stbi_write_hdr_to_func(stbi_write_func *func, void *context, int w, int h, int +comp, const float *data); int stbi_write_jpg_to_func(stbi_write_func *func, void +*context, int x, int y, int comp, const void *data, int quality); + + where the callback is: + void stbi_write_func(void *context, void *data, int size); + + You can configure it with these global variables: + int stbi_write_tga_with_rle; // defaults to true; set to 0 to +disable RLE int stbi_write_png_compression_level; // defaults to 8; set to +higher for more compression int stbi_write_force_png_filter; // defaults +to -1; set to 0..5 to force a filter mode + + + You can define STBI_WRITE_NO_STDIO to disable the file variant of these + functions, so the library will not use stdio.h at all. However, this will + also disable HDR writing, because it requires stdio for formatted output. + + Each function returns 0 on failure and non-0 on success. + + The functions create an image file defined by the parameters. The image + is a rectangle of pixels stored from left-to-right, top-to-bottom. + Each pixel contains 'comp' channels of data stored interleaved with 8-bits + per channel, in the following order: 1=Y, 2=YA, 3=RGB, 4=RGBA. (Y is + monochrome color.) The rectangle is 'w' pixels wide and 'h' pixels tall. + The *data pointer points to the first byte of the top-left-most pixel. + For PNG, "stride_in_bytes" is the distance in bytes from the first byte of + a row of pixels to the first byte of the next row of pixels. + + PNG creates output files with the same number of components as the input. + The BMP format expands Y to RGB in the file format and does not + output alpha. + + PNG supports writing rectangles of data even when the bytes storing rows of + data are not consecutive in memory (e.g. sub-rectangles of a larger image), + by supplying the stride between the beginning of adjacent rows. The other + formats do not. (Thus you cannot write a native-format BMP through the BMP + writer, both because it is in BGR order and because it may have padding + at the end of the line.) + + PNG allows you to set the deflate compression level by setting the global + variable 'stbi_write_png_compression_level' (it defaults to 8). + + HDR expects linear float data. Since the format is always 32-bit rgb(e) + data, alpha (if provided) is discarded, and for monochrome data it is + replicated across all three channels. + + TGA supports RLE or non-RLE compressed data. To use non-RLE-compressed + data, set the global variable 'stbi_write_tga_with_rle' to 0. + + JPEG does ignore alpha channels in input data; quality is between 1 and 100. + Higher quality looks better but results in a bigger image. + JPEG baseline (no JPEG progressive). + +CREDITS: + + + Sean Barrett - PNG/BMP/TGA + Baldur Karlsson - HDR + Jean-Sebastien Guay - TGA monochrome + Tim Kelsey - misc enhancements + Alan Hickman - TGA RLE + Emmanuel Julien - initial file IO callback implementation + Jon Olick - original jo_jpeg.cpp code + Daniel Gibson - integrate JPEG, allow external zlib + Aarni Koskela - allow choosing PNG filter + + bugfixes: + github:Chribba + Guillaume Chereau + github:jry2 + github:romigrou + Sergio Gonzalez + Jonas Karlsson + Filip Wasil + Thatcher Ulrich + github:poppolopoppo + Patrick Boettcher + github:xeekworx + Cap Petschulat + Simon Rodriguez + Ivan Tikhonov + github:ignotion + Adam Schackart + Andrew Kensler + +LICENSE + + See end of file for license information. + +*/ + +#ifndef INCLUDE_STB_IMAGE_WRITE_H +#define INCLUDE_STB_IMAGE_WRITE_H + +#include + +// if STB_IMAGE_WRITE_STATIC causes problems, try defining STBIWDEF to 'inline' +// or 'static inline' +#ifndef STBIWDEF +#ifdef STB_IMAGE_WRITE_STATIC +#define STBIWDEF static +#else +#ifdef __cplusplus +#define STBIWDEF extern "C" +#else +#define STBIWDEF extern +#endif +#endif +#endif + +#ifndef STB_IMAGE_WRITE_STATIC // C++ forbids static forward declarations +STBIWDEF int stbi_write_tga_with_rle; +STBIWDEF int stbi_write_png_compression_level; +STBIWDEF int stbi_write_force_png_filter; +#endif + +#ifndef STBI_WRITE_NO_STDIO +STBIWDEF int stbi_write_png(char const *filename, int w, int h, int comp, + const void *data, int stride_in_bytes); +STBIWDEF int stbi_write_bmp(char const *filename, int w, int h, int comp, + const void *data); +STBIWDEF int stbi_write_tga(char const *filename, int w, int h, int comp, + const void *data); +STBIWDEF int stbi_write_hdr(char const *filename, int w, int h, int comp, + const float *data); +STBIWDEF int stbi_write_jpg(char const *filename, int x, int y, int comp, + const void *data, int quality); + +#ifdef STBIW_WINDOWS_UTF8 +STBIWDEF int stbiw_convert_wchar_to_utf8(char *buffer, size_t bufferlen, + const wchar_t *input); +#endif +#endif + +typedef void stbi_write_func(void *context, void *data, int size); + +STBIWDEF int stbi_write_png_to_func(stbi_write_func *func, void *context, int w, + int h, int comp, const void *data, + int stride_in_bytes); +STBIWDEF int stbi_write_bmp_to_func(stbi_write_func *func, void *context, int w, + int h, int comp, const void *data); +STBIWDEF int stbi_write_tga_to_func(stbi_write_func *func, void *context, int w, + int h, int comp, const void *data); +STBIWDEF int stbi_write_hdr_to_func(stbi_write_func *func, void *context, int w, + int h, int comp, const float *data); +STBIWDEF int stbi_write_jpg_to_func(stbi_write_func *func, void *context, int x, + int y, int comp, const void *data, + int quality); + +STBIWDEF void stbi_flip_vertically_on_write(int flip_boolean); + +#endif // INCLUDE_STB_IMAGE_WRITE_H + +#ifdef STB_IMAGE_WRITE_IMPLEMENTATION + +#ifdef _WIN32 +#ifndef _CRT_SECURE_NO_WARNINGS +#define _CRT_SECURE_NO_WARNINGS +#endif +#ifndef _CRT_NONSTDC_NO_DEPRECATE +#define _CRT_NONSTDC_NO_DEPRECATE +#endif +#endif + +#ifndef STBI_WRITE_NO_STDIO +#include +#endif // STBI_WRITE_NO_STDIO + +#include +#include +#include +#include + +#if defined(STBIW_MALLOC) && defined(STBIW_FREE) && \ + (defined(STBIW_REALLOC) || defined(STBIW_REALLOC_SIZED)) +// ok +#elif !defined(STBIW_MALLOC) && !defined(STBIW_FREE) && \ + !defined(STBIW_REALLOC) && !defined(STBIW_REALLOC_SIZED) +// ok +#else +#error \ + "Must define all or none of STBIW_MALLOC, STBIW_FREE, and STBIW_REALLOC (or STBIW_REALLOC_SIZED)." +#endif + +#ifndef STBIW_MALLOC +#define STBIW_MALLOC(sz) malloc(sz) +#define STBIW_REALLOC(p, newsz) realloc(p, newsz) +#define STBIW_FREE(p) free(p) +#endif + +#ifndef STBIW_REALLOC_SIZED +#define STBIW_REALLOC_SIZED(p, oldsz, newsz) STBIW_REALLOC(p, newsz) +#endif + +#ifndef STBIW_MEMMOVE +#define STBIW_MEMMOVE(a, b, sz) memmove(a, b, sz) +#endif + +#ifndef STBIW_ASSERT +#include +#define STBIW_ASSERT(x) assert(x) +#endif + +#define STBIW_UCHAR(x) (unsigned char)((x) & 0xff) + +#ifdef STB_IMAGE_WRITE_STATIC +static int stbi_write_png_compression_level = 8; +static int stbi_write_tga_with_rle = 1; +static int stbi_write_force_png_filter = -1; +#else +int stbi_write_png_compression_level = 8; +int stbi_write_tga_with_rle = 1; +int stbi_write_force_png_filter = -1; +#endif + +static int stbi__flip_vertically_on_write = 0; + +STBIWDEF void stbi_flip_vertically_on_write(int flag) { + stbi__flip_vertically_on_write = flag; +} + +typedef struct { + stbi_write_func *func; + void *context; + unsigned char buffer[64]; + int buf_used; +} stbi__write_context; + +// initialize a callback-based context +static void stbi__start_write_callbacks(stbi__write_context *s, + stbi_write_func *c, void *context) { + s->func = c; + s->context = context; +} + +#ifndef STBI_WRITE_NO_STDIO + +static void stbi__stdio_write(void *context, void *data, int size) { + fwrite(data, 1, size, (FILE *)context); +} + +#if defined(_WIN32) && defined(STBIW_WINDOWS_UTF8) +#ifdef __cplusplus +#define STBIW_EXTERN extern "C" +#else +#define STBIW_EXTERN extern +#endif +STBIW_EXTERN __declspec(dllimport) int __stdcall MultiByteToWideChar( + unsigned int cp, unsigned long flags, const char *str, int cbmb, + wchar_t *widestr, int cchwide); +STBIW_EXTERN __declspec(dllimport) int __stdcall WideCharToMultiByte( + unsigned int cp, unsigned long flags, const wchar_t *widestr, int cchwide, + char *str, int cbmb, const char *defchar, int *used_default); + +STBIWDEF int stbiw_convert_wchar_to_utf8(char *buffer, size_t bufferlen, + const wchar_t *input) { + return WideCharToMultiByte(65001 /* UTF8 */, 0, input, -1, buffer, + (int)bufferlen, NULL, NULL); +} +#endif + +static FILE *stbiw__fopen(char const *filename, char const *mode) { + FILE *f; +#if defined(_WIN32) && defined(STBIW_WINDOWS_UTF8) + wchar_t wMode[64]; + wchar_t wFilename[1024]; + if (0 == MultiByteToWideChar(65001 /* UTF8 */, 0, filename, -1, wFilename, + sizeof(wFilename) / sizeof(*wFilename))) + return 0; + + if (0 == MultiByteToWideChar(65001 /* UTF8 */, 0, mode, -1, wMode, + sizeof(wMode) / sizeof(*wMode))) + return 0; + +#if defined(_MSC_VER) && _MSC_VER >= 1400 + if (0 != _wfopen_s(&f, wFilename, wMode)) + f = 0; +#else + f = _wfopen(wFilename, wMode); +#endif + +#elif defined(_MSC_VER) && _MSC_VER >= 1400 + if (0 != fopen_s(&f, filename, mode)) + f = 0; +#else + f = fopen(filename, mode); +#endif + return f; +} + +static int stbi__start_write_file(stbi__write_context *s, + const char *filename) { + FILE *f = stbiw__fopen(filename, "wb"); + stbi__start_write_callbacks(s, stbi__stdio_write, (void *)f); + return f != NULL; +} + +static void stbi__end_write_file(stbi__write_context *s) { + fclose((FILE *)s->context); +} + +#endif // !STBI_WRITE_NO_STDIO + +typedef unsigned int stbiw_uint32; +typedef int stb_image_write_test[sizeof(stbiw_uint32) == 4 ? 1 : -1]; + +static void stbiw__writefv(stbi__write_context *s, const char *fmt, va_list v) { + while (*fmt) { + switch (*fmt++) { + case ' ': + break; + case '1': { + unsigned char x = STBIW_UCHAR(va_arg(v, int)); + s->func(s->context, &x, 1); + break; + } + case '2': { + int x = va_arg(v, int); + unsigned char b[2]; + b[0] = STBIW_UCHAR(x); + b[1] = STBIW_UCHAR(x >> 8); + s->func(s->context, b, 2); + break; + } + case '4': { + stbiw_uint32 x = va_arg(v, int); + unsigned char b[4]; + b[0] = STBIW_UCHAR(x); + b[1] = STBIW_UCHAR(x >> 8); + b[2] = STBIW_UCHAR(x >> 16); + b[3] = STBIW_UCHAR(x >> 24); + s->func(s->context, b, 4); + break; + } + default: + STBIW_ASSERT(0); + return; + } + } +} + +static void stbiw__writef(stbi__write_context *s, const char *fmt, ...) { + va_list v; + va_start(v, fmt); + stbiw__writefv(s, fmt, v); + va_end(v); +} + +static void stbiw__write_flush(stbi__write_context *s) { + if (s->buf_used) { + s->func(s->context, &s->buffer, s->buf_used); + s->buf_used = 0; + } +} + +static void stbiw__putc(stbi__write_context *s, unsigned char c) { + s->func(s->context, &c, 1); +} + +static void stbiw__write1(stbi__write_context *s, unsigned char a) { + if ((size_t)s->buf_used + 1 > sizeof(s->buffer)) + stbiw__write_flush(s); + s->buffer[s->buf_used++] = a; +} + +static void stbiw__write3(stbi__write_context *s, unsigned char a, + unsigned char b, unsigned char c) { + int n; + if ((size_t)s->buf_used + 3 > sizeof(s->buffer)) + stbiw__write_flush(s); + n = s->buf_used; + s->buf_used = n + 3; + s->buffer[n + 0] = a; + s->buffer[n + 1] = b; + s->buffer[n + 2] = c; +} + +static void stbiw__write_pixel(stbi__write_context *s, int rgb_dir, int comp, + int write_alpha, int expand_mono, + unsigned char *d) { + unsigned char bg[3] = {255, 0, 255}, px[3]; + int k; + + if (write_alpha < 0) + stbiw__write1(s, d[comp - 1]); + + switch (comp) { + case 2: // 2 pixels = mono + alpha, alpha is written separately, so same as + // 1-channel case + case 1: + if (expand_mono) + stbiw__write3(s, d[0], d[0], d[0]); // monochrome bmp + else + stbiw__write1(s, d[0]); // monochrome TGA + break; + case 4: + if (!write_alpha) { + // composite against pink background + for (k = 0; k < 3; ++k) + px[k] = bg[k] + ((d[k] - bg[k]) * d[3]) / 255; + stbiw__write3(s, px[1 - rgb_dir], px[1], px[1 + rgb_dir]); + break; + } + /* FALLTHROUGH */ + case 3: + stbiw__write3(s, d[1 - rgb_dir], d[1], d[1 + rgb_dir]); + break; + } + if (write_alpha > 0) + stbiw__write1(s, d[comp - 1]); +} + +static void stbiw__write_pixels(stbi__write_context *s, int rgb_dir, int vdir, + int x, int y, int comp, void *data, + int write_alpha, int scanline_pad, + int expand_mono) { + stbiw_uint32 zero = 0; + int i, j, j_end; + + if (y <= 0) + return; + + if (stbi__flip_vertically_on_write) + vdir *= -1; + + if (vdir < 0) { + j_end = -1; + j = y - 1; + } else { + j_end = y; + j = 0; + } + + for (; j != j_end; j += vdir) { + for (i = 0; i < x; ++i) { + unsigned char *d = (unsigned char *)data + (j * x + i) * comp; + stbiw__write_pixel(s, rgb_dir, comp, write_alpha, expand_mono, d); + } + stbiw__write_flush(s); + s->func(s->context, &zero, scanline_pad); + } +} + +static int stbiw__outfile(stbi__write_context *s, int rgb_dir, int vdir, int x, + int y, int comp, int expand_mono, void *data, + int alpha, int pad, const char *fmt, ...) { + if (y < 0 || x < 0) { + return 0; + } else { + va_list v; + va_start(v, fmt); + stbiw__writefv(s, fmt, v); + va_end(v); + stbiw__write_pixels(s, rgb_dir, vdir, x, y, comp, data, alpha, pad, + expand_mono); + return 1; + } +} + +static int stbi_write_bmp_core(stbi__write_context *s, int x, int y, int comp, + const void *data) { + if (comp != 4) { + // write RGB bitmap + int pad = (-x * 3) & 3; + return stbiw__outfile(s, -1, -1, x, y, comp, 1, (void *)data, 0, pad, + "11 4 22 4" + "4 44 22 444444", + 'B', 'M', 14 + 40 + (x * 3 + pad) * y, 0, 0, + 14 + 40, // file header + 40, x, y, 1, 24, 0, 0, 0, 0, 0, 0); // bitmap header + } else { + // RGBA bitmaps need a v4 header + // use BI_BITFIELDS mode with 32bpp and alpha mask + // (straight BI_RGB with alpha mask doesn't work in most readers) + return stbiw__outfile( + s, -1, -1, x, y, comp, 1, (void *)data, 1, 0, + "11 4 22 4" + "4 44 22 444444 4444 4 444 444 444 444", + 'B', 'M', 14 + 108 + x * y * 4, 0, 0, 14 + 108, // file header + 108, x, y, 1, 32, 3, 0, 0, 0, 0, 0, 0xff0000, 0xff00, 0xff, 0xff000000u, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); // bitmap V4 header + } +} + +STBIWDEF int stbi_write_bmp_to_func(stbi_write_func *func, void *context, int x, + int y, int comp, const void *data) { + stbi__write_context s = {0}; + stbi__start_write_callbacks(&s, func, context); + return stbi_write_bmp_core(&s, x, y, comp, data); +} + +#ifndef STBI_WRITE_NO_STDIO +STBIWDEF int stbi_write_bmp(char const *filename, int x, int y, int comp, + const void *data) { + stbi__write_context s = {0}; + if (stbi__start_write_file(&s, filename)) { + int r = stbi_write_bmp_core(&s, x, y, comp, data); + stbi__end_write_file(&s); + return r; + } else + return 0; +} +#endif //! STBI_WRITE_NO_STDIO + +static int stbi_write_tga_core(stbi__write_context *s, int x, int y, int comp, + void *data) { + int has_alpha = (comp == 2 || comp == 4); + int colorbytes = has_alpha ? comp - 1 : comp; + int format = + colorbytes < 2 + ? 3 + : 2; // 3 color channels (RGB/RGBA) = 2, 1 color channel (Y/YA) = 3 + + if (y < 0 || x < 0) + return 0; + + if (!stbi_write_tga_with_rle) { + return stbiw__outfile(s, -1, -1, x, y, comp, 0, (void *)data, has_alpha, 0, + "111 221 2222 11", 0, 0, format, 0, 0, 0, 0, 0, x, y, + (colorbytes + has_alpha) * 8, has_alpha * 8); + } else { + int i, j, k; + int jend, jdir; + + stbiw__writef(s, "111 221 2222 11", 0, 0, format + 8, 0, 0, 0, 0, 0, x, y, + (colorbytes + has_alpha) * 8, has_alpha * 8); + + if (stbi__flip_vertically_on_write) { + j = 0; + jend = y; + jdir = 1; + } else { + j = y - 1; + jend = -1; + jdir = -1; + } + for (; j != jend; j += jdir) { + unsigned char *row = (unsigned char *)data + j * x * comp; + int len; + + for (i = 0; i < x; i += len) { + unsigned char *begin = row + i * comp; + int diff = 1; + len = 1; + + if (i < x - 1) { + ++len; + diff = memcmp(begin, row + (i + 1) * comp, comp); + if (diff) { + const unsigned char *prev = begin; + for (k = i + 2; k < x && len < 128; ++k) { + if (memcmp(prev, row + k * comp, comp)) { + prev += comp; + ++len; + } else { + --len; + break; + } + } + } else { + for (k = i + 2; k < x && len < 128; ++k) { + if (!memcmp(begin, row + k * comp, comp)) { + ++len; + } else { + break; + } + } + } + } + + if (diff) { + unsigned char header = STBIW_UCHAR(len - 1); + stbiw__write1(s, header); + for (k = 0; k < len; ++k) { + stbiw__write_pixel(s, -1, comp, has_alpha, 0, begin + k * comp); + } + } else { + unsigned char header = STBIW_UCHAR(len - 129); + stbiw__write1(s, header); + stbiw__write_pixel(s, -1, comp, has_alpha, 0, begin); + } + } + } + stbiw__write_flush(s); + } + return 1; +} + +// STBIWDEF int stbi_write_tga_to_func(stbi_write_func *func, void *context, int +// x, int y, int comp, const void *data) +// { +// stbi__write_context s = { 0 }; +// stbi__start_write_callbacks(&s, func, context); +// return stbi_write_tga_core(&s, x, y, comp, (void *) data); +// } + +#ifndef STBI_WRITE_NO_STDIO +STBIWDEF int stbi_write_tga(char const *filename, int x, int y, int comp, + const void *data) { + stbi__write_context s = {0}; + if (stbi__start_write_file(&s, filename)) { + int r = stbi_write_tga_core(&s, x, y, comp, (void *)data); + stbi__end_write_file(&s); + return r; + } else + return 0; +} +#endif + +// ************************************************************************************************* +// Radiance RGBE HDR writer +// by Baldur Karlsson + +#define stbiw__max(a, b) ((a) > (b) ? (a) : (b)) + +#ifndef STBI_WRITE_NO_STDIO + +static void stbiw__linear_to_rgbe(unsigned char *rgbe, float *linear) { + int exponent; + float maxcomp = stbiw__max(linear[0], stbiw__max(linear[1], linear[2])); + + if (maxcomp < 1e-32f) { + rgbe[0] = rgbe[1] = rgbe[2] = rgbe[3] = 0; + } else { + float normalize = (float)frexp(maxcomp, &exponent) * 256.0f / maxcomp; + + rgbe[0] = (unsigned char)(linear[0] * normalize); + rgbe[1] = (unsigned char)(linear[1] * normalize); + rgbe[2] = (unsigned char)(linear[2] * normalize); + rgbe[3] = (unsigned char)(exponent + 128); + } +} + +static void stbiw__write_run_data(stbi__write_context *s, int length, + unsigned char databyte) { + unsigned char lengthbyte = STBIW_UCHAR(length + 128); + STBIW_ASSERT(length + 128 <= 255); + s->func(s->context, &lengthbyte, 1); + s->func(s->context, &databyte, 1); +} + +static void stbiw__write_dump_data(stbi__write_context *s, int length, + unsigned char *data) { + unsigned char lengthbyte = STBIW_UCHAR(length); + STBIW_ASSERT(length <= + 128); // inconsistent with spec but consistent with official code + s->func(s->context, &lengthbyte, 1); + s->func(s->context, data, length); +} + +static void stbiw__write_hdr_scanline(stbi__write_context *s, int width, + int ncomp, unsigned char *scratch, + float *scanline) { + unsigned char scanlineheader[4] = {2, 2, 0, 0}; + unsigned char rgbe[4]; + float linear[3]; + int x; + + scanlineheader[2] = (width & 0xff00) >> 8; + scanlineheader[3] = (width & 0x00ff); + + /* skip RLE for images too small or large */ + if (width < 8 || width >= 32768) { + for (x = 0; x < width; x++) { + switch (ncomp) { + case 4: /* fallthrough */ + case 3: + linear[2] = scanline[x * ncomp + 2]; + linear[1] = scanline[x * ncomp + 1]; + linear[0] = scanline[x * ncomp + 0]; + break; + default: + linear[0] = linear[1] = linear[2] = scanline[x * ncomp + 0]; + break; + } + stbiw__linear_to_rgbe(rgbe, linear); + s->func(s->context, rgbe, 4); + } + } else { + int c, r; + /* encode into scratch buffer */ + for (x = 0; x < width; x++) { + switch (ncomp) { + case 4: /* fallthrough */ + case 3: + linear[2] = scanline[x * ncomp + 2]; + linear[1] = scanline[x * ncomp + 1]; + linear[0] = scanline[x * ncomp + 0]; + break; + default: + linear[0] = linear[1] = linear[2] = scanline[x * ncomp + 0]; + break; + } + stbiw__linear_to_rgbe(rgbe, linear); + scratch[x + width * 0] = rgbe[0]; + scratch[x + width * 1] = rgbe[1]; + scratch[x + width * 2] = rgbe[2]; + scratch[x + width * 3] = rgbe[3]; + } + + s->func(s->context, scanlineheader, 4); + + /* RLE each component separately */ + for (c = 0; c < 4; c++) { + unsigned char *comp = &scratch[width * c]; + + x = 0; + while (x < width) { + // find first run + r = x; + while (r + 2 < width) { + if (comp[r] == comp[r + 1] && comp[r] == comp[r + 2]) + break; + ++r; + } + if (r + 2 >= width) + r = width; + // dump up to first run + while (x < r) { + int len = r - x; + if (len > 128) + len = 128; + stbiw__write_dump_data(s, len, &comp[x]); + x += len; + } + // if there's a run, output it + if (r + 2 < width) { // same test as what we break out of in search + // loop, so only true if we break'd + // find next byte after run + while (r < width && comp[r] == comp[x]) + ++r; + // output run up to r + while (x < r) { + int len = r - x; + if (len > 127) + len = 127; + stbiw__write_run_data(s, len, comp[x]); + x += len; + } + } + } + } + } +} + +static int stbi_write_hdr_core(stbi__write_context *s, int x, int y, int comp, + float *data) { + if (y <= 0 || x <= 0 || data == NULL) + return 0; + else { + // Each component is stored separately. Allocate scratch space for full + // output scanline. + unsigned char *scratch = (unsigned char *)STBIW_MALLOC(x * 4); + int i, len; + char buffer[128]; + char header[] = + "#?RADIANCE\n# Written by stb_image_write.h\nFORMAT=32-bit_rle_rgbe\n"; + s->func(s->context, header, sizeof(header) - 1); + +#ifdef __STDC_LIB_EXT1__ + len = + sprintf_s(buffer, sizeof(buffer), + "EXPOSURE= 1.0000000000000\n\n-Y %d +X %d\n", y, x); +#else + len = sprintf(buffer, "EXPOSURE= 1.0000000000000\n\n-Y %d +X %d\n", + y, x); +#endif + s->func(s->context, buffer, len); + + for (i = 0; i < y; i++) + stbiw__write_hdr_scanline( + s, x, comp, scratch, + data + comp * x * (stbi__flip_vertically_on_write ? y - 1 - i : i)); + STBIW_FREE(scratch); + return 1; + } +} + +STBIWDEF int stbi_write_hdr_to_func(stbi_write_func *func, void *context, int x, + int y, int comp, const float *data) { + stbi__write_context s = {0}; + stbi__start_write_callbacks(&s, func, context); + return stbi_write_hdr_core(&s, x, y, comp, (float *)data); +} + +STBIWDEF int stbi_write_hdr(char const *filename, int x, int y, int comp, + const float *data) { + stbi__write_context s = {0}; + if (stbi__start_write_file(&s, filename)) { + int r = stbi_write_hdr_core(&s, x, y, comp, (float *)data); + stbi__end_write_file(&s); + return r; + } else + return 0; +} +#endif // STBI_WRITE_NO_STDIO + +////////////////////////////////////////////////////////////////////////////// +// +// PNG writer +// + +#ifndef STBIW_ZLIB_COMPRESS +// stretchy buffer; stbiw__sbpush() == vector<>::push_back() -- stbiw__sbcount() +// == vector<>::size() +#define stbiw__sbraw(a) ((int *)(void *)(a) - 2) +#define stbiw__sbm(a) stbiw__sbraw(a)[0] +#define stbiw__sbn(a) stbiw__sbraw(a)[1] + +#define stbiw__sbneedgrow(a, n) ((a) == 0 || stbiw__sbn(a) + n >= stbiw__sbm(a)) +#define stbiw__sbmaybegrow(a, n) \ + (stbiw__sbneedgrow(a, (n)) ? stbiw__sbgrow(a, n) : 0) +#define stbiw__sbgrow(a, n) stbiw__sbgrowf((void **)&(a), (n), sizeof(*(a))) + +#define stbiw__sbpush(a, v) \ + (stbiw__sbmaybegrow(a, 1), (a)[stbiw__sbn(a)++] = (v)) +#define stbiw__sbcount(a) ((a) ? stbiw__sbn(a) : 0) +#define stbiw__sbfree(a) ((a) ? STBIW_FREE(stbiw__sbraw(a)), 0 : 0) + +static void *stbiw__sbgrowf(void **arr, int increment, int itemsize) { + int m = *arr ? 2 * stbiw__sbm(*arr) + increment : increment + 1; + void *p = STBIW_REALLOC_SIZED( + *arr ? stbiw__sbraw(*arr) : 0, + *arr ? (stbiw__sbm(*arr) * itemsize + sizeof(int) * 2) : 0, + itemsize * m + sizeof(int) * 2); + STBIW_ASSERT(p); + if (p) { + if (!*arr) + ((int *)p)[1] = 0; + *arr = (void *)((int *)p + 2); + stbiw__sbm(*arr) = m; + } + return *arr; +} + +static unsigned char *stbiw__zlib_flushf(unsigned char *data, + unsigned int *bitbuffer, + int *bitcount) { + while (*bitcount >= 8) { + stbiw__sbpush(data, STBIW_UCHAR(*bitbuffer)); + *bitbuffer >>= 8; + *bitcount -= 8; + } + return data; +} + +static int stbiw__zlib_bitrev(int code, int codebits) { + int res = 0; + while (codebits--) { + res = (res << 1) | (code & 1); + code >>= 1; + } + return res; +} + +static unsigned int stbiw__zlib_countm(unsigned char *a, unsigned char *b, + int limit) { + int i; + for (i = 0; i < limit && i < 258; ++i) + if (a[i] != b[i]) + break; + return i; +} + +static unsigned int stbiw__zhash(unsigned char *data) { + stbiw_uint32 hash = data[0] + (data[1] << 8) + (data[2] << 16); + hash ^= hash << 3; + hash += hash >> 5; + hash ^= hash << 4; + hash += hash >> 17; + hash ^= hash << 25; + hash += hash >> 6; + return hash; +} + +#define stbiw__zlib_flush() (out = stbiw__zlib_flushf(out, &bitbuf, &bitcount)) +#define stbiw__zlib_add(code, codebits) \ + (bitbuf |= (code) << bitcount, bitcount += (codebits), stbiw__zlib_flush()) +#define stbiw__zlib_huffa(b, c) stbiw__zlib_add(stbiw__zlib_bitrev(b, c), c) +// default huffman tables +#define stbiw__zlib_huff1(n) stbiw__zlib_huffa(0x30 + (n), 8) +#define stbiw__zlib_huff2(n) stbiw__zlib_huffa(0x190 + (n) - 144, 9) +#define stbiw__zlib_huff3(n) stbiw__zlib_huffa(0 + (n) - 256, 7) +#define stbiw__zlib_huff4(n) stbiw__zlib_huffa(0xc0 + (n) - 280, 8) +#define stbiw__zlib_huff(n) \ + ((n) <= 143 ? stbiw__zlib_huff1(n) \ + : (n) <= 255 ? stbiw__zlib_huff2(n) \ + : (n) <= 279 ? stbiw__zlib_huff3(n) \ + : stbiw__zlib_huff4(n)) +#define stbiw__zlib_huffb(n) \ + ((n) <= 143 ? stbiw__zlib_huff1(n) : stbiw__zlib_huff2(n)) + +#define stbiw__ZHASH 16384 + +#endif // STBIW_ZLIB_COMPRESS + +STBIWDEF unsigned char *stbi_zlib_compress(unsigned char *data, int data_len, + int *out_len, int quality) { +#ifdef STBIW_ZLIB_COMPRESS + // user provided a zlib compress implementation, use that + return STBIW_ZLIB_COMPRESS(data, data_len, out_len, quality); +#else // use builtin + static unsigned short lengthc[] = { + 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 17, 19, 23, 27, + 31, 35, 43, 51, 59, 67, 83, 99, 115, 131, 163, 195, 227, 258, 259}; + static unsigned char lengtheb[] = {0, 0, 0, 0, 0, 0, 0, 0, 1, 1, + 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, + 4, 4, 4, 4, 5, 5, 5, 5, 0}; + static unsigned short distc[] = { + 1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, + 49, 65, 97, 129, 193, 257, 385, 513, 769, 1025, 1537, + 2049, 3073, 4097, 6145, 8193, 12289, 16385, 24577, 32768}; + static unsigned char disteb[] = {0, 0, 0, 0, 1, 1, 2, 2, 3, 3, + 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, + 9, 9, 10, 10, 11, 11, 12, 12, 13, 13}; + unsigned int bitbuf = 0; + int i, j, bitcount = 0; + unsigned char *out = NULL; + unsigned char ***hash_table = + (unsigned char ***)STBIW_MALLOC(stbiw__ZHASH * sizeof(unsigned char **)); + if (hash_table == NULL) + return NULL; + if (quality < 5) + quality = 5; + + stbiw__sbpush(out, 0x78); // DEFLATE 32K window + stbiw__sbpush(out, 0x5e); // FLEVEL = 1 + stbiw__zlib_add(1, 1); // BFINAL = 1 + stbiw__zlib_add(1, 2); // BTYPE = 1 -- fixed huffman + + for (i = 0; i < stbiw__ZHASH; ++i) + hash_table[i] = NULL; + + i = 0; + while (i < data_len - 3) { + // hash next 3 bytes of data to be compressed + int h = stbiw__zhash(data + i) & (stbiw__ZHASH - 1), best = 3; + unsigned char *bestloc = 0; + unsigned char **hlist = hash_table[h]; + int n = stbiw__sbcount(hlist); + for (j = 0; j < n; ++j) { + if (hlist[j] - data > i - 32768) { // if entry lies within window + int d = stbiw__zlib_countm(hlist[j], data + i, data_len - i); + if (d >= best) { + best = d; + bestloc = hlist[j]; + } + } + } + // when hash table entry is too long, delete half the entries + if (hash_table[h] && stbiw__sbn(hash_table[h]) == 2 * quality) { + STBIW_MEMMOVE(hash_table[h], hash_table[h] + quality, + sizeof(hash_table[h][0]) * quality); + stbiw__sbn(hash_table[h]) = quality; + } + stbiw__sbpush(hash_table[h], data + i); + + if (bestloc) { + // "lazy matching" - check match at *next* byte, and if it's better, do + // cur byte as literal + h = stbiw__zhash(data + i + 1) & (stbiw__ZHASH - 1); + hlist = hash_table[h]; + n = stbiw__sbcount(hlist); + for (j = 0; j < n; ++j) { + if (hlist[j] - data > i - 32767) { + int e = stbiw__zlib_countm(hlist[j], data + i + 1, data_len - i - 1); + if (e > best) { // if next match is better, bail on current match + bestloc = NULL; + break; + } + } + } + } + + if (bestloc) { + int d = (int)(data + i - bestloc); // distance back + STBIW_ASSERT(d <= 32767 && best <= 258); + for (j = 0; best > lengthc[j + 1] - 1; ++j) + ; + stbiw__zlib_huff(j + 257); + if (lengtheb[j]) + stbiw__zlib_add(best - lengthc[j], lengtheb[j]); + for (j = 0; d > distc[j + 1] - 1; ++j) + ; + stbiw__zlib_add(stbiw__zlib_bitrev(j, 5), 5); + if (disteb[j]) + stbiw__zlib_add(d - distc[j], disteb[j]); + i += best; + } else { + stbiw__zlib_huffb(data[i]); + ++i; + } + } + // write out final bytes + for (; i < data_len; ++i) + stbiw__zlib_huffb(data[i]); + stbiw__zlib_huff(256); // end of block + // pad with 0 bits to byte boundary + while (bitcount) + stbiw__zlib_add(0, 1); + + for (i = 0; i < stbiw__ZHASH; ++i) + (void)stbiw__sbfree(hash_table[i]); + STBIW_FREE(hash_table); + + // store uncompressed instead if compression was worse + if (stbiw__sbn(out) > data_len + 2 + ((data_len + 32766) / 32767) * 5) { + stbiw__sbn(out) = 2; // truncate to DEFLATE 32K window and FLEVEL = 1 + for (j = 0; j < data_len;) { + int blocklen = data_len - j; + if (blocklen > 32767) + blocklen = 32767; + stbiw__sbpush(out, + data_len - j == + blocklen); // BFINAL = ?, BTYPE = 0 -- no compression + stbiw__sbpush(out, STBIW_UCHAR(blocklen)); // LEN + stbiw__sbpush(out, STBIW_UCHAR(blocklen >> 8)); + stbiw__sbpush(out, STBIW_UCHAR(~blocklen)); // NLEN + stbiw__sbpush(out, STBIW_UCHAR(~blocklen >> 8)); + memcpy(out + stbiw__sbn(out), data + j, blocklen); + stbiw__sbn(out) += blocklen; + j += blocklen; + } + } + + { + // compute adler32 on input + unsigned int s1 = 1, s2 = 0; + int blocklen = (int)(data_len % 5552); + j = 0; + while (j < data_len) { + for (i = 0; i < blocklen; ++i) { + s1 += data[j + i]; + s2 += s1; + } + s1 %= 65521; + s2 %= 65521; + j += blocklen; + blocklen = 5552; + } + stbiw__sbpush(out, STBIW_UCHAR(s2 >> 8)); + stbiw__sbpush(out, STBIW_UCHAR(s2)); + stbiw__sbpush(out, STBIW_UCHAR(s1 >> 8)); + stbiw__sbpush(out, STBIW_UCHAR(s1)); + } + *out_len = stbiw__sbn(out); + // make returned pointer freeable + STBIW_MEMMOVE(stbiw__sbraw(out), out, *out_len); + return (unsigned char *)stbiw__sbraw(out); +#endif // STBIW_ZLIB_COMPRESS +} + +static unsigned int stbiw__crc32(unsigned char *buffer, int len) { +#ifdef STBIW_CRC32 + return STBIW_CRC32(buffer, len); +#else + static unsigned int crc_table[256] = { + 0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA, 0x076DC419, 0x706AF48F, + 0xE963A535, 0x9E6495A3, 0x0eDB8832, 0x79DCB8A4, 0xE0D5E91E, 0x97D2D988, + 0x09B64C2B, 0x7EB17CBD, 0xE7B82D07, 0x90BF1D91, 0x1DB71064, 0x6AB020F2, + 0xF3B97148, 0x84BE41DE, 0x1ADAD47D, 0x6DDDE4EB, 0xF4D4B551, 0x83D385C7, + 0x136C9856, 0x646BA8C0, 0xFD62F97A, 0x8A65C9EC, 0x14015C4F, 0x63066CD9, + 0xFA0F3D63, 0x8D080DF5, 0x3B6E20C8, 0x4C69105E, 0xD56041E4, 0xA2677172, + 0x3C03E4D1, 0x4B04D447, 0xD20D85FD, 0xA50AB56B, 0x35B5A8FA, 0x42B2986C, + 0xDBBBC9D6, 0xACBCF940, 0x32D86CE3, 0x45DF5C75, 0xDCD60DCF, 0xABD13D59, + 0x26D930AC, 0x51DE003A, 0xC8D75180, 0xBFD06116, 0x21B4F4B5, 0x56B3C423, + 0xCFBA9599, 0xB8BDA50F, 0x2802B89E, 0x5F058808, 0xC60CD9B2, 0xB10BE924, + 0x2F6F7C87, 0x58684C11, 0xC1611DAB, 0xB6662D3D, 0x76DC4190, 0x01DB7106, + 0x98D220BC, 0xEFD5102A, 0x71B18589, 0x06B6B51F, 0x9FBFE4A5, 0xE8B8D433, + 0x7807C9A2, 0x0F00F934, 0x9609A88E, 0xE10E9818, 0x7F6A0DBB, 0x086D3D2D, + 0x91646C97, 0xE6635C01, 0x6B6B51F4, 0x1C6C6162, 0x856530D8, 0xF262004E, + 0x6C0695ED, 0x1B01A57B, 0x8208F4C1, 0xF50FC457, 0x65B0D9C6, 0x12B7E950, + 0x8BBEB8EA, 0xFCB9887C, 0x62DD1DDF, 0x15DA2D49, 0x8CD37CF3, 0xFBD44C65, + 0x4DB26158, 0x3AB551CE, 0xA3BC0074, 0xD4BB30E2, 0x4ADFA541, 0x3DD895D7, + 0xA4D1C46D, 0xD3D6F4FB, 0x4369E96A, 0x346ED9FC, 0xAD678846, 0xDA60B8D0, + 0x44042D73, 0x33031DE5, 0xAA0A4C5F, 0xDD0D7CC9, 0x5005713C, 0x270241AA, + 0xBE0B1010, 0xC90C2086, 0x5768B525, 0x206F85B3, 0xB966D409, 0xCE61E49F, + 0x5EDEF90E, 0x29D9C998, 0xB0D09822, 0xC7D7A8B4, 0x59B33D17, 0x2EB40D81, + 0xB7BD5C3B, 0xC0BA6CAD, 0xEDB88320, 0x9ABFB3B6, 0x03B6E20C, 0x74B1D29A, + 0xEAD54739, 0x9DD277AF, 0x04DB2615, 0x73DC1683, 0xE3630B12, 0x94643B84, + 0x0D6D6A3E, 0x7A6A5AA8, 0xE40ECF0B, 0x9309FF9D, 0x0A00AE27, 0x7D079EB1, + 0xF00F9344, 0x8708A3D2, 0x1E01F268, 0x6906C2FE, 0xF762575D, 0x806567CB, + 0x196C3671, 0x6E6B06E7, 0xFED41B76, 0x89D32BE0, 0x10DA7A5A, 0x67DD4ACC, + 0xF9B9DF6F, 0x8EBEEFF9, 0x17B7BE43, 0x60B08ED5, 0xD6D6A3E8, 0xA1D1937E, + 0x38D8C2C4, 0x4FDFF252, 0xD1BB67F1, 0xA6BC5767, 0x3FB506DD, 0x48B2364B, + 0xD80D2BDA, 0xAF0A1B4C, 0x36034AF6, 0x41047A60, 0xDF60EFC3, 0xA867DF55, + 0x316E8EEF, 0x4669BE79, 0xCB61B38C, 0xBC66831A, 0x256FD2A0, 0x5268E236, + 0xCC0C7795, 0xBB0B4703, 0x220216B9, 0x5505262F, 0xC5BA3BBE, 0xB2BD0B28, + 0x2BB45A92, 0x5CB36A04, 0xC2D7FFA7, 0xB5D0CF31, 0x2CD99E8B, 0x5BDEAE1D, + 0x9B64C2B0, 0xEC63F226, 0x756AA39C, 0x026D930A, 0x9C0906A9, 0xEB0E363F, + 0x72076785, 0x05005713, 0x95BF4A82, 0xE2B87A14, 0x7BB12BAE, 0x0CB61B38, + 0x92D28E9B, 0xE5D5BE0D, 0x7CDCEFB7, 0x0BDBDF21, 0x86D3D2D4, 0xF1D4E242, + 0x68DDB3F8, 0x1FDA836E, 0x81BE16CD, 0xF6B9265B, 0x6FB077E1, 0x18B74777, + 0x88085AE6, 0xFF0F6A70, 0x66063BCA, 0x11010B5C, 0x8F659EFF, 0xF862AE69, + 0x616BFFD3, 0x166CCF45, 0xA00AE278, 0xD70DD2EE, 0x4E048354, 0x3903B3C2, + 0xA7672661, 0xD06016F7, 0x4969474D, 0x3E6E77DB, 0xAED16A4A, 0xD9D65ADC, + 0x40DF0B66, 0x37D83BF0, 0xA9BCAE53, 0xDEBB9EC5, 0x47B2CF7F, 0x30B5FFE9, + 0xBDBDF21C, 0xCABAC28A, 0x53B39330, 0x24B4A3A6, 0xBAD03605, 0xCDD70693, + 0x54DE5729, 0x23D967BF, 0xB3667A2E, 0xC4614AB8, 0x5D681B02, 0x2A6F2B94, + 0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D}; + + unsigned int crc = ~0u; + int i; + for (i = 0; i < len; ++i) + crc = (crc >> 8) ^ crc_table[buffer[i] ^ (crc & 0xff)]; + return ~crc; +#endif +} + +#define stbiw__wpng4(o, a, b, c, d) \ + ((o)[0] = STBIW_UCHAR(a), (o)[1] = STBIW_UCHAR(b), (o)[2] = STBIW_UCHAR(c), \ + (o)[3] = STBIW_UCHAR(d), (o) += 4) +#define stbiw__wp32(data, v) \ + stbiw__wpng4(data, (v) >> 24, (v) >> 16, (v) >> 8, (v)); +#define stbiw__wptag(data, s) stbiw__wpng4(data, s[0], s[1], s[2], s[3]) + +static void stbiw__wpcrc(unsigned char **data, int len) { + unsigned int crc = stbiw__crc32(*data - len - 4, len + 4); + stbiw__wp32(*data, crc); +} + +static unsigned char stbiw__paeth(int a, int b, int c) { + int p = a + b - c, pa = abs(p - a), pb = abs(p - b), pc = abs(p - c); + if (pa <= pb && pa <= pc) + return STBIW_UCHAR(a); + if (pb <= pc) + return STBIW_UCHAR(b); + return STBIW_UCHAR(c); +} + +// @OPTIMIZE: provide an option that always forces left-predict or paeth predict +static void stbiw__encode_png_line(unsigned char *pixels, int stride_bytes, + int width, int height, int y, int n, + int filter_type, signed char *line_buffer) { + static int mapping[] = {0, 1, 2, 3, 4}; + static int firstmap[] = {0, 1, 0, 5, 6}; + int *mymap = (y != 0) ? mapping : firstmap; + int i; + int type = mymap[filter_type]; + unsigned char *z = + pixels + + stride_bytes * (stbi__flip_vertically_on_write ? height - 1 - y : y); + int signed_stride = + stbi__flip_vertically_on_write ? -stride_bytes : stride_bytes; + + if (type == 0) { + memcpy(line_buffer, z, width * n); + return; + } + + // first loop isn't optimized since it's just one pixel + for (i = 0; i < n; ++i) { + switch (type) { + case 1: + line_buffer[i] = z[i]; + break; + case 2: + line_buffer[i] = z[i] - z[i - signed_stride]; + break; + case 3: + line_buffer[i] = z[i] - (z[i - signed_stride] >> 1); + break; + case 4: + line_buffer[i] = + (signed char)(z[i] - stbiw__paeth(0, z[i - signed_stride], 0)); + break; + case 5: + line_buffer[i] = z[i]; + break; + case 6: + line_buffer[i] = z[i]; + break; + } + } + switch (type) { + case 1: + for (i = n; i < width * n; ++i) + line_buffer[i] = z[i] - z[i - n]; + break; + case 2: + for (i = n; i < width * n; ++i) + line_buffer[i] = z[i] - z[i - signed_stride]; + break; + case 3: + for (i = n; i < width * n; ++i) + line_buffer[i] = z[i] - ((z[i - n] + z[i - signed_stride]) >> 1); + break; + case 4: + for (i = n; i < width * n; ++i) + line_buffer[i] = z[i] - stbiw__paeth(z[i - n], z[i - signed_stride], + z[i - signed_stride - n]); + break; + case 5: + for (i = n; i < width * n; ++i) + line_buffer[i] = z[i] - (z[i - n] >> 1); + break; + case 6: + for (i = n; i < width * n; ++i) + line_buffer[i] = z[i] - stbiw__paeth(z[i - n], 0, 0); + break; + } +} + +STBIWDEF unsigned char *stbi_write_png_to_mem(const unsigned char *pixels, + int stride_bytes, int x, int y, + int n, int *out_len) { + int force_filter = stbi_write_force_png_filter; + int ctype[5] = {-1, 0, 4, 2, 6}; + unsigned char sig[8] = {137, 80, 78, 71, 13, 10, 26, 10}; + unsigned char *out, *o, *filt, *zlib; + signed char *line_buffer; + int j, zlen; + + if (stride_bytes == 0) + stride_bytes = x * n; + + if (force_filter >= 5) { + force_filter = -1; + } + + filt = (unsigned char *)STBIW_MALLOC((x * n + 1) * y); + if (!filt) + return 0; + line_buffer = (signed char *)STBIW_MALLOC(x * n); + if (!line_buffer) { + STBIW_FREE(filt); + return 0; + } + for (j = 0; j < y; ++j) { + int filter_type; + if (force_filter > -1) { + filter_type = force_filter; + stbiw__encode_png_line((unsigned char *)(pixels), stride_bytes, x, y, j, + n, force_filter, line_buffer); + } else { // Estimate the best filter by running through all of them: + int best_filter = 0, best_filter_val = 0x7fffffff, est, i; + for (filter_type = 0; filter_type < 5; filter_type++) { + stbiw__encode_png_line((unsigned char *)(pixels), stride_bytes, x, y, j, + n, filter_type, line_buffer); + + // Estimate the entropy of the line using this filter; the less, the + // better. + est = 0; + for (i = 0; i < x * n; ++i) { + est += abs((signed char)line_buffer[i]); + } + if (est < best_filter_val) { + best_filter_val = est; + best_filter = filter_type; + } + } + if (filter_type != best_filter) { + // If the last iteration already got us the best filter, don't redo it + stbiw__encode_png_line((unsigned char *)(pixels), stride_bytes, x, y, j, + n, best_filter, line_buffer); + filter_type = best_filter; + } + } + // when we get here, filter_type contains the filter type, and + // line_buffer contains the data + filt[j * (x * n + 1)] = (unsigned char)filter_type; + STBIW_MEMMOVE(filt + j * (x * n + 1) + 1, line_buffer, x * n); + } + STBIW_FREE(line_buffer); + zlib = stbi_zlib_compress(filt, y * (x * n + 1), &zlen, + stbi_write_png_compression_level); + STBIW_FREE(filt); + if (!zlib) + return 0; + + // each tag requires 12 bytes of overhead + out = (unsigned char *)STBIW_MALLOC(8 + 12 + 13 + 12 + zlen + 12); + if (!out) + return 0; + *out_len = 8 + 12 + 13 + 12 + zlen + 12; + + o = out; + STBIW_MEMMOVE(o, sig, 8); + o += 8; + stbiw__wp32(o, 13); // header length + stbiw__wptag(o, "IHDR"); + stbiw__wp32(o, x); + stbiw__wp32(o, y); + *o++ = 8; + *o++ = STBIW_UCHAR(ctype[n]); + *o++ = 0; + *o++ = 0; + *o++ = 0; + stbiw__wpcrc(&o, 13); + + stbiw__wp32(o, zlen); + stbiw__wptag(o, "IDAT"); + STBIW_MEMMOVE(o, zlib, zlen); + o += zlen; + STBIW_FREE(zlib); + stbiw__wpcrc(&o, zlen); + + stbiw__wp32(o, 0); + stbiw__wptag(o, "IEND"); + stbiw__wpcrc(&o, 0); + + STBIW_ASSERT(o == out + *out_len); + + return out; +} + +#ifndef STBI_WRITE_NO_STDIO +STBIWDEF int stbi_write_png(char const *filename, int x, int y, int comp, + const void *data, int stride_bytes) { + FILE *f; + int len; + unsigned char *png = stbi_write_png_to_mem((const unsigned char *)data, + stride_bytes, x, y, comp, &len); + if (png == NULL) + return 0; + + f = stbiw__fopen(filename, "wb"); + if (!f) { + STBIW_FREE(png); + return 0; + } + fwrite(png, 1, len, f); + fclose(f); + STBIW_FREE(png); + return 1; +} +#endif + +STBIWDEF int stbi_write_png_to_func(stbi_write_func *func, void *context, int x, + int y, int comp, const void *data, + int stride_bytes) { + int len; + unsigned char *png = stbi_write_png_to_mem((const unsigned char *)data, + stride_bytes, x, y, comp, &len); + if (png == NULL) + return 0; + func(context, png, len); + STBIW_FREE(png); + return 1; +} + +/* *************************************************************************** + * + * JPEG writer + * + * This is based on Jon Olick's jo_jpeg.cpp: + * public domain Simple, Minimalistic JPEG writer - + * http://www.jonolick.com/code.html + */ + +static const unsigned char stbiw__jpg_ZigZag[] = { + 0, 1, 5, 6, 14, 15, 27, 28, 2, 4, 7, 13, 16, 26, 29, 42, + 3, 8, 12, 17, 25, 30, 41, 43, 9, 11, 18, 24, 31, 40, 44, 53, + 10, 19, 23, 32, 39, 45, 52, 54, 20, 22, 33, 38, 46, 51, 55, 60, + 21, 34, 37, 47, 50, 56, 59, 61, 35, 36, 48, 49, 57, 58, 62, 63}; + +static void stbiw__jpg_writeBits(stbi__write_context *s, int *bitBufP, + int *bitCntP, const unsigned short *bs) { + int bitBuf = *bitBufP, bitCnt = *bitCntP; + bitCnt += bs[1]; + bitBuf |= bs[0] << (24 - bitCnt); + while (bitCnt >= 8) { + unsigned char c = (bitBuf >> 16) & 255; + stbiw__putc(s, c); + if (c == 255) { + stbiw__putc(s, 0); + } + bitBuf <<= 8; + bitCnt -= 8; + } + *bitBufP = bitBuf; + *bitCntP = bitCnt; +} + +static void stbiw__jpg_DCT(float *d0p, float *d1p, float *d2p, float *d3p, + float *d4p, float *d5p, float *d6p, float *d7p) { + float d0 = *d0p, d1 = *d1p, d2 = *d2p, d3 = *d3p, d4 = *d4p, d5 = *d5p, + d6 = *d6p, d7 = *d7p; + float z1, z2, z3, z4, z5, z11, z13; + + float tmp0 = d0 + d7; + float tmp7 = d0 - d7; + float tmp1 = d1 + d6; + float tmp6 = d1 - d6; + float tmp2 = d2 + d5; + float tmp5 = d2 - d5; + float tmp3 = d3 + d4; + float tmp4 = d3 - d4; + + // Even part + float tmp10 = tmp0 + tmp3; // phase 2 + float tmp13 = tmp0 - tmp3; + float tmp11 = tmp1 + tmp2; + float tmp12 = tmp1 - tmp2; + + d0 = tmp10 + tmp11; // phase 3 + d4 = tmp10 - tmp11; + + z1 = (tmp12 + tmp13) * 0.707106781f; // c4 + d2 = tmp13 + z1; // phase 5 + d6 = tmp13 - z1; + + // Odd part + tmp10 = tmp4 + tmp5; // phase 2 + tmp11 = tmp5 + tmp6; + tmp12 = tmp6 + tmp7; + + // The rotator is modified from fig 4-8 to avoid extra negations. + z5 = (tmp10 - tmp12) * 0.382683433f; // c6 + z2 = tmp10 * 0.541196100f + z5; // c2-c6 + z4 = tmp12 * 1.306562965f + z5; // c2+c6 + z3 = tmp11 * 0.707106781f; // c4 + + z11 = tmp7 + z3; // phase 5 + z13 = tmp7 - z3; + + *d5p = z13 + z2; // phase 6 + *d3p = z13 - z2; + *d1p = z11 + z4; + *d7p = z11 - z4; + + *d0p = d0; + *d2p = d2; + *d4p = d4; + *d6p = d6; +} + +static void stbiw__jpg_calcBits(int val, unsigned short bits[2]) { + int tmp1 = val < 0 ? -val : val; + val = val < 0 ? val - 1 : val; + bits[1] = 1; + while (tmp1 >>= 1) { + ++bits[1]; + } + bits[0] = val & ((1 << bits[1]) - 1); +} + +static int stbiw__jpg_processDU(stbi__write_context *s, int *bitBuf, + int *bitCnt, float *CDU, int du_stride, + float *fdtbl, int DC, + const unsigned short HTDC[256][2], + const unsigned short HTAC[256][2]) { + const unsigned short EOB[2] = {HTAC[0x00][0], HTAC[0x00][1]}; + const unsigned short M16zeroes[2] = {HTAC[0xF0][0], HTAC[0xF0][1]}; + int dataOff, i, j, n, diff, end0pos, x, y; + int DU[64]; + + // DCT rows + for (dataOff = 0, n = du_stride * 8; dataOff < n; dataOff += du_stride) { + stbiw__jpg_DCT(&CDU[dataOff], &CDU[dataOff + 1], &CDU[dataOff + 2], + &CDU[dataOff + 3], &CDU[dataOff + 4], &CDU[dataOff + 5], + &CDU[dataOff + 6], &CDU[dataOff + 7]); + } + // DCT columns + for (dataOff = 0; dataOff < 8; ++dataOff) { + stbiw__jpg_DCT(&CDU[dataOff], &CDU[dataOff + du_stride], + &CDU[dataOff + du_stride * 2], &CDU[dataOff + du_stride * 3], + &CDU[dataOff + du_stride * 4], &CDU[dataOff + du_stride * 5], + &CDU[dataOff + du_stride * 6], + &CDU[dataOff + du_stride * 7]); + } + // Quantize/descale/zigzag the coefficients + for (y = 0, j = 0; y < 8; ++y) { + for (x = 0; x < 8; ++x, ++j) { + float v; + i = y * du_stride + x; + v = CDU[i] * fdtbl[j]; + // DU[stbiw__jpg_ZigZag[j]] = (int)(v < 0 ? ceilf(v - 0.5f) : floorf(v + + // 0.5f)); ceilf() and floorf() are C99, not C89, but I /think/ they're + // not needed here anyway? + DU[stbiw__jpg_ZigZag[j]] = (int)(v < 0 ? v - 0.5f : v + 0.5f); + } + } + + // Encode DC + diff = DU[0] - DC; + if (diff == 0) { + stbiw__jpg_writeBits(s, bitBuf, bitCnt, HTDC[0]); + } else { + unsigned short bits[2]; + stbiw__jpg_calcBits(diff, bits); + stbiw__jpg_writeBits(s, bitBuf, bitCnt, HTDC[bits[1]]); + stbiw__jpg_writeBits(s, bitBuf, bitCnt, bits); + } + // Encode ACs + end0pos = 63; + for (; (end0pos > 0) && (DU[end0pos] == 0); --end0pos) { + } + // end0pos = first element in reverse order !=0 + if (end0pos == 0) { + stbiw__jpg_writeBits(s, bitBuf, bitCnt, EOB); + return DU[0]; + } + for (i = 1; i <= end0pos; ++i) { + int startpos = i; + int nrzeroes; + unsigned short bits[2]; + for (; DU[i] == 0 && i <= end0pos; ++i) { + } + nrzeroes = i - startpos; + if (nrzeroes >= 16) { + int lng = nrzeroes >> 4; + int nrmarker; + for (nrmarker = 1; nrmarker <= lng; ++nrmarker) + stbiw__jpg_writeBits(s, bitBuf, bitCnt, M16zeroes); + nrzeroes &= 15; + } + stbiw__jpg_calcBits(DU[i], bits); + stbiw__jpg_writeBits(s, bitBuf, bitCnt, HTAC[(nrzeroes << 4) + bits[1]]); + stbiw__jpg_writeBits(s, bitBuf, bitCnt, bits); + } + if (end0pos != 63) { + stbiw__jpg_writeBits(s, bitBuf, bitCnt, EOB); + } + return DU[0]; +} + +static int stbi_write_jpg_core(stbi__write_context *s, int width, int height, + int comp, const void *data, int quality) { + // Constants that don't pollute global namespace + static const unsigned char std_dc_luminance_nrcodes[] = { + 0, 0, 1, 5, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0}; + static const unsigned char std_dc_luminance_values[] = {0, 1, 2, 3, 4, 5, + 6, 7, 8, 9, 10, 11}; + static const unsigned char std_ac_luminance_nrcodes[] = { + 0, 0, 2, 1, 3, 3, 2, 4, 3, 5, 5, 4, 4, 0, 0, 1, 0x7d}; + static const unsigned char std_ac_luminance_values[] = { + 0x01, 0x02, 0x03, 0x00, 0x04, 0x11, 0x05, 0x12, 0x21, 0x31, 0x41, 0x06, + 0x13, 0x51, 0x61, 0x07, 0x22, 0x71, 0x14, 0x32, 0x81, 0x91, 0xa1, 0x08, + 0x23, 0x42, 0xb1, 0xc1, 0x15, 0x52, 0xd1, 0xf0, 0x24, 0x33, 0x62, 0x72, + 0x82, 0x09, 0x0a, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x25, 0x26, 0x27, 0x28, + 0x29, 0x2a, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x43, 0x44, 0x45, + 0x46, 0x47, 0x48, 0x49, 0x4a, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, + 0x5a, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x73, 0x74, 0x75, + 0x76, 0x77, 0x78, 0x79, 0x7a, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, + 0x8a, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0xa2, 0xa3, + 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, + 0xb7, 0xb8, 0xb9, 0xba, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, + 0xca, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xe1, 0xe2, + 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xf1, 0xf2, 0xf3, 0xf4, + 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa}; + static const unsigned char std_dc_chrominance_nrcodes[] = { + 0, 0, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0}; + static const unsigned char std_dc_chrominance_values[] = {0, 1, 2, 3, 4, 5, + 6, 7, 8, 9, 10, 11}; + static const unsigned char std_ac_chrominance_nrcodes[] = { + 0, 0, 2, 1, 2, 4, 4, 3, 4, 7, 5, 4, 4, 0, 1, 2, 0x77}; + static const unsigned char std_ac_chrominance_values[] = { + 0x00, 0x01, 0x02, 0x03, 0x11, 0x04, 0x05, 0x21, 0x31, 0x06, 0x12, 0x41, + 0x51, 0x07, 0x61, 0x71, 0x13, 0x22, 0x32, 0x81, 0x08, 0x14, 0x42, 0x91, + 0xa1, 0xb1, 0xc1, 0x09, 0x23, 0x33, 0x52, 0xf0, 0x15, 0x62, 0x72, 0xd1, + 0x0a, 0x16, 0x24, 0x34, 0xe1, 0x25, 0xf1, 0x17, 0x18, 0x19, 0x1a, 0x26, + 0x27, 0x28, 0x29, 0x2a, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x43, 0x44, + 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, + 0x59, 0x5a, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x73, 0x74, + 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, + 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xb2, 0xb3, 0xb4, + 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, + 0xc8, 0xc9, 0xca, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, + 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xf2, 0xf3, 0xf4, + 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa}; + // Huffman tables + static const unsigned short YDC_HT[256][2] = { + {0, 2}, {2, 3}, {3, 3}, {4, 3}, {5, 3}, {6, 3}, + {14, 4}, {30, 5}, {62, 6}, {126, 7}, {254, 8}, {510, 9}}; + static const unsigned short UVDC_HT[256][2] = { + {0, 2}, {1, 2}, {2, 2}, {6, 3}, {14, 4}, {30, 5}, + {62, 6}, {126, 7}, {254, 8}, {510, 9}, {1022, 10}, {2046, 11}}; + static const unsigned short YAC_HT[256][2] = { + {10, 4}, {0, 2}, {1, 2}, {4, 3}, {11, 4}, + {26, 5}, {120, 7}, {248, 8}, {1014, 10}, {65410, 16}, + {65411, 16}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, + {0, 0}, {0, 0}, {12, 4}, {27, 5}, {121, 7}, + {502, 9}, {2038, 11}, {65412, 16}, {65413, 16}, {65414, 16}, + {65415, 16}, {65416, 16}, {0, 0}, {0, 0}, {0, 0}, + {0, 0}, {0, 0}, {0, 0}, {28, 5}, {249, 8}, + {1015, 10}, {4084, 12}, {65417, 16}, {65418, 16}, {65419, 16}, + {65420, 16}, {65421, 16}, {65422, 16}, {0, 0}, {0, 0}, + {0, 0}, {0, 0}, {0, 0}, {0, 0}, {58, 6}, + {503, 9}, {4085, 12}, {65423, 16}, {65424, 16}, {65425, 16}, + {65426, 16}, {65427, 16}, {65428, 16}, {65429, 16}, {0, 0}, + {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, + {59, 6}, {1016, 10}, {65430, 16}, {65431, 16}, {65432, 16}, + {65433, 16}, {65434, 16}, {65435, 16}, {65436, 16}, {65437, 16}, + {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, + {0, 0}, {122, 7}, {2039, 11}, {65438, 16}, {65439, 16}, + {65440, 16}, {65441, 16}, {65442, 16}, {65443, 16}, {65444, 16}, + {65445, 16}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, + {0, 0}, {0, 0}, {123, 7}, {4086, 12}, {65446, 16}, + {65447, 16}, {65448, 16}, {65449, 16}, {65450, 16}, {65451, 16}, + {65452, 16}, {65453, 16}, {0, 0}, {0, 0}, {0, 0}, + {0, 0}, {0, 0}, {0, 0}, {250, 8}, {4087, 12}, + {65454, 16}, {65455, 16}, {65456, 16}, {65457, 16}, {65458, 16}, + {65459, 16}, {65460, 16}, {65461, 16}, {0, 0}, {0, 0}, + {0, 0}, {0, 0}, {0, 0}, {0, 0}, {504, 9}, + {32704, 15}, {65462, 16}, {65463, 16}, {65464, 16}, {65465, 16}, + {65466, 16}, {65467, 16}, {65468, 16}, {65469, 16}, {0, 0}, + {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, + {505, 9}, {65470, 16}, {65471, 16}, {65472, 16}, {65473, 16}, + {65474, 16}, {65475, 16}, {65476, 16}, {65477, 16}, {65478, 16}, + {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, + {0, 0}, {506, 9}, {65479, 16}, {65480, 16}, {65481, 16}, + {65482, 16}, {65483, 16}, {65484, 16}, {65485, 16}, {65486, 16}, + {65487, 16}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, + {0, 0}, {0, 0}, {1017, 10}, {65488, 16}, {65489, 16}, + {65490, 16}, {65491, 16}, {65492, 16}, {65493, 16}, {65494, 16}, + {65495, 16}, {65496, 16}, {0, 0}, {0, 0}, {0, 0}, + {0, 0}, {0, 0}, {0, 0}, {1018, 10}, {65497, 16}, + {65498, 16}, {65499, 16}, {65500, 16}, {65501, 16}, {65502, 16}, + {65503, 16}, {65504, 16}, {65505, 16}, {0, 0}, {0, 0}, + {0, 0}, {0, 0}, {0, 0}, {0, 0}, {2040, 11}, + {65506, 16}, {65507, 16}, {65508, 16}, {65509, 16}, {65510, 16}, + {65511, 16}, {65512, 16}, {65513, 16}, {65514, 16}, {0, 0}, + {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, + {65515, 16}, {65516, 16}, {65517, 16}, {65518, 16}, {65519, 16}, + {65520, 16}, {65521, 16}, {65522, 16}, {65523, 16}, {65524, 16}, + {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, + {2041, 11}, {65525, 16}, {65526, 16}, {65527, 16}, {65528, 16}, + {65529, 16}, {65530, 16}, {65531, 16}, {65532, 16}, {65533, 16}, + {65534, 16}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, + {0, 0}}; + static const unsigned short UVAC_HT[256][2] = { + {0, 2}, {1, 2}, {4, 3}, {10, 4}, {24, 5}, + {25, 5}, {56, 6}, {120, 7}, {500, 9}, {1014, 10}, + {4084, 12}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, + {0, 0}, {0, 0}, {11, 4}, {57, 6}, {246, 8}, + {501, 9}, {2038, 11}, {4085, 12}, {65416, 16}, {65417, 16}, + {65418, 16}, {65419, 16}, {0, 0}, {0, 0}, {0, 0}, + {0, 0}, {0, 0}, {0, 0}, {26, 5}, {247, 8}, + {1015, 10}, {4086, 12}, {32706, 15}, {65420, 16}, {65421, 16}, + {65422, 16}, {65423, 16}, {65424, 16}, {0, 0}, {0, 0}, + {0, 0}, {0, 0}, {0, 0}, {0, 0}, {27, 5}, + {248, 8}, {1016, 10}, {4087, 12}, {65425, 16}, {65426, 16}, + {65427, 16}, {65428, 16}, {65429, 16}, {65430, 16}, {0, 0}, + {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, + {58, 6}, {502, 9}, {65431, 16}, {65432, 16}, {65433, 16}, + {65434, 16}, {65435, 16}, {65436, 16}, {65437, 16}, {65438, 16}, + {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, + {0, 0}, {59, 6}, {1017, 10}, {65439, 16}, {65440, 16}, + {65441, 16}, {65442, 16}, {65443, 16}, {65444, 16}, {65445, 16}, + {65446, 16}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, + {0, 0}, {0, 0}, {121, 7}, {2039, 11}, {65447, 16}, + {65448, 16}, {65449, 16}, {65450, 16}, {65451, 16}, {65452, 16}, + {65453, 16}, {65454, 16}, {0, 0}, {0, 0}, {0, 0}, + {0, 0}, {0, 0}, {0, 0}, {122, 7}, {2040, 11}, + {65455, 16}, {65456, 16}, {65457, 16}, {65458, 16}, {65459, 16}, + {65460, 16}, {65461, 16}, {65462, 16}, {0, 0}, {0, 0}, + {0, 0}, {0, 0}, {0, 0}, {0, 0}, {249, 8}, + {65463, 16}, {65464, 16}, {65465, 16}, {65466, 16}, {65467, 16}, + {65468, 16}, {65469, 16}, {65470, 16}, {65471, 16}, {0, 0}, + {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, + {503, 9}, {65472, 16}, {65473, 16}, {65474, 16}, {65475, 16}, + {65476, 16}, {65477, 16}, {65478, 16}, {65479, 16}, {65480, 16}, + {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, + {0, 0}, {504, 9}, {65481, 16}, {65482, 16}, {65483, 16}, + {65484, 16}, {65485, 16}, {65486, 16}, {65487, 16}, {65488, 16}, + {65489, 16}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, + {0, 0}, {0, 0}, {505, 9}, {65490, 16}, {65491, 16}, + {65492, 16}, {65493, 16}, {65494, 16}, {65495, 16}, {65496, 16}, + {65497, 16}, {65498, 16}, {0, 0}, {0, 0}, {0, 0}, + {0, 0}, {0, 0}, {0, 0}, {506, 9}, {65499, 16}, + {65500, 16}, {65501, 16}, {65502, 16}, {65503, 16}, {65504, 16}, + {65505, 16}, {65506, 16}, {65507, 16}, {0, 0}, {0, 0}, + {0, 0}, {0, 0}, {0, 0}, {0, 0}, {2041, 11}, + {65508, 16}, {65509, 16}, {65510, 16}, {65511, 16}, {65512, 16}, + {65513, 16}, {65514, 16}, {65515, 16}, {65516, 16}, {0, 0}, + {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, + {16352, 14}, {65517, 16}, {65518, 16}, {65519, 16}, {65520, 16}, + {65521, 16}, {65522, 16}, {65523, 16}, {65524, 16}, {65525, 16}, + {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, + {1018, 10}, {32707, 15}, {65526, 16}, {65527, 16}, {65528, 16}, + {65529, 16}, {65530, 16}, {65531, 16}, {65532, 16}, {65533, 16}, + {65534, 16}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, + {0, 0}}; + static const int YQT[] = { + 16, 11, 10, 16, 24, 40, 51, 61, 12, 12, 14, 19, 26, 58, 60, 55, + 14, 13, 16, 24, 40, 57, 69, 56, 14, 17, 22, 29, 51, 87, 80, 62, + 18, 22, 37, 56, 68, 109, 103, 77, 24, 35, 55, 64, 81, 104, 113, 92, + 49, 64, 78, 87, 103, 121, 120, 101, 72, 92, 95, 98, 112, 100, 103, 99}; + static const int UVQT[] = {17, 18, 24, 47, 99, 99, 99, 99, 18, 21, 26, 66, 99, + 99, 99, 99, 24, 26, 56, 99, 99, 99, 99, 99, 47, 66, + 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, + 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, + 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99}; + static const float aasf[] = { + 1.0f * 2.828427125f, 1.387039845f * 2.828427125f, + 1.306562965f * 2.828427125f, 1.175875602f * 2.828427125f, + 1.0f * 2.828427125f, 0.785694958f * 2.828427125f, + 0.541196100f * 2.828427125f, 0.275899379f * 2.828427125f}; + + int row, col, i, k, subsample; + float fdtbl_Y[64], fdtbl_UV[64]; + unsigned char YTable[64], UVTable[64]; + + if (!data || !width || !height || comp > 4 || comp < 1) { + return 0; + } + + quality = quality ? quality : 90; + subsample = quality <= 90 ? 1 : 0; + quality = quality < 1 ? 1 : quality > 100 ? 100 : quality; + quality = quality < 50 ? 5000 / quality : 200 - quality * 2; + + for (i = 0; i < 64; ++i) { + int uvti, yti = (YQT[i] * quality + 50) / 100; + YTable[stbiw__jpg_ZigZag[i]] = (unsigned char)(yti < 1 ? 1 + : yti > 255 ? 255 + : yti); + uvti = (UVQT[i] * quality + 50) / 100; + UVTable[stbiw__jpg_ZigZag[i]] = (unsigned char)(uvti < 1 ? 1 + : uvti > 255 ? 255 + : uvti); + } + + for (row = 0, k = 0; row < 8; ++row) { + for (col = 0; col < 8; ++col, ++k) { + fdtbl_Y[k] = 1 / (YTable[stbiw__jpg_ZigZag[k]] * aasf[row] * aasf[col]); + fdtbl_UV[k] = 1 / (UVTable[stbiw__jpg_ZigZag[k]] * aasf[row] * aasf[col]); + } + } + + // Write Headers + { + static const unsigned char head0[] = { + 0xFF, 0xD8, 0xFF, 0xE0, 0, 0x10, 'J', 'F', 'I', 'F', 0, 1, 1, + 0, 0, 1, 0, 1, 0, 0, 0xFF, 0xDB, 0, 0x84, 0}; + static const unsigned char head2[] = {0xFF, 0xDA, 0, 0xC, 3, 1, 0, + 2, 0x11, 3, 0x11, 0, 0x3F, 0}; + const unsigned char head1[] = {0xFF, + 0xC0, + 0, + 0x11, + 8, + (unsigned char)(height >> 8), + STBIW_UCHAR(height), + (unsigned char)(width >> 8), + STBIW_UCHAR(width), + 3, + 1, + (unsigned char)(subsample ? 0x22 : 0x11), + 0, + 2, + 0x11, + 1, + 3, + 0x11, + 1, + 0xFF, + 0xC4, + 0x01, + 0xA2, + 0}; + s->func(s->context, (void *)head0, sizeof(head0)); + s->func(s->context, (void *)YTable, sizeof(YTable)); + stbiw__putc(s, 1); + s->func(s->context, UVTable, sizeof(UVTable)); + s->func(s->context, (void *)head1, sizeof(head1)); + s->func(s->context, (void *)(std_dc_luminance_nrcodes + 1), + sizeof(std_dc_luminance_nrcodes) - 1); + s->func(s->context, (void *)std_dc_luminance_values, + sizeof(std_dc_luminance_values)); + stbiw__putc(s, 0x10); // HTYACinfo + s->func(s->context, (void *)(std_ac_luminance_nrcodes + 1), + sizeof(std_ac_luminance_nrcodes) - 1); + s->func(s->context, (void *)std_ac_luminance_values, + sizeof(std_ac_luminance_values)); + stbiw__putc(s, 1); // HTUDCinfo + s->func(s->context, (void *)(std_dc_chrominance_nrcodes + 1), + sizeof(std_dc_chrominance_nrcodes) - 1); + s->func(s->context, (void *)std_dc_chrominance_values, + sizeof(std_dc_chrominance_values)); + stbiw__putc(s, 0x11); // HTUACinfo + s->func(s->context, (void *)(std_ac_chrominance_nrcodes + 1), + sizeof(std_ac_chrominance_nrcodes) - 1); + s->func(s->context, (void *)std_ac_chrominance_values, + sizeof(std_ac_chrominance_values)); + s->func(s->context, (void *)head2, sizeof(head2)); + } + + // Encode 8x8 macroblocks + { + static const unsigned short fillBits[] = {0x7F, 7}; + int DCY = 0, DCU = 0, DCV = 0; + int bitBuf = 0, bitCnt = 0; + // comp == 2 is grey+alpha (alpha is ignored) + int ofsG = comp > 2 ? 1 : 0, ofsB = comp > 2 ? 2 : 0; + const unsigned char *dataR = (const unsigned char *)data; + const unsigned char *dataG = dataR + ofsG; + const unsigned char *dataB = dataR + ofsB; + int x, y, pos; + if (subsample) { + for (y = 0; y < height; y += 16) { + for (x = 0; x < width; x += 16) { + float Y[256], U[256], V[256]; + for (row = y, pos = 0; row < y + 16; ++row) { + // row >= height => use last input row + int clamped_row = (row < height) ? row : height - 1; + int base_p = + (stbi__flip_vertically_on_write ? (height - 1 - clamped_row) + : clamped_row) * + width * comp; + for (col = x; col < x + 16; ++col, ++pos) { + // if col >= width => use pixel from last input column + int p = base_p + ((col < width) ? col : (width - 1)) * comp; + float r = dataR[p], g = dataG[p], b = dataB[p]; + Y[pos] = +0.29900f * r + 0.58700f * g + 0.11400f * b - 128; + U[pos] = -0.16874f * r - 0.33126f * g + 0.50000f * b; + V[pos] = +0.50000f * r - 0.41869f * g - 0.08131f * b; + } + } + DCY = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, Y + 0, 16, fdtbl_Y, + DCY, YDC_HT, YAC_HT); + DCY = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, Y + 8, 16, fdtbl_Y, + DCY, YDC_HT, YAC_HT); + DCY = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, Y + 128, 16, fdtbl_Y, + DCY, YDC_HT, YAC_HT); + DCY = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, Y + 136, 16, fdtbl_Y, + DCY, YDC_HT, YAC_HT); + + // subsample U,V + { + float subU[64], subV[64]; + int yy, xx; + for (yy = 0, pos = 0; yy < 8; ++yy) { + for (xx = 0; xx < 8; ++xx, ++pos) { + int j = yy * 32 + xx * 2; + subU[pos] = + (U[j + 0] + U[j + 1] + U[j + 16] + U[j + 17]) * 0.25f; + subV[pos] = + (V[j + 0] + V[j + 1] + V[j + 16] + V[j + 17]) * 0.25f; + } + } + DCU = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, subU, 8, fdtbl_UV, + DCU, UVDC_HT, UVAC_HT); + DCV = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, subV, 8, fdtbl_UV, + DCV, UVDC_HT, UVAC_HT); + } + } + } + } else { + for (y = 0; y < height; y += 8) { + for (x = 0; x < width; x += 8) { + float Y[64], U[64], V[64]; + for (row = y, pos = 0; row < y + 8; ++row) { + // row >= height => use last input row + int clamped_row = (row < height) ? row : height - 1; + int base_p = + (stbi__flip_vertically_on_write ? (height - 1 - clamped_row) + : clamped_row) * + width * comp; + for (col = x; col < x + 8; ++col, ++pos) { + // if col >= width => use pixel from last input column + int p = base_p + ((col < width) ? col : (width - 1)) * comp; + float r = dataR[p], g = dataG[p], b = dataB[p]; + Y[pos] = +0.29900f * r + 0.58700f * g + 0.11400f * b - 128; + U[pos] = -0.16874f * r - 0.33126f * g + 0.50000f * b; + V[pos] = +0.50000f * r - 0.41869f * g - 0.08131f * b; + } + } + + DCY = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, Y, 8, fdtbl_Y, DCY, + YDC_HT, YAC_HT); + DCU = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, U, 8, fdtbl_UV, DCU, + UVDC_HT, UVAC_HT); + DCV = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, V, 8, fdtbl_UV, DCV, + UVDC_HT, UVAC_HT); + } + } + } + + // Do the bit alignment of the EOI marker + stbiw__jpg_writeBits(s, &bitBuf, &bitCnt, fillBits); + } + + // EOI + stbiw__putc(s, 0xFF); + stbiw__putc(s, 0xD9); + + return 1; +} + +STBIWDEF int stbi_write_jpg_to_func(stbi_write_func *func, void *context, int x, + int y, int comp, const void *data, + int quality) { + stbi__write_context s = {0}; + stbi__start_write_callbacks(&s, func, context); + return stbi_write_jpg_core(&s, x, y, comp, (void *)data, quality); +} + +#ifndef STBI_WRITE_NO_STDIO +STBIWDEF int stbi_write_jpg(char const *filename, int x, int y, int comp, + const void *data, int quality) { + stbi__write_context s = {0}; + if (stbi__start_write_file(&s, filename)) { + int r = stbi_write_jpg_core(&s, x, y, comp, data, quality); + stbi__end_write_file(&s); + return r; + } else + return 0; +} +#endif + +#endif // STB_IMAGE_WRITE_IMPLEMENTATION + +/* Revision history + 1.16 (2021-07-11) + make Deflate code emit uncompressed blocks when it would otherwise + expand support writing BMPs with alpha channel 1.15 (2020-07-13) unknown + 1.14 (2020-02-02) updated JPEG writer to downsample chroma channels + 1.13 + 1.12 + 1.11 (2019-08-11) + + 1.10 (2019-02-07) + support utf8 filenames in Windows; fix warnings and platform ifdefs + 1.09 (2018-02-11) + fix typo in zlib quality API, improve STB_I_W_STATIC in C++ + 1.08 (2018-01-29) + add stbi__flip_vertically_on_write, external zlib, zlib quality, + choose PNG filter 1.07 (2017-07-24) doc fix 1.06 (2017-07-23) writing JPEG + (using Jon Olick's code) 1.05 ??? 1.04 (2017-03-03) monochrome BMP + expansion 1.03 ??? 1.02 (2016-04-02) avoid allocating large structures on + the stack 1.01 (2016-01-16) STBIW_REALLOC_SIZED: support allocators with no + realloc support avoid race-condition in crc initialization minor compile + issues 1.00 (2015-09-14) installable file IO function 0.99 (2015-09-13) + warning fixes; TGA rle support + 0.98 (2015-04-08) + added STBIW_MALLOC, STBIW_ASSERT etc + 0.97 (2015-01-18) + fixed HDR asserts, rewrote HDR rle logic + 0.96 (2015-01-17) + add HDR output + fix monochrome BMP + 0.95 (2014-08-17) + add monochrome TGA output + 0.94 (2014-05-31) + rename private functions to avoid conflicts with stb_image.h + 0.93 (2014-05-27) + warning fixes + 0.92 (2010-08-01) + casts to unsigned char to fix warnings + 0.91 (2010-07-17) + first public release + 0.90 first internal release +*/ + +/* +------------------------------------------------------------------------------ +This software is available under 2 licenses -- choose whichever you prefer. +------------------------------------------------------------------------------ +ALTERNATIVE A - MIT License +Copyright (c) 2017 Sean Barrett +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +------------------------------------------------------------------------------ +ALTERNATIVE B - Public Domain (www.unlicense.org) +This is free and unencumbered software released into the public domain. +Anyone is free to copy, modify, publish, use, compile, sell, or distribute this +software, either in source code form or as a compiled binary, for any purpose, +commercial or non-commercial, and by any means. +In jurisdictions that recognize copyright laws, the author or authors of this +software dedicate any and all copyright interest in the software to the public +domain. We make this dedication for the benefit of the public at large and to +the detriment of our heirs and successors. We intend this dedication to be an +overt act of relinquishment in perpetuity of all present and future rights to +this software under copyright law. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +------------------------------------------------------------------------------ +*/ From 918574c7435bb4fa7d1bd0fcc3ae25c379906d72 Mon Sep 17 00:00:00 2001 From: richardlee <18626685+richardlee159@users.noreply.github.com> Date: Wed, 2 Apr 2025 11:25:43 +0200 Subject: [PATCH 14/19] Add hybrid implementation for the busy function --- functions/CMakeLists.txt | 4 +- functions/busy_hybrid/CMakeLists.txt | 17 ++ functions/busy_hybrid/busy_hybrid.c | 286 ++++++++++++++++++ .../busy_hybrid/linux_syscalls/CMakeLists.txt | 10 + .../arch/aarch64/syscall_arch.h | 80 +++++ .../linux_syscalls/arch/x86_64/syscall_arch.h | 88 ++++++ .../busy_hybrid/linux_syscalls/syscall.h | 105 +++++++ functions/busy_libc/CMakeLists.txt | 14 + functions/busy_libc/busy_libc.c | 116 +++++++ 9 files changed, 719 insertions(+), 1 deletion(-) create mode 100644 functions/busy_hybrid/CMakeLists.txt create mode 100644 functions/busy_hybrid/busy_hybrid.c create mode 100644 functions/busy_hybrid/linux_syscalls/CMakeLists.txt create mode 100644 functions/busy_hybrid/linux_syscalls/arch/aarch64/syscall_arch.h create mode 100644 functions/busy_hybrid/linux_syscalls/arch/x86_64/syscall_arch.h create mode 100644 functions/busy_hybrid/linux_syscalls/syscall.h create mode 100644 functions/busy_libc/CMakeLists.txt create mode 100644 functions/busy_libc/busy_libc.c diff --git a/functions/CMakeLists.txt b/functions/CMakeLists.txt index 43b3f82..a7a81ea 100644 --- a/functions/CMakeLists.txt +++ b/functions/CMakeLists.txt @@ -119,8 +119,10 @@ add_subdirectory(example_app) add_subdirectory(files) add_subdirectory(stdio) add_subdirectory(stdio_cxx) -add_subdirectory(example_app_hybrid) +# add_subdirectory(example_app_hybrid) add_subdirectory(compression) +add_subdirectory(busy_libc) +add_subdirectory(busy_hybrid) set(CONFIG_FLAG "--config ${CMAKE_BINARY_DIR}/dandelion.cfg") set(CONFIG_FLAGXX "--config ${CMAKE_BINARY_DIR}/dandelion++.cfg") diff --git a/functions/busy_hybrid/CMakeLists.txt b/functions/busy_hybrid/CMakeLists.txt new file mode 100644 index 0000000..2298912 --- /dev/null +++ b/functions/busy_hybrid/CMakeLists.txt @@ -0,0 +1,17 @@ +cmake_minimum_required(VERSION ${CMAKE_VERSION}) + +add_subdirectory(linux_syscalls) + +set(EXECUTABLE "busy_hybrid") + +add_executable(${EXECUTABLE} + busy_hybrid.c +) + +target_compile_options(${EXECUTABLE} PRIVATE -static -Os -flto) +target_link_options(${EXECUTABLE} PRIVATE -static -Os -flto) +target_link_libraries(${EXECUTABLE} PRIVATE + linux_syscalls + dlibc + dandelion_runtime +) \ No newline at end of file diff --git a/functions/busy_hybrid/busy_hybrid.c b/functions/busy_hybrid/busy_hybrid.c new file mode 100644 index 0000000..482b8d1 --- /dev/null +++ b/functions/busy_hybrid/busy_hybrid.c @@ -0,0 +1,286 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "syscall.h" +#include "unistd.h" + +#define ARRAY_ITEMS 16 +#define BUFFER_SIZE 1 * 1024 * 1024 +#define FETCH_REQUEST_PATH "/data/get_request" +#define STORE_PREAMBLE_PATH "/preamble/post_request" +#define STORE_RESPONSE_PATH "/store_request/post_response" + +typedef struct data_item { + int8_t value[ARRAY_ITEMS]; +} data_item; + +uint16_t my_htons(uint16_t port) { + return ((port >> 8) & 0x00FF) | ((port << 8) & 0xFF00); +} + +int my_inet_pton(int af, const char *src, void *dst) { + if (af == LINUX_AF_INET) { + uint8_t array[4] = {}; + for (int i = 0; i < 4; i++) { + char *end; + long value = strtol(src, &end, 10); + if (value < 0 || value > 255) { + return 0; + } + + if (*end != '\0' && *end != '.' && i < 3) { + return 0; + } + src = end + 1; + array[i] = (uint8_t)value; + } + + struct in_addr *addr = (struct in_addr *)dst; + memcpy(&addr->s_addr, array, 4); // Copy the 4 bytes into s_addr + + return 1; // Success + } + return -1; // Unsupported address family +} + +int extract_ip_port(const char *http_request, char *ip, size_t ip_len, + uint16_t *port) { + const char *prefix = "http://"; + const char *start = strstr(http_request, prefix); + if (!start) + return -1; + + start += strlen(prefix); + const char *colon = strchr(start, ':'); + if (!colon) + return -2; + + const char *slash = strchr(colon, '/'); + if (!slash) + return -3; + + // Extract IP + size_t ip_size = colon - start; + if (ip_size >= ip_len) + return -4; + strncpy(ip, start, ip_size); + ip[ip_size] = '\0'; + + // Extract port + char port_str[6] = {0}; + size_t port_size = slash - colon - 1; + if (port_size >= sizeof(port_str)) + return -5; + strncpy(port_str, colon + 1, port_size); + *port = atoi(port_str); + + return 0; +} + +int connect_to_server(const char *ip, uint16_t port) { + struct sockaddr_in server_addr; + + int sockfd = linux_socket(LINUX_AF_INET, LINUX_SOCK_STREAM, 0); + if (sockfd < 0) { + perror("Socket creation failed"); + return -1; + } + + server_addr.sin_family = LINUX_AF_INET; + server_addr.sin_port = my_htons(port); + + if (my_inet_pton(LINUX_AF_INET, ip, &server_addr.sin_addr) != 1) { + perror("Invalid IP address"); + close(sockfd); + return -1; + } + + int err = linux_connect(sockfd, &server_addr, sizeof(server_addr)); + if (err < 0) { + printf("Error: %s\n", strerror(-err)); + printf("size of server_addr: %ld\n", sizeof(server_addr)); + perror("Socket connection failed"); + close(sockfd); + return -1; + } + + return sockfd; +} + +ssize_t send_http_request(int sockfd, const char *request, size_t request_size, + char *response, size_t response_size) { + if (linux_send(sockfd, request, request_size, 0) < 0) { + perror("Error sending request"); + return -1; + } + + ssize_t total_received = 0, bytes; + // while ((bytes = linux_recv(sockfd, response + total_received, + // response_size - total_received - 1, 0)) > 0) { + // total_received += bytes; + // if (total_received >= response_size - 1) break; + // } + // blocks and waits forever if there is no next packate, could validate HTTP + // sice on the packages to see if we need more, this works for now + total_received = linux_recv(sockfd, response + total_received, + response_size - total_received - 1, 0); + if (total_received < 0) { + perror("Error receiving response"); + return -1; + } + response[total_received] = '\0'; + return total_received; +} + +void parse_http_body(const char *response, size_t response_size, + char **response_body, size_t *response_body_size) { + *response_body = strstr(response, "\r\n\r\n"); + if (*response_body == NULL) + return; + *response_body += 4; + *response_body_size = response_size - (*response_body - response); +} + +size_t read_file_to_string(const char *filename, char **content) { + // Open file in read mode + FILE *file = fopen(filename, "r"); + if (file == NULL) { + perror("Error opening file"); + return -1; + } + + // Get file size + fseek(file, 0, SEEK_END); + size_t file_size = ftell(file); + rewind(file); + + // Allocate memory for content (+1 for null terminator) + *content = (char *)malloc(file_size + 1); + if (*content == NULL) { + perror("Error allocating memory"); + return -1; // Memory allocation failed + } + + // Read file into the buffer + size_t bytes_read = fread(*content, 1, file_size, file); + (*content)[bytes_read] = '\0'; // Add null terminator + fclose(file); + + return bytes_read; // Return the length of the content +} + +int main() { + char *fetch_preamble = NULL; + size_t fetch_preamble_len = + read_file_to_string(FETCH_REQUEST_PATH, &fetch_preamble); + if (fetch_preamble < 0) + return 1; + + char server_ip[16]; + uint16_t server_port; + if (extract_ip_port(fetch_preamble, server_ip, 16, &server_port) < 0) + return 2; + + // fetching data + int sock = connect_to_server(server_ip, server_port); + if (sock < 0) + return 3; + + size_t fetch_request_len = fetch_preamble_len + 4; + char *fetch_request = malloc(fetch_request_len); + memcpy(fetch_request, fetch_preamble, fetch_preamble_len); + memcpy(fetch_request + fetch_preamble_len, "\r\n\r\n", 4); + + char *fetch_response = (char *)malloc(BUFFER_SIZE); + if (fetch_response == NULL) { + perror("Error allocating memory"); + return 4; + } + ssize_t fetch_response_len = send_http_request( + sock, fetch_request, fetch_request_len, fetch_response, BUFFER_SIZE); + if (fetch_response_len < 0) + return 5; + + char *fetch_response_body; + size_t fetch_response_body_len; + parse_http_body(fetch_response, fetch_response_len, &fetch_response_body, + &fetch_response_body_len); + if (fetch_response_body == NULL) + return 6; + + uint64_t iterations = *(uint64_t *)fetch_response_body; + data_item *data_items = (data_item *)(fetch_response_body + 8); + + // the igreggated statistics we want to collect + // sum, average, variance, min, max + int64_t sum = 0; + int64_t min = 256; + int64_t max = -256; + + size_t struct_index = 0; + size_t value_index = 0; + size_t max_index = (fetch_response_body_len - 8) / sizeof(data_item); + for (size_t iteration = 0; iteration < iterations; iteration++) { + for (size_t array_index = 0; array_index < ARRAY_ITEMS; array_index++) { + int8_t value = data_items[struct_index].value[array_index]; + if (value > max) + max = value; + if (value < min) + min = value; + sum += value; + } + // using size_t which has no sign, guarantees that the index is not + // negative when computing the remainder + struct_index = + (struct_index + data_items[struct_index].value[value_index]) % + max_index; + value_index = (value_index + 1) % ARRAY_ITEMS; + } + + char *store_preamble = NULL; + size_t store_preamble_len = + read_file_to_string(STORE_PREAMBLE_PATH, &store_preamble); + if (store_preamble_len < 0) + return 7; + store_preamble_len -= 2; // omit \n\n at the end + + char header[] = "\r\ncontent-length: 24\r\n\r\n"; // 3 * sizeof(int64_t) + size_t store_request_len = + store_preamble_len + (sizeof(header) - 1) + 3 * sizeof(int64_t); + char *store_request = malloc(store_request_len); + char *current_ptr = store_request; + memcpy(current_ptr, store_preamble, store_preamble_len); + current_ptr += store_preamble_len; + memcpy(current_ptr, header, sizeof(header) - 1); + current_ptr += sizeof(header) - 1; + memcpy(current_ptr, &sum, sizeof(int64_t)); + current_ptr += sizeof(int64_t); + memcpy(current_ptr, &min, sizeof(int64_t)); + current_ptr += sizeof(int64_t); + memcpy(current_ptr, &max, sizeof(int64_t)); + current_ptr += sizeof(int64_t); + + // posting data + char *store_response = fetch_response; + ssize_t store_response_len = send_http_request( + sock, store_request, store_request_len, store_response, BUFFER_SIZE); + if (store_response_len < 0) + return 8; + close(sock); + + FILE *store_response_file = fopen(STORE_RESPONSE_PATH, "wb"); + if (store_response_file == NULL) { + perror("Error opening file"); + return 9; + } + fwrite(store_response, 1, store_response_len, store_response_file); + fclose(store_response_file); + + return 0; +} diff --git a/functions/busy_hybrid/linux_syscalls/CMakeLists.txt b/functions/busy_hybrid/linux_syscalls/CMakeLists.txt new file mode 100644 index 0000000..d65c5d7 --- /dev/null +++ b/functions/busy_hybrid/linux_syscalls/CMakeLists.txt @@ -0,0 +1,10 @@ +add_library(linux_syscalls INTERFACE) + +target_include_directories(linux_syscalls INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}) +if(ARCHITECTURE MATCHES "aarch64|arm64|ARM64") + target_include_directories(linux_syscalls INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/arch/aarch64) +elseif(ARCHITECTURE MATCHES "x86_64|amd64|AMD64") + target_include_directories(linux_syscalls INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/arch/x86_64) +else() + message(FATAL_ERROR "Unsupported architecture: ${CMAKE_SYSTEM_PROCESSOR}") +endif() diff --git a/functions/busy_hybrid/linux_syscalls/arch/aarch64/syscall_arch.h b/functions/busy_hybrid/linux_syscalls/arch/aarch64/syscall_arch.h new file mode 100644 index 0000000..0e5a982 --- /dev/null +++ b/functions/busy_hybrid/linux_syscalls/arch/aarch64/syscall_arch.h @@ -0,0 +1,80 @@ +#define __SYSCALL_LL_E(x) (x) +#define __SYSCALL_LL_O(x) (x) + +#define SYS_openat 56 +#define SYS_close 57 +#define SYS_getdents64 61 +#define SYS_lseek 62 +#define SYS_read 63 +#define SYS_write 64 +#define SYS_ppoll 73 +#define SYS_exit_group 94 +#define SYS_socket 198 +#define SYS_connect 203 +#define SYS_sendto 206 +#define SYS_recvfrom 207 +#define SYS_mmap 222 + +#define __asm_syscall(...) \ + do { \ + __asm__ __volatile__("svc 0" : "=r"(x0) : __VA_ARGS__ : "memory", "cc"); \ + return x0; \ + } while (0) + +static inline long __syscall0(long n) { + register long x8 __asm__("x8") = n; + register long x0 __asm__("x0"); + __asm_syscall("r"(x8)); +} + +static inline long __syscall1(long n, long a) { + register long x8 __asm__("x8") = n; + register long x0 __asm__("x0") = a; + __asm_syscall("r"(x8), "0"(x0)); +} + +static inline long __syscall2(long n, long a, long b) { + register long x8 __asm__("x8") = n; + register long x0 __asm__("x0") = a; + register long x1 __asm__("x1") = b; + __asm_syscall("r"(x8), "0"(x0), "r"(x1)); +} + +static inline long __syscall3(long n, long a, long b, long c) { + register long x8 __asm__("x8") = n; + register long x0 __asm__("x0") = a; + register long x1 __asm__("x1") = b; + register long x2 __asm__("x2") = c; + __asm_syscall("r"(x8), "0"(x0), "r"(x1), "r"(x2)); +} + +static inline long __syscall4(long n, long a, long b, long c, long d) { + register long x8 __asm__("x8") = n; + register long x0 __asm__("x0") = a; + register long x1 __asm__("x1") = b; + register long x2 __asm__("x2") = c; + register long x3 __asm__("x3") = d; + __asm_syscall("r"(x8), "0"(x0), "r"(x1), "r"(x2), "r"(x3)); +} + +static inline long __syscall5(long n, long a, long b, long c, long d, long e) { + register long x8 __asm__("x8") = n; + register long x0 __asm__("x0") = a; + register long x1 __asm__("x1") = b; + register long x2 __asm__("x2") = c; + register long x3 __asm__("x3") = d; + register long x4 __asm__("x4") = e; + __asm_syscall("r"(x8), "0"(x0), "r"(x1), "r"(x2), "r"(x3), "r"(x4)); +} + +static inline long __syscall6(long n, long a, long b, long c, long d, long e, + long f) { + register long x8 __asm__("x8") = n; + register long x0 __asm__("x0") = a; + register long x1 __asm__("x1") = b; + register long x2 __asm__("x2") = c; + register long x3 __asm__("x3") = d; + register long x4 __asm__("x4") = e; + register long x5 __asm__("x5") = f; + __asm_syscall("r"(x8), "0"(x0), "r"(x1), "r"(x2), "r"(x3), "r"(x4), "r"(x5)); +} diff --git a/functions/busy_hybrid/linux_syscalls/arch/x86_64/syscall_arch.h b/functions/busy_hybrid/linux_syscalls/arch/x86_64/syscall_arch.h new file mode 100644 index 0000000..e6292bb --- /dev/null +++ b/functions/busy_hybrid/linux_syscalls/arch/x86_64/syscall_arch.h @@ -0,0 +1,88 @@ +#define __SYSCALL_LL_E(x) (x) +#define __SYSCALL_LL_O(x) (x) + +#define SYS_read 0 +#define SYS_write 1 +#define SYS_close 3 +#define SYS_lseek 8 +#define SYS_mmap 9 +#define SYS_socket 41 +#define SYS_connect 42 +#define SYS_sendto 44 +#define SYS_recvfrom 45 +#define SYS_arch_prctl 158 +#define SYS_exit_group 231 +#define SYS_openat 257 +#define SYS_getdents64 217 +#define SYS_ppoll 271 + +#define ARCH_SET_FS 0x1002 + +static __inline long __syscall0(long n) { + unsigned long ret; + __asm__ __volatile__("syscall" : "=a"(ret) : "a"(n) : "rcx", "r11", "memory"); + return ret; +} + +static __inline long __syscall1(long n, long a1) { + unsigned long ret; + __asm__ __volatile__("syscall" + : "=a"(ret) + : "a"(n), "D"(a1) + : "rcx", "r11", "memory"); + return ret; +} + +static __inline long __syscall2(long n, long a1, long a2) { + unsigned long ret; + __asm__ __volatile__("syscall" + : "=a"(ret) + : "a"(n), "D"(a1), "S"(a2) + : "rcx", "r11", "memory"); + return ret; +} + +static __inline long __syscall3(long n, long a1, long a2, long a3) { + unsigned long ret; + __asm__ __volatile__("syscall" + : "=a"(ret) + : "a"(n), "D"(a1), "S"(a2), "d"(a3) + : "rcx", "r11", "memory"); + return ret; +} + +static __inline long __syscall4(long n, long a1, long a2, long a3, long a4) { + unsigned long ret; + register long r10 __asm__("r10") = a4; + __asm__ __volatile__("syscall" + : "=a"(ret) + : "a"(n), "D"(a1), "S"(a2), "d"(a3), "r"(r10) + : "rcx", "r11", "memory"); + return ret; +} + +static __inline long __syscall5(long n, long a1, long a2, long a3, long a4, + long a5) { + unsigned long ret; + register long r10 __asm__("r10") = a4; + register long r8 __asm__("r8") = a5; + __asm__ __volatile__("syscall" + : "=a"(ret) + : "a"(n), "D"(a1), "S"(a2), "d"(a3), "r"(r10), "r"(r8) + : "rcx", "r11", "memory"); + return ret; +} + +static __inline long __syscall6(long n, long a1, long a2, long a3, long a4, + long a5, long a6) { + unsigned long ret; + register long r10 __asm__("r10") = a4; + register long r8 __asm__("r8") = a5; + register long r9 __asm__("r9") = a6; + __asm__ __volatile__("syscall" + : "=a"(ret) + : "a"(n), "D"(a1), "S"(a2), "d"(a3), "r"(r10), "r"(r8), + "r"(r9) + : "rcx", "r11", "memory"); + return ret; +} diff --git a/functions/busy_hybrid/linux_syscalls/syscall.h b/functions/busy_hybrid/linux_syscalls/syscall.h new file mode 100644 index 0000000..17c166a --- /dev/null +++ b/functions/busy_hybrid/linux_syscalls/syscall.h @@ -0,0 +1,105 @@ +#ifndef _LINUX_SYSCALL_H +#define _LINUX_SYSCALL_H + +#include "syscall_arch.h" + +#define LINUX_PROT_READ 0x1 +#define LINUX_PROT_WRITE 0x2 +#define LINUX_PROT_EXEC 0x4 + +#define LINUX_MAP_ANONYMOUS 0x20 +#define LINUX_MAP_PRIVATE 0x2 + +#define LINUX_O_RDONLY 00 +#define LINUX_AT_FDCWD -100 + +// networking defines +#define LINUX_AF_INET 2 +#define LINUX_SOCK_STREAM 1 + +// poll defines +#define POLLIN 0x0001 +#define POLLPRI 0x0002 +#define POLLOUT 0x0004 +#define POLLERR 0x0008 +#define POLLHUP 0x0010 +#define POLLNVAL 0x0020 + +#ifndef __scc +#define __scc(X) ((long)(X)) +typedef long syscall_arg_t; +#endif + +#define __syscall1(n, a) __syscall1(n, __scc(a)) +#define __syscall2(n, a, b) __syscall2(n, __scc(a), __scc(b)) +#define __syscall3(n, a, b, c) __syscall3(n, __scc(a), __scc(b), __scc(c)) +#define __syscall4(n, a, b, c, d) \ + __syscall4(n, __scc(a), __scc(b), __scc(c), __scc(d)) +#define __syscall5(n, a, b, c, d, e) \ + __syscall5(n, __scc(a), __scc(b), __scc(c), __scc(d), __scc(e)) +#define __syscall6(n, a, b, c, d, e, f) \ + __syscall6(n, __scc(a), __scc(b), __scc(c), __scc(d), __scc(e), __scc(f)) +#define __syscall7(n, a, b, c, d, e, f, g) \ + __syscall7(n, __scc(a), __scc(b), __scc(c), __scc(d), __scc(e), __scc(f), \ + __scc(g)) + +#define __SYSCALL_NARGS_X(a, b, c, d, e, f, g, h, n, ...) n +#define __SYSCALL_NARGS(...) \ + __SYSCALL_NARGS_X(__VA_ARGS__, 7, 6, 5, 4, 3, 2, 1, 0, ) +#define __SYSCALL_CONCAT_X(a, b) a##b +#define __SYSCALL_CONCAT(a, b) __SYSCALL_CONCAT_X(a, b) +#define __SYSCALL_DISP(b, ...) \ + __SYSCALL_CONCAT(b, __SYSCALL_NARGS(__VA_ARGS__))(__VA_ARGS__) + +#define __syscall(...) __SYSCALL_DISP(__syscall, __VA_ARGS__) + +static inline int linux_socket(int domain, int type, int protocol) { + return __syscall(SYS_socket, domain, type, protocol); +} + +struct sockaddr { + unsigned short sa_family; + char sa_data[14]; +}; + +struct in_addr { + uint32_t s_addr; +}; + +struct sockaddr_in { + unsigned short sin_family; + uint16_t sin_port; + struct in_addr sin_addr; + unsigned char sin_zero[sizeof (struct sockaddr) + - sizeof(unsigned short) + - sizeof(uint16_t) + - sizeof(struct in_addr)]; +}; + +static inline int linux_connect(int socketfd, const struct sockaddr_in* addr, int addrlen){ + return __syscall(SYS_connect, socketfd, addr, addrlen); +} + +static inline ptrdiff_t linux_send(int socketfd, const void* buf, size_t len, int flags){ + return __syscall(SYS_sendto, socketfd, buf, len, flags, NULL, 0); +} + +static inline ptrdiff_t linux_recv(int socketfd, void* buf, size_t len, int flags){ + return __syscall(SYS_recvfrom, socketfd, buf, len, flags, NULL, NULL); +} + +static inline int linux_close(int fd){ + return __syscall(SYS_close, fd); +} + +struct pollfd { + int fd; + short events; + short revents; +}; + +static inline int linux_ppoll(struct pollfd* fds, unsigned int nfds) { + return __syscall(SYS_ppoll, fds, nfds, NULL, NULL); +} + +#endif diff --git a/functions/busy_libc/CMakeLists.txt b/functions/busy_libc/CMakeLists.txt new file mode 100644 index 0000000..4fe9f07 --- /dev/null +++ b/functions/busy_libc/CMakeLists.txt @@ -0,0 +1,14 @@ +cmake_minimum_required(VERSION ${CMAKE_VERSION}) + +set(EXECUTABLE "busy_libc") + +add_executable(${EXECUTABLE} + busy_libc.c +) + +target_compile_options(${EXECUTABLE} PRIVATE -static -Os -flto) +target_link_options(${EXECUTABLE} PRIVATE -static -Os -flto) +target_link_libraries(${EXECUTABLE} PRIVATE + dlibc + dandelion_runtime +) \ No newline at end of file diff --git a/functions/busy_libc/busy_libc.c b/functions/busy_libc/busy_libc.c new file mode 100644 index 0000000..791d807 --- /dev/null +++ b/functions/busy_libc/busy_libc.c @@ -0,0 +1,116 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "unistd.h" + +#define ARRAY_ITEMS 16 +#define FETCH_REQUEST_PATH "/data/get_request" +#define STORE_PREAMBLE_PATH "/preamble/post_request" +#define STORE_RESPONSE_PATH "/store_request/post_response" + +typedef struct data_item { + int8_t value[ARRAY_ITEMS]; +} data_item; + +size_t read_file_to_string(const char *filename, char **content) { + // Open file in read mode + FILE *file = fopen(filename, "r"); + if (file == NULL) { + perror("Error opening file"); + return -1; + } + + // Get file size + fseek(file, 0, SEEK_END); + size_t file_size = ftell(file); + rewind(file); + + // Allocate memory for content (+1 for null terminator) + *content = (char *)malloc(file_size + 1); + if (*content == NULL) { + perror("Error allocating memory"); + return -1; // Memory allocation failed + } + + // Read file into the buffer + size_t bytes_read = fread(*content, 1, file_size, file); + (*content)[bytes_read] = '\0'; // Add null terminator + fclose(file); + + return bytes_read; // Return the length of the content +} + +int main() { + char *data = NULL; + size_t data_len = read_file_to_string(FETCH_REQUEST_PATH, &data); + if (data < 0) + return 1; + + uint64_t iterations = *(uint64_t *)data; + data_item *data_items = (data_item *)(data + 8); + + // the igreggated statistics we want to collect + // sum, average, variance, min, max + int64_t sum = 0; + int64_t min = 256; + int64_t max = -256; + + size_t struct_index = 0; + size_t value_index = 0; + size_t max_index = (data_len - 8) / sizeof(data_item); + for (size_t iteration = 0; iteration < iterations; iteration++) { + for (size_t array_index = 0; array_index < ARRAY_ITEMS; array_index++) { + int8_t value = data_items[struct_index].value[array_index]; + if (value > max) + max = value; + if (value < min) + min = value; + sum += value; + } + // using size_t which has no sign, guarantees that the index is not + // negative when computing the remainder + struct_index = + (struct_index + data_items[struct_index].value[value_index]) % + max_index; + value_index = (value_index + 1) % ARRAY_ITEMS; + } + + char *store_preamble = NULL; + size_t store_preamble_len = + read_file_to_string(STORE_PREAMBLE_PATH, &store_preamble); + if (store_preamble_len < 0) + return 7; + store_preamble_len -= 2; // omit \n\n at the end + + char header[] = "\nContent-Length: 24\n\n"; // 3 * sizeof(int64_t) + size_t store_request_len = + store_preamble_len + (sizeof(header) - 1) + 3 * sizeof(int64_t); + char *store_request = malloc(store_request_len); + char *current_ptr = store_request; + memcpy(current_ptr, store_preamble, store_preamble_len); + current_ptr += store_preamble_len; + memcpy(current_ptr, header, sizeof(header) - 1); + current_ptr += sizeof(header) - 1; + memcpy(current_ptr, &sum, sizeof(int64_t)); + current_ptr += sizeof(int64_t); + memcpy(current_ptr, &min, sizeof(int64_t)); + current_ptr += sizeof(int64_t); + memcpy(current_ptr, &max, sizeof(int64_t)); + current_ptr += sizeof(int64_t); + + FILE *store_response_file = fopen(STORE_RESPONSE_PATH, "wb"); + if (store_response_file == NULL) { + perror("Error opening file"); + return 9; + } + fwrite(store_request, 1, store_request_len, store_response_file); + fclose(store_response_file); + + return 0; +} From 79395d9bbb89fe8336fc97b7913df13cf8f35e5b Mon Sep 17 00:00:00 2001 From: richardlee <18626685+richardlee159@users.noreply.github.com> Date: Wed, 2 Apr 2025 13:59:18 +0200 Subject: [PATCH 15/19] Fix bugs --- functions/busy_hybrid/busy_hybrid.c | 6 ++---- functions/busy_libc/busy_libc.c | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/functions/busy_hybrid/busy_hybrid.c b/functions/busy_hybrid/busy_hybrid.c index 482b8d1..6fb8bbe 100644 --- a/functions/busy_hybrid/busy_hybrid.c +++ b/functions/busy_hybrid/busy_hybrid.c @@ -152,7 +152,6 @@ size_t read_file_to_string(const char *filename, char **content) { FILE *file = fopen(filename, "r"); if (file == NULL) { perror("Error opening file"); - return -1; } // Get file size @@ -164,7 +163,6 @@ size_t read_file_to_string(const char *filename, char **content) { *content = (char *)malloc(file_size + 1); if (*content == NULL) { perror("Error allocating memory"); - return -1; // Memory allocation failed } // Read file into the buffer @@ -179,7 +177,7 @@ int main() { char *fetch_preamble = NULL; size_t fetch_preamble_len = read_file_to_string(FETCH_REQUEST_PATH, &fetch_preamble); - if (fetch_preamble < 0) + if (fetch_preamble == NULL) return 1; char server_ip[16]; @@ -246,7 +244,7 @@ int main() { char *store_preamble = NULL; size_t store_preamble_len = read_file_to_string(STORE_PREAMBLE_PATH, &store_preamble); - if (store_preamble_len < 0) + if (store_preamble == NULL) return 7; store_preamble_len -= 2; // omit \n\n at the end diff --git a/functions/busy_libc/busy_libc.c b/functions/busy_libc/busy_libc.c index 791d807..9f1cda2 100644 --- a/functions/busy_libc/busy_libc.c +++ b/functions/busy_libc/busy_libc.c @@ -23,7 +23,6 @@ size_t read_file_to_string(const char *filename, char **content) { FILE *file = fopen(filename, "r"); if (file == NULL) { perror("Error opening file"); - return -1; } // Get file size @@ -35,7 +34,6 @@ size_t read_file_to_string(const char *filename, char **content) { *content = (char *)malloc(file_size + 1); if (*content == NULL) { perror("Error allocating memory"); - return -1; // Memory allocation failed } // Read file into the buffer @@ -49,7 +47,7 @@ size_t read_file_to_string(const char *filename, char **content) { int main() { char *data = NULL; size_t data_len = read_file_to_string(FETCH_REQUEST_PATH, &data); - if (data < 0) + if (data == NULL) return 1; uint64_t iterations = *(uint64_t *)data; @@ -84,7 +82,7 @@ int main() { char *store_preamble = NULL; size_t store_preamble_len = read_file_to_string(STORE_PREAMBLE_PATH, &store_preamble); - if (store_preamble_len < 0) + if (store_preamble == NULL) return 7; store_preamble_len -= 2; // omit \n\n at the end From d1b7734bfaa7fc6e7c2c82841b9b3b98ddc82c11 Mon Sep 17 00:00:00 2001 From: richardlee <18626685+richardlee159@users.noreply.github.com> Date: Wed, 2 Apr 2025 16:17:29 +0200 Subject: [PATCH 16/19] Fix more bugs --- functions/busy_hybrid/busy_hybrid.c | 1 + functions/busy_libc/busy_libc.c | 13 +++++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/functions/busy_hybrid/busy_hybrid.c b/functions/busy_hybrid/busy_hybrid.c index 6fb8bbe..e4a86f2 100644 --- a/functions/busy_hybrid/busy_hybrid.c +++ b/functions/busy_hybrid/busy_hybrid.c @@ -120,6 +120,7 @@ ssize_t send_http_request(int sockfd, const char *request, size_t request_size, return -1; } + // TODO: ensure the entire HTTP response has been received ssize_t total_received = 0, bytes; // while ((bytes = linux_recv(sockfd, response + total_received, // response_size - total_received - 1, 0)) > 0) { diff --git a/functions/busy_libc/busy_libc.c b/functions/busy_libc/busy_libc.c index 9f1cda2..4147084 100644 --- a/functions/busy_libc/busy_libc.c +++ b/functions/busy_libc/busy_libc.c @@ -7,6 +7,7 @@ #include #include +#include "dandelion/runtime.h" #include "unistd.h" #define ARRAY_ITEMS 16 @@ -45,10 +46,14 @@ size_t read_file_to_string(const char *filename, char **content) { } int main() { - char *data = NULL; - size_t data_len = read_file_to_string(FETCH_REQUEST_PATH, &data); - if (data == NULL) - return 1; + // use dandelion runtime interface directly to avoid data copy + IoBuffer *data_buffer = dandelion_get_input(0, 0); + char *data = (char *)data_buffer->data; + size_t data_len = data_buffer->data_len; + // char *data = NULL; + // size_t data_len = read_file_to_string(FETCH_REQUEST_PATH, &data); + // if (data == NULL) + // return 1; uint64_t iterations = *(uint64_t *)data; data_item *data_items = (data_item *)(data + 8); From 927becf820169d87a807f747e75791de417de758 Mon Sep 17 00:00:00 2001 From: richardlee <18626685+richardlee159@users.noreply.github.com> Date: Fri, 4 Apr 2025 11:09:48 +0200 Subject: [PATCH 17/19] Don't reuse connection in hybrid --- functions/busy_hybrid/busy_hybrid.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/functions/busy_hybrid/busy_hybrid.c b/functions/busy_hybrid/busy_hybrid.c index e4a86f2..edcfa17 100644 --- a/functions/busy_hybrid/busy_hybrid.c +++ b/functions/busy_hybrid/busy_hybrid.c @@ -205,6 +205,7 @@ int main() { sock, fetch_request, fetch_request_len, fetch_response, BUFFER_SIZE); if (fetch_response_len < 0) return 5; + close(sock); char *fetch_response_body; size_t fetch_response_body_len; @@ -267,6 +268,9 @@ int main() { // posting data char *store_response = fetch_response; + sock = connect_to_server(server_ip, server_port); + if (sock < 0) + return 3; ssize_t store_response_len = send_http_request( sock, store_request, store_request_len, store_response, BUFFER_SIZE); if (store_response_len < 0) From 7a0ae747bb540f07ebba74c24121d2b7c2579131 Mon Sep 17 00:00:00 2001 From: Tom Kuchler Date: Thu, 24 Apr 2025 12:17:30 +0200 Subject: [PATCH 18/19] Switch to using the SDK provided CMake definitions of dlibc dlibcxx --- functions/CMakeLists.txt | 104 ++++++++-------------------- functions/compression/compression.c | 1 + functions/stdio_cxx/CMakeLists.txt | 25 ------- 3 files changed, 31 insertions(+), 99 deletions(-) diff --git a/functions/CMakeLists.txt b/functions/CMakeLists.txt index a7a81ea..c1b95eb 100644 --- a/functions/CMakeLists.txt +++ b/functions/CMakeLists.txt @@ -39,51 +39,7 @@ execute_process( COMMAND_ERROR_IS_FATAL ANY ) -add_library(dandelion_runtime INTERFACE IMPORTED) -target_include_directories(dandelion_runtime INTERFACE - "${CMAKE_BINARY_DIR}/dandelion_sdk/include" -) -target_compile_options(dandelion_runtime INTERFACE -nostdinc -idirafter${COMPILER_RUNTIME_INCLUDE}) -target_link_libraries(dandelion_runtime INTERFACE - "${CMAKE_BINARY_DIR}/dandelion_sdk/lib/libdandelion_runtime.a" - "${CMAKE_BINARY_DIR}/dandelion_sdk/lib/libdandelion_system.a" -) -target_link_options(dandelion_runtime INTERFACE -static -nostdlib) - -add_library(dlibc INTERFACE IMPORTED) -target_include_directories(dlibc INTERFACE - "${CMAKE_BINARY_DIR}/dandelion_sdk/include" - "${CMAKE_BINARY_DIR}/dandelion_sdk/include/sys" -) -target_link_libraries(dlibc INTERFACE - "${CMAKE_BINARY_DIR}/dandelion_sdk/lib/libc.a" - "${CMAKE_BINARY_DIR}/dandelion_sdk/lib/libg.a" - "${CMAKE_BINARY_DIR}/dandelion_sdk/lib/libm.a" - "${CMAKE_BINARY_DIR}/dandelion_sdk/lib/libdandelion_file_system.a" -) -target_link_options(dlibc INTERFACE -T${CMAKE_BINARY_DIR}/dandelion_sdk/linker.ld) - -add_library(dlibcxx INTERFACE IMPORTED) -target_include_directories(dlibcxx INTERFACE "${CMAKE_BINARY_DIR}/dandelion_sdk/include/c++/v1") -target_link_libraries(dlibcxx INTERFACE - "${CMAKE_BINARY_DIR}/dandelion_sdk/lib/libc++.a" - "${CMAKE_BINARY_DIR}/dandelion_sdk/lib/libc++abi.a" - "${CMAKE_BINARY_DIR}/dandelion_sdk/lib/libunwind.a" -) - -# create compiler wrapper for use to compile non integrated programs -configure_file( - ${CMAKE_CURRENT_SOURCE_DIR}/cfg_template.txt - ${CMAKE_BINARY_DIR}/dandelion.cfg - FILE_PERMISSIONS OWNER_READ -) - -# create compiler wrapper for use to compile non integrated programs -configure_file( - ${CMAKE_CURRENT_SOURCE_DIR}/cfg++_template.txt - ${CMAKE_BINARY_DIR}/dandelion++.cfg - FILE_PERMISSIONS OWNER_READ -) +add_subdirectory("${CMAKE_BINARY_DIR}/dandelion_sdk" "${CMAKE_BINARY_DIR}/dandelion_sdk_build") if(CMAKE_BUILD_TYPE MATCHES "Debug") add_compile_options(-gdwarf-4) @@ -124,32 +80,32 @@ add_subdirectory(compression) add_subdirectory(busy_libc) add_subdirectory(busy_hybrid) -set(CONFIG_FLAG "--config ${CMAKE_BINARY_DIR}/dandelion.cfg") -set(CONFIG_FLAGXX "--config ${CMAKE_BINARY_DIR}/dandelion++.cfg") - -include(ExternalProject) -ExternalProject_Add( - llvmproject - PREFIX llvmproject - GIT_REPOSITORY https://github.com/llvm/llvm-project - GIT_TAG release/18.x - SOURCE_SUBDIR llvm - UPDATE_DISCONNECTED 1 - INSTALL_DIR ${CMAKE_BINARY_DIR}/llvm - LIST_SEPARATOR "," - CMAKE_ARGS - -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} - -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -DCMAKE_TRY_COMPILE_TARGET_TYPE=STATIC_LIBRARY - -DCMAKE_SYSTEM_NAME=Generic - -DCMAKE_INSTALL_PREFIX= - -DCMAKE_C_FLAGS:STRING=${CONFIG_FLAG} - -DCMAKE_CXX_FLAGS:STRING=${CONFIG_FLAGXX} - -DLLVM_ENABLE_PROJECTS=clang,lld - -DLLVM_TARGETS_TO_BUILD=AArch64,X86 - -DBUILD_SHARED_LIBS=OFF - -DLLVM_ENABLE_PIC=OFF - -DLLVM_ENABLE_THREADS=OFF - EXCLUDE_FROM_ALL TRUE -) \ No newline at end of file +# set(CONFIG_FLAG "--config ${CMAKE_BINARY_DIR}/dandelion.cfg") +# set(CONFIG_FLAGXX "--config ${CMAKE_BINARY_DIR}/dandelion++.cfg") + +# include(ExternalProject) +# ExternalProject_Add( +# llvmproject +# PREFIX llvmproject +# GIT_REPOSITORY https://github.com/llvm/llvm-project +# GIT_TAG release/18.x +# SOURCE_SUBDIR llvm +# UPDATE_DISCONNECTED 1 +# INSTALL_DIR ${CMAKE_BINARY_DIR}/llvm +# LIST_SEPARATOR "," +# CMAKE_ARGS +# -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} +# -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} +# -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} +# -DCMAKE_TRY_COMPILE_TARGET_TYPE=STATIC_LIBRARY +# -DCMAKE_SYSTEM_NAME=Generic +# -DCMAKE_INSTALL_PREFIX= +# -DCMAKE_C_FLAGS:STRING=${CONFIG_FLAG} +# -DCMAKE_CXX_FLAGS:STRING=${CONFIG_FLAGXX} +# -DLLVM_ENABLE_PROJECTS=clang,lld +# -DLLVM_TARGETS_TO_BUILD=AArch64,X86 +# -DBUILD_SHARED_LIBS=OFF +# -DLLVM_ENABLE_PIC=OFF +# -DLLVM_ENABLE_THREADS=OFF +# EXCLUDE_FROM_ALL TRUE +# ) \ No newline at end of file diff --git a/functions/compression/compression.c b/functions/compression/compression.c index 3bb9f24..9826247 100644 --- a/functions/compression/compression.c +++ b/functions/compression/compression.c @@ -27,6 +27,7 @@ void *realloc_sized(void *old, size_t old_size, size_t new_size) { } #define STBI_WRITE_NO_STDIO +#define STBIW_ASSERT(x) {} #define STBIW_MALLOC(size) dandelion_alloc(size, 8) #define STBIW_FREE(ptr) dandelion_free(ptr) #define STBIW_REALLOC_SIZED(ptr, old_size, new_size) \ diff --git a/functions/stdio_cxx/CMakeLists.txt b/functions/stdio_cxx/CMakeLists.txt index d226de1..1615e11 100644 --- a/functions/stdio_cxx/CMakeLists.txt +++ b/functions/stdio_cxx/CMakeLists.txt @@ -6,31 +6,6 @@ add_executable(${EXECUTABLE} stdio.cpp ) -# target_compile_options(${EXECUTABLE} PRIVATE -# -gdwarf-4 -# -static -# -O0 -# -nostdinc -# -I/mnt/c/Users/kuchlert/Documents/fuseArch/dandelionProj/dandelionFunctionExamples/build_debug/dandelion_sdk/include/c++/v1 -# # -I/mnt/c/Users/kuchlert/Documents/fuseArch/dandelionProj/dandelionFunctionExamples/build_debug/dandelion_sdk/include/machine -# # -I/mnt/c/Users/kuchlert/Documents/fuseArch/dandelionProj/dandelionFunctionExamples/build_debug/dandelion_sdk/include/ssp -# -I/mnt/c/Users/kuchlert/Documents/fuseArch/dandelionProj/dandelionFunctionExamples/build_debug/dandelion_sdk/include -# -idirafter/mnt/c/Users/kuchlert/Documents/fuseArch/dandelionProj/dandelionFunctionExamples/build_debug/dandelion_sdk/include/sys -# -I/usr/lib/llvm-14/lib/clang/14.0.6/include -# ) -# target_link_options(${EXECUTABLE} PRIVATE -static -# -nostdlib -# -L/mnt/c/Users/kuchlert/Documents/fuseArch/dandelionProj/dandelionFunctionExamples/build_debug/dandelion_sdk/lib -# -lc -# -lc++ -# -lc++abi -# -lg -# -lm -# -ldandelion_file_system -# -ldandelion_runtime -# -ldandelion_system -# -lunwind -# ) target_link_libraries(${EXECUTABLE} PRIVATE dlibcxx dlibc From 471fbb00afe930d6c6ec5943ab12262b2b341e1d Mon Sep 17 00:00:00 2001 From: Tom Kuchler Date: Wed, 30 Apr 2025 14:35:00 +0200 Subject: [PATCH 19/19] Prepare for open sourcing --- .github/workflows/file_checks.yml | 18 + LICENSE | 7 + LICENSES/MIT.txt | 1 + README.md | 11 + {functions => c_functions}/CMakeLists.txt | 0 c_functions/README.md | 29 + .../basic/CMakeLists.txt | 0 {functions => c_functions}/basic/basic.c | 0 .../busy/CMakeLists.txt | 0 {functions => c_functions}/busy/busy.c | 0 .../busy_hybrid/CMakeLists.txt | 0 c_functions/busy_hybrid/busy_hybrid.c | 289 ++++++++ .../busy_hybrid/linux_syscalls/CMakeLists.txt | 0 .../arch/aarch64/syscall_arch.h | 0 .../linux_syscalls/arch/x86_64/syscall_arch.h | 0 .../busy_hybrid}/linux_syscalls/syscall.h | 41 +- .../busy_libc/CMakeLists.txt | 0 c_functions/busy_libc/busy_libc.c | 119 ++++ {functions => c_functions}/cfg++_template.txt | 0 {functions => c_functions}/cfg_template.txt | 0 .../compression/CMakeLists.txt | 0 .../compression/compression.c | 4 +- {functions => c_functions}/compression/qoi.h | 625 +++++++++--------- .../compression/stb_image_write.h | 0 .../dirigent_busy/CMakeLists.txt | 0 c_functions/dirigent_busy/dirigent_busy.c | 87 +++ .../example_app/CMakeLists.txt | 0 .../example_app/fan_out.c | 26 +- .../example_app/handle.c | 8 +- c_functions/example_app/template.c | 123 ++++ .../example_app_hybrid/CMakeLists.txt | 0 c_functions/example_app_hybrid/app_hybrid.c | 410 ++++++++++++ .../linux_syscalls/CMakeLists.txt | 0 .../arch/aarch64/syscall_arch.h | 0 .../linux_syscalls/arch/x86_64/syscall_arch.h | 0 .../linux_syscalls/syscall.h | 41 +- .../example_app_nolibc/CMakeLists.txt | 0 .../example_app_nolibc/commons.h | 0 .../example_app_nolibc/fan_out.c | 0 .../example_app_nolibc/handle.c | 0 .../example_app_nolibc/template.c | 0 .../files/CMakeLists.txt | 0 {functions => c_functions}/files/fileio.c | 6 +- .../matmac/CMakeLists.txt | 0 {functions => c_functions}/matmac/matmac.c | 0 .../matmul/CMakeLists.txt | 0 {functions => c_functions}/matmul/matmul.c | 0 .../stdio/CMakeLists.txt | 0 {functions => c_functions}/stdio/stdio.c | 7 +- .../stdio_cxx/CMakeLists.txt | 0 .../stdio_cxx/stdio.cpp | 0 functions/README.md | 17 - functions/busy_hybrid/busy_hybrid.c | 289 -------- functions/busy_libc/busy_libc.c | 119 ---- functions/dirigent_busy/dirigent_busy.c | 85 --- functions/example_app/template.c | 129 ---- functions/example_app_hybrid/app_hybrid.c | 399 ----------- rust_functions/Cargo.toml | 18 - rust_functions/src/aes_func.rs | 27 - rust_functions/src/float_operation.rs | 13 - rust_functions/src/linpack.rs | 7 - rust_functions/src/main.rs | 47 -- rust_functions/src/matmul.rs | 34 - 63 files changed, 1460 insertions(+), 1576 deletions(-) create mode 100644 .github/workflows/file_checks.yml create mode 100644 LICENSE create mode 120000 LICENSES/MIT.txt create mode 100644 README.md rename {functions => c_functions}/CMakeLists.txt (100%) create mode 100644 c_functions/README.md rename {functions => c_functions}/basic/CMakeLists.txt (100%) rename {functions => c_functions}/basic/basic.c (100%) rename {functions => c_functions}/busy/CMakeLists.txt (100%) rename {functions => c_functions}/busy/busy.c (100%) rename {functions => c_functions}/busy_hybrid/CMakeLists.txt (100%) create mode 100644 c_functions/busy_hybrid/busy_hybrid.c rename {functions => c_functions}/busy_hybrid/linux_syscalls/CMakeLists.txt (100%) rename {functions => c_functions}/busy_hybrid/linux_syscalls/arch/aarch64/syscall_arch.h (100%) rename {functions => c_functions}/busy_hybrid/linux_syscalls/arch/x86_64/syscall_arch.h (100%) rename {functions/example_app_hybrid => c_functions/busy_hybrid}/linux_syscalls/syscall.h (74%) rename {functions => c_functions}/busy_libc/CMakeLists.txt (100%) create mode 100644 c_functions/busy_libc/busy_libc.c rename {functions => c_functions}/cfg++_template.txt (100%) rename {functions => c_functions}/cfg_template.txt (100%) rename {functions => c_functions}/compression/CMakeLists.txt (100%) rename {functions => c_functions}/compression/compression.c (92%) rename {functions => c_functions}/compression/qoi.h (56%) rename {functions => c_functions}/compression/stb_image_write.h (100%) rename {functions => c_functions}/dirigent_busy/CMakeLists.txt (100%) create mode 100644 c_functions/dirigent_busy/dirigent_busy.c rename {functions => c_functions}/example_app/CMakeLists.txt (100%) rename {functions => c_functions}/example_app/fan_out.c (64%) rename {functions => c_functions}/example_app/handle.c (84%) create mode 100644 c_functions/example_app/template.c rename {functions => c_functions}/example_app_hybrid/CMakeLists.txt (100%) create mode 100644 c_functions/example_app_hybrid/app_hybrid.c rename {functions => c_functions}/example_app_hybrid/linux_syscalls/CMakeLists.txt (100%) rename {functions => c_functions}/example_app_hybrid/linux_syscalls/arch/aarch64/syscall_arch.h (100%) rename {functions => c_functions}/example_app_hybrid/linux_syscalls/arch/x86_64/syscall_arch.h (100%) rename {functions/busy_hybrid => c_functions/example_app_hybrid}/linux_syscalls/syscall.h (74%) rename {functions => c_functions}/example_app_nolibc/CMakeLists.txt (100%) rename {functions => c_functions}/example_app_nolibc/commons.h (100%) rename {functions => c_functions}/example_app_nolibc/fan_out.c (100%) rename {functions => c_functions}/example_app_nolibc/handle.c (100%) rename {functions => c_functions}/example_app_nolibc/template.c (100%) rename {functions => c_functions}/files/CMakeLists.txt (100%) rename {functions => c_functions}/files/fileio.c (91%) rename {functions => c_functions}/matmac/CMakeLists.txt (100%) rename {functions => c_functions}/matmac/matmac.c (100%) rename {functions => c_functions}/matmul/CMakeLists.txt (100%) rename {functions => c_functions}/matmul/matmul.c (100%) rename {functions => c_functions}/stdio/CMakeLists.txt (100%) rename {functions => c_functions}/stdio/stdio.c (83%) rename {functions => c_functions}/stdio_cxx/CMakeLists.txt (100%) rename {functions => c_functions}/stdio_cxx/stdio.cpp (100%) delete mode 100644 functions/README.md delete mode 100644 functions/busy_hybrid/busy_hybrid.c delete mode 100644 functions/busy_libc/busy_libc.c delete mode 100644 functions/dirigent_busy/dirigent_busy.c delete mode 100644 functions/example_app/template.c delete mode 100644 functions/example_app_hybrid/app_hybrid.c delete mode 100644 rust_functions/Cargo.toml delete mode 100644 rust_functions/src/aes_func.rs delete mode 100644 rust_functions/src/float_operation.rs delete mode 100644 rust_functions/src/linpack.rs delete mode 100644 rust_functions/src/main.rs delete mode 100644 rust_functions/src/matmul.rs diff --git a/.github/workflows/file_checks.yml b/.github/workflows/file_checks.yml new file mode 100644 index 0000000..a462c32 --- /dev/null +++ b/.github/workflows/file_checks.yml @@ -0,0 +1,18 @@ +name: Format checks + +on: + push: + branches: [ "main" ] + + pull_request: + branches: [ "main" ] + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: C format check + run: find functions/ -iname '*.h' -o -iname '*.c' | clang-format --dry-run -Werror --files=/dev/stdin --style=LLVM \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d28f46c --- /dev/null +++ b/LICENSE @@ -0,0 +1,7 @@ +Copyright (c) 2025 ETH Zurich EASL + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the " Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/LICENSES/MIT.txt b/LICENSES/MIT.txt new file mode 120000 index 0000000..ea5b606 --- /dev/null +++ b/LICENSES/MIT.txt @@ -0,0 +1 @@ +../LICENSE \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..452bcd7 --- /dev/null +++ b/README.md @@ -0,0 +1,11 @@ +## Overview + +This repository contains examples of how to write functions to run on [Dandelion](https://github.com/eth-easl/dandelion) + +The different folders contain examples and more information for the supported languages. + +Curretly we only have full support for C and C++. +Examples for how to write functions, setup builds and other details can be found in the `c_functons` folder. + +There is also support for functions compiled to Wasm. +The tooling to compile functions this way is in the `wasm-compiler` folder. \ No newline at end of file diff --git a/functions/CMakeLists.txt b/c_functions/CMakeLists.txt similarity index 100% rename from functions/CMakeLists.txt rename to c_functions/CMakeLists.txt diff --git a/c_functions/README.md b/c_functions/README.md new file mode 100644 index 0000000..f0a5959 --- /dev/null +++ b/c_functions/README.md @@ -0,0 +1,29 @@ +# Functions + +This folder contains a series of functions used for basic functionality tests or as example applications. +The functions are split into multiple executables that can be built individually. + +All C and C++ functions depend on the [dandelion SDK](https://github.com/eth-easl/dandelion). +Functions can either only use the lowest level interface `dandelion_runtime` or the `dlibc`/`dlibcxx`. +The runtime provides basic wrapper to start a function and interact directly with input and output sets and buffers. +The libc variants also support reading and writing inputs and outputs to a filesystem abstraction, which is further explained in the SDK documentation. + +# Building + +The build for this repository is set up to allow building functions for all backends and architectures supported by the SDK. +To set this up use the following commands: + +``` +mkdir build +cd build +cmake -DPLATFORM= -DARCHITECTURE= -DCMAKE_BUILD_TYPE- . +``` + +The platforms supported are `CHERI`, `MMU_FREEBSD`, `MMU_LINUX`, `KVM`, `WASM`, `DEBUG`. \ +We support `x86_64` and `aarch64` architectures. \ +The build type can be either `Debug` or `Release`. + +When one of the parameters is not given, it defaults to the `DEBUG` platform, the build system architecture and `Debug` build type. + +The CMake in this project is set up to automatically download the latest version of the dandelion SDK when it is initially set up. +The easiest way to use a different version, is to replace the `dandelion_sdk` folder in the build folder with a symlink to the version to be used. \ No newline at end of file diff --git a/functions/basic/CMakeLists.txt b/c_functions/basic/CMakeLists.txt similarity index 100% rename from functions/basic/CMakeLists.txt rename to c_functions/basic/CMakeLists.txt diff --git a/functions/basic/basic.c b/c_functions/basic/basic.c similarity index 100% rename from functions/basic/basic.c rename to c_functions/basic/basic.c diff --git a/functions/busy/CMakeLists.txt b/c_functions/busy/CMakeLists.txt similarity index 100% rename from functions/busy/CMakeLists.txt rename to c_functions/busy/CMakeLists.txt diff --git a/functions/busy/busy.c b/c_functions/busy/busy.c similarity index 100% rename from functions/busy/busy.c rename to c_functions/busy/busy.c diff --git a/functions/busy_hybrid/CMakeLists.txt b/c_functions/busy_hybrid/CMakeLists.txt similarity index 100% rename from functions/busy_hybrid/CMakeLists.txt rename to c_functions/busy_hybrid/CMakeLists.txt diff --git a/c_functions/busy_hybrid/busy_hybrid.c b/c_functions/busy_hybrid/busy_hybrid.c new file mode 100644 index 0000000..d8b01ab --- /dev/null +++ b/c_functions/busy_hybrid/busy_hybrid.c @@ -0,0 +1,289 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "syscall.h" +#include "unistd.h" + +#define ARRAY_ITEMS 16 +#define BUFFER_SIZE 1 * 1024 * 1024 +#define FETCH_REQUEST_PATH "/data/get_request" +#define STORE_PREAMBLE_PATH "/preamble/post_request" +#define STORE_RESPONSE_PATH "/store_request/post_response" + +typedef struct data_item { + int8_t value[ARRAY_ITEMS]; +} data_item; + +uint16_t my_htons(uint16_t port) { + return ((port >> 8) & 0x00FF) | ((port << 8) & 0xFF00); +} + +int my_inet_pton(int af, const char *src, void *dst) { + if (af == LINUX_AF_INET) { + uint8_t array[4] = {}; + for (int i = 0; i < 4; i++) { + char *end; + long value = strtol(src, &end, 10); + if (value < 0 || value > 255) { + return 0; + } + + if (*end != '\0' && *end != '.' && i < 3) { + return 0; + } + src = end + 1; + array[i] = (uint8_t)value; + } + + struct in_addr *addr = (struct in_addr *)dst; + memcpy(&addr->s_addr, array, 4); // Copy the 4 bytes into s_addr + + return 1; // Success + } + return -1; // Unsupported address family +} + +int extract_ip_port(const char *http_request, char *ip, size_t ip_len, + uint16_t *port) { + const char *prefix = "http://"; + const char *start = strstr(http_request, prefix); + if (!start) + return -1; + + start += strlen(prefix); + const char *colon = strchr(start, ':'); + if (!colon) + return -2; + + const char *slash = strchr(colon, '/'); + if (!slash) + return -3; + + // Extract IP + size_t ip_size = colon - start; + if (ip_size >= ip_len) + return -4; + strncpy(ip, start, ip_size); + ip[ip_size] = '\0'; + + // Extract port + char port_str[6] = {0}; + size_t port_size = slash - colon - 1; + if (port_size >= sizeof(port_str)) + return -5; + strncpy(port_str, colon + 1, port_size); + *port = atoi(port_str); + + return 0; +} + +int connect_to_server(const char *ip, uint16_t port) { + struct sockaddr_in server_addr; + + int sockfd = linux_socket(LINUX_AF_INET, LINUX_SOCK_STREAM, 0); + if (sockfd < 0) { + perror("Socket creation failed"); + return -1; + } + + server_addr.sin_family = LINUX_AF_INET; + server_addr.sin_port = my_htons(port); + + if (my_inet_pton(LINUX_AF_INET, ip, &server_addr.sin_addr) != 1) { + perror("Invalid IP address"); + close(sockfd); + return -1; + } + + int err = linux_connect(sockfd, &server_addr, sizeof(server_addr)); + if (err < 0) { + printf("Error: %s\n", strerror(-err)); + printf("size of server_addr: %ld\n", sizeof(server_addr)); + perror("Socket connection failed"); + close(sockfd); + return -1; + } + + return sockfd; +} + +ssize_t send_http_request(int sockfd, const char *request, size_t request_size, + char *response, size_t response_size) { + if (linux_send(sockfd, request, request_size, 0) < 0) { + perror("Error sending request"); + return -1; + } + + // TODO: ensure the entire HTTP response has been received + ssize_t total_received = 0, bytes; + // while ((bytes = linux_recv(sockfd, response + total_received, + // response_size - total_received - 1, 0)) > 0) { + // total_received += bytes; + // if (total_received >= response_size - 1) break; + // } + // blocks and waits forever if there is no next packate, could validate HTTP + // sice on the packages to see if we need more, this works for now + total_received = linux_recv(sockfd, response + total_received, + response_size - total_received - 1, 0); + if (total_received < 0) { + perror("Error receiving response"); + return -1; + } + response[total_received] = '\0'; + return total_received; +} + +void parse_http_body(const char *response, size_t response_size, + char **response_body, size_t *response_body_size) { + *response_body = strstr(response, "\r\n\r\n"); + if (*response_body == NULL) + return; + *response_body += 4; + *response_body_size = response_size - (*response_body - response); +} + +size_t read_file_to_string(const char *filename, char **content) { + // Open file in read mode + FILE *file = fopen(filename, "r"); + if (file == NULL) { + perror("Error opening file"); + } + + // Get file size + fseek(file, 0, SEEK_END); + size_t file_size = ftell(file); + rewind(file); + + // Allocate memory for content (+1 for null terminator) + *content = (char *)malloc(file_size + 1); + if (*content == NULL) { + perror("Error allocating memory"); + } + + // Read file into the buffer + size_t bytes_read = fread(*content, 1, file_size, file); + (*content)[bytes_read] = '\0'; // Add null terminator + fclose(file); + + return bytes_read; // Return the length of the content +} + +int main() { + char *fetch_preamble = NULL; + size_t fetch_preamble_len = + read_file_to_string(FETCH_REQUEST_PATH, &fetch_preamble); + if (fetch_preamble == NULL) + return 1; + + char server_ip[16]; + uint16_t server_port; + if (extract_ip_port(fetch_preamble, server_ip, 16, &server_port) < 0) + return 2; + + // fetching data + int sock = connect_to_server(server_ip, server_port); + if (sock < 0) + return 3; + + size_t fetch_request_len = fetch_preamble_len + 4; + char *fetch_request = malloc(fetch_request_len); + memcpy(fetch_request, fetch_preamble, fetch_preamble_len); + memcpy(fetch_request + fetch_preamble_len, "\r\n\r\n", 4); + + char *fetch_response = (char *)malloc(BUFFER_SIZE); + if (fetch_response == NULL) { + perror("Error allocating memory"); + return 4; + } + ssize_t fetch_response_len = send_http_request( + sock, fetch_request, fetch_request_len, fetch_response, BUFFER_SIZE); + if (fetch_response_len < 0) + return 5; + close(sock); + + char *fetch_response_body; + size_t fetch_response_body_len; + parse_http_body(fetch_response, fetch_response_len, &fetch_response_body, + &fetch_response_body_len); + if (fetch_response_body == NULL) + return 6; + + uint64_t iterations = *(uint64_t *)fetch_response_body; + data_item *data_items = (data_item *)(fetch_response_body + 8); + + // the igreggated statistics we want to collect + // sum, average, variance, min, max + int64_t sum = 0; + int64_t min = 256; + int64_t max = -256; + + size_t struct_index = 0; + size_t value_index = 0; + size_t max_index = (fetch_response_body_len - 8) / sizeof(data_item); + for (size_t iteration = 0; iteration < iterations; iteration++) { + for (size_t array_index = 0; array_index < ARRAY_ITEMS; array_index++) { + int8_t value = data_items[struct_index].value[array_index]; + if (value > max) + max = value; + if (value < min) + min = value; + sum += value; + } + // using size_t which has no sign, guarantees that the index is not + // negative when computing the remainder + struct_index = + (struct_index + data_items[struct_index].value[value_index]) % + max_index; + value_index = (value_index + 1) % ARRAY_ITEMS; + } + + char *store_preamble = NULL; + size_t store_preamble_len = + read_file_to_string(STORE_PREAMBLE_PATH, &store_preamble); + if (store_preamble == NULL) + return 7; + store_preamble_len -= 2; // omit \n\n at the end + + char header[] = "\r\ncontent-length: 24\r\n\r\n"; // 3 * sizeof(int64_t) + size_t store_request_len = + store_preamble_len + (sizeof(header) - 1) + 3 * sizeof(int64_t); + char *store_request = malloc(store_request_len); + char *current_ptr = store_request; + memcpy(current_ptr, store_preamble, store_preamble_len); + current_ptr += store_preamble_len; + memcpy(current_ptr, header, sizeof(header) - 1); + current_ptr += sizeof(header) - 1; + memcpy(current_ptr, &sum, sizeof(int64_t)); + current_ptr += sizeof(int64_t); + memcpy(current_ptr, &min, sizeof(int64_t)); + current_ptr += sizeof(int64_t); + memcpy(current_ptr, &max, sizeof(int64_t)); + current_ptr += sizeof(int64_t); + + // posting data + char *store_response = fetch_response; + sock = connect_to_server(server_ip, server_port); + if (sock < 0) + return 3; + ssize_t store_response_len = send_http_request( + sock, store_request, store_request_len, store_response, BUFFER_SIZE); + if (store_response_len < 0) + return 8; + close(sock); + + FILE *store_response_file = fopen(STORE_RESPONSE_PATH, "wb"); + if (store_response_file == NULL) { + perror("Error opening file"); + return 9; + } + fwrite(store_response, 1, store_response_len, store_response_file); + fclose(store_response_file); + + return 0; +} diff --git a/functions/busy_hybrid/linux_syscalls/CMakeLists.txt b/c_functions/busy_hybrid/linux_syscalls/CMakeLists.txt similarity index 100% rename from functions/busy_hybrid/linux_syscalls/CMakeLists.txt rename to c_functions/busy_hybrid/linux_syscalls/CMakeLists.txt diff --git a/functions/busy_hybrid/linux_syscalls/arch/aarch64/syscall_arch.h b/c_functions/busy_hybrid/linux_syscalls/arch/aarch64/syscall_arch.h similarity index 100% rename from functions/busy_hybrid/linux_syscalls/arch/aarch64/syscall_arch.h rename to c_functions/busy_hybrid/linux_syscalls/arch/aarch64/syscall_arch.h diff --git a/functions/busy_hybrid/linux_syscalls/arch/x86_64/syscall_arch.h b/c_functions/busy_hybrid/linux_syscalls/arch/x86_64/syscall_arch.h similarity index 100% rename from functions/busy_hybrid/linux_syscalls/arch/x86_64/syscall_arch.h rename to c_functions/busy_hybrid/linux_syscalls/arch/x86_64/syscall_arch.h diff --git a/functions/example_app_hybrid/linux_syscalls/syscall.h b/c_functions/busy_hybrid/linux_syscalls/syscall.h similarity index 74% rename from functions/example_app_hybrid/linux_syscalls/syscall.h rename to c_functions/busy_hybrid/linux_syscalls/syscall.h index 17c166a..723eceb 100644 --- a/functions/example_app_hybrid/linux_syscalls/syscall.h +++ b/c_functions/busy_hybrid/linux_syscalls/syscall.h @@ -18,12 +18,12 @@ #define LINUX_SOCK_STREAM 1 // poll defines -#define POLLIN 0x0001 -#define POLLPRI 0x0002 -#define POLLOUT 0x0004 -#define POLLERR 0x0008 -#define POLLHUP 0x0010 -#define POLLNVAL 0x0020 +#define POLLIN 0x0001 +#define POLLPRI 0x0002 +#define POLLOUT 0x0004 +#define POLLERR 0x0008 +#define POLLHUP 0x0010 +#define POLLNVAL 0x0020 #ifndef __scc #define __scc(X) ((long)(X)) @@ -63,34 +63,33 @@ struct sockaddr { }; struct in_addr { - uint32_t s_addr; + uint32_t s_addr; }; struct sockaddr_in { - unsigned short sin_family; - uint16_t sin_port; - struct in_addr sin_addr; - unsigned char sin_zero[sizeof (struct sockaddr) - - sizeof(unsigned short) - - sizeof(uint16_t) - - sizeof(struct in_addr)]; + unsigned short sin_family; + uint16_t sin_port; + struct in_addr sin_addr; + unsigned char sin_zero[sizeof(struct sockaddr) - sizeof(unsigned short) - + sizeof(uint16_t) - sizeof(struct in_addr)]; }; -static inline int linux_connect(int socketfd, const struct sockaddr_in* addr, int addrlen){ +static inline int linux_connect(int socketfd, const struct sockaddr_in *addr, + int addrlen) { return __syscall(SYS_connect, socketfd, addr, addrlen); } -static inline ptrdiff_t linux_send(int socketfd, const void* buf, size_t len, int flags){ +static inline ptrdiff_t linux_send(int socketfd, const void *buf, size_t len, + int flags) { return __syscall(SYS_sendto, socketfd, buf, len, flags, NULL, 0); } -static inline ptrdiff_t linux_recv(int socketfd, void* buf, size_t len, int flags){ +static inline ptrdiff_t linux_recv(int socketfd, void *buf, size_t len, + int flags) { return __syscall(SYS_recvfrom, socketfd, buf, len, flags, NULL, NULL); } -static inline int linux_close(int fd){ - return __syscall(SYS_close, fd); -} +static inline int linux_close(int fd) { return __syscall(SYS_close, fd); } struct pollfd { int fd; @@ -98,7 +97,7 @@ struct pollfd { short revents; }; -static inline int linux_ppoll(struct pollfd* fds, unsigned int nfds) { +static inline int linux_ppoll(struct pollfd *fds, unsigned int nfds) { return __syscall(SYS_ppoll, fds, nfds, NULL, NULL); } diff --git a/functions/busy_libc/CMakeLists.txt b/c_functions/busy_libc/CMakeLists.txt similarity index 100% rename from functions/busy_libc/CMakeLists.txt rename to c_functions/busy_libc/CMakeLists.txt diff --git a/c_functions/busy_libc/busy_libc.c b/c_functions/busy_libc/busy_libc.c new file mode 100644 index 0000000..6297f62 --- /dev/null +++ b/c_functions/busy_libc/busy_libc.c @@ -0,0 +1,119 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dandelion/runtime.h" +#include "unistd.h" + +#define ARRAY_ITEMS 16 +#define FETCH_REQUEST_PATH "/data/get_request" +#define STORE_PREAMBLE_PATH "/preamble/post_request" +#define STORE_RESPONSE_PATH "/store_request/post_response" + +typedef struct data_item { + int8_t value[ARRAY_ITEMS]; +} data_item; + +size_t read_file_to_string(const char *filename, char **content) { + // Open file in read mode + FILE *file = fopen(filename, "r"); + if (file == NULL) { + perror("Error opening file"); + } + + // Get file size + fseek(file, 0, SEEK_END); + size_t file_size = ftell(file); + rewind(file); + + // Allocate memory for content (+1 for null terminator) + *content = (char *)malloc(file_size + 1); + if (*content == NULL) { + perror("Error allocating memory"); + } + + // Read file into the buffer + size_t bytes_read = fread(*content, 1, file_size, file); + (*content)[bytes_read] = '\0'; // Add null terminator + fclose(file); + + return bytes_read; // Return the length of the content +} + +int main() { + // use dandelion runtime interface directly to avoid data copy + IoBuffer *data_buffer = dandelion_get_input(0, 0); + char *data = (char *)data_buffer->data; + size_t data_len = data_buffer->data_len; + // char *data = NULL; + // size_t data_len = read_file_to_string(FETCH_REQUEST_PATH, &data); + // if (data == NULL) + // return 1; + + uint64_t iterations = *(uint64_t *)data; + data_item *data_items = (data_item *)(data + 8); + + // the igreggated statistics we want to collect + // sum, average, variance, min, max + int64_t sum = 0; + int64_t min = 256; + int64_t max = -256; + + size_t struct_index = 0; + size_t value_index = 0; + size_t max_index = (data_len - 8) / sizeof(data_item); + for (size_t iteration = 0; iteration < iterations; iteration++) { + for (size_t array_index = 0; array_index < ARRAY_ITEMS; array_index++) { + int8_t value = data_items[struct_index].value[array_index]; + if (value > max) + max = value; + if (value < min) + min = value; + sum += value; + } + // using size_t which has no sign, guarantees that the index is not + // negative when computing the remainder + struct_index = + (struct_index + data_items[struct_index].value[value_index]) % + max_index; + value_index = (value_index + 1) % ARRAY_ITEMS; + } + + char *store_preamble = NULL; + size_t store_preamble_len = + read_file_to_string(STORE_PREAMBLE_PATH, &store_preamble); + if (store_preamble == NULL) + return 7; + store_preamble_len -= 2; // omit \n\n at the end + + char header[] = "\nContent-Length: 24\n\n"; // 3 * sizeof(int64_t) + size_t store_request_len = + store_preamble_len + (sizeof(header) - 1) + 3 * sizeof(int64_t); + char *store_request = malloc(store_request_len); + char *current_ptr = store_request; + memcpy(current_ptr, store_preamble, store_preamble_len); + current_ptr += store_preamble_len; + memcpy(current_ptr, header, sizeof(header) - 1); + current_ptr += sizeof(header) - 1; + memcpy(current_ptr, &sum, sizeof(int64_t)); + current_ptr += sizeof(int64_t); + memcpy(current_ptr, &min, sizeof(int64_t)); + current_ptr += sizeof(int64_t); + memcpy(current_ptr, &max, sizeof(int64_t)); + current_ptr += sizeof(int64_t); + + FILE *store_response_file = fopen(STORE_RESPONSE_PATH, "wb"); + if (store_response_file == NULL) { + perror("Error opening file"); + return 9; + } + fwrite(store_request, 1, store_request_len, store_response_file); + fclose(store_response_file); + + return 0; +} diff --git a/functions/cfg++_template.txt b/c_functions/cfg++_template.txt similarity index 100% rename from functions/cfg++_template.txt rename to c_functions/cfg++_template.txt diff --git a/functions/cfg_template.txt b/c_functions/cfg_template.txt similarity index 100% rename from functions/cfg_template.txt rename to c_functions/cfg_template.txt diff --git a/functions/compression/CMakeLists.txt b/c_functions/compression/CMakeLists.txt similarity index 100% rename from functions/compression/CMakeLists.txt rename to c_functions/compression/CMakeLists.txt diff --git a/functions/compression/compression.c b/c_functions/compression/compression.c similarity index 92% rename from functions/compression/compression.c rename to c_functions/compression/compression.c index 9826247..a86011e 100644 --- a/functions/compression/compression.c +++ b/c_functions/compression/compression.c @@ -27,7 +27,9 @@ void *realloc_sized(void *old, size_t old_size, size_t new_size) { } #define STBI_WRITE_NO_STDIO -#define STBIW_ASSERT(x) {} +#define STBIW_ASSERT(x) \ + { \ + } #define STBIW_MALLOC(size) dandelion_alloc(size, 8) #define STBIW_FREE(ptr) dandelion_free(ptr) #define STBIW_REALLOC_SIZED(ptr, old_size, new_size) \ diff --git a/functions/compression/qoi.h b/c_functions/compression/qoi.h similarity index 56% rename from functions/compression/qoi.h rename to c_functions/compression/qoi.h index f2800b0..9a3a48d 100644 --- a/functions/compression/qoi.h +++ b/c_functions/compression/qoi.h @@ -24,10 +24,10 @@ stb_image_write QOI offers 20x-50x faster encoding, 3x-4x faster decoding and // Encode and store an RGBA buffer to the file system. The qoi_desc describes // the input pixel data. qoi_write("image_new.qoi", rgba_pixels, &(qoi_desc){ - .width = 1920, - .height = 1080, - .channels = 4, - .colorspace = QOI_SRGB + .width = 1920, + .height = 1080, + .channels = 4, + .colorspace = QOI_SRGB }); // Load and decode a QOI image from the file system into a 32bbp RGBA buffer. @@ -64,11 +64,12 @@ A QOI file has a 14 byte header, followed by any number of data "chunks" and an 8-byte end marker. struct qoi_header_t { - char magic[4]; // magic bytes "qoif" - uint32_t width; // image width in pixels (BE) - uint32_t height; // image height in pixels (BE) - uint8_t channels; // 3 = RGB, 4 = RGBA - uint8_t colorspace; // 0 = sRGB with linear alpha, 1 = all channels linear + char magic[4]; // magic bytes "qoif" + uint32_t width; // image width in pixels (BE) + uint32_t height; // image height in pixels (BE) + uint8_t channels; // 3 = RGB, 4 = RGBA + uint8_t colorspace; // 0 = sRGB with linear alpha, 1 = all channels +linear }; Images are encoded row by row, left to right, top to bottom. The decoder and @@ -91,7 +92,7 @@ the color value. In the encoder, if the pixel value at the index matches the current pixel, this index position is written to the stream as QOI_OP_INDEX. The hash function for the index is: - index_position = (r * 3 + g * 5 + b * 7 + a * 11) % 64 + index_position = (r * 3 + g * 5 + b * 7 + a * 11) % 64 Each chunk starts with a 2- or 8-bit tag, followed by a number of data bits. The bit length of chunks is divisible by 8 - i.e. all chunks are byte aligned. All @@ -153,8 +154,8 @@ The alpha value remains unchanged from the previous pixel. The green channel is used to indicate the general direction of change and is encoded in 6 bits. The red and blue channels (dr and db) base their diffs off of the green channel difference and are encoded in 4 bits. I.e.: - dr_dg = (cur_px.r - prev_px.r) - (cur_px.g - prev_px.g) - db_dg = (cur_px.b - prev_px.b) - (cur_px.g - prev_px.g) + dr_dg = (cur_px.r - prev_px.r) - (cur_px.g - prev_px.g) + db_dg = (cur_px.b - prev_px.b) - (cur_px.g - prev_px.g) The difference to the current channel values are using a wraparound operation, so "10 - 13" will result in 253, while "250 + 7" will result in 1. @@ -207,7 +208,6 @@ The alpha value remains unchanged from the previous pixel. */ - /* ----------------------------------------------------------------------------- Header - Public functions */ @@ -224,20 +224,20 @@ filled with the description read from the file header (for qoi_read and qoi_decode). The colorspace in this qoi_desc is an enum where - 0 = sRGB, i.e. gamma scaled RGB channels and a linear alpha channel - 1 = all channels are linear + 0 = sRGB, i.e. gamma scaled RGB channels and a linear alpha channel + 1 = all channels are linear You may use the constants QOI_SRGB or QOI_LINEAR. The colorspace is purely informative. It will be saved to the file header, but does not affect how chunks are en-/decoded. */ -#define QOI_SRGB 0 +#define QOI_SRGB 0 #define QOI_LINEAR 1 typedef struct { - unsigned int width; - unsigned int height; - unsigned char channels; - unsigned char colorspace; + unsigned int width; + unsigned int height; + unsigned char channels; + unsigned char colorspace; } qoi_desc; #ifndef QOI_NO_STDIO @@ -251,7 +251,6 @@ failed) or the number of bytes written on success. */ int qoi_write(const char *filename, const void *data, const qoi_desc *desc); - /* Read and decode a QOI image from the file system. If channels is 0, the number of channels from the file header is used. If channels is 3 or 4 the output format will be forced into this number of channels. @@ -266,7 +265,6 @@ void *qoi_read(const char *filename, qoi_desc *desc, int channels); #endif /* QOI_NO_STDIO */ - /* Encode raw RGB or RGBA pixels into a QOI image in memory. The function either returns NULL on failure (invalid parameters or malloc @@ -277,7 +275,6 @@ The returned qoi data should be free()d after use. */ void *qoi_encode(const void *data, const qoi_desc *desc, int *out_len); - /* Decode a QOI image from memory. The function either returns NULL on failure (invalid parameters or malloc @@ -288,13 +285,11 @@ The returned pixel data should be free()d after use. */ void *qoi_decode(const void *data, int size, qoi_desc *desc, int channels); - #ifdef __cplusplus } #endif #endif /* QOI_H */ - /* ----------------------------------------------------------------------------- Implementation */ @@ -303,26 +298,27 @@ Implementation */ #include #ifndef QOI_MALLOC - #define QOI_MALLOC(sz) malloc(sz) - #define QOI_FREE(p) free(p) +#define QOI_MALLOC(sz) malloc(sz) +#define QOI_FREE(p) free(p) #endif #ifndef QOI_ZEROARR - #define QOI_ZEROARR(a) memset((a),0,sizeof(a)) +#define QOI_ZEROARR(a) memset((a), 0, sizeof(a)) #endif -#define QOI_OP_INDEX 0x00 /* 00xxxxxx */ -#define QOI_OP_DIFF 0x40 /* 01xxxxxx */ -#define QOI_OP_LUMA 0x80 /* 10xxxxxx */ -#define QOI_OP_RUN 0xc0 /* 11xxxxxx */ -#define QOI_OP_RGB 0xfe /* 11111110 */ -#define QOI_OP_RGBA 0xff /* 11111111 */ +#define QOI_OP_INDEX 0x00 /* 00xxxxxx */ +#define QOI_OP_DIFF 0x40 /* 01xxxxxx */ +#define QOI_OP_LUMA 0x80 /* 10xxxxxx */ +#define QOI_OP_RUN 0xc0 /* 11xxxxxx */ +#define QOI_OP_RGB 0xfe /* 11111110 */ +#define QOI_OP_RGBA 0xff /* 11111111 */ -#define QOI_MASK_2 0xc0 /* 11000000 */ +#define QOI_MASK_2 0xc0 /* 11000000 */ -#define QOI_COLOR_HASH(C) (C.rgba.r*3 + C.rgba.g*5 + C.rgba.b*7 + C.rgba.a*11) -#define QOI_MAGIC \ - (((unsigned int)'q') << 24 | ((unsigned int)'o') << 16 | \ - ((unsigned int)'i') << 8 | ((unsigned int)'f')) +#define QOI_COLOR_HASH(C) \ + (C.rgba.r * 3 + C.rgba.g * 5 + C.rgba.b * 7 + C.rgba.a * 11) +#define QOI_MAGIC \ + (((unsigned int)'q') << 24 | ((unsigned int)'o') << 16 | \ + ((unsigned int)'i') << 8 | ((unsigned int)'f')) #define QOI_HEADER_SIZE 14 /* 2GB is the max file size that this implementation can safely handle. We guard @@ -332,317 +328,290 @@ enough for anybody. */ #define QOI_PIXELS_MAX ((unsigned int)400000000) typedef union { - struct { unsigned char r, g, b, a; } rgba; - unsigned int v; + struct { + unsigned char r, g, b, a; + } rgba; + unsigned int v; } qoi_rgba_t; -static const unsigned char qoi_padding[8] = {0,0,0,0,0,0,0,1}; +static const unsigned char qoi_padding[8] = {0, 0, 0, 0, 0, 0, 0, 1}; static void qoi_write_32(unsigned char *bytes, int *p, unsigned int v) { - bytes[(*p)++] = (0xff000000 & v) >> 24; - bytes[(*p)++] = (0x00ff0000 & v) >> 16; - bytes[(*p)++] = (0x0000ff00 & v) >> 8; - bytes[(*p)++] = (0x000000ff & v); + bytes[(*p)++] = (0xff000000 & v) >> 24; + bytes[(*p)++] = (0x00ff0000 & v) >> 16; + bytes[(*p)++] = (0x0000ff00 & v) >> 8; + bytes[(*p)++] = (0x000000ff & v); } static unsigned int qoi_read_32(const unsigned char *bytes, int *p) { - unsigned int a = bytes[(*p)++]; - unsigned int b = bytes[(*p)++]; - unsigned int c = bytes[(*p)++]; - unsigned int d = bytes[(*p)++]; - return a << 24 | b << 16 | c << 8 | d; + unsigned int a = bytes[(*p)++]; + unsigned int b = bytes[(*p)++]; + unsigned int c = bytes[(*p)++]; + unsigned int d = bytes[(*p)++]; + return a << 24 | b << 16 | c << 8 | d; } void *qoi_encode(const void *data, const qoi_desc *desc, int *out_len) { - int i, max_size, p, run; - int px_len, px_end, px_pos, channels; - unsigned char *bytes; - const unsigned char *pixels; - qoi_rgba_t index[64]; - qoi_rgba_t px, px_prev; - - if ( - data == NULL || out_len == NULL || desc == NULL || - desc->width == 0 || desc->height == 0 || - desc->channels < 3 || desc->channels > 4 || - desc->colorspace > 1 || - desc->height >= QOI_PIXELS_MAX / desc->width - ) { - return NULL; - } - - max_size = - desc->width * desc->height * (desc->channels + 1) + - QOI_HEADER_SIZE + sizeof(qoi_padding); - - p = 0; - bytes = (unsigned char *) QOI_MALLOC(max_size); - if (!bytes) { - return NULL; - } - - qoi_write_32(bytes, &p, QOI_MAGIC); - qoi_write_32(bytes, &p, desc->width); - qoi_write_32(bytes, &p, desc->height); - bytes[p++] = desc->channels; - bytes[p++] = desc->colorspace; - - - pixels = (const unsigned char *)data; - - QOI_ZEROARR(index); - - run = 0; - px_prev.rgba.r = 0; - px_prev.rgba.g = 0; - px_prev.rgba.b = 0; - px_prev.rgba.a = 255; - px = px_prev; - - px_len = desc->width * desc->height * desc->channels; - px_end = px_len - desc->channels; - channels = desc->channels; - - for (px_pos = 0; px_pos < px_len; px_pos += channels) { - px.rgba.r = pixels[px_pos + 0]; - px.rgba.g = pixels[px_pos + 1]; - px.rgba.b = pixels[px_pos + 2]; - - if (channels == 4) { - px.rgba.a = pixels[px_pos + 3]; - } - - if (px.v == px_prev.v) { - run++; - if (run == 62 || px_pos == px_end) { - bytes[p++] = QOI_OP_RUN | (run - 1); - run = 0; - } - } - else { - int index_pos; - - if (run > 0) { - bytes[p++] = QOI_OP_RUN | (run - 1); - run = 0; - } - - index_pos = QOI_COLOR_HASH(px) % 64; - - if (index[index_pos].v == px.v) { - bytes[p++] = QOI_OP_INDEX | index_pos; - } - else { - index[index_pos] = px; - - if (px.rgba.a == px_prev.rgba.a) { - signed char vr = px.rgba.r - px_prev.rgba.r; - signed char vg = px.rgba.g - px_prev.rgba.g; - signed char vb = px.rgba.b - px_prev.rgba.b; - - signed char vg_r = vr - vg; - signed char vg_b = vb - vg; - - if ( - vr > -3 && vr < 2 && - vg > -3 && vg < 2 && - vb > -3 && vb < 2 - ) { - bytes[p++] = QOI_OP_DIFF | (vr + 2) << 4 | (vg + 2) << 2 | (vb + 2); - } - else if ( - vg_r > -9 && vg_r < 8 && - vg > -33 && vg < 32 && - vg_b > -9 && vg_b < 8 - ) { - bytes[p++] = QOI_OP_LUMA | (vg + 32); - bytes[p++] = (vg_r + 8) << 4 | (vg_b + 8); - } - else { - bytes[p++] = QOI_OP_RGB; - bytes[p++] = px.rgba.r; - bytes[p++] = px.rgba.g; - bytes[p++] = px.rgba.b; - } - } - else { - bytes[p++] = QOI_OP_RGBA; - bytes[p++] = px.rgba.r; - bytes[p++] = px.rgba.g; - bytes[p++] = px.rgba.b; - bytes[p++] = px.rgba.a; - } - } - } - px_prev = px; - } - - for (i = 0; i < (int)sizeof(qoi_padding); i++) { - bytes[p++] = qoi_padding[i]; - } - - *out_len = p; - return bytes; + int i, max_size, p, run; + int px_len, px_end, px_pos, channels; + unsigned char *bytes; + const unsigned char *pixels; + qoi_rgba_t index[64]; + qoi_rgba_t px, px_prev; + + if (data == NULL || out_len == NULL || desc == NULL || desc->width == 0 || + desc->height == 0 || desc->channels < 3 || desc->channels > 4 || + desc->colorspace > 1 || desc->height >= QOI_PIXELS_MAX / desc->width) { + return NULL; + } + + max_size = desc->width * desc->height * (desc->channels + 1) + + QOI_HEADER_SIZE + sizeof(qoi_padding); + + p = 0; + bytes = (unsigned char *)QOI_MALLOC(max_size); + if (!bytes) { + return NULL; + } + + qoi_write_32(bytes, &p, QOI_MAGIC); + qoi_write_32(bytes, &p, desc->width); + qoi_write_32(bytes, &p, desc->height); + bytes[p++] = desc->channels; + bytes[p++] = desc->colorspace; + + pixels = (const unsigned char *)data; + + QOI_ZEROARR(index); + + run = 0; + px_prev.rgba.r = 0; + px_prev.rgba.g = 0; + px_prev.rgba.b = 0; + px_prev.rgba.a = 255; + px = px_prev; + + px_len = desc->width * desc->height * desc->channels; + px_end = px_len - desc->channels; + channels = desc->channels; + + for (px_pos = 0; px_pos < px_len; px_pos += channels) { + px.rgba.r = pixels[px_pos + 0]; + px.rgba.g = pixels[px_pos + 1]; + px.rgba.b = pixels[px_pos + 2]; + + if (channels == 4) { + px.rgba.a = pixels[px_pos + 3]; + } + + if (px.v == px_prev.v) { + run++; + if (run == 62 || px_pos == px_end) { + bytes[p++] = QOI_OP_RUN | (run - 1); + run = 0; + } + } else { + int index_pos; + + if (run > 0) { + bytes[p++] = QOI_OP_RUN | (run - 1); + run = 0; + } + + index_pos = QOI_COLOR_HASH(px) % 64; + + if (index[index_pos].v == px.v) { + bytes[p++] = QOI_OP_INDEX | index_pos; + } else { + index[index_pos] = px; + + if (px.rgba.a == px_prev.rgba.a) { + signed char vr = px.rgba.r - px_prev.rgba.r; + signed char vg = px.rgba.g - px_prev.rgba.g; + signed char vb = px.rgba.b - px_prev.rgba.b; + + signed char vg_r = vr - vg; + signed char vg_b = vb - vg; + + if (vr > -3 && vr < 2 && vg > -3 && vg < 2 && vb > -3 && vb < 2) { + bytes[p++] = QOI_OP_DIFF | (vr + 2) << 4 | (vg + 2) << 2 | (vb + 2); + } else if (vg_r > -9 && vg_r < 8 && vg > -33 && vg < 32 && + vg_b > -9 && vg_b < 8) { + bytes[p++] = QOI_OP_LUMA | (vg + 32); + bytes[p++] = (vg_r + 8) << 4 | (vg_b + 8); + } else { + bytes[p++] = QOI_OP_RGB; + bytes[p++] = px.rgba.r; + bytes[p++] = px.rgba.g; + bytes[p++] = px.rgba.b; + } + } else { + bytes[p++] = QOI_OP_RGBA; + bytes[p++] = px.rgba.r; + bytes[p++] = px.rgba.g; + bytes[p++] = px.rgba.b; + bytes[p++] = px.rgba.a; + } + } + } + px_prev = px; + } + + for (i = 0; i < (int)sizeof(qoi_padding); i++) { + bytes[p++] = qoi_padding[i]; + } + + *out_len = p; + return bytes; } void *qoi_decode(const void *data, int size, qoi_desc *desc, int channels) { - const unsigned char *bytes; - unsigned int header_magic; - unsigned char *pixels; - qoi_rgba_t index[64]; - qoi_rgba_t px; - int px_len, chunks_len, px_pos; - int p = 0, run = 0; - - if ( - data == NULL || desc == NULL || - (channels != 0 && channels != 3 && channels != 4) || - size < QOI_HEADER_SIZE + (int)sizeof(qoi_padding) - ) { - return NULL; - } - - bytes = (const unsigned char *)data; - - header_magic = qoi_read_32(bytes, &p); - desc->width = qoi_read_32(bytes, &p); - desc->height = qoi_read_32(bytes, &p); - desc->channels = bytes[p++]; - desc->colorspace = bytes[p++]; - - if ( - desc->width == 0 || desc->height == 0 || - desc->channels < 3 || desc->channels > 4 || - desc->colorspace > 1 || - header_magic != QOI_MAGIC || - desc->height >= QOI_PIXELS_MAX / desc->width - ) { - return NULL; - } - - if (channels == 0) { - channels = desc->channels; - } - - px_len = desc->width * desc->height * channels; - pixels = (unsigned char *) QOI_MALLOC(px_len); - if (!pixels) { - return NULL; - } - - QOI_ZEROARR(index); - px.rgba.r = 0; - px.rgba.g = 0; - px.rgba.b = 0; - px.rgba.a = 255; - - chunks_len = size - (int)sizeof(qoi_padding); - for (px_pos = 0; px_pos < px_len; px_pos += channels) { - if (run > 0) { - run--; - } - else if (p < chunks_len) { - int b1 = bytes[p++]; - - if (b1 == QOI_OP_RGB) { - px.rgba.r = bytes[p++]; - px.rgba.g = bytes[p++]; - px.rgba.b = bytes[p++]; - } - else if (b1 == QOI_OP_RGBA) { - px.rgba.r = bytes[p++]; - px.rgba.g = bytes[p++]; - px.rgba.b = bytes[p++]; - px.rgba.a = bytes[p++]; - } - else if ((b1 & QOI_MASK_2) == QOI_OP_INDEX) { - px = index[b1]; - } - else if ((b1 & QOI_MASK_2) == QOI_OP_DIFF) { - px.rgba.r += ((b1 >> 4) & 0x03) - 2; - px.rgba.g += ((b1 >> 2) & 0x03) - 2; - px.rgba.b += ( b1 & 0x03) - 2; - } - else if ((b1 & QOI_MASK_2) == QOI_OP_LUMA) { - int b2 = bytes[p++]; - int vg = (b1 & 0x3f) - 32; - px.rgba.r += vg - 8 + ((b2 >> 4) & 0x0f); - px.rgba.g += vg; - px.rgba.b += vg - 8 + (b2 & 0x0f); - } - else if ((b1 & QOI_MASK_2) == QOI_OP_RUN) { - run = (b1 & 0x3f); - } - - index[QOI_COLOR_HASH(px) % 64] = px; - } - - pixels[px_pos + 0] = px.rgba.r; - pixels[px_pos + 1] = px.rgba.g; - pixels[px_pos + 2] = px.rgba.b; - - if (channels == 4) { - pixels[px_pos + 3] = px.rgba.a; - } - } - - return pixels; + const unsigned char *bytes; + unsigned int header_magic; + unsigned char *pixels; + qoi_rgba_t index[64]; + qoi_rgba_t px; + int px_len, chunks_len, px_pos; + int p = 0, run = 0; + + if (data == NULL || desc == NULL || + (channels != 0 && channels != 3 && channels != 4) || + size < QOI_HEADER_SIZE + (int)sizeof(qoi_padding)) { + return NULL; + } + + bytes = (const unsigned char *)data; + + header_magic = qoi_read_32(bytes, &p); + desc->width = qoi_read_32(bytes, &p); + desc->height = qoi_read_32(bytes, &p); + desc->channels = bytes[p++]; + desc->colorspace = bytes[p++]; + + if (desc->width == 0 || desc->height == 0 || desc->channels < 3 || + desc->channels > 4 || desc->colorspace > 1 || header_magic != QOI_MAGIC || + desc->height >= QOI_PIXELS_MAX / desc->width) { + return NULL; + } + + if (channels == 0) { + channels = desc->channels; + } + + px_len = desc->width * desc->height * channels; + pixels = (unsigned char *)QOI_MALLOC(px_len); + if (!pixels) { + return NULL; + } + + QOI_ZEROARR(index); + px.rgba.r = 0; + px.rgba.g = 0; + px.rgba.b = 0; + px.rgba.a = 255; + + chunks_len = size - (int)sizeof(qoi_padding); + for (px_pos = 0; px_pos < px_len; px_pos += channels) { + if (run > 0) { + run--; + } else if (p < chunks_len) { + int b1 = bytes[p++]; + + if (b1 == QOI_OP_RGB) { + px.rgba.r = bytes[p++]; + px.rgba.g = bytes[p++]; + px.rgba.b = bytes[p++]; + } else if (b1 == QOI_OP_RGBA) { + px.rgba.r = bytes[p++]; + px.rgba.g = bytes[p++]; + px.rgba.b = bytes[p++]; + px.rgba.a = bytes[p++]; + } else if ((b1 & QOI_MASK_2) == QOI_OP_INDEX) { + px = index[b1]; + } else if ((b1 & QOI_MASK_2) == QOI_OP_DIFF) { + px.rgba.r += ((b1 >> 4) & 0x03) - 2; + px.rgba.g += ((b1 >> 2) & 0x03) - 2; + px.rgba.b += (b1 & 0x03) - 2; + } else if ((b1 & QOI_MASK_2) == QOI_OP_LUMA) { + int b2 = bytes[p++]; + int vg = (b1 & 0x3f) - 32; + px.rgba.r += vg - 8 + ((b2 >> 4) & 0x0f); + px.rgba.g += vg; + px.rgba.b += vg - 8 + (b2 & 0x0f); + } else if ((b1 & QOI_MASK_2) == QOI_OP_RUN) { + run = (b1 & 0x3f); + } + + index[QOI_COLOR_HASH(px) % 64] = px; + } + + pixels[px_pos + 0] = px.rgba.r; + pixels[px_pos + 1] = px.rgba.g; + pixels[px_pos + 2] = px.rgba.b; + + if (channels == 4) { + pixels[px_pos + 3] = px.rgba.a; + } + } + + return pixels; } #ifndef QOI_NO_STDIO #include int qoi_write(const char *filename, const void *data, const qoi_desc *desc) { - FILE *f = fopen(filename, "wb"); - int size, err; - void *encoded; - - if (!f) { - return 0; - } - - encoded = qoi_encode(data, desc, &size); - if (!encoded) { - fclose(f); - return 0; - } - - fwrite(encoded, 1, size, f); - fflush(f); - err = ferror(f); - fclose(f); - - QOI_FREE(encoded); - return err ? 0 : size; + FILE *f = fopen(filename, "wb"); + int size, err; + void *encoded; + + if (!f) { + return 0; + } + + encoded = qoi_encode(data, desc, &size); + if (!encoded) { + fclose(f); + return 0; + } + + fwrite(encoded, 1, size, f); + fflush(f); + err = ferror(f); + fclose(f); + + QOI_FREE(encoded); + return err ? 0 : size; } void *qoi_read(const char *filename, qoi_desc *desc, int channels) { - FILE *f = fopen(filename, "rb"); - int size, bytes_read; - void *pixels, *data; - - if (!f) { - return NULL; - } - - fseek(f, 0, SEEK_END); - size = ftell(f); - if (size <= 0 || fseek(f, 0, SEEK_SET) != 0) { - fclose(f); - return NULL; - } - - data = QOI_MALLOC(size); - if (!data) { - fclose(f); - return NULL; - } - - bytes_read = fread(data, 1, size, f); - fclose(f); - pixels = (bytes_read != size) ? NULL : qoi_decode(data, bytes_read, desc, channels); - QOI_FREE(data); - return pixels; + FILE *f = fopen(filename, "rb"); + int size, bytes_read; + void *pixels, *data; + + if (!f) { + return NULL; + } + + fseek(f, 0, SEEK_END); + size = ftell(f); + if (size <= 0 || fseek(f, 0, SEEK_SET) != 0) { + fclose(f); + return NULL; + } + + data = QOI_MALLOC(size); + if (!data) { + fclose(f); + return NULL; + } + + bytes_read = fread(data, 1, size, f); + fclose(f); + pixels = (bytes_read != size) ? NULL + : qoi_decode(data, bytes_read, desc, channels); + QOI_FREE(data); + return pixels; } #endif /* QOI_NO_STDIO */ diff --git a/functions/compression/stb_image_write.h b/c_functions/compression/stb_image_write.h similarity index 100% rename from functions/compression/stb_image_write.h rename to c_functions/compression/stb_image_write.h diff --git a/functions/dirigent_busy/CMakeLists.txt b/c_functions/dirigent_busy/CMakeLists.txt similarity index 100% rename from functions/dirigent_busy/CMakeLists.txt rename to c_functions/dirigent_busy/CMakeLists.txt diff --git a/c_functions/dirigent_busy/dirigent_busy.c b/c_functions/dirigent_busy/dirigent_busy.c new file mode 100644 index 0000000..a7ab4f5 --- /dev/null +++ b/c_functions/dirigent_busy/dirigent_busy.c @@ -0,0 +1,87 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "unistd.h" + +int main(int argc, char const *argv[]) { + // parse input + FILE *input_data = fopen("/input/input.csv", "r"); + if (input_data == NULL) { + perror("could not open /input/input.csv"); + return -1; + } + FILE *output_data = fopen("/output/output.csv", "w+"); + if (output_data == NULL) { + perror("could not open /output/output.csv"); + return -1; + } + + char *workload = NULL; + size_t workload_size = 0; + if (__getdelim(&workload, &workload_size, ',', input_data) < 0) { + perror("failed to read workload"); + return -1; + } + workload[strlen(workload) - 1] = '\0'; + + char *function = NULL; + size_t function_size = 0; + if (__getdelim(&function, &function_size, ',', input_data) < 0) { + perror("failed to read function"); + return -1; + } + function[strlen(function) - 1] = '\0'; + + char *requestedCpu = NULL; + size_t requestedCpu_size = 0; + if (__getdelim(&requestedCpu, &requestedCpu_size, ',', input_data) < 0) { + perror("failed to read requestedCpu"); + return -1; + } + requestedCpu[strlen(requestedCpu) - 1] = '\0'; + + char *multiplier = NULL; + size_t multiplier_size = 0; + if (__getdelim(&multiplier, &multiplier_size, ',', input_data) < 0) { + perror("failed to read multiplier"); + return -1; + } + + char *trace_string = "trace"; + char *empyt_string = "empty"; + + if (strcmp(trace_string, function) == 0) { + long multiplier_num = strtol(multiplier, NULL, 10); + long requested_cpu_num = strtol(requestedCpu, NULL, 10); + + long total_iterations = multiplier_num * requested_cpu_num; + + // printf("%ld, %ld, %ld\n", multiplier_num, requested_cpu_num, + // total_iterations); + + volatile double result = 0.0; + volatile double input = 10.0; + volatile long iteration; + for (iteration = 0; iteration < total_iterations; iteration++) { + result = sqrt(input); + } + + fprintf(output_data, "\"OK\",\"%s\",\"dandelionServer\",%ld,%f", function, + iteration, result); + + return 0; + } else if (strcmp(empyt_string, function) == 0) { + fprintf(output_data, "\"OK - EMPTY\",\"dandelionServer\",\"%s\",0", + function); + + return 0; + } + + return -1; +} diff --git a/functions/example_app/CMakeLists.txt b/c_functions/example_app/CMakeLists.txt similarity index 100% rename from functions/example_app/CMakeLists.txt rename to c_functions/example_app/CMakeLists.txt diff --git a/functions/example_app/fan_out.c b/c_functions/example_app/fan_out.c similarity index 64% rename from functions/example_app/fan_out.c rename to c_functions/example_app/fan_out.c index e32c13e..d451f05 100644 --- a/functions/example_app/fan_out.c +++ b/c_functions/example_app/fan_out.c @@ -8,7 +8,7 @@ #include "unistd.h" -const unsigned int SERVER_NUMBER = 10; +const unsigned int SERVER_NUMBER = 10; int main(int argc, char const *argv[]) { @@ -18,15 +18,15 @@ int main(int argc, char const *argv[]) { perror("Failed to open file to get log server\n"); return -1; } - char* server_line = NULL; + char *server_line = NULL; size_t server_line_len = 0; - if(__getline(&server_line, &server_line_len, log_server) < 0) { + if (__getline(&server_line, &server_line_len, log_server) < 0) { perror("Fauled to read line from auth server file\n"); return -1; } - + // read auth response - FILE* auth_file = fopen("/responses/auth", "r"); + FILE *auth_file = fopen("/responses/auth", "r"); if (auth_file == NULL) { perror("Failed to open auth file\n"); return -1; @@ -34,20 +34,24 @@ int main(int argc, char const *argv[]) { // get the json with the athorization char username[21] = {0}; char token[257] = {0}; - if (fscanf(auth_file, "%*[^\"]\"authorized\":\"%20[^\"]\",\"token\":\"%256[^\"]\"}", username, token) < 0) { + if (fscanf(auth_file, + "%*[^\"]\"authorized\":\"%20[^\"]\",\"token\":\"%256[^\"]\"}", + username, token) < 0) { perror("Failed to parse line from token file"); return -1; } - for(unsigned int server_index = 0; server_index < SERVER_NUMBER; server_index++){ + for (unsigned int server_index = 0; server_index < SERVER_NUMBER; + server_index++) { char request_path[100] = {0}; - if (snprintf(request_path, 100, "/requests/server_%d", server_index) < 0){ - perror("Failed to format file path for log request"); - return -1; + if (snprintf(request_path, 100, "/requests/server_%d", server_index) < 0) { + perror("Failed to format file path for log request"); + return -1; } // write authorization request FILE *log_request = fopen(request_path, "w+"); - fprintf(log_request, "GET http://%s/logs/%02d HTTP/1.1\n\n", server_line, server_index); + fprintf(log_request, "GET http://%s/logs/%02d HTTP/1.1\n\n", server_line, + server_index); fprintf(log_request, "{\"username\": \"%s\"}", username); } diff --git a/functions/example_app/handle.c b/c_functions/example_app/handle.c similarity index 84% rename from functions/example_app/handle.c rename to c_functions/example_app/handle.c index f812947..6f77ba8 100644 --- a/functions/example_app/handle.c +++ b/c_functions/example_app/handle.c @@ -16,15 +16,15 @@ int main(int argc, char const *argv[]) { perror("Failed to open file to get auth server"); return -1; } - char* server_line = NULL; + char *server_line = NULL; size_t server_line_len = 0; - if(__getline(&server_line, &server_line_len, auth_server) < 0) { + if (__getline(&server_line, &server_line_len, auth_server) < 0) { perror("Fauled to read line from auth server file\n"); return -1; } - + // read token from incomming file - FILE* token_file = fopen("/responses/Authorization", "r"); + FILE *token_file = fopen("/responses/Authorization", "r"); if (token_file == NULL) { perror("Failed to open file with token"); return -1; diff --git a/c_functions/example_app/template.c b/c_functions/example_app/template.c new file mode 100644 index 0000000..75b0cd9 --- /dev/null +++ b/c_functions/example_app/template.c @@ -0,0 +1,123 @@ +#include +#include +#include +#include +#include +#include +#include + +#include "unistd.h" + +typedef struct log_node { + struct log_node *next; + char timestamp[100]; + char server_id[100]; + char type[100]; + char details[100]; +} log_node; + +char event_template[] = "{" + "\"details\":\"%100[^\"]\"," + "\"event_type\":\"%100[^\"]\"," + "\"server_id\":\"%100[^\"]\"," + "\"timestamp\":\"%100[^\"]\"" + "%*[^{]"; + +int main(int argc, char const *argv[]) { + + log_node *log_root = NULL; + + // read all events and put the in list sorted by timestamps + DIR *log_dir = opendir("/responses"); + if (log_dir == NULL) { + perror("Could not open log directory"); + return -1; + } + + struct dirent *entry; + while ((entry = readdir(log_dir))) { + if (entry->d_type == DT_REG) { + char filepath[256] = {0}; + if (snprintf(filepath, 256, "/responses/%s", entry->d_name) < 0) { + perror("Failed to format log filepath\n"); + return -1; + } + FILE *log_file = fopen(filepath, "r"); + size_t line_size = 0; + char *line_buffer = NULL; + // read until and including initial [ + if (__getdelim(&line_buffer, &line_size, '[', log_file) < 0) { + perror("Failed to read initial json {"); + return -1; + } + free(line_buffer); + // keep reading until we find no more } + int items_found = 0; + log_node *current = malloc(sizeof(log_node)); + while ((items_found = fscanf(log_file, event_template, current->details, + current->type, current->server_id, + current->timestamp)) == 4) { + current->next = NULL; + if (log_root == NULL) { + log_root = current; + } else { + if (strcmp(current->timestamp, log_root->timestamp) <= 0) { + current->next = log_root; + log_root = current; + } else { + log_node *previous = log_root; + while (previous->next != NULL) { + log_node *next = previous->next; + if (strcmp(current->timestamp, next->timestamp) <= 0) { + current->next = next; + break; + } + previous = next; + } + previous->next = current; + } + } + current = malloc(sizeof(log_node)); + } + free(current); + } + } + + // open output file + FILE *log_file = fopen("/requests/log.txt", "w+"); + if (log_file == NULL) { + perror("Failed to open file to get auth server\n"); + return -1; + } + // write preamble + fprintf(log_file, "\n"); + fprintf(log_file, "\n"); + fprintf(log_file, "\n"); + fprintf(log_file, " \n"); + fprintf(log_file, " Logs\n"); + fprintf(log_file, "\n"); + fprintf(log_file, "\n"); + fprintf(log_file, " \n"); + fprintf(log_file, " \n"); + fprintf(log_file, " \n"); + fprintf(log_file, " \n"); + fprintf(log_file, " \n"); + fprintf(log_file, " \n"); + fprintf(log_file, " \n"); + + // write log information + for (log_node *current = log_root; current != NULL; current = current->next) { + fprintf(log_file, " \n"); + fprintf(log_file, " \n", current->timestamp); + fprintf(log_file, " \n", current->server_id); + fprintf(log_file, " \n", current->type); + fprintf(log_file, " \n", current->details); + fprintf(log_file, " \n"); + } + + // write ending + fprintf(log_file, "
TimestampServer IDEvent TypeDetails
%s%s%s%s
\n"); + fprintf(log_file, "\n"); + + return -1; +} diff --git a/functions/example_app_hybrid/CMakeLists.txt b/c_functions/example_app_hybrid/CMakeLists.txt similarity index 100% rename from functions/example_app_hybrid/CMakeLists.txt rename to c_functions/example_app_hybrid/CMakeLists.txt diff --git a/c_functions/example_app_hybrid/app_hybrid.c b/c_functions/example_app_hybrid/app_hybrid.c new file mode 100644 index 0000000..9860e38 --- /dev/null +++ b/c_functions/example_app_hybrid/app_hybrid.c @@ -0,0 +1,410 @@ +#include "unistd.h" +#include +#include +#include +#include +#include +#include +#include +#include + +#include "syscall.h" + +#define BUFFER_SIZE 4096 +#define SERVER_NUMBER 10 +const uint16_t AUTH_SERVER_PORT = 8000; + +// Structure to store log event details +typedef struct log_node { + struct log_node *next; + char timestamp[100]; + char server_id[100]; + char type[100]; + char details[100]; +} log_node; + +char event_template[] = "%*[^{]{" + "\"details\":\"%100[^\"]\"," + "\"event_type\":\"%100[^\"]\"," + "\"server_id\":\"%100[^\"]\"," + "\"timestamp\":\"%100[^\"]\"" + "%}"; + +uint16_t my_htons(uint16_t port) { + return ((port >> 8) & 0x00FF) | ((port << 8) & 0xFF00); +} + +int my_inet_pton(int af, const char *src, void *dst) { + if (af == LINUX_AF_INET) { + uint8_t array[4] = {}; + for (int i = 0; i < 4; i++) { + char *end; + long value = strtol(src, &end, 10); + if (value < 0 || value > 255) { + return 0; + } + + if (*end != '\0' && *end != '.' && i < 3) { + return 0; + } + src = end + 1; + array[i] = (uint8_t)value; + } + + struct in_addr *addr = (struct in_addr *)dst; + memcpy(&addr->s_addr, array, 4); // Copy the 4 bytes into s_addr + + return 1; // Success + } + return -1; // Unsupported address family +} + +int connect_to_server(const char *ip, uint16_t port) { + struct sockaddr_in server_addr; + + int sockfd = linux_socket(LINUX_AF_INET, LINUX_SOCK_STREAM, 0); + if (sockfd < 0) { + perror("Socket creation failed"); + return -1; + } + + server_addr.sin_family = LINUX_AF_INET; + server_addr.sin_port = my_htons(port); + + if (my_inet_pton(LINUX_AF_INET, ip, &server_addr.sin_addr) != 1) { + perror("Invalid IP address"); + close(sockfd); + return -1; + } + + int err = linux_connect(sockfd, &server_addr, sizeof(server_addr)); + if (err < 0) { + printf("Error: %s\n", strerror(-err)); + printf("size of server_addr: %ld\n", sizeof(server_addr)); + perror("Socket connection failed"); + close(sockfd); + return -1; + } + + return sockfd; +} + +int send_http_request(int sockfd, const char *request, size_t request_size, + char *response, size_t response_size) { + if (linux_send(sockfd, request, request_size, 0) < 0) { + perror("Send failed"); + return -1; + } + + ssize_t total_received = 0, bytes; + // while ((bytes = linux_recv(sockfd, response + total_received, response_size + // - total_received - 1, 0)) > 0) { + // total_received += bytes; + // if (total_received >= response_size - 1) break; + // } + // blocks and waits forever if there is no next packate, could validate HTTP + // sice on the packages to see if we need more, this works for now + total_received = linux_recv(sockfd, response + total_received, + response_size - total_received - 1, 0); + if (total_received < 0) { + return -1; + } + response[total_received] = '\0'; + return 0; +} + +void parse_auth_response(const char *response, char *token) { + const char *start = strstr(response, "\"token\":"); + if (start) { + sscanf(start, "\"token\": \"%[^\"]", token); + } +} + +void parse_log_events(const char *response, log_node **log_root) { + log_node *current = NULL; + + const char *start = strstr(response, "\"events\":"); + if (start) { + start += 9; + while (1) { + current = malloc(sizeof(log_node)); + int found = sscanf(start, event_template, current->details, current->type, + current->server_id, current->timestamp); + if (found != 4) { + free(current); + break; + } + + current->next = *log_root; + *log_root = current; + + start = strstr(start, "},{"); + if (!start) + break; + start += 1; + } + } +} + +// Render logs to an HTML file +void render_logs_to_html(log_node *log_root) { + FILE *log_file = fopen("/requests/log.html", "w+"); + if (!log_file) { + perror("Failed to open HTML file"); + return; + } + + // sort by date + // get number + size_t num_entries = 0; + log_node *current = log_root; + while (current) { + num_entries++; + current = current->next; + } + log_node **log_heap = malloc(sizeof(log_node *) * num_entries); + current = log_root; + for (size_t index = 0; index < num_entries; index++) { + log_heap[index] = current; + current = current->next; + for (size_t insert = index; insert > 0; insert = insert / 2) { + if (strcmp(log_heap[insert / 2]->timestamp, + log_heap[insert]->timestamp) <= 0) { + break; + } else { + log_node *tmp = log_heap[insert]; + log_heap[insert] = log_heap[insert / 2]; + log_heap[insert / 2] = tmp; + } + } + } + + // write preamble + fprintf(log_file, "\n"); + fprintf(log_file, "\n"); + fprintf(log_file, "\n"); + fprintf(log_file, " \n"); + fprintf(log_file, " Logs\n"); + fprintf(log_file, "\n"); + fprintf(log_file, "\n"); + fprintf(log_file, " \n"); + fprintf(log_file, " \n"); + fprintf(log_file, " \n"); + fprintf(log_file, " \n"); + fprintf(log_file, " \n"); + fprintf(log_file, " \n"); + fprintf(log_file, " \n"); + + // write log information + for (; num_entries > 0; num_entries--) { + current = log_heap[0]; + fprintf(log_file, " \n"); + fprintf(log_file, " \n", current->timestamp); + fprintf(log_file, " \n", current->server_id); + fprintf(log_file, " \n", current->type); + fprintf(log_file, " \n", current->details); + fprintf(log_file, " \n"); + // remove current root from heap + size_t index = 0; + log_heap[0] = log_heap[num_entries - 1]; + // trade down until no more children + while (index * 2 + 1 < num_entries) { + // compare with first child + int is_bigger = strcmp(log_heap[index]->timestamp, + log_heap[index * 2 + 1]->timestamp); + // check if second child exists + if (index * 2 + 2 >= num_entries) { + // second child does not exist, if is bigger than child swap + if (is_bigger > 0) { + log_node *tmp = log_heap[index]; + log_heap[index] = log_heap[index * 2 + 1]; + log_heap[index * 2 + 1] = tmp; + } + break; + } + // know fist child is smaller, if second is smaller than first swap with + // second otherwise with first + if (is_bigger > 0) { + size_t swap_index; + if (strcmp(log_heap[index * 2 + 1]->timestamp, + log_heap[index * 2 + 2]->timestamp) <= 0) { + // first is smaller + swap_index = index * 2 + 1; + } else { + swap_index = index * 2 + 2; + } + log_node *tmp = log_heap[index]; + log_heap[index] = log_heap[swap_index]; + log_heap[swap_index] = tmp; + index = swap_index; + } else { + // first is not smaller so need to check second if swap is neccessary + if (strcmp(log_heap[index]->timestamp, + log_heap[index * 2 + 2]->timestamp) > 0) { + // is bigger so need to swap + log_node *tmp = log_heap[index]; + log_heap[index] = log_heap[index * 2 + 2]; + log_heap[index * 2 + 2] = tmp; + index = index * 2 + 2; + } else { + break; + } + } + } + } + + // write ending + fprintf(log_file, "
TimestampServer IDEvent TypeDetails
%s%s%s%s
\n"); + fprintf(log_file, "\n"); + + fclose(log_file); +} + +// Main function to handle authorization, log fetching, and rendering +int main() { + // read authentification server ip from file + FILE *auth_server = fopen("/servers/server.txt", "r"); + if (auth_server == NULL) { + perror("Failed to open file to get auth server"); + return -1; + } + char *auth_server_ip = NULL; + size_t server_line_len = 0; + size_t read_chars = __getline(&auth_server_ip, &server_line_len, auth_server); + if (read_chars < 0) { + perror("Fauled to read line from auth server file\n"); + return -1; + } + // strip possible endln from the getline + if (auth_server_ip[read_chars - 1] == '\n') { + auth_server_ip[read_chars - 1] = '\0'; + } + + // read token from incomming file + FILE *token_file = fopen("/responses/Authorization", "r"); + if (token_file == NULL) { + perror("Failed to open file with token"); + return -1; + } + char auth_token[257] = {0}; + if (fscanf(token_file, "Bearer %256s", auth_token) < 0) { + perror("Failed to parse line from token file"); + return -1; + } + + // handle + int auth_sock = connect_to_server(auth_server_ip, AUTH_SERVER_PORT); + if (auth_sock < 0) + return 1; + + char auth_request[BUFFER_SIZE]; + int written = snprintf(auth_request, sizeof(auth_request), + "POST /authorize HTTP/1.1\n" + "Host: %s\n" + "Content-Type: application/json\n" + "Content-Length: %zu\n\n" + "{\"token\": \"%s\"}", + auth_server_ip, strlen(auth_token) + 13, auth_token); + if (written < 0 || written > BUFFER_SIZE) { + perror("Failed to format auth request"); + return -1; + } + char auth_response[BUFFER_SIZE]; + if (send_http_request(auth_sock, auth_request, written, auth_response, + BUFFER_SIZE) < 0) { + close(auth_sock); + return 1; + } + + close(auth_sock); + + char token[256]; + parse_auth_response(auth_response, token); + + // fan_out + log_node *log_root = NULL; + char log_request[SERVER_NUMBER][BUFFER_SIZE]; + int request_length[SERVER_NUMBER]; + char log_response[SERVER_NUMBER][BUFFER_SIZE]; + size_t response_read[SERVER_NUMBER]; + struct pollfd poll_fds[SERVER_NUMBER]; + + int log_sockets[SERVER_NUMBER]; + // open sockets + for (int server = 0; server < SERVER_NUMBER; server++) { + log_sockets[server] = connect_to_server(auth_server_ip, AUTH_SERVER_PORT); + if (log_sockets[server] < 0) { + perror("Failed to open log socket"); + return -1; + } + } + // format requests + for (int server = 0; server < SERVER_NUMBER; server++) { + request_length[server] = + snprintf(log_request[server], sizeof(log_request[server]), + "GET /logs/%d HTTP/1.1\n" + "Host: %s\n" + "Authorization: Bearer %s\n\n", + server, auth_server_ip, token); + if (request_length < 0) { + perror("Could not format server log request"); + return -1; + } + } + // issue all requests + for (int server = 0; server < SERVER_NUMBER; server++) { + if (linux_send(log_sockets[server], log_request[server], + request_length[server], 0) < 0) { + perror("Send failed"); + return -1; + } + response_read[server] = 0; + poll_fds[server].fd = log_sockets[server]; + poll_fds[server].events = POLLIN; + poll_fds[server].revents = 0; + } + unsigned int received_responses = 0; + while (received_responses < SERVER_NUMBER) { + int poll_result = linux_ppoll(poll_fds, SERVER_NUMBER); + if (poll_result < 0) { + perror("Polling failed"); + return -1; + } + // check all the file descriptors for arrived data + for (int server = 0; server < SERVER_NUMBER; server++) { + if (poll_fds[server].fd > 0 && + (poll_fds[server].revents & (POLLIN | POLLHUP))) { + received_responses++; + // disable further polling on this socket + poll_fds[server].fd = -1; + int new_data = linux_recv(log_sockets[server], + log_response[server] + response_read[server], + BUFFER_SIZE - response_read[server] - 1, 0); + if (new_data < 0) { + perror("Failed to read from socket"); + return -1; + } + response_read[server] += new_data; + log_response[server][new_data] = '\0'; + parse_log_events(log_response[server], &log_root); + } + } + } + // close sockets + for (int server = 0; server < SERVER_NUMBER; server++) { + close(log_sockets[server]); + } + + // render + render_logs_to_html(log_root); + + // Free log nodes + log_node *current = log_root; + while (current) { + log_node *next = current->next; + free(current); + current = next; + } + + return 0; +} \ No newline at end of file diff --git a/functions/example_app_hybrid/linux_syscalls/CMakeLists.txt b/c_functions/example_app_hybrid/linux_syscalls/CMakeLists.txt similarity index 100% rename from functions/example_app_hybrid/linux_syscalls/CMakeLists.txt rename to c_functions/example_app_hybrid/linux_syscalls/CMakeLists.txt diff --git a/functions/example_app_hybrid/linux_syscalls/arch/aarch64/syscall_arch.h b/c_functions/example_app_hybrid/linux_syscalls/arch/aarch64/syscall_arch.h similarity index 100% rename from functions/example_app_hybrid/linux_syscalls/arch/aarch64/syscall_arch.h rename to c_functions/example_app_hybrid/linux_syscalls/arch/aarch64/syscall_arch.h diff --git a/functions/example_app_hybrid/linux_syscalls/arch/x86_64/syscall_arch.h b/c_functions/example_app_hybrid/linux_syscalls/arch/x86_64/syscall_arch.h similarity index 100% rename from functions/example_app_hybrid/linux_syscalls/arch/x86_64/syscall_arch.h rename to c_functions/example_app_hybrid/linux_syscalls/arch/x86_64/syscall_arch.h diff --git a/functions/busy_hybrid/linux_syscalls/syscall.h b/c_functions/example_app_hybrid/linux_syscalls/syscall.h similarity index 74% rename from functions/busy_hybrid/linux_syscalls/syscall.h rename to c_functions/example_app_hybrid/linux_syscalls/syscall.h index 17c166a..723eceb 100644 --- a/functions/busy_hybrid/linux_syscalls/syscall.h +++ b/c_functions/example_app_hybrid/linux_syscalls/syscall.h @@ -18,12 +18,12 @@ #define LINUX_SOCK_STREAM 1 // poll defines -#define POLLIN 0x0001 -#define POLLPRI 0x0002 -#define POLLOUT 0x0004 -#define POLLERR 0x0008 -#define POLLHUP 0x0010 -#define POLLNVAL 0x0020 +#define POLLIN 0x0001 +#define POLLPRI 0x0002 +#define POLLOUT 0x0004 +#define POLLERR 0x0008 +#define POLLHUP 0x0010 +#define POLLNVAL 0x0020 #ifndef __scc #define __scc(X) ((long)(X)) @@ -63,34 +63,33 @@ struct sockaddr { }; struct in_addr { - uint32_t s_addr; + uint32_t s_addr; }; struct sockaddr_in { - unsigned short sin_family; - uint16_t sin_port; - struct in_addr sin_addr; - unsigned char sin_zero[sizeof (struct sockaddr) - - sizeof(unsigned short) - - sizeof(uint16_t) - - sizeof(struct in_addr)]; + unsigned short sin_family; + uint16_t sin_port; + struct in_addr sin_addr; + unsigned char sin_zero[sizeof(struct sockaddr) - sizeof(unsigned short) - + sizeof(uint16_t) - sizeof(struct in_addr)]; }; -static inline int linux_connect(int socketfd, const struct sockaddr_in* addr, int addrlen){ +static inline int linux_connect(int socketfd, const struct sockaddr_in *addr, + int addrlen) { return __syscall(SYS_connect, socketfd, addr, addrlen); } -static inline ptrdiff_t linux_send(int socketfd, const void* buf, size_t len, int flags){ +static inline ptrdiff_t linux_send(int socketfd, const void *buf, size_t len, + int flags) { return __syscall(SYS_sendto, socketfd, buf, len, flags, NULL, 0); } -static inline ptrdiff_t linux_recv(int socketfd, void* buf, size_t len, int flags){ +static inline ptrdiff_t linux_recv(int socketfd, void *buf, size_t len, + int flags) { return __syscall(SYS_recvfrom, socketfd, buf, len, flags, NULL, NULL); } -static inline int linux_close(int fd){ - return __syscall(SYS_close, fd); -} +static inline int linux_close(int fd) { return __syscall(SYS_close, fd); } struct pollfd { int fd; @@ -98,7 +97,7 @@ struct pollfd { short revents; }; -static inline int linux_ppoll(struct pollfd* fds, unsigned int nfds) { +static inline int linux_ppoll(struct pollfd *fds, unsigned int nfds) { return __syscall(SYS_ppoll, fds, nfds, NULL, NULL); } diff --git a/functions/example_app_nolibc/CMakeLists.txt b/c_functions/example_app_nolibc/CMakeLists.txt similarity index 100% rename from functions/example_app_nolibc/CMakeLists.txt rename to c_functions/example_app_nolibc/CMakeLists.txt diff --git a/functions/example_app_nolibc/commons.h b/c_functions/example_app_nolibc/commons.h similarity index 100% rename from functions/example_app_nolibc/commons.h rename to c_functions/example_app_nolibc/commons.h diff --git a/functions/example_app_nolibc/fan_out.c b/c_functions/example_app_nolibc/fan_out.c similarity index 100% rename from functions/example_app_nolibc/fan_out.c rename to c_functions/example_app_nolibc/fan_out.c diff --git a/functions/example_app_nolibc/handle.c b/c_functions/example_app_nolibc/handle.c similarity index 100% rename from functions/example_app_nolibc/handle.c rename to c_functions/example_app_nolibc/handle.c diff --git a/functions/example_app_nolibc/template.c b/c_functions/example_app_nolibc/template.c similarity index 100% rename from functions/example_app_nolibc/template.c rename to c_functions/example_app_nolibc/template.c diff --git a/functions/files/CMakeLists.txt b/c_functions/files/CMakeLists.txt similarity index 100% rename from functions/files/CMakeLists.txt rename to c_functions/files/CMakeLists.txt diff --git a/functions/files/fileio.c b/c_functions/files/fileio.c similarity index 91% rename from functions/files/fileio.c rename to c_functions/files/fileio.c index b74a382..5fefae4 100644 --- a/functions/files/fileio.c +++ b/c_functions/files/fileio.c @@ -41,13 +41,13 @@ int copy_folder(char *in_parent, char *out_parent) { char *new_in = new_path(in_parent, entry->d_name); if (entry->d_type == DT_DIR) { // check if folder exists - DIR* out_parent = opendir(new_out); - if(out_parent == NULL && errno == ENOENT){ + DIR *out_parent = opendir(new_out); + if (out_parent == NULL && errno == ENOENT) { if (mkdir(new_out, 0x777) < 0) { perror("Failed to create new directory"); return -1; } - } else if(out_parent == NULL) { + } else if (out_parent == NULL) { perror("Failed check for existing directory"); return -1; } else { diff --git a/functions/matmac/CMakeLists.txt b/c_functions/matmac/CMakeLists.txt similarity index 100% rename from functions/matmac/CMakeLists.txt rename to c_functions/matmac/CMakeLists.txt diff --git a/functions/matmac/matmac.c b/c_functions/matmac/matmac.c similarity index 100% rename from functions/matmac/matmac.c rename to c_functions/matmac/matmac.c diff --git a/functions/matmul/CMakeLists.txt b/c_functions/matmul/CMakeLists.txt similarity index 100% rename from functions/matmul/CMakeLists.txt rename to c_functions/matmul/CMakeLists.txt diff --git a/functions/matmul/matmul.c b/c_functions/matmul/matmul.c similarity index 100% rename from functions/matmul/matmul.c rename to c_functions/matmul/matmul.c diff --git a/functions/stdio/CMakeLists.txt b/c_functions/stdio/CMakeLists.txt similarity index 100% rename from functions/stdio/CMakeLists.txt rename to c_functions/stdio/CMakeLists.txt diff --git a/functions/stdio/stdio.c b/c_functions/stdio/stdio.c similarity index 83% rename from functions/stdio/stdio.c rename to c_functions/stdio/stdio.c index b0e2190..d5766b4 100644 --- a/functions/stdio/stdio.c +++ b/c_functions/stdio/stdio.c @@ -4,7 +4,7 @@ #include "unistd.h" -int main(int argc, char const* argv[]) { +int main(int argc, char const *argv[]) { // print to std out and std err with the usual mode int err; if ((err = printf("Test string to stdout\n")) < 0) { @@ -22,12 +22,13 @@ int main(int argc, char const* argv[]) { } printf("read %zi characters from stdin\n", read_chars); ssize_t written = fwrite(in_buffer, 1, read_chars, stdout); - if (written != read_chars) return -4; + if (written != read_chars) + return -4; // print string arguments for (int iter = 0; iter < argc; iter++) { printf("argument %d is %s\n", iter, argv[iter]); } - char* env_home = getenv("HOME"); + char *env_home = getenv("HOME"); printf("environmental variable HOME is %s\n", env_home); return 0; } diff --git a/functions/stdio_cxx/CMakeLists.txt b/c_functions/stdio_cxx/CMakeLists.txt similarity index 100% rename from functions/stdio_cxx/CMakeLists.txt rename to c_functions/stdio_cxx/CMakeLists.txt diff --git a/functions/stdio_cxx/stdio.cpp b/c_functions/stdio_cxx/stdio.cpp similarity index 100% rename from functions/stdio_cxx/stdio.cpp rename to c_functions/stdio_cxx/stdio.cpp diff --git a/functions/README.md b/functions/README.md deleted file mode 100644 index 32f422e..0000000 --- a/functions/README.md +++ /dev/null @@ -1,17 +0,0 @@ -# Functions - -This folder contains a series of functions used to test basic functionality. -The functions are split into multiple executables that can be built individually. -Some functions depend on the function interface directly while others use it -through mlibc. - -# Building the C functions -The cmake build process will build the functions based on the latest version of the dandelion SDK. -For this cmake meeds to know the platform to build for (cheri, mmu_freebsd, mmu_linux, debug) and architecture (x86_64, aarch64) -The build type can be Release or Debug. -From the top folder functions can be built with: -``` -mkdir build -cd build -cmake -DPLATFORM= -DARCHITECTURE= -DCMAKE_BUILD_TYPE- ../functions -``` diff --git a/functions/busy_hybrid/busy_hybrid.c b/functions/busy_hybrid/busy_hybrid.c deleted file mode 100644 index edcfa17..0000000 --- a/functions/busy_hybrid/busy_hybrid.c +++ /dev/null @@ -1,289 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include - -#include "syscall.h" -#include "unistd.h" - -#define ARRAY_ITEMS 16 -#define BUFFER_SIZE 1 * 1024 * 1024 -#define FETCH_REQUEST_PATH "/data/get_request" -#define STORE_PREAMBLE_PATH "/preamble/post_request" -#define STORE_RESPONSE_PATH "/store_request/post_response" - -typedef struct data_item { - int8_t value[ARRAY_ITEMS]; -} data_item; - -uint16_t my_htons(uint16_t port) { - return ((port >> 8) & 0x00FF) | ((port << 8) & 0xFF00); -} - -int my_inet_pton(int af, const char *src, void *dst) { - if (af == LINUX_AF_INET) { - uint8_t array[4] = {}; - for (int i = 0; i < 4; i++) { - char *end; - long value = strtol(src, &end, 10); - if (value < 0 || value > 255) { - return 0; - } - - if (*end != '\0' && *end != '.' && i < 3) { - return 0; - } - src = end + 1; - array[i] = (uint8_t)value; - } - - struct in_addr *addr = (struct in_addr *)dst; - memcpy(&addr->s_addr, array, 4); // Copy the 4 bytes into s_addr - - return 1; // Success - } - return -1; // Unsupported address family -} - -int extract_ip_port(const char *http_request, char *ip, size_t ip_len, - uint16_t *port) { - const char *prefix = "http://"; - const char *start = strstr(http_request, prefix); - if (!start) - return -1; - - start += strlen(prefix); - const char *colon = strchr(start, ':'); - if (!colon) - return -2; - - const char *slash = strchr(colon, '/'); - if (!slash) - return -3; - - // Extract IP - size_t ip_size = colon - start; - if (ip_size >= ip_len) - return -4; - strncpy(ip, start, ip_size); - ip[ip_size] = '\0'; - - // Extract port - char port_str[6] = {0}; - size_t port_size = slash - colon - 1; - if (port_size >= sizeof(port_str)) - return -5; - strncpy(port_str, colon + 1, port_size); - *port = atoi(port_str); - - return 0; -} - -int connect_to_server(const char *ip, uint16_t port) { - struct sockaddr_in server_addr; - - int sockfd = linux_socket(LINUX_AF_INET, LINUX_SOCK_STREAM, 0); - if (sockfd < 0) { - perror("Socket creation failed"); - return -1; - } - - server_addr.sin_family = LINUX_AF_INET; - server_addr.sin_port = my_htons(port); - - if (my_inet_pton(LINUX_AF_INET, ip, &server_addr.sin_addr) != 1) { - perror("Invalid IP address"); - close(sockfd); - return -1; - } - - int err = linux_connect(sockfd, &server_addr, sizeof(server_addr)); - if (err < 0) { - printf("Error: %s\n", strerror(-err)); - printf("size of server_addr: %ld\n", sizeof(server_addr)); - perror("Socket connection failed"); - close(sockfd); - return -1; - } - - return sockfd; -} - -ssize_t send_http_request(int sockfd, const char *request, size_t request_size, - char *response, size_t response_size) { - if (linux_send(sockfd, request, request_size, 0) < 0) { - perror("Error sending request"); - return -1; - } - - // TODO: ensure the entire HTTP response has been received - ssize_t total_received = 0, bytes; - // while ((bytes = linux_recv(sockfd, response + total_received, - // response_size - total_received - 1, 0)) > 0) { - // total_received += bytes; - // if (total_received >= response_size - 1) break; - // } - // blocks and waits forever if there is no next packate, could validate HTTP - // sice on the packages to see if we need more, this works for now - total_received = linux_recv(sockfd, response + total_received, - response_size - total_received - 1, 0); - if (total_received < 0) { - perror("Error receiving response"); - return -1; - } - response[total_received] = '\0'; - return total_received; -} - -void parse_http_body(const char *response, size_t response_size, - char **response_body, size_t *response_body_size) { - *response_body = strstr(response, "\r\n\r\n"); - if (*response_body == NULL) - return; - *response_body += 4; - *response_body_size = response_size - (*response_body - response); -} - -size_t read_file_to_string(const char *filename, char **content) { - // Open file in read mode - FILE *file = fopen(filename, "r"); - if (file == NULL) { - perror("Error opening file"); - } - - // Get file size - fseek(file, 0, SEEK_END); - size_t file_size = ftell(file); - rewind(file); - - // Allocate memory for content (+1 for null terminator) - *content = (char *)malloc(file_size + 1); - if (*content == NULL) { - perror("Error allocating memory"); - } - - // Read file into the buffer - size_t bytes_read = fread(*content, 1, file_size, file); - (*content)[bytes_read] = '\0'; // Add null terminator - fclose(file); - - return bytes_read; // Return the length of the content -} - -int main() { - char *fetch_preamble = NULL; - size_t fetch_preamble_len = - read_file_to_string(FETCH_REQUEST_PATH, &fetch_preamble); - if (fetch_preamble == NULL) - return 1; - - char server_ip[16]; - uint16_t server_port; - if (extract_ip_port(fetch_preamble, server_ip, 16, &server_port) < 0) - return 2; - - // fetching data - int sock = connect_to_server(server_ip, server_port); - if (sock < 0) - return 3; - - size_t fetch_request_len = fetch_preamble_len + 4; - char *fetch_request = malloc(fetch_request_len); - memcpy(fetch_request, fetch_preamble, fetch_preamble_len); - memcpy(fetch_request + fetch_preamble_len, "\r\n\r\n", 4); - - char *fetch_response = (char *)malloc(BUFFER_SIZE); - if (fetch_response == NULL) { - perror("Error allocating memory"); - return 4; - } - ssize_t fetch_response_len = send_http_request( - sock, fetch_request, fetch_request_len, fetch_response, BUFFER_SIZE); - if (fetch_response_len < 0) - return 5; - close(sock); - - char *fetch_response_body; - size_t fetch_response_body_len; - parse_http_body(fetch_response, fetch_response_len, &fetch_response_body, - &fetch_response_body_len); - if (fetch_response_body == NULL) - return 6; - - uint64_t iterations = *(uint64_t *)fetch_response_body; - data_item *data_items = (data_item *)(fetch_response_body + 8); - - // the igreggated statistics we want to collect - // sum, average, variance, min, max - int64_t sum = 0; - int64_t min = 256; - int64_t max = -256; - - size_t struct_index = 0; - size_t value_index = 0; - size_t max_index = (fetch_response_body_len - 8) / sizeof(data_item); - for (size_t iteration = 0; iteration < iterations; iteration++) { - for (size_t array_index = 0; array_index < ARRAY_ITEMS; array_index++) { - int8_t value = data_items[struct_index].value[array_index]; - if (value > max) - max = value; - if (value < min) - min = value; - sum += value; - } - // using size_t which has no sign, guarantees that the index is not - // negative when computing the remainder - struct_index = - (struct_index + data_items[struct_index].value[value_index]) % - max_index; - value_index = (value_index + 1) % ARRAY_ITEMS; - } - - char *store_preamble = NULL; - size_t store_preamble_len = - read_file_to_string(STORE_PREAMBLE_PATH, &store_preamble); - if (store_preamble == NULL) - return 7; - store_preamble_len -= 2; // omit \n\n at the end - - char header[] = "\r\ncontent-length: 24\r\n\r\n"; // 3 * sizeof(int64_t) - size_t store_request_len = - store_preamble_len + (sizeof(header) - 1) + 3 * sizeof(int64_t); - char *store_request = malloc(store_request_len); - char *current_ptr = store_request; - memcpy(current_ptr, store_preamble, store_preamble_len); - current_ptr += store_preamble_len; - memcpy(current_ptr, header, sizeof(header) - 1); - current_ptr += sizeof(header) - 1; - memcpy(current_ptr, &sum, sizeof(int64_t)); - current_ptr += sizeof(int64_t); - memcpy(current_ptr, &min, sizeof(int64_t)); - current_ptr += sizeof(int64_t); - memcpy(current_ptr, &max, sizeof(int64_t)); - current_ptr += sizeof(int64_t); - - // posting data - char *store_response = fetch_response; - sock = connect_to_server(server_ip, server_port); - if (sock < 0) - return 3; - ssize_t store_response_len = send_http_request( - sock, store_request, store_request_len, store_response, BUFFER_SIZE); - if (store_response_len < 0) - return 8; - close(sock); - - FILE *store_response_file = fopen(STORE_RESPONSE_PATH, "wb"); - if (store_response_file == NULL) { - perror("Error opening file"); - return 9; - } - fwrite(store_response, 1, store_response_len, store_response_file); - fclose(store_response_file); - - return 0; -} diff --git a/functions/busy_libc/busy_libc.c b/functions/busy_libc/busy_libc.c deleted file mode 100644 index 4147084..0000000 --- a/functions/busy_libc/busy_libc.c +++ /dev/null @@ -1,119 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include - -#include "dandelion/runtime.h" -#include "unistd.h" - -#define ARRAY_ITEMS 16 -#define FETCH_REQUEST_PATH "/data/get_request" -#define STORE_PREAMBLE_PATH "/preamble/post_request" -#define STORE_RESPONSE_PATH "/store_request/post_response" - -typedef struct data_item { - int8_t value[ARRAY_ITEMS]; -} data_item; - -size_t read_file_to_string(const char *filename, char **content) { - // Open file in read mode - FILE *file = fopen(filename, "r"); - if (file == NULL) { - perror("Error opening file"); - } - - // Get file size - fseek(file, 0, SEEK_END); - size_t file_size = ftell(file); - rewind(file); - - // Allocate memory for content (+1 for null terminator) - *content = (char *)malloc(file_size + 1); - if (*content == NULL) { - perror("Error allocating memory"); - } - - // Read file into the buffer - size_t bytes_read = fread(*content, 1, file_size, file); - (*content)[bytes_read] = '\0'; // Add null terminator - fclose(file); - - return bytes_read; // Return the length of the content -} - -int main() { - // use dandelion runtime interface directly to avoid data copy - IoBuffer *data_buffer = dandelion_get_input(0, 0); - char *data = (char *)data_buffer->data; - size_t data_len = data_buffer->data_len; - // char *data = NULL; - // size_t data_len = read_file_to_string(FETCH_REQUEST_PATH, &data); - // if (data == NULL) - // return 1; - - uint64_t iterations = *(uint64_t *)data; - data_item *data_items = (data_item *)(data + 8); - - // the igreggated statistics we want to collect - // sum, average, variance, min, max - int64_t sum = 0; - int64_t min = 256; - int64_t max = -256; - - size_t struct_index = 0; - size_t value_index = 0; - size_t max_index = (data_len - 8) / sizeof(data_item); - for (size_t iteration = 0; iteration < iterations; iteration++) { - for (size_t array_index = 0; array_index < ARRAY_ITEMS; array_index++) { - int8_t value = data_items[struct_index].value[array_index]; - if (value > max) - max = value; - if (value < min) - min = value; - sum += value; - } - // using size_t which has no sign, guarantees that the index is not - // negative when computing the remainder - struct_index = - (struct_index + data_items[struct_index].value[value_index]) % - max_index; - value_index = (value_index + 1) % ARRAY_ITEMS; - } - - char *store_preamble = NULL; - size_t store_preamble_len = - read_file_to_string(STORE_PREAMBLE_PATH, &store_preamble); - if (store_preamble == NULL) - return 7; - store_preamble_len -= 2; // omit \n\n at the end - - char header[] = "\nContent-Length: 24\n\n"; // 3 * sizeof(int64_t) - size_t store_request_len = - store_preamble_len + (sizeof(header) - 1) + 3 * sizeof(int64_t); - char *store_request = malloc(store_request_len); - char *current_ptr = store_request; - memcpy(current_ptr, store_preamble, store_preamble_len); - current_ptr += store_preamble_len; - memcpy(current_ptr, header, sizeof(header) - 1); - current_ptr += sizeof(header) - 1; - memcpy(current_ptr, &sum, sizeof(int64_t)); - current_ptr += sizeof(int64_t); - memcpy(current_ptr, &min, sizeof(int64_t)); - current_ptr += sizeof(int64_t); - memcpy(current_ptr, &max, sizeof(int64_t)); - current_ptr += sizeof(int64_t); - - FILE *store_response_file = fopen(STORE_RESPONSE_PATH, "wb"); - if (store_response_file == NULL) { - perror("Error opening file"); - return 9; - } - fwrite(store_request, 1, store_request_len, store_response_file); - fclose(store_response_file); - - return 0; -} diff --git a/functions/dirigent_busy/dirigent_busy.c b/functions/dirigent_busy/dirigent_busy.c deleted file mode 100644 index 551d81f..0000000 --- a/functions/dirigent_busy/dirigent_busy.c +++ /dev/null @@ -1,85 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include - -#include "unistd.h" - -int main(int argc, char const *argv[]) { - // parse input - FILE *input_data = fopen("/input/input.csv", "r"); - if (input_data == NULL) { - perror("could not open /input/input.csv"); - return -1; - } - FILE *output_data = fopen("/output/output.csv", "w+"); - if (output_data == NULL) { - perror("could not open /output/output.csv"); - return -1; - } - - char *workload = NULL; - size_t workload_size = 0; - if (__getdelim(&workload, &workload_size, ',', input_data) < 0) { - perror("failed to read workload"); - return -1; - } - workload[strlen(workload) - 1] = '\0'; - - char *function = NULL; - size_t function_size = 0; - if (__getdelim(&function, &function_size, ',', input_data) < 0) { - perror("failed to read function"); - return -1; - } - function[strlen(function) - 1] = '\0'; - - char *requestedCpu = NULL; - size_t requestedCpu_size = 0; - if (__getdelim(&requestedCpu, &requestedCpu_size, ',', input_data) < 0) { - perror("failed to read requestedCpu"); - return -1; - } - requestedCpu[strlen(requestedCpu) - 1] = '\0'; - - char *multiplier = NULL; - size_t multiplier_size = 0; - if (__getdelim(&multiplier, &multiplier_size, ',', input_data) < 0) { - perror("failed to read multiplier"); - return -1; - } - - char *trace_string = "trace"; - char *empyt_string = "empty"; - - if (strcmp(trace_string, function) == 0) { - long multiplier_num = strtol(multiplier, NULL, 10); - long requested_cpu_num = strtol(requestedCpu, NULL, 10); - - long total_iterations = multiplier_num * requested_cpu_num; - - //printf("%ld, %ld, %ld\n", multiplier_num, requested_cpu_num, total_iterations); - - volatile double result = 0.0; - volatile double input = 10.0; - volatile long iteration; - for (iteration = 0; iteration < total_iterations; iteration++) { - result = sqrt(input); - } - - fprintf(output_data, "\"OK\",\"%s\",\"dandelionServer\",%ld,%f", function, iteration, result); - - return 0; - } else if (strcmp(empyt_string, function) == 0) { - fprintf(output_data, "\"OK - EMPTY\",\"dandelionServer\",\"%s\",0", function); - - return 0; - } - - return -1; - -} diff --git a/functions/example_app/template.c b/functions/example_app/template.c deleted file mode 100644 index 1b495cd..0000000 --- a/functions/example_app/template.c +++ /dev/null @@ -1,129 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include - -#include "unistd.h" - -typedef struct log_node{ - struct log_node* next; - char timestamp[100]; - char server_id[100]; - char type[100]; - char details[100]; -} log_node; - -char event_template[] = -"{" -"\"details\":\"%100[^\"]\"," -"\"event_type\":\"%100[^\"]\"," -"\"server_id\":\"%100[^\"]\"," -"\"timestamp\":\"%100[^\"]\"" -"%*[^{]"; - -int main(int argc, char const *argv[]) { - - log_node* log_root = NULL; - - // read all events and put the in list sorted by timestamps - DIR* log_dir = opendir("/responses"); - if (log_dir == NULL) { - perror("Could not open log directory"); - return -1; - } - - struct dirent* entry; - while((entry = readdir(log_dir))) { - if(entry->d_type == DT_REG) { - char filepath[256] = {0}; - if(snprintf(filepath, 256, "/responses/%s", entry->d_name) < 0){ - perror("Failed to format log filepath\n"); - return -1; - } - FILE* log_file = fopen(filepath, "r"); - size_t line_size = 0; - char* line_buffer = NULL; - // read until and including initial [ - if(__getdelim(&line_buffer, &line_size,'[', log_file) < 0){ - perror("Failed to read initial json {"); - return -1; - } - free(line_buffer); - // keep reading until we find no more } - int items_found = 0; - log_node* current = malloc(sizeof(log_node)); - while( - (items_found = - fscanf(log_file, - event_template, - current->details, - current->type, - current->server_id, - current->timestamp)) == 4){ - current->next = NULL; - if(log_root == NULL) { - log_root = current; - } else { - if (strcmp(current->timestamp, log_root->timestamp) <= 0){ - current->next = log_root; - log_root = current; - } else { - log_node* previous = log_root; - while(previous->next != NULL) { - log_node* next = previous->next; - if(strcmp(current->timestamp, next->timestamp) <= 0){ - current->next = next; - break; - } - previous = next; - } - previous->next = current; - } - } - current = malloc(sizeof(log_node)); - } - free(current); - } - } - - // open output file - FILE *log_file = fopen("/requests/log.txt", "w+"); - if (log_file == NULL) { - perror("Failed to open file to get auth server\n"); - return -1; - } - // write preamble - fprintf(log_file, "\n"); - fprintf(log_file, "\n"); - fprintf(log_file, "\n"); - fprintf(log_file, " \n"); - fprintf(log_file, " Logs\n"); - fprintf(log_file, "\n"); - fprintf(log_file, "\n"); - fprintf(log_file, " \n"); - fprintf(log_file, " \n"); - fprintf(log_file, " \n"); - fprintf(log_file, " \n"); - fprintf(log_file, " \n"); - fprintf(log_file, " \n"); - fprintf(log_file, " \n"); - - // write log information - for(log_node* current = log_root; current != NULL; current = current->next){ - fprintf(log_file, " \n"); - fprintf(log_file, " \n", current->timestamp); - fprintf(log_file, " \n", current->server_id); - fprintf(log_file, " \n", current->type); - fprintf(log_file, " \n", current->details); - fprintf(log_file, " \n"); - } - - // write ending - fprintf(log_file, "
TimestampServer IDEvent TypeDetails
%s%s%s%s
\n"); - fprintf(log_file, "\n"); - - return -1; -} diff --git a/functions/example_app_hybrid/app_hybrid.c b/functions/example_app_hybrid/app_hybrid.c deleted file mode 100644 index 69279bc..0000000 --- a/functions/example_app_hybrid/app_hybrid.c +++ /dev/null @@ -1,399 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include "unistd.h" - -#include "syscall.h" - -#define BUFFER_SIZE 4096 -#define SERVER_NUMBER 10 -const uint16_t AUTH_SERVER_PORT = 8000; - -// Structure to store log event details -typedef struct log_node { - struct log_node *next; - char timestamp[100]; - char server_id[100]; - char type[100]; - char details[100]; -} log_node; - -char event_template[] = -"%*[^{]{" -"\"details\":\"%100[^\"]\"," -"\"event_type\":\"%100[^\"]\"," -"\"server_id\":\"%100[^\"]\"," -"\"timestamp\":\"%100[^\"]\"" -"%}"; - - -uint16_t my_htons(uint16_t port) { - return ((port >> 8) & 0x00FF) | ((port << 8) & 0xFF00); -} - -int my_inet_pton(int af, const char *src, void *dst) { - if (af == LINUX_AF_INET) { - uint8_t array[4] = {}; - for (int i = 0; i < 4; i++) { - char *end; - long value = strtol(src, &end, 10); - if (value < 0 || value > 255) { - return 0; - } - - if (*end != '\0' && *end != '.' && i < 3) { - return 0; - } - src = end + 1; - array[i] = (uint8_t)value; - } - - struct in_addr *addr = (struct in_addr *)dst; - memcpy(&addr->s_addr, array, 4); // Copy the 4 bytes into s_addr - - return 1; // Success - } - return -1; // Unsupported address family -} - -int connect_to_server(const char *ip, uint16_t port) { - struct sockaddr_in server_addr; - - int sockfd = linux_socket(LINUX_AF_INET, LINUX_SOCK_STREAM, 0); - if (sockfd < 0) { - perror("Socket creation failed"); - return -1; - } - - server_addr.sin_family = LINUX_AF_INET; - server_addr.sin_port = my_htons(port); - - if (my_inet_pton(LINUX_AF_INET, ip, &server_addr.sin_addr) != 1) { - perror("Invalid IP address"); - close(sockfd); - return -1; - } - - int err = linux_connect(sockfd, &server_addr, sizeof(server_addr)); - if (err < 0) { - printf("Error: %s\n", strerror(-err)); - printf("size of server_addr: %ld\n", sizeof(server_addr)); - perror("Socket connection failed"); - close(sockfd); - return -1; - } - - return sockfd; -} - - -int send_http_request(int sockfd, const char *request, size_t request_size, char *response, size_t response_size) { - if (linux_send(sockfd, request, request_size, 0) < 0) { - perror("Send failed"); - return -1; - } - - ssize_t total_received = 0, bytes; - // while ((bytes = linux_recv(sockfd, response + total_received, response_size - total_received - 1, 0)) > 0) { - // total_received += bytes; - // if (total_received >= response_size - 1) break; - // } - // blocks and waits forever if there is no next packate, could validate HTTP sice on the packages to see if we need more, this works for now - total_received = linux_recv(sockfd, response + total_received, response_size - total_received - 1, 0); - if(total_received < 0){ - return -1; - } - response[total_received] = '\0'; - return 0; -} - - -void parse_auth_response(const char *response, char *token) { - const char *start = strstr(response, "\"token\":"); - if (start) { - sscanf(start, "\"token\": \"%[^\"]", token); - } -} - -void parse_log_events(const char *response, log_node **log_root) { - log_node *current = NULL; - - const char *start = strstr(response, "\"events\":"); - if (start) { - start += 9; - while (1) { - current = malloc(sizeof(log_node)); - int found = sscanf(start, - event_template, - current->details, - current->type, - current->server_id, - current->timestamp); - if (found != 4) { - free(current); - break; - } - - current->next = *log_root; - *log_root = current; - - start = strstr(start, "},{"); - if (!start) break; - start += 1; - } - } -} - -// Render logs to an HTML file -void render_logs_to_html(log_node *log_root) { - FILE *log_file = fopen("/requests/log.html", "w+"); - if (!log_file) { - perror("Failed to open HTML file"); - return; - } - - // sort by date - // get number - size_t num_entries = 0; - log_node* current = log_root; - while(current){ - num_entries++; - current = current->next; - } - log_node** log_heap = malloc(sizeof(log_node*) * num_entries); - current = log_root; - for(size_t index = 0; index < num_entries; index++){ - log_heap[index] = current; - current = current->next; - for(size_t insert = index; insert > 0; insert = insert / 2){ - if(strcmp(log_heap[insert/2]->timestamp, log_heap[insert]->timestamp) <= 0){ - break; - } else { - log_node* tmp = log_heap[insert]; - log_heap[insert] = log_heap[insert/2]; - log_heap[insert/2] = tmp; - } - } - } - - // write preamble - fprintf(log_file, "\n"); - fprintf(log_file, "\n"); - fprintf(log_file, "\n"); - fprintf(log_file, " \n"); - fprintf(log_file, " Logs\n"); - fprintf(log_file, "\n"); - fprintf(log_file, "\n"); - fprintf(log_file, " \n"); - fprintf(log_file, " \n"); - fprintf(log_file, " \n"); - fprintf(log_file, " \n"); - fprintf(log_file, " \n"); - fprintf(log_file, " \n"); - fprintf(log_file, " \n"); - - // write log information - for(; num_entries > 0; num_entries--){ - current = log_heap[0]; - fprintf(log_file, " \n"); - fprintf(log_file, " \n", current->timestamp); - fprintf(log_file, " \n", current->server_id); - fprintf(log_file, " \n", current->type); - fprintf(log_file, " \n", current->details); - fprintf(log_file, " \n"); - // remove current root from heap - size_t index = 0; - log_heap[0] = log_heap[num_entries-1]; - // trade down until no more children - while(index*2 + 1 < num_entries){ - // compare with first child - int is_bigger = strcmp(log_heap[index]->timestamp, log_heap[index*2+1]->timestamp); - // check if second child exists - if(index*2 + 2 >= num_entries){ - // second child does not exist, if is bigger than child swap - if(is_bigger > 0){ - log_node* tmp = log_heap[index]; - log_heap[index] = log_heap[index*2+1]; - log_heap[index*2+1] = tmp; - } - break; - } - // know fist child is smaller, if second is smaller than first swap with second otherwise with first - if(is_bigger > 0){ - size_t swap_index; - if(strcmp(log_heap[index*2+1]->timestamp, log_heap[index*2+2]->timestamp) <= 0){ - // first is smaller - swap_index = index*2+1; - } else { - swap_index = index*2+2; - } - log_node* tmp = log_heap[index]; - log_heap[index] = log_heap[swap_index]; - log_heap[swap_index] = tmp; - index = swap_index; - } else { - // first is not smaller so need to check second if swap is neccessary - if(strcmp(log_heap[index]->timestamp, log_heap[index*2+2]->timestamp) > 0){ - // is bigger so need to swap - log_node* tmp = log_heap[index]; - log_heap[index] = log_heap[index*2+2]; - log_heap[index*2+2] = tmp; - index = index*2+2; - } else { - break; - } - } - } - } - - // write ending - fprintf(log_file, "
TimestampServer IDEvent TypeDetails
%s%s%s%s
\n"); - fprintf(log_file, "\n"); - - fclose(log_file); -} - -// Main function to handle authorization, log fetching, and rendering -int main() { - // read authentification server ip from file - FILE *auth_server = fopen("/servers/server.txt", "r"); - if (auth_server == NULL) { - perror("Failed to open file to get auth server"); - return -1; - } - char *auth_server_ip = NULL; - size_t server_line_len = 0; - size_t read_chars = __getline(&auth_server_ip, &server_line_len, auth_server); - if (read_chars < 0) { - perror("Fauled to read line from auth server file\n"); - return -1; - } - // strip possible endln from the getline - if(auth_server_ip[read_chars-1] == '\n'){ - auth_server_ip[read_chars-1] = '\0'; - } - - // read token from incomming file - FILE *token_file = fopen("/responses/Authorization", "r"); - if (token_file == NULL) { - perror("Failed to open file with token"); - return -1; - } - char auth_token[257] = {0}; - if (fscanf(token_file, "Bearer %256s", auth_token) < 0) { - perror("Failed to parse line from token file"); - return -1; - } - - // handle - int auth_sock = connect_to_server(auth_server_ip, AUTH_SERVER_PORT); - if (auth_sock < 0) return 1; - - char auth_request[BUFFER_SIZE]; - int written = snprintf(auth_request, sizeof(auth_request), - "POST /authorize HTTP/1.1\n" - "Host: %s\n" - "Content-Type: application/json\n" - "Content-Length: %zu\n\n" - "{\"token\": \"%s\"}", auth_server_ip, strlen(auth_token) + 13, auth_token); - if(written < 0 || written > BUFFER_SIZE){ - perror("Failed to format auth request"); - return -1; - } - char auth_response[BUFFER_SIZE]; - if (send_http_request(auth_sock, auth_request, written, auth_response, BUFFER_SIZE) < 0) { - close(auth_sock); - return 1; - } - - close(auth_sock); - - char token[256]; - parse_auth_response(auth_response, token); - - // fan_out - log_node *log_root = NULL; - char log_request[SERVER_NUMBER][BUFFER_SIZE]; - int request_length[SERVER_NUMBER]; - char log_response[SERVER_NUMBER][BUFFER_SIZE]; - size_t response_read[SERVER_NUMBER]; - struct pollfd poll_fds[SERVER_NUMBER]; - - int log_sockets[SERVER_NUMBER]; - // open sockets - for(int server = 0; server < SERVER_NUMBER; server++){ - log_sockets[server] = connect_to_server(auth_server_ip, AUTH_SERVER_PORT); - if(log_sockets[server] < 0){ - perror("Failed to open log socket"); - return -1; - } - } - // format requests - for (int server = 0; server < SERVER_NUMBER; server++) { - request_length[server] = snprintf(log_request[server], sizeof(log_request[server]), - "GET /logs/%d HTTP/1.1\n" - "Host: %s\n" - "Authorization: Bearer %s\n\n", server, auth_server_ip, token); - if(request_length < 0){ - perror("Could not format server log request"); - return -1; - } - } - // issue all requests - for (int server = 0; server < SERVER_NUMBER; server++){ - if (linux_send(log_sockets[server], log_request[server], request_length[server], 0) < 0) { - perror("Send failed"); - return -1; - } - response_read[server] = 0; - poll_fds[server].fd = log_sockets[server]; - poll_fds[server].events = POLLIN; - poll_fds[server].revents = 0; - } - unsigned int received_responses = 0; - while(received_responses < SERVER_NUMBER){ - int poll_result = linux_ppoll(poll_fds, SERVER_NUMBER); - if(poll_result < 0) { - perror("Polling failed"); - return -1; - } - // check all the file descriptors for arrived data - for(int server = 0; server < SERVER_NUMBER; server++){ - if(poll_fds[server].fd > 0 && (poll_fds[server].revents & (POLLIN | POLLHUP))){ - received_responses++; - // disable further polling on this socket - poll_fds[server].fd = -1; - int new_data = linux_recv(log_sockets[server], log_response[server] + response_read[server], BUFFER_SIZE - response_read[server] - 1, 0); - if(new_data < 0) { - perror("Failed to read from socket"); - return -1; - } - response_read[server] += new_data; - log_response[server][new_data] = '\0'; - parse_log_events(log_response[server], &log_root); - } - } - } - // close sockets - for(int server = 0; server < SERVER_NUMBER; server++){ - close(log_sockets[server]); - } - - // render - render_logs_to_html(log_root); - - // Free log nodes - log_node *current = log_root; - while (current) { - log_node *next = current->next; - free(current); - current = next; - } - - return 0; -} \ No newline at end of file diff --git a/rust_functions/Cargo.toml b/rust_functions/Cargo.toml deleted file mode 100644 index 4aedf38..0000000 --- a/rust_functions/Cargo.toml +++ /dev/null @@ -1,18 +0,0 @@ -[package] -name = "rust_fass_functions" -version = "0.1.0" -authors = ["mgiardino"] -edition = "2018" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] - -ndarray = { version = "0.15.0", features = ["blas"] } -blas-src = { version = "0.8", features = ["openblas"] } -openblas-src = { version = "0.10", features = ["cblas", "system"] } - -ndarray-linalg = "0.16.0" -ndarray-rand = { version = "0.14.0" } -aes = "0.8.2" -generic-array = "0.14.6" diff --git a/rust_functions/src/aes_func.rs b/rust_functions/src/aes_func.rs deleted file mode 100644 index 1b80693..0000000 --- a/rust_functions/src/aes_func.rs +++ /dev/null @@ -1,27 +0,0 @@ -use aes::Aes128; -use aes::cipher::{ - BlockCipher, BlockEncrypt, BlockDecrypt, KeyInit, -}; -use generic_array::GenericArray; -use generic_array::ArrayLength; - -pub fn aes_encrypt(key: GenericArray>, - block: GenericArray>) - -> GenericArray> { - let cipher = Aes128::new(&key); - - cipher.encrypt_block(&mut block); - - block -} - -pub fn aes_decrypt(key: GenericArray>, - block: GenericArray>) - -> GenericArray> { - let cipher = Aes128::new(&key); - - cipher.decrypt_block(&mut block); - - block -} - diff --git a/rust_functions/src/float_operation.rs b/rust_functions/src/float_operation.rs deleted file mode 100644 index 0dc8ec6..0000000 --- a/rust_functions/src/float_operation.rs +++ /dev/null @@ -1,13 +0,0 @@ -pub fn float_operations(n: i64) { - for i in 0..n { - let i: f64 = i as f64; - let sin_i = i.sin(); - let cos_i = i.cos(); - let sqrt_i = i.sqrt(); - - //println!("{n}: {sin_i} {cos_i} {sqrt_i}"); - println!("{0}: {1} {2} {3}", i, sin_i, cos_i, sqrt_i); - } -} - - diff --git a/rust_functions/src/linpack.rs b/rust_functions/src/linpack.rs deleted file mode 100644 index 0dc6713..0000000 --- a/rust_functions/src/linpack.rs +++ /dev/null @@ -1,7 +0,0 @@ -use ndarray::prelude::*; -use ndarray_linalg::Solve; - -pub fn linpack(a: Array2, b: Array1) -> Array1 { - let c = a.solve_into(b).unwrap(); - c -} diff --git a/rust_functions/src/main.rs b/rust_functions/src/main.rs deleted file mode 100644 index 6f20e30..0000000 --- a/rust_functions/src/main.rs +++ /dev/null @@ -1,47 +0,0 @@ -mod float_operation; -use float_operation::float_operations; -use ndarray::{Array1, Array2}; - -mod matmul; -use matmul::gen_matrix; -use matmul::gen_vector; -use matmul::matmul; - -mod linpack; -use linpack::linpack; - -mod aes_func; -use aes_func::aes_decrypt; -use aes_func::aes_encrypt; -use generic_array::GenericArray; - -extern crate blas_src; - -fn main() { - let i = 5; - println!("Running float operations"); - float_operations(i); - println!("Running matmul"); - let n: usize = 128; - let a = gen_matrix(n, n); - let b = gen_matrix(n, n); - let mut c = Array2::::zeros((n,n)); - c = matmul(a, b, c); - println!("{:?}", c); - - println!("Running linpack"); - let a = gen_matrix(n, n); - let b: Array1 = gen_vector(n); -// let b = arr1(b.slice(s![0..1, ..])); - let c = linpack(a, b); - println!("{:?}", c); -// AES doesn't work yet :( -// println!("Running AES"); -// let key = GenericArray::from([0u8; 16]); -// let mut block = GenericArray::from([42u8; 16]); -// block = aes_encrypt(key, &block); -// println!("{:?}", block); -// block = aes_decrypt(key, &block); -// println!("{:?}", block); -} - diff --git a/rust_functions/src/matmul.rs b/rust_functions/src/matmul.rs deleted file mode 100644 index 2651e03..0000000 --- a/rust_functions/src/matmul.rs +++ /dev/null @@ -1,34 +0,0 @@ -use ndarray::prelude::*; -use ndarray::{Array}; -use ndarray_rand::RandomExt; -use ndarray_rand::rand_distr::Uniform; - -pub fn matmul(a: Array2, b: Array2, mut c: Array2) -> Array2 { - let (n_a, m_a) = a.dim(); - let (n_b, m_b) = b.dim(); - let (n_c, m_c) = c.dim(); - if m_a != n_b { panic!{"matrix mismatch! m_a != n_b"}; } - if m_c != m_b { panic!{"matrix mismatch! m_c != m_b"}; } - if n_c != n_a { panic!{"matrix mismatch! n_c != n_a"}; } - - c = a.dot(&b); - c -} - -fn print_type_of(_: &T) { - println!("{}", std::any::type_name::()) -} - - -pub fn gen_matrix(n: usize, m: usize) -> Array> { - let a = Array::random((n, m), Uniform::new(-0.5, 0.5)); - print_type_of(&a); - // let a = Array::::zeros((n, n).f()); - a -} - -pub fn gen_vector(n: usize) -> Array1 { - let a = Array::random(n, Uniform::new(-0.5, 0.5)); - print_type_of(&a); - a -}