From b09bbcdd0815b64abfa814585d24ff7115422597 Mon Sep 17 00:00:00 2001 From: theMackabu Date: Fri, 1 May 2026 15:01:51 -0700 Subject: [PATCH] add clang-format target --- .clang-format | 9 + demos/async/client.c | 3 +- demos/async/server.c | 33 +- demos/bench.c | 69 +- demos/client.c | 13 +- demos/server.c | 24 +- include/arena.h | 18 +- include/backend.h | 9 +- include/minicoro.h | 1828 +++++++++++++++++++------------------- include/routes.h | 8 +- include/rpc/client.h | 12 +- include/rpc/protocol.h | 3 +- include/rpc/server.h | 9 +- include/rpc/trace.h | 33 +- include/scheduler.h | 6 +- meson.build | 1 + src/backend/kqueue.c | 63 +- src/client.c | 124 +-- src/payload.c | 110 +-- src/protocol.c | 20 +- src/routes.c | 60 +- src/scheduler.c | 80 +- src/server.c | 341 ++----- src/trace.c | 210 ++--- tests/test_integration.c | 7 +- tests/test_routes.c | 6 +- tests/test_scheduler.c | 6 +- 27 files changed, 1321 insertions(+), 1784 deletions(-) create mode 100644 .clang-format diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..a94263d --- /dev/null +++ b/.clang-format @@ -0,0 +1,9 @@ +BasedOnStyle: LLVM +IndentWidth: 2 +ColumnLimit: 120 +PointerAlignment: Right +AlignAfterOpenBracket: Align +AllowShortFunctionsOnASingleLine: Empty +AllowShortIfStatementsOnASingleLine: AllIfsAndElse +AllowShortBlocksOnASingleLine: Always +SortIncludes: true diff --git a/demos/async/client.c b/demos/async/client.c index 01a6a4e..3b77070 100644 --- a/demos/async/client.c +++ b/demos/async/client.c @@ -30,8 +30,7 @@ int main(int argc, char **argv) { } if (result_count == 1 && result[0].type == RPC_TYPE_I64) { - printf("async %lld + %lld = %lld\n", (long long)a, (long long)b, - (long long)result[0].as.i64); + printf("async %lld + %lld = %lld\n", (long long)a, (long long)b, (long long)result[0].as.i64); } else { fprintf(stderr, "unexpected response\n"); } diff --git a/demos/async/server.c b/demos/async/server.c index edf9d83..3d8fbe0 100644 --- a/demos/async/server.c +++ b/demos/async/server.c @@ -49,9 +49,7 @@ static async_job *queue_pop(async_queue *queue) { async_job *job = queue->head; if (job) { queue->head = job->next; - if (!queue->head) { - queue->tail = NULL; - } + if (!queue->head) { queue->tail = NULL; } } pthread_mutex_unlock(&queue->mutex); return job; @@ -61,9 +59,7 @@ static void *queue_worker(void *arg) { async_queue *queue = arg; for (;;) { async_job *job = queue_pop(queue); - if (!job) { - return NULL; - } + if (!job) { return NULL; } struct timespec delay = { .tv_sec = 0, @@ -77,9 +73,7 @@ static void *queue_worker(void *arg) { static int queue_start(async_queue *queue) { memset(queue, 0, sizeof(*queue)); - if (pthread_mutex_init(&queue->mutex, NULL) != 0) { - return -1; - } + if (pthread_mutex_init(&queue->mutex, NULL) != 0) { return -1; } if (pthread_cond_init(&queue->cond, NULL) != 0) { pthread_mutex_destroy(&queue->mutex); return -1; @@ -104,18 +98,12 @@ static void queue_stop(async_queue *queue) { static void on_signal(int signo) { (void)signo; - if (g_server) { - rpc_server_stop(g_server); - } + if (g_server) { rpc_server_stop(g_server); } } -static int async_add(rpc_ctx *ctx, const rpc_value *args, size_t argc, - rpc_writer *out, void *user_data) { +static int async_add(rpc_ctx *ctx, const rpc_value *args, size_t argc, rpc_writer *out, void *user_data) { async_queue *queue = user_data; - if (argc != 2 || args[0].type != RPC_TYPE_I64 || - args[1].type != RPC_TYPE_I64) { - return -1; - } + if (argc != 2 || args[0].type != RPC_TYPE_I64 || args[1].type != RPC_TYPE_I64) { return -1; } async_job job; memset(&job, 0, sizeof(job)); @@ -141,10 +129,8 @@ int main(int argc, char **argv) { return 1; } - if (rpc_server_init(&g_server) != 0 || - rpc_server_set_workers(g_server, workers) != 0 || - rpc_server_add_async_route(g_server, 3, async_add, &g_queue) != 0 || - rpc_server_bind(g_server, host, port) != 0 || + if (rpc_server_init(&g_server) != 0 || rpc_server_set_workers(g_server, workers) != 0 || + rpc_server_add_async_route(g_server, 3, async_add, &g_queue) != 0 || rpc_server_bind(g_server, host, port) != 0 || rpc_server_listen(g_server) != 0) { fprintf(stderr, "failed to start async RPC server\n"); rpc_server_destroy(g_server); @@ -154,8 +140,7 @@ int main(int argc, char **argv) { signal(SIGINT, on_signal); signal(SIGTERM, on_signal); - printf("async rpc demo server listening on %s:%u\n", host, - rpc_server_port(g_server)); + printf("async rpc demo server listening on %s:%u\n", host, rpc_server_port(g_server)); int rc = rpc_server_run(g_server); rpc_server_destroy(g_server); diff --git a/demos/bench.c b/demos/bench.c index cc21e1d..6573f0f 100644 --- a/demos/bench.c +++ b/demos/bench.c @@ -50,13 +50,10 @@ static uint64_t now_ns(void) { return (uint64_t)ts.tv_sec * 1000000000ull + (uint64_t)ts.tv_nsec; } -static int add_handler(rpc_ctx *ctx, const rpc_value *args, size_t argc, - rpc_writer *out, void *user_data) { +static int add_handler(rpc_ctx *ctx, const rpc_value *args, size_t argc, rpc_writer *out, void *user_data) { (void)ctx; (void)user_data; - if (argc != 2 || args[0].type != RPC_TYPE_I64 || args[1].type != RPC_TYPE_I64) { - return -1; - } + if (argc != 2 || args[0].type != RPC_TYPE_I64 || args[1].type != RPC_TYPE_I64) { return -1; } return rpc_writer_i64(out, args[0].as.i64 + args[1].as.i64); } @@ -68,11 +65,9 @@ static void *server_main(void *arg) { static int start_server(bench_server *server, char port[16]) { if (rpc_server_init(&server->rpc) != 0 || - (server->workers != 0 && - rpc_server_set_workers(server->rpc, server->workers) != 0) || + (server->workers != 0 && rpc_server_set_workers(server->rpc, server->workers) != 0) || rpc_server_add_route(server->rpc, 1, add_handler, NULL) != 0 || - rpc_server_bind(server->rpc, "127.0.0.1", "0") != 0 || - rpc_server_listen(server->rpc) != 0) { + rpc_server_bind(server->rpc, "127.0.0.1", "0") != 0 || rpc_server_listen(server->rpc) != 0) { return -1; } @@ -86,9 +81,7 @@ static int start_server(bench_server *server, char port[16]) { } static void stop_server(bench_server *server) { - if (!server->rpc) { - return; - } + if (!server->rpc) { return; } rpc_server_stop(server->rpc); pthread_join(server->thread, NULL); rpc_server_destroy(server->rpc); @@ -99,10 +92,7 @@ static int run_one_call(rpc_client *client, rpc_writer *payload, int64_t expecte rpc_value *values = NULL; size_t count = 0; int rc = rpc_client_call(client, 1, payload, &values, &count); - if (rc == 0 && - (count != 1 || values[0].type != RPC_TYPE_I64 || values[0].as.i64 != expected)) { - rc = -1; - } + if (rc == 0 && (count != 1 || values[0].type != RPC_TYPE_I64 || values[0].as.i64 != expected)) { rc = -1; } rpc_values_free(values); return rc; } @@ -181,8 +171,7 @@ static void *worker_main(void *arg) { uint64_t call_id = 0; rpc_value *values = NULL; size_t count = 0; - if (rpc_client_recv_response(client, &call_id, &values, &count) != 0 || - !validate_response(values, count, 42)) { + if (rpc_client_recv_response(client, &call_id, &values, &count) != 0 || !validate_response(values, count, 42)) { rpc_values_free(values); worker->failed = 1; goto pipeline_done; @@ -194,8 +183,7 @@ static void *worker_main(void *arg) { worker->failed = 1; goto pipeline_done; } - worker->latencies_ns[worker->latency_offset + received] = - now_ns() - starts[slot]; + worker->latencies_ns[worker->latency_offset + received] = now_ns() - starts[slot]; rpc_values_free(values); received++; @@ -218,9 +206,7 @@ pipeline_done: free(starts); done: - if (!announced_gate) { - (void)gate_ready_and_wait(worker->gate); - } + if (!announced_gate) { (void)gate_ready_and_wait(worker->gate); } rpc_writer_free(&payload); rpc_client_close(client); return NULL; @@ -245,36 +231,26 @@ static void format_duration(uint64_t ns, char out[16]) { } static uint64_t percentile(const uint64_t *values, uint64_t count, double pct) { - if (count == 0) { - return 0; - } + if (count == 0) { return 0; } uint64_t idx = (uint64_t)((pct / 100.0) * (double)(count - 1)); return values[idx]; } static uint64_t parse_u64_arg(const char *text, uint64_t fallback) { - if (!text || !*text) { - return fallback; - } + if (!text || !*text) { return fallback; } char *end = NULL; errno = 0; unsigned long long value = strtoull(text, &end, 10); - if (errno != 0 || !end || *end != '\0' || value == 0) { - return fallback; - } + if (errno != 0 || !end || *end != '\0' || value == 0) { return fallback; } return (uint64_t)value; } static int parse_bool_arg(const char *text, int fallback) { - if (!text || !*text) { - return fallback; - } - if (strcmp(text, "1") == 0 || strcmp(text, "true") == 0 || - strcmp(text, "on") == 0 || strcmp(text, "yes") == 0) { + if (!text || !*text) { return fallback; } + if (strcmp(text, "1") == 0 || strcmp(text, "true") == 0 || strcmp(text, "on") == 0 || strcmp(text, "yes") == 0) { return 1; } - if (strcmp(text, "0") == 0 || strcmp(text, "false") == 0 || - strcmp(text, "off") == 0 || strcmp(text, "no") == 0) { + if (strcmp(text, "0") == 0 || strcmp(text, "false") == 0 || strcmp(text, "off") == 0 || strcmp(text, "no") == 0) { return 0; } return fallback; @@ -342,9 +318,7 @@ int main(int argc, char **argv) { uint64_t server_workers = argc > 4 ? parse_u64_arg(argv[4], 0) : 0; uint64_t pipeline = argc > 5 ? parse_u64_arg(argv[5], 1) : 1; int trace_enabled = argc > 6 ? parse_bool_arg(argv[6], 0) : 0; - if (clients > total_requests) { - clients = total_requests; - } + if (clients > total_requests) { clients = total_requests; } uint64_t *latencies = calloc(total_requests, sizeof(*latencies)); pthread_t *threads = calloc(clients, sizeof(*threads)); @@ -424,9 +398,7 @@ int main(int argc, char **argv) { } for (uint64_t i = 0; i < clients; ++i) { - if (workers[i].started) { - pthread_join(threads[i], NULL); - } + if (workers[i].started) { pthread_join(threads[i], NULL); } failed |= workers[i].failed; } rpc_trace_set_enabled(0); @@ -465,8 +437,7 @@ int main(int argc, char **argv) { printf("requests: %" PRIu64 "\n", total_requests); printf("clients: %" PRIu64 "\n", clients); printf("warmup: %" PRIu64 "\n", warmup_total); - printf("server workers: %" PRIu64 "%s\n", server_workers, - server_workers == 0 ? " (auto)" : ""); + printf("server workers: %" PRIu64 "%s\n", server_workers, server_workers == 0 ? " (auto)" : ""); printf("pipeline depth: %" PRIu64 "\n", pipeline); printf("trace: %s\n", trace_enabled ? "on" : "off"); printf("elapsed: %s\n", elapsed_buf); @@ -476,9 +447,7 @@ int main(int argc, char **argv) { printf("latency p95: %s\n", p95_buf); printf("latency p99: %s\n", p99_buf); printf("latency max: %s\n", max_buf); - if (trace_enabled) { - rpc_trace_dump(stdout); - } + if (trace_enabled) { rpc_trace_dump(stdout); } pthread_cond_destroy(&gate.cond); pthread_mutex_destroy(&gate.mutex); diff --git a/demos/client.c b/demos/client.c index b989f79..41900dc 100644 --- a/demos/client.c +++ b/demos/client.c @@ -8,7 +8,7 @@ typedef long long longer; int main(int argc, char **argv) { const char *host = argc > 1 ? argv[1] : "127.0.0.1"; const char *port = argc > 2 ? argv[2] : "7000"; - + int64_t a = argc > 3 ? strtoll(argv[3], NULL, 10) : 20; int64_t b = argc > 4 ? strtoll(argv[4], NULL, 10) : 22; @@ -25,7 +25,7 @@ int main(int argc, char **argv) { rpc_value *result = NULL; size_t result_count = 0; - + if (rpc_client_call(client, 1, &args, &result, &result_count) != 0) { fprintf(stderr, "RPC error: %s\n", rpc_client_error(client)); rpc_writer_free(&args); @@ -33,14 +33,13 @@ int main(int argc, char **argv) { return 1; } - if (result_count == 1 && result[0].type == RPC_TYPE_I64) printf( - "%lld + %lld = %lld\n", - (longer)a, (longer)b, (longer)result[0].as.i64 - ); else fprintf(stderr, "unexpected response\n"); + if (result_count == 1 && result[0].type == RPC_TYPE_I64) + printf("%lld + %lld = %lld\n", (longer)a, (longer)b, (longer)result[0].as.i64); + else fprintf(stderr, "unexpected response\n"); rpc_values_free(result); rpc_writer_free(&args); rpc_client_close(client); - + return 0; } diff --git a/demos/server.c b/demos/server.c index 665fca9..318b578 100644 --- a/demos/server.c +++ b/demos/server.c @@ -9,27 +9,19 @@ static rpc_server *g_server; static void on_signal(int signo) { (void)signo; - if (g_server) { - rpc_server_stop(g_server); - } + if (g_server) { rpc_server_stop(g_server); } } -static int add_i64(rpc_ctx *ctx, const rpc_value *args, size_t argc, - rpc_writer *out, void *user_data) { +static int add_i64(rpc_ctx *ctx, const rpc_value *args, size_t argc, rpc_writer *out, void *user_data) { (void)ctx; (void)user_data; - if (argc != 2 || args[0].type != RPC_TYPE_I64 || args[1].type != RPC_TYPE_I64) { - return -1; - } + if (argc != 2 || args[0].type != RPC_TYPE_I64 || args[1].type != RPC_TYPE_I64) { return -1; } return rpc_writer_i64(out, args[0].as.i64 + args[1].as.i64); } -static int echo_string(rpc_ctx *ctx, const rpc_value *args, size_t argc, - rpc_writer *out, void *user_data) { +static int echo_string(rpc_ctx *ctx, const rpc_value *args, size_t argc, rpc_writer *out, void *user_data) { (void)user_data; - if (argc != 1 || args[0].type != RPC_TYPE_STRING) { - return -1; - } + if (argc != 1 || args[0].type != RPC_TYPE_STRING) { return -1; } rpc_ctx_yield(ctx); return rpc_writer_string(out, args[0].as.string.data, args[0].as.string.len); } @@ -39,11 +31,9 @@ int main(int argc, char **argv) { const char *port = argc > 2 ? argv[2] : "7000"; uint32_t workers = argc > 3 ? (uint32_t)strtoul(argv[3], NULL, 10) : 0; - if (rpc_server_init(&g_server) != 0 || - (workers != 0 && rpc_server_set_workers(g_server, workers) != 0) || + if (rpc_server_init(&g_server) != 0 || (workers != 0 && rpc_server_set_workers(g_server, workers) != 0) || rpc_server_add_route(g_server, 1, add_i64, NULL) != 0 || - rpc_server_add_async_route(g_server, 2, echo_string, NULL) != 0 || - rpc_server_bind(g_server, host, port) != 0 || + rpc_server_add_async_route(g_server, 2, echo_string, NULL) != 0 || rpc_server_bind(g_server, host, port) != 0 || rpc_server_listen(g_server) != 0) { fprintf(stderr, "failed to start RPC server\n"); rpc_server_destroy(g_server); diff --git a/include/arena.h b/include/arena.h index 810a0c2..54e93a2 100644 --- a/include/arena.h +++ b/include/arena.h @@ -20,14 +20,12 @@ static inline size_t rpc_arena_align(size_t n) { return (n + a - 1u) & ~(a - 1u); } -static inline int rpc_fixed_arena_init(rpc_fixed_arena *arena, size_t elem_size, - size_t capacity) { +static inline int rpc_fixed_arena_init(rpc_fixed_arena *arena, size_t elem_size, size_t capacity) { memset(arena, 0, sizeof(*arena)); arena->elem_size = rpc_arena_align(elem_size < sizeof(void *) ? sizeof(void *) : elem_size); arena->capacity = capacity; arena->bytes = arena->elem_size * capacity; - arena->base = mmap(NULL, arena->bytes, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANON, -1, 0); + arena->base = mmap(NULL, arena->bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); if (arena->base == MAP_FAILED) { memset(arena, 0, sizeof(*arena)); return -1; @@ -36,9 +34,7 @@ static inline int rpc_fixed_arena_init(rpc_fixed_arena *arena, size_t elem_size, } static inline void rpc_fixed_arena_destroy(rpc_fixed_arena *arena) { - if (arena->base) { - munmap(arena->base, arena->bytes); - } + if (arena->base) { munmap(arena->base, arena->bytes); } memset(arena, 0, sizeof(*arena)); } @@ -50,16 +46,12 @@ static inline void *rpc_fixed_arena_alloc(rpc_fixed_arena *arena) { } else if (arena->watermark < arena->capacity) { p = arena->base + arena->watermark++ * arena->elem_size; } - if (p) { - memset(p, 0, arena->elem_size); - } + if (p) { memset(p, 0, arena->elem_size); } return p; } static inline void rpc_fixed_arena_free(rpc_fixed_arena *arena, void *p) { - if (!p) { - return; - } + if (!p) { return; } *(void **)p = arena->free_list; arena->free_list = p; } diff --git a/include/backend.h b/include/backend.h index 66f2183..a866502 100644 --- a/include/backend.h +++ b/include/backend.h @@ -19,13 +19,10 @@ typedef struct rpc_backend rpc_backend; int rpc_backend_kqueue_create(rpc_backend **out); void rpc_backend_destroy(rpc_backend *backend); -int rpc_backend_register(rpc_backend *backend, int fd, uint32_t events, - uintptr_t user); -int rpc_backend_modify(rpc_backend *backend, int fd, uint32_t events, - uintptr_t user); +int rpc_backend_register(rpc_backend *backend, int fd, uint32_t events, uintptr_t user); +int rpc_backend_modify(rpc_backend *backend, int fd, uint32_t events, uintptr_t user); int rpc_backend_remove(rpc_backend *backend, int fd); int rpc_backend_wake(rpc_backend *backend); -int rpc_backend_poll(rpc_backend *backend, rpc_backend_event *events, int max_events, - int timeout_ms); +int rpc_backend_poll(rpc_backend *backend, rpc_backend_event *events, int max_events, int timeout_ms); #endif diff --git a/include/minicoro.h b/include/minicoro.h index bb691a7..17fc10f 100644 --- a/include/minicoro.h +++ b/include/minicoro.h @@ -14,7 +14,8 @@ The API is inspired by Lua coroutines but with C use in mind. - Supports custom allocators. - Storage system to allow passing values between yield and resume. - Customizable stack size. -- Supports growable stacks and low memory footprint when enabling the virtual memory allocator. +- Supports growable stacks and low memory footprint when enabling the virtual memory +allocator. - Coroutine API design inspired by Lua with C use in mind. - Yield across any C function. - Made to work in multithread applications. @@ -50,14 +51,22 @@ to create, resume, yield or destroy a coroutine. # Caveats -- Avoid using coroutines with C++ exceptions, this is not recommended, it may not behave as you expect. -- When using C++ RAII (i.e. destructors) you must resume the coroutine until it dies to properly execute all destructors. +- Avoid using coroutines with C++ exceptions, this is not recommended, it may not behave +as you expect. +- When using C++ RAII (i.e. destructors) you must resume the coroutine until it dies to +properly execute all destructors. - Some unsupported sanitizers for C may trigger false warnings when using coroutines. -- The `mco_coro` object is not thread safe, you should use a mutex for manipulating it in multithread applications. -- To use in multithread applications, you must compile with C compiler that supports `thread_local` qualifier. -- Avoid using `thread_local` inside coroutine code, the compiler may cache thread local variables pointers which can be invalid when a coroutine switch threads. -- Stack space is limited. By default it has 56KB of space, this can be changed on coroutine creation, or by enabling the virtual memory backed allocator to make it 2040KB. -- Take care to not cause stack overflows (run out of stack space), otherwise your program may crash or not, the behavior is undefined. +- The `mco_coro` object is not thread safe, you should use a mutex for manipulating it +in multithread applications. +- To use in multithread applications, you must compile with C compiler that supports +`thread_local` qualifier. +- Avoid using `thread_local` inside coroutine code, the compiler may cache thread local +variables pointers which can be invalid when a coroutine switch threads. +- Stack space is limited. By default it has 56KB of space, this can be changed on +coroutine creation, or by enabling the virtual memory backed allocator to make it +2040KB. +- Take care to not cause stack overflows (run out of stack space), otherwise your +program may crash or not, the behavior is undefined. - On WebAssembly you must compile with Emscripten flag `-s ASYNCIFY=1`. - The WebAssembly Binaryen asyncify method can be used when explicitly enabled, you may want to do this only to use minicoro with WebAssembly native interpreters @@ -72,24 +81,28 @@ a coroutine only suspends its execution by explicitly calling a yield function. You create a coroutine by calling `mco_create`. Its sole argument is a `mco_desc` structure with a description for the coroutine. -The `mco_create` function only creates a new coroutine and returns a handle to it, it does not start the coroutine. +The `mco_create` function only creates a new coroutine and returns a handle to it, it +does not start the coroutine. You execute a coroutine by calling `mco_resume`. -When calling a resume function the coroutine starts its execution by calling its body function. -After the coroutine starts running, it runs until it terminates or yields. +When calling a resume function the coroutine starts its execution by calling its body +function. After the coroutine starts running, it runs until it terminates or yields. A coroutine yields by calling `mco_yield`. When a coroutine yields, the corresponding resume returns immediately, -even if the yield happens inside nested function calls (that is, not in the main function). -The next time you resume the same coroutine, it continues its execution from the point where it yielded. +even if the yield happens inside nested function calls (that is, not in the main +function). The next time you resume the same coroutine, it continues its execution from +the point where it yielded. To associate a persistent value with the coroutine, -you can optionally set `user_data` on its creation and later retrieve with `mco_get_user_data`. +you can optionally set `user_data` on its creation and later retrieve with +`mco_get_user_data`. To pass values between resume and yield, you can optionally use `mco_push` and `mco_pop` APIs, they are intended to pass temporary values using a LIFO style buffer. -The storage system can also be used to send and receive initial values on coroutine creation or before it finishes. +The storage system can also be used to send and receive initial values on coroutine +creation or before it finishes. # Usage @@ -100,7 +113,8 @@ To use minicoro, do the following in one .c file: #include "minicoro.h" ``` -You can do `#include "minicoro.h"` in other parts of the program just like any other header. +You can do `#include "minicoro.h"` in other parts of the program just like any other +header. ## Minimal Example @@ -122,8 +136,8 @@ void coro_entry(mco_coro* co) { int main() { // First initialize a `desc` object through `mco_desc_init`. mco_desc desc = mco_desc_init(coro_entry, 0); - // Configure `desc` fields when needed (e.g. customize user_data or allocation functions). - desc.user_data = NULL; + // Configure `desc` fields when needed (e.g. customize user_data or allocation +functions). desc.user_data = NULL; // Call `mco_create` with the output coroutine pointer and `desc` pointer. mco_coro* co; mco_result res = mco_create(&co, &desc); @@ -173,38 +187,41 @@ the user is encouraged to handle them properly. ## Virtual memory backed allocator -The new compile time option `MCO_USE_VMEM_ALLOCATOR` enables a virtual memory backed allocator. +The new compile time option `MCO_USE_VMEM_ALLOCATOR` enables a virtual memory backed +allocator. Every stackful coroutine usually have to reserve memory for its full stack, -this typically makes the total memory usage very high when allocating thousands of coroutines, -for example, an application with 100 thousands coroutine with stacks of 56KB would consume as high -as 5GB of memory, however your application may not really full stack usage for every coroutine. +this typically makes the total memory usage very high when allocating thousands of +coroutines, for example, an application with 100 thousands coroutine with stacks of 56KB +would consume as high as 5GB of memory, however your application may not really full +stack usage for every coroutine. Some developers often prefer stackless coroutines over stackful coroutines -because of this problem, stackless memory footprint is low, therefore often considered more lightweight. -However stackless have many other limitations, like you cannot run unconstrained code inside them. +because of this problem, stackless memory footprint is low, therefore often considered +more lightweight. However stackless have many other limitations, like you cannot run +unconstrained code inside them. One remedy to the solution is to make stackful coroutines growable, to only use physical memory on demand when its really needed, and there is a nice way to do this relying on virtual memory allocation when supported by the operating system. -The virtual memory backed allocator will reserve virtual memory in the OS for each coroutine stack, -but not trigger real physical memory usage yet. -While the application virtual memory usage will be high, -the physical memory usage will be low and actually grow on demand (usually every 4KB chunk in Linux). +The virtual memory backed allocator will reserve virtual memory in the OS for each +coroutine stack, but not trigger real physical memory usage yet. While the application +virtual memory usage will be high, the physical memory usage will be low and actually +grow on demand (usually every 4KB chunk in Linux). The virtual memory backed allocator also raises the default stack size to about 2MB, typically the size of extra threads in Linux, so you have more space in your coroutines and the risk of stack overflow is low. As an example, allocating 100 thousands coroutines with nearly 2MB stack reserved space -with the virtual memory allocator uses 783MB of physical memory usage, that is about 8KB per coroutine, -however the virtual memory usage will be at 98GB. +with the virtual memory allocator uses 783MB of physical memory usage, that is about 8KB +per coroutine, however the virtual memory usage will be at 98GB. -It is recommended to enable this option only if you plan to spawn thousands of coroutines -while wanting to have a low memory footprint. -Not all environments have an OS with virtual memory support, therefore this option is disabled by default. +It is recommended to enable this option only if you plan to spawn thousands of +coroutines while wanting to have a low memory footprint. Not all environments have an OS +with virtual memory support, therefore this option is disabled by default. This option may add an order of magnitude overhead to `mco_create()`/`mco_destroy()`, because they will request the OS to manage virtual memory page tables, @@ -215,29 +232,37 @@ if this is a problem for you, please customize a custom allocator for your own n The following can be defined to change the library behavior: - `MCO_API` - Public API qualifier. Default is `extern`. -- `MCO_MIN_STACK_SIZE` - Minimum stack size when creating a coroutine. Default is 32768 (32KB). +- `MCO_MIN_STACK_SIZE` - Minimum stack size when creating a coroutine. Default is +32768 (32KB). - `MCO_DEFAULT_STORAGE_SIZE` - Size of coroutine storage buffer. Default is 1024. -- `MCO_DEFAULT_STACK_SIZE` - Default stack size when creating a coroutine. Default is 57344 (56KB). When `MCO_USE_VMEM_ALLOCATOR` is true the default is 2040KB (nearly 2MB). +- `MCO_DEFAULT_STACK_SIZE` - Default stack size when creating a coroutine. Default is +57344 (56KB). When `MCO_USE_VMEM_ALLOCATOR` is true the default is 2040KB (nearly 2MB). - `MCO_ALLOC` - Default allocation function. Default is `calloc`. - `MCO_DEALLOC` - Default deallocation function. Default is `free`. -- `MCO_USE_VMEM_ALLOCATOR` - Use virtual memory backed allocator, improving memory footprint per coroutine. -- `MCO_NO_DEFAULT_ALLOCATOR` - Disable the default allocator using `MCO_ALLOC` and `MCO_DEALLOC`. -- `MCO_ZERO_MEMORY` - Zero memory of stack when poping storage, intended for garbage collected environments. -- `MCO_DEBUG` - Enable debug mode, logging any runtime error to stdout. Defined automatically unless `NDEBUG` or `MCO_NO_DEBUG` is defined. +- `MCO_USE_VMEM_ALLOCATOR` - Use virtual memory backed allocator, improving memory +footprint per coroutine. +- `MCO_NO_DEFAULT_ALLOCATOR` - Disable the default allocator using `MCO_ALLOC` and +`MCO_DEALLOC`. +- `MCO_ZERO_MEMORY` - Zero memory of stack when poping storage, intended for +garbage collected environments. +- `MCO_DEBUG` - Enable debug mode, logging any runtime error to stdout. +Defined automatically unless `NDEBUG` or `MCO_NO_DEBUG` is defined. - `MCO_NO_DEBUG` - Disable debug mode. -- `MCO_NO_MULTITHREAD` - Disable multithread usage. Multithread is supported when `thread_local` is supported. +- `MCO_NO_MULTITHREAD` - Disable multithread usage. Multithread is supported when +`thread_local` is supported. - `MCO_USE_ASM` - Force use of assembly context switch implementation. - `MCO_USE_UCONTEXT` - Force use of ucontext context switch implementation. - `MCO_USE_FIBERS` - Force use of fibers context switch implementation. -- `MCO_USE_ASYNCIFY` - Force use of Binaryen asyncify context switch implementation. -- `MCO_USE_VALGRIND` - Define if you want run with valgrind to fix accessing memory errors. +- `MCO_USE_ASYNCIFY` - Force use of Binaryen asyncify context switch +implementation. +- `MCO_USE_VALGRIND` - Define if you want run with valgrind to fix accessing +memory errors. # License Your choice of either Public Domain or MIT No Attribution, see end of file. */ - #ifndef MINICORO_H #define MINICORO_H @@ -257,14 +282,18 @@ extern "C" { #include /* for size_t */ -/* ---------------------------------------------------------------------------------------------- */ +/* ---------------------------------------------------------------------------------------------- + */ /* Coroutine states. */ typedef enum mco_state { - MCO_DEAD = 0, /* The coroutine has finished normally or was uninitialized before finishing. */ - MCO_NORMAL, /* The coroutine is active but not running (that is, it has resumed another coroutine). */ - MCO_RUNNING, /* The coroutine is active and running. */ - MCO_SUSPENDED /* The coroutine is suspended (in a call to yield, or it has not started running yet). */ + MCO_DEAD = 0, /* The coroutine has finished normally or was uninitialized before + finishing. */ + MCO_NORMAL, /* The coroutine is active but not running (that is, it has resumed + another coroutine). */ + MCO_RUNNING, /* The coroutine is active and running. */ + MCO_SUSPENDED /* The coroutine is suspended (in a call to yield, or it has not started + running yet). */ } mco_state; /* Coroutine result codes. */ @@ -287,60 +316,69 @@ typedef enum mco_result { /* Coroutine structure. */ typedef struct mco_coro mco_coro; struct mco_coro { - void* context; + void *context; mco_state state; - void (*func)(mco_coro* co); - mco_coro* prev_co; - void* user_data; + void (*func)(mco_coro *co); + mco_coro *prev_co; + void *user_data; size_t coro_size; - void* allocator_data; - void (*dealloc_cb)(void* ptr, size_t size, void* allocator_data); - void* stack_base; /* Stack base address, can be used to scan memory in a garbage collector. */ + void *allocator_data; + void (*dealloc_cb)(void *ptr, size_t size, void *allocator_data); + void *stack_base; /* Stack base address, can be used to scan memory in a garbage + collector. */ size_t stack_size; - unsigned char* storage; + unsigned char *storage; size_t bytes_stored; size_t storage_size; - void* asan_prev_stack; /* Used by address sanitizer. */ - void* tsan_prev_fiber; /* Used by thread sanitizer. */ - void* tsan_fiber; /* Used by thread sanitizer. */ - size_t magic_number; /* Used to check stack overflow. */ + void *asan_prev_stack; /* Used by address sanitizer. */ + void *tsan_prev_fiber; /* Used by thread sanitizer. */ + void *tsan_fiber; /* Used by thread sanitizer. */ + size_t magic_number; /* Used to check stack overflow. */ }; /* Structure used to initialize a coroutine. */ typedef struct mco_desc { - void (*func)(mco_coro* co); /* Entry point function for the coroutine. */ - void* user_data; /* Coroutine user data, can be get with `mco_get_user_data`. */ + void (*func)(mco_coro *co); /* Entry point function for the coroutine. */ + void *user_data; /* Coroutine user data, can be get with `mco_get_user_data`. */ /* Custom allocation interface. */ - void* (*alloc_cb)(size_t size, void* allocator_data); /* Custom allocation function. */ - void (*dealloc_cb)(void* ptr, size_t size, void* allocator_data); /* Custom deallocation function. */ - void* allocator_data; /* User data pointer passed to `alloc`/`dealloc` allocation functions. */ - size_t storage_size; /* Coroutine storage size, to be used with the storage APIs. */ + void *(*alloc_cb)(size_t size, void *allocator_data); /* Custom allocation function. */ + void (*dealloc_cb)(void *ptr, size_t size, void *allocator_data); /* Custom deallocation function. */ + void *allocator_data; /* User data pointer passed to `alloc`/`dealloc` allocation + functions. */ + size_t storage_size; /* Coroutine storage size, to be used with the storage APIs. */ /* These must be initialized only through `mco_init_desc`. */ - size_t coro_size; /* Coroutine structure size. */ - size_t stack_size; /* Coroutine stack size. */ + size_t coro_size; /* Coroutine structure size. */ + size_t stack_size; /* Coroutine stack size. */ } mco_desc; /* Coroutine functions. */ -MCO_API mco_desc mco_desc_init(void (*func)(mco_coro* co), size_t stack_size); /* Initialize description of a coroutine. When stack size is 0 then MCO_DEFAULT_STACK_SIZE is used. */ -MCO_API mco_result mco_init(mco_coro* co, mco_desc* desc); /* Initialize the coroutine. */ -MCO_API mco_result mco_uninit(mco_coro* co); /* Uninitialize the coroutine, may fail if it's not dead or suspended. */ -MCO_API mco_result mco_create(mco_coro** out_co, mco_desc* desc); /* Allocates and initializes a new coroutine. */ -MCO_API mco_result mco_destroy(mco_coro* co); /* Uninitialize and deallocate the coroutine, may fail if it's not dead or suspended. */ -MCO_API mco_result mco_resume(mco_coro* co); /* Starts or continues the execution of the coroutine. */ -MCO_API mco_result mco_yield(mco_coro* co); /* Suspends the execution of a coroutine. */ -MCO_API mco_state mco_status(mco_coro* co); /* Returns the status of the coroutine. */ -MCO_API void* mco_get_user_data(mco_coro* co); /* Get coroutine user data supplied on coroutine creation. */ +MCO_API mco_desc mco_desc_init(void (*func)(mco_coro *co), + size_t stack_size); /* Initialize description of a coroutine. When stack + size is 0 then MCO_DEFAULT_STACK_SIZE is used. */ +MCO_API mco_result mco_init(mco_coro *co, mco_desc *desc); /* Initialize the coroutine. */ +MCO_API mco_result mco_uninit(mco_coro *co); /* Uninitialize the coroutine, may fail if + it's not dead or suspended. */ +MCO_API mco_result mco_create(mco_coro **out_co, mco_desc *desc); /* Allocates and initializes a new coroutine. */ +MCO_API mco_result mco_destroy(mco_coro *co); /* Uninitialize and deallocate the coroutine, may fail if + it's not dead or suspended. */ +MCO_API mco_result mco_resume(mco_coro *co); /* Starts or continues the execution of the coroutine. */ +MCO_API mco_result mco_yield(mco_coro *co); /* Suspends the execution of a coroutine. */ +MCO_API mco_state mco_status(mco_coro *co); /* Returns the status of the coroutine. */ +MCO_API void *mco_get_user_data(mco_coro *co); /* Get coroutine user data supplied on coroutine creation. */ /* Storage interface functions, used to pass values between yield and resume. */ -MCO_API mco_result mco_push(mco_coro* co, const void* src, size_t len); /* Push bytes to the coroutine storage. Use to send values between yield and resume. */ -MCO_API mco_result mco_pop(mco_coro* co, void* dest, size_t len); /* Pop bytes from the coroutine storage. Use to get values between yield and resume. */ -MCO_API mco_result mco_peek(mco_coro* co, void* dest, size_t len); /* Like `mco_pop` but it does not consumes the storage. */ -MCO_API size_t mco_get_bytes_stored(mco_coro* co); /* Get the available bytes that can be retrieved with a `mco_pop`. */ -MCO_API size_t mco_get_storage_size(mco_coro* co); /* Get the total storage size. */ +MCO_API mco_result mco_push(mco_coro *co, const void *src, size_t len); /* Push bytes to the coroutine storage. Use to + send values between yield and resume. */ +MCO_API mco_result mco_pop(mco_coro *co, void *dest, size_t len); /* Pop bytes from the coroutine storage. Use to + get values between yield and resume. */ +MCO_API mco_result mco_peek(mco_coro *co, void *dest, + size_t len); /* Like `mco_pop` but it does not consumes the storage. */ +MCO_API size_t mco_get_bytes_stored(mco_coro *co); /* Get the available bytes that can be retrieved with a `mco_pop`. */ +MCO_API size_t mco_get_storage_size(mco_coro *co); /* Get the total storage size. */ /* Misc functions. */ -MCO_API mco_coro* mco_running(void); /* Returns the running coroutine for the current thread. */ -MCO_API const char* mco_result_description(mco_result res); /* Get the description of a result. */ +MCO_API mco_coro *mco_running(void); /* Returns the running coroutine for the current thread. */ +MCO_API const char *mco_result_description(mco_result res); /* Get the description of a result. */ #ifdef __cplusplus } @@ -354,7 +392,8 @@ MCO_API const char* mco_result_description(mco_result res); /* Get the descripti extern "C" { #endif -/* ---------------------------------------------------------------------------------------------- */ +/* ---------------------------------------------------------------------------------------------- + */ /* Minimum stack size when creating a coroutine. */ #ifndef MCO_MIN_STACK_SIZE @@ -363,11 +402,12 @@ extern "C" { /* Default stack size when creating a coroutine. */ #ifndef MCO_DEFAULT_STACK_SIZE -/* Use multiples of 64KB minus 8KB, because 8KB is reserved for coroutine internal structures. */ +/* Use multiples of 64KB minus 8KB, because 8KB is reserved for coroutine internal + * structures. */ #ifdef MCO_USE_VMEM_ALLOCATOR -#define MCO_DEFAULT_STACK_SIZE 2040*1024 /* 2040KB, nearly the same stack size of a thread in x86_64 Linux. */ +#define MCO_DEFAULT_STACK_SIZE 2040 * 1024 /* 2040KB, nearly the same stack size of a thread in x86_64 Linux. */ #else -#define MCO_DEFAULT_STACK_SIZE 56*1024 /* 56KB */ +#define MCO_DEFAULT_STACK_SIZE 56 * 1024 /* 56KB */ #endif #endif @@ -376,32 +416,30 @@ extern "C" { /* Detect implementation based on OS, arch and compiler. */ #if !defined(MCO_USE_UCONTEXT) && !defined(MCO_USE_FIBERS) && !defined(MCO_USE_ASM) && !defined(MCO_USE_ASYNCIFY) - #if defined(_WIN32) - #if (defined(__GNUC__) && defined(__x86_64__)) || (defined(_MSC_VER) && defined(_M_X64)) - #define MCO_USE_ASM - #else - #define MCO_USE_FIBERS - #endif - #elif defined(__CYGWIN__) /* MSYS */ - #define MCO_USE_UCONTEXT - #elif defined(__EMSCRIPTEN__) - #define MCO_USE_FIBERS - #elif defined(__wasm__) - #define MCO_USE_ASYNCIFY - #else - #if __GNUC__ >= 3 /* Assembly extension supported. */ - #if defined(__x86_64__) || \ - defined(__i386) || defined(__i386__) || \ - defined(__ARM_EABI__) || defined(__aarch64__) || \ - defined(__riscv) - #define MCO_USE_ASM - #else - #define MCO_USE_UCONTEXT - #endif - #else - #define MCO_USE_UCONTEXT - #endif - #endif +#if defined(_WIN32) +#if (defined(__GNUC__) && defined(__x86_64__)) || (defined(_MSC_VER) && defined(_M_X64)) +#define MCO_USE_ASM +#else +#define MCO_USE_FIBERS +#endif +#elif defined(__CYGWIN__) /* MSYS */ +#define MCO_USE_UCONTEXT +#elif defined(__EMSCRIPTEN__) +#define MCO_USE_FIBERS +#elif defined(__wasm__) +#define MCO_USE_ASYNCIFY +#else +#if __GNUC__ >= 3 /* Assembly extension supported. */ +#if defined(__x86_64__) || defined(__i386) || defined(__i386__) || defined(__ARM_EABI__) || defined(__aarch64__) || \ + defined(__riscv) +#define MCO_USE_ASM +#else +#define MCO_USE_UCONTEXT +#endif +#else +#define MCO_USE_UCONTEXT +#endif +#endif #endif #define _MCO_UNUSED(x) (void)(x) @@ -411,174 +449,176 @@ extern "C" { #endif #ifndef MCO_LOG - #ifdef MCO_DEBUG - #include - #define MCO_LOG(s) puts(s) - #else - #define MCO_LOG(s) - #endif +#ifdef MCO_DEBUG +#include +#define MCO_LOG(s) puts(s) +#else +#define MCO_LOG(s) +#endif #endif #ifndef MCO_ASSERT - #ifdef MCO_DEBUG - #include - #define MCO_ASSERT(c) assert(c) - #else - #define MCO_ASSERT(c) - #endif +#ifdef MCO_DEBUG +#include +#define MCO_ASSERT(c) assert(c) +#else +#define MCO_ASSERT(c) +#endif #endif #ifndef MCO_THREAD_LOCAL - #ifdef MCO_NO_MULTITHREAD - #define MCO_THREAD_LOCAL - #else - #ifdef thread_local - #define MCO_THREAD_LOCAL thread_local - #elif __STDC_VERSION__ >= 201112 && !defined(__STDC_NO_THREADS__) - #define MCO_THREAD_LOCAL _Thread_local - #elif defined(_WIN32) && (defined(_MSC_VER) || defined(__ICL) || defined(__DMC__) || defined(__BORLANDC__)) - #define MCO_THREAD_LOCAL __declspec(thread) - #elif defined(__GNUC__) || defined(__SUNPRO_C) || defined(__xlC__) - #define MCO_THREAD_LOCAL __thread - #else /* No thread local support, `mco_running` will be thread unsafe. */ - #define MCO_THREAD_LOCAL - #define MCO_NO_MULTITHREAD - #endif - #endif +#ifdef MCO_NO_MULTITHREAD +#define MCO_THREAD_LOCAL +#else +#ifdef thread_local +#define MCO_THREAD_LOCAL thread_local +#elif __STDC_VERSION__ >= 201112 && !defined(__STDC_NO_THREADS__) +#define MCO_THREAD_LOCAL _Thread_local +#elif defined(_WIN32) && (defined(_MSC_VER) || defined(__ICL) || defined(__DMC__) || defined(__BORLANDC__)) +#define MCO_THREAD_LOCAL __declspec(thread) +#elif defined(__GNUC__) || defined(__SUNPRO_C) || defined(__xlC__) +#define MCO_THREAD_LOCAL __thread +#else /* No thread local support, `mco_running` will be thread unsafe. */ +#define MCO_THREAD_LOCAL +#define MCO_NO_MULTITHREAD +#endif +#endif #endif #ifndef MCO_FORCE_INLINE - #ifdef _MSC_VER - #define MCO_FORCE_INLINE __forceinline - #elif defined(__GNUC__) - #if defined(__STRICT_ANSI__) - #define MCO_FORCE_INLINE __inline__ __attribute__((always_inline)) - #else - #define MCO_FORCE_INLINE inline __attribute__((always_inline)) - #endif - #elif defined(__BORLANDC__) || defined(__DMC__) || defined(__SC__) || defined(__WATCOMC__) || defined(__LCC__) || defined(__DECC) - #define MCO_FORCE_INLINE __inline - #else /* No inline support. */ - #define MCO_FORCE_INLINE - #endif +#ifdef _MSC_VER +#define MCO_FORCE_INLINE __forceinline +#elif defined(__GNUC__) +#if defined(__STRICT_ANSI__) +#define MCO_FORCE_INLINE __inline__ __attribute__((always_inline)) +#else +#define MCO_FORCE_INLINE inline __attribute__((always_inline)) +#endif +#elif defined(__BORLANDC__) || defined(__DMC__) || defined(__SC__) || defined(__WATCOMC__) || defined(__LCC__) || \ + defined(__DECC) +#define MCO_FORCE_INLINE __inline +#else /* No inline support. */ +#define MCO_FORCE_INLINE +#endif #endif #ifndef MCO_NO_INLINE - #ifdef __GNUC__ - #define MCO_NO_INLINE __attribute__((noinline)) - #elif defined(_MSC_VER) - #define MCO_NO_INLINE __declspec(noinline) - #else - #define MCO_NO_INLINE - #endif +#ifdef __GNUC__ +#define MCO_NO_INLINE __attribute__((noinline)) +#elif defined(_MSC_VER) +#define MCO_NO_INLINE __declspec(noinline) +#else +#define MCO_NO_INLINE +#endif #endif #if defined(_WIN32) && (defined(MCO_USE_FIBERS) || defined(MCO_USE_VMEM_ALLOCATOR)) - #ifndef _WIN32_WINNT - #define _WIN32_WINNT 0x0400 - #endif - #ifndef WIN32_LEAN_AND_MEAN - #define WIN32_LEAN_AND_MEAN - #endif - #include +#ifndef _WIN32_WINNT +#define _WIN32_WINNT 0x0400 +#endif +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include #endif #ifndef MCO_NO_DEFAULT_ALLOCATOR - #if defined(MCO_USE_VMEM_ALLOCATOR) && defined(_WIN32) - static void* mco_alloc(size_t size, void* allocator_data) { - _MCO_UNUSED(allocator_data); - return VirtualAlloc(NULL, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); - } - static void mco_dealloc(void* ptr, size_t size, void* allocator_data) { - _MCO_UNUSED(allocator_data); - _MCO_UNUSED(size); - int res = VirtualFree(ptr, 0, MEM_RELEASE); - _MCO_UNUSED(res); - MCO_ASSERT(res != 0); - } - #elif defined(MCO_USE_VMEM_ALLOCATOR) /* POSIX virtual memory allocator */ - #include - static void* mco_alloc(size_t size, void* allocator_data) { - _MCO_UNUSED(allocator_data); - void *ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - return ptr != MAP_FAILED ? ptr : NULL; - } - static void mco_dealloc(void* ptr, size_t size, void* allocator_data) { - _MCO_UNUSED(allocator_data); - int res = munmap(ptr, size); - _MCO_UNUSED(res); - MCO_ASSERT(res == 0); - } - #else /* C allocator */ - #ifndef MCO_ALLOC - #include - /* We use calloc() so we give a chance for the OS to reserve virtual memory without really using physical memory, - calloc() also has the nice property of initializing the stack to zeros. */ - #define MCO_ALLOC(size) calloc(1, size) - #define MCO_DEALLOC(ptr, size) free(ptr) - #endif - static void* mco_alloc(size_t size, void* allocator_data) { - _MCO_UNUSED(allocator_data); - return MCO_ALLOC(size); - } - static void mco_dealloc(void* ptr, size_t size, void* allocator_data) { - _MCO_UNUSED(size); - _MCO_UNUSED(allocator_data); - MCO_DEALLOC(ptr, size); - } - #endif /* MCO_USE_VMEM_ALLOCATOR */ +#if defined(MCO_USE_VMEM_ALLOCATOR) && defined(_WIN32) +static void *mco_alloc(size_t size, void *allocator_data) { + _MCO_UNUSED(allocator_data); + return VirtualAlloc(NULL, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); +} +static void mco_dealloc(void *ptr, size_t size, void *allocator_data) { + _MCO_UNUSED(allocator_data); + _MCO_UNUSED(size); + int res = VirtualFree(ptr, 0, MEM_RELEASE); + _MCO_UNUSED(res); + MCO_ASSERT(res != 0); +} +#elif defined(MCO_USE_VMEM_ALLOCATOR) /* POSIX virtual memory allocator */ +#include +static void *mco_alloc(size_t size, void *allocator_data) { + _MCO_UNUSED(allocator_data); + void *ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + return ptr != MAP_FAILED ? ptr : NULL; +} +static void mco_dealloc(void *ptr, size_t size, void *allocator_data) { + _MCO_UNUSED(allocator_data); + int res = munmap(ptr, size); + _MCO_UNUSED(res); + MCO_ASSERT(res == 0); +} +#else /* C allocator */ +#ifndef MCO_ALLOC +#include +/* We use calloc() so we give a chance for the OS to reserve virtual memory without + really using physical memory, calloc() also has the nice property of initializing the + stack to zeros. */ +#define MCO_ALLOC(size) calloc(1, size) +#define MCO_DEALLOC(ptr, size) free(ptr) +#endif +static void *mco_alloc(size_t size, void *allocator_data) { + _MCO_UNUSED(allocator_data); + return MCO_ALLOC(size); +} +static void mco_dealloc(void *ptr, size_t size, void *allocator_data) { + _MCO_UNUSED(size); + _MCO_UNUSED(allocator_data); + MCO_DEALLOC(ptr, size); +} +#endif /* MCO_USE_VMEM_ALLOCATOR */ #endif /* MCO_NO_DEFAULT_ALLOCATOR */ #if defined(__has_feature) - #if __has_feature(address_sanitizer) - #define _MCO_USE_ASAN - #endif - #if __has_feature(thread_sanitizer) - #define _MCO_USE_TSAN - #endif +#if __has_feature(address_sanitizer) +#define _MCO_USE_ASAN +#endif +#if __has_feature(thread_sanitizer) +#define _MCO_USE_TSAN +#endif #endif #if defined(__SANITIZE_ADDRESS__) - #define _MCO_USE_ASAN +#define _MCO_USE_ASAN #endif #if defined(__SANITIZE_THREAD__) - #define _MCO_USE_TSAN +#define _MCO_USE_TSAN #endif #ifdef _MCO_USE_ASAN -void __sanitizer_start_switch_fiber(void** fake_stack_save, const void *bottom, size_t size); -void __sanitizer_finish_switch_fiber(void* fake_stack_save, const void **bottom_old, size_t *size_old); +void __sanitizer_start_switch_fiber(void **fake_stack_save, const void *bottom, size_t size); +void __sanitizer_finish_switch_fiber(void *fake_stack_save, const void **bottom_old, size_t *size_old); #endif #ifdef _MCO_USE_TSAN -void* __tsan_get_current_fiber(void); -void* __tsan_create_fiber(unsigned flags); -void __tsan_destroy_fiber(void* fiber); -void __tsan_switch_to_fiber(void* fiber, unsigned flags); +void *__tsan_get_current_fiber(void); +void *__tsan_create_fiber(unsigned flags); +void __tsan_destroy_fiber(void *fiber); +void __tsan_switch_to_fiber(void *fiber, unsigned flags); #endif #include /* For memcpy and memset. */ /* Utility for aligning addresses. */ static MCO_FORCE_INLINE size_t _mco_align_forward(size_t addr, size_t align) { - return (addr + (align-1)) & ~(align-1); + return (addr + (align - 1)) & ~(align - 1); } /* Variable holding the current running coroutine per thread. */ -static MCO_THREAD_LOCAL mco_coro* mco_current_co = NULL; +static MCO_THREAD_LOCAL mco_coro *mco_current_co = NULL; -static MCO_FORCE_INLINE void _mco_prepare_jumpin(mco_coro* co) { +static MCO_FORCE_INLINE void _mco_prepare_jumpin(mco_coro *co) { /* Set the old coroutine to normal state and update it. */ - mco_coro* prev_co = mco_running(); /* Must access through `mco_running`. */ + mco_coro *prev_co = mco_running(); /* Must access through `mco_running`. */ MCO_ASSERT(co->prev_co == NULL); co->prev_co = prev_co; - if(prev_co) { + if (prev_co) { MCO_ASSERT(prev_co->state == MCO_RUNNING); prev_co->state = MCO_NORMAL; } mco_current_co = co; #ifdef _MCO_USE_ASAN - if(prev_co) { - void* bottom_old = NULL; + if (prev_co) { + void *bottom_old = NULL; size_t size_old = 0; - __sanitizer_finish_switch_fiber(prev_co->asan_prev_stack, (const void**)&bottom_old, &size_old); + __sanitizer_finish_switch_fiber(prev_co->asan_prev_stack, (const void **)&bottom_old, &size_old); prev_co->asan_prev_stack = NULL; } __sanitizer_start_switch_fiber(&co->asan_prev_stack, co->stack_base, co->stack_size); @@ -589,42 +629,41 @@ static MCO_FORCE_INLINE void _mco_prepare_jumpin(mco_coro* co) { #endif } -static MCO_FORCE_INLINE void _mco_prepare_jumpout(mco_coro* co) { +static MCO_FORCE_INLINE void _mco_prepare_jumpout(mco_coro *co) { /* Switch back to the previous running coroutine. */ /* MCO_ASSERT(mco_running() == co); */ - mco_coro* prev_co = co->prev_co; + mco_coro *prev_co = co->prev_co; co->prev_co = NULL; - if(prev_co) { + if (prev_co) { /* MCO_ASSERT(prev_co->state == MCO_NORMAL); */ prev_co->state = MCO_RUNNING; } mco_current_co = prev_co; #ifdef _MCO_USE_ASAN - void* bottom_old = NULL; + void *bottom_old = NULL; size_t size_old = 0; - __sanitizer_finish_switch_fiber(co->asan_prev_stack, (const void**)&bottom_old, &size_old); + __sanitizer_finish_switch_fiber(co->asan_prev_stack, (const void **)&bottom_old, &size_old); co->asan_prev_stack = NULL; - if(prev_co) { - __sanitizer_start_switch_fiber(&prev_co->asan_prev_stack, bottom_old, size_old); - } + if (prev_co) { __sanitizer_start_switch_fiber(&prev_co->asan_prev_stack, bottom_old, size_old); } #endif #ifdef _MCO_USE_TSAN - void* tsan_prev_fiber = co->tsan_prev_fiber; + void *tsan_prev_fiber = co->tsan_prev_fiber; co->tsan_prev_fiber = NULL; __tsan_switch_to_fiber(tsan_prev_fiber, 0); #endif } -static void _mco_jumpin(mco_coro* co); -static void _mco_jumpout(mco_coro* co); +static void _mco_jumpin(mco_coro *co); +static void _mco_jumpout(mco_coro *co); -static MCO_NO_INLINE void _mco_main(mco_coro* co) { - co->func(co); /* Run the coroutine function. */ +static MCO_NO_INLINE void _mco_main(mco_coro *co) { + co->func(co); /* Run the coroutine function. */ co->state = MCO_DEAD; /* Coroutine finished successfully, set state to dead. */ - _mco_jumpout(co); /* Jump back to the old context .*/ + _mco_jumpout(co); /* Jump back to the old context .*/ } -/* ---------------------------------------------------------------------------------------------- */ +/* ---------------------------------------------------------------------------------------------- + */ #if defined(MCO_USE_UCONTEXT) || defined(MCO_USE_ASM) @@ -664,11 +703,11 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. typedef struct _mco_ctxbuf { void *rip, *rsp, *rbp, *rbx, *r12, *r13, *r14, *r15, *rdi, *rsi; - void* xmm[20]; /* xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15 */ - void* fiber_storage; - void* dealloc_stack; - void* stack_limit; - void* stack_base; + void *xmm[20]; /* xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15 */ + void *fiber_storage; + void *dealloc_stack; + void *stack_limit; + void *stack_base; } _mco_ctxbuf; #if defined(__GNUC__) @@ -679,163 +718,162 @@ typedef struct _mco_ctxbuf { #endif _MCO_ASM_BLOB static unsigned char _mco_wrap_main_code[] = { - 0x4c, 0x89, 0xe9, /* mov %r13,%rcx */ - 0x41, 0xff, 0xe4, /* jmpq *%r12 */ - 0xc3, /* retq */ - 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90 /* nop */ + 0x4c, 0x89, 0xe9, /* mov %r13,%rcx */ + 0x41, 0xff, 0xe4, /* jmpq *%r12 */ + 0xc3, /* retq */ + 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90 /* nop */ }; _MCO_ASM_BLOB static unsigned char _mco_switch_code[] = { - 0x48, 0x8d, 0x05, 0x3e, 0x01, 0x00, 0x00, /* lea 0x13e(%rip),%rax */ - 0x48, 0x89, 0x01, /* mov %rax,(%rcx) */ - 0x48, 0x89, 0x61, 0x08, /* mov %rsp,0x8(%rcx) */ - 0x48, 0x89, 0x69, 0x10, /* mov %rbp,0x10(%rcx) */ - 0x48, 0x89, 0x59, 0x18, /* mov %rbx,0x18(%rcx) */ - 0x4c, 0x89, 0x61, 0x20, /* mov %r12,0x20(%rcx) */ - 0x4c, 0x89, 0x69, 0x28, /* mov %r13,0x28(%rcx) */ - 0x4c, 0x89, 0x71, 0x30, /* mov %r14,0x30(%rcx) */ - 0x4c, 0x89, 0x79, 0x38, /* mov %r15,0x38(%rcx) */ - 0x48, 0x89, 0x79, 0x40, /* mov %rdi,0x40(%rcx) */ - 0x48, 0x89, 0x71, 0x48, /* mov %rsi,0x48(%rcx) */ - 0x0f, 0x11, 0x71, 0x50, /* movups %xmm6,0x50(%rcx) */ - 0x0f, 0x11, 0x79, 0x60, /* movups %xmm7,0x60(%rcx) */ - 0x44, 0x0f, 0x11, 0x41, 0x70, /* movups %xmm8,0x70(%rcx) */ - 0x44, 0x0f, 0x11, 0x89, 0x80, 0x00, 0x00, 0x00, /* movups %xmm9,0x80(%rcx) */ - 0x44, 0x0f, 0x11, 0x91, 0x90, 0x00, 0x00, 0x00, /* movups %xmm10,0x90(%rcx) */ - 0x44, 0x0f, 0x11, 0x99, 0xa0, 0x00, 0x00, 0x00, /* movups %xmm11,0xa0(%rcx) */ - 0x44, 0x0f, 0x11, 0xa1, 0xb0, 0x00, 0x00, 0x00, /* movups %xmm12,0xb0(%rcx) */ - 0x44, 0x0f, 0x11, 0xa9, 0xc0, 0x00, 0x00, 0x00, /* movups %xmm13,0xc0(%rcx) */ - 0x44, 0x0f, 0x11, 0xb1, 0xd0, 0x00, 0x00, 0x00, /* movups %xmm14,0xd0(%rcx) */ - 0x44, 0x0f, 0x11, 0xb9, 0xe0, 0x00, 0x00, 0x00, /* movups %xmm15,0xe0(%rcx) */ - 0x65, 0x4c, 0x8b, 0x14, 0x25, 0x30, 0x00, 0x00, 0x00, /* mov %gs:0x30,%r10 */ - 0x49, 0x8b, 0x42, 0x20, /* mov 0x20(%r10),%rax */ - 0x48, 0x89, 0x81, 0xf0, 0x00, 0x00, 0x00, /* mov %rax,0xf0(%rcx) */ - 0x49, 0x8b, 0x82, 0x78, 0x14, 0x00, 0x00, /* mov 0x1478(%r10),%rax */ - 0x48, 0x89, 0x81, 0xf8, 0x00, 0x00, 0x00, /* mov %rax,0xf8(%rcx) */ - 0x49, 0x8b, 0x42, 0x10, /* mov 0x10(%r10),%rax */ - 0x48, 0x89, 0x81, 0x00, 0x01, 0x00, 0x00, /* mov %rax,0x100(%rcx) */ - 0x49, 0x8b, 0x42, 0x08, /* mov 0x8(%r10),%rax */ - 0x48, 0x89, 0x81, 0x08, 0x01, 0x00, 0x00, /* mov %rax,0x108(%rcx) */ - 0x48, 0x8b, 0x82, 0x08, 0x01, 0x00, 0x00, /* mov 0x108(%rdx),%rax */ - 0x49, 0x89, 0x42, 0x08, /* mov %rax,0x8(%r10) */ - 0x48, 0x8b, 0x82, 0x00, 0x01, 0x00, 0x00, /* mov 0x100(%rdx),%rax */ - 0x49, 0x89, 0x42, 0x10, /* mov %rax,0x10(%r10) */ - 0x48, 0x8b, 0x82, 0xf8, 0x00, 0x00, 0x00, /* mov 0xf8(%rdx),%rax */ - 0x49, 0x89, 0x82, 0x78, 0x14, 0x00, 0x00, /* mov %rax,0x1478(%r10) */ - 0x48, 0x8b, 0x82, 0xf0, 0x00, 0x00, 0x00, /* mov 0xf0(%rdx),%rax */ - 0x49, 0x89, 0x42, 0x20, /* mov %rax,0x20(%r10) */ - 0x44, 0x0f, 0x10, 0xba, 0xe0, 0x00, 0x00, 0x00, /* movups 0xe0(%rdx),%xmm15 */ - 0x44, 0x0f, 0x10, 0xb2, 0xd0, 0x00, 0x00, 0x00, /* movups 0xd0(%rdx),%xmm14 */ - 0x44, 0x0f, 0x10, 0xaa, 0xc0, 0x00, 0x00, 0x00, /* movups 0xc0(%rdx),%xmm13 */ - 0x44, 0x0f, 0x10, 0xa2, 0xb0, 0x00, 0x00, 0x00, /* movups 0xb0(%rdx),%xmm12 */ - 0x44, 0x0f, 0x10, 0x9a, 0xa0, 0x00, 0x00, 0x00, /* movups 0xa0(%rdx),%xmm11 */ - 0x44, 0x0f, 0x10, 0x92, 0x90, 0x00, 0x00, 0x00, /* movups 0x90(%rdx),%xmm10 */ - 0x44, 0x0f, 0x10, 0x8a, 0x80, 0x00, 0x00, 0x00, /* movups 0x80(%rdx),%xmm9 */ - 0x44, 0x0f, 0x10, 0x42, 0x70, /* movups 0x70(%rdx),%xmm8 */ - 0x0f, 0x10, 0x7a, 0x60, /* movups 0x60(%rdx),%xmm7 */ - 0x0f, 0x10, 0x72, 0x50, /* movups 0x50(%rdx),%xmm6 */ - 0x48, 0x8b, 0x72, 0x48, /* mov 0x48(%rdx),%rsi */ - 0x48, 0x8b, 0x7a, 0x40, /* mov 0x40(%rdx),%rdi */ - 0x4c, 0x8b, 0x7a, 0x38, /* mov 0x38(%rdx),%r15 */ - 0x4c, 0x8b, 0x72, 0x30, /* mov 0x30(%rdx),%r14 */ - 0x4c, 0x8b, 0x6a, 0x28, /* mov 0x28(%rdx),%r13 */ - 0x4c, 0x8b, 0x62, 0x20, /* mov 0x20(%rdx),%r12 */ - 0x48, 0x8b, 0x5a, 0x18, /* mov 0x18(%rdx),%rbx */ - 0x48, 0x8b, 0x6a, 0x10, /* mov 0x10(%rdx),%rbp */ - 0x48, 0x8b, 0x62, 0x08, /* mov 0x8(%rdx),%rsp */ - 0xff, 0x22, /* jmpq *(%rdx) */ - 0xc3, /* retq */ - 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, /* nop */ - 0x90, 0x90, /* nop */ + 0x48, 0x8d, 0x05, 0x3e, 0x01, 0x00, 0x00, /* lea 0x13e(%rip),%rax */ + 0x48, 0x89, 0x01, /* mov %rax,(%rcx) */ + 0x48, 0x89, 0x61, 0x08, /* mov %rsp,0x8(%rcx) */ + 0x48, 0x89, 0x69, 0x10, /* mov %rbp,0x10(%rcx) */ + 0x48, 0x89, 0x59, 0x18, /* mov %rbx,0x18(%rcx) */ + 0x4c, 0x89, 0x61, 0x20, /* mov %r12,0x20(%rcx) */ + 0x4c, 0x89, 0x69, 0x28, /* mov %r13,0x28(%rcx) */ + 0x4c, 0x89, 0x71, 0x30, /* mov %r14,0x30(%rcx) */ + 0x4c, 0x89, 0x79, 0x38, /* mov %r15,0x38(%rcx) */ + 0x48, 0x89, 0x79, 0x40, /* mov %rdi,0x40(%rcx) */ + 0x48, 0x89, 0x71, 0x48, /* mov %rsi,0x48(%rcx) */ + 0x0f, 0x11, 0x71, 0x50, /* movups %xmm6,0x50(%rcx) */ + 0x0f, 0x11, 0x79, 0x60, /* movups %xmm7,0x60(%rcx) */ + 0x44, 0x0f, 0x11, 0x41, 0x70, /* movups %xmm8,0x70(%rcx) */ + 0x44, 0x0f, 0x11, 0x89, 0x80, 0x00, 0x00, 0x00, /* movups %xmm9,0x80(%rcx) */ + 0x44, 0x0f, 0x11, 0x91, 0x90, 0x00, 0x00, 0x00, /* movups %xmm10,0x90(%rcx) */ + 0x44, 0x0f, 0x11, 0x99, 0xa0, 0x00, 0x00, 0x00, /* movups %xmm11,0xa0(%rcx) */ + 0x44, 0x0f, 0x11, 0xa1, 0xb0, 0x00, 0x00, 0x00, /* movups %xmm12,0xb0(%rcx) */ + 0x44, 0x0f, 0x11, 0xa9, 0xc0, 0x00, 0x00, 0x00, /* movups %xmm13,0xc0(%rcx) */ + 0x44, 0x0f, 0x11, 0xb1, 0xd0, 0x00, 0x00, 0x00, /* movups %xmm14,0xd0(%rcx) */ + 0x44, 0x0f, 0x11, 0xb9, 0xe0, 0x00, 0x00, 0x00, /* movups %xmm15,0xe0(%rcx) */ + 0x65, 0x4c, 0x8b, 0x14, 0x25, 0x30, 0x00, 0x00, 0x00, /* mov %gs:0x30,%r10 */ + 0x49, 0x8b, 0x42, 0x20, /* mov 0x20(%r10),%rax */ + 0x48, 0x89, 0x81, 0xf0, 0x00, 0x00, 0x00, /* mov %rax,0xf0(%rcx) */ + 0x49, 0x8b, 0x82, 0x78, 0x14, 0x00, 0x00, /* mov 0x1478(%r10),%rax */ + 0x48, 0x89, 0x81, 0xf8, 0x00, 0x00, 0x00, /* mov %rax,0xf8(%rcx) */ + 0x49, 0x8b, 0x42, 0x10, /* mov 0x10(%r10),%rax */ + 0x48, 0x89, 0x81, 0x00, 0x01, 0x00, 0x00, /* mov %rax,0x100(%rcx) */ + 0x49, 0x8b, 0x42, 0x08, /* mov 0x8(%r10),%rax */ + 0x48, 0x89, 0x81, 0x08, 0x01, 0x00, 0x00, /* mov %rax,0x108(%rcx) */ + 0x48, 0x8b, 0x82, 0x08, 0x01, 0x00, 0x00, /* mov 0x108(%rdx),%rax */ + 0x49, 0x89, 0x42, 0x08, /* mov %rax,0x8(%r10) */ + 0x48, 0x8b, 0x82, 0x00, 0x01, 0x00, 0x00, /* mov 0x100(%rdx),%rax */ + 0x49, 0x89, 0x42, 0x10, /* mov %rax,0x10(%r10) */ + 0x48, 0x8b, 0x82, 0xf8, 0x00, 0x00, 0x00, /* mov 0xf8(%rdx),%rax */ + 0x49, 0x89, 0x82, 0x78, 0x14, 0x00, 0x00, /* mov %rax,0x1478(%r10) */ + 0x48, 0x8b, 0x82, 0xf0, 0x00, 0x00, 0x00, /* mov 0xf0(%rdx),%rax */ + 0x49, 0x89, 0x42, 0x20, /* mov %rax,0x20(%r10) */ + 0x44, 0x0f, 0x10, 0xba, 0xe0, 0x00, 0x00, 0x00, /* movups 0xe0(%rdx),%xmm15 */ + 0x44, 0x0f, 0x10, 0xb2, 0xd0, 0x00, 0x00, 0x00, /* movups 0xd0(%rdx),%xmm14 */ + 0x44, 0x0f, 0x10, 0xaa, 0xc0, 0x00, 0x00, 0x00, /* movups 0xc0(%rdx),%xmm13 */ + 0x44, 0x0f, 0x10, 0xa2, 0xb0, 0x00, 0x00, 0x00, /* movups 0xb0(%rdx),%xmm12 */ + 0x44, 0x0f, 0x10, 0x9a, 0xa0, 0x00, 0x00, 0x00, /* movups 0xa0(%rdx),%xmm11 */ + 0x44, 0x0f, 0x10, 0x92, 0x90, 0x00, 0x00, 0x00, /* movups 0x90(%rdx),%xmm10 */ + 0x44, 0x0f, 0x10, 0x8a, 0x80, 0x00, 0x00, 0x00, /* movups 0x80(%rdx),%xmm9 */ + 0x44, 0x0f, 0x10, 0x42, 0x70, /* movups 0x70(%rdx),%xmm8 */ + 0x0f, 0x10, 0x7a, 0x60, /* movups 0x60(%rdx),%xmm7 */ + 0x0f, 0x10, 0x72, 0x50, /* movups 0x50(%rdx),%xmm6 */ + 0x48, 0x8b, 0x72, 0x48, /* mov 0x48(%rdx),%rsi */ + 0x48, 0x8b, 0x7a, 0x40, /* mov 0x40(%rdx),%rdi */ + 0x4c, 0x8b, 0x7a, 0x38, /* mov 0x38(%rdx),%r15 */ + 0x4c, 0x8b, 0x72, 0x30, /* mov 0x30(%rdx),%r14 */ + 0x4c, 0x8b, 0x6a, 0x28, /* mov 0x28(%rdx),%r13 */ + 0x4c, 0x8b, 0x62, 0x20, /* mov 0x20(%rdx),%r12 */ + 0x48, 0x8b, 0x5a, 0x18, /* mov 0x18(%rdx),%rbx */ + 0x48, 0x8b, 0x6a, 0x10, /* mov 0x10(%rdx),%rbp */ + 0x48, 0x8b, 0x62, 0x08, /* mov 0x8(%rdx),%rsp */ + 0xff, 0x22, /* jmpq *(%rdx) */ + 0xc3, /* retq */ + 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, /* nop */ + 0x90, 0x90, /* nop */ }; -void (*_mco_wrap_main)(void) = (void(*)(void))(void*)_mco_wrap_main_code; -void (*_mco_switch)(_mco_ctxbuf* from, _mco_ctxbuf* to) = (void(*)(_mco_ctxbuf* from, _mco_ctxbuf* to))(void*)_mco_switch_code; +void (*_mco_wrap_main)(void) = (void (*)(void))(void *)_mco_wrap_main_code; +void (*_mco_switch)(_mco_ctxbuf *from, + _mco_ctxbuf *to) = (void (*)(_mco_ctxbuf *from, _mco_ctxbuf *to))(void *)_mco_switch_code; -static mco_result _mco_makectx(mco_coro* co, _mco_ctxbuf* ctx, void* stack_base, size_t stack_size) { +static mco_result _mco_makectx(mco_coro *co, _mco_ctxbuf *ctx, void *stack_base, size_t stack_size) { stack_size = stack_size - 32; /* Reserve 32 bytes for the shadow space. */ - void** stack_high_ptr = (void**)((size_t)stack_base + stack_size - sizeof(size_t)); - stack_high_ptr[0] = (void*)(0xdeaddeaddeaddead); /* Dummy return address. */ - ctx->rip = (void*)(_mco_wrap_main); - ctx->rsp = (void*)(stack_high_ptr); - ctx->r12 = (void*)(_mco_main); - ctx->r13 = (void*)(co); - void* stack_top = (void*)((size_t)stack_base + stack_size); + void **stack_high_ptr = (void **)((size_t)stack_base + stack_size - sizeof(size_t)); + stack_high_ptr[0] = (void *)(0xdeaddeaddeaddead); /* Dummy return address. */ + ctx->rip = (void *)(_mco_wrap_main); + ctx->rsp = (void *)(stack_high_ptr); + ctx->r12 = (void *)(_mco_main); + ctx->r13 = (void *)(co); + void *stack_top = (void *)((size_t)stack_base + stack_size); ctx->stack_base = stack_top; ctx->stack_limit = stack_base; ctx->dealloc_stack = stack_base; return MCO_SUCCESS; } -#else /* not _WIN32 */ +#else /* not _WIN32 */ typedef struct _mco_ctxbuf { void *rip, *rsp, *rbp, *rbx, *r12, *r13, *r14, *r15; } _mco_ctxbuf; void _mco_wrap_main(void); -int _mco_switch(_mco_ctxbuf* from, _mco_ctxbuf* to); +int _mco_switch(_mco_ctxbuf *from, _mco_ctxbuf *to); -__asm__( - ".text\n" +__asm__(".text\n" #ifdef __MACH__ /* Mac OS X assembler */ - ".globl __mco_wrap_main\n" - "__mco_wrap_main:\n" -#else /* Linux assembler */ - ".globl _mco_wrap_main\n" - ".type _mco_wrap_main @function\n" - ".hidden _mco_wrap_main\n" - "_mco_wrap_main:\n" -#endif - " movq %r13, %rdi\n" - " jmpq *%r12\n" + ".globl __mco_wrap_main\n" + "__mco_wrap_main:\n" +#else /* Linux assembler */ + ".globl _mco_wrap_main\n" + ".type _mco_wrap_main @function\n" + ".hidden _mco_wrap_main\n" + "_mco_wrap_main:\n" +#endif + " movq %r13, %rdi\n" + " jmpq *%r12\n" #ifndef __MACH__ - ".size _mco_wrap_main, .-_mco_wrap_main\n" + ".size _mco_wrap_main, .-_mco_wrap_main\n" #endif ); -__asm__( - ".text\n" +__asm__(".text\n" #ifdef __MACH__ /* Mac OS assembler */ - ".globl __mco_switch\n" - "__mco_switch:\n" -#else /* Linux assembler */ - ".globl _mco_switch\n" - ".type _mco_switch @function\n" - ".hidden _mco_switch\n" - "_mco_switch:\n" -#endif - " leaq 0x3d(%rip), %rax\n" - " movq %rax, (%rdi)\n" - " movq %rsp, 8(%rdi)\n" - " movq %rbp, 16(%rdi)\n" - " movq %rbx, 24(%rdi)\n" - " movq %r12, 32(%rdi)\n" - " movq %r13, 40(%rdi)\n" - " movq %r14, 48(%rdi)\n" - " movq %r15, 56(%rdi)\n" - " movq 56(%rsi), %r15\n" - " movq 48(%rsi), %r14\n" - " movq 40(%rsi), %r13\n" - " movq 32(%rsi), %r12\n" - " movq 24(%rsi), %rbx\n" - " movq 16(%rsi), %rbp\n" - " movq 8(%rsi), %rsp\n" - " jmpq *(%rsi)\n" - " ret\n" + ".globl __mco_switch\n" + "__mco_switch:\n" +#else /* Linux assembler */ + ".globl _mco_switch\n" + ".type _mco_switch @function\n" + ".hidden _mco_switch\n" + "_mco_switch:\n" +#endif + " leaq 0x3d(%rip), %rax\n" + " movq %rax, (%rdi)\n" + " movq %rsp, 8(%rdi)\n" + " movq %rbp, 16(%rdi)\n" + " movq %rbx, 24(%rdi)\n" + " movq %r12, 32(%rdi)\n" + " movq %r13, 40(%rdi)\n" + " movq %r14, 48(%rdi)\n" + " movq %r15, 56(%rdi)\n" + " movq 56(%rsi), %r15\n" + " movq 48(%rsi), %r14\n" + " movq 40(%rsi), %r13\n" + " movq 32(%rsi), %r12\n" + " movq 24(%rsi), %rbx\n" + " movq 16(%rsi), %rbp\n" + " movq 8(%rsi), %rsp\n" + " jmpq *(%rsi)\n" + " ret\n" #ifndef __MACH__ - ".size _mco_switch, .-_mco_switch\n" + ".size _mco_switch, .-_mco_switch\n" #endif ); -static mco_result _mco_makectx(mco_coro* co, _mco_ctxbuf* ctx, void* stack_base, size_t stack_size) { +static mco_result _mco_makectx(mco_coro *co, _mco_ctxbuf *ctx, void *stack_base, size_t stack_size) { stack_size = stack_size - 128; /* Reserve 128 bytes for the Red Zone space (System V AMD64 ABI). */ - void** stack_high_ptr = (void**)((size_t)stack_base + stack_size - sizeof(size_t)); - stack_high_ptr[0] = (void*)(0xdeaddeaddeaddead); /* Dummy return address. */ - ctx->rip = (void*)(_mco_wrap_main); - ctx->rsp = (void*)(stack_high_ptr); - ctx->r12 = (void*)(_mco_main); - ctx->r13 = (void*)(co); + void **stack_high_ptr = (void **)((size_t)stack_base + stack_size - sizeof(size_t)); + stack_high_ptr[0] = (void *)(0xdeaddeaddeaddead); /* Dummy return address. */ + ctx->rip = (void *)(_mco_wrap_main); + ctx->rsp = (void *)(stack_high_ptr); + ctx->r12 = (void *)(_mco_main); + ctx->r13 = (void *)(co); return MCO_SUCCESS; } @@ -844,10 +882,10 @@ static mco_result _mco_makectx(mco_coro* co, _mco_ctxbuf* ctx, void* stack_base, #elif defined(__riscv) typedef struct _mco_ctxbuf { - void* s[12]; /* s0-s11 */ - void* ra; - void* pc; - void* sp; + void *s[12]; /* s0-s11 */ + void *ra; + void *pc; + void *sp; #ifdef __riscv_flen #if __riscv_flen == 64 double fs[12]; /* fs0-fs11 */ @@ -858,190 +896,186 @@ typedef struct _mco_ctxbuf { } _mco_ctxbuf; void _mco_wrap_main(void); -int _mco_switch(_mco_ctxbuf* from, _mco_ctxbuf* to); - -__asm__( - ".text\n" - ".globl _mco_wrap_main\n" - ".type _mco_wrap_main @function\n" - ".hidden _mco_wrap_main\n" - "_mco_wrap_main:\n" - " mv a0, s0\n" - " jr s1\n" - ".size _mco_wrap_main, .-_mco_wrap_main\n" -); - -__asm__( - ".text\n" - ".globl _mco_switch\n" - ".type _mco_switch @function\n" - ".hidden _mco_switch\n" - "_mco_switch:\n" - #if __riscv_xlen == 64 - " sd s0, 0x00(a0)\n" - " sd s1, 0x08(a0)\n" - " sd s2, 0x10(a0)\n" - " sd s3, 0x18(a0)\n" - " sd s4, 0x20(a0)\n" - " sd s5, 0x28(a0)\n" - " sd s6, 0x30(a0)\n" - " sd s7, 0x38(a0)\n" - " sd s8, 0x40(a0)\n" - " sd s9, 0x48(a0)\n" - " sd s10, 0x50(a0)\n" - " sd s11, 0x58(a0)\n" - " sd ra, 0x60(a0)\n" - " sd ra, 0x68(a0)\n" /* pc */ - " sd sp, 0x70(a0)\n" - #ifdef __riscv_flen - #if __riscv_flen == 64 - " fsd fs0, 0x78(a0)\n" - " fsd fs1, 0x80(a0)\n" - " fsd fs2, 0x88(a0)\n" - " fsd fs3, 0x90(a0)\n" - " fsd fs4, 0x98(a0)\n" - " fsd fs5, 0xa0(a0)\n" - " fsd fs6, 0xa8(a0)\n" - " fsd fs7, 0xb0(a0)\n" - " fsd fs8, 0xb8(a0)\n" - " fsd fs9, 0xc0(a0)\n" - " fsd fs10, 0xc8(a0)\n" - " fsd fs11, 0xd0(a0)\n" - " fld fs0, 0x78(a1)\n" - " fld fs1, 0x80(a1)\n" - " fld fs2, 0x88(a1)\n" - " fld fs3, 0x90(a1)\n" - " fld fs4, 0x98(a1)\n" - " fld fs5, 0xa0(a1)\n" - " fld fs6, 0xa8(a1)\n" - " fld fs7, 0xb0(a1)\n" - " fld fs8, 0xb8(a1)\n" - " fld fs9, 0xc0(a1)\n" - " fld fs10, 0xc8(a1)\n" - " fld fs11, 0xd0(a1)\n" - #else - #error "Unsupported RISC-V FLEN" - #endif - #endif /* __riscv_flen */ - " ld s0, 0x00(a1)\n" - " ld s1, 0x08(a1)\n" - " ld s2, 0x10(a1)\n" - " ld s3, 0x18(a1)\n" - " ld s4, 0x20(a1)\n" - " ld s5, 0x28(a1)\n" - " ld s6, 0x30(a1)\n" - " ld s7, 0x38(a1)\n" - " ld s8, 0x40(a1)\n" - " ld s9, 0x48(a1)\n" - " ld s10, 0x50(a1)\n" - " ld s11, 0x58(a1)\n" - " ld ra, 0x60(a1)\n" - " ld a2, 0x68(a1)\n" /* pc */ - " ld sp, 0x70(a1)\n" - " jr a2\n" - #elif __riscv_xlen == 32 - " sw s0, 0x00(a0)\n" - " sw s1, 0x04(a0)\n" - " sw s2, 0x08(a0)\n" - " sw s3, 0x0c(a0)\n" - " sw s4, 0x10(a0)\n" - " sw s5, 0x14(a0)\n" - " sw s6, 0x18(a0)\n" - " sw s7, 0x1c(a0)\n" - " sw s8, 0x20(a0)\n" - " sw s9, 0x24(a0)\n" - " sw s10, 0x28(a0)\n" - " sw s11, 0x2c(a0)\n" - " sw ra, 0x30(a0)\n" - " sw ra, 0x34(a0)\n" /* pc */ - " sw sp, 0x38(a0)\n" - #ifdef __riscv_flen - #if __riscv_flen == 64 - " fsd fs0, 0x3c(a0)\n" - " fsd fs1, 0x44(a0)\n" - " fsd fs2, 0x4c(a0)\n" - " fsd fs3, 0x54(a0)\n" - " fsd fs4, 0x5c(a0)\n" - " fsd fs5, 0x64(a0)\n" - " fsd fs6, 0x6c(a0)\n" - " fsd fs7, 0x74(a0)\n" - " fsd fs8, 0x7c(a0)\n" - " fsd fs9, 0x84(a0)\n" - " fsd fs10, 0x8c(a0)\n" - " fsd fs11, 0x94(a0)\n" - " fld fs0, 0x3c(a1)\n" - " fld fs1, 0x44(a1)\n" - " fld fs2, 0x4c(a1)\n" - " fld fs3, 0x54(a1)\n" - " fld fs4, 0x5c(a1)\n" - " fld fs5, 0x64(a1)\n" - " fld fs6, 0x6c(a1)\n" - " fld fs7, 0x74(a1)\n" - " fld fs8, 0x7c(a1)\n" - " fld fs9, 0x84(a1)\n" - " fld fs10, 0x8c(a1)\n" - " fld fs11, 0x94(a1)\n" - #elif __riscv_flen == 32 - " fsw fs0, 0x3c(a0)\n" - " fsw fs1, 0x40(a0)\n" - " fsw fs2, 0x44(a0)\n" - " fsw fs3, 0x48(a0)\n" - " fsw fs4, 0x4c(a0)\n" - " fsw fs5, 0x50(a0)\n" - " fsw fs6, 0x54(a0)\n" - " fsw fs7, 0x58(a0)\n" - " fsw fs8, 0x5c(a0)\n" - " fsw fs9, 0x60(a0)\n" - " fsw fs10, 0x64(a0)\n" - " fsw fs11, 0x68(a0)\n" - " flw fs0, 0x3c(a1)\n" - " flw fs1, 0x40(a1)\n" - " flw fs2, 0x44(a1)\n" - " flw fs3, 0x48(a1)\n" - " flw fs4, 0x4c(a1)\n" - " flw fs5, 0x50(a1)\n" - " flw fs6, 0x54(a1)\n" - " flw fs7, 0x58(a1)\n" - " flw fs8, 0x5c(a1)\n" - " flw fs9, 0x60(a1)\n" - " flw fs10, 0x64(a1)\n" - " flw fs11, 0x68(a1)\n" - #else - #error "Unsupported RISC-V FLEN" - #endif - #endif /* __riscv_flen */ - " lw s0, 0x00(a1)\n" - " lw s1, 0x04(a1)\n" - " lw s2, 0x08(a1)\n" - " lw s3, 0x0c(a1)\n" - " lw s4, 0x10(a1)\n" - " lw s5, 0x14(a1)\n" - " lw s6, 0x18(a1)\n" - " lw s7, 0x1c(a1)\n" - " lw s8, 0x20(a1)\n" - " lw s9, 0x24(a1)\n" - " lw s10, 0x28(a1)\n" - " lw s11, 0x2c(a1)\n" - " lw ra, 0x30(a1)\n" - " lw a2, 0x34(a1)\n" /* pc */ - " lw sp, 0x38(a1)\n" - " jr a2\n" - #else - #error "Unsupported RISC-V XLEN" - #endif /* __riscv_xlen */ - ".size _mco_switch, .-_mco_switch\n" -); - -static mco_result _mco_makectx(mco_coro* co, _mco_ctxbuf* ctx, void* stack_base, size_t stack_size) { - ctx->s[0] = (void*)(co); - ctx->s[1] = (void*)(_mco_main); - ctx->pc = (void*)(_mco_wrap_main); +int _mco_switch(_mco_ctxbuf *from, _mco_ctxbuf *to); + +__asm__(".text\n" + ".globl _mco_wrap_main\n" + ".type _mco_wrap_main @function\n" + ".hidden _mco_wrap_main\n" + "_mco_wrap_main:\n" + " mv a0, s0\n" + " jr s1\n" + ".size _mco_wrap_main, .-_mco_wrap_main\n"); + +__asm__(".text\n" + ".globl _mco_switch\n" + ".type _mco_switch @function\n" + ".hidden _mco_switch\n" + "_mco_switch:\n" +#if __riscv_xlen == 64 + " sd s0, 0x00(a0)\n" + " sd s1, 0x08(a0)\n" + " sd s2, 0x10(a0)\n" + " sd s3, 0x18(a0)\n" + " sd s4, 0x20(a0)\n" + " sd s5, 0x28(a0)\n" + " sd s6, 0x30(a0)\n" + " sd s7, 0x38(a0)\n" + " sd s8, 0x40(a0)\n" + " sd s9, 0x48(a0)\n" + " sd s10, 0x50(a0)\n" + " sd s11, 0x58(a0)\n" + " sd ra, 0x60(a0)\n" + " sd ra, 0x68(a0)\n" /* pc */ + " sd sp, 0x70(a0)\n" +#ifdef __riscv_flen +#if __riscv_flen == 64 + " fsd fs0, 0x78(a0)\n" + " fsd fs1, 0x80(a0)\n" + " fsd fs2, 0x88(a0)\n" + " fsd fs3, 0x90(a0)\n" + " fsd fs4, 0x98(a0)\n" + " fsd fs5, 0xa0(a0)\n" + " fsd fs6, 0xa8(a0)\n" + " fsd fs7, 0xb0(a0)\n" + " fsd fs8, 0xb8(a0)\n" + " fsd fs9, 0xc0(a0)\n" + " fsd fs10, 0xc8(a0)\n" + " fsd fs11, 0xd0(a0)\n" + " fld fs0, 0x78(a1)\n" + " fld fs1, 0x80(a1)\n" + " fld fs2, 0x88(a1)\n" + " fld fs3, 0x90(a1)\n" + " fld fs4, 0x98(a1)\n" + " fld fs5, 0xa0(a1)\n" + " fld fs6, 0xa8(a1)\n" + " fld fs7, 0xb0(a1)\n" + " fld fs8, 0xb8(a1)\n" + " fld fs9, 0xc0(a1)\n" + " fld fs10, 0xc8(a1)\n" + " fld fs11, 0xd0(a1)\n" +#else +#error "Unsupported RISC-V FLEN" +#endif +#endif /* __riscv_flen */ + " ld s0, 0x00(a1)\n" + " ld s1, 0x08(a1)\n" + " ld s2, 0x10(a1)\n" + " ld s3, 0x18(a1)\n" + " ld s4, 0x20(a1)\n" + " ld s5, 0x28(a1)\n" + " ld s6, 0x30(a1)\n" + " ld s7, 0x38(a1)\n" + " ld s8, 0x40(a1)\n" + " ld s9, 0x48(a1)\n" + " ld s10, 0x50(a1)\n" + " ld s11, 0x58(a1)\n" + " ld ra, 0x60(a1)\n" + " ld a2, 0x68(a1)\n" /* pc */ + " ld sp, 0x70(a1)\n" + " jr a2\n" +#elif __riscv_xlen == 32 + " sw s0, 0x00(a0)\n" + " sw s1, 0x04(a0)\n" + " sw s2, 0x08(a0)\n" + " sw s3, 0x0c(a0)\n" + " sw s4, 0x10(a0)\n" + " sw s5, 0x14(a0)\n" + " sw s6, 0x18(a0)\n" + " sw s7, 0x1c(a0)\n" + " sw s8, 0x20(a0)\n" + " sw s9, 0x24(a0)\n" + " sw s10, 0x28(a0)\n" + " sw s11, 0x2c(a0)\n" + " sw ra, 0x30(a0)\n" + " sw ra, 0x34(a0)\n" /* pc */ + " sw sp, 0x38(a0)\n" +#ifdef __riscv_flen +#if __riscv_flen == 64 + " fsd fs0, 0x3c(a0)\n" + " fsd fs1, 0x44(a0)\n" + " fsd fs2, 0x4c(a0)\n" + " fsd fs3, 0x54(a0)\n" + " fsd fs4, 0x5c(a0)\n" + " fsd fs5, 0x64(a0)\n" + " fsd fs6, 0x6c(a0)\n" + " fsd fs7, 0x74(a0)\n" + " fsd fs8, 0x7c(a0)\n" + " fsd fs9, 0x84(a0)\n" + " fsd fs10, 0x8c(a0)\n" + " fsd fs11, 0x94(a0)\n" + " fld fs0, 0x3c(a1)\n" + " fld fs1, 0x44(a1)\n" + " fld fs2, 0x4c(a1)\n" + " fld fs3, 0x54(a1)\n" + " fld fs4, 0x5c(a1)\n" + " fld fs5, 0x64(a1)\n" + " fld fs6, 0x6c(a1)\n" + " fld fs7, 0x74(a1)\n" + " fld fs8, 0x7c(a1)\n" + " fld fs9, 0x84(a1)\n" + " fld fs10, 0x8c(a1)\n" + " fld fs11, 0x94(a1)\n" +#elif __riscv_flen == 32 + " fsw fs0, 0x3c(a0)\n" + " fsw fs1, 0x40(a0)\n" + " fsw fs2, 0x44(a0)\n" + " fsw fs3, 0x48(a0)\n" + " fsw fs4, 0x4c(a0)\n" + " fsw fs5, 0x50(a0)\n" + " fsw fs6, 0x54(a0)\n" + " fsw fs7, 0x58(a0)\n" + " fsw fs8, 0x5c(a0)\n" + " fsw fs9, 0x60(a0)\n" + " fsw fs10, 0x64(a0)\n" + " fsw fs11, 0x68(a0)\n" + " flw fs0, 0x3c(a1)\n" + " flw fs1, 0x40(a1)\n" + " flw fs2, 0x44(a1)\n" + " flw fs3, 0x48(a1)\n" + " flw fs4, 0x4c(a1)\n" + " flw fs5, 0x50(a1)\n" + " flw fs6, 0x54(a1)\n" + " flw fs7, 0x58(a1)\n" + " flw fs8, 0x5c(a1)\n" + " flw fs9, 0x60(a1)\n" + " flw fs10, 0x64(a1)\n" + " flw fs11, 0x68(a1)\n" +#else +#error "Unsupported RISC-V FLEN" +#endif +#endif /* __riscv_flen */ + " lw s0, 0x00(a1)\n" + " lw s1, 0x04(a1)\n" + " lw s2, 0x08(a1)\n" + " lw s3, 0x0c(a1)\n" + " lw s4, 0x10(a1)\n" + " lw s5, 0x14(a1)\n" + " lw s6, 0x18(a1)\n" + " lw s7, 0x1c(a1)\n" + " lw s8, 0x20(a1)\n" + " lw s9, 0x24(a1)\n" + " lw s10, 0x28(a1)\n" + " lw s11, 0x2c(a1)\n" + " lw ra, 0x30(a1)\n" + " lw a2, 0x34(a1)\n" /* pc */ + " lw sp, 0x38(a1)\n" + " jr a2\n" +#else +#error "Unsupported RISC-V XLEN" +#endif /* __riscv_xlen */ + ".size _mco_switch, .-_mco_switch\n"); + +static mco_result _mco_makectx(mco_coro *co, _mco_ctxbuf *ctx, void *stack_base, size_t stack_size) { + ctx->s[0] = (void *)(co); + ctx->s[1] = (void *)(_mco_main); + ctx->pc = (void *)(_mco_wrap_main); #if __riscv_xlen == 64 - ctx->ra = (void*)(0xdeaddeaddeaddead); + ctx->ra = (void *)(0xdeaddeaddeaddead); #elif __riscv_xlen == 32 - ctx->ra = (void*)(0xdeaddead); + ctx->ra = (void *)(0xdeaddead); #endif - ctx->sp = (void*)((size_t)stack_base + stack_size); + ctx->sp = (void *)((size_t)stack_base + stack_size); return MCO_SUCCESS; } @@ -1051,49 +1085,49 @@ typedef struct _mco_ctxbuf { void *eip, *esp, *ebp, *ebx, *esi, *edi; } _mco_ctxbuf; -void _mco_switch(_mco_ctxbuf* from, _mco_ctxbuf* to); +void _mco_switch(_mco_ctxbuf *from, _mco_ctxbuf *to); __asm__( #ifdef __DJGPP__ /* DOS compiler */ - "__mco_switch:\n" + "__mco_switch:\n" #else - ".text\n" - ".globl _mco_switch\n" - ".type _mco_switch @function\n" - ".hidden _mco_switch\n" - "_mco_switch:\n" -#endif - " call 1f\n" - " 1:\n" - " popl %ecx\n" - " addl $(2f-1b), %ecx\n" - " movl 4(%esp), %eax\n" - " movl 8(%esp), %edx\n" - " movl %ecx, (%eax)\n" - " movl %esp, 4(%eax)\n" - " movl %ebp, 8(%eax)\n" - " movl %ebx, 12(%eax)\n" - " movl %esi, 16(%eax)\n" - " movl %edi, 20(%eax)\n" - " movl 20(%edx), %edi\n" - " movl 16(%edx), %esi\n" - " movl 12(%edx), %ebx\n" - " movl 8(%edx), %ebp\n" - " movl 4(%edx), %esp\n" - " jmp *(%edx)\n" - " 2:\n" - " ret\n" + ".text\n" + ".globl _mco_switch\n" + ".type _mco_switch @function\n" + ".hidden _mco_switch\n" + "_mco_switch:\n" +#endif + " call 1f\n" + " 1:\n" + " popl %ecx\n" + " addl $(2f-1b), %ecx\n" + " movl 4(%esp), %eax\n" + " movl 8(%esp), %edx\n" + " movl %ecx, (%eax)\n" + " movl %esp, 4(%eax)\n" + " movl %ebp, 8(%eax)\n" + " movl %ebx, 12(%eax)\n" + " movl %esi, 16(%eax)\n" + " movl %edi, 20(%eax)\n" + " movl 20(%edx), %edi\n" + " movl 16(%edx), %esi\n" + " movl 12(%edx), %ebx\n" + " movl 8(%edx), %ebp\n" + " movl 4(%edx), %esp\n" + " jmp *(%edx)\n" + " 2:\n" + " ret\n" #ifndef __DJGPP__ - ".size _mco_switch, .-_mco_switch\n" + ".size _mco_switch, .-_mco_switch\n" #endif ); -static mco_result _mco_makectx(mco_coro* co, _mco_ctxbuf* ctx, void* stack_base, size_t stack_size) { - void** stack_high_ptr = (void**)((size_t)stack_base + stack_size - 16 - 1*sizeof(size_t)); - stack_high_ptr[0] = (void*)(0xdeaddead); /* Dummy return address. */ - stack_high_ptr[1] = (void*)(co); - ctx->eip = (void*)(_mco_main); - ctx->esp = (void*)(stack_high_ptr); +static mco_result _mco_makectx(mco_coro *co, _mco_ctxbuf *ctx, void *stack_base, size_t stack_size) { + void **stack_high_ptr = (void **)((size_t)stack_base + stack_size - 16 - 1 * sizeof(size_t)); + stack_high_ptr[0] = (void *)(0xdeaddead); /* Dummy return address. */ + stack_high_ptr[1] = (void *)(co); + ctx->eip = (void *)(_mco_main); + ctx->esp = (void *)(stack_high_ptr); return MCO_SUCCESS; } @@ -1101,7 +1135,7 @@ static mco_result _mco_makectx(mco_coro* co, _mco_ctxbuf* ctx, void* stack_base, typedef struct _mco_ctxbuf { #ifndef __SOFTFP__ - void* f[16]; + void *f[16]; #endif void *d[4]; /* d8-d15 */ void *r[4]; /* r4-r11 */ @@ -1110,60 +1144,58 @@ typedef struct _mco_ctxbuf { } _mco_ctxbuf; void _mco_wrap_main(void); -int _mco_switch(_mco_ctxbuf* from, _mco_ctxbuf* to); +int _mco_switch(_mco_ctxbuf *from, _mco_ctxbuf *to); -__asm__( - ".text\n" +__asm__(".text\n" #ifdef __APPLE__ - ".globl __mco_switch\n" - "__mco_switch:\n" + ".globl __mco_switch\n" + "__mco_switch:\n" #else - ".globl _mco_switch\n" - ".type _mco_switch #function\n" - ".hidden _mco_switch\n" - "_mco_switch:\n" + ".globl _mco_switch\n" + ".type _mco_switch #function\n" + ".hidden _mco_switch\n" + "_mco_switch:\n" #endif #ifndef __SOFTFP__ - " vstmia r0!, {d8-d15}\n" + " vstmia r0!, {d8-d15}\n" #endif - " stmia r0, {r4-r11, lr}\n" - " str sp, [r0, #9*4]\n" + " stmia r0, {r4-r11, lr}\n" + " str sp, [r0, #9*4]\n" #ifndef __SOFTFP__ - " vldmia r1!, {d8-d15}\n" + " vldmia r1!, {d8-d15}\n" #endif - " ldr sp, [r1, #9*4]\n" - " ldmia r1, {r4-r11, pc}\n" + " ldr sp, [r1, #9*4]\n" + " ldmia r1, {r4-r11, pc}\n" #ifndef __APPLE__ - ".size _mco_switch, .-_mco_switch\n" + ".size _mco_switch, .-_mco_switch\n" #endif ); -__asm__( - ".text\n" +__asm__(".text\n" #ifdef __APPLE__ - ".globl __mco_wrap_main\n" - "__mco_wrap_main:\n" + ".globl __mco_wrap_main\n" + "__mco_wrap_main:\n" #else - ".globl _mco_wrap_main\n" - ".type _mco_wrap_main #function\n" - ".hidden _mco_wrap_main\n" - "_mco_wrap_main:\n" -#endif - " mov r0, r4\n" - " mov ip, r5\n" - " mov lr, r6\n" - " bx ip\n" + ".globl _mco_wrap_main\n" + ".type _mco_wrap_main #function\n" + ".hidden _mco_wrap_main\n" + "_mco_wrap_main:\n" +#endif + " mov r0, r4\n" + " mov ip, r5\n" + " mov lr, r6\n" + " bx ip\n" #ifndef __APPLE__ - ".size _mco_wrap_main, .-_mco_wrap_main\n" + ".size _mco_wrap_main, .-_mco_wrap_main\n" #endif ); -static mco_result _mco_makectx(mco_coro* co, _mco_ctxbuf* ctx, void* stack_base, size_t stack_size) { - ctx->d[0] = (void*)(co); - ctx->d[1] = (void*)(_mco_main); - ctx->d[2] = (void*)(0xdeaddead); /* Dummy return address. */ - ctx->lr = (void*)(_mco_wrap_main); - ctx->sp = (void*)((size_t)stack_base + stack_size); +static mco_result _mco_makectx(mco_coro *co, _mco_ctxbuf *ctx, void *stack_base, size_t stack_size) { + ctx->d[0] = (void *)(co); + ctx->d[1] = (void *)(_mco_main); + ctx->d[2] = (void *)(0xdeaddead); /* Dummy return address. */ + ctx->lr = (void *)(_mco_wrap_main); + ctx->sp = (void *)((size_t)stack_base + stack_size); return MCO_SUCCESS; } @@ -1177,76 +1209,74 @@ typedef struct _mco_ctxbuf { } _mco_ctxbuf; void _mco_wrap_main(void); -int _mco_switch(_mco_ctxbuf* from, _mco_ctxbuf* to); +int _mco_switch(_mco_ctxbuf *from, _mco_ctxbuf *to); -__asm__( - ".text\n" +__asm__(".text\n" #ifdef __APPLE__ - ".globl __mco_switch\n" - "__mco_switch:\n" + ".globl __mco_switch\n" + "__mco_switch:\n" #else - ".globl _mco_switch\n" - ".type _mco_switch #function\n" - ".hidden _mco_switch\n" - "_mco_switch:\n" -#endif - - " mov x10, sp\n" - " mov x11, x30\n" - " stp x19, x20, [x0, #(0*16)]\n" - " stp x21, x22, [x0, #(1*16)]\n" - " stp d8, d9, [x0, #(7*16)]\n" - " stp x23, x24, [x0, #(2*16)]\n" - " stp d10, d11, [x0, #(8*16)]\n" - " stp x25, x26, [x0, #(3*16)]\n" - " stp d12, d13, [x0, #(9*16)]\n" - " stp x27, x28, [x0, #(4*16)]\n" - " stp d14, d15, [x0, #(10*16)]\n" - " stp x29, x30, [x0, #(5*16)]\n" - " stp x10, x11, [x0, #(6*16)]\n" - " ldp x19, x20, [x1, #(0*16)]\n" - " ldp x21, x22, [x1, #(1*16)]\n" - " ldp d8, d9, [x1, #(7*16)]\n" - " ldp x23, x24, [x1, #(2*16)]\n" - " ldp d10, d11, [x1, #(8*16)]\n" - " ldp x25, x26, [x1, #(3*16)]\n" - " ldp d12, d13, [x1, #(9*16)]\n" - " ldp x27, x28, [x1, #(4*16)]\n" - " ldp d14, d15, [x1, #(10*16)]\n" - " ldp x29, x30, [x1, #(5*16)]\n" - " ldp x10, x11, [x1, #(6*16)]\n" - " mov sp, x10\n" - " br x11\n" + ".globl _mco_switch\n" + ".type _mco_switch #function\n" + ".hidden _mco_switch\n" + "_mco_switch:\n" +#endif + + " mov x10, sp\n" + " mov x11, x30\n" + " stp x19, x20, [x0, #(0*16)]\n" + " stp x21, x22, [x0, #(1*16)]\n" + " stp d8, d9, [x0, #(7*16)]\n" + " stp x23, x24, [x0, #(2*16)]\n" + " stp d10, d11, [x0, #(8*16)]\n" + " stp x25, x26, [x0, #(3*16)]\n" + " stp d12, d13, [x0, #(9*16)]\n" + " stp x27, x28, [x0, #(4*16)]\n" + " stp d14, d15, [x0, #(10*16)]\n" + " stp x29, x30, [x0, #(5*16)]\n" + " stp x10, x11, [x0, #(6*16)]\n" + " ldp x19, x20, [x1, #(0*16)]\n" + " ldp x21, x22, [x1, #(1*16)]\n" + " ldp d8, d9, [x1, #(7*16)]\n" + " ldp x23, x24, [x1, #(2*16)]\n" + " ldp d10, d11, [x1, #(8*16)]\n" + " ldp x25, x26, [x1, #(3*16)]\n" + " ldp d12, d13, [x1, #(9*16)]\n" + " ldp x27, x28, [x1, #(4*16)]\n" + " ldp d14, d15, [x1, #(10*16)]\n" + " ldp x29, x30, [x1, #(5*16)]\n" + " ldp x10, x11, [x1, #(6*16)]\n" + " mov sp, x10\n" + " br x11\n" #ifndef __APPLE__ - ".size _mco_switch, .-_mco_switch\n" + ".size _mco_switch, .-_mco_switch\n" #endif ); -__asm__( - ".text\n" +__asm__(".text\n" #ifdef __APPLE__ - ".globl __mco_wrap_main\n" - "__mco_wrap_main:\n" + ".globl __mco_wrap_main\n" + "__mco_wrap_main:\n" #else - ".globl _mco_wrap_main\n" - ".type _mco_wrap_main #function\n" - ".hidden _mco_wrap_main\n" - "_mco_wrap_main:\n" -#endif - " mov x0, x19\n" - " mov x30, x21\n" - " br x20\n" + ".globl _mco_wrap_main\n" + ".type _mco_wrap_main #function\n" + ".hidden _mco_wrap_main\n" + "_mco_wrap_main:\n" +#endif + " mov x0, x19\n" + " mov x30, x21\n" + " br x20\n" #ifndef __APPLE__ - ".size _mco_wrap_main, .-_mco_wrap_main\n" + ".size _mco_wrap_main, .-_mco_wrap_main\n" #endif ); -static mco_result _mco_makectx(mco_coro* co, _mco_ctxbuf* ctx, void* stack_base, size_t stack_size) { - ctx->x[0] = (void*)(co); - ctx->x[1] = (void*)(_mco_main); - ctx->x[2] = (void*)(0xdeaddeaddeaddead); /* Dummy return address. */ - ctx->sp = (void*)((size_t)stack_base + stack_size); - ctx->lr = (void*)(_mco_wrap_main); +static mco_result _mco_makectx(mco_coro *co, _mco_ctxbuf *ctx, void *stack_base, size_t stack_size) { + ctx->x[0] = (void *)(co); + ctx->x[1] = (void *)(_mco_main); + ctx->x[2] = (void *)(0xdeaddeaddeaddead); /* Dummy return address. */ + ctx->sp = (void *)((size_t)stack_base + stack_size); + ctx->lr = (void *)(_mco_wrap_main); return MCO_SUCCESS; } @@ -1264,34 +1294,34 @@ typedef ucontext_t _mco_ctxbuf; #if defined(_LP64) || defined(__LP64__) static void _mco_wrap_main(unsigned int lo, unsigned int hi) { - mco_coro* co = (mco_coro*)(((size_t)lo) | (((size_t)hi) << 32)); /* Extract coroutine pointer. */ + mco_coro *co = (mco_coro *)(((size_t)lo) | (((size_t)hi) << 32)); /* Extract coroutine pointer. */ _mco_main(co); } #else static void _mco_wrap_main(unsigned int lo) { - mco_coro* co = (mco_coro*)((size_t)lo); /* Extract coroutine pointer. */ + mco_coro *co = (mco_coro *)((size_t)lo); /* Extract coroutine pointer. */ _mco_main(co); } #endif -static MCO_FORCE_INLINE void _mco_switch(_mco_ctxbuf* from, _mco_ctxbuf* to) { +static MCO_FORCE_INLINE void _mco_switch(_mco_ctxbuf *from, _mco_ctxbuf *to) { int res = swapcontext(from, to); _MCO_UNUSED(res); MCO_ASSERT(res == 0); } -static mco_result _mco_makectx(mco_coro* co, _mco_ctxbuf* ctx, void* stack_base, size_t stack_size) { +static mco_result _mco_makectx(mco_coro *co, _mco_ctxbuf *ctx, void *stack_base, size_t stack_size) { /* Initialize ucontext. */ - if(getcontext(ctx) != 0) { + if (getcontext(ctx) != 0) { MCO_LOG("failed to get ucontext"); return MCO_MAKE_CONTEXT_ERROR; } - ctx->uc_link = NULL; /* We never exit from _mco_wrap_main. */ + ctx->uc_link = NULL; /* We never exit from _mco_wrap_main. */ ctx->uc_stack.ss_sp = stack_base; ctx->uc_stack.ss_size = stack_size; unsigned int lo = (unsigned int)((size_t)co); #if defined(_LP64) || defined(__LP64__) - unsigned int hi = (unsigned int)(((size_t)co)>>32); + unsigned int hi = (unsigned int)(((size_t)co) >> 32); makecontext(ctx, (void (*)(void))_mco_wrap_main, 2, lo, hi); #else makecontext(ctx, (void (*)(void))_mco_wrap_main, 1, lo); @@ -1313,37 +1343,35 @@ typedef struct _mco_context { _mco_ctxbuf back_ctx; } _mco_context; -static void _mco_jumpin(mco_coro* co) { - _mco_context* context = (_mco_context*)co->context; +static void _mco_jumpin(mco_coro *co) { + _mco_context *context = (_mco_context *)co->context; _mco_prepare_jumpin(co); _mco_switch(&context->back_ctx, &context->ctx); /* Do the context switch. */ } -static void _mco_jumpout(mco_coro* co) { - _mco_context* context = (_mco_context*)co->context; +static void _mco_jumpout(mco_coro *co) { + _mco_context *context = (_mco_context *)co->context; _mco_prepare_jumpout(co); _mco_switch(&context->ctx, &context->back_ctx); /* Do the context switch. */ } -static mco_result _mco_create_context(mco_coro* co, mco_desc* desc) { +static mco_result _mco_create_context(mco_coro *co, mco_desc *desc) { /* Determine the context and stack address. */ size_t co_addr = (size_t)co; size_t context_addr = _mco_align_forward(co_addr + sizeof(mco_coro), 16); size_t storage_addr = _mco_align_forward(context_addr + sizeof(_mco_context), 16); size_t stack_addr = _mco_align_forward(storage_addr + desc->storage_size, 16); /* Initialize context. */ - _mco_context* context = (_mco_context*)context_addr; + _mco_context *context = (_mco_context *)context_addr; memset(context, 0, sizeof(_mco_context)); /* Initialize storage. */ - unsigned char* storage = (unsigned char*)storage_addr; + unsigned char *storage = (unsigned char *)storage_addr; /* Initialize stack. */ - void *stack_base = (void*)stack_addr; + void *stack_base = (void *)stack_addr; size_t stack_size = desc->stack_size; /* Make the context. */ mco_result res = _mco_makectx(co, &context->ctx, stack_base, stack_size); - if(res != MCO_SUCCESS) { - return res; - } + if (res != MCO_SUCCESS) { return res; } #ifdef MCO_USE_VALGRIND context->valgrind_stack_id = VALGRIND_STACK_REGISTER(stack_addr, stack_addr + stack_size); #endif @@ -1355,10 +1383,10 @@ static mco_result _mco_create_context(mco_coro* co, mco_desc* desc) { return MCO_SUCCESS; } -static void _mco_destroy_context(mco_coro* co) { +static void _mco_destroy_context(mco_coro *co) { #ifdef MCO_USE_VALGRIND - _mco_context* context = (_mco_context*)co->context; - if(context && context->valgrind_stack_id != 0) { + _mco_context *context = (_mco_context *)co->context; + if (context && context->valgrind_stack_id != 0) { VALGRIND_STACK_DEREGISTER(context->valgrind_stack_id); context->valgrind_stack_id = 0; } @@ -1367,46 +1395,47 @@ static void _mco_destroy_context(mco_coro* co) { #endif } -static MCO_FORCE_INLINE void _mco_init_desc_sizes(mco_desc* desc, size_t stack_size) { - desc->coro_size = _mco_align_forward(sizeof(mco_coro), 16) + - _mco_align_forward(sizeof(_mco_context), 16) + - _mco_align_forward(desc->storage_size, 16) + - stack_size + 16; +static MCO_FORCE_INLINE void _mco_init_desc_sizes(mco_desc *desc, size_t stack_size) { + desc->coro_size = _mco_align_forward(sizeof(mco_coro), 16) + _mco_align_forward(sizeof(_mco_context), 16) + + _mco_align_forward(desc->storage_size, 16) + stack_size + 16; desc->stack_size = stack_size; /* This is just a hint, it won't be the real one. */ } #endif /* defined(MCO_USE_UCONTEXT) || defined(MCO_USE_ASM) */ -/* ---------------------------------------------------------------------------------------------- */ +/* ---------------------------------------------------------------------------------------------- + */ #ifdef MCO_USE_FIBERS #ifdef _WIN32 typedef struct _mco_context { - void* fib; - void* back_fib; + void *fib; + void *back_fib; } _mco_context; -static void _mco_jumpin(mco_coro* co) { +static void _mco_jumpin(mco_coro *co) { void *cur_fib = GetCurrentFiber(); - if(!cur_fib || cur_fib == (void*)0x1e00) { /* See http://blogs.msdn.com/oldnewthing/archive/2004/12/31/344799.aspx */ + if (!cur_fib || cur_fib == (void *)0x1e00) { /* See + http://blogs.msdn.com/oldnewthing/archive/2004/12/31/344799.aspx + */ cur_fib = ConvertThreadToFiber(NULL); } MCO_ASSERT(cur_fib != NULL); - _mco_context* context = (_mco_context*)co->context; + _mco_context *context = (_mco_context *)co->context; context->back_fib = cur_fib; _mco_prepare_jumpin(co); SwitchToFiber(context->fib); } -static void CALLBACK _mco_wrap_main(void* co) { - _mco_main((mco_coro*)co); +static void CALLBACK _mco_wrap_main(void *co) { + _mco_main((mco_coro *)co); } -static void _mco_jumpout(mco_coro* co) { - _mco_context* context = (_mco_context*)co->context; - void* back_fib = context->back_fib; +static void _mco_jumpout(mco_coro *co) { + _mco_context *context = (_mco_context *)co->context; + void *back_fib = context->back_fib; MCO_ASSERT(back_fib != NULL); context->back_fib = NULL; _mco_prepare_jumpout(co); @@ -1416,54 +1445,53 @@ static void _mco_jumpout(mco_coro* co) { /* Reverse engineered Fiber struct, used to get stack base. */ typedef struct _mco_fiber { LPVOID param; /* fiber param */ - void* except; /* saved exception handlers list */ - void* stack_base; /* top of fiber stack */ - void* stack_limit; /* fiber stack low-water mark */ - void* stack_allocation; /* base of the fiber stack allocation */ + void *except; /* saved exception handlers list */ + void *stack_base; /* top of fiber stack */ + void *stack_limit; /* fiber stack low-water mark */ + void *stack_allocation; /* base of the fiber stack allocation */ CONTEXT context; /* fiber context */ DWORD flags; /* fiber flags */ LPFIBER_START_ROUTINE start; /* start routine */ void **fls_slots; /* fiber storage slots */ } _mco_fiber; -static mco_result _mco_create_context(mco_coro* co, mco_desc* desc) { +static mco_result _mco_create_context(mco_coro *co, mco_desc *desc) { /* Determine the context address. */ size_t co_addr = (size_t)co; size_t context_addr = _mco_align_forward(co_addr + sizeof(mco_coro), 16); size_t storage_addr = _mco_align_forward(context_addr + sizeof(_mco_context), 16); /* Initialize context. */ - _mco_context* context = (_mco_context*)context_addr; + _mco_context *context = (_mco_context *)context_addr; memset(context, 0, sizeof(_mco_context)); /* Initialize storage. */ - unsigned char* storage = (unsigned char*)storage_addr; + unsigned char *storage = (unsigned char *)storage_addr; /* Create the fiber. */ - _mco_fiber* fib = (_mco_fiber*)CreateFiberEx(desc->stack_size, desc->stack_size, FIBER_FLAG_FLOAT_SWITCH, _mco_wrap_main, co); - if(!fib) { + _mco_fiber *fib = + (_mco_fiber *)CreateFiberEx(desc->stack_size, desc->stack_size, FIBER_FLAG_FLOAT_SWITCH, _mco_wrap_main, co); + if (!fib) { MCO_LOG("failed to create fiber"); return MCO_MAKE_CONTEXT_ERROR; } context->fib = fib; co->context = context; - co->stack_base = (void*)((size_t)fib->stack_base - desc->stack_size); + co->stack_base = (void *)((size_t)fib->stack_base - desc->stack_size); co->stack_size = desc->stack_size; co->storage = storage; co->storage_size = desc->storage_size; return MCO_SUCCESS; } -static void _mco_destroy_context(mco_coro* co) { - _mco_context* context = (_mco_context*)co->context; - if(context && context->fib) { +static void _mco_destroy_context(mco_coro *co) { + _mco_context *context = (_mco_context *)co->context; + if (context && context->fib) { DeleteFiber(context->fib); context->fib = NULL; } } -static MCO_FORCE_INLINE void _mco_init_desc_sizes(mco_desc* desc, size_t stack_size) { - desc->coro_size = _mco_align_forward(sizeof(mco_coro), 16) + - _mco_align_forward(sizeof(_mco_context), 16) + - _mco_align_forward(desc->storage_size, 16) + - 16; +static MCO_FORCE_INLINE void _mco_init_desc_sizes(mco_desc *desc, size_t stack_size) { + desc->coro_size = _mco_align_forward(sizeof(mco_coro), 16) + _mco_align_forward(sizeof(_mco_context), 16) + + _mco_align_forward(desc->storage_size, 16) + 16; desc->stack_size = stack_size; } @@ -1477,21 +1505,21 @@ static MCO_FORCE_INLINE void _mco_init_desc_sizes(mco_desc* desc, size_t stack_s typedef struct _mco_context { emscripten_fiber_t fib; - emscripten_fiber_t* back_fib; + emscripten_fiber_t *back_fib; } _mco_context; -static emscripten_fiber_t* running_fib = NULL; +static emscripten_fiber_t *running_fib = NULL; static unsigned char main_asyncify_stack[MCO_ASYNCFY_STACK_SIZE]; static emscripten_fiber_t main_fib; -static void _mco_wrap_main(void* co) { - _mco_main((mco_coro*)co); +static void _mco_wrap_main(void *co) { + _mco_main((mco_coro *)co); } -static void _mco_jumpin(mco_coro* co) { - _mco_context* context = (_mco_context*)co->context; - emscripten_fiber_t* back_fib = running_fib; - if(!back_fib) { +static void _mco_jumpin(mco_coro *co) { + _mco_context *context = (_mco_context *)co->context; + emscripten_fiber_t *back_fib = running_fib; + if (!back_fib) { back_fib = &main_fib; emscripten_fiber_init_from_current_context(back_fib, main_asyncify_stack, MCO_ASYNCFY_STACK_SIZE); } @@ -1501,15 +1529,15 @@ static void _mco_jumpin(mco_coro* co) { emscripten_fiber_swap(back_fib, &context->fib); /* Do the context switch. */ } -static void _mco_jumpout(mco_coro* co) { - _mco_context* context = (_mco_context*)co->context; +static void _mco_jumpout(mco_coro *co) { + _mco_context *context = (_mco_context *)co->context; running_fib = context->back_fib; _mco_prepare_jumpout(co); emscripten_fiber_swap(&context->fib, context->back_fib); /* Do the context switch. */ } -static mco_result _mco_create_context(mco_coro* co, mco_desc* desc) { - if(emscripten_has_asyncify() != 1) { +static mco_result _mco_create_context(mco_coro *co, mco_desc *desc) { + if (emscripten_has_asyncify() != 1) { MCO_LOG("failed to create fiber because ASYNCIFY is not enabled"); return MCO_MAKE_CONTEXT_ERROR; } @@ -1520,17 +1548,18 @@ static mco_result _mco_create_context(mco_coro* co, mco_desc* desc) { size_t stack_addr = _mco_align_forward(storage_addr + desc->storage_size, 16); size_t asyncify_stack_addr = _mco_align_forward(stack_addr + desc->stack_size, 16); /* Initialize context. */ - _mco_context* context = (_mco_context*)context_addr; + _mco_context *context = (_mco_context *)context_addr; memset(context, 0, sizeof(_mco_context)); /* Initialize storage. */ - unsigned char* storage = (unsigned char*)storage_addr; + unsigned char *storage = (unsigned char *)storage_addr; /* Initialize stack. */ - void *stack_base = (void*)stack_addr; + void *stack_base = (void *)stack_addr; size_t stack_size = asyncify_stack_addr - stack_addr; - void *asyncify_stack_base = (void*)asyncify_stack_addr; + void *asyncify_stack_base = (void *)asyncify_stack_addr; size_t asyncify_stack_size = co_addr + desc->coro_size - asyncify_stack_addr; /* Create the fiber. */ - emscripten_fiber_init(&context->fib, _mco_wrap_main, co, stack_base, stack_size, asyncify_stack_base, asyncify_stack_size); + emscripten_fiber_init(&context->fib, _mco_wrap_main, co, stack_base, stack_size, asyncify_stack_base, + asyncify_stack_size); co->context = context; co->stack_base = stack_base; co->stack_size = stack_size; @@ -1539,18 +1568,15 @@ static mco_result _mco_create_context(mco_coro* co, mco_desc* desc) { return MCO_SUCCESS; } -static void _mco_destroy_context(mco_coro* co) { +static void _mco_destroy_context(mco_coro *co) { /* Nothing to do. */ _MCO_UNUSED(co); } -static MCO_FORCE_INLINE void _mco_init_desc_sizes(mco_desc* desc, size_t stack_size) { - desc->coro_size = _mco_align_forward(sizeof(mco_coro), 16) + - _mco_align_forward(sizeof(_mco_context), 16) + - _mco_align_forward(desc->storage_size, 16) + - _mco_align_forward(stack_size, 16) + - _mco_align_forward(MCO_ASYNCFY_STACK_SIZE, 16) + - 16; +static MCO_FORCE_INLINE void _mco_init_desc_sizes(mco_desc *desc, size_t stack_size) { + desc->coro_size = _mco_align_forward(sizeof(mco_coro), 16) + _mco_align_forward(sizeof(_mco_context), 16) + + _mco_align_forward(desc->storage_size, 16) + _mco_align_forward(stack_size, 16) + + _mco_align_forward(MCO_ASYNCFY_STACK_SIZE, 16) + 16; desc->stack_size = stack_size; /* This is just a hint, it won't be the real one. */ } @@ -1562,13 +1588,14 @@ static MCO_FORCE_INLINE void _mco_init_desc_sizes(mco_desc* desc, size_t stack_s #endif /* MCO_USE_FIBERS */ -/* ---------------------------------------------------------------------------------------------- */ +/* ---------------------------------------------------------------------------------------------- + */ #ifdef MCO_USE_ASYNCIFY typedef struct _asyncify_stack_region { - void* start; - void* limit; + void *start; + void *limit; } _asyncify_stack_region; typedef struct _mco_context { @@ -1576,36 +1603,37 @@ typedef struct _mco_context { _asyncify_stack_region stack_region; } _mco_context; -__attribute__((import_module("asyncify"), import_name("start_unwind"))) void _asyncify_start_unwind(void*); -__attribute__((import_module("asyncify"), import_name("stop_unwind"))) void _asyncify_stop_unwind(); -__attribute__((import_module("asyncify"), import_name("start_rewind"))) void _asyncify_start_rewind(void*); -__attribute__((import_module("asyncify"), import_name("stop_rewind"))) void _asyncify_stop_rewind(); +__attribute__((import_module("asyncify"), import_name("start_unwind"))) void _asyncify_start_unwind(void *); +__attribute__((import_module("asyncify"), import_name("stop_unwind"))) void _asyncify_stop_unwind(); +__attribute__((import_module("asyncify"), import_name("start_rewind"))) void _asyncify_start_rewind(void *); +__attribute__((import_module("asyncify"), import_name("stop_rewind"))) void _asyncify_stop_rewind(); -MCO_NO_INLINE void _mco_jumpin(mco_coro* co) { - _mco_context* context = (_mco_context*)co->context; +MCO_NO_INLINE void _mco_jumpin(mco_coro *co) { + _mco_context *context = (_mco_context *)co->context; _mco_prepare_jumpin(co); - if(context->rewind_id > 0) { /* Begin rewinding until last yield point. */ + if (context->rewind_id > 0) { /* Begin rewinding until last yield point. */ _asyncify_start_rewind(&context->stack_region); } - _mco_main(co); /* Run the coroutine function. */ + _mco_main(co); /* Run the coroutine function. */ _asyncify_stop_unwind(); /* Stop saving coroutine stack. */ } -static MCO_NO_INLINE void _mco_finish_jumpout(mco_coro* co, volatile int rewind_id) { - _mco_context* context = (_mco_context*)co->context; +static MCO_NO_INLINE void _mco_finish_jumpout(mco_coro *co, volatile int rewind_id) { + _mco_context *context = (_mco_context *)co->context; int next_rewind_id = context->rewind_id + 1; - if(rewind_id == next_rewind_id) { /* Begins unwinding the stack (save locals and call stack to rewind later) */ + if (rewind_id == next_rewind_id) { /* Begins unwinding the stack (save locals and call + stack to rewind later) */ _mco_prepare_jumpout(co); context->rewind_id = next_rewind_id; _asyncify_start_unwind(&context->stack_region); - } else if(rewind_id == context->rewind_id) { /* Continue from yield point. */ + } else if (rewind_id == context->rewind_id) { /* Continue from yield point. */ _asyncify_stop_rewind(); } /* Otherwise, we should be rewinding, let it continue... */ } -MCO_NO_INLINE void _mco_jumpout(mco_coro* co) { - _mco_context* context = (_mco_context*)co->context; +MCO_NO_INLINE void _mco_jumpout(mco_coro *co) { + _mco_context *context = (_mco_context *)co->context; /* Save rewind point into a local, that should be restored when rewinding. That is "rewind_id != co->rewind_id + 1" may be true when rewinding. @@ -1615,22 +1643,22 @@ MCO_NO_INLINE void _mco_jumpout(mco_coro* co) { _mco_finish_jumpout(co, rewind_id); } -static mco_result _mco_create_context(mco_coro* co, mco_desc* desc) { +static mco_result _mco_create_context(mco_coro *co, mco_desc *desc) { /* Determine the context address. */ size_t co_addr = (size_t)co; size_t context_addr = _mco_align_forward(co_addr + sizeof(mco_coro), 16); size_t storage_addr = _mco_align_forward(context_addr + sizeof(_mco_context), 16); size_t stack_addr = _mco_align_forward(storage_addr + desc->storage_size, 16); /* Initialize context. */ - _mco_context* context = (_mco_context*)context_addr; + _mco_context *context = (_mco_context *)context_addr; memset(context, 0, sizeof(_mco_context)); /* Initialize storage. */ - unsigned char* storage = (unsigned char*)storage_addr; + unsigned char *storage = (unsigned char *)storage_addr; /* Initialize stack. */ - void *stack_base = (void*)stack_addr; + void *stack_base = (void *)stack_addr; size_t stack_size = desc->stack_size; context->stack_region.start = stack_base; - context->stack_region.limit = (void*)((size_t)stack_base + stack_size); + context->stack_region.limit = (void *)((size_t)stack_base + stack_size); co->context = context; co->stack_base = stack_base; co->stack_size = stack_size; @@ -1639,30 +1667,26 @@ static mco_result _mco_create_context(mco_coro* co, mco_desc* desc) { return MCO_SUCCESS; } -static void _mco_destroy_context(mco_coro* co) { +static void _mco_destroy_context(mco_coro *co) { /* Nothing to do. */ _MCO_UNUSED(co); } -static MCO_FORCE_INLINE void _mco_init_desc_sizes(mco_desc* desc, size_t stack_size) { - desc->coro_size = _mco_align_forward(sizeof(mco_coro), 16) + - _mco_align_forward(sizeof(_mco_context), 16) + - _mco_align_forward(desc->storage_size, 16) + - _mco_align_forward(stack_size, 16) + - 16; +static MCO_FORCE_INLINE void _mco_init_desc_sizes(mco_desc *desc, size_t stack_size) { + desc->coro_size = _mco_align_forward(sizeof(mco_coro), 16) + _mco_align_forward(sizeof(_mco_context), 16) + + _mco_align_forward(desc->storage_size, 16) + _mco_align_forward(stack_size, 16) + 16; desc->stack_size = stack_size; /* This is just a hint, it won't be the real one. */ } #endif /* MCO_USE_ASYNCIFY */ -/* ---------------------------------------------------------------------------------------------- */ +/* ---------------------------------------------------------------------------------------------- + */ -mco_desc mco_desc_init(void (*func)(mco_coro* co), size_t stack_size) { - if(stack_size != 0) { +mco_desc mco_desc_init(void (*func)(mco_coro *co), size_t stack_size) { + if (stack_size != 0) { /* Stack size should be at least `MCO_MIN_STACK_SIZE`. */ - if(stack_size < MCO_MIN_STACK_SIZE) { - stack_size = MCO_MIN_STACK_SIZE; - } + if (stack_size < MCO_MIN_STACK_SIZE) { stack_size = MCO_MIN_STACK_SIZE; } } else { stack_size = MCO_DEFAULT_STACK_SIZE; } @@ -1680,40 +1704,38 @@ mco_desc mco_desc_init(void (*func)(mco_coro* co), size_t stack_size) { return desc; } -static mco_result _mco_validate_desc(mco_desc* desc) { - if(!desc) { +static mco_result _mco_validate_desc(mco_desc *desc) { + if (!desc) { MCO_LOG("coroutine description is NULL"); return MCO_INVALID_ARGUMENTS; } - if(!desc->func) { + if (!desc->func) { MCO_LOG("coroutine function in invalid"); return MCO_INVALID_ARGUMENTS; } - if(desc->stack_size < MCO_MIN_STACK_SIZE) { + if (desc->stack_size < MCO_MIN_STACK_SIZE) { MCO_LOG("coroutine stack size is too small"); return MCO_INVALID_ARGUMENTS; } - if(desc->coro_size < sizeof(mco_coro)) { + if (desc->coro_size < sizeof(mco_coro)) { MCO_LOG("coroutine size is invalid"); return MCO_INVALID_ARGUMENTS; } return MCO_SUCCESS; } -mco_result mco_init(mco_coro* co, mco_desc* desc) { - if(!co) { +mco_result mco_init(mco_coro *co, mco_desc *desc) { + if (!co) { MCO_LOG("attempt to initialize an invalid coroutine"); return MCO_INVALID_COROUTINE; } memset(co, 0, sizeof(mco_coro)); /* Validate coroutine description. */ mco_result res = _mco_validate_desc(desc); - if(res != MCO_SUCCESS) - return res; + if (res != MCO_SUCCESS) return res; /* Create the coroutine. */ res = _mco_create_context(co, desc); - if(res != MCO_SUCCESS) - return res; + if (res != MCO_SUCCESS) return res; co->state = MCO_SUSPENDED; /* We initialize in suspended state. */ co->dealloc_cb = desc->dealloc_cb; co->coro_size = desc->coro_size; @@ -1727,20 +1749,20 @@ mco_result mco_init(mco_coro* co, mco_desc* desc) { return MCO_SUCCESS; } -mco_result mco_uninit(mco_coro* co) { - if(!co) { +mco_result mco_uninit(mco_coro *co) { + if (!co) { MCO_LOG("attempt to uninitialize an invalid coroutine"); return MCO_INVALID_COROUTINE; } /* Cannot uninitialize while running. */ - if(!(co->state == MCO_SUSPENDED || co->state == MCO_DEAD)) { + if (!(co->state == MCO_SUSPENDED || co->state == MCO_DEAD)) { MCO_LOG("attempt to uninitialize a coroutine that is not dead or suspended"); return MCO_INVALID_OPERATION; } /* The coroutine is now dead and cannot be used anymore. */ co->state = MCO_DEAD; #ifdef _MCO_USE_TSAN - if(co->tsan_fiber != NULL) { + if (co->tsan_fiber != NULL) { __tsan_destroy_fiber(co->tsan_fiber); co->tsan_fiber = NULL; } @@ -1749,27 +1771,27 @@ mco_result mco_uninit(mco_coro* co) { return MCO_SUCCESS; } -mco_result mco_create(mco_coro** out_co, mco_desc* desc) { +mco_result mco_create(mco_coro **out_co, mco_desc *desc) { /* Validate input. */ - if(!out_co) { + if (!out_co) { MCO_LOG("coroutine output pointer is NULL"); return MCO_INVALID_POINTER; } - if(!desc || !desc->alloc_cb || !desc->dealloc_cb) { + if (!desc || !desc->alloc_cb || !desc->dealloc_cb) { *out_co = NULL; MCO_LOG("coroutine allocator description is not set"); return MCO_INVALID_ARGUMENTS; } /* Allocate the coroutine. */ - mco_coro* co = (mco_coro*)desc->alloc_cb(desc->coro_size, desc->allocator_data); - if(!co) { + mco_coro *co = (mco_coro *)desc->alloc_cb(desc->coro_size, desc->allocator_data); + if (!co) { MCO_LOG("coroutine allocation failed"); *out_co = NULL; return MCO_OUT_OF_MEMORY; } /* Initialize the coroutine. */ mco_result res = mco_init(co, desc); - if(res != MCO_SUCCESS) { + if (res != MCO_SUCCESS) { desc->dealloc_cb(co, desc->coro_size, desc->allocator_data); *out_co = NULL; return res; @@ -1778,17 +1800,16 @@ mco_result mco_create(mco_coro** out_co, mco_desc* desc) { return MCO_SUCCESS; } -mco_result mco_destroy(mco_coro* co) { - if(!co) { +mco_result mco_destroy(mco_coro *co) { + if (!co) { MCO_LOG("attempt to destroy an invalid coroutine"); return MCO_INVALID_COROUTINE; } /* Uninitialize the coroutine first. */ mco_result res = mco_uninit(co); - if(res != MCO_SUCCESS) - return res; + if (res != MCO_SUCCESS) return res; /* Free the coroutine. */ - if(!co->dealloc_cb) { + if (!co->dealloc_cb) { MCO_LOG("attempt destroy a coroutine that has no free callback"); return MCO_INVALID_POINTER; } @@ -1796,12 +1817,12 @@ mco_result mco_destroy(mco_coro* co) { return MCO_SUCCESS; } -mco_result mco_resume(mco_coro* co) { - if(!co) { +mco_result mco_resume(mco_coro *co) { + if (!co) { MCO_LOG("attempt to resume an invalid coroutine"); return MCO_INVALID_COROUTINE; } - if(co->state != MCO_SUSPENDED) { /* Can only resume coroutines that are suspended. */ + if (co->state != MCO_SUSPENDED) { /* Can only resume coroutines that are suspended. */ MCO_LOG("attempt to resume a coroutine that is not suspended"); return MCO_NOT_SUSPENDED; } @@ -1810,25 +1831,26 @@ mco_result mco_resume(mco_coro* co) { return MCO_SUCCESS; } -mco_result mco_yield(mco_coro* co) { - if(!co) { +mco_result mco_yield(mco_coro *co) { + if (!co) { MCO_LOG("attempt to yield an invalid coroutine"); return MCO_INVALID_COROUTINE; } #ifdef MCO_USE_ASYNCIFY /* Asyncify already checks for stack overflow. */ #else - /* This check happens when the stack overflow already happened, but better later than never. */ + /* This check happens when the stack overflow already happened, but better later than + * never. */ volatile size_t dummy; size_t stack_addr = (size_t)&dummy; size_t stack_min = (size_t)co->stack_base; size_t stack_max = stack_min + co->stack_size; - if(co->magic_number != MCO_MAGIC_NUMBER || stack_addr < stack_min || stack_addr > stack_max) { /* Stack overflow. */ + if (co->magic_number != MCO_MAGIC_NUMBER || stack_addr < stack_min || stack_addr > stack_max) { /* Stack overflow. */ MCO_LOG("coroutine stack overflow, try increasing the stack size"); return MCO_STACK_OVERFLOW; } #endif - if(co->state != MCO_RUNNING) { /* Can only yield coroutines that are running. */ + if (co->state != MCO_RUNNING) { /* Can only yield coroutines that are running. */ MCO_LOG("attempt to yield a coroutine that is not running"); return MCO_NOT_RUNNING; } @@ -1837,31 +1859,27 @@ mco_result mco_yield(mco_coro* co) { return MCO_SUCCESS; } -mco_state mco_status(mco_coro* co) { - if(co != NULL) { - return co->state; - } +mco_state mco_status(mco_coro *co) { + if (co != NULL) { return co->state; } return MCO_DEAD; } -void* mco_get_user_data(mco_coro* co) { - if(co != NULL) { - return co->user_data; - } +void *mco_get_user_data(mco_coro *co) { + if (co != NULL) { return co->user_data; } return NULL; } -mco_result mco_push(mco_coro* co, const void* src, size_t len) { - if(!co) { +mco_result mco_push(mco_coro *co, const void *src, size_t len) { + if (!co) { MCO_LOG("attempt to use an invalid coroutine"); return MCO_INVALID_COROUTINE; - } else if(len > 0) { + } else if (len > 0) { size_t bytes_stored = co->bytes_stored + len; - if(bytes_stored > co->storage_size) { + if (bytes_stored > co->storage_size) { MCO_LOG("attempt to push too many bytes into coroutine storage"); return MCO_NOT_ENOUGH_SPACE; } - if(!src) { + if (!src) { MCO_LOG("attempt push a null pointer into coroutine storage"); return MCO_INVALID_POINTER; } @@ -1871,19 +1889,17 @@ mco_result mco_push(mco_coro* co, const void* src, size_t len) { return MCO_SUCCESS; } -mco_result mco_pop(mco_coro* co, void* dest, size_t len) { - if(!co) { +mco_result mco_pop(mco_coro *co, void *dest, size_t len) { + if (!co) { MCO_LOG("attempt to use an invalid coroutine"); return MCO_INVALID_COROUTINE; - } else if(len > 0) { - if(len > co->bytes_stored) { + } else if (len > 0) { + if (len > co->bytes_stored) { MCO_LOG("attempt to pop too many bytes from coroutine storage"); return MCO_NOT_ENOUGH_SPACE; } size_t bytes_stored = co->bytes_stored - len; - if(dest) { - memcpy(dest, &co->storage[bytes_stored], len); - } + if (dest) { memcpy(dest, &co->storage[bytes_stored], len); } co->bytes_stored = bytes_stored; #ifdef MCO_ZERO_MEMORY /* Clear garbage in the discarded storage. */ @@ -1893,16 +1909,16 @@ mco_result mco_pop(mco_coro* co, void* dest, size_t len) { return MCO_SUCCESS; } -mco_result mco_peek(mco_coro* co, void* dest, size_t len) { - if(!co) { +mco_result mco_peek(mco_coro *co, void *dest, size_t len) { + if (!co) { MCO_LOG("attempt to use an invalid coroutine"); return MCO_INVALID_COROUTINE; - } else if(len > 0) { - if(len > co->bytes_stored) { + } else if (len > 0) { + if (len > co->bytes_stored) { MCO_LOG("attempt to peek too many bytes from coroutine storage"); return MCO_NOT_ENOUGH_SPACE; } - if(!dest) { + if (!dest) { MCO_LOG("attempt peek into a null pointer"); return MCO_INVALID_POINTER; } @@ -1911,67 +1927,63 @@ mco_result mco_peek(mco_coro* co, void* dest, size_t len) { return MCO_SUCCESS; } -size_t mco_get_bytes_stored(mco_coro* co) { - if(co == NULL) { - return 0; - } +size_t mco_get_bytes_stored(mco_coro *co) { + if (co == NULL) { return 0; } return co->bytes_stored; } -size_t mco_get_storage_size(mco_coro* co) { - if(co == NULL) { - return 0; - } +size_t mco_get_storage_size(mco_coro *co) { + if (co == NULL) { return 0; } return co->storage_size; } #ifdef MCO_NO_MULTITHREAD -mco_coro* mco_running(void) { +mco_coro *mco_running(void) { return mco_current_co; } #else -static MCO_NO_INLINE mco_coro* _mco_running(void) { +static MCO_NO_INLINE mco_coro *_mco_running(void) { return mco_current_co; } -mco_coro* mco_running(void) { +mco_coro *mco_running(void) { /* Compilers aggressively optimize the use of TLS by caching loads. Since fiber code can migrate between threads it’s possible for the load to be stale. To prevent this from happening we avoid inline functions. */ - mco_coro* (*volatile func)(void) = _mco_running; + mco_coro *(*volatile func)(void) = _mco_running; return func(); } #endif -const char* mco_result_description(mco_result res) { - switch(res) { - case MCO_SUCCESS: - return "No error"; - case MCO_GENERIC_ERROR: - return "Generic error"; - case MCO_INVALID_POINTER: - return "Invalid pointer"; - case MCO_INVALID_COROUTINE: - return "Invalid coroutine"; - case MCO_NOT_SUSPENDED: - return "Coroutine not suspended"; - case MCO_NOT_RUNNING: - return "Coroutine not running"; - case MCO_MAKE_CONTEXT_ERROR: - return "Make context error"; - case MCO_SWITCH_CONTEXT_ERROR: - return "Switch context error"; - case MCO_NOT_ENOUGH_SPACE: - return "Not enough space"; - case MCO_OUT_OF_MEMORY: - return "Out of memory"; - case MCO_INVALID_ARGUMENTS: - return "Invalid arguments"; - case MCO_INVALID_OPERATION: - return "Invalid operation"; - case MCO_STACK_OVERFLOW: - return "Stack overflow"; +const char *mco_result_description(mco_result res) { + switch (res) { + case MCO_SUCCESS: + return "No error"; + case MCO_GENERIC_ERROR: + return "Generic error"; + case MCO_INVALID_POINTER: + return "Invalid pointer"; + case MCO_INVALID_COROUTINE: + return "Invalid coroutine"; + case MCO_NOT_SUSPENDED: + return "Coroutine not suspended"; + case MCO_NOT_RUNNING: + return "Coroutine not running"; + case MCO_MAKE_CONTEXT_ERROR: + return "Make context error"; + case MCO_SWITCH_CONTEXT_ERROR: + return "Switch context error"; + case MCO_NOT_ENOUGH_SPACE: + return "Not enough space"; + case MCO_OUT_OF_MEMORY: + return "Out of memory"; + case MCO_INVALID_ARGUMENTS: + return "Invalid arguments"; + case MCO_INVALID_OPERATION: + return "Invalid operation"; + case MCO_STACK_OVERFLOW: + return "Stack overflow"; } return "Unknown error"; } diff --git a/include/routes.h b/include/routes.h index d282f3f..174006f 100644 --- a/include/routes.h +++ b/include/routes.h @@ -3,8 +3,8 @@ #include "rpc/server.h" -#include #include +#include #define RPC_ROUTE_PAGE_BITS 16u #define RPC_ROUTE_PAGE_SIZE (1u << RPC_ROUTE_PAGE_BITS) @@ -36,10 +36,8 @@ typedef struct rpc_routes { int rpc_routes_init(rpc_routes *routes); void rpc_routes_destroy(rpc_routes *routes); -int rpc_routes_add(rpc_routes *routes, uint32_t proc_id, rpc_handler_fn handler, - void *user_data); -int rpc_routes_add_ex(rpc_routes *routes, uint32_t proc_id, - rpc_handler_fn handler, void *user_data, int is_async); +int rpc_routes_add(rpc_routes *routes, uint32_t proc_id, rpc_handler_fn handler, void *user_data); +int rpc_routes_add_ex(rpc_routes *routes, uint32_t proc_id, rpc_handler_fn handler, void *user_data, int is_async); int rpc_routes_remove(rpc_routes *routes, uint32_t proc_id); int rpc_routes_lookup(rpc_routes *routes, uint32_t proc_id, rpc_route *out); diff --git a/include/rpc/client.h b/include/rpc/client.h index c2204ab..82d8eda 100644 --- a/include/rpc/client.h +++ b/include/rpc/client.h @@ -9,18 +9,14 @@ extern "C" { typedef struct rpc_client rpc_client; -int rpc_client_connect(rpc_client **out_client, const char *host, - const char *port); +int rpc_client_connect(rpc_client **out_client, const char *host, const char *port); void rpc_client_close(rpc_client *client); int rpc_client_ping(rpc_client *client); -int rpc_client_call(rpc_client *client, uint32_t proc_id, - const rpc_writer *args, rpc_value **out_values, +int rpc_client_call(rpc_client *client, uint32_t proc_id, const rpc_writer *args, rpc_value **out_values, size_t *out_count); -int rpc_client_send_call(rpc_client *client, uint32_t proc_id, - const rpc_writer *args, uint64_t *out_call_id); -int rpc_client_recv_response(rpc_client *client, uint64_t *out_call_id, - rpc_value **out_values, size_t *out_count); +int rpc_client_send_call(rpc_client *client, uint32_t proc_id, const rpc_writer *args, uint64_t *out_call_id); +int rpc_client_recv_response(rpc_client *client, uint64_t *out_call_id, rpc_value **out_values, size_t *out_count); const char *rpc_client_error(const rpc_client *client); diff --git a/include/rpc/protocol.h b/include/rpc/protocol.h index 6c4b962..89f7785 100644 --- a/include/rpc/protocol.h +++ b/include/rpc/protocol.h @@ -81,8 +81,7 @@ int rpc_writer_f64(rpc_writer *writer, double value); int rpc_writer_bytes(rpc_writer *writer, const void *data, uint32_t len); int rpc_writer_string(rpc_writer *writer, const char *data, uint32_t len); -int rpc_payload_decode(const uint8_t *data, size_t len, rpc_value **out_values, - size_t *out_count); +int rpc_payload_decode(const uint8_t *data, size_t len, rpc_value **out_values, size_t *out_count); void rpc_values_free(rpc_value *values); #ifdef __cplusplus diff --git a/include/rpc/server.h b/include/rpc/server.h index b5ab0fd..9944c30 100644 --- a/include/rpc/server.h +++ b/include/rpc/server.h @@ -10,8 +10,7 @@ extern "C" { typedef struct rpc_ctx rpc_ctx; typedef struct rpc_server rpc_server; -typedef int (*rpc_handler_fn)(rpc_ctx *ctx, const rpc_value *args, size_t argc, - rpc_writer *out, void *user_data); +typedef int (*rpc_handler_fn)(rpc_ctx *ctx, const rpc_value *args, size_t argc, rpc_writer *out, void *user_data); uint64_t rpc_ctx_call_id(const rpc_ctx *ctx); uint32_t rpc_ctx_proc_id(const rpc_ctx *ctx); @@ -26,10 +25,8 @@ int rpc_server_run(rpc_server *server); void rpc_server_stop(rpc_server *server); void rpc_server_destroy(rpc_server *server); -int rpc_server_add_route(rpc_server *server, uint32_t proc_id, - rpc_handler_fn handler, void *user_data); -int rpc_server_add_async_route(rpc_server *server, uint32_t proc_id, - rpc_handler_fn handler, void *user_data); +int rpc_server_add_route(rpc_server *server, uint32_t proc_id, rpc_handler_fn handler, void *user_data); +int rpc_server_add_async_route(rpc_server *server, uint32_t proc_id, rpc_handler_fn handler, void *user_data); int rpc_server_remove_route(rpc_server *server, uint32_t proc_id); #ifdef __cplusplus diff --git a/include/rpc/trace.h b/include/rpc/trace.h index af454d3..fc00041 100644 --- a/include/rpc/trace.h +++ b/include/rpc/trace.h @@ -57,10 +57,8 @@ void rpc_trace_set_enabled(int enabled); uint64_t rpc_trace_begin_slow(void); void rpc_trace_end_slow(rpc_trace_metric metric, uint64_t start_ns); void rpc_trace_add_slow(rpc_trace_metric metric, uint64_t value); -void rpc_trace_worker_end_slow(uint32_t worker, rpc_trace_worker_metric metric, - uint64_t start_ns); -void rpc_trace_worker_add_slow(uint32_t worker, rpc_trace_worker_metric metric, - uint64_t value); +void rpc_trace_worker_end_slow(uint32_t worker, rpc_trace_worker_metric metric, uint64_t start_ns); +void rpc_trace_worker_add_slow(uint32_t worker, rpc_trace_worker_metric metric, uint64_t value); void rpc_trace_reset(void); void rpc_trace_snapshot(rpc_trace_stat out[RPC_TRACE_COUNT]); void rpc_trace_dump(FILE *out); @@ -73,8 +71,7 @@ void rpc_trace_dump(FILE *out); static inline int rpc_trace_is_enabled(void) { #if defined(__GNUC__) || defined(__clang__) - return RPC_TRACE_UNLIKELY( - __atomic_load_n(&rpc_trace_enabled, __ATOMIC_RELAXED)); + return RPC_TRACE_UNLIKELY(__atomic_load_n(&rpc_trace_enabled, __ATOMIC_RELAXED)); #else return RPC_TRACE_UNLIKELY(rpc_trace_enabled); #endif @@ -85,31 +82,19 @@ static inline uint64_t rpc_trace_begin(void) { } static inline void rpc_trace_end(rpc_trace_metric metric, uint64_t start_ns) { - if (rpc_trace_is_enabled() && start_ns != 0) { - rpc_trace_end_slow(metric, start_ns); - } + if (rpc_trace_is_enabled() && start_ns != 0) { rpc_trace_end_slow(metric, start_ns); } } static inline void rpc_trace_add(rpc_trace_metric metric, uint64_t value) { - if (rpc_trace_is_enabled()) { - rpc_trace_add_slow(metric, value); - } + if (rpc_trace_is_enabled()) { rpc_trace_add_slow(metric, value); } } -static inline void rpc_trace_worker_end(uint32_t worker, - rpc_trace_worker_metric metric, - uint64_t start_ns) { - if (rpc_trace_is_enabled() && start_ns != 0) { - rpc_trace_worker_end_slow(worker, metric, start_ns); - } +static inline void rpc_trace_worker_end(uint32_t worker, rpc_trace_worker_metric metric, uint64_t start_ns) { + if (rpc_trace_is_enabled() && start_ns != 0) { rpc_trace_worker_end_slow(worker, metric, start_ns); } } -static inline void rpc_trace_worker_add(uint32_t worker, - rpc_trace_worker_metric metric, - uint64_t value) { - if (rpc_trace_is_enabled()) { - rpc_trace_worker_add_slow(worker, metric, value); - } +static inline void rpc_trace_worker_add(uint32_t worker, rpc_trace_worker_metric metric, uint64_t value) { + if (rpc_trace_is_enabled()) { rpc_trace_worker_add_slow(worker, metric, value); } } #ifdef __cplusplus diff --git a/include/scheduler.h b/include/scheduler.h index e48fcb3..c32ab21 100644 --- a/include/scheduler.h +++ b/include/scheduler.h @@ -17,10 +17,8 @@ struct rpc_ctx { int rpc_scheduler_init(rpc_scheduler **out); void rpc_scheduler_destroy(rpc_scheduler *scheduler); -int rpc_scheduler_submit(rpc_scheduler *scheduler, uint64_t call_id, - uint32_t proc_id, rpc_handler_fn handler, - void *handler_data, const uint8_t *payload, - size_t payload_len, rpc_call_done_fn done, +int rpc_scheduler_submit(rpc_scheduler *scheduler, uint64_t call_id, uint32_t proc_id, rpc_handler_fn handler, + void *handler_data, const uint8_t *payload, size_t payload_len, rpc_call_done_fn done, void *done_data); void rpc_scheduler_run_ready(rpc_scheduler *scheduler); diff --git a/meson.build b/meson.build index 22c11d5..02f6ab9 100644 --- a/meson.build +++ b/meson.build @@ -9,6 +9,7 @@ project('rpc', 'c', default_options: [ ], version: '0.0.1') include = include_directories('include') +clang_format = find_program('clang-format', required: false) core_sources = files( 'src/backend/kqueue.c', diff --git a/src/backend/kqueue.c b/src/backend/kqueue.c index a3dba38..2c804fc 100644 --- a/src/backend/kqueue.c +++ b/src/backend/kqueue.c @@ -13,19 +13,15 @@ struct rpc_backend { int kq; }; -static int apply_filter(rpc_backend *backend, int fd, int16_t filter, - uint16_t flags, uintptr_t user) { +static int apply_filter(rpc_backend *backend, int fd, int16_t filter, uint16_t flags, uintptr_t user) { struct kevent change; EV_SET(&change, (uintptr_t)fd, filter, flags, 0, 0, (void *)user); int rc = kevent(backend->kq, &change, 1, NULL, 0, NULL); - if (rc != 0 && errno == ENOENT && (flags & EV_DELETE)) { - return 0; - } + if (rc != 0 && errno == ENOENT && (flags & EV_DELETE)) { return 0; } return rc; } -static int set_interest(rpc_backend *backend, int fd, uint32_t events, - uintptr_t user) { +static int set_interest(rpc_backend *backend, int fd, uint32_t events, uintptr_t user) { int rc = 0; if (events & RPC_BACKEND_READ) { rc |= apply_filter(backend, fd, EVFILT_READ, EV_ADD | EV_ENABLE, user); @@ -41,13 +37,9 @@ static int set_interest(rpc_backend *backend, int fd, uint32_t events, } int rpc_backend_kqueue_create(rpc_backend **out) { - if (!out) { - return -1; - } + if (!out) { return -1; } rpc_backend *backend = calloc(1, sizeof(*backend)); - if (!backend) { - return -1; - } + if (!backend) { return -1; } backend->kq = kqueue(); if (backend->kq < 0) { free(backend); @@ -68,49 +60,36 @@ int rpc_backend_kqueue_create(rpc_backend **out) { void rpc_backend_destroy(rpc_backend *backend) { if (backend) { - if (backend->kq >= 0) { - close(backend->kq); - } + if (backend->kq >= 0) { close(backend->kq); } free(backend); } } -int rpc_backend_register(rpc_backend *backend, int fd, uint32_t events, - uintptr_t user) { - if (!backend || fd < 0) { - return -1; - } +int rpc_backend_register(rpc_backend *backend, int fd, uint32_t events, uintptr_t user) { + if (!backend || fd < 0) { return -1; } return set_interest(backend, fd, events, user); } -int rpc_backend_modify(rpc_backend *backend, int fd, uint32_t events, - uintptr_t user) { +int rpc_backend_modify(rpc_backend *backend, int fd, uint32_t events, uintptr_t user) { return rpc_backend_register(backend, fd, events, user); } int rpc_backend_remove(rpc_backend *backend, int fd) { - if (!backend || fd < 0) { - return -1; - } + if (!backend || fd < 0) { return -1; } (void)apply_filter(backend, fd, EVFILT_READ, EV_DELETE, 0); (void)apply_filter(backend, fd, EVFILT_WRITE, EV_DELETE, 0); return 0; } int rpc_backend_wake(rpc_backend *backend) { - if (!backend) { - return -1; - } + if (!backend) { return -1; } struct kevent wake; EV_SET(&wake, 1, EVFILT_USER, 0, NOTE_TRIGGER, 0, NULL); return kevent(backend->kq, &wake, 1, NULL, 0, NULL); } -int rpc_backend_poll(rpc_backend *backend, rpc_backend_event *events, int max_events, - int timeout_ms) { - if (!backend || !events || max_events <= 0) { - return -1; - } +int rpc_backend_poll(rpc_backend *backend, rpc_backend_event *events, int max_events, int timeout_ms) { + if (!backend || !events || max_events <= 0) { return -1; } struct timespec timeout; struct timespec *timeout_ptr = NULL; @@ -124,9 +103,7 @@ int rpc_backend_poll(rpc_backend *backend, rpc_backend_event *events, int max_ev int limit = max_events < RPC_KQUEUE_MAX_EVENTS ? max_events : RPC_KQUEUE_MAX_EVENTS; int n = kevent(backend->kq, NULL, 0, kev, limit, timeout_ptr); if (n < 0) { - if (errno == EINTR) { - return 0; - } + if (errno == EINTR) { return 0; } return -1; } @@ -134,15 +111,9 @@ int rpc_backend_poll(rpc_backend *backend, rpc_backend_event *events, int max_ev events[i].fd = (int)kev[i].ident; events[i].events = 0; events[i].user = (uintptr_t)kev[i].udata; - if (kev[i].filter == EVFILT_USER) { - events[i].events |= RPC_BACKEND_WAKE; - } - if (kev[i].filter == EVFILT_READ) { - events[i].events |= RPC_BACKEND_READ; - } - if (kev[i].filter == EVFILT_WRITE) { - events[i].events |= RPC_BACKEND_WRITE; - } + if (kev[i].filter == EVFILT_USER) { events[i].events |= RPC_BACKEND_WAKE; } + if (kev[i].filter == EVFILT_READ) { events[i].events |= RPC_BACKEND_READ; } + if (kev[i].filter == EVFILT_WRITE) { events[i].events |= RPC_BACKEND_WRITE; } } return n; } diff --git a/src/client.c b/src/client.c index 71b8683..90fc3ef 100644 --- a/src/client.c +++ b/src/client.c @@ -27,16 +27,12 @@ struct rpc_client { }; static void set_error(rpc_client *client, const char *message) { - if (client) { - snprintf(client->error, sizeof(client->error), "%s", message); - } + if (client) { snprintf(client->error, sizeof(client->error), "%s", message); } } static int send_iov_full(int fd, const struct iovec *iov, int iov_count) { struct iovec local[2]; - if (iov_count <= 0 || iov_count > 2) { - return -1; - } + if (iov_count <= 0 || iov_count > 2) { return -1; } memcpy(local, iov, (size_t)iov_count * sizeof(*iov)); while (iov_count > 0) { @@ -59,9 +55,7 @@ static int send_iov_full(int fd, const struct iovec *iov, int iov_count) { } continue; } - if (n < 0 && errno == EINTR) { - continue; - } + if (n < 0 && errno == EINTR) { continue; } return -1; } return 0; @@ -72,27 +66,20 @@ static size_t client_read_available(const rpc_client *client) { } static int client_read_reserve(rpc_client *client, size_t need) { - if (client_read_available(client) >= need) { - return 0; - } + if (client_read_available(client) >= need) { return 0; } if (client->read_off > 0) { - memmove(client->read_buf, client->read_buf + client->read_off, - client_read_available(client)); + memmove(client->read_buf, client->read_buf + client->read_off, client_read_available(client)); client->read_len -= client->read_off; client->read_off = 0; } - if (client->read_cap >= need) { - return 0; - } + if (client->read_cap >= need) { return 0; } size_t next_cap = client->read_cap ? client->read_cap : 65536u; while (next_cap < need) { next_cap *= 2u; } uint8_t *next = realloc(client->read_buf, next_cap); - if (!next) { - return -1; - } + if (!next) { return -1; } client->read_buf = next; client->read_cap = next_cap; return 0; @@ -100,29 +87,20 @@ static int client_read_reserve(rpc_client *client, size_t need) { static int client_read_fill(rpc_client *client, size_t need) { while (client_read_available(client) < need) { - if (client_read_reserve(client, need) != 0) { - return -1; - } - if (client->read_len == client->read_cap && - client_read_reserve(client, client->read_cap + 1u) != 0) { - return -1; - } - ssize_t n = recv(client->fd, client->read_buf + client->read_len, - client->read_cap - client->read_len, 0); + if (client_read_reserve(client, need) != 0) { return -1; } + if (client->read_len == client->read_cap && client_read_reserve(client, client->read_cap + 1u) != 0) { return -1; } + ssize_t n = recv(client->fd, client->read_buf + client->read_len, client->read_cap - client->read_len, 0); if (n > 0) { client->read_len += (size_t)n; continue; } - if (n < 0 && errno == EINTR) { - continue; - } + if (n < 0 && errno == EINTR) { continue; } return -1; } return 0; } -static int send_packet(rpc_client *client, rpc_op op, uint32_t proc_id, - uint64_t call_id, const rpc_writer *payload) { +static int send_packet(rpc_client *client, rpc_op op, uint32_t proc_id, uint64_t call_id, const rpc_writer *payload) { uint64_t trace = rpc_trace_begin(); rpc_header header = { .op = op, @@ -167,8 +145,7 @@ static int recv_packet(rpc_client *client, rpc_header *header, uint8_t **body) { *body = NULL; if (client_read_fill(client, RPC_HEADER_SIZE) != 0 || - rpc_header_decode(client->read_buf + client->read_off, header) != 0 || - header->size > RPC_MAX_PAYLOAD_SIZE) { + rpc_header_decode(client->read_buf + client->read_off, header) != 0 || header->size > RPC_MAX_PAYLOAD_SIZE) { set_error(client, "read failed"); rpc_trace_end(RPC_TRACE_CLIENT_RECV, trace); return -1; @@ -187,10 +164,7 @@ static int recv_packet(rpc_client *client, rpc_header *header, uint8_t **body) { rpc_trace_end(RPC_TRACE_CLIENT_RECV, trace); return -1; } - if (header->size > 0) { - memcpy(*body, client->read_buf + client->read_off + RPC_HEADER_SIZE, - header->size); - } + if (header->size > 0) { memcpy(*body, client->read_buf + client->read_off + RPC_HEADER_SIZE, header->size); } client->read_off += packet_size; if (client->read_off == client->read_len) { client->read_off = 0; @@ -200,16 +174,11 @@ static int recv_packet(rpc_client *client, rpc_header *header, uint8_t **body) { return 0; } -int rpc_client_connect(rpc_client **out_client, const char *host, - const char *port) { - if (!out_client || !port) { - return -1; - } +int rpc_client_connect(rpc_client **out_client, const char *host, const char *port) { + if (!out_client || !port) { return -1; } rpc_client *client = calloc(1, sizeof(*client)); - if (!client) { - return -1; - } + if (!client) { return -1; } client->fd = -1; client->next_call_id = 1; @@ -226,13 +195,10 @@ int rpc_client_connect(rpc_client **out_client, const char *host, for (struct addrinfo *ai = res; ai; ai = ai->ai_next) { int fd = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol); - if (fd < 0) { - continue; - } + if (fd < 0) { continue; } #ifdef SO_NOSIGPIPE int no_sigpipe = 1; - (void)setsockopt(fd, SOL_SOCKET, SO_NOSIGPIPE, &no_sigpipe, - sizeof(no_sigpipe)); + (void)setsockopt(fd, SOL_SOCKET, SO_NOSIGPIPE, &no_sigpipe, sizeof(no_sigpipe)); #endif if (connect(fd, ai->ai_addr, ai->ai_addrlen) == 0) { client->fd = fd; @@ -252,9 +218,7 @@ int rpc_client_connect(rpc_client **out_client, const char *host, } void rpc_client_close(rpc_client *client) { - if (!client) { - return; - } + if (!client) { return; } if (client->fd >= 0) { (void)send_packet(client, RPC_OP_DISCONNECT, 0, client->next_call_id++, NULL); close(client->fd); @@ -264,31 +228,23 @@ void rpc_client_close(rpc_client *client) { } int rpc_client_ping(rpc_client *client) { - if (!client || client->fd < 0) { - return -1; - } + if (!client || client->fd < 0) { return -1; } uint64_t call_id = client->next_call_id++; - if (send_packet(client, RPC_OP_PING, 0, call_id, NULL) != 0) { - return -1; - } + if (send_packet(client, RPC_OP_PING, 0, call_id, NULL) != 0) { return -1; } rpc_header header; uint8_t *body = NULL; - if (recv_packet(client, &header, &body) != 0) { - return -1; - } + if (recv_packet(client, &header, &body) != 0) { return -1; } free(body); - if (header.op != RPC_OP_RESPONSE || header.call_id != call_id || - header.size != 0) { + if (header.op != RPC_OP_RESPONSE || header.call_id != call_id || header.size != 0) { set_error(client, "unexpected ping response"); return -1; } return 0; } -int rpc_client_call(rpc_client *client, uint32_t proc_id, - const rpc_writer *args, rpc_value **out_values, +int rpc_client_call(rpc_client *client, uint32_t proc_id, const rpc_writer *args, rpc_value **out_values, size_t *out_count) { uint64_t trace_call = rpc_trace_begin(); if (!client || client->fd < 0 || !out_values || !out_count) { @@ -318,33 +274,23 @@ int rpc_client_call(rpc_client *client, uint32_t proc_id, return rc; } -int rpc_client_send_call(rpc_client *client, uint32_t proc_id, - const rpc_writer *args, uint64_t *out_call_id) { - if (!client || client->fd < 0 || !out_call_id) { - return -1; - } +int rpc_client_send_call(rpc_client *client, uint32_t proc_id, const rpc_writer *args, uint64_t *out_call_id) { + if (!client || client->fd < 0 || !out_call_id) { return -1; } uint64_t call_id = client->next_call_id++; - if (send_packet(client, RPC_OP_RPC, proc_id, call_id, args) != 0) { - return -1; - } + if (send_packet(client, RPC_OP_RPC, proc_id, call_id, args) != 0) { return -1; } *out_call_id = call_id; return 0; } -int rpc_client_recv_response(rpc_client *client, uint64_t *out_call_id, - rpc_value **out_values, size_t *out_count) { - if (!client || client->fd < 0 || !out_call_id || !out_values || !out_count) { - return -1; - } +int rpc_client_recv_response(rpc_client *client, uint64_t *out_call_id, rpc_value **out_values, size_t *out_count) { + if (!client || client->fd < 0 || !out_call_id || !out_values || !out_count) { return -1; } *out_call_id = 0; *out_values = NULL; *out_count = 0; rpc_header header; uint8_t *body = NULL; - if (recv_packet(client, &header, &body) != 0) { - return -1; - } + if (recv_packet(client, &header, &body) != 0) { return -1; } int rc = -1; *out_call_id = header.call_id; @@ -359,12 +305,10 @@ int rpc_client_recv_response(rpc_client *client, uint64_t *out_call_id, } else if (header.op == RPC_OP_ERROR) { rpc_value *values = NULL; size_t count = 0; - if (rpc_payload_decode(body, header.size, &values, &count) == 0 && - count == 1 && values[0].type == RPC_TYPE_STRING) { + if (rpc_payload_decode(body, header.size, &values, &count) == 0 && count == 1 && + values[0].type == RPC_TYPE_STRING) { size_t n = values[0].as.string.len; - if (n >= sizeof(client->error)) { - n = sizeof(client->error) - 1u; - } + if (n >= sizeof(client->error)) { n = sizeof(client->error) - 1u; } memcpy(client->error, values[0].as.string.data, n); client->error[n] = '\0'; } else { diff --git a/src/payload.c b/src/payload.c index d0e0fc2..bece0cc 100644 --- a/src/payload.c +++ b/src/payload.c @@ -19,8 +19,7 @@ static void put_u64(uint8_t *out, uint64_t value) { } static uint32_t get_u32(const uint8_t *in) { - return ((uint32_t)in[0] << 24u) | ((uint32_t)in[1] << 16u) | - ((uint32_t)in[2] << 8u) | (uint32_t)in[3]; + return ((uint32_t)in[0] << 24u) | ((uint32_t)in[1] << 16u) | ((uint32_t)in[2] << 8u) | (uint32_t)in[3]; } static uint64_t get_u64(const uint8_t *in) { @@ -32,14 +31,9 @@ static uint64_t get_u64(const uint8_t *in) { } static int writer_reserve(rpc_writer *writer, size_t extra) { - if (!writer || extra > RPC_MAX_PAYLOAD_SIZE || - writer->len > RPC_MAX_PAYLOAD_SIZE - extra) { - return -1; - } + if (!writer || extra > RPC_MAX_PAYLOAD_SIZE || writer->len > RPC_MAX_PAYLOAD_SIZE - extra) { return -1; } size_t need = writer->len + extra; - if (need <= writer->cap) { - return 0; - } + if (need <= writer->cap) { return 0; } size_t cap = writer->cap ? writer->cap : 64u; while (cap < need) { @@ -50,35 +44,25 @@ static int writer_reserve(rpc_writer *writer, size_t extra) { cap *= 2u; } uint8_t *next = realloc(writer->data, cap); - if (!next) { - return -1; - } + if (!next) { return -1; } writer->data = next; writer->cap = cap; return 0; } static int writer_push(rpc_writer *writer, const void *data, size_t len) { - if (writer_reserve(writer, len) != 0) { - return -1; - } - if (len > 0) { - memcpy(writer->data + writer->len, data, len); - } + if (writer_reserve(writer, len) != 0) { return -1; } + if (len > 0) { memcpy(writer->data + writer->len, data, len); } writer->len += len; return 0; } void rpc_writer_init(rpc_writer *writer) { - if (writer) { - memset(writer, 0, sizeof(*writer)); - } + if (writer) { memset(writer, 0, sizeof(*writer)); } } void rpc_writer_reset(rpc_writer *writer) { - if (writer) { - writer->len = 0; - } + if (writer) { writer->len = 0; } } void rpc_writer_free(rpc_writer *writer) { @@ -123,40 +107,27 @@ int rpc_writer_f64(rpc_writer *writer, double value) { int rpc_writer_bytes(rpc_writer *writer, const void *data, uint32_t len) { uint8_t prefix[5]; - if (len > 0 && !data) { - return -1; - } + if (len > 0 && !data) { return -1; } prefix[0] = RPC_TYPE_BYTES; put_u32(prefix + 1, len); - if (writer_push(writer, prefix, sizeof(prefix)) != 0) { - return -1; - } + if (writer_push(writer, prefix, sizeof(prefix)) != 0) { return -1; } return writer_push(writer, data, len); } int rpc_writer_string(rpc_writer *writer, const char *data, uint32_t len) { uint8_t prefix[5]; - if (len > 0 && !data) { - return -1; - } + if (len > 0 && !data) { return -1; } prefix[0] = RPC_TYPE_STRING; put_u32(prefix + 1, len); - if (writer_push(writer, prefix, sizeof(prefix)) != 0) { - return -1; - } + if (writer_push(writer, prefix, sizeof(prefix)) != 0) { return -1; } return writer_push(writer, data, len); } -int rpc_payload_decode(const uint8_t *data, size_t len, rpc_value **out_values, - size_t *out_count) { +int rpc_payload_decode(const uint8_t *data, size_t len, rpc_value **out_values, size_t *out_count) { uint64_t trace = rpc_trace_begin(); static const void *dispatch[] = { - [RPC_TYPE_NULL] = &&type_null, - [RPC_TYPE_BOOL] = &&type_bool, - [RPC_TYPE_I64] = &&type_i64, - [RPC_TYPE_U64] = &&type_u64, - [RPC_TYPE_F64] = &&type_f64, - [RPC_TYPE_BYTES] = &&type_bytes, + [RPC_TYPE_NULL] = &&type_null, [RPC_TYPE_BOOL] = &&type_bool, [RPC_TYPE_I64] = &&type_i64, + [RPC_TYPE_U64] = &&type_u64, [RPC_TYPE_F64] = &&type_f64, [RPC_TYPE_BYTES] = &&type_bytes, [RPC_TYPE_STRING] = &&type_string, }; @@ -187,58 +158,43 @@ int rpc_payload_decode(const uint8_t *data, size_t len, rpc_value **out_values, memset(&value, 0, sizeof(value)); value.type = (rpc_type)data[off++]; - if ((size_t)value.type >= sizeof(dispatch) / sizeof(*dispatch) || - !dispatch[value.type]) { - goto malformed; - } + if ((size_t)value.type >= sizeof(dispatch) / sizeof(*dispatch) || !dispatch[value.type]) { goto malformed; } goto *dispatch[value.type]; -type_null: + type_null: goto store; -type_bool: - if (off + 1u > len || (data[off] != 0u && data[off] != 1u)) { - goto malformed; - } + type_bool: + if (off + 1u > len || (data[off] != 0u && data[off] != 1u)) { goto malformed; } value.as.boolean = data[off++] != 0u; goto store; -type_i64: - if (off + 8u > len) { - goto malformed; - } + type_i64: + if (off + 8u > len) { goto malformed; } value.as.i64 = (int64_t)get_u64(data + off); off += 8u; goto store; -type_u64: - if (off + 8u > len) { - goto malformed; - } + type_u64: + if (off + 8u > len) { goto malformed; } value.as.u64 = get_u64(data + off); off += 8u; goto store; -type_f64: { - if (off + 8u > len) { - goto malformed; - } + type_f64: { + if (off + 8u > len) { goto malformed; } uint64_t bits = get_u64(data + off); memcpy(&value.as.f64, &bits, sizeof(bits)); off += 8u; goto store; } -type_bytes: -type_string: { - if (off + 4u > len) { - goto malformed; - } + type_bytes: + type_string: { + if (off + 4u > len) { goto malformed; } uint32_t value_len = get_u32(data + off); off += 4u; - if (off + value_len > len) { - goto malformed; - } + if (off + value_len > len) { goto malformed; } if (value.type == RPC_TYPE_BYTES) { value.as.bytes.data = data + off; value.as.bytes.len = value_len; @@ -250,11 +206,11 @@ type_string: { goto store; } -store: + store: values[count++] = value; continue; -malformed: + malformed: free(values); rpc_trace_end(RPC_TRACE_PAYLOAD_DECODE, trace); return -1; @@ -266,4 +222,6 @@ malformed: return 0; } -void rpc_values_free(rpc_value *values) { free(values); } +void rpc_values_free(rpc_value *values) { + free(values); +} diff --git a/src/protocol.c b/src/protocol.c index 29787d5..4133c40 100644 --- a/src/protocol.c +++ b/src/protocol.c @@ -16,8 +16,7 @@ static void put_u64(uint8_t *out, uint64_t value) { } static uint32_t get_u32(const uint8_t *in) { - return ((uint32_t)in[0] << 24u) | ((uint32_t)in[1] << 16u) | - ((uint32_t)in[2] << 8u) | (uint32_t)in[3]; + return ((uint32_t)in[0] << 24u) | ((uint32_t)in[1] << 16u) | ((uint32_t)in[2] << 8u) | (uint32_t)in[3]; } static uint64_t get_u64(const uint8_t *in) { @@ -29,15 +28,12 @@ static uint64_t get_u64(const uint8_t *in) { } static int valid_op(uint8_t op) { - return op == RPC_OP_RPC || op == RPC_OP_PING || op == RPC_OP_DISCONNECT || - op == RPC_OP_RESPONSE || op == RPC_OP_ERROR; + return op == RPC_OP_RPC || op == RPC_OP_PING || op == RPC_OP_DISCONNECT || op == RPC_OP_RESPONSE || + op == RPC_OP_ERROR; } int rpc_header_encode(const rpc_header *header, uint8_t out[RPC_HEADER_SIZE]) { - if (!header || !out || !valid_op((uint8_t)header->op) || - header->size > RPC_MAX_PAYLOAD_SIZE) { - return -1; - } + if (!header || !out || !valid_op((uint8_t)header->op) || header->size > RPC_MAX_PAYLOAD_SIZE) { return -1; } out[0] = (uint8_t)header->op; out[1] = header->flags; @@ -48,9 +44,7 @@ int rpc_header_encode(const rpc_header *header, uint8_t out[RPC_HEADER_SIZE]) { } int rpc_header_decode(const uint8_t in[RPC_HEADER_SIZE], rpc_header *out) { - if (!in || !out || !valid_op(in[0])) { - return -1; - } + if (!in || !out || !valid_op(in[0])) { return -1; } memset(out, 0, sizeof(*out)); out->op = (rpc_op)in[0]; @@ -58,8 +52,6 @@ int rpc_header_decode(const uint8_t in[RPC_HEADER_SIZE], rpc_header *out) { out->proc_id = get_u32(in + 2); out->size = get_u32(in + 6); out->call_id = get_u64(in + 10); - if (out->size > RPC_MAX_PAYLOAD_SIZE) { - return -1; - } + if (out->size > RPC_MAX_PAYLOAD_SIZE) { return -1; } return 0; } diff --git a/src/routes.c b/src/routes.c index 3fa5d32..2ed4a5c 100644 --- a/src/routes.c +++ b/src/routes.c @@ -20,7 +20,7 @@ static void free_retired(rpc_routes *routes) { rpc_retired_route *node = routes->retired; routes->retired = NULL; - + while (node) { rpc_retired_route *next = node->next; free(node->route); @@ -34,12 +34,12 @@ static int retire_route(rpc_routes *routes, rpc_route *route) { rpc_retired_route *node = calloc(1, sizeof(*node)); if (!node) return -1; - + node->route = route; node->next = routes->retired; routes->retired = node; free_retired(routes); - + return 0; } @@ -50,23 +50,23 @@ int rpc_routes_init(rpc_routes *routes) { routes->root_bytes = RPC_ROUTE_ROOT_SIZE * sizeof(*routes->pages); routes->page_bytes = RPC_ROUTE_PAGE_SIZE * sizeof(rpc_route_slot); routes->pages = route_mmap(routes->root_bytes); - + if (!routes->pages) return -1; - + if (pthread_mutex_init(&routes->mutate_lock, NULL) != 0) { munmap(routes->pages, routes->root_bytes); memset(routes, 0, sizeof(*routes)); return -1; } - + atomic_init(&routes->active_readers, 0); - + return 0; } void rpc_routes_destroy(rpc_routes *routes) { if (!routes) return; - + pthread_mutex_lock(&routes->mutate_lock); if (!routes->pages) goto retired; @@ -81,17 +81,17 @@ void rpc_routes_destroy(rpc_routes *routes) { munmap(page, routes->page_bytes); } munmap(routes->pages, routes->root_bytes); - + retired: rpc_retired_route *node = routes->retired; - + while (node) { rpc_retired_route *next = node->next; free(node->route); free(node); node = next; } - + routes->retired = NULL; pthread_mutex_unlock(&routes->mutate_lock); pthread_mutex_destroy(&routes->mutate_lock); @@ -101,19 +101,13 @@ int rpc_routes_add(rpc_routes *routes, uint32_t proc_id, rpc_handler_fn handler, return rpc_routes_add_ex(routes, proc_id, handler, user_data, 0); } -int rpc_routes_add_ex(rpc_routes *routes, uint32_t proc_id, - rpc_handler_fn handler, void *user_data, int is_async) { +int rpc_routes_add_ex(rpc_routes *routes, uint32_t proc_id, rpc_handler_fn handler, void *user_data, int is_async) { if (!routes || !handler) return -1; rpc_route *route = calloc(1, sizeof(*route)); if (!route) return -1; - *route = (rpc_route){ - .proc_id = proc_id, - .handler = handler, - .user_data = user_data, - .is_async = is_async ? 1 : 0 - }; + *route = (rpc_route){.proc_id = proc_id, .handler = handler, .user_data = user_data, .is_async = is_async ? 1 : 0}; pthread_mutex_lock(&routes->mutate_lock); uint32_t page_idx = proc_id >> RPC_ROUTE_PAGE_BITS; @@ -128,14 +122,11 @@ int rpc_routes_add_ex(rpc_routes *routes, uint32_t proc_id, } atomic_store_explicit(&routes->pages[page_idx], page, memory_order_release); } - rpc_route *old = atomic_exchange_explicit( - &page[slot_idx], - route, memory_order_acq_rel - ); - + rpc_route *old = atomic_exchange_explicit(&page[slot_idx], route, memory_order_acq_rel); + int rc = retire_route(routes, old); pthread_mutex_unlock(&routes->mutate_lock); - + return rc; } @@ -146,13 +137,11 @@ int rpc_routes_remove(rpc_routes *routes, uint32_t proc_id) { uint32_t page_idx = proc_id >> RPC_ROUTE_PAGE_BITS; uint32_t slot_idx = proc_id & RPC_ROUTE_PAGE_MASK; rpc_route_slot *page = atomic_load_explicit(&routes->pages[page_idx], memory_order_acquire); - rpc_route *old = page - ? atomic_exchange_explicit(&page[slot_idx], NULL, memory_order_acq_rel) - : NULL; - + rpc_route *old = page ? atomic_exchange_explicit(&page[slot_idx], NULL, memory_order_acq_rel) : NULL; + int rc = old ? retire_route(routes, old) : -1; pthread_mutex_unlock(&routes->mutate_lock); - + return rc; } @@ -167,14 +156,9 @@ int rpc_routes_lookup(rpc_routes *routes, uint32_t proc_id, rpc_route *out) { uint32_t page_idx = proc_id >> RPC_ROUTE_PAGE_BITS; uint32_t slot_idx = proc_id & RPC_ROUTE_PAGE_MASK; rpc_route_slot *page = atomic_load_explicit(&routes->pages[page_idx], memory_order_acquire); - rpc_route *route = page - ? atomic_load_explicit(&page[slot_idx], memory_order_acquire) - : NULL; - if (route) { - *out = *route; - } - unsigned old = - atomic_fetch_sub_explicit(&routes->active_readers, 1u, memory_order_release); + rpc_route *route = page ? atomic_load_explicit(&page[slot_idx], memory_order_acquire) : NULL; + if (route) { *out = *route; } + unsigned old = atomic_fetch_sub_explicit(&routes->active_readers, 1u, memory_order_release); if (old == 1u) { pthread_mutex_lock(&routes->mutate_lock); free_retired(routes); diff --git a/src/scheduler.c b/src/scheduler.c index 508bc26..cab968a 100644 --- a/src/scheduler.c +++ b/src/scheduler.c @@ -40,12 +40,8 @@ struct rpc_call { }; static void call_free(rpc_scheduler *scheduler, rpc_call *call) { - if (!call) { - return; - } - if (call->co) { - (void)mco_destroy(call->co); - } + if (!call) { return; } + if (call->co) { (void)mco_destroy(call->co); } rpc_values_free(call->args); rpc_writer_free(&call->response); free(call->payload); @@ -65,32 +61,24 @@ static int enqueue(rpc_scheduler *scheduler, rpc_call *call) { static rpc_call *dequeue(rpc_scheduler *scheduler) { rpc_call *call = scheduler->head; - if (!call) { - return NULL; - } + if (!call) { return NULL; } scheduler->head = call->next_ready; - if (!scheduler->head) { - scheduler->tail = NULL; - } + if (!scheduler->head) { scheduler->tail = NULL; } call->next_ready = NULL; return call; } static void call_entry(mco_coro *co) { rpc_call *call = mco_get_user_data(co); - call->result = call->handler(&call->ctx, call->args, call->argc, - &call->response, call->handler_data); + call->result = call->handler(&call->ctx, call->args, call->argc, &call->response, call->handler_data); call->completed = 1; if (call->result != 0 && call->error[0] == '\0') { - snprintf(call->error, sizeof(call->error), "procedure returned %d", - call->result); + snprintf(call->error, sizeof(call->error), "procedure returned %d", call->result); } } static void call_finish(rpc_scheduler *scheduler, rpc_call *call) { - if (call->done) { - call->done(call, call->done_data); - } + if (call->done) { call->done(call, call->done_data); } call_free(scheduler, call); } @@ -99,8 +87,7 @@ static int call_resume(rpc_call *call) { mco_result rc = mco_resume(call->co); rpc_trace_end(RPC_TRACE_SCHED_RESUME, trace_resume); if (rc != MCO_SUCCESS) { - snprintf(call->error, sizeof(call->error), "coroutine resume failed: %s", - mco_result_description(rc)); + snprintf(call->error, sizeof(call->error), "coroutine resume failed: %s", mco_result_description(rc)); call->result = -1; call->completed = 1; return -1; @@ -109,15 +96,10 @@ static int call_resume(rpc_call *call) { } int rpc_scheduler_init(rpc_scheduler **out) { - if (!out) { - return -1; - } + if (!out) { return -1; } rpc_scheduler *scheduler = calloc(1, sizeof(*scheduler)); - if (!scheduler) { - return -1; - } - if (rpc_fixed_arena_init(&scheduler->call_arena, sizeof(rpc_call), - RPC_CALL_ARENA_CAPACITY) != 0) { + if (!scheduler) { return -1; } + if (rpc_fixed_arena_init(&scheduler->call_arena, sizeof(rpc_call), RPC_CALL_ARENA_CAPACITY) != 0) { free(scheduler); return -1; } @@ -126,9 +108,7 @@ int rpc_scheduler_init(rpc_scheduler **out) { } void rpc_scheduler_destroy(rpc_scheduler *scheduler) { - if (!scheduler) { - return; - } + if (!scheduler) { return; } rpc_call *call = NULL; while ((call = dequeue(scheduler)) != NULL) { call_free(scheduler, call); @@ -137,10 +117,8 @@ void rpc_scheduler_destroy(rpc_scheduler *scheduler) { free(scheduler); } -int rpc_scheduler_submit(rpc_scheduler *scheduler, uint64_t call_id, - uint32_t proc_id, rpc_handler_fn handler, - void *handler_data, const uint8_t *payload, - size_t payload_len, rpc_call_done_fn done, +int rpc_scheduler_submit(rpc_scheduler *scheduler, uint64_t call_id, uint32_t proc_id, rpc_handler_fn handler, + void *handler_data, const uint8_t *payload, size_t payload_len, rpc_call_done_fn done, void *done_data) { uint64_t trace_submit = rpc_trace_begin(); if (!scheduler || !handler || (!payload && payload_len > 0)) { @@ -173,8 +151,7 @@ int rpc_scheduler_submit(rpc_scheduler *scheduler, uint64_t call_id, } uint64_t trace_decode = rpc_trace_begin(); - if (rpc_payload_decode(call->payload, call->payload_len, &call->args, - &call->argc) != 0) { + if (rpc_payload_decode(call->payload, call->payload_len, &call->args, &call->argc) != 0) { rpc_trace_end(RPC_TRACE_SCHED_DECODE, trace_decode); snprintf(call->error, sizeof(call->error), "malformed payload"); call->result = -1; @@ -192,8 +169,7 @@ int rpc_scheduler_submit(rpc_scheduler *scheduler, uint64_t call_id, mco_result rc = mco_create(&call->co, &desc); rpc_trace_end(RPC_TRACE_SCHED_CORO_CREATE, trace_create); if (rc != MCO_SUCCESS) { - snprintf(call->error, sizeof(call->error), "coroutine create failed: %s", - mco_result_description(rc)); + snprintf(call->error, sizeof(call->error), "coroutine create failed: %s", mco_result_description(rc)); call->result = -1; call->completed = 1; done(call, done_data); @@ -212,18 +188,14 @@ int rpc_scheduler_submit(rpc_scheduler *scheduler, uint64_t call_id, } void rpc_scheduler_run_ready(rpc_scheduler *scheduler) { - if (!scheduler) { - return; - } + if (!scheduler) { return; } rpc_call *call = NULL; while ((call = dequeue(scheduler)) != NULL) { (void)call_resume(call); if (!call->completed && mco_status(call->co) == MCO_SUSPENDED) { - if (enqueue(scheduler, call) == 0) { - continue; - } + if (enqueue(scheduler, call) == 0) { continue; } snprintf(call->error, sizeof(call->error), "scheduler enqueue failed"); call->result = -1; call->completed = 1; @@ -233,7 +205,9 @@ void rpc_scheduler_run_ready(rpc_scheduler *scheduler) { } } -int rpc_call_result(const rpc_call *call) { return call ? call->result : -1; } +int rpc_call_result(const rpc_call *call) { + return call ? call->result : -1; +} const rpc_writer *rpc_call_response(const rpc_call *call) { return call ? &call->response : NULL; @@ -251,14 +225,16 @@ uint32_t rpc_call_proc_id(const rpc_call *call) { return call ? call->ctx.proc_id : 0; } -uint64_t rpc_ctx_call_id(const rpc_ctx *ctx) { return ctx ? ctx->call_id : 0; } +uint64_t rpc_ctx_call_id(const rpc_ctx *ctx) { + return ctx ? ctx->call_id : 0; +} -uint32_t rpc_ctx_proc_id(const rpc_ctx *ctx) { return ctx ? ctx->proc_id : 0; } +uint32_t rpc_ctx_proc_id(const rpc_ctx *ctx) { + return ctx ? ctx->proc_id : 0; +} void rpc_ctx_yield(rpc_ctx *ctx) { (void)ctx; mco_coro *co = mco_running(); - if (co) { - (void)mco_yield(co); - } + if (co) { (void)mco_yield(co); } } diff --git a/src/server.c b/src/server.c index f4b97d6..2fe9383 100644 --- a/src/server.c +++ b/src/server.c @@ -90,33 +90,22 @@ static void worker_queue_write(rpc_worker *worker, rpc_connection *conn); static void worker_flush_writes(rpc_worker *worker); static int connection_set_interests(rpc_connection *conn, uint32_t interests) { - if (conn->interests == interests) { - return 0; - } - if (rpc_backend_modify(conn->worker->backend, conn->fd, interests, - (uintptr_t)conn) != 0) { - return -1; - } + if (conn->interests == interests) { return 0; } + if (rpc_backend_modify(conn->worker->backend, conn->fd, interests, (uintptr_t)conn) != 0) { return -1; } conn->interests = interests; return 0; } static int set_nonblock(int fd) { int flags = fcntl(fd, F_GETFL, 0); - if (flags < 0) { - return -1; - } + if (flags < 0) { return -1; } return fcntl(fd, F_SETFL, flags | O_NONBLOCK); } static uint32_t cpu_count(void) { long n = sysconf(_SC_NPROCESSORS_ONLN); - if (n <= 0) { - return 1; - } - if ((unsigned long)n > RPC_MAX_WORKERS) { - return RPC_MAX_WORKERS; - } + if (n <= 0) { return 1; } + if ((unsigned long)n > RPC_MAX_WORKERS) { return RPC_MAX_WORKERS; } return (uint32_t)n; } @@ -129,23 +118,14 @@ static void sockaddr_set_port(struct sockaddr *addr, uint16_t port) { } static uint16_t sockaddr_get_port(const struct sockaddr *addr) { - if (addr->sa_family == AF_INET) { - return ntohs(((const struct sockaddr_in *)addr)->sin_port); - } - if (addr->sa_family == AF_INET6) { - return ntohs(((const struct sockaddr_in6 *)addr)->sin6_port); - } + if (addr->sa_family == AF_INET) { return ntohs(((const struct sockaddr_in *)addr)->sin_port); } + if (addr->sa_family == AF_INET6) { return ntohs(((const struct sockaddr_in6 *)addr)->sin6_port); } return 0; } -static int append_bytes(uint8_t **buf, size_t *len, size_t *cap, - const void *data, size_t data_len) { - if (data_len == 0) { - return 0; - } - if (*len > SIZE_MAX - data_len) { - return -1; - } +static int append_bytes(uint8_t **buf, size_t *len, size_t *cap, const void *data, size_t data_len) { + if (data_len == 0) { return 0; } + if (*len > SIZE_MAX - data_len) { return -1; } size_t need = *len + data_len; if (need > *cap) { size_t next_cap = *cap ? *cap : 4096u; @@ -153,42 +133,31 @@ static int append_bytes(uint8_t **buf, size_t *len, size_t *cap, next_cap *= 2u; } uint8_t *next = realloc(*buf, next_cap); - if (!next) { - return -1; - } + if (!next) { return -1; } *buf = next; *cap = next_cap; } - if (data) { - memcpy(*buf + *len, data, data_len); - } + if (data) { memcpy(*buf + *len, data, data_len); } *len += data_len; return 0; } static int reserve_bytes(uint8_t **buf, size_t *cap, size_t need) { - if (need <= *cap) { - return 0; - } + if (need <= *cap) { return 0; } size_t next_cap = *cap ? *cap : 4096u; while (next_cap < need) { next_cap *= 2u; } uint8_t *next = realloc(*buf, next_cap); - if (!next) { - return -1; - } + if (!next) { return -1; } *buf = next; *cap = next_cap; return 0; } -static int queue_packet(rpc_connection *conn, rpc_op op, uint8_t flags, - uint32_t proc_id, uint64_t call_id, +static int queue_packet(rpc_connection *conn, rpc_op op, uint8_t flags, uint32_t proc_id, uint64_t call_id, const uint8_t *payload, size_t payload_len) { - if (payload_len > RPC_MAX_PAYLOAD_SIZE) { - return -1; - } + if (payload_len > RPC_MAX_PAYLOAD_SIZE) { return -1; } uint8_t header_buf[RPC_HEADER_SIZE]; rpc_header header = { .op = op, @@ -197,39 +166,27 @@ static int queue_packet(rpc_connection *conn, rpc_op op, uint8_t flags, .size = (uint32_t)payload_len, .call_id = call_id, }; - if (rpc_header_encode(&header, header_buf) != 0) { - return -1; - } - if (append_bytes(&conn->write_buf, &conn->write_len, &conn->write_cap, - header_buf, sizeof(header_buf)) != 0) { - return -1; - } - if (append_bytes(&conn->write_buf, &conn->write_len, &conn->write_cap, payload, - payload_len) != 0) { + if (rpc_header_encode(&header, header_buf) != 0) { return -1; } + if (append_bytes(&conn->write_buf, &conn->write_len, &conn->write_cap, header_buf, sizeof(header_buf)) != 0) { return -1; } + if (append_bytes(&conn->write_buf, &conn->write_len, &conn->write_cap, payload, payload_len) != 0) { return -1; } worker_queue_write(conn->worker, conn); return 0; } -static int queue_string_error(rpc_connection *conn, uint32_t proc_id, - uint64_t call_id, const char *message) { +static int queue_string_error(rpc_connection *conn, uint32_t proc_id, uint64_t call_id, const char *message) { rpc_writer writer; rpc_writer_init(&writer); int rc = rpc_writer_string(&writer, message, (uint32_t)strlen(message)); - if (rc == 0) { - rc = queue_packet(conn, RPC_OP_ERROR, RPC_FLAG_NONE, proc_id, call_id, - writer.data, writer.len); - } + if (rc == 0) { rc = queue_packet(conn, RPC_OP_ERROR, RPC_FLAG_NONE, proc_id, call_id, writer.data, writer.len); } rpc_writer_free(&writer); return rc; } static void connection_destroy(rpc_connection *conn) { - if (!conn) { - return; - } + if (!conn) { return; } rpc_worker *worker = conn->worker; (void)rpc_backend_remove(worker->backend, conn->fd); close(conn->fd); @@ -240,17 +197,13 @@ static void connection_destroy(rpc_connection *conn) { while (*link && *link != conn) { link = &(*link)->next; } - if (*link == conn) { - *link = conn->next; - } + if (*link == conn) { *link = conn->next; } if (conn->write_queued) { rpc_connection **write_link = &worker->pending_writes; while (*write_link && *write_link != conn) { write_link = &(*write_link)->write_next; } - if (*write_link == conn) { - *write_link = conn->write_next; - } + if (*write_link == conn) { *write_link = conn->write_next; } conn->write_queued = 0; conn->write_next = NULL; } @@ -258,15 +211,11 @@ static void connection_destroy(rpc_connection *conn) { } static void maybe_close(rpc_connection *conn) { - if (conn->closing && conn->write_len == conn->write_off) { - connection_destroy(conn); - } + if (conn->closing && conn->write_len == conn->write_off) { connection_destroy(conn); } } static void worker_queue_write(rpc_worker *worker, rpc_connection *conn) { - if (conn->write_queued) { - return; - } + if (conn->write_queued) { return; } conn->write_queued = 1; conn->write_next = worker->pending_writes; worker->pending_writes = conn; @@ -293,22 +242,18 @@ static void on_call_done(rpc_call *call, void *user_data) { rpc_connection *conn = user_data; const rpc_writer *response = rpc_call_response(call); if (rpc_call_result(call) == 0) { - (void)queue_packet(conn, RPC_OP_RESPONSE, RPC_FLAG_NONE, - rpc_call_proc_id(call), rpc_call_id(call), response->data, + (void)queue_packet(conn, RPC_OP_RESPONSE, RPC_FLAG_NONE, rpc_call_proc_id(call), rpc_call_id(call), response->data, response->len); } else { - (void)queue_string_error(conn, rpc_call_proc_id(call), rpc_call_id(call), - rpc_call_error(call)); + (void)queue_string_error(conn, rpc_call_proc_id(call), rpc_call_id(call), rpc_call_error(call)); } } -static int handle_sync_call(rpc_connection *conn, const rpc_header *header, - rpc_route *route, const uint8_t *payload) { +static int handle_sync_call(rpc_connection *conn, const rpc_header *header, rpc_route *route, const uint8_t *payload) { rpc_value *args = NULL; size_t argc = 0; if (rpc_payload_decode(payload, header->size, &args, &argc) != 0) { - return queue_string_error(conn, header->proc_id, header->call_id, - "malformed payload"); + return queue_string_error(conn, header->proc_id, header->call_id, "malformed payload"); } rpc_ctx ctx = { @@ -320,19 +265,17 @@ static int handle_sync_call(rpc_connection *conn, const rpc_header *header, int result = route->handler(&ctx, args, argc, &response, route->user_data); int rc = 0; if (result == 0) { - rc = queue_packet(conn, RPC_OP_RESPONSE, RPC_FLAG_NONE, header->proc_id, - header->call_id, response.data, response.len); + rc = queue_packet(conn, RPC_OP_RESPONSE, RPC_FLAG_NONE, header->proc_id, header->call_id, response.data, + response.len); } else { - rc = queue_string_error(conn, header->proc_id, header->call_id, - "procedure failed"); + rc = queue_string_error(conn, header->proc_id, header->call_id, "procedure failed"); } rpc_writer_free(&response); rpc_values_free(args); return rc; } -static int handle_packet(rpc_connection *conn, const rpc_header *header, - const uint8_t *payload) { +static int handle_packet(rpc_connection *conn, const rpc_header *header, const uint8_t *payload) { static const void *dispatch[] = { [RPC_OP_RPC] = &&op_rpc, [RPC_OP_PING] = &&op_ping, @@ -341,50 +284,40 @@ static int handle_packet(rpc_connection *conn, const rpc_header *header, [RPC_OP_ERROR] = &&op_unsupported, }; - if ((size_t)header->op >= sizeof(dispatch) / sizeof(*dispatch) || - !dispatch[header->op]) { - goto op_unsupported; - } + if ((size_t)header->op >= sizeof(dispatch) / sizeof(*dispatch) || !dispatch[header->op]) { goto op_unsupported; } goto *dispatch[header->op]; op_ping: - return queue_packet(conn, RPC_OP_RESPONSE, RPC_FLAG_NONE, 0, header->call_id, - NULL, 0); + return queue_packet(conn, RPC_OP_RESPONSE, RPC_FLAG_NONE, 0, header->call_id, NULL, 0); op_disconnect: - conn->closing = 1; - return 0; + conn->closing = 1; + return 0; op_rpc: { - rpc_trace_worker_add(conn->worker->index, RPC_TRACE_WORKER_RPCS, 1); - rpc_route route; - uint64_t trace_route = rpc_trace_begin(); - if (rpc_routes_lookup(&conn->worker->server->routes, header->proc_id, &route) != 0) { - rpc_trace_end(RPC_TRACE_SERVER_ROUTE, trace_route); - return queue_string_error(conn, header->proc_id, header->call_id, - "unknown procedure"); - } + rpc_trace_worker_add(conn->worker->index, RPC_TRACE_WORKER_RPCS, 1); + rpc_route route; + uint64_t trace_route = rpc_trace_begin(); + if (rpc_routes_lookup(&conn->worker->server->routes, header->proc_id, &route) != 0) { rpc_trace_end(RPC_TRACE_SERVER_ROUTE, trace_route); + return queue_string_error(conn, header->proc_id, header->call_id, "unknown procedure"); + } + rpc_trace_end(RPC_TRACE_SERVER_ROUTE, trace_route); - if (!route.is_async) { - return handle_sync_call(conn, header, &route, payload); - } + if (!route.is_async) { return handle_sync_call(conn, header, &route, payload); } - uint64_t trace_schedule = rpc_trace_begin(); - if (rpc_scheduler_submit(conn->worker->scheduler, header->call_id, - header->proc_id, route.handler, route.user_data, - payload, header->size, on_call_done, conn) != 0) { - rpc_trace_end(RPC_TRACE_SERVER_SCHEDULE, trace_schedule); - return queue_string_error(conn, header->proc_id, header->call_id, - "scheduler failure"); - } + uint64_t trace_schedule = rpc_trace_begin(); + if (rpc_scheduler_submit(conn->worker->scheduler, header->call_id, header->proc_id, route.handler, route.user_data, + payload, header->size, on_call_done, conn) != 0) { rpc_trace_end(RPC_TRACE_SERVER_SCHEDULE, trace_schedule); - return 0; + return queue_string_error(conn, header->proc_id, header->call_id, "scheduler failure"); } + rpc_trace_end(RPC_TRACE_SERVER_SCHEDULE, trace_schedule); + return 0; +} op_unsupported: - return queue_string_error(conn, header->proc_id, header->call_id, - "unsupported operation"); + return queue_string_error(conn, header->proc_id, header->call_id, "unsupported operation"); } static void parse_available(rpc_connection *conn) { @@ -396,9 +329,7 @@ static void parse_available(rpc_connection *conn) { conn->closing = 1; break; } - if (conn->read_len - off - RPC_HEADER_SIZE < header.size) { - break; - } + if (conn->read_len - off - RPC_HEADER_SIZE < header.size) { break; } const uint8_t *payload = conn->read_buf + off + RPC_HEADER_SIZE; if (handle_packet(conn, &header, payload) != 0) { conn->closing = 1; @@ -419,15 +350,13 @@ static void connection_read(rpc_connection *conn) { uint64_t trace = rpc_trace_begin(); for (;;) { if (conn->read_cap - conn->read_len < RPC_READ_CHUNK) { - if (reserve_bytes(&conn->read_buf, &conn->read_cap, - conn->read_len + RPC_READ_CHUNK) != 0) { + if (reserve_bytes(&conn->read_buf, &conn->read_cap, conn->read_len + RPC_READ_CHUNK) != 0) { conn->closing = 1; rpc_trace_end(RPC_TRACE_SERVER_READ, trace); return; } } - ssize_t n = recv(conn->fd, conn->read_buf + conn->read_len, - conn->read_cap - conn->read_len, 0); + ssize_t n = recv(conn->fd, conn->read_buf + conn->read_len, conn->read_cap - conn->read_len, 0); if (n > 0) { conn->read_len += (size_t)n; parse_available(conn); @@ -446,9 +375,7 @@ static void connection_read(rpc_connection *conn) { rpc_trace_end(RPC_TRACE_SERVER_READ, trace); return; } - if (errno == EINTR) { - continue; - } + if (errno == EINTR) { continue; } conn->closing = 1; rpc_trace_end(RPC_TRACE_SERVER_READ, trace); return; @@ -459,15 +386,12 @@ static void connection_write(rpc_connection *conn) { rpc_trace_worker_add(conn->worker->index, RPC_TRACE_WORKER_WRITES, 1); uint64_t trace = rpc_trace_begin(); while (conn->write_off < conn->write_len) { - ssize_t n = send(conn->fd, conn->write_buf + conn->write_off, - conn->write_len - conn->write_off, 0); + ssize_t n = send(conn->fd, conn->write_buf + conn->write_off, conn->write_len - conn->write_off, 0); if (n > 0) { conn->write_off += (size_t)n; continue; } - if (n < 0 && errno == EINTR) { - continue; - } + if (n < 0 && errno == EINTR) { continue; } if (n < 0 && (errno == EAGAIN || errno == EWOULDBLOCK)) { (void)connection_set_interests(conn, RPC_BACKEND_READ | RPC_BACKEND_WRITE); rpc_trace_end(RPC_TRACE_SERVER_WRITE, trace); @@ -494,8 +418,7 @@ static int worker_add_connection(rpc_worker *worker, int fd) { conn->worker = worker; conn->next = worker->connections; worker->connections = conn; - if (rpc_backend_register(worker->backend, fd, RPC_BACKEND_READ, - (uintptr_t)conn) != 0) { + if (rpc_backend_register(worker->backend, fd, RPC_BACKEND_READ, (uintptr_t)conn) != 0) { connection_destroy(conn); return -1; } @@ -549,9 +472,7 @@ static void accept_ready(rpc_listener *listener) { socklen_t addr_len = sizeof(addr); int fd = accept(listener->fd, (struct sockaddr *)&addr, &addr_len); if (fd < 0) { - if (errno == EINTR) { - continue; - } + if (errno == EINTR) { continue; } rpc_trace_end(RPC_TRACE_SERVER_ACCEPT, trace); return; } @@ -559,10 +480,7 @@ static void accept_ready(rpc_listener *listener) { close(fd); continue; } - uint32_t idx = - atomic_fetch_add_explicit(&server->next_worker, 1, - memory_order_relaxed) % - server->worker_count; + uint32_t idx = atomic_fetch_add_explicit(&server->next_worker, 1, memory_order_relaxed) % server->worker_count; worker_enqueue_connection(&server->workers[idx], fd); } } @@ -574,27 +492,18 @@ static int worker_init(rpc_server *server, rpc_worker *worker, uint32_t index) { for (size_t i = 0; i < RPC_MAX_LISTENERS; ++i) { worker->listeners[i].fd = -1; } - if (pthread_mutex_init(&worker->pending_mutex, NULL) != 0) { - return -1; - } + if (pthread_mutex_init(&worker->pending_mutex, NULL) != 0) { return -1; } worker->pending_mutex_ready = 1; - if (rpc_backend_kqueue_create(&worker->backend) != 0) { - return -1; - } - if (rpc_fixed_arena_init(&worker->connection_arena, sizeof(rpc_connection), - RPC_CONNECTION_ARENA_CAPACITY) != 0) { - return -1; - } - if (rpc_scheduler_init(&worker->scheduler) != 0) { + if (rpc_backend_kqueue_create(&worker->backend) != 0) { return -1; } + if (rpc_fixed_arena_init(&worker->connection_arena, sizeof(rpc_connection), RPC_CONNECTION_ARENA_CAPACITY) != 0) { return -1; } + if (rpc_scheduler_init(&worker->scheduler) != 0) { return -1; } return 0; } static void worker_destroy(rpc_worker *worker) { - if (!worker) { - return; - } + if (!worker) { return; } rpc_connection *conn = worker->connections; while (conn) { rpc_connection *next = conn->next; @@ -629,16 +538,10 @@ static void worker_destroy(rpc_worker *worker) { } static int server_ensure_workers(rpc_server *server) { - if (server->workers_ready) { - return 0; - } - if (server->worker_count == 0) { - server->worker_count = cpu_count(); - } + if (server->workers_ready) { return 0; } + if (server->worker_count == 0) { server->worker_count = cpu_count(); } server->workers = calloc(server->worker_count, sizeof(*server->workers)); - if (!server->workers) { - return -1; - } + if (!server->workers) { return -1; } for (uint32_t i = 0; i < server->worker_count; ++i) { if (worker_init(server, &server->workers[i], i) != 0) { for (uint32_t j = 0; j <= i; ++j) { @@ -654,13 +557,9 @@ static int server_ensure_workers(rpc_server *server) { } int rpc_server_init(rpc_server **out_server) { - if (!out_server) { - return -1; - } + if (!out_server) { return -1; } rpc_server *server = calloc(1, sizeof(*server)); - if (!server) { - return -1; - } + if (!server) { return -1; } atomic_init(&server->stopping, false); atomic_init(&server->next_worker, 0); server->worker_count = cpu_count(); @@ -674,8 +573,7 @@ int rpc_server_init(rpc_server **out_server) { } int rpc_server_set_workers(rpc_server *server, uint32_t worker_count) { - if (!server || server->workers_ready || server->listening || worker_count == 0 || - worker_count > RPC_MAX_WORKERS) { + if (!server || server->workers_ready || server->listening || worker_count == 0 || worker_count > RPC_MAX_WORKERS) { return -1; } server->worker_count = worker_count; @@ -683,9 +581,7 @@ int rpc_server_set_workers(rpc_server *server, uint32_t worker_count) { } int rpc_server_bind(rpc_server *server, const char *host, const char *port) { - if (!server || !port || server_ensure_workers(server) != 0) { - return -1; - } + if (!server || !port || server_ensure_workers(server) != 0) { return -1; } struct addrinfo hints; memset(&hints, 0, sizeof(hints)); @@ -694,32 +590,23 @@ int rpc_server_bind(rpc_server *server, const char *host, const char *port) { hints.ai_flags = AI_PASSIVE; struct addrinfo *res = NULL; - if (getaddrinfo(host, port, &hints, &res) != 0) { - return -1; - } + if (getaddrinfo(host, port, &hints, &res) != 0) { return -1; } int ok = -1; for (struct addrinfo *ai = res; ai; ai = ai->ai_next) { - if (server->workers[0].listener_count >= RPC_MAX_LISTENERS) { - break; - } + if (server->workers[0].listener_count >= RPC_MAX_LISTENERS) { break; } rpc_worker *worker = &server->workers[0]; int fd = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol); - if (fd < 0) { - continue; - } + if (fd < 0) { continue; } int yes = 1; (void)setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)); struct sockaddr_storage addr; memcpy(&addr, ai->ai_addr, ai->ai_addrlen); - if (server->port != 0) { - sockaddr_set_port((struct sockaddr *)&addr, server->port); - } + if (server->port != 0) { sockaddr_set_port((struct sockaddr *)&addr, server->port); } - if (set_nonblock(fd) != 0 || - bind(fd, (struct sockaddr *)&addr, ai->ai_addrlen) != 0) { + if (set_nonblock(fd) != 0 || bind(fd, (struct sockaddr *)&addr, ai->ai_addrlen) != 0) { close(fd); continue; } @@ -747,21 +634,14 @@ uint16_t rpc_server_port(const rpc_server *server) { } int rpc_server_listen(rpc_server *server) { - if (!server) { - return -1; - } + if (!server) { return -1; } for (uint32_t wi = 0; wi < server->worker_count; ++wi) { rpc_worker *worker = &server->workers[wi]; for (size_t i = 0; i < worker->listener_count; ++i) { rpc_listener *listener = &worker->listeners[i]; - if (listen(listener->fd, SOMAXCONN) != 0) { - return -1; - } + if (listen(listener->fd, SOMAXCONN) != 0) { return -1; } uintptr_t user = ((uintptr_t)listener) | 1u; - if (rpc_backend_register(worker->backend, listener->fd, RPC_BACKEND_READ, - user) != 0) { - return -1; - } + if (rpc_backend_register(worker->backend, listener->fd, RPC_BACKEND_READ, user) != 0) { return -1; } } } server->listening = 1; @@ -777,12 +657,9 @@ static int worker_run(rpc_worker *worker) { uint64_t trace_poll = rpc_trace_begin(); int n = rpc_backend_poll(worker->backend, events, RPC_EVENT_BATCH, -1); rpc_trace_end(RPC_TRACE_SERVER_POLL_WAIT, trace_poll); - if (n < 0) { - return -1; - } + if (n < 0) { return -1; } rpc_trace_add(RPC_TRACE_SERVER_POLL_EVENTS, (uint64_t)n); - rpc_trace_worker_add(worker->index, RPC_TRACE_WORKER_POLL_EVENTS, - (uint64_t)n); + rpc_trace_worker_add(worker->index, RPC_TRACE_WORKER_POLL_EVENTS, (uint64_t)n); uint64_t trace_active = rpc_trace_begin(); for (int i = 0; i < n; ++i) { if (events[i].events & RPC_BACKEND_WAKE) { @@ -794,12 +671,8 @@ static int worker_run(rpc_worker *worker) { accept_ready(listener); } else { rpc_connection *conn = (rpc_connection *)events[i].user; - if ((events[i].events & RPC_BACKEND_READ) && !conn->closing) { - connection_read(conn); - } - if ((events[i].events & RPC_BACKEND_WRITE) && !conn->closing) { - connection_write(conn); - } + if ((events[i].events & RPC_BACKEND_READ) && !conn->closing) { connection_read(conn); } + if ((events[i].events & RPC_BACKEND_WRITE) && !conn->closing) { connection_write(conn); } maybe_close(conn); } } @@ -819,17 +692,12 @@ static void *worker_run_main(void *arg) { } int rpc_server_run(rpc_server *server) { - if (!server || !server->listening) { - return -1; - } + if (!server || !server->listening) { return -1; } - if (server->worker_count == 1) { - return worker_run(&server->workers[0]); - } + if (server->worker_count == 1) { return worker_run(&server->workers[0]); } for (uint32_t i = 0; i < server->worker_count; ++i) { - if (pthread_create(&server->workers[i].thread, NULL, worker_run_main, - &server->workers[i]) != 0) { + if (pthread_create(&server->workers[i].thread, NULL, worker_run_main, &server->workers[i]) != 0) { rpc_server_stop(server); for (uint32_t j = 0; j < i; ++j) { pthread_join(server->workers[j].thread, NULL); @@ -848,21 +716,15 @@ int rpc_server_run(rpc_server *server) { } void rpc_server_stop(rpc_server *server) { - if (!server) { - return; - } + if (!server) { return; } atomic_store_explicit(&server->stopping, true, memory_order_release); for (uint32_t i = 0; i < server->worker_count; ++i) { - if (server->workers && server->workers[i].backend) { - (void)rpc_backend_wake(server->workers[i].backend); - } + if (server->workers && server->workers[i].backend) { (void)rpc_backend_wake(server->workers[i].backend); } } } void rpc_server_destroy(rpc_server *server) { - if (!server) { - return; - } + if (!server) { return; } if (server->workers) { rpc_server_stop(server); for (uint32_t i = 0; i < server->worker_count; ++i) { @@ -874,23 +736,16 @@ void rpc_server_destroy(rpc_server *server) { } free(server->workers); } - if (server->routes_ready) { - rpc_routes_destroy(&server->routes); - } + if (server->routes_ready) { rpc_routes_destroy(&server->routes); } free(server); } -int rpc_server_add_route(rpc_server *server, uint32_t proc_id, - rpc_handler_fn handler, void *user_data) { - return server ? rpc_routes_add(&server->routes, proc_id, handler, user_data) - : -1; +int rpc_server_add_route(rpc_server *server, uint32_t proc_id, rpc_handler_fn handler, void *user_data) { + return server ? rpc_routes_add(&server->routes, proc_id, handler, user_data) : -1; } -int rpc_server_add_async_route(rpc_server *server, uint32_t proc_id, - rpc_handler_fn handler, void *user_data) { - return server ? rpc_routes_add_ex(&server->routes, proc_id, handler, - user_data, 1) - : -1; +int rpc_server_add_async_route(rpc_server *server, uint32_t proc_id, rpc_handler_fn handler, void *user_data) { + return server ? rpc_routes_add_ex(&server->routes, proc_id, handler, user_data, 1) : -1; } int rpc_server_remove_route(rpc_server *server, uint32_t proc_id) { diff --git a/src/trace.c b/src/trace.c index c4e1b3d..4516fef 100644 --- a/src/trace.c +++ b/src/trace.c @@ -10,59 +10,46 @@ typedef struct rpc_trace_counter { } rpc_trace_counter; static const char *trace_names[RPC_TRACE_COUNT] = { - [RPC_TRACE_CLIENT_CALL] = "client.call", - [RPC_TRACE_CLIENT_SEND] = "client.send", - [RPC_TRACE_CLIENT_RECV] = "client.recv", - [RPC_TRACE_CLIENT_DECODE] = "client.decode", - [RPC_TRACE_SERVER_ACCEPT] = "server.accept", - [RPC_TRACE_SERVER_POLL_WAIT] = "server.poll_wait", - [RPC_TRACE_SERVER_POLL_EVENTS] = "server.poll_events", - [RPC_TRACE_SERVER_LOOP_ACTIVE] = "server.loop_active", - [RPC_TRACE_SERVER_READ] = "server.read", - [RPC_TRACE_SERVER_PARSE] = "server.parse", - [RPC_TRACE_SERVER_ROUTE] = "server.route", - [RPC_TRACE_SERVER_SCHEDULE] = "server.schedule", - [RPC_TRACE_SERVER_WRITE] = "server.write", - [RPC_TRACE_SCHED_SUBMIT] = "sched.submit", - [RPC_TRACE_SCHED_DECODE] = "sched.decode", - [RPC_TRACE_SCHED_CORO_CREATE] = "sched.coro_create", - [RPC_TRACE_SCHED_RESUME] = "sched.resume", - [RPC_TRACE_PAYLOAD_DECODE] = "payload.decode", - [RPC_TRACE_ROUTE_LOOKUP] = "route.lookup", + [RPC_TRACE_CLIENT_CALL] = "client.call", + [RPC_TRACE_CLIENT_SEND] = "client.send", + [RPC_TRACE_CLIENT_RECV] = "client.recv", + [RPC_TRACE_CLIENT_DECODE] = "client.decode", + [RPC_TRACE_SERVER_ACCEPT] = "server.accept", + [RPC_TRACE_SERVER_POLL_WAIT] = "server.poll_wait", + [RPC_TRACE_SERVER_POLL_EVENTS] = "server.poll_events", + [RPC_TRACE_SERVER_LOOP_ACTIVE] = "server.loop_active", + [RPC_TRACE_SERVER_READ] = "server.read", + [RPC_TRACE_SERVER_PARSE] = "server.parse", + [RPC_TRACE_SERVER_ROUTE] = "server.route", + [RPC_TRACE_SERVER_SCHEDULE] = "server.schedule", + [RPC_TRACE_SERVER_WRITE] = "server.write", + [RPC_TRACE_SCHED_SUBMIT] = "sched.submit", + [RPC_TRACE_SCHED_DECODE] = "sched.decode", + [RPC_TRACE_SCHED_CORO_CREATE] = "sched.coro_create", + [RPC_TRACE_SCHED_RESUME] = "sched.resume", + [RPC_TRACE_PAYLOAD_DECODE] = "payload.decode", + [RPC_TRACE_ROUTE_LOOKUP] = "route.lookup", }; static const int trace_is_time[RPC_TRACE_COUNT] = { - [RPC_TRACE_CLIENT_CALL] = 1, - [RPC_TRACE_CLIENT_SEND] = 1, - [RPC_TRACE_CLIENT_RECV] = 1, - [RPC_TRACE_CLIENT_DECODE] = 1, - [RPC_TRACE_SERVER_ACCEPT] = 1, - [RPC_TRACE_SERVER_POLL_WAIT] = 1, - [RPC_TRACE_SERVER_LOOP_ACTIVE] = 1, - [RPC_TRACE_SERVER_READ] = 1, - [RPC_TRACE_SERVER_PARSE] = 1, - [RPC_TRACE_SERVER_ROUTE] = 1, - [RPC_TRACE_SERVER_SCHEDULE] = 1, - [RPC_TRACE_SERVER_WRITE] = 1, - [RPC_TRACE_SCHED_SUBMIT] = 1, - [RPC_TRACE_SCHED_DECODE] = 1, - [RPC_TRACE_SCHED_CORO_CREATE] = 1, - [RPC_TRACE_SCHED_RESUME] = 1, - [RPC_TRACE_PAYLOAD_DECODE] = 1, - [RPC_TRACE_ROUTE_LOOKUP] = 1, + [RPC_TRACE_CLIENT_CALL] = 1, [RPC_TRACE_CLIENT_SEND] = 1, [RPC_TRACE_CLIENT_RECV] = 1, + [RPC_TRACE_CLIENT_DECODE] = 1, [RPC_TRACE_SERVER_ACCEPT] = 1, [RPC_TRACE_SERVER_POLL_WAIT] = 1, + [RPC_TRACE_SERVER_LOOP_ACTIVE] = 1, [RPC_TRACE_SERVER_READ] = 1, [RPC_TRACE_SERVER_PARSE] = 1, + [RPC_TRACE_SERVER_ROUTE] = 1, [RPC_TRACE_SERVER_SCHEDULE] = 1, [RPC_TRACE_SERVER_WRITE] = 1, + [RPC_TRACE_SCHED_SUBMIT] = 1, [RPC_TRACE_SCHED_DECODE] = 1, [RPC_TRACE_SCHED_CORO_CREATE] = 1, + [RPC_TRACE_SCHED_RESUME] = 1, [RPC_TRACE_PAYLOAD_DECODE] = 1, [RPC_TRACE_ROUTE_LOOKUP] = 1, }; static const char *trace_avg_units[RPC_TRACE_COUNT] = { - [RPC_TRACE_SERVER_POLL_EVENTS] = "evt", + [RPC_TRACE_SERVER_POLL_EVENTS] = "evt", }; static const char *trace_max_units[RPC_TRACE_COUNT] = { - [RPC_TRACE_SERVER_POLL_EVENTS] = "events", + [RPC_TRACE_SERVER_POLL_EVENTS] = "events", }; static rpc_trace_counter counters[RPC_TRACE_COUNT]; -static rpc_trace_counter worker_counters[RPC_TRACE_MAX_WORKERS] - [RPC_TRACE_WORKER_COUNT]; +static rpc_trace_counter worker_counters[RPC_TRACE_MAX_WORKERS][RPC_TRACE_WORKER_COUNT]; int rpc_trace_enabled = 0; @@ -97,19 +84,14 @@ void rpc_trace_end_slow(rpc_trace_metric metric, uint64_t start_ns) { } void rpc_trace_add_slow(rpc_trace_metric metric, uint64_t value) { - if ((unsigned)metric >= RPC_TRACE_COUNT) { - return; - } + if ((unsigned)metric >= RPC_TRACE_COUNT) { return; } rpc_trace_counter *counter = &counters[metric]; atomic_fetch_add_explicit(&counter->count, 1, memory_order_relaxed); atomic_fetch_add_explicit(&counter->total_ns, value, memory_order_relaxed); uint64_t old = atomic_load_explicit(&counter->max_ns, memory_order_relaxed); - while (old < value && - !atomic_compare_exchange_weak_explicit(&counter->max_ns, &old, value, - memory_order_relaxed, - memory_order_relaxed)) { - } + while (old < value && !atomic_compare_exchange_weak_explicit(&counter->max_ns, &old, value, memory_order_relaxed, + memory_order_relaxed)) {} } static void trace_counter_add(rpc_trace_counter *counter, uint64_t value) { @@ -117,24 +99,16 @@ static void trace_counter_add(rpc_trace_counter *counter, uint64_t value) { atomic_fetch_add_explicit(&counter->total_ns, value, memory_order_relaxed); uint64_t old = atomic_load_explicit(&counter->max_ns, memory_order_relaxed); - while (old < value && - !atomic_compare_exchange_weak_explicit(&counter->max_ns, &old, value, - memory_order_relaxed, - memory_order_relaxed)) { - } + while (old < value && !atomic_compare_exchange_weak_explicit(&counter->max_ns, &old, value, memory_order_relaxed, + memory_order_relaxed)) {} } -void rpc_trace_worker_end_slow(uint32_t worker, rpc_trace_worker_metric metric, - uint64_t start_ns) { +void rpc_trace_worker_end_slow(uint32_t worker, rpc_trace_worker_metric metric, uint64_t start_ns) { rpc_trace_worker_add_slow(worker, metric, rpc_trace_begin_slow() - start_ns); } -void rpc_trace_worker_add_slow(uint32_t worker, rpc_trace_worker_metric metric, - uint64_t value) { - if (worker >= RPC_TRACE_MAX_WORKERS || - (unsigned)metric >= RPC_TRACE_WORKER_COUNT) { - return; - } +void rpc_trace_worker_add_slow(uint32_t worker, rpc_trace_worker_metric metric, uint64_t value) { + if (worker >= RPC_TRACE_MAX_WORKERS || (unsigned)metric >= RPC_TRACE_WORKER_COUNT) { return; } trace_counter_add(&worker_counters[worker][metric], value); } @@ -146,12 +120,9 @@ void rpc_trace_reset(void) { } for (size_t worker = 0; worker < RPC_TRACE_MAX_WORKERS; ++worker) { for (size_t metric = 0; metric < RPC_TRACE_WORKER_COUNT; ++metric) { - atomic_store_explicit(&worker_counters[worker][metric].count, 0, - memory_order_relaxed); - atomic_store_explicit(&worker_counters[worker][metric].total_ns, 0, - memory_order_relaxed); - atomic_store_explicit(&worker_counters[worker][metric].max_ns, 0, - memory_order_relaxed); + atomic_store_explicit(&worker_counters[worker][metric].count, 0, memory_order_relaxed); + atomic_store_explicit(&worker_counters[worker][metric].total_ns, 0, memory_order_relaxed); + atomic_store_explicit(&worker_counters[worker][metric].max_ns, 0, memory_order_relaxed); } } } @@ -161,10 +132,8 @@ void rpc_trace_snapshot(rpc_trace_stat out[RPC_TRACE_COUNT]) { out[i] = (rpc_trace_stat){ .name = trace_names[i], .count = atomic_load_explicit(&counters[i].count, memory_order_relaxed), - .total = - atomic_load_explicit(&counters[i].total_ns, memory_order_relaxed), - .max = - atomic_load_explicit(&counters[i].max_ns, memory_order_relaxed), + .total = atomic_load_explicit(&counters[i].total_ns, memory_order_relaxed), + .max = atomic_load_explicit(&counters[i].max_ns, memory_order_relaxed), .is_time = trace_is_time[i], }; } @@ -173,24 +142,18 @@ void rpc_trace_snapshot(rpc_trace_stat out[RPC_TRACE_COUNT]) { void rpc_trace_dump(FILE *out) { rpc_trace_stat stats[RPC_TRACE_COUNT]; rpc_trace_snapshot(stats); - if (!out) { - out = stderr; - } + if (!out) { out = stderr; } fprintf(out, "\ntrace:\n"); - fprintf(out, " %-22s %12s %12s %12s\n", "metric", "count", "avg", - "max"); + fprintf(out, " %-22s %12s %12s %12s\n", "metric", "count", "avg", "max"); for (size_t i = 0; i < RPC_TRACE_COUNT; ++i) { - if (stats[i].count == 0) { - continue; - } + if (stats[i].count == 0) { continue; } if (stats[i].is_time) { char avg[16]; char max[16]; format_duration(stats[i].total / stats[i].count, avg); format_duration(stats[i].max, max); - fprintf(out, " %-22s %12llu %12s %12s\n", stats[i].name, - (unsigned long long)stats[i].count, avg, max); + fprintf(out, " %-22s %12llu %12s %12s\n", stats[i].name, (unsigned long long)stats[i].count, avg, max); } else { double avg = (double)stats[i].total / (double)stats[i].count; const char *avg_unit = trace_avg_units[i]; @@ -199,13 +162,10 @@ void rpc_trace_dump(FILE *out) { char avg_buf[24]; char max_buf[24]; snprintf(avg_buf, sizeof(avg_buf), "%.2f %s", avg, avg_unit); - snprintf(max_buf, sizeof(max_buf), "%llu %s", - (unsigned long long)stats[i].max, max_unit); - fprintf(out, " %-22s %12llu %12s %12s\n", stats[i].name, - (unsigned long long)stats[i].count, avg_buf, max_buf); + snprintf(max_buf, sizeof(max_buf), "%llu %s", (unsigned long long)stats[i].max, max_unit); + fprintf(out, " %-22s %12llu %12s %12s\n", stats[i].name, (unsigned long long)stats[i].count, avg_buf, max_buf); } else { - fprintf(out, " %-22s %12llu %12.2f %12llu\n", stats[i].name, - (unsigned long long)stats[i].count, avg, + fprintf(out, " %-22s %12llu %12.2f %12llu\n", stats[i].name, (unsigned long long)stats[i].count, avg, (unsigned long long)stats[i].max); } } @@ -213,67 +173,45 @@ void rpc_trace_dump(FILE *out) { int printed_workers = 0; for (size_t worker = 0; worker < RPC_TRACE_MAX_WORKERS; ++worker) { - rpc_trace_counter *poll_events = - &worker_counters[worker][RPC_TRACE_WORKER_POLL_EVENTS]; - rpc_trace_counter *accepts = - &worker_counters[worker][RPC_TRACE_WORKER_ACCEPTS]; - rpc_trace_counter *reads = - &worker_counters[worker][RPC_TRACE_WORKER_READS]; - rpc_trace_counter *rpcs = - &worker_counters[worker][RPC_TRACE_WORKER_RPCS]; - rpc_trace_counter *writes = - &worker_counters[worker][RPC_TRACE_WORKER_WRITES]; - rpc_trace_counter *active = - &worker_counters[worker][RPC_TRACE_WORKER_ACTIVE]; - - uint64_t polls_count = - atomic_load_explicit(&poll_events->count, memory_order_relaxed); - uint64_t polls_total = - atomic_load_explicit(&poll_events->total_ns, memory_order_relaxed); - uint64_t polls_max = - atomic_load_explicit(&poll_events->max_ns, memory_order_relaxed); - uint64_t accept_total = - atomic_load_explicit(&accepts->total_ns, memory_order_relaxed); - uint64_t read_total = - atomic_load_explicit(&reads->total_ns, memory_order_relaxed); - uint64_t rpc_total = - atomic_load_explicit(&rpcs->total_ns, memory_order_relaxed); - uint64_t write_total = - atomic_load_explicit(&writes->total_ns, memory_order_relaxed); - uint64_t active_count = - atomic_load_explicit(&active->count, memory_order_relaxed); - uint64_t active_total = - atomic_load_explicit(&active->total_ns, memory_order_relaxed); - uint64_t active_max = - atomic_load_explicit(&active->max_ns, memory_order_relaxed); - - if (polls_count == 0 && accept_total == 0 && read_total == 0 && - rpc_total == 0 && write_total == 0 && active_count == 0) { + rpc_trace_counter *poll_events = &worker_counters[worker][RPC_TRACE_WORKER_POLL_EVENTS]; + rpc_trace_counter *accepts = &worker_counters[worker][RPC_TRACE_WORKER_ACCEPTS]; + rpc_trace_counter *reads = &worker_counters[worker][RPC_TRACE_WORKER_READS]; + rpc_trace_counter *rpcs = &worker_counters[worker][RPC_TRACE_WORKER_RPCS]; + rpc_trace_counter *writes = &worker_counters[worker][RPC_TRACE_WORKER_WRITES]; + rpc_trace_counter *active = &worker_counters[worker][RPC_TRACE_WORKER_ACTIVE]; + + uint64_t polls_count = atomic_load_explicit(&poll_events->count, memory_order_relaxed); + uint64_t polls_total = atomic_load_explicit(&poll_events->total_ns, memory_order_relaxed); + uint64_t polls_max = atomic_load_explicit(&poll_events->max_ns, memory_order_relaxed); + uint64_t accept_total = atomic_load_explicit(&accepts->total_ns, memory_order_relaxed); + uint64_t read_total = atomic_load_explicit(&reads->total_ns, memory_order_relaxed); + uint64_t rpc_total = atomic_load_explicit(&rpcs->total_ns, memory_order_relaxed); + uint64_t write_total = atomic_load_explicit(&writes->total_ns, memory_order_relaxed); + uint64_t active_count = atomic_load_explicit(&active->count, memory_order_relaxed); + uint64_t active_total = atomic_load_explicit(&active->total_ns, memory_order_relaxed); + uint64_t active_max = atomic_load_explicit(&active->max_ns, memory_order_relaxed); + + if (polls_count == 0 && accept_total == 0 && read_total == 0 && rpc_total == 0 && write_total == 0 && + active_count == 0) { continue; } if (!printed_workers) { fprintf(out, "\nworker trace:\n"); - fprintf(out, - " %6s %8s %14s %10s %9s %9s %9s %9s %12s %12s\n", - "worker", "polls", "evt", "max_ev", "accepts", + fprintf(out, " %6s %8s %14s %10s %9s %9s %9s %9s %12s %12s\n", "worker", "polls", "evt", "max_ev", "accepts", "reads", "rpcs", "writes", "active_avg", "active_max"); printed_workers = 1; } char active_avg_buf[16]; char active_max_buf[16]; - format_duration(active_count ? active_total / active_count : 0, - active_avg_buf); + format_duration(active_count ? active_total / active_count : 0, active_avg_buf); format_duration(active_max, active_max_buf); - double events_per_poll = - polls_count ? (double)polls_total / (double)polls_count : 0.0; + double events_per_poll = polls_count ? (double)polls_total / (double)polls_count : 0.0; - fprintf(out, - " %6zu %8llu %14.2f %10llu %9llu %9llu %9llu %9llu %12s %12s\n", - worker, (unsigned long long)polls_count, events_per_poll, - (unsigned long long)polls_max, (unsigned long long)accept_total, - (unsigned long long)read_total, (unsigned long long)rpc_total, + fprintf(out, " %6zu %8llu %14.2f %10llu %9llu %9llu %9llu %9llu %12s %12s\n", worker, + (unsigned long long)polls_count, events_per_poll, (unsigned long long)polls_max, + (unsigned long long)accept_total, (unsigned long long)read_total, (unsigned long long)rpc_total, (unsigned long long)write_total, active_avg_buf, active_max_buf); } } diff --git a/tests/test_integration.c b/tests/test_integration.c index 02f9678..fff947c 100644 --- a/tests/test_integration.c +++ b/tests/test_integration.c @@ -5,12 +5,9 @@ #include #include -static int add_handler(rpc_ctx *ctx, const rpc_value *args, size_t argc, - rpc_writer *out, void *user_data) { +static int add_handler(rpc_ctx *ctx, const rpc_value *args, size_t argc, rpc_writer *out, void *user_data) { (void)user_data; - if (argc != 2 || args[0].type != RPC_TYPE_I64 || args[1].type != RPC_TYPE_I64) { - return -1; - } + if (argc != 2 || args[0].type != RPC_TYPE_I64 || args[1].type != RPC_TYPE_I64) { return -1; } rpc_ctx_yield(ctx); return rpc_writer_i64(out, args[0].as.i64 + args[1].as.i64); } diff --git a/tests/test_routes.c b/tests/test_routes.c index 56a8a89..01bcd1c 100644 --- a/tests/test_routes.c +++ b/tests/test_routes.c @@ -2,8 +2,7 @@ #include -static int handler_a(rpc_ctx *ctx, const rpc_value *args, size_t argc, - rpc_writer *out, void *user_data) { +static int handler_a(rpc_ctx *ctx, const rpc_value *args, size_t argc, rpc_writer *out, void *user_data) { (void)ctx; (void)args; (void)argc; @@ -12,8 +11,7 @@ static int handler_a(rpc_ctx *ctx, const rpc_value *args, size_t argc, return 0; } -static int handler_b(rpc_ctx *ctx, const rpc_value *args, size_t argc, - rpc_writer *out, void *user_data) { +static int handler_b(rpc_ctx *ctx, const rpc_value *args, size_t argc, rpc_writer *out, void *user_data) { (void)ctx; (void)args; (void)argc; diff --git a/tests/test_scheduler.c b/tests/test_scheduler.c index b2b1e49..bbd5162 100644 --- a/tests/test_scheduler.c +++ b/tests/test_scheduler.c @@ -4,8 +4,7 @@ static int done_count; -static int yielding_handler(rpc_ctx *ctx, const rpc_value *args, size_t argc, - rpc_writer *out, void *user_data) { +static int yielding_handler(rpc_ctx *ctx, const rpc_value *args, size_t argc, rpc_writer *out, void *user_data) { int *seen = user_data; assert(argc == 1); assert(args[0].type == RPC_TYPE_I64); @@ -36,8 +35,7 @@ int main(void) { rpc_writer_init(&payload); assert(rpc_writer_i64(&payload, 41) == 0); assert(rpc_scheduler_init(&scheduler) == 0); - assert(rpc_scheduler_submit(scheduler, 7, 9, yielding_handler, &seen, - payload.data, payload.len, on_done, NULL) == 0); + assert(rpc_scheduler_submit(scheduler, 7, 9, yielding_handler, &seen, payload.data, payload.len, on_done, NULL) == 0); rpc_scheduler_run_ready(scheduler); assert(seen == 2); assert(done_count == 1); -- 2.51.2