diff --git a/example.c b/example.c index fc9151c..8ff4ac7 100644 --- a/example.c +++ b/example.c @@ -8,62 +8,52 @@ int main() { textstore_t store; - if(textstore_init(&store, 64)) { - fprintf(stderr, "ERROR: textstore_init() failed.\n"); + if(textstore_init(&store, 64)) return 1; - } // A reference counted string handle. char *handle1_expected = "Hello world!"; textstore_handle handle1 = textstore_intern(&store, handle1_expected); - if (strcmp(textstore_string(&store, handle1), handle1_expected) != 0) { - fprintf(stderr, "ERROR: handle1 does not equal \"%s\"\n", handle1_expected); + if (strcmp(textstore_string(&store, handle1), handle1_expected) != 0) return 1; - } char *handle2_expected = "A different string!"; textstore_handle handle2 = textstore_intern(&store, handle2_expected); - if (strcmp(textstore_string(&store, handle2), handle2_expected)) { - fprintf(stderr, "ERROR: handle2 does not equal \"%s\"\n", handle2_expected); + if (strcmp(textstore_string(&store, handle2), handle2_expected)) return 1; - } // A handle may become invalidated after all references to it are released. textstore_release(&store, handle1); - if (textstore_gc(&store) != 0) { - fprintf(stderr, "ERROR: Garbage collector failed.\n"); + if (textstore_gc(&store) != 0) return 1; - } // Trying to get the data of an inactive handle should return null. char *invalid_string = textstore_string(&store, handle1); - if (invalid_string != NULL) { - fprintf(stderr, "ERROR: handle1 should be null, but isn't: \"%s\"\n", invalid_string); + if (invalid_string != NULL) return 1; - } // Interning new strings may re-use previously gc'd handle indexes. // But they are not equal due to their cycle bits. textstore_handle handle3 = textstore_intern(&store, "Hello world."); - if (handle1 == handle3) { - fprintf(stderr, "ERROR: handle1 and handle3 should not be equal.\n"); + if (handle1 == handle3) return 1; - } // And trying to access a previously gc'd handle will still return an error, // even if its index is in use. invalid_string = textstore_string(&store, handle1); - if (invalid_string != NULL) { - fprintf(stderr, "ERROR: handle1 should be null, but isn't: \"%s\"\n", invalid_string); + if (invalid_string != NULL) return 1; - } // Likewise, interning the same string will result in identical handles. textstore_handle handle4 = textstore_intern(&store, "Hello world."); - if (handle3 != handle4) { - fprintf(stderr, "ERROR: handle3 and handle4 should be equal.\n"); + if (handle3 != handle4) + return 1; + + // It's also possible to intern strings that aren't null-terminated. + textstore_handle handle5 = textstore_intern_len(&store, &(char){'.'}, 1); + // But the internally stored string is null-terminated for ecosystem compatibility. + if (strcmp(textstore_string(&store, handle5), ".") != 0) return 1; - } { textstore_handle temp_handle; @@ -77,18 +67,15 @@ int main() { textstore_release(&store, temp_handle); } - textstore_handle handle5 = textstore_intern(&store, "All your bases..."); + textstore_handle handle6 = textstore_intern(&store, "All your bases..."); textstore_gc(&store); // Even after a GC and the underlying text being invalidated and shifted // about, valid handles can still be used. - if (strcmp(textstore_string(&store, handle5), "All your bases...") != 0) { - fprintf(stderr, "ERROR: handle5 is wrong? \"%s\"\n", textstore_string(&store, handle5)); + if (strcmp(textstore_string(&store, handle6), "All your bases...") != 0) return 1; - } textstore_free(&store); printf("All tests passed.\n"); return 0; } - diff --git a/include/textstore.h b/include/textstore.h index 169b917..ab60bf4 100644 --- a/include/textstore.h +++ b/include/textstore.h @@ -52,7 +52,7 @@ typedef struct textstore_t { bool *active; /* Whether the given handle is currently active. */ uint32_t count; /* The number of string entries. */ - uint32_t capacity; /* The current capacity of the hashmap. */ + uint32_t hashmap_cap; /* The current capacity of the hashmap. */ } textstore_t; int textstore_init(textstore_t *store, uint32_t initial_buffer); diff --git a/src/textstore.c b/src/textstore.c index 79269d4..7f4c574 100644 --- a/src/textstore.c +++ b/src/textstore.c @@ -27,6 +27,14 @@ #include "rapidhash.h" +/** + * TODO: Make all of this easier to follow. + * In hindsight, I realize how much all of this relies on specific pointer math and such. + * It may be better to split some things into functions, macros or structs? + * This may be helped by splitting the free-queue into its own allocation, rather than + * sharing space with the text buffer? + */ + /* Size of struct-of-array member's items. */ #define SoASize (sizeof(uint32_t) + sizeof(textstore_slice) + (sizeof(uint32_t) * 2) + sizeof(bool)) @@ -42,7 +50,7 @@ static inline textstore_handle to_handle(uint32_t index, uint32_t generation) { * Note: Use on a temporary copy. */ static inline void textstore_alloc(textstore_t *store, size_t n) { - store->map = (uint32_t *)malloc(SoASize * n); + store->map = (uint32_t *)calloc(n, SoASize); store->key = (textstore_slice *)(store->map + n); store->ref = (uint32_t *)(store->key + n); store->generation = (uint32_t *)(store->ref + n); @@ -51,7 +59,7 @@ static inline void textstore_alloc(textstore_t *store, size_t n) { static void textstore_put(textstore_t *store, char *str, uint32_t len, uint32_t value) { uint32_t *map = store->map; - uint32_t capacity = store->capacity; + uint32_t capacity = store->hashmap_cap; uint32_t idx = rapidhash(str, len) % capacity; while (map[idx] != 0) { /* The 0 index is already reserved for empty string. */ idx = (idx + 1) % capacity; @@ -60,11 +68,11 @@ static void textstore_put(textstore_t *store, char *str, uint32_t len, uint32_t } static void textstore_rehash(textstore_t *store) { - char *buffer = store->buffer; - textstore_slice *slices = store->key + 1; /* Skip index 0, empty string. */ - bool *active = store->active + 1; - uint32_t count = store->count - 1; - memset(store->map, 0, sizeof(uint32_t) * store->capacity); + char *buffer = store->buffer; + textstore_slice *slices = store->key + 1; /* Skip index 0, empty string. */ + bool *active = store->active + 1; + uint32_t count = store->count - 1; + memset(store->map, 0, sizeof(uint32_t) * store->hashmap_cap); for (uint32_t i = 0; i < count; i++) { if (active[i]) { textstore_slice slice = slices[i]; @@ -77,20 +85,20 @@ static textstore_handle textstore_find(textstore_t *store, char *str, uint32_t l assert(str != NULL); assert(len != 0); - char *buffer = store->buffer; - uint32_t capacity = store->capacity; - uint32_t *map = store->map; - textstore_slice *key = store->key; - bool *active = store->active; + char *buffer = store->buffer; + uint32_t hashmap_cap = store->hashmap_cap; + uint32_t *map = store->map; + textstore_slice *key = store->key; + bool *active = store->active; - uint32_t index = rapidhash(str, len) % capacity; + uint32_t index = rapidhash(str, len) % hashmap_cap; while (map[index] != 0) { uint32_t idx = map[index]; if (key[idx].len != len || !active[idx]) { - index = (index + 1) % capacity; + index = (index + 1) % hashmap_cap; continue; } - if (strcmp(str, &buffer[key[idx].offset]) == 0) { + if (memcmp(str, &buffer[key[idx].offset], len) == 0) { return to_handle(idx, store->generation[idx]); } } @@ -108,6 +116,7 @@ static void textstore_append(textstore_t *store, char *str, uint32_t len) { new_capacity *= 2; } store->buffer = realloc(store->buffer, new_capacity); + memset(&store->buffer[store->buffer_cap], 0, new_capacity - store->buffer_cap); store->buffer_cap = new_capacity; } memcpy(&store->buffer[store->length], str, len); @@ -154,15 +163,15 @@ textstore_handle textstore_intern_len(textstore_t *store, char *str, uint32_t le store->ref[index] = 1; store->active[index] = true; - if (index > ((store->capacity/4)*3)) { + if (index > ((store->hashmap_cap/4)*3)) { /* At 3/4 capacity, grow the hash-table. */ textstore_t copy = *store; - textstore_alloc(©, copy.capacity * 2); + textstore_alloc(©, copy.hashmap_cap * 2); memcpy(copy.key, store->key, copy.count * sizeof(textstore_slice)); memcpy(copy.ref, store->ref, copy.count * sizeof(uint32_t)); memcpy(copy.generation, store->generation, copy.count * sizeof(uint32_t)); memcpy(copy.active, store->active, copy.count * sizeof(bool)); - copy.capacity *= 2; + copy.hashmap_cap *= 2; free(store->map); *store = copy; textstore_rehash(store); @@ -176,15 +185,37 @@ int textstore_gc(textstore_t *store) { uint32_t *refs = store->ref; bool *actives = store->active; uint32_t count = store->count; + uint32_t available = store->available; + + /** + * As a note for whoever is unfortunate enough to interact with my code (likely myself). + * To minimize the number of allocations, the textstore keeps the queue of freed handles + * and the actual text, in contiguous memory. The free queue is at the start of the buffer, + * and new strings get added to the end of the buffer. This works out, because the queue list + * only grows when textstore_gc() is caled, when all handles with 0 references are collected. + * + * The free-queue is last-in-first-out. + * + * [ 01 ] [ 02 ] H e l l o W o r l d ! ... + * ^ + * store->available = 2 + * + * So when a new string is interned, and there are handles available to use, the last item is + * popped off the stack. + * + * [ 01 ] [ 02 ] H e l l o W o r l d ! ... + * ^ + * store->available = 1 + */ // Pass 1: Count how many strings need to be collected, and determine how // much space is needed for the new buffer. - uint32_t collect = store->available; - uint32_t new_length = collect * sizeof(uint32_t); + uint32_t collect = 0; + uint32_t new_length = available * sizeof(uint32_t); for (uint32_t i = 0; i < count; i++) { if (actives[i]) { if (refs[i] > 0) { - new_length += strings[i].len; + new_length += strings[i].len + 1; } else { collect += 1; new_length += sizeof(uint32_t); @@ -196,17 +227,17 @@ int textstore_gc(textstore_t *store) { if (collect == 0) return 0; // Pass 2: Generate new buffer from the active strings. + if (new_length < store->buffer_cap) + new_length = store->buffer_cap; char *new_buffer = malloc(new_length); if (new_buffer == NULL) return 1; memset(new_buffer, 0, new_length); - uint32_t new_offset = collect * sizeof(uint32_t); + uint32_t new_offset = (collect + available) * sizeof(uint32_t); // copy existing free queue to the new one. - memcpy(new_buffer, store->buffer, store->available * sizeof(uint32_t)); + if (available) + memcpy(new_buffer, store->buffer, available * sizeof(uint32_t)); uint32_t *queue_buffer = (uint32_t *)new_buffer; - uint32_t queue_offset = store->available; - - // char * text_buffer = (char *)(&queue_buffer[collect]); - // uint32_t text_length = new_length - (sizeof(uint32_t) * collect); + uint32_t queue_offset = available; for (uint32_t i = 0; i < count; i++) { if (actives[i]) { @@ -216,6 +247,7 @@ int textstore_gc(textstore_t *store) { strings[i] = (textstore_slice){0}; continue; } + // Copy string to new location, and shift offsets. memcpy(&new_buffer[new_offset], &store->buffer[strings[i].offset], strings[i].len); strings[i].offset = new_offset; new_offset += strings[i].len + 1; @@ -224,7 +256,8 @@ int textstore_gc(textstore_t *store) { free(store->buffer); store->buffer = new_buffer; - store->available = collect; + store->buffer_cap = new_length; + store->available = collect + available; textstore_rehash(store); return 0; @@ -233,13 +266,12 @@ int textstore_gc(textstore_t *store) { int textstore_init(textstore_t *store, uint32_t initial_buffer) { textstore_t temp = {0}; if (initial_buffer == 0) initial_buffer = 1; - temp.buffer = (char *)malloc(initial_buffer); + temp.buffer = (char *)calloc(initial_buffer, 1); if (temp.buffer == NULL) return 1; temp.buffer_cap = initial_buffer; - memset(temp.buffer, 0, initial_buffer); - temp.capacity = 8; - textstore_alloc(&temp, temp.capacity); + temp.hashmap_cap = 8; + textstore_alloc(&temp, temp.hashmap_cap); /* Special-case empty string ("") for index 0. */ temp.key[0] = (textstore_slice){ 0, 0 }; diff --git a/xmake.lua b/xmake.lua index bca24df..241b17a 100644 --- a/xmake.lua +++ b/xmake.lua @@ -1,18 +1,19 @@ set_project("textstore") set_version("0.0.1") +set_description("Reference-counted, de-duplicated strings.") add_rules("mode.debug", "mode.release") +set_warnings("allextra", "pedantic") +set_languages("c99") + target("example") set_kind("binary") + set_default(false) add_deps("textstore") - set_warnings("allextra", "pedantic") - set_languages("c99") add_files("example.c") target("textstore") set_kind("object") - set_warnings("allextra", "pedantic") - set_languages("c99") add_headerfiles("include/textstore.h") add_includedirs("include", {public = true}) add_files("src/textstore.c")