From 34a860e4dd408e0bff97520464360921f20ba082 Mon Sep 17 00:00:00 2001 From: Hamcha Date: Fri, 19 Sep 2014 22:39:06 +0200 Subject: [PATCH 01/43] Get rid of the Erlang frontend I never liked port_driver.c anyway --- Makefile | 43 +-- c_src/port_driver.c | 620 -------------------------------------------- include/olegdb.hrl | 7 - run_server.sh | 47 ---- src/ol_database.erl | 135 ---------- src/ol_http.erl | 105 -------- src/ol_parse.erl | 89 ------- src/ol_route.erl | 113 -------- src/ol_util.erl | 69 ----- src/olegdb.app.src | 18 -- src/olegdb.erl | 124 --------- 11 files changed, 4 insertions(+), 1366 deletions(-) delete mode 100644 c_src/port_driver.c delete mode 100644 include/olegdb.hrl delete mode 100755 run_server.sh delete mode 100644 src/ol_database.erl delete mode 100644 src/ol_http.erl delete mode 100644 src/ol_parse.erl delete mode 100644 src/ol_route.erl delete mode 100644 src/ol_util.erl delete mode 100644 src/olegdb.app.src delete mode 100644 src/olegdb.erl diff --git a/Makefile b/Makefile index feac3a3..d8c5e55 100644 --- a/Makefile +++ b/Makefile @@ -13,13 +13,6 @@ INSTALL_LIB=$(PREFIX)/lib/ INSTALL_BIN=$(PREFIX)/bin/ INSTALL_INCLUDE=$(PREFIX)/include/ -ERL_LIB_LOOKFOR=-DLIBLOCATION=\"./build/lib/\" -ERLFLAGS=-smp -W1 -Werror -b beam -I./include -o $(BIN_DIR) $(ERL_LIB_LOOKFOR) -ERL_DIR=$(shell echo 'io:format("~s~n",[code:root_dir()]),init:stop().' | erl | sed -n '/^1>/s/^1> //p') -ERLI_DIR=$(shell echo 'io:format("~s~n",[code:lib_dir(erl_interface)]),init:stop().' | erl | sed -n '/^1>/s/^1> //p') -ERLINCLUDES=-I$(ERL_DIR)/usr/include/ -I$(ERLI_DIR)/include/ -ERLLIBS=-L$(ERL_DIR)/usr/lib/ -L$(ERLI_DIR)/lib/ -ERL_ODB_INSTALL_DIR=$(ERL_DIR)/lib/olegdb-$(VERSION) INCLUDES=-I./include MATH_LINKER= @@ -36,20 +29,11 @@ test.o: ./c_src/test.c main.o: ./c_src/main.c $(CC) $(CFLAGS) $(INCLUDES) -c $< -port_driver.o: ./c_src/port_driver.c - $(CC) $(CFLAGS) $(INCLUDES) $(ERLINCLUDES) -c $< -fpic - %.o: ./c_src/%.c $(CC) $(CFLAGS) $(INCLUDES) -c -fPIC $< FORCE: -$(BIN_DIR)ol_database.beam: ./src/ol_database.erl FORCE - erlc $(ERLFLAGS) $< - -$(BIN_DIR)%.beam: ./src/%.erl - erlc $(ERLFLAGS) $< - oleg_test: $(BIN_DIR)oleg_test $(BIN_DIR)oleg_test: liboleg test.o main.o $(CC) $(CFLAGS) $(INCLUDES) -L$(LIB_DIR) -o $(BIN_DIR)oleg_test test.o main.o $(MATH_LINKER) -loleg @@ -58,34 +42,15 @@ liboleg: $(LIB_DIR)liboleg.so $(LIB_DIR)liboleg.so: murmur3.o oleg.o logging.o aol.o rehash.o file.o utils.o tree.o lz4.o stack.o cursor.o data.o $(CC) $(CFLAGS) $(INCLUDES) -o $(LIB_DIR)liboleg.so $^ -fpic -shared $(MATH_LINKER) -server: $(BIN_DIR)ol_database.beam $(BIN_DIR)ol_http.beam \ - $(BIN_DIR)ol_parse.beam $(BIN_DIR)ol_util.beam $(BIN_DIR)olegdb.beam \ - $(BIN_DIR)ol_route.beam $(LIB_DIR)liboleg.so port_driver.o - $(CC) $(CFLAGS) $(INCLUDES) $(ERLINCLUDES) $(ERLLIBS) -L$(LIB_DIR) -o $(LIB_DIR)libolegserver.so port_driver.o -fpic -shared $(MATH_LINKER) -loleg -lei - uninstall: rm -rf $(INSTALL_LIB)/liboleg* rm -rf $(INSTALL_BIN)/olegdb - rm -rf $(ERL_ODB_INSTALL_DIR) -install: erlinstall +server: + +install: goinstall -# The reason we have install twice here is because the variable needs to be compiled install -# when we are installing. It tells erlang where to look -erlinstall: ERL_LIB_LOOKFOR=-DLIBLOCATION=\"$(INSTALL_LIB)\" -erlinstall: libinstall server - @mkdir -p $(INSTALL_BIN) - install $(LIB_DIR)libolegserver.so $(INSTALL_LIB)libolegserver.so.$(VERSION) - ln -fs $(INSTALL_LIB)libolegserver.so.$(VERSION) $(INSTALL_LIB)libolegserver.so - ln -fs $(INSTALL_LIB)libolegserver.so.$(VERSION) $(INSTALL_LIB)libolegserver.so.$(SOVERSION) - @mkdir -p $(ERL_ODB_INSTALL_DIR)/src - @mkdir -p $(ERL_ODB_INSTALL_DIR)/include - @mkdir -p $(ERL_ODB_INSTALL_DIR)/ebin - install $(BIN_DIR)*.beam $(ERL_ODB_INSTALL_DIR)/ebin - install ./include/*.hrl $(ERL_ODB_INSTALL_DIR)/include - install ./src/*.erl $(ERL_ODB_INSTALL_DIR)/src - install ./src/*.app.src $(ERL_ODB_INSTALL_DIR)/ebin - cp ./run_server.sh $(INSTALL_BIN)olegdb +goinstall: libinstall libinstall: liboleg @mkdir -p $(INSTALL_LIB) diff --git a/c_src/port_driver.c b/c_src/port_driver.c deleted file mode 100644 index f0bf95c..0000000 --- a/c_src/port_driver.c +++ /dev/null @@ -1,620 +0,0 @@ -/* -* This is where the magic happens. -*/ -#include -#include "ei.h" -#include "erl_driver.h" - -#include "aol.h" -#include "errhandle.h" -#include "file.h" -#include "cursor.h" -#include "oleg.h" -#include "logging.h" -#include "tree.h" - -/* Needed for R14B or earlier */ -#if (ERL_DRV_EXTENDED_MAJOR_VERSION < 2) -#define ErlDrvSizeT int -#endif - -/* This is used to store and manipulate state. */ -typedef struct { - ErlDrvPort port; - ol_splay_tree *databases; - char db_loc[255]; -} oleg_data; - -/* This should match the object in olegdb.htl */ -typedef struct { - char database_name[DB_NAME_SIZE]; - char *key; - size_t klen; - size_t data_len; - int version; - long expiration; - unsigned char *data; -} ol_record; - -static ErlDrvData oleg_start(ErlDrvPort port, char *buff) { - oleg_data *d = (oleg_data*)driver_alloc(sizeof(oleg_data)); - d->port = port; - d->databases = NULL; - d->db_loc[0] = '\0'; - return (ErlDrvData)d; -} - -static void oleg_stop(ErlDrvData data) { - oleg_data *d = (oleg_data*)data; - ol_splay_tree *tree = d->databases; - - ol_cursor cursor; - check(olc_generic_init(tree, &cursor), "Could not init cursor."); - while(olc_step(&cursor)) { - ol_splay_tree_node *node = _olc_get_node(&cursor); - check(node != NULL, "Could not retrieve a node."); - ol_database *db = (ol_database *)node->ref_obj; - - if (db != NULL) - ol_close(db); - } - -error: - /* Fall through to here. */ - driver_free(data); - ols_close(tree); -} - -static void _gen_atom(ei_x_buff *to_send, const char *str) { - ei_x_new_with_version(to_send); - ei_x_encode_atom(to_send, str); -} - -/* Converts a string of binary data from erlang into something we can use */ -/* Buf is what we're reading from, index is where to start. */ -static ol_record *read_record(char *buf, int index) { - ol_record *new_obj = driver_alloc(sizeof(ol_record)); - int arity = 0; - char atom[64]; - long len = 0; /* Not really used, but ei_decode_binary wants it */ - int type = 0; /* Also not used. */ - int data_size = 0; /* Used to get the length of the data we'll be storing. */ - - /* TODO: Error checking in here somewhere. */ - if (ei_decode_version(buf, &index, &new_obj->version)) - ol_log_msg(LOG_WARN, "Could not decode version.\n"); - - /* Gives us how many items are in the tuple */ - if (ei_decode_tuple_header(buf, &index, &arity)) - ol_log_msg(LOG_WARN, "Could not decode tuple header.\n"); - - if (arity != 6) - ol_log_msg(LOG_WARN, "Arity was not as expected.\n"); - - if (ei_decode_atom(buf, &index, atom)) - ol_log_msg(LOG_WARN, "Could not decode ol_record atom\n"); - if (ei_decode_binary(buf, &index, new_obj->database_name, &len)) - ol_log_msg(LOG_WARN, "Could not get database name.\n"); - new_obj->database_name[len] = '\0'; - - if (ei_decode_long(buf, &index, &new_obj->expiration)) - ol_log_msg(LOG_WARN, "Could not get expiration.\n"); - - int key_size = -1; - int erl_type = -1; - if (ei_get_type(buf, &index, &erl_type, &key_size)) - ol_log_msg(LOG_WARN, "Could not get key size.\n"); - new_obj->key = driver_alloc(key_size); - - if (ei_decode_binary(buf, &index, new_obj->key, &len)) - ol_log_msg(LOG_WARN, "Could not get key.\n"); - new_obj->klen = len; - debug("Key: %s", new_obj->key); - debug("get_type klen: %zu", new_obj->klen); - - /* This stuff is all to get the data. */ - ei_get_type(buf, &index, &type, &data_size); - new_obj->data = NULL; - if (data_size > 0) { - new_obj->data = driver_alloc(data_size); - if (ei_decode_binary(buf, &index, new_obj->data, &len)) - ol_log_msg(LOG_WARN, "Could not get data.\n"); - new_obj->data_len = len; - debug("Data length: %zu", len); - } - - return new_obj; -} - -static void port_driver_init(oleg_data *d, char *cmd) { - /* ol_init */ - int tmp_index = 1; - int version = 0; - - if (ei_decode_version(cmd, &tmp_index, &version)) - ol_log_msg(LOG_WARN, "Could not decode version.\n"); - - /* Decode passed string into our persistent data structure: */ - if (ei_decode_string(cmd, &tmp_index, d->db_loc)) - ol_log_msg(LOG_WARN, "Could not get database location.\n"); - - /* Send back 'ok' */ - ei_x_buff to_send; - _gen_atom(&to_send, "ok"); - driver_output(d->port, to_send.buff, to_send.index); - ei_x_free(&to_send); -} - -static void port_driver_jar(oleg_data *d, ol_database *db, ol_record *obj) { - int res = 0; - res = ol_jar(db, obj->key, obj->klen, obj->data, obj->data_len); - - debug("Expiration: %li", obj->expiration); - if (obj->expiration != -1) { - struct tm new_expire; - time_t passed_time = (time_t)obj->expiration; - localtime_r(&passed_time, &new_expire); - ol_spoil(db, obj->key, obj->klen, &new_expire); - } - ei_x_buff to_send; - ei_x_new_with_version(&to_send); - - if (res != 0) - _gen_atom(&to_send, "error"); - else - _gen_atom(&to_send, "ok"); - - driver_output(d->port, to_send.buff, to_send.index); - ei_x_free(&to_send); -} - -static void port_driver_unjar(oleg_data *d, ol_database *db, ol_record *obj) { - size_t val_size; - /* TODO: Fix this when we have one clean function to retrieve content type - * and data at the same time. - */ - unsigned char *data = NULL; - int ret = ol_unjar_ds(db, obj->key, obj->klen, &data, &val_size); - ei_x_buff to_send; - ei_x_new_with_version(&to_send); - if (ret == 0) { - ei_x_encode_tuple_header(&to_send, 2); - ei_x_encode_atom(&to_send, "ok"); - ei_x_encode_binary(&to_send, data, val_size); - } else { - ei_x_encode_atom(&to_send, "not_found"); - } - driver_output(d->port, to_send.buff, to_send.index); - ei_x_free(&to_send); - free(data); -} - -static void port_driver_scoop(oleg_data *d, ol_database *db, ol_record *obj) { - int ret = ol_scoop(db, obj->key, obj->klen); - ei_x_buff to_send; - ei_x_new_with_version(&to_send); - if (ret == 0) - ei_x_encode_atom(&to_send, "ok"); - else - ei_x_encode_atom(&to_send, "not_found"); - driver_output(d->port, to_send.buff, to_send.index); - ei_x_free(&to_send); -} - -static void port_driver_bucket_meta(oleg_data *d, ol_database *db, ol_record *obj) { - int exists = ol_exists(db, obj->key, obj->klen); - ei_x_buff to_send; - ei_x_new_with_version(&to_send); - if (exists == 0) { - struct tm *time_retrieved = NULL; - time_retrieved = ol_expiration_time(db, obj->key, obj->klen); - - /* If we have an expiration time, send it back with the content type. */ - if (time_retrieved != NULL) { - ei_x_encode_tuple_header(&to_send, 3); - ei_x_encode_atom(&to_send, "ok"); - ei_x_encode_long(&to_send, (long)db->rcrd_cnt); - ei_x_encode_long(&to_send, (long)mktime(time_retrieved)); - } else { - ei_x_encode_tuple_header(&to_send, 2); - ei_x_encode_atom(&to_send, "ok"); - ei_x_encode_long(&to_send, (long)db->rcrd_cnt); - } - } else { - ei_x_encode_atom(&to_send, "not_found"); - } - driver_output(d->port, to_send.buff, to_send.index); - ei_x_free(&to_send); -} - -static void port_driver_not_found(oleg_data *d) { - /* Send something back so we're not blocking. */ - ei_x_buff to_send; - _gen_atom(&to_send, "not_found"); - driver_output(d->port, to_send.buff, to_send.index); - ei_x_free(&to_send); -} - -static void port_driver_error(oleg_data *d, const char *msg) { - /* Send something back so we're not blocking. */ - ei_x_buff to_send; - ei_x_new_with_version(&to_send); - ei_x_encode_tuple_header(&to_send, 2); - ei_x_encode_atom(&to_send, "error"); - ei_x_encode_binary(&to_send, msg, strlen(msg)); - driver_output(d->port, to_send.buff, to_send.index); - ei_x_free(&to_send); -} - -/* Common successful return methods used by _first, _last, _next and _prev. */ -static inline void port_driver_cursor_response(oleg_data *d, const ol_bucket *bucket, - unsigned char *data, const size_t val_size) { - - ei_x_buff to_send; - ei_x_new_with_version(&to_send); - ei_x_encode_tuple_header(&to_send, 3); - - /* Send back ok, content type, key for bucket, and value for bucket */ - ei_x_encode_atom(&to_send, "ok"); - ei_x_encode_binary(&to_send, bucket->key, bucket->klen); - ei_x_encode_binary(&to_send, data, val_size); - - driver_output(d->port, to_send.buff, to_send.index); - - ei_x_free(&to_send); - free(data); - return; -} - -static void port_driver_cursor_next(oleg_data *d, ol_database *db, ol_record *obj) { - char _key[KEY_SIZE] = {'\0'}; - size_t _klen = 0; - - ol_bucket *bucket = ol_get_bucket(db, obj->key, obj->klen, &_key, &_klen); - - if (bucket == NULL) - return port_driver_not_found(d); - - ol_splay_tree_node *node = bucket->node; - ol_splay_tree_node *maximum = ols_subtree_maximum(db->tree->root); - - /* Get the next successor to this node. */ - if (!_olc_next(&node, maximum) || node == bucket->node) { - /* Could not find next node. */ - port_driver_not_found(d); - return; - } - - /* Found next node. */ - unsigned char *data = NULL; - size_t val_size; - - ol_bucket *next_bucket = (ol_bucket *)node->ref_obj; - - /* Let ol_unjar_ds handle decompression and whatever else for us: */ - int ret = ol_unjar_ds(db, next_bucket->key, next_bucket->klen, &data, &val_size); - if (ret == 0) { - port_driver_cursor_response(d, next_bucket, data, val_size); - return; - } - - ol_log_msg(LOG_ERR, "Something went horribly wrong. We couldn't get the data of a bucket we just found in the tree."); - port_driver_not_found(d); - return; -} - -static void port_driver_cursor_prev(oleg_data *d, ol_database *db, ol_record *obj) { - char _key[KEY_SIZE] = {'\0'}; - size_t _klen = 0; - - ol_bucket *bucket = ol_get_bucket(db, obj->key, obj->klen, &_key, &_klen); - - if (bucket == NULL) - return port_driver_not_found(d); - - ol_splay_tree_node *node = bucket->node; - ol_splay_tree_node *minimum = ols_subtree_minimum(db->tree->root); - - /* Get the next successor to this node. */ - if (!_olc_prev(&node, minimum) || node == bucket->node) { - /* Could not find next node. */ - port_driver_not_found(d); - return; - } - - /* Found next node. */ - unsigned char *data = NULL; - size_t val_size; - - ol_bucket *prev_bucket = (ol_bucket *)node->ref_obj; - - /* Let ol_unjar_ds handle decompression and whatever else for us: */ - int ret = ol_unjar_ds(db, prev_bucket->key, prev_bucket->klen, &data, &val_size); - if (ret == 0) { - port_driver_cursor_response(d, prev_bucket, data, val_size); - return; - } - - ol_log_msg(LOG_ERR, "Something went horribly wrong. We couldn't get the data of a bucket we just found in the tree."); - port_driver_not_found(d); - return; -} - -static void port_driver_cursor_first(oleg_data *d, ol_database *db, ol_record *obj) { - ol_splay_tree_node *minimum = ols_subtree_minimum(db->tree->root); - - if (minimum == NULL) { - return port_driver_not_found(d); - } - - /* Found next node. */ - unsigned char *data = NULL; - size_t val_size; - - ol_bucket *bucket = (ol_bucket *)minimum->ref_obj; - - /* Let ol_unjar_ds handle decompression and whatever else for us: */ - int ret = ol_unjar_ds(db, bucket->key, bucket->klen, &data, &val_size); - if (ret == 0) { - port_driver_cursor_response(d, bucket, data, val_size); - return; - } - - ol_log_msg(LOG_ERR, "Something went horribly wrong. We couldn't get the data of a bucket we just found in the tree."); - port_driver_not_found(d); - return; -} - -static void port_driver_cursor_last(oleg_data *d, ol_database *db, ol_record *obj) { - ol_splay_tree_node *maximum = ols_subtree_maximum(db->tree->root); - - if (maximum == NULL) { - return port_driver_not_found(d); - } - - /* Found next node. */ - unsigned char *data = NULL; - size_t val_size; - - ol_bucket *bucket = (ol_bucket *)maximum->ref_obj; - - /* Let ol_unjar_ds handle decompression and whatever else for us: */ - int ret = ol_unjar_ds(db, bucket->key, bucket->klen, &data, &val_size); - if (ret == 0) { - port_driver_cursor_response(d, bucket, data, val_size); - return; - } - - ol_log_msg(LOG_ERR, "Something went horribly wrong. We couldn't get the data of a bucket we just found in the tree."); - port_driver_not_found(d); - return; -} - -static void port_driver_prefix_match(oleg_data *d, ol_database *db, ol_record *obj) { - ol_val_array matches_list = NULL; - int match_num = ol_prefix_match(db, obj->key, obj->klen, &matches_list); - if (match_num > 0) { - ei_x_buff to_send; - ei_x_new_with_version(&to_send); - - /* Response looks like this: */ - /* {ok, ["list", "of", "matches"]} */ - ei_x_encode_tuple_header(&to_send, 3); - ei_x_encode_atom(&to_send, "ok"); - ei_x_encode_long(&to_send, (long)match_num); - ei_x_encode_list_header(&to_send, match_num); - - int i = 0; - for(; i < match_num; i++) { - /* Matches should be null terminated. Unless somebody fucks with - * something. */ - ei_x_encode_binary(&to_send, matches_list[i], strlen(matches_list[i])); - } - /* Apparently erlang is dumb and you need to end a list with an - * empty one. */ - ei_x_encode_empty_list(&to_send); - - driver_output(d->port, to_send.buff, to_send.index); - ei_x_free(&to_send); - - /* Free everything we just sent over the wire. */ - for (i = 0; i < match_num; i++) { - free(matches_list[i]); - } - free(matches_list); - - return; - } - - return port_driver_not_found(d); -} - -static int port_driver_squish(oleg_data *d, ol_database *db) { - if (db == NULL) - return 0; - - return ol_squish(db); -} - -static int port_driver_sync(oleg_data *d, ol_database *db) { - if (db == NULL) - return 0; - - return ol_sync(db); -} - -/* So this is where all the magic happens. If you want to know how we switch - * on different commands, go look at ol_database:encode/1. - */ -static void oleg_output(ErlDrvData data, char *cmd, ErlDrvSizeT clen) { - oleg_data *d = (oleg_data*)data; - int fn = cmd[0]; - ol_record *obj = NULL; - - debug("Command from server: %i", fn); - if (fn == 0) { - return port_driver_init(d, cmd); - } else if (fn == 9) { - if (d->databases == NULL) - return port_driver_error(d, "No databases initialized."); - - /* This is one of the more unique commands in that we don't - * need a decoded obj. We aren't even given one. Squish everyone. */ - ol_cursor cursor; - olc_generic_init(d->databases, &cursor); - - int ret = 1; - while(olc_step(&cursor)) { - ol_splay_tree_node *node = _olc_get_node(&cursor); - ol_database *db = (ol_database *)node->ref_obj; - - if (db != NULL) - ret = ret && port_driver_squish(d, db); - } - - if (ret) { - ei_x_buff to_send; - _gen_atom(&to_send, "ok"); - driver_output(d->port, to_send.buff, to_send.index); - ei_x_free(&to_send); - } else { - port_driver_error(d, "Could not squish all databases."); - } - - /* Don't do anything else. */ - return; - } else if (fn == 11) { - if (d->databases == NULL) - return port_driver_error(d, "No databases initialized."); - - /* Similar to squish above. */ - ol_cursor cursor; - olc_generic_init(d->databases, &cursor); - - int ret = 0; - while(olc_step(&cursor)) { - ol_splay_tree_node *node = _olc_get_node(&cursor); - ol_database *db = (ol_database *)node->ref_obj; - - if (db != NULL) - ret = ret && port_driver_sync(d, db); - } - - if (ret) { - ei_x_buff to_send; - _gen_atom(&to_send, "ok"); - driver_output(d->port, to_send.buff, to_send.index); - ei_x_free(&to_send); - } else { - port_driver_error(d, "Could not fsync all databases."); - } - - /* Don't do anything else. */ - return; - } - - /* Check to see if someone called ol_init */ - if (d->db_loc[0] == '\0') { - ei_x_buff to_send; - _gen_atom(&to_send, "no_db_location"); - driver_output(d->port, to_send.buff, to_send.index); - ei_x_free(&to_send); - return; - } - - /* Turn Erlang into Oleg */ - obj = read_record(cmd, 1); - - /* Open up a db if we don't have on already */ - if (d->databases == NULL) { - ols_init(&(d->databases)); - } - - /* Find the database in the list of opened databases. */ - const size_t dbname_len = strnlen(obj->database_name, DB_NAME_SIZE); - ol_splay_tree_node *found = ols_find(d->databases, obj->database_name, dbname_len); - - /* If we didn't find it, create it. */ - if (found == NULL && obj->database_name != NULL) { - ol_database *db = NULL; - db = ol_open(d->db_loc, obj->database_name, OL_F_APPENDONLY | OL_F_LZ4 | OL_F_SPLAYTREE); - if (db == NULL) - return port_driver_error(d, "Could not open database."); - ol_splay_tree_node *node = ols_insert(d->databases, obj->database_name, dbname_len, db); - found = node; - } else if (obj->database_name == NULL) { - return port_driver_error(d, "Something went wrong, trying to operate on a null database."); - } - - ol_database *found_db = (ol_database *)found->ref_obj; - - /* Figure out what the frontend wants us to do */ - switch (fn) { - case 1: - port_driver_jar(d, found_db, obj); - break; - case 2: - port_driver_unjar(d, found_db, obj); - break; - case 3: - port_driver_scoop(d, found_db, obj); - break; - case 4: - port_driver_bucket_meta(d, found_db, obj); - break; - case 5: - port_driver_cursor_next(d, found_db, obj); - break; - case 6: - port_driver_cursor_prev(d, found_db, obj); - break; - case 7: - port_driver_cursor_first(d, found_db, obj); - break; - case 8: - port_driver_cursor_last(d, found_db, obj); - break; - case 10: - port_driver_prefix_match(d, found_db, obj); - break; - default: - port_driver_not_found(d); - } - - if (obj) { - if (obj->data) driver_free(obj->data); - driver_free(obj); - driver_free(obj->key); - } -} - -/* Various callbacks */ -ErlDrvEntry ol_driver_entry = { - NULL, - oleg_start, - oleg_stop, - oleg_output, - NULL, - NULL, - "libolegserver", - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - ERL_DRV_EXTENDED_MARKER, - ERL_DRV_EXTENDED_MAJOR_VERSION, - ERL_DRV_EXTENDED_MINOR_VERSION, - 0, - NULL, - NULL, - NULL -}; - -DRIVER_INIT(libolegserver) { - return &ol_driver_entry; -} diff --git a/include/olegdb.hrl b/include/olegdb.hrl deleted file mode 100644 index d38af3a..0000000 --- a/include/olegdb.hrl +++ /dev/null @@ -1,7 +0,0 @@ -%%% Erlang record shared between many functions. --define(DEFAULT_DBNAME, <<"oleg">>). --record(ol_record, {database = ?DEFAULT_DBNAME, - expiration_time=-1, - key, % Binary - value = <<>>, - content_length=0}). diff --git a/run_server.sh b/run_server.sh deleted file mode 100755 index d2795bb..0000000 --- a/run_server.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash - -ATTEMPT=0 -handle_close() { - if [ $ATTEMPT -eq 0 ]; then - ATTEMPT=1 - #echo "Shutting down oleg..." - KILLSTRING="{satan, olegdb@$(hostname)} ! {shutdown, self()}, receive _ -> halt() end." - #echo "Running $KILLSTRING" - erl +Bd -noinput -sname the_judge_$$ -eval "$KILLSTRING" - #echo "Shutdown." - else - echo "Already tried." - exit 0 - fi -} -trap handle_close SIGINT SIGTERM - -export ERL_MAX_PORTS=4096 -RUN_DIR=/run/olegdb -PID_FILE=$RUN_DIR/pid -#TODO: +Bc here might be the trick -ERL_OPTS="+Bi -sname olegdb -noinput +K true -smp enable -run olegdb main ${1+"$@"}" - - -CMD_NAME=$(basename $0) -if [ "$CMD_NAME" = 'olegdb' ]; then - if [ -e $PID_FILE ] - then - echo "Another instance of OlegDB is already running." - exit -1 - else - echo "Starting OlegDB..." - echo $$ > $PID_FILE - fi -else - ERL_OPTS="-pa ./build/bin "$ERL_OPTS - # DEBUG mode - export PATH=./build/bin/:$PATH - export LD_LIBRARY_PATH=./build/lib/:$LD_LIBRARY_PATH -fi - -# TODO: Have the trap up above intercept the ctrl+c before this does -#erl $ERL_OPTS -erl $ERL_OPTS & -ERL_PID=$! -wait $ERL_PID diff --git a/src/ol_database.erl b/src/ol_database.erl deleted file mode 100644 index 4f202bc..0000000 --- a/src/ol_database.erl +++ /dev/null @@ -1,135 +0,0 @@ -%%% This is mainly where communcation with liboleg happens. You will -%%% see functions dealing with encoding, internal calls and the port -%%% driver interface. --module(ol_database). --export([start/0, - init/1, - ol_init/1, - ol_jar/1, - ol_unjar/1, - ol_bucket_meta/1, - ol_next_key/1, - ol_prev_key/1, - ol_first_key/1, - ol_last_key/1, - ol_squish/0, - ol_sync/0, - ol_prefix_match/1, - ol_scoop/1]). - --include("olegdb.hrl"). --define(SHAREDLIB, "libolegserver"). - -start() -> - code:add_path(?LIBLOCATION), - case erl_ddll:load_driver(?LIBLOCATION, ?SHAREDLIB) of - ok -> ok; - {error, already_loaded} -> ok; - {error, ErrorDesc} -> exit(erl_ddll:format_error(ErrorDesc)) - end, - %% Spawn of the looper - Me = self(), - spawn(fun() -> ?MODULE:init(Me) end), - receive - spawned -> ok - end. - -init(Papa) -> - register(olegdb_port_driver, self()), - Port = open_port({spawn, ?SHAREDLIB}, [binary]), - Papa ! spawned, - loop(Port). - -encode({ol_init, X}) -> [0, term_to_binary(X)]; -encode({ol_jar, X}) -> [1, term_to_binary(X)]; -encode({ol_unjar, X}) -> [2, term_to_binary(X)]; -encode({ol_scoop, X}) -> [3, term_to_binary(X)]; -encode({ol_bucket_meta, X}) -> [4, term_to_binary(X)]; -encode({ol_next_key, X}) -> [5, term_to_binary(X)]; -encode({ol_prev_key, X}) -> [6, term_to_binary(X)]; -encode({ol_first_key, X}) -> [7, term_to_binary(X)]; -encode({ol_last_key, X}) -> [8, term_to_binary(X)]; -encode({ol_squish}) -> [9]; -encode({ol_prefix_match, X}) -> [10, term_to_binary(X)]; -encode({ol_sync}) -> [11]; -encode(_) -> - io:format("Don't know how to decode that.~n"), - exit(unknown_call). - -%% TODO: Keep some kind of 'last-compacted' state here so we avoid having -%% unnecessary IO. -loop(Port) -> - %% Wait for someone to call for something - %io:format("Queue size: ~p~n", [erlang:process_info(self(), message_queue_len)]), - receive - {call, Caller, Msg} -> - %% Send that to liboleg - Port ! {self(), {command, encode(Msg)}}, - receive - %% Give the caller our result - {Port, {data, Data}} -> - Caller ! {olegdb_port_driver, binary_to_term(<>)}; - badarg -> - io:format("Badarg ~n"), - exit(port_terminated) - end, - loop(Port); - {shutdown, From} -> - io:format("[-] Shutting down.~n"), - Port ! {self(), close}, - receive - {Port, closed} -> - From ! {ok, self()}, - loop(Port) - end; - {'EXIT', Port, Reason} -> - io:format("~p ~n", [Reason]), - exit(port_terminated) - end. - -call_port(Msg) -> - olegdb_port_driver ! {call, self(), Msg}, - receive - {olegdb_port_driver, Result} -> - Result - end. - -ol_init(DbLocation) -> - call_port({ol_init, DbLocation}). - -ol_jar(OlRecord) -> - %io:format("[-] Expiration at this point: ~p~n", [OlRecord#ol_record.expiration_time]), - if - byte_size(OlRecord#ol_record.value) > 0 -> - call_port({ol_jar, OlRecord}); - true -> {error, <<"No data posted.">>} - end. - -ol_unjar(OlRecord) -> - call_port({ol_unjar, OlRecord}). - -ol_scoop(OlRecord) -> - call_port({ol_scoop, OlRecord}). - -ol_bucket_meta(OlRecord) -> - call_port({ol_bucket_meta, OlRecord}). - -ol_next_key(OlRecord) -> - call_port({ol_next_key, OlRecord}). - -ol_prev_key(OlRecord) -> - call_port({ol_prev_key, OlRecord}). - -ol_first_key(OlRecord) -> - call_port({ol_first_key, OlRecord}). - -ol_last_key(OlRecord) -> - call_port({ol_last_key, OlRecord}). - -ol_squish() -> - call_port({ol_squish}). - -ol_prefix_match(OlRecord) -> - call_port({ol_prefix_match, OlRecord}). - -ol_sync() -> call_port({ol_sync}). diff --git a/src/ol_http.erl b/src/ol_http.erl deleted file mode 100644 index a07e981..0000000 --- a/src/ol_http.erl +++ /dev/null @@ -1,105 +0,0 @@ -%%% HTTP responses/headers. --module(ol_http). --export([get_response/1, - not_found_response/0, - post_response/0, - deleted_response/0, - continue_you_shit_response/0, - bucket_meta_response/2, - bucket_meta_response/1, - cursor_bucket_response/2, - error_response/1, - prefix_response/2 - ]). - -get_response(Data) -> - io_lib:format( - <<"HTTP/1.1 200 OK\r\n" - "Server: OlegDB/fresh_cuts_n_jams\r\n" - "Content-Type: application/octet-stream\r\n" - "Content-Length: ~p\r\n" - "Connection: close\r\n" - "\r\n~s">>, [byte_size(Data), Data]). - -cursor_bucket_response(CursorKey, CursorData) -> - io_lib:format( - <<"HTTP/1.1 200 OK\r\n" - "Server: OlegDB/fresh_cuts_n_jams\r\n" - "Content-Type: application/octet-stream\r\n" - "Content-Length: ~p\r\n" - "Connection: close\r\n" - "X-OlegDB-Key: ~s\r\n" - "\r\n~s">>, [byte_size(CursorData), CursorKey, CursorData]). - -prefix_response(NumMatches, MatchesList) -> - Converted = ol_util:list_to_bad_json(MatchesList), - io_lib:format( - <<"HTTP/1.1 200 OK\r\n" - "Server: OlegDB/fresh_cuts_n_jams\r\n" - "Content-Type: application/json\r\n" - "Content-Length: ~p\r\n" - "Connection: close\r\n" - "X-OlegDB-Num-Matches: ~p\r\n" - "\r\n~s">>, [byte_size(Converted), NumMatches, Converted]). - -not_found_response() -> - <<"HTTP/1.1 404 Not Found\r\n" - "Status: 404 Not Found\r\n" - "Server: OlegDB/fresh_cuts_n_jams\r\n" - "Content-Length: 26\r\n" - "Connection: close\r\n" - "Content-Type: text/plain\r\n" - "\r\n" - "These aren't your ghosts.\n">>. - -post_response() -> - <<"HTTP/1.1 200 OK\r\n" - "Server: OlegDB/fresh_cuts_n_jams\r\n" - "Content-Type: text/plain\r\n" - "Connection: close\r\n" - "Content-Length: 7\r\n" - "\r\n" - "無駄\n">>. - -deleted_response() -> - <<"HTTP/1.1 200 OK\r\n" - "Server: OlegDB/fresh_cuts_n_jams\r\n" - "Content-Type: text/plain\r\n" - "Content-Length: 44\r\n" - "Connection: close\r\n" - "\r\n" - "The wind whispers through your empty forest.\n">>. - -continue_you_shit_response() -> - <<"HTTP/1.1 100 Continue\r\n" - "Server: OlegDB/fresh_cuts_n_jams\r\n" - "Content-Length: 0\r\n" - "\r\n">>. - -bucket_meta_response(RcrdCnt, Expires) -> - io_lib:format( - <<"HTTP/1.1 200 OK\r\n" - "Server: OlegDB/fresh_cuts_n_jams\r\n" - "Content-Length: 0\r\n" - "Content-Type: application/octet-stream\r\n" - "X-OlegDB-Rcrd-Cnt: ~p\r\n" - "Expires: ~p\r\n" - "\r\n">>, [RcrdCnt, Expires]). - -bucket_meta_response(RcrdCnt) -> - io_lib:format( - <<"HTTP/1.1 200 OK\r\n" - "Server: OlegDB/fresh_cuts_n_jams\r\n" - "Content-Length: 0\r\n" - "Content-Type: application/octet-stream\r\n" - "X-OlegDB-Rcrd-Cnt: ~p\r\n" - "\r\n">>, [RcrdCnt]). - -error_response(Data) -> - io:format("[x] Error: ~p~n", [Data]), - io_lib:format( - <<"HTTP/1.1 500 Internal Server Error\r\n" - "Server: OlegDB/fresh_cuts_n_jams\r\n" - "Content-Length: ~p\r\n" - "\r\n" - "~s">>, [byte_size(Data), Data]). diff --git a/src/ol_parse.erl b/src/ol_parse.erl deleted file mode 100644 index 221fc94..0000000 --- a/src/ol_parse.erl +++ /dev/null @@ -1,89 +0,0 @@ -%%% HTTP parsing. --module(ol_parse). --include("olegdb.hrl"). --export([parse_http/1]). - -parse_db_name_and_key(Data) -> - [FirstLine|_] = binary:split(Data, [<<"\r\n">>]), - % Actually Verb Url HttpVersion\r\n: - [Verb, Url|_] = binary:split(FirstLine, [<<" ">>], [global]), - ParsedUrl = parse_url(Url), - case Verb of - <<"GET">> -> {get, ParsedUrl}; - <<"POST">> -> {post, ParsedUrl}; - <<"HEAD">> -> {head, ParsedUrl}; - <<"DELETE">> -> {delete, ParsedUrl}; - Chunk -> - {error, <<"Didn't understand your verb.">>, Chunk} - end. - -parse_url(Url) -> - Split = binary:split(Url, [<<"/">>], [global]), - %io:format("S: ~p~n", [Split]), - case Split of - [<<>>, <<>>] -> {error, <<"No database or key specified.">>}; - % Url was like //key. Bad! - [_, <<>>, <<_/binary>>|_] -> {error, <<"No database specified.">>}; - % These handle cursor iteration: - [_, <>, <>, <<"_next">>|_] -> {ok, DB_Name, Key, {cursor, next}}; - [_, <>, <>, <<"_prev">>|_] -> {ok, DB_Name, Key, {cursor, prev}}; - [_, <>, <>, <<"_first">>|_] -> {ok, DB_Name, Key, {cursor, first}}; - [_, <>, <>, <<"_last">>|_] -> {ok, DB_Name, Key, {cursor, last}}; - % Prefix matching: - [_, <>, <>, <<"_match">>|_] -> {ok, DB_Name, Key, {prefix, match}}; - % Url was like /users/1 or /pictures/thing - [_, <>, <> |_] -> {ok, DB_Name, Key}; - % The url was like /test or /what, so just assume the default DB. - [_, <> |_] -> {ok, Key} - end. - -parse_http(Data) -> - case parse_db_name_and_key(Data) of - % The only difference between these two is // vs. / - {ReqType, {ok, DB_Name, Key}} -> - {ok, ReqType, parse_header(Data, #ol_record{database=DB_Name, key=Key} - )}; - % Default to whatever database name is specified in include/olegdb.hrl: - {ReqType, {ok, Key}} -> - {ok, ReqType, parse_header(Data, #ol_record{key=Key})}; - {ReqType, {ok, DB_Name, Key, Operand}} -> - Parsed = parse_header(Data, #ol_record{database=DB_Name, key=Key}), - {ok, ReqType, Parsed, Operand}; - {ReqType, {error, ErrorDesc}} -> {ReqType, {error, ErrorDesc}}; - X -> io:format("[-] Could not parse http in a sane way: ~p~n", [X]), - throw(oleg_bad_parse) - end. - -parse_header(Data, Record) -> - Split = binary:split(Data, [<<"\r\n\r\n">>]), - %io:format("Split: ~p~n", [Split]), - case Split of - [Header,PostedData|_] -> - LowercaseHeader = ol_util:bits_to_lower(Header), - parse_header1(binary:split(LowercaseHeader, [<<"\r\n">>], [global]), - { Record#ol_record{value=PostedData}, []}); - [Header|_] -> - LowercaseHeader = ol_util:bits_to_lower(Header), - parse_header1(binary:split(LowercaseHeader, [<<"\r\n">>], [global]), - { Record#ol_record{}, []}); - X -> io:format("[-] Could not parse header in a sane way: ~p~n", [X]), - throw(bad_parse) - end. - -%% Tail recursive function that maps over the lines in a header to fill out -%% an ol_record to later. -parse_header1([], {Record, Options}) -> {Record, Options}; -parse_header1([Line|Header], {Record, Options}) -> - case Line of - <<"expect: 100-continue">> -> - parse_header1(Header, {Record, Options ++ [send_100]}); - <<"content-length: ", CLength/binary>> -> - % This is only used for 100 requests - Len = list_to_integer(binary_to_list(CLength)), - parse_header1(Header, {Record#ol_record{content_length=Len}, Options}); - <<"x-olegdb-use-by: ", Timestamp/binary>> -> - Time = list_to_integer(binary_to_list(Timestamp)), - parse_header1(Header, {Record#ol_record{expiration_time=Time}, Options}); - _ -> - parse_header1(Header, {Record, Options}) - end. diff --git a/src/ol_route.erl b/src/ol_route.erl deleted file mode 100644 index 5c21850..0000000 --- a/src/ol_route.erl +++ /dev/null @@ -1,113 +0,0 @@ -%% General routery. --module(ol_route). --include("olegdb.hrl"). --export([cursor_operand/1, route/2]). - -cursor_response(Response) -> - case Response of - {ok, Key, Data} -> - ol_http:cursor_bucket_response(Key, Data); - {error, ErrMsg} -> ol_http:error_response(ErrMsg); - not_found -> - ol_http:not_found_response(); - _ -> ol_http:error_response(<<"Something went wrong.">>) - end. - -cursor_operand({ReqType, Header, Operand}) -> - if - ReqType == get -> - %io:format("[-] Requesting cursor ~p~n", [Header#ol_record.key]), - case Operand of - first -> cursor_response(ol_database:ol_first_key(Header)); - next -> cursor_response(ol_database:ol_next_key(Header)); - prev -> cursor_response(ol_database:ol_prev_key(Header)); - last -> cursor_response(ol_database:ol_last_key(Header)) - end; - true -> - ol_http:error_response(<<"Cursors do not support the requested verb.">>) - end. - -prefix_operand({ReqType, Header, Operand}) -> - if - ReqType == get -> - % Well we've only got one cursor operand right now but CASE - % STATEMENT ANYWAY - case Operand of - match -> - case ol_database:ol_prefix_match(Header) of - {ok, MatchNum, MatchesList} -> ol_http:prefix_response(MatchNum, MatchesList); - {error, ErrMsg} -> ol_http:error_response(ErrMsg); - not_found -> ol_http:not_found_response(); - X -> ol_http:error_response(X) - end - end; - true -> - ol_http:error_response(<<"Prefix matching does not support the requested verb.">>) - end. - -route(Bits, Socket) -> - case ol_parse:parse_http(Bits) of - {ok, _, {Header, [send_100|_]}} -> - hundred_handler(Header, Socket); - {ok, ReqType, {Header, _}, {cursor, Operand}} -> - cursor_operand({ReqType, Header, Operand}); - {ok, ReqType, {Header, _}, {prefix, Operand}} -> - prefix_operand({ReqType, Header, Operand}); - {ok, ReqType, {Header, _}} -> - case ReqType of - get -> - %io:format("[-] Requesting ~p~n", [Header#ol_record.key]), - %ol_http:not_found_response(); - case ol_database:ol_unjar(Header) of - {ok, Data} -> - ol_http:get_response(Data); - {error, ErrMsg} -> ol_http:error_response(ErrMsg); - _ -> ol_http:not_found_response() - end; - head -> - %io:format("[-] HEAD ~p~n", [Header#ol_record.key]), - case ol_database:ol_bucket_meta(Header) of - {ok, RcrdCnt} -> - ol_http:bucket_meta_response(RcrdCnt); - {ok, RcrdCnt, Expires} -> - ol_http:bucket_meta_response(RcrdCnt, Expires); - {error, ErrMsg} -> ol_http:error_response(ErrMsg); - _ -> ol_http:not_found_response() - end; - post -> - %io:format("[-] Posting to ~p~n", [Header#ol_record.key]), - NewHeader = ol_util:read_remaining_data(Header, Socket), - case ol_database:ol_jar(NewHeader) of - ok -> ol_http:post_response(); - {error, ErrMsg} -> ol_http:error_response(ErrMsg); - _ -> ol_http:not_found_response() - end; - delete -> - %io:format("[-] Deleting ~p~n", [Header#ol_record.key]), - case ol_database:ol_scoop(Header) of - ok -> ol_http:deleted_response(); - {error, ErrMsg} -> ol_http:error_response(ErrMsg); - _ -> ol_http:not_found_response() - end; - {error, ErrMsg} -> - ol_http:error_response(ErrMsg) - end; - {_, {error, ErrMsg}} -> - io:format("[X] Error ~p~n", [ErrMsg]), - ol_http:error_response(ErrMsg); - X -> - io:format("[X] Somebody requested something weird: ~p~n", [X]), - ol_http:error_response(<<"Make a better request next time.">>) - end. - -hundred_handler(Header, Socket) -> - case gen_tcp:send(Socket, ol_http:continue_you_shit_response()) of - ok -> - Data = ol_util:read_all_data(Socket, Header#ol_record.content_length), - case ol_database:ol_jar(Header#ol_record{value=Data}) of - ok -> ol_http:post_response(); - _ -> ol_http:not_found_response() - end; - {error, Reason} -> - io:format("[-] Could not send to socket: ~p~n", [Reason]) - end. diff --git a/src/ol_util.erl b/src/ol_util.erl deleted file mode 100644 index 430357d..0000000 --- a/src/ol_util.erl +++ /dev/null @@ -1,69 +0,0 @@ -%%% Common utility functions. --module(ol_util). --include("olegdb.hrl"). --export([read_all_data/2, read_remaining_data/2, bits_to_lower/1, list_to_bad_json/1]). -read_remaining_data(Header, Socket) -> - ExpectedLength = Header#ol_record.content_length, - if - byte_size(Header#ol_record.value) < ExpectedLength -> - Header#ol_record{value = - read_all_data1(Socket, - Header#ol_record.content_length, - Header#ol_record.value) - }; - true -> - Header - end. - -read_all_data(Socket, ExpectedLength) -> - read_all_data1(Socket, ExpectedLength, <<>>). -read_all_data1(Socket, ExpectedLength, Data) -> - case gen_tcp:recv(Socket, 0, 60000) of - {ok, ReadData} -> - Combined = <>, - if - byte_size(Combined) < ExpectedLength -> - %io:format("[-] Continuing to read. Byte size: ~p~n", [byte_size(Combined)]), - read_all_data1(Socket, ExpectedLength, Combined); - true -> - Combined - end; - {error, closed} -> - ok; - {error, timeout} -> - io:format("[-] Client timed out.~n"), - ok - end. - -bits_to_lower(<>) -> <<(string:to_lower(X)), (bits_to_lower(Rest))/bits>>; -bits_to_lower(_) -> <<>>. - -escape_special_func({ReplaceMe, ReplaceWith}, Accum) -> - %% Val is a 2-tuple of {replaceMe, replaceWith} - binary:replace(<>, ReplaceMe, ReplaceWith, [global]). - -escape_special(BinaryString) -> - AllChars = [ {<<"\\">>, <<"\\\\">>} - , {<<"\"">>, <<"\\\"">>} - , {<<"\b">>, <<"\\\b">>} - , {<<"\f">>, <<"\\\f">>} - , {<<"\n">>, <<"\\\n">>} - , {<<"\r">>, <<"\\\r">>} - , {<<"\t">>, <<"\\\t">>} - , {<<"\v">>, <<"\\\v">>} - , {<<"'">>, <<"\\\'">>} - ], - lists:foldl(fun(Val, Accum) -> escape_special_func(Val, Accum) end, <>, AllChars). - -list_to_bad_json1([], ByteString) -> <>; -list_to_bad_json1([ByteString|ListOfBits], Accumulator) when Accumulator == <<"[">> -> - % Don't encode a comma after the first value. - %io:format("Replacing ~p~n", [ByteString]), - Replaced = escape_special(ByteString), - list_to_bad_json1(ListOfBits, <>); -list_to_bad_json1([ByteString|ListOfBits], Accumulator) -> - %io:format("Replacing ~p~n", [ByteString]), - Replaced = escape_special(ByteString), - list_to_bad_json1(ListOfBits, <>). - -list_to_bad_json(ListOfBits) -> list_to_bad_json1(ListOfBits, <<"[">>). diff --git a/src/olegdb.app.src b/src/olegdb.app.src deleted file mode 100644 index 8b5a90f..0000000 --- a/src/olegdb.app.src +++ /dev/null @@ -1,18 +0,0 @@ -{application, OlegDB, [ - {description, "OlegDB, the Layman's Database"}, - {vsn, "0.1.5"}, - {modules, - [ ol_database - , olegdb - , ol_http - , ol_parse - , ol_util - ] - }, - {registered, [olegdb]}, - {env, - [ {} - ] - }, - {applications, [kernel, stdlib]} -]} diff --git a/src/olegdb.erl b/src/olegdb.erl deleted file mode 100644 index 24a92c7..0000000 --- a/src/olegdb.erl +++ /dev/null @@ -1,124 +0,0 @@ -%%% This is the main runtime of OlegDB. main/1 is where the magic happens. --module(olegdb). --include("olegdb.hrl"). --export([main/0, main/1, request_handler/1, do_accept/1]). - --define(DEFAULT_HOST, "localhost"). --define(DEFAULT_PORT, 8080). --define(ACCEPTOR_POOL_NUM, 64). - -% Recurring tasks: --define(COMPACTION_INTERVAL, 60 * 10000). % Five minutes --define(SYNC_INTERVAL, 10 * 1000). % 10 Seconds - -server_manager(Caller) -> - server_manager(Caller, ?DEFAULT_HOST, ?DEFAULT_PORT). - -server_manager(Caller, Port) -> - server_manager(Caller, ?DEFAULT_HOST, Port). - -server_manager(Caller, Hostname, Port) -> - {ok, Ip} = inet:getaddr(Hostname, inet), - case gen_tcp:listen(Port, [binary, {ip, Ip}, {active, false}, {reuseaddr, true}, {nodelay, true}, {backlog, 100}]) of - {ok, Sock} -> - io:format("[-] Listening on IP ~p, port ~p~n", [Ip, Port]), - gen_tcp:controlling_process(Sock, Caller), - % Spawn our acceptor pool workers - [spawn(?MODULE, do_accept, [Sock]) || _ <- lists:seq(0, ?ACCEPTOR_POOL_NUM)]; - {error, Reason} -> - io:format("[X] Could not listen: ~p~n", [Reason]) - end. - -%% Responsible for accepting new connections and spawning request handlers. -do_accept(Sock) -> - case gen_tcp:accept(Sock) of - {ok, Accepted} -> - %io:format("[-] Connection accepted!~n"), - spawn(?MODULE, request_handler, [Accepted]), - do_accept(Sock); - {error, Error} -> - io:format("[X] Could not accept a connection. Error: ~p~n", [Error]); - X -> X - end. - -request_handler(Accepted) -> - % Read in all data, timeout after 60 seconds - case gen_tcp:recv(Accepted, 0, 60000) of - {ok, Data} -> - send_handler(Data, Accepted); - {error, closed} -> ok; - {error, timeout} -> - io:format("[-] Client timed out.~n"), - ok - end. - -send_handler(Data, Accepted) -> - Resp = ol_route:route(Data, Accepted), - case gen_tcp:send(Accepted, Resp) of - {error, Reason} -> - io:format("[-] Could not send to socket: ~p~n", [Reason]); - _ -> ok - end, - ok = gen_tcp:close(Accepted). - -supervise() -> - %% Eventually this function will do something interesting. - %% We just sit here and block so that the parent process doesn't die - %% while it's children are off living fulfilling lives. - receive - {shutdown, From} -> - io:format("[-] Telling port driver to shut down.~n"), - io:format("[-] No.~n"), - olegdb_port_driver ! {shutdown, self()}, - receive - {ok, _} -> - From ! {ok, self()}, - io:format("[-] Night night.~n"), - halt() - end; - {compact} -> % We don't really care who the message is from. - %io:format("[-] Compacting...~n"), - ol_database:ol_squish(), - %io:format("[-] Done compacting.~n"), - erlang:send_after(?COMPACTION_INTERVAL, satan, {compact}); - {fsync} -> - % Sync every ten seconds - ol_database:ol_sync(), - erlang:send_after(?SYNC_INTERVAL, satan, {fsync}); - X -> io:format("[-] Receieved message: ~p~n", [X]) - end, - supervise(). - -main() -> main([]). -main([]) -> - io:format("[X] You must specify a location to store aol/dump files.~n"), - exit(not_enough_args); -main([DbLocation|Args]) -> - case filelib:is_dir(DbLocation) of - true -> - io:format("[-] Starting server.~n"), - ol_database:start(), - ol_database:ol_init(DbLocation), - io:format("[-] Args: ~p~n", [Args]), - case Args of - [] -> server_manager(self()); - [Port] -> - {PortNum, _} = string:to_integer(Port), - server_manager(self(), PortNum); - [Hostname, Port] -> - {PortNum, _} = string:to_integer(Port), - server_manager(self(), Hostname, PortNum) - end, - register(satan, self()), - io:format("[-] Death spiral. Running as ~p~n", [node()]), - - % Setup the compaction routine: - erlang:send_after(?COMPACTION_INTERVAL, satan, {compact}), - % Setup the fsync routine: - erlang:send_after(?SYNC_INTERVAL, satan, {fsync}), - - supervise(); - _ -> - io:format("[X] Specified database directory does not exist.~n"), - exit(location_does_not_exst) - end. From a6e70678d7776d599c14d99031a44fbdd8ac95ad Mon Sep 17 00:00:00 2001 From: Hamcha Date: Fri, 19 Sep 2014 23:44:14 +0200 Subject: [PATCH 02/43] Semiworking server (from goleg/d projects) Not working correctly yet --- Makefile | 10 +- frontend/goleg/highlevel.go | 54 +++++++++++ frontend/goleg/wrapper.go | 164 +++++++++++++++++++++++++++++++++ frontend/main.go | 176 ++++++++++++++++++++++++++++++++++++ olegdb.conf.sample | 7 ++ 5 files changed, 407 insertions(+), 4 deletions(-) create mode 100644 frontend/goleg/highlevel.go create mode 100644 frontend/goleg/wrapper.go create mode 100644 frontend/main.go create mode 100644 olegdb.conf.sample diff --git a/Makefile b/Makefile index d8c5e55..a24230c 100644 --- a/Makefile +++ b/Makefile @@ -43,14 +43,16 @@ $(LIB_DIR)liboleg.so: murmur3.o oleg.o logging.o aol.o rehash.o file.o utils.o t $(CC) $(CFLAGS) $(INCLUDES) -o $(LIB_DIR)liboleg.so $^ -fpic -shared $(MATH_LINKER) uninstall: - rm -rf $(INSTALL_LIB)/liboleg* - rm -rf $(INSTALL_BIN)/olegdb + rm -rf $(INSTALL_LIB)liboleg* + rm -rf $(INSTALL_BIN)olegdb -server: +server: liboleg + go build -o $(BIN_DIR)olegdb ./frontend install: goinstall -goinstall: libinstall +goinstall: server libinstall + cp $(BIN_DIR)olegdb $(INSTALL_BIN)olegdb libinstall: liboleg @mkdir -p $(INSTALL_LIB) diff --git a/frontend/goleg/highlevel.go b/frontend/goleg/highlevel.go new file mode 100644 index 0000000..232f30f --- /dev/null +++ b/frontend/goleg/highlevel.go @@ -0,0 +1,54 @@ +package goleg + +/* +#cgo CFLAGS: -I../include/ +#cgo LDFLAGS: -L../build/lib/ -loleg +#include +*/ +import "C" +import "time" + +type Database struct { + db *C.ol_database + RecordCount *C.int +} + +func Open(path, name string, features int) Database { + var database Database + database.db = COpen(path, name, features) + database.RecordCount = &database.db.rcrd_cnt + return database +} + +func (d Database) Close() int { + return CClose(d.db) +} + +func (d Database) Unjar(key string) []byte { + var dsize uintptr + return CUnjarDs(d.db, key, uintptr(len(key)), &dsize) +} + +func (d Database) Jar(key string, value []byte) int { + return CJar(d.db, key, uintptr(len(key)), value, uintptr(len(value))) +} + +func (d Database) Scoop(key string) int { + return CScoop(d.db, key, uintptr(len(key))) +} + +func (d Database) Uptime() int { + return CUptime(d.db) +} + +func (d Database) Expiration(key string) (time.Time, bool) { + return CExpirationTime(d.db, key, uintptr(len(key))) +} + +func (d Database) Spoil(key string, expiration time.Time) int { + return CSpoil(d.db, key, uintptr(len(key)), expiration) +} + +func (d Database) Exists(key string) bool { + return CExists(d.db, key, uintptr(len(key))) == 0 +} diff --git a/frontend/goleg/wrapper.go b/frontend/goleg/wrapper.go new file mode 100644 index 0000000..f378ca3 --- /dev/null +++ b/frontend/goleg/wrapper.go @@ -0,0 +1,164 @@ +package goleg + +/* +#cgo LDFLAGS: -loleg +#include +#include +*/ +import "C" +import "unsafe" +import "time" + +const F_APPENDONLY = C.OL_F_APPENDONLY +const F_LZ4 = C.OL_F_LZ4 +const F_SPLAYTREE = C.OL_F_SPLAYTREE + +func COpen(path, name string, features int) *C.ol_database { + // Turn parameters into their C counterparts + cpath := C.CString(path) + defer C.free(unsafe.Pointer(cpath)) + + cname := C.CString(name) + defer C.free(unsafe.Pointer(cname)) + + cfeats := C.int(features) + + // Pass them to ol_open + return C.ol_open(cpath, cname, cfeats) +} + +func CClose(database *C.ol_database) int { + return int(C.ol_close(database)) +} + +func CUnjar(db *C.ol_database, key string, klen uintptr, dsize uintptr) []byte { + // Turn parameters into their C counterparts + ckey := C.CString(key) + defer C.free(unsafe.Pointer(ckey)) + + cklen := (C.size_t)(klen) + + // Pass them to ol_unjar + var ptr *C.uchar + res := C.ol_unjar(db, ckey, cklen, &ptr) + if res == 1 { + return nil + } + // Retrieve data in Go []bytes + data := C.GoBytes(unsafe.Pointer(ptr), C.int(dsize)) + + // Free C pointer + C.free(unsafe.Pointer(ptr)) + + return data +} + +func CUnjarDs(db *C.ol_database, key string, klen uintptr, dsize *uintptr) []byte { + // Turn parameters into their C counterparts + ckey := C.CString(key) + defer C.free(unsafe.Pointer(ckey)) + + cklen := (C.size_t)(klen) + cdsize := (*C.size_t)(unsafe.Pointer(dsize)) + + // Pass them to ol_unjar_ds + var ptr *C.uchar + res := C.ol_unjar_ds(db, ckey, cklen, &ptr, cdsize) + if res == 1 { + return nil + } + // Retrieve data in Go []bytes + data := C.GoBytes(unsafe.Pointer(ptr), C.int(*dsize)) + + // Free C pointer + C.free(unsafe.Pointer(ptr)) + + return data +} + +func CJar(db *C.ol_database, key string, klen uintptr, value []byte, vsize uintptr) int { + // Turn parameters into their C counterparts + ckey := C.CString(key) + defer C.free(unsafe.Pointer(ckey)) + + cklen := (C.size_t)(klen) + cvsize := (C.size_t)(vsize) + + cvalue := (*C.uchar)(unsafe.Pointer(&value[0])) + + // Pass them to ol_jar + return int(C.ol_jar(db, ckey, cklen, cvalue, cvsize)) +} + +func CScoop(db *C.ol_database, key string, klen uintptr) int { + // Turn parameters into their C counterparts + ckey := C.CString(key) + defer C.free(unsafe.Pointer(ckey)) + + cklen := (C.size_t)(klen) + + // Pass them to ol_scoop + return int(C.ol_scoop(db, ckey, cklen)) +} + +func CUptime(db *C.ol_database) int { + return int(C.ol_uptime(db)) +} + +func CExpirationTime(db *C.ol_database, key string, klen uintptr) (time.Time, bool) { + // Turn parameters into their C counterparts + ckey := C.CString(key) + defer C.free(unsafe.Pointer(ckey)) + + cklen := (C.size_t)(klen) + + // Pass them to ol_expiration_time + ctime := C.ol_expiration_time(db, ckey, cklen) + + // Does the expiration exist? If no, return false as second param + if ctime == nil { + return time.Now(), false + } + + // turn ctime into a Go datatype + gotime := time.Date(int(ctime.tm_year)+1900, + time.Month(int(ctime.tm_mon)+1), + int(ctime.tm_mday), + int(ctime.tm_hour), + int(ctime.tm_min), + int(ctime.tm_sec), + 0, time.Local) + return gotime, true +} + +func CSpoil(db *C.ol_database, key string, klen uintptr, expiration time.Time) int { + // Turn parameters into their C counterparts + ckey := C.CString(key) + defer C.free(unsafe.Pointer(ckey)) + + cklen := (C.size_t)(klen) + + exp := expiration + + var ctime C.struct_tm + ctime.tm_year = C.int(exp.Year() - 1900) + ctime.tm_mon = C.int(int(exp.Month()) - 1) + ctime.tm_mday = C.int(exp.Day()) + ctime.tm_hour = C.int(exp.Hour()) + ctime.tm_min = C.int(exp.Minute()) + ctime.tm_sec = C.int(exp.Second()) + + // Pass them to ol_spoil + return int(C.ol_spoil(db, ckey, cklen, &ctime)) + +} + +func CExists(db *C.ol_database, key string, klen uintptr) int { + // Turn parameters into their C counterparts + ckey := C.CString(key) + defer C.free(unsafe.Pointer(ckey)) + + cklen := (C.size_t)(klen) + + return int(C.ol_exists(db, ckey, cklen)) +} diff --git a/frontend/main.go b/frontend/main.go new file mode 100644 index 0000000..46335c8 --- /dev/null +++ b/frontend/main.go @@ -0,0 +1,176 @@ +package main + +import ( + "./goleg" + "encoding/json" + "flag" + "fmt" + "io/ioutil" + "net/http" + "strconv" + "strings" + "time" +) + +type Config struct { + // Base settings (required) + Listen string + DataDir string + // HTTPS settings + UseHTTPS bool + CertFile string + PkeyFile string +} + +var config Config + +var databases map[string]goleg.Database + +func handler(w http.ResponseWriter, r *http.Request) { + // Check for a valid url + if r.URL.Path == "/" { + http.Error(w, "http://"+r.Host+"//", 400) + return + } + params := strings.Split(r.URL.Path[1:], "/") + if len(params) < 2 { + http.Error(w, "next time do a proper request", 400) + return + } + + // Get parameters + dbname := params[0] + key := params[1] + + // Get the database if loaded, show error otherwise + var database goleg.Database + var ok bool + if database, ok = databases[dbname]; !ok { + databases[dbname] = goleg.Open(config.DataDir, dbname, goleg.F_APPENDONLY|goleg.F_LZ4|goleg.F_SPLAYTREE) + //TODO Find a way to close unused databases + } + + // Get request? Get the value + if r.Method == "GET" { + value := database.Unjar(key) + // Check if the item existed + if value == nil { + http.Error(w, "Are you sure this is mayo?", 404) + return + } + + // Print value + fmt.Fprintf(w, string(value)) + return + } + + // Post request? Submit a value + if r.Method == "POST" { + value, err := ioutil.ReadAll(r.Body) + if err != nil { + http.Error(w, "Your post body is in klingon or something", 500) + return + } + + // Check if value already existed + exists := database.Exists(key) + + res := database.Jar(key, value) + if res == 0 { + // Status 201 if created, 200 if updated + if exists { + w.WriteHeader(200) + } else { + w.WriteHeader(201) + } + fmt.Fprintf(w, "Got a full jar!") + } else { + http.Error(w, "Wait, the jar vanished!", 500) + return + } + + // Try to set expiration, if provided + if eep, ok := r.Header["X-Olegdb-Use-By"]; ok { + ep, err := strconv.Atoi(eep[0]) + if err != nil { + http.Error(w, "The expiration date does not meet my hipster requirements", 400) + return + } + date := time.Unix(int64(ep), 0) + database.Spoil(key, date) + fmt.Fprintf(w, "\r\nThe jar is spoiling!") + return + } + + } + + // Delete request? Delete a value + if r.Method == "DELETE" { + res := database.Scoop(key) + if res == 0 { + fmt.Fprintf(w, "Scooping done, that'll be $5.10") + return + } else { + http.Error(w, "Got mayo all over my shirt!", 500) + return + } + } + + // Head request? Get key info + if r.Method == "HEAD" { + // Does it even exists? + if !database.Exists(key) { + http.Error(w, "Are you sure this is mayo?", 404) + return + } + + // Get and set Expiration + res, bl := database.Expiration(key) + if bl == true { + w.Header().Add("Expires", strconv.Itoa(int(res.UTC().Unix()))) + } + + // Add Record count + w.Header().Add("X-Olegdb-Rcrd-Cnt", strconv.Itoa(int(*database.RecordCount))) + + fmt.Fprintf(w, "\r\n") + } +} + +func main() { + // Parse command line flags (if there are) + directory := flag.String("dir", "", "Directory where to store dumps and data") + configfile := flag.String("config", "olegdb.conf", "Config file to read settings from") + bindaddr := flag.String("bind", "", "Address and port to bind, host:port") + + flag.Parse() + + // Parse config file + rawconf, err := ioutil.ReadFile(*configfile) + if err != nil { + panic(err.Error()) + } + err = json.Unmarshal(rawconf, &config) + if err != nil { + panic(err.Error()) + } + + databases = make(map[string]goleg.Database) + + // Override config option with command line ones + if *directory != "" { + config.DataDir = *directory + } + + if *bindaddr != "" { + config.Listen = *bindaddr + } + + http.HandleFunc("/", handler) + fmt.Println("Listening on " + config.Listen) + if config.UseHTTPS { + http.ListenAndServeTLS(config.Listen, config.CertFile, config.PkeyFile, nil) + } else { + http.ListenAndServe(config.Listen, nil) + } +} diff --git a/olegdb.conf.sample b/olegdb.conf.sample new file mode 100644 index 0000000..4f7e1dd --- /dev/null +++ b/olegdb.conf.sample @@ -0,0 +1,7 @@ +{ + "Listen" : "localhost:8080", + "DataDir" : "data", + "UseHTTPS" : false, + "CertFile" : "", + "PkeyFile" : "" +} \ No newline at end of file From a6920a3bd2fe66942cb48dffa133b7218969de03 Mon Sep 17 00:00:00 2001 From: Hamcha Date: Sat, 20 Sep 2014 02:15:44 +0200 Subject: [PATCH 03/43] Updated README --- README.md | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index a36e36c..6f58a65 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Dependencies ============ * A healthy fear of the end -* Erlang +* Go (>= 1.1) Installation ============ @@ -35,7 +35,7 @@ Currently builds are tested against gcc and clang. ```bash # Building everything: make -# Just the erlang beam files: +# Just the golang frontend: make server # Just the C library: make liboleg @@ -52,16 +52,11 @@ To run tests: To run the erlang server: ```bash -olegdb +olegdb [-conf olegdb.conf] [-bind localhost:8080] [-dir data] ``` +For an explaination of the command line parameters, do `olegdb -h` -You can optionally specify a port, host or both. But not just a host. - -```bash -olegdb [[host] port] -``` - -curl2sudo® install script coming soon. +curl2sudo© install script coming soon. Documentation ============= From 88af29c6f82867401fa64e9c711b829fdef816b9 Mon Sep 17 00:00:00 2001 From: Hamcha Date: Sat, 20 Sep 2014 02:24:22 +0200 Subject: [PATCH 04/43] More detailed CONTRIBUTING.md --- CONTRIBUTING.md | 7 +++++-- README.md | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 30ed503..b1d28d5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -44,20 +44,23 @@ with this one. ol_bucket *c, *v, *u, *zx; ```` We're probably not going to accept that pull request. +* No tabs, just spaces +* All the above applies for the C code, for the Go frontend we use `gofmt` rules (default settings) Architecture Decisions ---------------------- Communication is key. If you're making a rather big architecture decision, discuss it with others. A second pair of eyes often does wonders to reveal weird -or bad ideas. +or bad ideas. +If you don't feel like making an issue or want quick feedback on it, hit us on [IRC](https://olegdb.org/community.html#irc), we do most of our discussions in there. Code Review ----------- All code should be code-reviewed by at least one other person before being merged into master. Master is considered stable and should always pass tests. -Ping @qpfiffer for quick commentary. +Ping [@qpfiffer](github.com/qpfiffer) for quick commentary. Documentation ------------- diff --git a/README.md b/README.md index 6f58a65..c5ac15e 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ olegdb [-conf olegdb.conf] [-bind localhost:8080] [-dir data] ``` For an explaination of the command line parameters, do `olegdb -h` -curl2sudo© install script coming soon. +curl2sudo™ install script coming soon. Documentation ============= From fda97c568586825b72548211d48038ac58292676 Mon Sep 17 00:00:00 2001 From: Hamcha Date: Sun, 21 Sep 2014 02:45:24 +0200 Subject: [PATCH 05/43] Big refactor of Golegd to be better organized --- README.md | 2 +- frontend/dispatcher.go | 95 ++++++++++++++++++++++++++++++++++ frontend/main.go | 115 ----------------------------------------- frontend/operations.go | 99 +++++++++++++++++++++++++++++++++++ 4 files changed, 195 insertions(+), 116 deletions(-) create mode 100644 frontend/dispatcher.go create mode 100644 frontend/operations.go diff --git a/README.md b/README.md index c5ac15e..188b675 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ olegdb [-conf olegdb.conf] [-bind localhost:8080] [-dir data] ``` For an explaination of the command line parameters, do `olegdb -h` -curl2sudo™ install script coming soon. +curl2sudo® install script coming soon. Documentation ============= diff --git a/frontend/dispatcher.go b/frontend/dispatcher.go new file mode 100644 index 0000000..8a80d5b --- /dev/null +++ b/frontend/dispatcher.go @@ -0,0 +1,95 @@ +package main + +import ( + "./goleg" + "net/http" + "strings" +) + +type Operation struct { + Database *goleg.Database + Key string + Operation string +} + +type HTTPError struct { + Code int + Message string +} + +func handler(w http.ResponseWriter, r *http.Request) { + dbname, key, opname, err := getRequestInfo(r) + if err != nil { + http.Error(w, err.Message, err.Code) + return + } + + // Get the database if loaded, show error otherwise + var database goleg.Database + var ok bool + if database, ok = databases[dbname]; !ok { + databases[dbname] = goleg.Open(config.DataDir, dbname, goleg.F_APPENDONLY|goleg.F_LZ4|goleg.F_SPLAYTREE) + database = databases[dbname] + //TODO Find a way to close unused databases + } + + operation := Operation{ + Database: &database, + Key: key, + Operation: opname, + } + + switch operation.Operation { + case OpGet: + err = httpGet(w, operation) + case OpSet: + err = httpSet(w, operation, r) + case OpInfo: + err = httpInfo(w, operation) + case OpDelete: + err = httpDelete(w, operation) + default: + err = &HTTPError{Message: "I don't get what you're trying to do", Code: 400} + } + + if err != nil { + http.Error(w, err.Message, err.Code) + } +} + +func getRequestInfo(r *http.Request) (database, key, operation string, err *HTTPError) { + params := strings.Split(r.URL.Path[1:], "/") + if len(params) < 2 { + return "", "", "", &HTTPError{Code: 400, Message: "Not enought arguments in URL"} + } + + // Get parameters + database = params[0] + key = params[1] + operation = OpGet + + // Get operation name, it can either be: + // 1. An HTTP method that's not GET + if strings.ToUpper(r.Method) != "GET" { + operation = "/" + r.Method + } else + // 2. A third argument (Cursor iteration) + if len(params) > 2 { + operation = "." + params[2] + } else + // 3. A second argument starting with a _ (but not another _) + if key[0] == '_' { + // To get _key you do __key + if key[1] == '_' { + key = key[1:] + } else { + operation = key + key = "" + } + } + + // Case insensitive + operation = strings.ToLower(operation) + + return database, key, operation, nil +} diff --git a/frontend/main.go b/frontend/main.go index 46335c8..44cbc9d 100644 --- a/frontend/main.go +++ b/frontend/main.go @@ -7,9 +7,6 @@ import ( "fmt" "io/ioutil" "net/http" - "strconv" - "strings" - "time" ) type Config struct { @@ -23,120 +20,8 @@ type Config struct { } var config Config - var databases map[string]goleg.Database -func handler(w http.ResponseWriter, r *http.Request) { - // Check for a valid url - if r.URL.Path == "/" { - http.Error(w, "http://"+r.Host+"//", 400) - return - } - params := strings.Split(r.URL.Path[1:], "/") - if len(params) < 2 { - http.Error(w, "next time do a proper request", 400) - return - } - - // Get parameters - dbname := params[0] - key := params[1] - - // Get the database if loaded, show error otherwise - var database goleg.Database - var ok bool - if database, ok = databases[dbname]; !ok { - databases[dbname] = goleg.Open(config.DataDir, dbname, goleg.F_APPENDONLY|goleg.F_LZ4|goleg.F_SPLAYTREE) - //TODO Find a way to close unused databases - } - - // Get request? Get the value - if r.Method == "GET" { - value := database.Unjar(key) - // Check if the item existed - if value == nil { - http.Error(w, "Are you sure this is mayo?", 404) - return - } - - // Print value - fmt.Fprintf(w, string(value)) - return - } - - // Post request? Submit a value - if r.Method == "POST" { - value, err := ioutil.ReadAll(r.Body) - if err != nil { - http.Error(w, "Your post body is in klingon or something", 500) - return - } - - // Check if value already existed - exists := database.Exists(key) - - res := database.Jar(key, value) - if res == 0 { - // Status 201 if created, 200 if updated - if exists { - w.WriteHeader(200) - } else { - w.WriteHeader(201) - } - fmt.Fprintf(w, "Got a full jar!") - } else { - http.Error(w, "Wait, the jar vanished!", 500) - return - } - - // Try to set expiration, if provided - if eep, ok := r.Header["X-Olegdb-Use-By"]; ok { - ep, err := strconv.Atoi(eep[0]) - if err != nil { - http.Error(w, "The expiration date does not meet my hipster requirements", 400) - return - } - date := time.Unix(int64(ep), 0) - database.Spoil(key, date) - fmt.Fprintf(w, "\r\nThe jar is spoiling!") - return - } - - } - - // Delete request? Delete a value - if r.Method == "DELETE" { - res := database.Scoop(key) - if res == 0 { - fmt.Fprintf(w, "Scooping done, that'll be $5.10") - return - } else { - http.Error(w, "Got mayo all over my shirt!", 500) - return - } - } - - // Head request? Get key info - if r.Method == "HEAD" { - // Does it even exists? - if !database.Exists(key) { - http.Error(w, "Are you sure this is mayo?", 404) - return - } - - // Get and set Expiration - res, bl := database.Expiration(key) - if bl == true { - w.Header().Add("Expires", strconv.Itoa(int(res.UTC().Unix()))) - } - - // Add Record count - w.Header().Add("X-Olegdb-Rcrd-Cnt", strconv.Itoa(int(*database.RecordCount))) - - fmt.Fprintf(w, "\r\n") - } -} - func main() { // Parse command line flags (if there are) directory := flag.String("dir", "", "Directory where to store dumps and data") diff --git a/frontend/operations.go b/frontend/operations.go new file mode 100644 index 0000000..5214ff3 --- /dev/null +++ b/frontend/operations.go @@ -0,0 +1,99 @@ +package main + +import ( + "fmt" + "io/ioutil" + "net/http" + "strconv" + "time" +) + +const ( + OpGet = "/get" + OpSet = "/post" + OpInfo = "._info" + OpDelete = "/delete" + OpCursorFirst = "_first" + OpCursorLast = "_last" + OpCursorNext = "._next" + OpCursorPrev = "._prev" + OpPrefixMatch = "_match" +) + +func httpGet(w http.ResponseWriter, op Operation) *HTTPError { + value := op.Database.Unjar(op.Key) + // Check if the item existed + if value == nil { + return &HTTPError{Code: 404, Message: "Key not found in database"} + } + + // Send value + fmt.Fprintf(w, string(value)) + return nil +} + +func httpSet(w http.ResponseWriter, op Operation, r *http.Request) *HTTPError { + value, err := ioutil.ReadAll(r.Body) + if err != nil { + return &HTTPError{Code: 500, Message: "Your post body is messed up!"} + } + + // Check if value already existed + exists := op.Database.Exists(op.Key) + + res := op.Database.Jar(op.Key, value) + if res == 0 { + // Status 201 if created, 200 if updated + if exists { + w.WriteHeader(200) + } else { + w.WriteHeader(201) + } + fmt.Fprintf(w, "Value set successfully!") + } else { + return &HTTPError{Code: 500, Message: "Something went horribly wrong..."} + } + + // Try to set expiration, if provided + if eep, ok := r.Header["X-Olegdb-Use-By"]; ok { + ep, err := strconv.Atoi(eep[0]) + if err != nil { + return &HTTPError{Code: 500, Message: "The expiration format is wrong!"} + } + date := time.Unix(int64(ep), 0) + op.Database.Spoil(op.Key, date) + fmt.Fprintf(w, "\r\nThe jar is spoiling!") + } + + return nil +} + +func httpInfo(w http.ResponseWriter, op Operation) *HTTPError { + // Does it even exists? + if !op.Database.Exists(op.Key) { + return &HTTPError{Code: 404, Message: "Key not found in database"} + } + + // Get and set Expiration + res, doesExpire := op.Database.Expiration(op.Key) + if doesExpire { + w.Header().Add("Expires", strconv.Itoa(int(res.UTC().Unix()))) + } + + // Add Record count + w.Header().Add("X-Olegdb-Rcrd-Cnt", strconv.Itoa(int(*op.Database.RecordCount))) + + // Send empty body + fmt.Fprintf(w, "\r\n") + return nil +} + +func httpDelete(w http.ResponseWriter, op Operation) *HTTPError { + res := op.Database.Scoop(op.Key) + if res != 0 { + return &HTTPError{Code: 500, Message: "Something went horribly wrong..."} + } + + fmt.Fprintf(w, "Key deleted successfully!") + return nil +} From f630f32e7dd686337f96373800bde37c1704e5e0 Mon Sep 17 00:00:00 2001 From: Hamcha Date: Sun, 21 Sep 2014 02:52:48 +0200 Subject: [PATCH 06/43] Add F_AOL_FFLUSH to fix hippie filesystems --- frontend/dispatcher.go | 2 +- frontend/goleg/wrapper.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/dispatcher.go b/frontend/dispatcher.go index 8a80d5b..ef013ec 100644 --- a/frontend/dispatcher.go +++ b/frontend/dispatcher.go @@ -28,7 +28,7 @@ func handler(w http.ResponseWriter, r *http.Request) { var database goleg.Database var ok bool if database, ok = databases[dbname]; !ok { - databases[dbname] = goleg.Open(config.DataDir, dbname, goleg.F_APPENDONLY|goleg.F_LZ4|goleg.F_SPLAYTREE) + databases[dbname] = goleg.Open(config.DataDir, dbname, goleg.F_APPENDONLY|goleg.F_AOL_FFLUSH|goleg.F_LZ4|goleg.F_SPLAYTREE) database = databases[dbname] //TODO Find a way to close unused databases } diff --git a/frontend/goleg/wrapper.go b/frontend/goleg/wrapper.go index f378ca3..39bce3c 100644 --- a/frontend/goleg/wrapper.go +++ b/frontend/goleg/wrapper.go @@ -12,6 +12,7 @@ import "time" const F_APPENDONLY = C.OL_F_APPENDONLY const F_LZ4 = C.OL_F_LZ4 const F_SPLAYTREE = C.OL_F_SPLAYTREE +const F_AOL_FFLUSH = C.OL_F_AOL_FFLUSH func COpen(path, name string, features int) *C.ol_database { // Turn parameters into their C counterparts From 32c5708172f2459746edb491e2f0115cd13a3d43 Mon Sep 17 00:00:00 2001 From: Hamcha Date: Sun, 21 Sep 2014 02:57:30 +0200 Subject: [PATCH 07/43] Close all databases before closing --- frontend/dispatcher.go | 1 - frontend/main.go | 7 +++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/frontend/dispatcher.go b/frontend/dispatcher.go index ef013ec..47831da 100644 --- a/frontend/dispatcher.go +++ b/frontend/dispatcher.go @@ -30,7 +30,6 @@ func handler(w http.ResponseWriter, r *http.Request) { if database, ok = databases[dbname]; !ok { databases[dbname] = goleg.Open(config.DataDir, dbname, goleg.F_APPENDONLY|goleg.F_AOL_FFLUSH|goleg.F_LZ4|goleg.F_SPLAYTREE) database = databases[dbname] - //TODO Find a way to close unused databases } operation := Operation{ diff --git a/frontend/main.go b/frontend/main.go index 44cbc9d..569699c 100644 --- a/frontend/main.go +++ b/frontend/main.go @@ -52,6 +52,7 @@ func main() { } http.HandleFunc("/", handler) + defer unload() fmt.Println("Listening on " + config.Listen) if config.UseHTTPS { http.ListenAndServeTLS(config.Listen, config.CertFile, config.PkeyFile, nil) @@ -59,3 +60,9 @@ func main() { http.ListenAndServe(config.Listen, nil) } } + +func unload() { + for v := range databases { + databases[v].Close() + } +} From 7dca943554d2907b9cee4200e1aa1d87f0074e9f Mon Sep 17 00:00:00 2001 From: Hamcha Date: Sun, 21 Sep 2014 03:46:20 +0200 Subject: [PATCH 08/43] Experimental PrefixMatch function (aka Won't work) --- frontend/goleg/highlevel.go | 8 +++++++ frontend/goleg/wrapper.go | 46 +++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/frontend/goleg/highlevel.go b/frontend/goleg/highlevel.go index 232f30f..5f2336e 100644 --- a/frontend/goleg/highlevel.go +++ b/frontend/goleg/highlevel.go @@ -52,3 +52,11 @@ func (d Database) Spoil(key string, expiration time.Time) int { func (d Database) Exists(key string) bool { return CExists(d.db, key, uintptr(len(key))) == 0 } + +func (d Database) Squish() bool { + return CSquish(d.db) == 1 +} + +func (d Database) PrefixMatch(prefix string) []string { + return CPrefixMatch(d.db, prefix, uintptr(len(prefix))) +} diff --git a/frontend/goleg/wrapper.go b/frontend/goleg/wrapper.go index 39bce3c..6a80ef2 100644 --- a/frontend/goleg/wrapper.go +++ b/frontend/goleg/wrapper.go @@ -8,6 +8,7 @@ package goleg import "C" import "unsafe" import "time" +import "reflect" const F_APPENDONLY = C.OL_F_APPENDONLY const F_LZ4 = C.OL_F_LZ4 @@ -163,3 +164,48 @@ func CExists(db *C.ol_database, key string, klen uintptr) int { return int(C.ol_exists(db, ckey, cklen)) } + +func CSquish(db *C.ol_database) int { + return int(C.ol_squish(db)) +} + +func CCas(db *C.ol_database, key string, klen uintptr, value []byte, vsize uintptr, ovalue []byte, ovsize uintptr) int { + // Turn parameters into their C counterparts + ckey := C.CString(key) + defer C.free(unsafe.Pointer(ckey)) + + cklen := (C.size_t)(klen) + cvsize := (C.size_t)(vsize) + covsize := (C.size_t)(ovsize) + + cvalue := (*C.uchar)(unsafe.Pointer(&value[0])) + covalue := (*C.uchar)(unsafe.Pointer(&ovalue[0])) + + // Pass them to ol_jar + return int(C.ol_cas(db, ckey, cklen, cvalue, cvsize, covalue, covsize)) +} + +func CPrefixMatch(db *C.ol_database, prefix string, plen uintptr) []string { + // Turn parameters into their C counterparts + cprefix := C.CString(prefix) + defer C.free(unsafe.Pointer(cprefix)) + + cplen := (C.size_t)(plen) + + var ptr *C.ol_val_array + length := int(C.ol_prefix_match(db, cprefix, cplen, ptr)) + + hdr := reflect.SliceHeader{ + Data: uintptr(unsafe.Pointer(ptr)), + Len: length, + Cap: length, + } + strSlice := *(*[]*C.char)(unsafe.Pointer(&hdr)) + out := make([]string, 0) + for i := range strSlice { + out = append(out, C.GoString(strSlice[i])) + C.free(unsafe.Pointer(strSlice[i])) + } + C.free(unsafe.Pointer(ptr)) + return out +} From 03579f031cb86a9378b9f13b935d7e06e5461f26 Mon Sep 17 00:00:00 2001 From: Kyle Terry Date: Sat, 20 Sep 2014 22:32:24 -0700 Subject: [PATCH 09/43] Added golang installs for testing go-frontend --- .travis.yml | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5260782..2dd1274 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,14 +1,21 @@ language: c compiler: - - gcc - - clang -script: make liboleg oleg_test && export LD_LIBRARY_PATH=./build/lib:$LD_LIBRARY_PATH && ./build/bin/oleg_test + - gcc + - clang +script: + - make liboleg oleg_test + - export LD_LIBRARY_PATH=./build/lib:$LD_LIBRARY_PATH + - ./build/bin/oleg_test + - go test ./... +before_install: + - sudo apt-get update + - sudo apt-get install golang notifications: email: - on_success: never - on_failure: change - recipients: - - "qpfiffer@gmail.com" + on_success: never + on_failure: change + recipients: + - "qpfiffer@gmail.com" env: global: - secure: "GbSMNCqnGnjiuDqJJ55/EaNifx9L7t6JQVPw6illEnGZXL3T5RiC7alrpOQl2qGJG/gwRudMpFmEyvEqs0Rlpgx3x1DvcLCMlC2Rjd/2YklD0KVW0D91Mrrkk7ERzQTO8MXB2AUTxjSAwsZtewybebQhoqr7frulM4TclDZ/Cq4=" From 4c10ef2a167e2c19b810052d65fee3f2bc6d9dd6 Mon Sep 17 00:00:00 2001 From: Kyle Terry Date: Sat, 20 Sep 2014 22:45:29 -0700 Subject: [PATCH 10/43] Added cd to frontend --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2dd1274..cdba9ca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ script: - make liboleg oleg_test - export LD_LIBRARY_PATH=./build/lib:$LD_LIBRARY_PATH - ./build/bin/oleg_test - - go test ./... + - cd frontend && go test ./... before_install: - sudo apt-get update - sudo apt-get install golang From 8c08dc20c844d80594c6860ef98d48b07ff4a744 Mon Sep 17 00:00:00 2001 From: Kyle Terry Date: Sat, 20 Sep 2014 22:55:47 -0700 Subject: [PATCH 11/43] Trying this? --- frontend/goleg/wrapper.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/goleg/wrapper.go b/frontend/goleg/wrapper.go index 6a80ef2..e51d2b6 100644 --- a/frontend/goleg/wrapper.go +++ b/frontend/goleg/wrapper.go @@ -1,7 +1,8 @@ package goleg /* -#cgo LDFLAGS: -loleg +#cgo CFLAGS: -I../include/ +#cgo LDFLAGS: -L../build/lib/ -loleg #include #include */ From fddf17efc048e2ee487d4d683fd7f38d7fac4645 Mon Sep 17 00:00:00 2001 From: Kyle Terry Date: Sat, 20 Sep 2014 23:03:26 -0700 Subject: [PATCH 12/43] fuck a cd --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index cdba9ca..2dd1274 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ script: - make liboleg oleg_test - export LD_LIBRARY_PATH=./build/lib:$LD_LIBRARY_PATH - ./build/bin/oleg_test - - cd frontend && go test ./... + - go test ./... before_install: - sudo apt-get update - sudo apt-get install golang From 6b96c32b9a66c266470dc0820ed90ed0bdf19f98 Mon Sep 17 00:00:00 2001 From: Kyle Terry Date: Tue, 23 Sep 2014 19:54:16 -0700 Subject: [PATCH 13/43] Replacing instances of erlang with Go --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 188b675..6f699c5 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ olegdb: No. OlegDB is a ~~single-threaded, non-concurrent~~, transactionless NoSQL database written by bitter SQL-lovers in a futile attempt to hop on the schemaless trend before everyone realizes it was a bad move. It is primarily a C library with an -Erlang frontend for communication. +Go frontend for communication. Dependencies ============ @@ -26,7 +26,7 @@ Dependencies Installation ============ -OlegDB consists of a server written in Erlang and a C library for all of the +OlegDB consists of a server written in Go and a C library for all of the heavy lifting. Binaries are in `build/bin/` and the library is in `build/lib/`. Beam files are also thrown in `build/bin/`. From 3e050e9b528d695bd8ffb4caf38f2430749a28de Mon Sep 17 00:00:00 2001 From: Kyle Terry Date: Tue, 23 Sep 2014 23:56:03 -0700 Subject: [PATCH 14/43] Consistent imports --- frontend/goleg/highlevel.go | 6 ++++-- frontend/goleg/wrapper.go | 10 ++++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/frontend/goleg/highlevel.go b/frontend/goleg/highlevel.go index 5f2336e..f45bce0 100644 --- a/frontend/goleg/highlevel.go +++ b/frontend/goleg/highlevel.go @@ -5,8 +5,10 @@ package goleg #cgo LDFLAGS: -L../build/lib/ -loleg #include */ -import "C" -import "time" +import ( + "C" + "time" +) type Database struct { db *C.ol_database diff --git a/frontend/goleg/wrapper.go b/frontend/goleg/wrapper.go index e51d2b6..3e457b9 100644 --- a/frontend/goleg/wrapper.go +++ b/frontend/goleg/wrapper.go @@ -6,10 +6,12 @@ package goleg #include #include */ -import "C" -import "unsafe" -import "time" -import "reflect" +import ( + "C" + "unsafe" + "time" + "reflect" +) const F_APPENDONLY = C.OL_F_APPENDONLY const F_LZ4 = C.OL_F_LZ4 From ef54951207e5a24668fdd2cf216dbefe8701e722 Mon Sep 17 00:00:00 2001 From: Hamcha Date: Fri, 26 Sep 2014 16:10:44 +0200 Subject: [PATCH 15/43] Prefix matching on frontend --- frontend/dispatcher.go | 2 ++ frontend/goleg/highlevel.go | 5 +++-- frontend/goleg/wrapper.go | 15 +++++++++++---- frontend/operations.go | 16 +++++++++++++++- 4 files changed, 31 insertions(+), 7 deletions(-) diff --git a/frontend/dispatcher.go b/frontend/dispatcher.go index 47831da..590f783 100644 --- a/frontend/dispatcher.go +++ b/frontend/dispatcher.go @@ -47,6 +47,8 @@ func handler(w http.ResponseWriter, r *http.Request) { err = httpInfo(w, operation) case OpDelete: err = httpDelete(w, operation) + case OpPrefixMatch: + err = httpMatch(w, operation) default: err = &HTTPError{Message: "I don't get what you're trying to do", Code: 400} } diff --git a/frontend/goleg/highlevel.go b/frontend/goleg/highlevel.go index f45bce0..977de0e 100644 --- a/frontend/goleg/highlevel.go +++ b/frontend/goleg/highlevel.go @@ -59,6 +59,7 @@ func (d Database) Squish() bool { return CSquish(d.db) == 1 } -func (d Database) PrefixMatch(prefix string) []string { - return CPrefixMatch(d.db, prefix, uintptr(len(prefix))) +func (d Database) PrefixMatch(prefix string) (bool, []string) { + len, out := CPrefixMatch(d.db, prefix, uintptr(len(prefix))) + return len >= 0, out } diff --git a/frontend/goleg/wrapper.go b/frontend/goleg/wrapper.go index 3e457b9..1cb9d85 100644 --- a/frontend/goleg/wrapper.go +++ b/frontend/goleg/wrapper.go @@ -188,27 +188,34 @@ func CCas(db *C.ol_database, key string, klen uintptr, value []byte, vsize uintp return int(C.ol_cas(db, ckey, cklen, cvalue, cvsize, covalue, covsize)) } -func CPrefixMatch(db *C.ol_database, prefix string, plen uintptr) []string { +func CPrefixMatch(db *C.ol_database, prefix string, plen uintptr) (int, []string) { // Turn parameters into their C counterparts cprefix := C.CString(prefix) defer C.free(unsafe.Pointer(cprefix)) cplen := (C.size_t)(plen) - var ptr *C.ol_val_array - length := int(C.ol_prefix_match(db, cprefix, cplen, ptr)) + // Call native function + var ptr C.ol_val_array + length := int(C.ol_prefix_match(db, cprefix, cplen, &ptr)) + if length < 0 { + return length, nil + } + // Set array structure hdr := reflect.SliceHeader{ Data: uintptr(unsafe.Pointer(ptr)), Len: length, Cap: length, } strSlice := *(*[]*C.char)(unsafe.Pointer(&hdr)) + // Create GoString array out := make([]string, 0) for i := range strSlice { out = append(out, C.GoString(strSlice[i])) C.free(unsafe.Pointer(strSlice[i])) } + // Free structure C.free(unsafe.Pointer(ptr)) - return out + return length, out } diff --git a/frontend/operations.go b/frontend/operations.go index 5214ff3..6feacb8 100644 --- a/frontend/operations.go +++ b/frontend/operations.go @@ -5,6 +5,7 @@ import ( "io/ioutil" "net/http" "strconv" + "strings" "time" ) @@ -17,7 +18,7 @@ const ( OpCursorLast = "_last" OpCursorNext = "._next" OpCursorPrev = "._prev" - OpPrefixMatch = "_match" + OpPrefixMatch = "._match" ) func httpGet(w http.ResponseWriter, op Operation) *HTTPError { @@ -97,3 +98,16 @@ func httpDelete(w http.ResponseWriter, op Operation) *HTTPError { fmt.Fprintf(w, "Key deleted successfully!") return nil } + +func httpMatch(w http.ResponseWriter, op Operation) *HTTPError { + has, res := op.Database.PrefixMatch(op.Key) + if !has { + return &HTTPError{Code: 500, Message: "Something went horribly wrong..."} + } + if len(res) == 0 { + return &HTTPError{Code: 404, Message: "No matches found"} + } + + fmt.Fprintf(w, strings.Join(res, "\n")) + return nil +} From 68967e61319bab939f39bfb65abb9da4a3b3e4e6 Mon Sep 17 00:00:00 2001 From: Hamcha Date: Sat, 27 Sep 2014 00:14:57 +0200 Subject: [PATCH 16/43] Include/lib dirs fixed, Prefix matching changed --- frontend/goleg/highlevel.go | 6 +++--- frontend/goleg/wrapper.go | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/frontend/goleg/highlevel.go b/frontend/goleg/highlevel.go index 977de0e..5994c61 100644 --- a/frontend/goleg/highlevel.go +++ b/frontend/goleg/highlevel.go @@ -1,9 +1,9 @@ package goleg /* -#cgo CFLAGS: -I../include/ -#cgo LDFLAGS: -L../build/lib/ -loleg -#include +#cgo CFLAGS: -I../../include +#cgo LDFLAGS: -L../../build/lib -loleg +#include "oleg.h" */ import ( "C" diff --git a/frontend/goleg/wrapper.go b/frontend/goleg/wrapper.go index 1cb9d85..7fbbd97 100644 --- a/frontend/goleg/wrapper.go +++ b/frontend/goleg/wrapper.go @@ -1,10 +1,10 @@ package goleg /* -#cgo CFLAGS: -I../include/ -#cgo LDFLAGS: -L../build/lib/ -loleg +#cgo CFLAGS: -I../../include +#cgo LDFLAGS: -L../../build/lib -loleg #include -#include +#include "oleg.h" */ import ( "C" @@ -125,7 +125,7 @@ func CExpirationTime(db *C.ol_database, key string, klen uintptr) (time.Time, bo return time.Now(), false } - // turn ctime into a Go datatype + // Turn ctime into a Go datatype gotime := time.Date(int(ctime.tm_year)+1900, time.Month(int(ctime.tm_mon)+1), int(ctime.tm_mday), @@ -196,7 +196,7 @@ func CPrefixMatch(db *C.ol_database, prefix string, plen uintptr) (int, []string cplen := (C.size_t)(plen) // Call native function - var ptr C.ol_val_array + var ptr C.ol_key_array length := int(C.ol_prefix_match(db, cprefix, cplen, &ptr)) if length < 0 { return length, nil From 4b21c97d43401b23fad1086f49e65ad34ac6391e Mon Sep 17 00:00:00 2001 From: Hamcha Date: Sun, 28 Sep 2014 04:13:30 +0200 Subject: [PATCH 17/43] Preliminary cursor functions (not finished) --- frontend/goleg/highlevel.go | 22 +++++++++- frontend/goleg/wrapper.go | 82 ++++++++++++++++++++++++++++++++++--- 2 files changed, 97 insertions(+), 7 deletions(-) diff --git a/frontend/goleg/highlevel.go b/frontend/goleg/highlevel.go index 5994c61..2352140 100644 --- a/frontend/goleg/highlevel.go +++ b/frontend/goleg/highlevel.go @@ -5,8 +5,8 @@ package goleg #cgo LDFLAGS: -L../../build/lib -loleg #include "oleg.h" */ +import "C" import ( - "C" "time" ) @@ -63,3 +63,23 @@ func (d Database) PrefixMatch(prefix string) (bool, []string) { len, out := CPrefixMatch(d.db, prefix, uintptr(len(prefix))) return len >= 0, out } + +func (d Database) First() (string, []byte) { + cursor := CCurFirst(d.db) + return CCurGet(cursor) +} + +func (d Database) Last() (string, []byte) { + cursor := CCurLast(d.db) + return CCurGet(cursor) +} + +func (d Database) Next(key string) (string, []byte) { + cursor := CCurNext(d.db, key, uintptr(len(key))) + return CCurGet(cursor) +} + +func (d Database) Prev() (string, []byte) { + cursor := CCurPrev(d.db, key, uintptr(len(key))) + return CCurGet(cursor) +} diff --git a/frontend/goleg/wrapper.go b/frontend/goleg/wrapper.go index 7fbbd97..d5dcc61 100644 --- a/frontend/goleg/wrapper.go +++ b/frontend/goleg/wrapper.go @@ -5,12 +5,13 @@ package goleg #cgo LDFLAGS: -L../../build/lib -loleg #include #include "oleg.h" +#include "cursor.h" */ +import "C" import ( - "C" - "unsafe" - "time" "reflect" + "time" + "unsafe" ) const F_APPENDONLY = C.OL_F_APPENDONLY @@ -172,17 +173,17 @@ func CSquish(db *C.ol_database) int { return int(C.ol_squish(db)) } -func CCas(db *C.ol_database, key string, klen uintptr, value []byte, vsize uintptr, ovalue []byte, ovsize uintptr) int { +func CCas(db *C.ol_database, key string, klen uintptr, value []byte, vsize uintptr, ovalue *[]byte, ovsize *uintptr) int { // Turn parameters into their C counterparts ckey := C.CString(key) defer C.free(unsafe.Pointer(ckey)) cklen := (C.size_t)(klen) cvsize := (C.size_t)(vsize) - covsize := (C.size_t)(ovsize) + covsize := (C.size_t)(*ovsize) cvalue := (*C.uchar)(unsafe.Pointer(&value[0])) - covalue := (*C.uchar)(unsafe.Pointer(&ovalue[0])) + covalue := (*C.uchar)(unsafe.Pointer(ovalue)) // Pass them to ol_jar return int(C.ol_cas(db, ckey, cklen, cvalue, cvsize, covalue, covsize)) @@ -219,3 +220,72 @@ func CPrefixMatch(db *C.ol_database, prefix string, plen uintptr) (int, []string C.free(unsafe.Pointer(ptr)) return length, out } + +func CGetBucket(db *C.ol_database, key string, klen uintptr, _key *string, _klen *uintptr) *C.ol_bucket { + // Turn parameters into their C counterparts + ckey := C.CString(key) + defer C.free(unsafe.Pointer(ckey)) + + var c_key [C.KEY_SIZE]C.char + + cklen := (C.size_t)(klen) + c_klen := (*C.size_t)(unsafe.Pointer(_klen)) + + bucket := C.ol_get_bucket(db, ckey, cklen, &c_key, c_klen) + + *_key = C.GoStringN((*C.char)(unsafe.Pointer(&c_key)), C.int(*c_klen)) + + return bucket +} + +func CCurFirst(db *C.ol_database) *C.ol_cursor { + minimum := C.ols_subtree_minimum(db.tree.root) + return minimum +} + +func CCurLast(db *C.ol_database) *C.ol_cursor { + maximum := C.ols_subtree_maximum(db.tree.root) + return maximum +} + +func CCurNext(db *C.ol_database, key string, klen uintptr) *C.ol_cursor { + var _key string + var _klen uintptr + bucket := CGetBucket(db, key, klen, &_key, &_klen) + + if bucket == nil { + return bucket + } + + node := bucket.node + maximum := C.ols_subtree_maximum(db.tree.root) + ret := int(C._olc_next(&node, maximum)) + if ret != 0 { + return nil + } + + return node +} + +func CCurPrev(db *C.ol_database, key string, klen uintptr) *C.ol_cursor { + var _key string + var _klen uintptr + bucket := CGetBucket(db, key, klen, &_key, &_klen) + + if bucket == nil { + return bucket + } + + node := bucket.node + minimum := C.ols_subtree_minimum(db.tree.root) + ret := int(C._olc_next(&node, minimum)) + if ret != 0 { + return nil + } + + return node +} + +func CCurGet(cursor *C.ol_cursor) (string, []byte) { + +} From 6d6ed7384e421759242a6937d8b7a0692361ab10 Mon Sep 17 00:00:00 2001 From: Hamcha Date: Sun, 28 Sep 2014 08:30:59 +0200 Subject: [PATCH 18/43] Cursor operations are a-go! --- frontend/dispatcher.go | 8 ++++++ frontend/goleg/highlevel.go | 38 ++++++++++++++++--------- frontend/goleg/wrapper.go | 57 ++++++++++++++++--------------------- frontend/operations.go | 42 ++++++++++++++++++++++++++- 4 files changed, 98 insertions(+), 47 deletions(-) diff --git a/frontend/dispatcher.go b/frontend/dispatcher.go index 590f783..eff45c9 100644 --- a/frontend/dispatcher.go +++ b/frontend/dispatcher.go @@ -49,6 +49,14 @@ func handler(w http.ResponseWriter, r *http.Request) { err = httpDelete(w, operation) case OpPrefixMatch: err = httpMatch(w, operation) + case OpCursorFirst: + err = httpCurFirst(w, operation) + case OpCursorLast: + err = httpCurLast(w, operation) + case OpCursorNext: + err = httpCurNext(w, operation) + case OpCursorPrev: + err = httpCurPrev(w, operation) default: err = &HTTPError{Message: "I don't get what you're trying to do", Code: 400} } diff --git a/frontend/goleg/highlevel.go b/frontend/goleg/highlevel.go index 2352140..6b51e0e 100644 --- a/frontend/goleg/highlevel.go +++ b/frontend/goleg/highlevel.go @@ -28,7 +28,7 @@ func (d Database) Close() int { func (d Database) Unjar(key string) []byte { var dsize uintptr - return CUnjarDs(d.db, key, uintptr(len(key)), &dsize) + return CUnjar(d.db, key, uintptr(len(key)), &dsize) } func (d Database) Jar(key string, value []byte) int { @@ -64,22 +64,34 @@ func (d Database) PrefixMatch(prefix string) (bool, []string) { return len >= 0, out } -func (d Database) First() (string, []byte) { - cursor := CCurFirst(d.db) - return CCurGet(cursor) +func (d Database) First() (bool, string, []byte) { + node := CNodeFirst(d.db) + if node == nil { + return false, "", nil + } + return CNodeGet(d.db, node) } -func (d Database) Last() (string, []byte) { - cursor := CCurLast(d.db) - return CCurGet(cursor) +func (d Database) Last() (bool, string, []byte) { + node := CNodeLast(d.db) + if node == nil { + return false, "", nil + } + return CNodeGet(d.db, node) } -func (d Database) Next(key string) (string, []byte) { - cursor := CCurNext(d.db, key, uintptr(len(key))) - return CCurGet(cursor) +func (d Database) Next(key string) (bool, string, []byte) { + node := CNodeNext(d.db, key, uintptr(len(key))) + if node == nil { + return false, "", nil + } + return CNodeGet(d.db, node) } -func (d Database) Prev() (string, []byte) { - cursor := CCurPrev(d.db, key, uintptr(len(key))) - return CCurGet(cursor) +func (d Database) Prev(key string) (bool, string, []byte) { + node := CNodePrev(d.db, key, uintptr(len(key))) + if node == nil { + return false, "", nil + } + return CNodeGet(d.db, node) } diff --git a/frontend/goleg/wrapper.go b/frontend/goleg/wrapper.go index d5dcc61..14cebdc 100644 --- a/frontend/goleg/wrapper.go +++ b/frontend/goleg/wrapper.go @@ -37,29 +37,7 @@ func CClose(database *C.ol_database) int { return int(C.ol_close(database)) } -func CUnjar(db *C.ol_database, key string, klen uintptr, dsize uintptr) []byte { - // Turn parameters into their C counterparts - ckey := C.CString(key) - defer C.free(unsafe.Pointer(ckey)) - - cklen := (C.size_t)(klen) - - // Pass them to ol_unjar - var ptr *C.uchar - res := C.ol_unjar(db, ckey, cklen, &ptr) - if res == 1 { - return nil - } - // Retrieve data in Go []bytes - data := C.GoBytes(unsafe.Pointer(ptr), C.int(dsize)) - - // Free C pointer - C.free(unsafe.Pointer(ptr)) - - return data -} - -func CUnjarDs(db *C.ol_database, key string, klen uintptr, dsize *uintptr) []byte { +func CUnjar(db *C.ol_database, key string, klen uintptr, dsize *uintptr) []byte { // Turn parameters into their C counterparts ckey := C.CString(key) defer C.free(unsafe.Pointer(ckey)) @@ -238,54 +216,67 @@ func CGetBucket(db *C.ol_database, key string, klen uintptr, _key *string, _klen return bucket } -func CCurFirst(db *C.ol_database) *C.ol_cursor { +func CNodeFirst(db *C.ol_database) *C.ol_splay_tree_node { minimum := C.ols_subtree_minimum(db.tree.root) return minimum } -func CCurLast(db *C.ol_database) *C.ol_cursor { +func CNodeLast(db *C.ol_database) *C.ol_splay_tree_node { maximum := C.ols_subtree_maximum(db.tree.root) return maximum } -func CCurNext(db *C.ol_database, key string, klen uintptr) *C.ol_cursor { +func CNodeNext(db *C.ol_database, key string, klen uintptr) *C.ol_splay_tree_node { var _key string var _klen uintptr bucket := CGetBucket(db, key, klen, &_key, &_klen) if bucket == nil { - return bucket + return nil } node := bucket.node maximum := C.ols_subtree_maximum(db.tree.root) ret := int(C._olc_next(&node, maximum)) - if ret != 0 { + if ret == 0 || node == bucket.node { return nil } return node } -func CCurPrev(db *C.ol_database, key string, klen uintptr) *C.ol_cursor { +func CNodePrev(db *C.ol_database, key string, klen uintptr) *C.ol_splay_tree_node { var _key string var _klen uintptr bucket := CGetBucket(db, key, klen, &_key, &_klen) if bucket == nil { - return bucket + return nil } node := bucket.node minimum := C.ols_subtree_minimum(db.tree.root) - ret := int(C._olc_next(&node, minimum)) - if ret != 0 { + ret := int(C._olc_prev(&node, minimum)) + if ret == 0 || node == bucket.node { return nil } return node } -func CCurGet(cursor *C.ol_cursor) (string, []byte) { +func CNodeGet(db *C.ol_database, node *C.ol_splay_tree_node) (bool, string, []byte) { + // Get associated bucket + bucket := (*C.ol_bucket)(node.ref_obj) + + if bucket == nil { + return false, "", nil + } + + // Get key and data + key := C.GoStringN((*C.char)(unsafe.Pointer(&bucket.key)), C.int(bucket.klen)) + + var dsize uintptr + data := CUnjar(db, key, uintptr(len(key)), &dsize) + return true, key, data } diff --git a/frontend/operations.go b/frontend/operations.go index 6feacb8..4139fce 100644 --- a/frontend/operations.go +++ b/frontend/operations.go @@ -29,7 +29,7 @@ func httpGet(w http.ResponseWriter, op Operation) *HTTPError { } // Send value - fmt.Fprintf(w, string(value)) + w.Write(value) return nil } @@ -111,3 +111,43 @@ func httpMatch(w http.ResponseWriter, op Operation) *HTTPError { fmt.Fprintf(w, strings.Join(res, "\n")) return nil } + +func httpCurFirst(w http.ResponseWriter, op Operation) *HTTPError { + has, key, data := op.Database.First() + if !has { + return &HTTPError{Code: 404, Message: "No record found"} + } + w.Header().Add("X-Olegdb-Key", key) + w.Write(data) + return nil +} + +func httpCurLast(w http.ResponseWriter, op Operation) *HTTPError { + has, key, data := op.Database.Last() + if !has { + return &HTTPError{Code: 404, Message: "No record found"} + } + w.Header().Add("X-Olegdb-Key", key) + w.Write(data) + return nil +} + +func httpCurNext(w http.ResponseWriter, op Operation) *HTTPError { + has, key, data := op.Database.Next(op.Key) + if !has { + return &HTTPError{Code: 404, Message: "No record found"} + } + w.Header().Add("X-Olegdb-Key", key) + w.Write(data) + return nil +} + +func httpCurPrev(w http.ResponseWriter, op Operation) *HTTPError { + has, key, data := op.Database.Prev(op.Key) + if !has { + return &HTTPError{Code: 404, Message: "No record found"} + } + w.Header().Add("X-Olegdb-Key", key) + w.Write(data) + return nil +} From 1407b9f366c5450c4491db7b4e03fabde031e9bb Mon Sep 17 00:00:00 2001 From: Hamcha Date: Sun, 28 Sep 2014 08:37:34 +0200 Subject: [PATCH 19/43] Typo! --- frontend/operations.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/frontend/operations.go b/frontend/operations.go index 4139fce..c760c95 100644 --- a/frontend/operations.go +++ b/frontend/operations.go @@ -115,7 +115,7 @@ func httpMatch(w http.ResponseWriter, op Operation) *HTTPError { func httpCurFirst(w http.ResponseWriter, op Operation) *HTTPError { has, key, data := op.Database.First() if !has { - return &HTTPError{Code: 404, Message: "No record found"} + return &HTTPError{Code: 404, Message: "No records found"} } w.Header().Add("X-Olegdb-Key", key) w.Write(data) @@ -125,7 +125,7 @@ func httpCurFirst(w http.ResponseWriter, op Operation) *HTTPError { func httpCurLast(w http.ResponseWriter, op Operation) *HTTPError { has, key, data := op.Database.Last() if !has { - return &HTTPError{Code: 404, Message: "No record found"} + return &HTTPError{Code: 404, Message: "No records found"} } w.Header().Add("X-Olegdb-Key", key) w.Write(data) @@ -135,7 +135,7 @@ func httpCurLast(w http.ResponseWriter, op Operation) *HTTPError { func httpCurNext(w http.ResponseWriter, op Operation) *HTTPError { has, key, data := op.Database.Next(op.Key) if !has { - return &HTTPError{Code: 404, Message: "No record found"} + return &HTTPError{Code: 404, Message: "No records found"} } w.Header().Add("X-Olegdb-Key", key) w.Write(data) @@ -145,7 +145,7 @@ func httpCurNext(w http.ResponseWriter, op Operation) *HTTPError { func httpCurPrev(w http.ResponseWriter, op Operation) *HTTPError { has, key, data := op.Database.Prev(op.Key) if !has { - return &HTTPError{Code: 404, Message: "No record found"} + return &HTTPError{Code: 404, Message: "No records found"} } w.Header().Add("X-Olegdb-Key", key) w.Write(data) From 0f7db0846b68a22239e58ef9790d475274df00f4 Mon Sep 17 00:00:00 2001 From: Hamcha Date: Sun, 28 Sep 2014 09:16:24 +0200 Subject: [PATCH 20/43] Setup for tests --- frontend/dispatcher.go | 7 ++++++- frontend/goleg/goleg_test.go | 39 ++++++++++++++++++++++++++++++++++++ frontend/goleg/highlevel.go | 8 ++++++-- 3 files changed, 51 insertions(+), 3 deletions(-) create mode 100644 frontend/goleg/goleg_test.go diff --git a/frontend/dispatcher.go b/frontend/dispatcher.go index eff45c9..996172f 100644 --- a/frontend/dispatcher.go +++ b/frontend/dispatcher.go @@ -28,7 +28,12 @@ func handler(w http.ResponseWriter, r *http.Request) { var database goleg.Database var ok bool if database, ok = databases[dbname]; !ok { - databases[dbname] = goleg.Open(config.DataDir, dbname, goleg.F_APPENDONLY|goleg.F_AOL_FFLUSH|goleg.F_LZ4|goleg.F_SPLAYTREE) + var dberr error + databases[dbname], dberr = goleg.Open(config.DataDir, dbname, goleg.F_APPENDONLY|goleg.F_AOL_FFLUSH|goleg.F_LZ4|goleg.F_SPLAYTREE) + if dberr != nil { + http.Error(w, "Cannot open database", 500) + return + } database = databases[dbname] } diff --git a/frontend/goleg/goleg_test.go b/frontend/goleg/goleg_test.go new file mode 100644 index 0000000..cb3820b --- /dev/null +++ b/frontend/goleg/goleg_test.go @@ -0,0 +1,39 @@ +package goleg + +import ( + "io/ioutil" + "os" + "testing" +) + +func openRandomDB() (Database, string, error) { + name, err := ioutil.TempDir("/tmp", "goleg") + if err != nil { + return Database{}, "", err + } + + database, err := Open(name, "test", F_APPENDONLY|F_AOL_FFLUSH|F_LZ4|F_SPLAYTREE) + if err != nil { + return Database{}, "", err + } + + return database, name, nil +} + +func cleanTemp(dir string) { + os.RemoveAll(dir) +} + +func TestOpen(t *testing.T) { + if testing.Short() { + t.Skip("Skipping in short mode") + } + + database, dir, err := openRandomDB() + if err != nil { + t.Fatalf("Can't open database: %s", err.Error()) + } + + database.Close() + cleanTemp(dir) +} diff --git a/frontend/goleg/highlevel.go b/frontend/goleg/highlevel.go index 6b51e0e..769ebb0 100644 --- a/frontend/goleg/highlevel.go +++ b/frontend/goleg/highlevel.go @@ -7,6 +7,7 @@ package goleg */ import "C" import ( + "errors" "time" ) @@ -15,11 +16,14 @@ type Database struct { RecordCount *C.int } -func Open(path, name string, features int) Database { +func Open(path, name string, features int) (Database, error) { var database Database database.db = COpen(path, name, features) + if database.db == nil { + return Database{}, errors.New("Can't open database (NULL returned)") + } database.RecordCount = &database.db.rcrd_cnt - return database + return database, nil } func (d Database) Close() int { From 2ababa0f3b5cf5c6a417dd25347a9fba9c97f825 Mon Sep 17 00:00:00 2001 From: Hamcha Date: Sun, 28 Sep 2014 09:17:47 +0200 Subject: [PATCH 21/43] Harcoded old test directory begone Also we don't need bash --- run_tests.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index 9e0731f..f89e626 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,9 +1,7 @@ -#!/usr/bin/env bash +#!/bin/sh export LD_LIBRARY_PATH=./build/lib:$LD_LIBRARY_PATH -rm -rf /tmp/oleg_is_king - if [ "$1" == "valgrind" ] then valgrind --leak-check=full --track-origins=yes ./build/bin/oleg_test From 2ac307113934b8dd0b0309c51b60bf59261b6d40 Mon Sep 17 00:00:00 2001 From: Hamcha Date: Sun, 28 Sep 2014 09:28:46 +0200 Subject: [PATCH 22/43] Modified documentation to reflect frontend changes --- docs/Overview/04_Cursor_Iteration.markdown | 142 ++++++++------------- docs/Overview/05_Prefix_Matching.markdown | 36 +++--- 2 files changed, 65 insertions(+), 113 deletions(-) diff --git a/docs/Overview/04_Cursor_Iteration.markdown b/docs/Overview/04_Cursor_Iteration.markdown index 7f943ee..0f9bd37 100644 --- a/docs/Overview/04_Cursor_Iteration.markdown +++ b/docs/Overview/04_Cursor_Iteration.markdown @@ -3,31 +3,24 @@ database via the frontend. It's a pretty simple interface and follows the rest of the current URL idioms. Each cursor operand is of the form `/database/key/operand`. In some -operands (`_last` and `_first`) the `key` option is ignored. Using them is +operands (`_last` and `_first`) the `key` option is the operand. Using them is trivial. Using any of the operands will return both the value of the key you requested (`_next` will return the next value, `_prev` will return the previous value, etc.) -and the HTTP header `X-OlegDB-Key` followed by the key paired to the value you +and the HTTP header `X-Olegdb-Key` followed by the key paired to the value you just retrieved. For example, say we have two keys in the database, `aaa` and `bbb`. To begin with, I can request the first key in the database: ```` -$ curl -v localhost:8080/oleg//_first -> GET /oleg//_first HTTP/1.1 -> User-Agent: curl/7.22.0 (x86_64-pc-linux-gnu) libcurl/7.22.0 OpenSSL/1.0.1 -> zlib/1.2.3.4 libidn/1.23 librtmp/2.3 -> Host: localhost:8080 -> Accept: */* -> -< HTTP/1.1 200 OK -< Server: OlegDB/fresh_cuts_n_jams -< Content-Type: application/octet-stream -< Content-Length: 22 -< Connection: close -< X-OlegDB-Key: aaa -< -I am the value of aaa. +$ curl -i localhost:8080/oleg/_first +HTTP/1.1 200 OK +X-Olegdb-Key: aaa +Date: Sun, 28 Sep 2014 07:23:39 GMT +Content-Length: 21 +Content-Type: text/plain; charset=utf-8 + +I am the value of aaa ```` As you can see, the key `aaa` is the first one in the tree of ordered keys. If @@ -37,21 +30,14 @@ because the `key` is not used in this command. It will, however, be used in the next: ```` -$ curl -v localhost:8080/oleg/aaa/_next -> GET /oleg/aaa/_next HTTP/1.1 -> User-Agent: curl/7.22.0 (x86_64-pc-linux-gnu) libcurl/7.22.0 OpenSSL/1.0.1 -> zlib/1.2.3.4 libidn/1.23 librtmp/2.3 -> Host: localhost:8080 -> Accept: */* -> -< HTTP/1.1 200 OK -< Server: OlegDB/fresh_cuts_n_jams -< Content-Type: application/octet-stream -< Content-Length: 22 -< Connection: close -< X-OlegDB-Key: bbb -< -I am the value of bbb. +$ curl -i localhost:8080/oleg/aaa/_next +HTTP/1.1 200 OK +X-Olegdb-Key: bbb +Date: Sun, 28 Sep 2014 07:24:16 GMT +Content-Length: 21 +Content-Type: text/plain; charset=utf-8 + +I am the value of bbb ```` Logically, the key `bbb` follows the key `aaa`. Nice. In our request, we asked @@ -61,21 +47,13 @@ corresponds to that value. Lets see what happens if we try to get the next key, knowing that we only have two keys (`aaa` and `bbb`) in our database: ```` -$ curl -v localhost:8080/oleg/bbb/_next -> GET /oleg/bbb/_next HTTP/1.1 -> User-Agent: curl/7.22.0 (x86_64-pc-linux-gnu) libcurl/7.22.0 OpenSSL/1.0.1 -> zlib/1.2.3.4 libidn/1.23 librtmp/2.3 -> Host: localhost:8080 -> Accept: */* -> -< HTTP/1.1 404 Not Found -< Status: 404 Not Found -< Server: OlegDB/fresh_cuts_n_jams -< Content-Length: 26 -< Connection: close -< Content-Type: text/plain -< -These aren't your ghosts. +$ curl -i localhost:8080/oleg/bbb/_next +HTTP/1.1 404 Not Found +Content-Type: text/plain; charset=utf-8 +Date: Sun, 28 Sep 2014 07:24:26 GMT +Content-Length: 17 + +No records found ```` We get a 404 statuscode and a message to match. This informs us that we cannot @@ -86,49 +64,29 @@ the database and `_prev` to iterate backwards. The usage of these commands is identical to those above: ```` -$ curl -v localhost:8080/oleg//_last -> GET /oleg//_last HTTP/1.1 -> User-Agent: curl/7.22.0 (x86_64-pc-linux-gnu) libcurl/7.22.0 OpenSSL/1.0.1 -> zlib/1.2.3.4 libidn/1.23 librtmp/2.3 -> Host: localhost:8080 -> Accept: */* -> -< HTTP/1.1 200 OK -< Server: OlegDB/fresh_cuts_n_jams -< Content-Type: application/octet-stream -< Content-Length: 22 -< Connection: close -< X-OlegDB-Key: bbb -< -I am the value of bbb. -$ curl -v localhost:8080/oleg/bbb/_prev -> GET /oleg/bbb/_prev HTTP/1.1 -> User-Agent: curl/7.22.0 (x86_64-pc-linux-gnu) libcurl/7.22.0 OpenSSL/1.0.1 -> zlib/1.2.3.4 libidn/1.23 librtmp/2.3 -> Host: localhost:8080 -> Accept: */* -> -< HTTP/1.1 200 OK -< Server: OlegDB/fresh_cuts_n_jams -< Content-Type: application/octet-stream -< Content-Length: 22 -< Connection: close -< X-OlegDB-Key: aaa -< -I am the value of aaa. -$ curl -v localhost:8080/oleg/aaa/_prev -> GET /oleg/aaa/_prev HTTP/1.1 -> User-Agent: curl/7.22.0 (x86_64-pc-linux-gnu) libcurl/7.22.0 OpenSSL/1.0.1 -> zlib/1.2.3.4 libidn/1.23 librtmp/2.3 -> Host: localhost:8080 -> Accept: */* -> -< HTTP/1.1 404 Not Found -< Status: 404 Not Found -< Server: OlegDB/fresh_cuts_n_jams -< Content-Length: 26 -< Connection: close -< Content-Type: text/plain -< -These aren't your ghosts. +$ curl -i localhost:8080/oleg/_last +HTTP/1.1 200 OK +X-Olegdb-Key: bbb +Date: Sun, 28 Sep 2014 07:24:50 GMT +Content-Length: 21 +Content-Type: text/plain; charset=utf-8 + +I am the value of bbb + +$ curl -i localhost:8080/oleg/bbb/_prev +HTTP/1.1 200 OK +X-Olegdb-Key: aaa +Date: Sun, 28 Sep 2014 07:25:06 GMT +Content-Length: 21 +Content-Type: text/plain; charset=utf-8 + +I am the value of aaa + +$ curl -i localhost:8080/oleg/aaa/_prev +HTTP/1.1 404 Not Found +Content-Type: text/plain; charset=utf-8 +Date: Sun, 28 Sep 2014 07:25:17 GMT +Content-Length: 17 + +No records found ```` diff --git a/docs/Overview/05_Prefix_Matching.markdown b/docs/Overview/05_Prefix_Matching.markdown index af8e916..7ba3ed2 100644 --- a/docs/Overview/05_Prefix_Matching.markdown +++ b/docs/Overview/05_Prefix_Matching.markdown @@ -1,32 +1,26 @@ In addition to [cursor iteration](#cursor_iteration) `0.1.2` added the ability -to return the values of keys that match a given prefix. Use of this feature -follows the same URL layout as it's predeccesors, mainly via the use of the +to return the keys that match a given prefix. Use of this feature follows the +same URL layout as it's predeccesors, mainly via the use of the `_match` qualifier. For example, say I have three keys in the database, `test_a`, `test_b` and -`test_c`. I can easily find the values of all of these keys in one operation by -using the `_match` operand. To demonstrate: +`test_c`. I can easily find these keys in one operation by using the `_match` +operand. To demonstrate: ```` -$ curl -v localhost:8080/oleg/test/_match -> GET /oleg/test/_match HTTP/1.1 -> User-Agent: curl/7.22.0 (x86_64-pc-linux-gnu) libcurl/7.22.0 OpenSSL/1.0.1 -> zlib/1.2.3.4 libidn/1.23 librtmp/2.3 -> Host: localhost:8080 -> Accept: */* -> -< HTTP/1.1 200 OK -< Server: OlegDB/fresh_cuts_n_jams -< Content-Type: application/json -< Content-Length: 25 -< Connection: close -< X-OlegDB-Num-Matches: 3 -< -["test3","test2","test1"] +$ curl -i localhost:8080/oleg/test/_match +HTTP/1.1 200 OK +Date: Sun, 28 Sep 2014 07:26:35 GMT +Content-Length: 20 +Content-Type: text/plain; charset=utf-8 + +test_a +test_b +test_c ```` -This returns a JSON-encoded list of all of your values. Also of note is the -`X-OlegDB-Num-Matches` header which specifies the number of keys that matched +This returns a list of all the keys separated by `\n`. Also of note is the +`X-Olegdb-Num-Matches` header which specifies the number of keys that matched the given prefix. If no matches are present, a 404 is returned. From 3f0406069d452d2f71a40b2c136743281efaa6b9 Mon Sep 17 00:00:00 2001 From: Hamcha Date: Sun, 28 Sep 2014 09:31:03 +0200 Subject: [PATCH 23/43] Fix bug in prefix matching api --- frontend/operations.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/frontend/operations.go b/frontend/operations.go index c760c95..fa56c3f 100644 --- a/frontend/operations.go +++ b/frontend/operations.go @@ -102,12 +102,9 @@ func httpDelete(w http.ResponseWriter, op Operation) *HTTPError { func httpMatch(w http.ResponseWriter, op Operation) *HTTPError { has, res := op.Database.PrefixMatch(op.Key) if !has { - return &HTTPError{Code: 500, Message: "Something went horribly wrong..."} - } - if len(res) == 0 { return &HTTPError{Code: 404, Message: "No matches found"} } - + w.Header().Add("X-Olegdb-Num-Matches", strconv.Itoa(len(res))) fmt.Fprintf(w, strings.Join(res, "\n")) return nil } From 349d3bbba50f0f6504802926c14129ccdc0ba64d Mon Sep 17 00:00:00 2001 From: Hamcha Date: Sun, 28 Sep 2014 12:19:39 +0200 Subject: [PATCH 24/43] Fix compilation issue the ugly way --- Makefile | 4 +++- frontend/goleg/highlevel.go | 2 -- frontend/goleg/wrapper.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 63dc0a6..2a56f65 100644 --- a/Makefile +++ b/Makefile @@ -5,13 +5,15 @@ ifndef CC endif VERSION=0.1.5 SOVERSION=0 -BUILD_DIR=$(shell pwd)/build/ +CUR_DIR=$(shell pwd) +BUILD_DIR=$(CUR_DIR)/build/ LIB_DIR=$(BUILD_DIR)lib/ BIN_DIR=$(BUILD_DIR)bin/ PREFIX?=/usr/local INSTALL_LIB=$(PREFIX)/lib/ INSTALL_BIN=$(PREFIX)/bin/ INSTALL_INCLUDE=$(PREFIX)/include/olegdb/ +export CGO_LDFLAGS=-L$(BUILD_DIR)lib INCLUDES=-I./include diff --git a/frontend/goleg/highlevel.go b/frontend/goleg/highlevel.go index 769ebb0..7ffc95f 100644 --- a/frontend/goleg/highlevel.go +++ b/frontend/goleg/highlevel.go @@ -1,8 +1,6 @@ package goleg /* -#cgo CFLAGS: -I../../include -#cgo LDFLAGS: -L../../build/lib -loleg #include "oleg.h" */ import "C" diff --git a/frontend/goleg/wrapper.go b/frontend/goleg/wrapper.go index 14cebdc..b443112 100644 --- a/frontend/goleg/wrapper.go +++ b/frontend/goleg/wrapper.go @@ -2,7 +2,7 @@ package goleg /* #cgo CFLAGS: -I../../include -#cgo LDFLAGS: -L../../build/lib -loleg +#cgo LDFLAGS: -loleg #include #include "oleg.h" #include "cursor.h" From 80544a10b186f6fb2d7a058a7e3a54216b76bbb6 Mon Sep 17 00:00:00 2001 From: Hamcha Date: Sun, 28 Sep 2014 12:46:08 +0200 Subject: [PATCH 25/43] Fix potential sigsegv caught by Clang 3.3 --- c_src/test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/c_src/test.c b/c_src/test.c index 058c61d..2270f72 100644 --- a/c_src/test.c +++ b/c_src/test.c @@ -175,6 +175,7 @@ int test_jar(const ol_feature_flags features) { int test_can_jump_cursor(const ol_feature_flags features) { ol_database *db = _test_db_open(features); int max_records = 10; + unsigned char *r_val = NULL; unsigned char to_insert[] = "roadkill"; int i; for (i = 0; i < max_records; i++) { @@ -212,7 +213,6 @@ int test_can_jump_cursor(const ol_feature_flags features) { check(node != NULL, "Could not retrieve node."); /* Prep some variables so we can check them */ - unsigned char *r_val = NULL; char r_key[KEY_SIZE] = {'0'}; size_t r_vsize; From a603f16629be88d517b9a8c91ef8c0d64995acb3 Mon Sep 17 00:00:00 2001 From: Hamcha Date: Tue, 7 Oct 2014 07:01:16 +0200 Subject: [PATCH 26/43] Couple more tests --- frontend/goleg/goleg_test.go | 51 ++++++++++++++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/frontend/goleg/goleg_test.go b/frontend/goleg/goleg_test.go index cb3820b..c174d2b 100644 --- a/frontend/goleg/goleg_test.go +++ b/frontend/goleg/goleg_test.go @@ -3,16 +3,18 @@ package goleg import ( "io/ioutil" "os" + "strconv" "testing" ) -func openRandomDB() (Database, string, error) { +func openRandomDB(features int) (Database, string, error) { name, err := ioutil.TempDir("/tmp", "goleg") if err != nil { return Database{}, "", err } - database, err := Open(name, "test", F_APPENDONLY|F_AOL_FFLUSH|F_LZ4|F_SPLAYTREE) + //F_APPENDONLY|F_AOL_FFLUSH|F_LZ4|F_SPLAYTREE + database, err := Open(name, "test", features) if err != nil { return Database{}, "", err } @@ -37,3 +39,48 @@ func TestOpen(t *testing.T) { database.Close() cleanTemp(dir) } + +const JARN = 1000 + +func TestJar(t *testing.T) { + if testing.Short() { + t.Skip("Skipping in short mode") + } + + database, dir, err := openRandomDB(F_LZ4 | F_SPLAYTREE) + if err != nil { + t.Fatalf("Can't open database: %s", err.Error()) + } + + for i := 0; i < JARN; i++ { + if !database.Jar("record"+strconv.Itoa(i), []byte("value"+strconv.Itoa(i))) { + t.Fatalf("Can't jar value #%d", i) + } + } + + database.Close() + cleanTemp(dir) +} + +func TestUnjar(t *testing.T) { + database, dir, err := openRandomDB(F_LZ4 | F_SPLAYTREE) + if err != nil { + t.Fatalf("Can't open database: %s", err.Error()) + } + + for i := 0; i < JARN; i++ { + if !database.Jar("record"+strconv.Itoa(i), []byte("value"+strconv.Itoa(i))) { + t.Fatalf("Can't jar value #%d", i) + } + } + + for i := 0; i < JARN; i++ { + val := database.Unjar("record" + strconv.Itoa(i)) + if val != []byte("value"+strconv.Itoa(i)) { + t.Errorf("Value #%d doesn't match", i) + } + } + + database.Close() + cleanTemp(dir) +} From e33f4caf0d5ebe49a3a8033a6fde4df09d272f28 Mon Sep 17 00:00:00 2001 From: Kyle Terry Date: Sun, 9 Nov 2014 13:37:36 -0800 Subject: [PATCH 27/43] Cleaner errors, still needs some work --- frontend/main.go | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/frontend/main.go b/frontend/main.go index 569699c..3578602 100644 --- a/frontend/main.go +++ b/frontend/main.go @@ -7,6 +7,8 @@ import ( "fmt" "io/ioutil" "net/http" + "log" + "os" ) type Config struct { @@ -22,6 +24,24 @@ type Config struct { var config Config var databases map[string]goleg.Database +const ( +Usage = ` +Usage: %s -config [options] + Config path: + Path to a configuration file. This is required. +Options: + -bind + Override Listen directive in configuration. + -dir + Override db storage location in configuration. + -v + Version + -h + This help. +` +) + + func main() { // Parse command line flags (if there are) directory := flag.String("dir", "", "Directory where to store dumps and data") @@ -30,10 +50,16 @@ func main() { flag.Parse() + if configfile == nil { + text := fmt.Sprintf(Usage, os.Args[0]) + log.Print("Config file is required") + log.Fatal(text) + } + // Parse config file rawconf, err := ioutil.ReadFile(*configfile) if err != nil { - panic(err.Error()) + log.Fatal("Could not read configuration. Please see the documentation.") } err = json.Unmarshal(rawconf, &config) if err != nil { From ec8cbaa8dea601818990bebd1431b8df745d6594 Mon Sep 17 00:00:00 2001 From: Kyle Terry Date: Sun, 9 Nov 2014 14:03:43 -0800 Subject: [PATCH 28/43] Lock because oleg is like that --- frontend/goleg/highlevel.go | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/frontend/goleg/highlevel.go b/frontend/goleg/highlevel.go index 7ffc95f..c5a7699 100644 --- a/frontend/goleg/highlevel.go +++ b/frontend/goleg/highlevel.go @@ -7,11 +7,13 @@ import "C" import ( "errors" "time" + "sync" ) type Database struct { db *C.ol_database RecordCount *C.int + mutex *sync.Mutex } func Open(path, name string, features int) (Database, error) { @@ -21,52 +23,75 @@ func Open(path, name string, features int) (Database, error) { return Database{}, errors.New("Can't open database (NULL returned)") } database.RecordCount = &database.db.rcrd_cnt + database.mutex = &sync.Mutex{} return database, nil } func (d Database) Close() int { + d.mutex.Lock() + defer d.mutex.Unlock() return CClose(d.db) } func (d Database) Unjar(key string) []byte { + d.mutex.Lock() + defer d.mutex.Unlock() var dsize uintptr return CUnjar(d.db, key, uintptr(len(key)), &dsize) } func (d Database) Jar(key string, value []byte) int { + d.mutex.Lock() + defer d.mutex.Unlock() return CJar(d.db, key, uintptr(len(key)), value, uintptr(len(value))) } func (d Database) Scoop(key string) int { + d.mutex.Lock() + defer d.mutex.Unlock() return CScoop(d.db, key, uintptr(len(key))) } func (d Database) Uptime() int { + d.mutex.Lock() + defer d.mutex.Unlock() return CUptime(d.db) } func (d Database) Expiration(key string) (time.Time, bool) { + d.mutex.Lock() + defer d.mutex.Unlock() return CExpirationTime(d.db, key, uintptr(len(key))) } func (d Database) Spoil(key string, expiration time.Time) int { + d.mutex.Lock() + defer d.mutex.Unlock() return CSpoil(d.db, key, uintptr(len(key)), expiration) } func (d Database) Exists(key string) bool { + d.mutex.Lock() + defer d.mutex.Unlock() return CExists(d.db, key, uintptr(len(key))) == 0 } func (d Database) Squish() bool { + d.mutex.Lock() + defer d.mutex.Unlock() return CSquish(d.db) == 1 } func (d Database) PrefixMatch(prefix string) (bool, []string) { + d.mutex.Lock() + defer d.mutex.Unlock() len, out := CPrefixMatch(d.db, prefix, uintptr(len(prefix))) return len >= 0, out } func (d Database) First() (bool, string, []byte) { + d.mutex.Lock() + defer d.mutex.Unlock() node := CNodeFirst(d.db) if node == nil { return false, "", nil @@ -75,6 +100,8 @@ func (d Database) First() (bool, string, []byte) { } func (d Database) Last() (bool, string, []byte) { + d.mutex.Lock() + defer d.mutex.Unlock() node := CNodeLast(d.db) if node == nil { return false, "", nil @@ -83,6 +110,8 @@ func (d Database) Last() (bool, string, []byte) { } func (d Database) Next(key string) (bool, string, []byte) { + d.mutex.Lock() + defer d.mutex.Unlock() node := CNodeNext(d.db, key, uintptr(len(key))) if node == nil { return false, "", nil @@ -91,6 +120,8 @@ func (d Database) Next(key string) (bool, string, []byte) { } func (d Database) Prev(key string) (bool, string, []byte) { + d.mutex.Lock() + defer d.mutex.Unlock() node := CNodePrev(d.db, key, uintptr(len(key))) if node == nil { return false, "", nil From 116765caddd48a1dd25c3d611a1529707c8fa5fd Mon Sep 17 00:00:00 2001 From: Kyle Terry Date: Sun, 9 Nov 2014 14:10:43 -0800 Subject: [PATCH 29/43] Added data dir to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index f0d94db..41a1184 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ olegdb *.o *.dump +data From 21f7391f563d104a721ba8a9af83400ceb9f366e Mon Sep 17 00:00:00 2001 From: Kyle Terry Date: Sun, 9 Nov 2014 14:56:50 -0800 Subject: [PATCH 30/43] Couple of fixes. Q can fix the rest --- scripts/torture.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/torture.py b/scripts/torture.py index e2f19c1..9342c9b 100755 --- a/scripts/torture.py +++ b/scripts/torture.py @@ -30,11 +30,11 @@ def thread_burn(thread_id): requests.post(connection_str, data=compressed, headers={ - "X-OlegDB-use-by": expiration} + "X-Olegdb-Use-By": expiration} ) - duff = requests.head(connection_str) # For code coverage + duff = requests.get(connection_str + "/_info") # For code coverage if duff.status_code not in [404, 500]: - known_count = duff.headers['x-olegdb-rcrd-cnt'] + known_count = duff.headers['X-Olegdb-Rcrd-Cnt'] resp = requests.get(connection_str, stream=True) raw = resp.raw.read() From d62d926c0e3aa96762800b22b651cc30f13c1b4b Mon Sep 17 00:00:00 2001 From: Kyle Terry Date: Sun, 9 Nov 2014 19:16:31 -0800 Subject: [PATCH 31/43] Stray erlang words --- README.md | 2 +- docs/Overview/01_What_is_OlegDB.markdown | 6 +++--- docs/Overview/02_Installation.markdown | 26 ++++++++---------------- 3 files changed, 12 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 6f699c5..d163457 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ To run tests: ./run_tests.sh ``` -To run the erlang server: +To run the Go server: ```bash olegdb [-conf olegdb.conf] [-bind localhost:8080] [-dir data] diff --git a/docs/Overview/01_What_is_OlegDB.markdown b/docs/Overview/01_What_is_OlegDB.markdown index 94cf3fa..d423154 100644 --- a/docs/Overview/01_What_is_OlegDB.markdown +++ b/docs/Overview/01_What_is_OlegDB.markdown @@ -1,7 +1,7 @@ -OlegDB is a concurrent, pretty fast K/V hashtable with an Erlang frontend. +OlegDB is a concurrent, pretty fast K/V hashtable with an Go frontend. It uses the Murmur3 hashing algorithm to hash and index keys. We chose -Erlang for the server because it is functional, uses the actor model and -the pattern matching is ridiculous. +Go for the server because it is easy to rapidly create an HTTP frontend that is +performant and has all the tools in core to prevent race conditions. In addition to this, liboleg is a C library that powers everything. liboleg exports a relatively simplistic API for use in other applications. We build the main diff --git a/docs/Overview/02_Installation.markdown b/docs/Overview/02_Installation.markdown index b7c751d..78f70a1 100644 --- a/docs/Overview/02_Installation.markdown +++ b/docs/Overview/02_Installation.markdown @@ -1,5 +1,5 @@ Installing OlegDB is pretty simple, you only need a POSIX compliant system, -make, gcc/clang (thats all we test) and Erlang. You'll also need the source +make, gcc/clang (thats all we test) and Go. You'll also need the source code for OlegDB. Once you have your fanciful medley of computer science tools, you're ready to @@ -11,34 +11,24 @@ I'm going to assume you've extracted the source tarball into a folder called `~/src/olegdb` and that you haven't cd'd into it yet. Lets smash some electrons together: -```` +```bash $ cd ~/src/olegdb $ make $ sudo make install -```` +``` If you really wanted to, you could specify a different installation directory. The default is `/usr/local`. You can do this by setting the `PREFIX` variable before compilation: -```` +```bash $ sudo make PREFIX=/usr/ install -```` +``` Actually running OlegDB and getting it do stuff after this point is trivial, if your installation prefix is in your `PATH` variable you should just be able to run something like the following: -```` -$ olegdb [data_directory] -```` - -...where `[data_directory]` is the place you want OlegDB to store persistent data. -Make it `/dev/null` if you want, I don't care. You can also specify -IP/port information from the commandline: - -```` -$ olegdb /tmp 1978 #Starts OlegDB listening on port 1978 -$ olegdb /tmp 0.0.0.0 1337 #Starts OlegDB listening on the 0.0.0.0 IP, with port 1337 -$ olegdb /tmp data.shithouse.tv 666 #Hostnames work too -```` +```bash +$ olegdb -config /path/to/json/config +``` From 7873e1c4fa3b68ea9a81ff7ea01d296fd0dc563c Mon Sep 17 00:00:00 2001 From: Kyle Terry Date: Tue, 25 Nov 2014 13:26:36 -0800 Subject: [PATCH 32/43] Git leaving messes behind --- c_src/test.c | 1 - 1 file changed, 1 deletion(-) diff --git a/c_src/test.c b/c_src/test.c index 6711aa0..bb2d84f 100644 --- a/c_src/test.c +++ b/c_src/test.c @@ -209,7 +209,6 @@ int test_can_jump_cursor(const ol_feature_flags features) { check(tx != NULL, "Could not begin transaction."); int max_records = 10; - unsigned char *r_val = NULL; unsigned char to_insert[] = "roadkill"; int i; for (i = 0; i < max_records; i++) { From 08d1461ccdff4970efe8996486431429796dc81b Mon Sep 17 00:00:00 2001 From: Kyle Terry Date: Tue, 25 Nov 2014 13:43:09 -0800 Subject: [PATCH 33/43] Compiles, but still can't find liboleg --- frontend/goleg/wrapper.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/goleg/wrapper.go b/frontend/goleg/wrapper.go index b443112..412e4e3 100644 --- a/frontend/goleg/wrapper.go +++ b/frontend/goleg/wrapper.go @@ -45,9 +45,9 @@ func CUnjar(db *C.ol_database, key string, klen uintptr, dsize *uintptr) []byte cklen := (C.size_t)(klen) cdsize := (*C.size_t)(unsafe.Pointer(dsize)) - // Pass them to ol_unjar_ds + // Pass them to ol_unjar var ptr *C.uchar - res := C.ol_unjar_ds(db, ckey, cklen, &ptr, cdsize) + res := C.ol_unjar(db, ckey, cklen, &ptr, cdsize) if res == 1 { return nil } From c0379dc4bf2cf0ff42f4013be51ab01ed7b34749 Mon Sep 17 00:00:00 2001 From: Quinlan Pfiffer Date: Tue, 25 Nov 2014 22:31:45 -0800 Subject: [PATCH 34/43] Makefile actually works now. --- Makefile | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 0863299..1c9316b 100644 --- a/Makefile +++ b/Makefile @@ -28,16 +28,15 @@ all: liboleg oleg_test server test.o: ./c_src/test.c $(CC) $(CFLAGS) $(INCLUDES) -c $< + main.o: ./c_src/main.c $(CC) $(CFLAGS) $(INCLUDES) -c $< %.o: ./c_src/%.c $(CC) $(CFLAGS) $(INCLUDES) -c -fPIC $< -FORCE: - -oleg_test: $(BIN_DIR)oleg_test -$(BIN_DIR)oleg_test: liboleg test.o main.o +oleg_test: $(BIN_DIR)oleg_test liboleg +$(BIN_DIR)oleg_test: test.o main.o $(CC) $(CFLAGS) $(INCLUDES) -L$(LIB_DIR) -o $(BIN_DIR)oleg_test test.o main.o $(MATH_LINKER) -loleg liboleg: $(LIB_DIR)liboleg.so @@ -48,8 +47,11 @@ uninstall: rm -rf $(INSTALL_LIB)liboleg* rm -rf $(INSTALL_BIN)olegdb -server: liboleg - go build -o $(BIN_DIR)olegdb ./frontend +frontend: $(BIN_DIR)olegdb +$(BIN_DIR)olegdb: + go build -o $(BIN_DIR)olegdb ./frontend/ + +server: liboleg frontend install: goinstall From 45cd984d71449e910616c79913f9072409496ad9 Mon Sep 17 00:00:00 2001 From: Quinlan Pfiffer Date: Tue, 25 Nov 2014 22:36:18 -0800 Subject: [PATCH 35/43] I think this fixes the linker bugs. --- Makefile | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 1c9316b..731ee06 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ else MATH_LINKER=-lm endif -all: liboleg oleg_test server +all: oleg_test frontend test.o: ./c_src/test.c $(CC) $(CFLAGS) $(INCLUDES) -c $< @@ -35,8 +35,8 @@ main.o: ./c_src/main.c %.o: ./c_src/%.c $(CC) $(CFLAGS) $(INCLUDES) -c -fPIC $< -oleg_test: $(BIN_DIR)oleg_test liboleg -$(BIN_DIR)oleg_test: test.o main.o +oleg_test: $(BIN_DIR)oleg_test +$(BIN_DIR)oleg_test: liboleg test.o main.o $(CC) $(CFLAGS) $(INCLUDES) -L$(LIB_DIR) -o $(BIN_DIR)oleg_test test.o main.o $(MATH_LINKER) -loleg liboleg: $(LIB_DIR)liboleg.so @@ -48,14 +48,12 @@ uninstall: rm -rf $(INSTALL_BIN)olegdb frontend: $(BIN_DIR)olegdb -$(BIN_DIR)olegdb: +$(BIN_DIR)olegdb: liboleg go build -o $(BIN_DIR)olegdb ./frontend/ -server: liboleg frontend - install: goinstall -goinstall: server libinstall +goinstall: frontend libinstall cp $(BIN_DIR)olegdb $(INSTALL_BIN)olegdb libinstall: liboleg From 99f7018c662bac66209c294a21f11c6644a87a43 Mon Sep 17 00:00:00 2001 From: Kyle Terry Date: Tue, 25 Nov 2014 15:06:15 -0800 Subject: [PATCH 36/43] Cleaned up logging and config example --- frontend/main.go | 18 +++++++++++------- olegdb.conf.sample | 12 ++++++------ 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/frontend/main.go b/frontend/main.go index 3578602..52d3a86 100644 --- a/frontend/main.go +++ b/frontend/main.go @@ -45,21 +45,23 @@ Options: func main() { // Parse command line flags (if there are) directory := flag.String("dir", "", "Directory where to store dumps and data") - configfile := flag.String("config", "olegdb.conf", "Config file to read settings from") + configfile := flag.String("config", "", "Config file to read settings from") bindaddr := flag.String("bind", "", "Address and port to bind, host:port") flag.Parse() - if configfile == nil { - text := fmt.Sprintf(Usage, os.Args[0]) - log.Print("Config file is required") - log.Fatal(text) + if *configfile == "" { + fmt.Println("Config file is required") + fmt.Printf(Usage, os.Args[0]) + os.Exit(1) } // Parse config file rawconf, err := ioutil.ReadFile(*configfile) if err != nil { - log.Fatal("Could not read configuration. Please see the documentation.") + fmt.Println("Could not read configuration. Please see the documentation.") + fmt.Printf(Usage, os.Args[0]) + os.Exit(1) } err = json.Unmarshal(rawconf, &config) if err != nil { @@ -79,10 +81,12 @@ func main() { http.HandleFunc("/", handler) defer unload() - fmt.Println("Listening on " + config.Listen) + log.Println("Starting server...") if config.UseHTTPS { + log.Println("Listening on https://" + config.Listen) http.ListenAndServeTLS(config.Listen, config.CertFile, config.PkeyFile, nil) } else { + log.Println("Listening on http://" + config.Listen) http.ListenAndServe(config.Listen, nil) } } diff --git a/olegdb.conf.sample b/olegdb.conf.sample index 4f7e1dd..9ae4053 100644 --- a/olegdb.conf.sample +++ b/olegdb.conf.sample @@ -1,7 +1,7 @@ { - "Listen" : "localhost:8080", - "DataDir" : "data", - "UseHTTPS" : false, - "CertFile" : "", - "PkeyFile" : "" -} \ No newline at end of file + "Listen": "localhost:38080", + "DataDir": "data", + "UseHTTPS": false, + "CertFile": "", + "PkeyFile": "" +} From 114e016108165569f0092f86e211305c2dc8a46e Mon Sep 17 00:00:00 2001 From: Kyle Terry Date: Tue, 25 Nov 2014 15:11:58 -0800 Subject: [PATCH 37/43] More json-esk --- frontend/main.go | 10 +++++----- olegdb.conf.sample | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/frontend/main.go b/frontend/main.go index 52d3a86..c65b16b 100644 --- a/frontend/main.go +++ b/frontend/main.go @@ -13,12 +13,12 @@ import ( type Config struct { // Base settings (required) - Listen string - DataDir string + Listen string `json:"listen"` + DataDir string `json: "datadir"` // HTTPS settings - UseHTTPS bool - CertFile string - PkeyFile string + UseHTTPS bool `json:"usehttps"` + CertFile string `json:"certfile"` + PkeyFile string `json:"pkeyfile"` } var config Config diff --git a/olegdb.conf.sample b/olegdb.conf.sample index 9ae4053..8ff3e06 100644 --- a/olegdb.conf.sample +++ b/olegdb.conf.sample @@ -1,7 +1,7 @@ { - "Listen": "localhost:38080", - "DataDir": "data", - "UseHTTPS": false, - "CertFile": "", - "PkeyFile": "" + "listen": "localhost:38080", + "datadir": "data", + "usehttps": false, + "certfile": "", + "pkeyfile": "" } From c3e584090a96bd605b393a71ab04544a10c4d00e Mon Sep 17 00:00:00 2001 From: Kyle Terry Date: Tue, 25 Nov 2014 15:40:02 -0800 Subject: [PATCH 38/43] Fixed some tests --- frontend/goleg/goleg_test.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/frontend/goleg/goleg_test.go b/frontend/goleg/goleg_test.go index c174d2b..d91aa94 100644 --- a/frontend/goleg/goleg_test.go +++ b/frontend/goleg/goleg_test.go @@ -5,6 +5,8 @@ import ( "os" "strconv" "testing" + "fmt" + "bytes" ) func openRandomDB(features int) (Database, string, error) { @@ -31,7 +33,7 @@ func TestOpen(t *testing.T) { t.Skip("Skipping in short mode") } - database, dir, err := openRandomDB() + database, dir, err := openRandomDB(F_APPENDONLY) if err != nil { t.Fatalf("Can't open database: %s", err.Error()) } @@ -53,7 +55,7 @@ func TestJar(t *testing.T) { } for i := 0; i < JARN; i++ { - if !database.Jar("record"+strconv.Itoa(i), []byte("value"+strconv.Itoa(i))) { + if database.Jar("record"+strconv.Itoa(i), []byte("value"+strconv.Itoa(i))) != 0 { t.Fatalf("Can't jar value #%d", i) } } @@ -69,14 +71,15 @@ func TestUnjar(t *testing.T) { } for i := 0; i < JARN; i++ { - if !database.Jar("record"+strconv.Itoa(i), []byte("value"+strconv.Itoa(i))) { + if database.Jar("record"+strconv.Itoa(i), []byte("value"+strconv.Itoa(i))) != 0 { t.Fatalf("Can't jar value #%d", i) } } for i := 0; i < JARN; i++ { val := database.Unjar("record" + strconv.Itoa(i)) - if val != []byte("value"+strconv.Itoa(i)) { + fmt.Printf("%#v\n", val) + if bytes.Equal(val, []byte("value"+strconv.Itoa(i))) { t.Errorf("Value #%d doesn't match", i) } } From 30936fc33c8056e417cbd6d8743a29f9241d80ef Mon Sep 17 00:00:00 2001 From: Kyle Terry Date: Tue, 25 Nov 2014 16:08:16 -0800 Subject: [PATCH 39/43] oops, this passes now --- frontend/goleg/goleg_test.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/frontend/goleg/goleg_test.go b/frontend/goleg/goleg_test.go index d91aa94..08d1aa0 100644 --- a/frontend/goleg/goleg_test.go +++ b/frontend/goleg/goleg_test.go @@ -5,7 +5,6 @@ import ( "os" "strconv" "testing" - "fmt" "bytes" ) @@ -78,8 +77,7 @@ func TestUnjar(t *testing.T) { for i := 0; i < JARN; i++ { val := database.Unjar("record" + strconv.Itoa(i)) - fmt.Printf("%#v\n", val) - if bytes.Equal(val, []byte("value"+strconv.Itoa(i))) { + if !bytes.Equal(val, []byte("value"+strconv.Itoa(i))) { t.Errorf("Value #%d doesn't match", i) } } From 969fdd6ed1f7dffef13a49660668a5b07a64b18a Mon Sep 17 00:00:00 2001 From: Kyle Terry Date: Tue, 25 Nov 2014 16:08:34 -0800 Subject: [PATCH 40/43] We only need a few samples since oleg backend covers most of this --- frontend/goleg/goleg_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/goleg/goleg_test.go b/frontend/goleg/goleg_test.go index 08d1aa0..49769b1 100644 --- a/frontend/goleg/goleg_test.go +++ b/frontend/goleg/goleg_test.go @@ -41,7 +41,7 @@ func TestOpen(t *testing.T) { cleanTemp(dir) } -const JARN = 1000 +const JARN = 10 func TestJar(t *testing.T) { if testing.Short() { From 0a9330328f3b8aff88bd01c7479eca85de4f37ed Mon Sep 17 00:00:00 2001 From: Kyle Terry Date: Tue, 25 Nov 2014 16:15:15 -0800 Subject: [PATCH 41/43] Hopefully this hack heals travis --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 2dd1274..fd9f2fb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,7 @@ compiler: - clang script: - make liboleg oleg_test + - make install - export LD_LIBRARY_PATH=./build/lib:$LD_LIBRARY_PATH - ./build/bin/oleg_test - go test ./... From 1d6430f733939d238cb1310ee0d310ccb6a363ae Mon Sep 17 00:00:00 2001 From: Kyle Terry Date: Tue, 25 Nov 2014 16:16:48 -0800 Subject: [PATCH 42/43] sudo --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index fd9f2fb..040262e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ compiler: - clang script: - make liboleg oleg_test - - make install + - sudo make install - export LD_LIBRARY_PATH=./build/lib:$LD_LIBRARY_PATH - ./build/bin/oleg_test - go test ./... From a8665dff573bf12bdbf773d5bc3a785a752c148d Mon Sep 17 00:00:00 2001 From: Kyle Terry Date: Tue, 25 Nov 2014 16:19:53 -0800 Subject: [PATCH 43/43] I'm despirate --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 040262e..9617a07 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ script: - sudo make install - export LD_LIBRARY_PATH=./build/lib:$LD_LIBRARY_PATH - ./build/bin/oleg_test - - go test ./... + - LD_LIBRARY_PATH=./build/lib:/usr/local/lib:/usr/lib:$LD_LIBRARY_PATH go test ./... before_install: - sudo apt-get update - sudo apt-get install golang