From 256c79efb97b56b215891a743575e92ef9d475c6 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Sat, 26 Oct 2024 13:37:06 +1100 Subject: [PATCH 01/85] Create the minimal viable mainloop --- Makefile | 1 + src/edge_utils.c | 60 +++++++++------------------------------ src/mainloop.c | 74 ++++++++++++++++++++++++++++++++++++++++++++++++ src/mainloop.h | 14 +++++++++ 4 files changed, 102 insertions(+), 47 deletions(-) create mode 100644 src/mainloop.c create mode 100644 src/mainloop.h diff --git a/Makefile b/Makefile index 20b5a85..8b7ad94 100644 --- a/Makefile +++ b/Makefile @@ -103,6 +103,7 @@ OBJS=\ src/initfuncs.o \ src/json.o \ src/logging.o \ + src/mainloop.o \ src/management.o \ src/metrics.o \ src/minilzo.o \ diff --git a/src/edge_utils.c b/src/edge_utils.c index 9be89b3..a9d63a5 100644 --- a/src/edge_utils.c +++ b/src/edge_utils.c @@ -45,6 +45,7 @@ #include // for gethostname, sleep #include "auth.h" // for generate_private_key #include "header_encryption.h" // for packet_header_encrypt, packet_he... +#include "mainloop.h" // for mainloop_runonce #include "management.h" // for readFromMgmtSocket #include "n2n.h" // for n3n_runtime_data, n2n_edge_... #include "n2n_wire.h" // for fill_sockaddr, decod... @@ -3012,51 +3013,12 @@ int run_edge_loop (struct n3n_runtime_data *eee) { while(*eee->keep_running) { - int rc, max_sock = 0; + int rc; fd_set readers; fd_set writers; time_t now; - FD_ZERO(&readers); - FD_ZERO(&writers); - - if(eee->sock >= 0) { - FD_SET(eee->sock, &readers); - max_sock = max(max_sock, eee->sock); - } -#ifndef SKIP_MULTICAST_PEERS_DISCOVERY - if((eee->conf.allow_p2p) - && (eee->conf.preferred_sock.family == (uint8_t)AF_INVALID)) { - FD_SET(eee->udp_multicast_sock, &readers); - max_sock = max(max_sock, eee->udp_multicast_sock); - } -#endif - -#ifndef _WIN32 - FD_SET(eee->device.fd, &readers); - max_sock = max(max_sock, eee->device.fd); -#endif - - slots_t *slots = eee->mgmt_slots; - max_sock = max( - max_sock, - slots_fdset( - slots, - &readers, - &writers - ) - ); - - // FIXME: - // unlock the windows tun reader thread before select() and lock it - // again after select(). It currently works by accident, but the - // structures it manipulates are not thread-safe, so try to make it - // work by /design/ - - struct timeval wait_time; - wait_time.tv_sec = (eee->sn_wait) ? (SOCKET_TIMEOUT_INTERVAL_SECS / 10 + 1) : (SOCKET_TIMEOUT_INTERVAL_SECS); - wait_time.tv_usec = 0; - rc = select(max_sock + 1, &readers, &writers, NULL, &wait_time); + rc = mainloop_runonce(&readers, &writers, eee); now = time(NULL); if(rc > 0) { @@ -3109,7 +3071,11 @@ int run_edge_loop (struct n3n_runtime_data *eee) { } #endif - int slots_ready = slots_fdset_loop(slots, &readers, &writers); + int slots_ready = slots_fdset_loop( + eee->mgmt_slots, + &readers, + &writers + ); if(slots_ready < 0) { traceEvent( @@ -3124,20 +3090,20 @@ int run_edge_loop (struct n3n_runtime_data *eee) { // for each OS supported) // This should only be a concern if we are doing a large // number of slot connections - for(int i=0; inr_slots; i++) { - if(slots->conn[i].fd == -1) { + for(int i=0; imgmt_slots->nr_slots; i++) { + if(eee->mgmt_slots->conn[i].fd == -1) { continue; } - if(slots->conn[i].state == CONN_READY) { - mgmt_api_handler(eee, &slots->conn[i]); + if(eee->mgmt_slots->conn[i].state == CONN_READY) { + mgmt_api_handler(eee, &eee->mgmt_slots->conn[i]); } } } } // check for timed out slots - slots_closeidle(slots); + slots_closeidle(eee->mgmt_slots); // If anything we recieved caused us to stop.. if(!(*eee->keep_running)) diff --git a/src/mainloop.c b/src/mainloop.c new file mode 100644 index 0000000..4bc2f91 --- /dev/null +++ b/src/mainloop.c @@ -0,0 +1,74 @@ +/** + * Copyright (C) Hamish Coleman + * SPDX-License-Identifier: GPL-3.0-only + * + */ + +#include // for slots_fdset +#include // for select, FD_ZERO, +#include // for n3n_runtime_data + +#ifndef max +#define max(a, b) (((a) < (b)) ? (b) : (a)) +#endif + +#ifndef min +#define min(a, b) (((a) >(b)) ? (b) : (a)) +#endif + +static int setup_select(fd_set *rd, fd_set *wr, struct n3n_runtime_data *eee) { + FD_ZERO(rd); + FD_ZERO(wr); + int max_sock = 0; + + if(eee->sock >= 0) { + FD_SET(eee->sock, rd); + max_sock = max(max_sock, eee->sock); + } +#ifndef SKIP_MULTICAST_PEERS_DISCOVERY + if((eee->conf.allow_p2p) + && (eee->conf.preferred_sock.family == (uint8_t)AF_INVALID)) { + FD_SET(eee->udp_multicast_sock, rd); + max_sock = max(max_sock, eee->udp_multicast_sock); + } +#endif + +#ifndef _WIN32 + FD_SET(eee->device.fd, rd); + max_sock = max(max_sock, eee->device.fd); +#endif + + max_sock = max( + max_sock, + slots_fdset( + eee->mgmt_slots, + rd, + wr + ) + ); + + return max_sock; +} + +int mainloop_runonce(fd_set *rd, fd_set *wr, struct n3n_runtime_data *eee) { + + int maxfd = setup_select(rd, wr, eee); + + // FIXME: + // unlock the windows tun reader thread before select() and lock it + // again after select(). It currently works by accident, but the + // structures it manipulates are not thread-safe, so try to make it + // work by /design/ + + struct timeval wait_time; + if(eee->sn_wait) { + wait_time.tv_sec = (SOCKET_TIMEOUT_INTERVAL_SECS / 10 + 1); + } else { + wait_time.tv_sec = (SOCKET_TIMEOUT_INTERVAL_SECS); + } + wait_time.tv_usec = 0; + + int rc = select(maxfd + 1, rd, wr, NULL, &wait_time); + + return rc; +} diff --git a/src/mainloop.h b/src/mainloop.h new file mode 100644 index 0000000..928bd8c --- /dev/null +++ b/src/mainloop.h @@ -0,0 +1,14 @@ +/** + * Copyright (C) Hamish Coleman + * + * non public structure and function definitions + */ + +#ifndef _MAINLOOP_H_ +#define _MAINLOOP_H_ + +#include // for n3n_runtime_data + +int mainloop_runonce(fd_set *, fd_set *, struct n3n_runtime_data *); + +#endif From c101ad4fac733900986f82c29bce3a02f8788533 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Sat, 26 Oct 2024 14:11:22 +1100 Subject: [PATCH 02/85] Add a helper target to run the include-what-you-use checker --- Makefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Makefile b/Makefile index 8b7ad94..34a373e 100644 --- a/Makefile +++ b/Makefile @@ -339,6 +339,11 @@ clean.cov: apps/*.gcno apps/*.gcda \ tools/*.gcno tools/*.gcda +.PHONY: iwyu +iwyu: iwyu.out +iwyu.out: + CFLAGS="-Xiwyu --error_always" $(MAKE) -k CC=include-what-you-use 2> iwyu.out + .PHONY: clean clean: clean.cov rm -rf $(OBJS) $(SUBDIR_LIBS) $(DOCS) $(COVERAGEDIR)/ *.dSYM *~ From a5b8a5b67fd4a17072fecd680ee17929cf3d8fe4 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Sat, 26 Oct 2024 14:41:02 +1100 Subject: [PATCH 03/85] Address iwyu notes --- include/auth.h | 2 ++ include/speck.h | 1 + libs/connslot/connslot.c | 29 +++++++++++++---------------- libs/connslot/connslot.h | 8 +++++--- libs/connslot/strbuf.c | 15 +++++++-------- src/aes.c | 2 +- src/auth.c | 4 +++- src/cc20.c | 5 +++-- src/conffile.c | 10 ++++++++++ src/conffile_defs.c | 2 ++ src/edge_utils.c | 10 +++++++--- src/header_encryption.c | 4 ++-- src/mainloop.c | 6 +++++- src/management.c | 11 ++++++++++- src/management.h | 3 +++ src/metrics.c | 1 + src/n2n.c | 10 +++++++--- src/n2n_port_mapping.c | 6 ------ src/network_traffic_filter.c | 3 ++- src/peer_info.c | 5 +++++ src/peer_info.h | 6 ++++++ src/random_numbers.c | 1 + src/resolve.c | 15 +++++++++++---- src/resolve.h | 7 ++++++- src/sn_selection.c | 3 +++ src/sn_utils.c | 10 +++++++--- src/test_hashing.c | 3 +-- src/transform_aes.c | 3 +++ src/transform_cc20.c | 3 +++ src/transform_lzo.c | 5 ++++- src/transform_none.c | 2 ++ src/transform_null.c | 3 ++- src/transform_speck.c | 3 +++ src/transform_tf.c | 3 +++ src/tuntap_linux.c | 4 ++++ src/wire.c | 8 +++++--- 36 files changed, 153 insertions(+), 63 deletions(-) diff --git a/include/auth.h b/include/auth.h index a360b66..712af07 100644 --- a/include/auth.h +++ b/include/auth.h @@ -23,7 +23,9 @@ #include // for size_t #include // for uint8_t, uint32_t + #include "n2n.h" // for n2n_private_public_key_t, n2n_community_t, N2N_A... +#include "n2n_typedefs.h" int bin_to_ascii(char *out, uint8_t *in, size_t in_len); diff --git a/include/speck.h b/include/speck.h index 9f9cb18..18399b4 100644 --- a/include/speck.h +++ b/include/speck.h @@ -27,6 +27,7 @@ #include // for uint64_t, uint32_t +#include #define u32 uint32_t diff --git a/libs/connslot/connslot.c b/libs/connslot/connslot.c index 66db4fb..00149ed 100644 --- a/libs/connslot/connslot.c +++ b/libs/connslot/connslot.c @@ -6,26 +6,22 @@ */ #define _GNU_SOURCE +#include // for errno, EAGAIN, ENOENT, EWOULDBLOCK +#include // for fcntl, F_SETFL, O_NONBLOCK #ifndef _WIN32 -#include +#include // for htons, htonl, sockaddr_in, sock... #endif -#include -#include +#include // for remove +#include // for free, abort, malloc, strtoul +#include // for memmem, memcpy, strlen, strncpy #ifndef _WIN32 -#include +#include // for socket, bind, listen, setsockopt +#include // for chmod +#include // for writev +#include // for sockaddr_un #endif -#include -#include -#include -#ifndef _WIN32 -#include -#include -#include -#include -#include -#endif -#include -#include +#include // for NULL, time, size_t +#include // for close, chown #ifdef _WIN32 #include @@ -33,6 +29,7 @@ #endif #include "connslot.h" +#include "strbuf.h" // for sb_reprintf, sb_len, strbuf_t #ifdef _WIN32 // Windows is a strange place to live, if you are a POSIX programmer diff --git a/libs/connslot/connslot.h b/libs/connslot/connslot.h index d90d2e0..f1a341e 100644 --- a/libs/connslot/connslot.h +++ b/libs/connslot/connslot.h @@ -8,16 +8,18 @@ #ifndef CONNSLOT_H #define CONNSLOT_H -#include +#include // for bool +#include // for ssize_t + #ifndef _WIN32 -#include +#include // for fd_set #endif #ifdef _WIN32 #include #endif -#include "strbuf.h" +#include "strbuf.h" // for strbuf_t #ifdef _WIN32 void *memmem(void *haystack, size_t haystack_len, void * needle, size_t needle_len); diff --git a/libs/connslot/strbuf.c b/libs/connslot/strbuf.c index 2406436..7e0ef87 100644 --- a/libs/connslot/strbuf.c +++ b/libs/connslot/strbuf.c @@ -6,18 +6,17 @@ * SPDX-License-Identifier: LGPL-2.1-only */ -#include -#include -#include -#include -#include -#include -#include +#include // for va_list, va_end, va_start +#include // for bool, true, false +#include // for size_t, NULL +#include // for printf, vsnprintf +#include // for malloc, realloc +#include // for memcpy #ifdef _WIN32 #include #else -#include +#include // for recv, send #endif #include "strbuf.h" diff --git a/src/aes.c b/src/aes.c index e1ab836..0a7879a 100644 --- a/src/aes.c +++ b/src/aes.c @@ -18,7 +18,7 @@ * */ -#include "config.h" +#include "config.h" // for HAVE_LIBCRYPTO #include // for traceEvent #include // for uint32_t, uint8_t diff --git a/src/auth.c b/src/auth.c index 227c25a..aa59a2c 100644 --- a/src/auth.c +++ b/src/auth.c @@ -20,12 +20,14 @@ #include "auth.h" + #include // for traceEvent -#include // for calloc, free #include // for strlen, size_t + #include "curve25519.h" // for curve25519 #include "pearson.h" // for pearson_hash_128, pearson_hash_256 #include "speck.h" // for speck_context_t, speck_128_encrypt, speck_init +#include "n2n.h" // mapping six binary bits to printable ascii character diff --git a/src/cc20.c b/src/cc20.c index 0bde7b7..a6317eb 100644 --- a/src/cc20.c +++ b/src/cc20.c @@ -22,8 +22,10 @@ #include // for traceEvent #include // for calloc, free, size_t #include // for memcpy +#include +#include + #include "cc20.h" -#include "config.h" // HAVE_LIBCRYPTO #include "portable_endian.h" // for htole32 @@ -91,7 +93,6 @@ int cc20_crypt (unsigned char *out, const unsigned char *in, size_t in_len, // https://github.com/Ginurx/chacha20-c (public domain) -#include // for _mm_xor_si128, _mm_add_epi32, _mm_slli_epi32 #include // for _MM_SHUFFLE diff --git a/src/conffile.c b/src/conffile.c index cb5250e..1952428 100644 --- a/src/conffile.c +++ b/src/conffile.c @@ -19,10 +19,20 @@ #include // for strcmp #include // for mkdir #include // for access +#include +#include +#include +#include +#include + #include "peer_info.h" // for struct peer_info +#include "n2n_typedefs.h" +#include "n3n/ethernet.h" +#include "uthash.h" #ifdef _WIN32 #include "win32/defs.h" + #include // for _mkdir #else #include // for getgrnam diff --git a/src/conffile_defs.c b/src/conffile_defs.c index 7263a5f..fd2b564 100644 --- a/src/conffile_defs.c +++ b/src/conffile_defs.c @@ -9,6 +9,8 @@ #include // for n2n_edge_conf_t #include +#include "n2n_define.h" + static struct n3n_conf_option section_auth[] = { { diff --git a/src/edge_utils.c b/src/edge_utils.c index a9d63a5..c3c4b53 100644 --- a/src/edge_utils.c +++ b/src/edge_utils.c @@ -39,11 +39,13 @@ #include // for snprintf, sprintf #include // for free, calloc, getenv #include // for memcpy, memset, NULL, memcmp -#include // for timeval #include // for time_t, ssize_t, u_int #include // for time #include // for gethostname, sleep -#include "auth.h" // for generate_private_key +#include +#include +#include + #include "header_encryption.h" // for packet_header_encrypt, packet_he... #include "mainloop.h" // for mainloop_runonce #include "management.h" // for readFromMgmtSocket @@ -51,14 +53,16 @@ #include "n2n_wire.h" // for fill_sockaddr, decod... #include "pearson.h" // for pearson_hash_128, pearson_hash_64 #include "peer_info.h" // for peer_info, clear_peer_list, ... -#include "portable_endian.h" // for be16toh, htobe16 #include "resolve.h" // for resolve_create_thread, resolve_c... #include "sn_selection.h" // for sn_selection_criterion_common_da... #include "speck.h" // for speck_128_decrypt, speck_128_enc... #include "uthash.h" // for UT_hash_handle, HASH_COUNT, HASH... +#include "n2n_define.h" +#include "n2n_typedefs.h" #ifdef _WIN32 #include // for _mkdir + #include "win32/edge_utils_win32.h" #else #include // for inet_ntoa, inet_addr, inet_ntop diff --git a/src/header_encryption.c b/src/header_encryption.c index 2d7ca17..00e2025 100644 --- a/src/header_encryption.c +++ b/src/header_encryption.c @@ -22,10 +22,8 @@ #include // for traceEvent #include // for n3n_rand #include // for uint32_t, uint8_t, uint64_t, uint16_t -#include // for calloc #include // for memcpy #include "header_encryption.h" // for packet_header_change_dynamic_key, pac... -#include "n2n.h" // for N2N_COMMUNITY_SIZE... #include "n2n_define.h" // for N2N_COMMUNITY_SIZE #include "n2n_typedefs.h" // for N2N_AUTH_CHALLENGE_SIZE #include "pearson.h" // for pearson_hash_128, pearson_hash_64 @@ -33,6 +31,8 @@ #include "speck.h" // for speck_init, speck_context_t, speck_ctr #include "uthash.h" // for HASH_FIND_STR +struct speck_context_t; + #define HASH_FIND_COMMUNITY(head, name, out) HASH_FIND_STR(head, name, out) diff --git a/src/mainloop.c b/src/mainloop.c index 4bc2f91..16c2224 100644 --- a/src/mainloop.c +++ b/src/mainloop.c @@ -5,8 +5,12 @@ */ #include // for slots_fdset -#include // for select, FD_ZERO, #include // for n3n_runtime_data +#include +#include +#include // for select, FD_ZERO, + +#include "n2n_define.h" #ifndef max #define max(a, b) (((a) < (b)) ? (b) : (a)) diff --git a/src/management.c b/src/management.c index 69157e6..a5cc379 100644 --- a/src/management.c +++ b/src/management.c @@ -14,14 +14,23 @@ #include // for n3n_metrics_render #include // for ip_subnet_to_str, sock_to_cstr #include // for load_allowed_sn_community +#include #include // for sn_selection_criterion_str #include -#include // for snprintf, NULL, size_t +#include +#include #include // for strtoul #include // for strtok, strlen, strncpy +#include +#include + #include "base64.h" // for base64decode +#include "connslot/strbuf.h" #include "management.h" +#include "n2n.h" +#include "n2n_typedefs.h" #include "peer_info.h" // for peer_info +#include "uthash.h" #ifdef _WIN32 #include "win32/defs.h" diff --git a/src/management.h b/src/management.h index 68f3e43..bc0c814 100644 --- a/src/management.h +++ b/src/management.h @@ -18,8 +18,11 @@ #include // for size_t #include // for uint64_t #include // for ssize_t + #include "n2n_define.h" // for n2n_event_topic +struct n3n_runtime_data; + #ifdef _WIN32 #include #else diff --git a/src/metrics.c b/src/metrics.c index edf747e..e564f55 100644 --- a/src/metrics.c +++ b/src/metrics.c @@ -7,6 +7,7 @@ #include #include +#include #include static struct n3n_metrics_module *registered_metrics; diff --git a/src/n2n.c b/src/n2n.c index b57a2ef..3759199 100644 --- a/src/n2n.c +++ b/src/n2n.c @@ -24,16 +24,20 @@ #include // for traceEvent #include // for n3n_rand #include // for ip_subnet_to_str, sock_to_cstr -#include +#include +#include +#include #include // for free, atoi, calloc, strtol #include // for memcmp, memcpy, memset, strlen, strerror #include // for gettimeofday, timeval -#include // for time, localtime, strftime + #include "n2n.h" -#include "uthash.h" // for UT_hash_handle, HASH_DEL, HASH_ITER, HAS... +#include "n2n_define.h" +#include "n2n_typedefs.h" #ifdef _WIN32 #include "win32/defs.h" + #include #else #include // for inet_ntop diff --git a/src/n2n_port_mapping.c b/src/n2n_port_mapping.c index 814b651..a63fba5 100644 --- a/src/n2n_port_mapping.c +++ b/src/n2n_port_mapping.c @@ -54,17 +54,11 @@ */ -#include // for N2N_NETMASK_STR_SIZE -#include // for traceEvent #include // for uint16_t -#include // for memcpy #ifdef _WIN32 #include "win32/defs.h" #else -#include // for inet_ntoa -#include // for in_addr, htonl, in_addr_t -#include // for sendto, recvfrom, sockaddr_storage #endif #include "n2n_port_mapping.h" // for n2n_del_port_mapping, n2n_set_port_map... diff --git a/src/network_traffic_filter.c b/src/network_traffic_filter.c index 99a2e4b..f727648 100644 --- a/src/network_traffic_filter.c +++ b/src/network_traffic_filter.c @@ -25,8 +25,9 @@ #include // for sprintf #include // for free, malloc, atoi #include // for memcpy, strcpy, NULL, memset -#include "n2n.h" // for filter_rule_t, filter_rule_pair_... + #include "uthash.h" // for UT_hash_handle, HASH_ITER, HASH_DEL +#include "n2n_typedefs.h" #ifdef _WIN32 #include "win32/defs.h" diff --git a/src/peer_info.c b/src/peer_info.c index d5ca9e7..61ab458 100644 --- a/src/peer_info.c +++ b/src/peer_info.c @@ -11,6 +11,11 @@ #include // for traceEvent #include // for sn_selection_criterion_default #include +#include +#include +#include +#include + #include "management.h" // for mgmt_event_post #include "peer_info.h" #include "resolve.h" // for supernode2sock diff --git a/src/peer_info.h b/src/peer_info.h index 7d5290b..a1f5408 100644 --- a/src/peer_info.h +++ b/src/peer_info.h @@ -9,6 +9,12 @@ #define _PEER_INFO_H_ #include // for n2n_mac_t, n2n_ip_subnet_t, n2n_desc_t, n2n_sock_t +#include +#include + +#include "n2n_typedefs.h" +#include "n3n/ethernet.h" +#include "uthash.h" #define HASH_ADD_PEER(head,add) \ HASH_ADD(hh,head,mac_addr,sizeof(n2n_mac_t),add) diff --git a/src/random_numbers.c b/src/random_numbers.c index f8223d7..820c2ab 100644 --- a/src/random_numbers.c +++ b/src/random_numbers.c @@ -25,6 +25,7 @@ #include // for NULL, size_t #include // for clock, time #include // for syscall +#include // syscall and inquiring random number from hardware generators might fail, so // we will retry diff --git a/src/resolve.c b/src/resolve.c index 1603f0d..ac03b90 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -6,14 +6,21 @@ * The resolver thread code and functions */ -#include // for sock_equal #include #include #include // for n3n_resolve_parameter_t -#include // for sock_to_cstr -#include // for sleep -#include "resolve.h" +#include +#include +#include +#include +#include + #include "config.h" // for HAVE_LIBPTHREAD +#include "resolve.h" +#include "n2n_define.h" +#include "n2n_typedefs.h" + +struct peer_info; #ifdef HAVE_LIBPTHREAD #include diff --git a/src/resolve.h b/src/resolve.h index 974c989..6c4f147 100644 --- a/src/resolve.h +++ b/src/resolve.h @@ -9,12 +9,17 @@ #ifndef _RESOLVE_H_ #define _RESOLVE_H_ -#include // for n2n_resolve_parameter_t #include // for n2n_sock_t +#include // for n2n_resolve_parameter_t +#include +#include #include // for UT_hash_handle + #include "config.h" // for HAVE_LIBPTHREAD #include "peer_info.h" // for struct peer_info +struct peer_info; + #ifdef HAVE_LIBPTHREAD struct n3n_resolve_ip_sock { char *org_ip; /* pointer to original ip/named address string (used read only) */ diff --git a/src/sn_selection.c b/src/sn_selection.c index e4c27a8..be7d749 100644 --- a/src/sn_selection.c +++ b/src/sn_selection.c @@ -24,6 +24,9 @@ #include // for snprintf, NULL #include // for memcpy, memset #include "n2n.h" // for n3n_runtime_data, SN_SELECTION_CRIT... +#include "n2n_define.h" +#include "n2n_typedefs.h" +#include "n3n/ethernet.h" #include "peer_info.h" // for peer_info_t #include "portable_endian.h" // for be32toh, be64toh, htobe64 #include "sn_selection.h" // for selection_criterion_str_t, sn_selection_cr... diff --git a/src/sn_utils.c b/src/sn_utils.c index 99915df..993f8d9 100644 --- a/src/sn_utils.c +++ b/src/sn_utils.c @@ -26,20 +26,23 @@ #include // for n3n_rand, n3n_rand_sqr #include // for ip_subnet_to_str, sock_to_cstr #include // for load_allowed_sn_community, calculate_... +#include #include #include // for uint8_t, uint32_t, uint16_t, uint64_t #include // for sscanf, snprintf, fclose, fgets, fopen #include // for free, calloc, getenv #include // for memcpy, NULL, memset, size_t, strerror #include // for MAX -#include // for timeval -#include // for ssize_t #include // for time_t, time +#include + #include "auth.h" // for ascii_to_bin, calculate_dynamic_key #include "header_encryption.h" // for packet_header_encrypt, packet_header_... #include "management.h" // for process_mgmt #include "n2n.h" // for sn_community, n3n_runtime_data +#include "n2n_define.h" #include "n2n_regex.h" // for re_matchp, re_compile +#include "n2n_typedefs.h" #include "n2n_wire.h" // for encode_buf, encode_PEER_INFO, encode_... #include "pearson.h" // for pearson_hash_128, pearson_hash_32 #include "peer_info.h" // for purge_peer_list, clear_peer_list @@ -50,8 +53,9 @@ #include "uthash.h" // for UT_hash_handle, HASH_ITER, HASH_DEL #ifdef _WIN32 -#include // for _rmdir #include "win32/defs.h" + +#include // for _rmdir #else #include // for inet_addr, inet_ntoa #include // for ntohl, in_addr_t, sockaddr_in, INADDR... diff --git a/src/test_hashing.c b/src/test_hashing.c index a996b3e..6d37556 100644 --- a/src/test_hashing.c +++ b/src/test_hashing.c @@ -5,10 +5,9 @@ */ #include // for PRIx64, PRIx16, PRIx32 -#include // for uint8_t, uint16_t, uint32_t, uint64_t #include // for printf, fprintf, stderr, stdout #include // for memcmp -#include "n2n.h" + #include "hexdump.h" // for fhexdump #include "pearson.h" // for pearson_hash_128, pearson_hash_16, pearson_has... diff --git a/src/transform_aes.c b/src/transform_aes.c index f1ac92c..81622d9 100644 --- a/src/transform_aes.c +++ b/src/transform_aes.c @@ -26,8 +26,11 @@ #include // for calloc, free #include // for memcpy, size_t, memset, memcmp, strlen #include // for u_char, ssize_t, time_t + #include "aes.h" // for AES_BLOCK_SIZE, aes_cbc_decrypt, aes_cbc... #include "n2n.h" // for n2n_trans_op_t +#include "n2n_define.h" +#include "n2n_typedefs.h" #include "pearson.h" // for pearson_hash_256 diff --git a/src/transform_cc20.c b/src/transform_cc20.c index fc57dd4..bb10da0 100644 --- a/src/transform_cc20.c +++ b/src/transform_cc20.c @@ -26,8 +26,11 @@ #include // for size_t, calloc, free #include // for memset, strlen #include // for u_char, ssize_t, time_t + #include "cc20.h" // for CC20_IV_SIZE, cc20_crypt, cc20_deinit #include "n2n.h" // for n2n_trans_op_t +#include "n2n_define.h" +#include "n2n_typedefs.h" #include "pearson.h" // for pearson_hash_256 diff --git a/src/transform_lzo.c b/src/transform_lzo.c index 75089b8..ee201ef 100644 --- a/src/transform_lzo.c +++ b/src/transform_lzo.c @@ -24,9 +24,12 @@ #include // for uint8_t #include // for size_t, calloc, free, NULL #include // for memset -#include // for time_t +#include + #include "minilzo.h" // for lzo1x_1_compress, lzo1x_decompress, LZO1X_1_M... #include "n2n.h" // for n2n_trans_op_t, N2N_... +#include "n2n_define.h" +#include "n2n_typedefs.h" /* heap allocation for compression as per lzo example doc */ diff --git a/src/transform_none.c b/src/transform_none.c index 0aa3ad0..4e1ba11 100644 --- a/src/transform_none.c +++ b/src/transform_none.c @@ -5,6 +5,8 @@ */ #include // for n3n_transform_register +#include + #include "n2n_define.h" // for N2N_COMPRESSION_ID_NONE // A dummy transform struct for the no-op compression diff --git a/src/transform_null.c b/src/transform_null.c index 991f32d..19094fa 100644 --- a/src/transform_null.c +++ b/src/transform_null.c @@ -23,8 +23,9 @@ #include // for n3n_transform_register #include // for uint8_t #include // for memcpy, size_t, memset -#include // for time_t + #include "n2n.h" // for n2n_trans_op_t, N2N_... +#include "n2n_typedefs.h" static int transop_deinit_null (n2n_trans_op_t *arg ) { diff --git a/src/transform_speck.c b/src/transform_speck.c index 9e2cdf6..7b98174 100644 --- a/src/transform_speck.c +++ b/src/transform_speck.c @@ -26,7 +26,10 @@ #include // for size_t, calloc, free #include // for memset, strlen #include // for u_char, ssize_t, time_t + #include "n2n.h" // for n2n_trans_op_t +#include "n2n_define.h" +#include "n2n_typedefs.h" #include "pearson.h" // for pearson_hash_256 #include "speck.h" // for N2N_SPECK_IVEC_SIZE, speck_ctr, speck_de... diff --git a/src/transform_tf.c b/src/transform_tf.c index 83a2f9b..d78ad7c 100644 --- a/src/transform_tf.c +++ b/src/transform_tf.c @@ -26,7 +26,10 @@ #include // for calloc, free #include // for memcpy, size_t, memset, memcmp, strlen #include // for u_char, ssize_t, time_t + #include "n2n.h" // for n2n_trans_op_t +#include "n2n_define.h" +#include "n2n_typedefs.h" #include "pearson.h" // for pearson_hash_256 #include "tf.h" // for TF_BLOCK_SIZE, tf_cbc_decrypt, tf_cbc_en... diff --git a/src/tuntap_linux.c b/src/tuntap_linux.c index fa990ae..5a87215 100644 --- a/src/tuntap_linux.c +++ b/src/tuntap_linux.c @@ -38,8 +38,12 @@ #include // for ioctl, SIOCGIFADDR, SIOCGIFFLAGS #include // for MIN #include // for socket, msghdr, AF_INET, sockaddr +#include // for iovec #include // for close, getpid, read, write, ssi... + #include "n2n.h" // for tuntap_dev, ... +#include "n2n_typedefs.h" +#include "n3n/ethernet.h" static int setup_ifname (int fd, const char *ifname, diff --git a/src/wire.c b/src/wire.c index 0663d63..7e70d5b 100644 --- a/src/wire.c +++ b/src/wire.c @@ -29,16 +29,18 @@ #include // for uint8_t, uint16_t, uint32_t, uint64_t #include // for size_t, memset, memcpy -#include "portable_endian.h" // for be64toh, htobe64 -#include "n2n.h" // for n2n_sock_t, n2n_common_t, n2n_auth_t, n2n_RE... + +#include "n2n_define.h" +#include "n2n_typedefs.h" #include "n2n_wire.h" // for decode_PACKET, decode_PEER_INFO, decode_QUER... +#include "n3n/ethernet.h" +#include "portable_endian.h" // for be64toh, htobe64 #ifdef _WIN32 #include "win32/defs.h" #else #include // for sockaddr_in, sockaddr_in6, in6_addr, in_addr #include // for AF_INET, AF_INET6, SOCK_STREAM, SOCK_DGRAM -#include // for sa_family_t #endif From 04d21e8d5b2748abf3eed1ab36e113afdd2f26c4 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Sat, 26 Oct 2024 14:45:39 +1100 Subject: [PATCH 04/85] Address linter concerns --- src/edge_utils.c | 6 +++--- src/mainloop.c | 6 +++--- src/mainloop.h | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/edge_utils.c b/src/edge_utils.c index c3c4b53..4c8827b 100644 --- a/src/edge_utils.c +++ b/src/edge_utils.c @@ -3076,9 +3076,9 @@ int run_edge_loop (struct n3n_runtime_data *eee) { #endif int slots_ready = slots_fdset_loop( - eee->mgmt_slots, - &readers, - &writers + eee->mgmt_slots, + &readers, + &writers ); if(slots_ready < 0) { diff --git a/src/mainloop.c b/src/mainloop.c index 16c2224..b88e50f 100644 --- a/src/mainloop.c +++ b/src/mainloop.c @@ -20,7 +20,7 @@ #define min(a, b) (((a) >(b)) ? (b) : (a)) #endif -static int setup_select(fd_set *rd, fd_set *wr, struct n3n_runtime_data *eee) { +static int setup_select (fd_set *rd, fd_set *wr, struct n3n_runtime_data *eee) { FD_ZERO(rd); FD_ZERO(wr); int max_sock = 0; @@ -54,7 +54,7 @@ static int setup_select(fd_set *rd, fd_set *wr, struct n3n_runtime_data *eee) { return max_sock; } -int mainloop_runonce(fd_set *rd, fd_set *wr, struct n3n_runtime_data *eee) { +int mainloop_runonce (fd_set *rd, fd_set *wr, struct n3n_runtime_data *eee) { int maxfd = setup_select(rd, wr, eee); @@ -71,7 +71,7 @@ int mainloop_runonce(fd_set *rd, fd_set *wr, struct n3n_runtime_data *eee) { wait_time.tv_sec = (SOCKET_TIMEOUT_INTERVAL_SECS); } wait_time.tv_usec = 0; - + int rc = select(maxfd + 1, rd, wr, NULL, &wait_time); return rc; diff --git a/src/mainloop.h b/src/mainloop.h index 928bd8c..159b762 100644 --- a/src/mainloop.h +++ b/src/mainloop.h @@ -9,6 +9,6 @@ #include // for n3n_runtime_data -int mainloop_runonce(fd_set *, fd_set *, struct n3n_runtime_data *); +int mainloop_runonce (fd_set *, fd_set *, struct n3n_runtime_data *); #endif From 38600297a777882fc8e57df6e610fb96a5bb70e0 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Sat, 26 Oct 2024 14:48:47 +1100 Subject: [PATCH 05/85] Address windows CI issues --- src/conffile.c | 2 +- src/edge_utils.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/conffile.c b/src/conffile.c index 1952428..eff289b 100644 --- a/src/conffile.c +++ b/src/conffile.c @@ -19,7 +19,6 @@ #include // for strcmp #include // for mkdir #include // for access -#include #include #include #include @@ -35,6 +34,7 @@ #include // for _mkdir #else +#include #include // for getgrnam #include // for sockaddr_in #endif diff --git a/src/edge_utils.c b/src/edge_utils.c index 4c8827b..31540d0 100644 --- a/src/edge_utils.c +++ b/src/edge_utils.c @@ -42,7 +42,6 @@ #include // for time_t, ssize_t, u_int #include // for time #include // for gethostname, sleep -#include #include #include @@ -53,6 +52,7 @@ #include "n2n_wire.h" // for fill_sockaddr, decod... #include "pearson.h" // for pearson_hash_128, pearson_hash_64 #include "peer_info.h" // for peer_info, clear_peer_list, ... +#include "portable_endian.h" // for be16toh, htobe16 #include "resolve.h" // for resolve_create_thread, resolve_c... #include "sn_selection.h" // for sn_selection_criterion_common_da... #include "speck.h" // for speck_128_decrypt, speck_128_enc... From 32b13f502fd3191f416b91ea6327c45d49b17137 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Sat, 26 Oct 2024 14:50:35 +1100 Subject: [PATCH 06/85] Address issues compiling on BSD and Windows --- src/conffile.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/conffile.c b/src/conffile.c index eff289b..02ec3a0 100644 --- a/src/conffile.c +++ b/src/conffile.c @@ -7,22 +7,21 @@ #include // for isprint and friends #include // for errno +#include #include -#include // for n3n_peer_add_by_hostname #include // for setTraceLevel -#include // for n3n_transform_lookup_ #include +#include // for n3n_peer_add_by_hostname +#include // for n3n_transform_lookup_ +#include #include // for true, false #include // for uint32_t #include // for printf #include // for malloc #include // for strcmp +#include #include // for mkdir #include // for access -#include -#include -#include -#include #include "peer_info.h" // for struct peer_info #include "n2n_typedefs.h" From 51bd519e396d61798dd5e83967a09aa295a7938c Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Sat, 26 Oct 2024 14:53:36 +1100 Subject: [PATCH 07/85] Address issues compiling on Windows --- src/conffile.c | 4 ++-- src/edge_utils.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/conffile.c b/src/conffile.c index 02ec3a0..5aaeb35 100644 --- a/src/conffile.c +++ b/src/conffile.c @@ -13,13 +13,11 @@ #include #include // for n3n_peer_add_by_hostname #include // for n3n_transform_lookup_ -#include #include // for true, false #include // for uint32_t #include // for printf #include // for malloc #include // for strcmp -#include #include // for mkdir #include // for access @@ -36,6 +34,8 @@ #include #include // for getgrnam #include // for sockaddr_in +#include +#include #endif #include // for generate_private_key diff --git a/src/edge_utils.c b/src/edge_utils.c index 31540d0..ec22060 100644 --- a/src/edge_utils.c +++ b/src/edge_utils.c @@ -42,7 +42,6 @@ #include // for time_t, ssize_t, u_int #include // for time #include // for gethostname, sleep -#include #include #include "header_encryption.h" // for packet_header_encrypt, packet_he... @@ -68,6 +67,7 @@ #include // for inet_ntoa, inet_addr, inet_ntop #include // for sockaddr_in, ntohl, IPPROTO_IP #include // for TCP_NODELAY +#include #include // for select, FD_SET, FD_ISSET, FD_ZERO #include // for setsockopt, AF_INET, connect #endif From 0527053f4da4b11f1afefc1c59de11e933d2c9ac Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Sat, 26 Oct 2024 15:00:08 +1100 Subject: [PATCH 08/85] Continue compiling after errors - shows all errors at once --- .github/workflows/quick_windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/quick_windows.yml b/.github/workflows/quick_windows.yml index 17dd0b1..0042656 100644 --- a/.github/workflows/quick_windows.yml +++ b/.github/workflows/quick_windows.yml @@ -24,7 +24,7 @@ jobs: export CFLAGS="-fprofile-arcs -ftest-coverage" export LDFLAGS="--coverage" ./scripts/hack_fakeautoconf.sh - make -j4 + make -k -j4 make test.builtin test.units shell: bash From da33cab2534b61209e8297a398a7dfa73e2d2073 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Sat, 26 Oct 2024 15:05:59 +1100 Subject: [PATCH 09/85] Address issues compiling on Windows --- src/mainloop.c | 3 +++ src/management.c | 2 +- src/n2n.c | 2 +- src/peer_info.c | 3 +++ src/resolve.c | 2 +- src/sn_utils.c | 2 +- 6 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/mainloop.c b/src/mainloop.c index b88e50f..32c5064 100644 --- a/src/mainloop.c +++ b/src/mainloop.c @@ -8,7 +8,10 @@ #include // for n3n_runtime_data #include #include + +#ifndef _WIN32 #include // for select, FD_ZERO, +#endif #include "n2n_define.h" diff --git a/src/management.c b/src/management.c index a5cc379..967bf43 100644 --- a/src/management.c +++ b/src/management.c @@ -14,7 +14,6 @@ #include // for n3n_metrics_render #include // for ip_subnet_to_str, sock_to_cstr #include // for load_allowed_sn_community -#include #include // for sn_selection_criterion_str #include #include @@ -36,6 +35,7 @@ #include "win32/defs.h" #else #include // for getnameinfo, NI_NUMERICHOST, NI_NUMERICSERV +#include #include // for sendto, sockaddr #endif diff --git a/src/n2n.c b/src/n2n.c index 3759199..ac6e09e 100644 --- a/src/n2n.c +++ b/src/n2n.c @@ -24,7 +24,6 @@ #include // for traceEvent #include // for n3n_rand #include // for ip_subnet_to_str, sock_to_cstr -#include #include #include #include // for free, atoi, calloc, strtol @@ -41,6 +40,7 @@ #include #else #include // for inet_ntop +#include #include // for AF_INET, PF_INET, bind, setsockopt, shut... #endif diff --git a/src/peer_info.c b/src/peer_info.c index 61ab458..98c2f4c 100644 --- a/src/peer_info.c +++ b/src/peer_info.c @@ -14,7 +14,10 @@ #include #include #include + +#ifndef _WIN32 #include +#endif #include "management.h" // for mgmt_event_post #include "peer_info.h" diff --git a/src/resolve.c b/src/resolve.c index ac03b90..d857ee7 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -9,7 +9,6 @@ #include #include #include // for n3n_resolve_parameter_t -#include #include #include #include @@ -31,6 +30,7 @@ struct peer_info; #include #else #include // for addrinfo, freeaddrinfo, gai_strerror +#include #include // for AF_INET, PF_INET #include // for gettimeofday, timersub #endif diff --git a/src/sn_utils.c b/src/sn_utils.c index 993f8d9..db40b0d 100644 --- a/src/sn_utils.c +++ b/src/sn_utils.c @@ -26,7 +26,6 @@ #include // for n3n_rand, n3n_rand_sqr #include // for ip_subnet_to_str, sock_to_cstr #include // for load_allowed_sn_community, calculate_... -#include #include #include // for uint8_t, uint32_t, uint16_t, uint64_t #include // for sscanf, snprintf, fclose, fgets, fopen @@ -60,6 +59,7 @@ #include // for inet_addr, inet_ntoa #include // for ntohl, in_addr_t, sockaddr_in, INADDR... #include // for TCP_NODELAY +#include #include // for FD_ISSET, FD_SET, select, FD_SETSIZE #include // for recvfrom, shutdown, sockaddr_storage #endif From a4d6abd0ad1a2bb03b5937b0487c8071e8d6a447 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Sat, 26 Oct 2024 15:38:22 +1100 Subject: [PATCH 10/85] Refactor some simple cases into the new mainloop --- include/n2n.h | 1 - src/edge_utils.c | 42 +++-------------------------- src/mainloop.c | 52 ++++++++++++++++++++++++++++++++++-- src/win32/edge_utils_win32.c | 1 + 4 files changed, 54 insertions(+), 42 deletions(-) diff --git a/include/n2n.h b/include/n2n.h index a8cb148..aa21253 100644 --- a/include/n2n.h +++ b/include/n2n.h @@ -141,7 +141,6 @@ void update_supernode_reg (struct n3n_runtime_data * eee, time_t nowTime); void readFromIPSocket (struct n3n_runtime_data * eee, int in_sock); void edge_term (struct n3n_runtime_data *eee); void edge_send_packet2net (struct n3n_runtime_data *eee, uint8_t *tap_pkt, size_t len); -void edge_read_from_tap (struct n3n_runtime_data *eee); int run_edge_loop (struct n3n_runtime_data *eee); int quick_edge_init (char *device_name, char *community_name, char *encrypt_key, char *device_mac, diff --git a/src/edge_utils.c b/src/edge_utils.c index ec22060..18c1b99 100644 --- a/src/edge_utils.c +++ b/src/edge_utils.c @@ -44,9 +44,10 @@ #include // for gethostname, sleep #include +#include "edge_utils.h" #include "header_encryption.h" // for packet_header_encrypt, packet_he... #include "mainloop.h" // for mainloop_runonce -#include "management.h" // for readFromMgmtSocket +#include "management.h" // for mgmt_event_post #include "n2n.h" // for n3n_runtime_data, n2n_edge_... #include "n2n_wire.h" // for fill_sockaddr, decod... #include "pearson.h" // for pearson_hash_128, pearson_hash_64 @@ -3020,10 +3021,9 @@ int run_edge_loop (struct n3n_runtime_data *eee) { int rc; fd_set readers; fd_set writers; - time_t now; rc = mainloop_runonce(&readers, &writers, eee); - now = time(NULL); + time_t now = time(NULL); if(rc > 0) { // any or all of the FDs could have input; check them all @@ -3068,42 +3068,6 @@ int run_edge_loop (struct n3n_runtime_data *eee) { } #endif -#ifndef _WIN32 - if((eee->device.fd != -1) && FD_ISSET(eee->device.fd, &readers)) { - // read an ethernet frame from the TAP socket; write on the IP socket - edge_read_from_tap(eee); - } -#endif - - int slots_ready = slots_fdset_loop( - eee->mgmt_slots, - &readers, - &writers - ); - - if(slots_ready < 0) { - traceEvent( - TRACE_ERROR, - "slots_fdset_loop returns %i (Is daemon exiting?)", slots_ready - ); - } else if(slots_ready > 0) { - // A linear scan is not ideal, but this is a select() loop - // not one built for performance. - // - update connslot to have callbacks instead of scan - // - switch to a modern poll loop (and reimplement differently - // for each OS supported) - // This should only be a concern if we are doing a large - // number of slot connections - for(int i=0; imgmt_slots->nr_slots; i++) { - if(eee->mgmt_slots->conn[i].fd == -1) { - continue; - } - - if(eee->mgmt_slots->conn[i].state == CONN_READY) { - mgmt_api_handler(eee, &eee->mgmt_slots->conn[i]); - } - } - } } // check for timed out slots diff --git a/src/mainloop.c b/src/mainloop.c index 32c5064..33b0661 100644 --- a/src/mainloop.c +++ b/src/mainloop.c @@ -6,6 +6,7 @@ #include // for slots_fdset #include // for n3n_runtime_data +#include // for traceEvent #include #include @@ -13,6 +14,8 @@ #include // for select, FD_ZERO, #endif +#include "edge_utils.h" // for edge_read_from_tap +#include "management.h" // for readFromMgmtSocket #include "n2n_define.h" #ifndef max @@ -75,7 +78,52 @@ int mainloop_runonce (fd_set *rd, fd_set *wr, struct n3n_runtime_data *eee) { } wait_time.tv_usec = 0; - int rc = select(maxfd + 1, rd, wr, NULL, &wait_time); + int ready = select(maxfd + 1, rd, wr, NULL, &wait_time); - return rc; + if(ready < 1) { + // Nothing ready or an error + return ready; + } + + // One timestamp to use for this entire loop iteration + // time_t now = time(NULL); + +#ifndef _WIN32 + if((eee->device.fd != -1) && FD_ISSET(eee->device.fd, rd)) { + // read an ethernet frame from the TAP socket; write on the IP socket + edge_read_from_tap(eee); + } +#endif + + int slots_ready = slots_fdset_loop( + eee->mgmt_slots, + rd, + wr + ); + + if(slots_ready < 0) { + traceEvent( + TRACE_ERROR, + "slots_fdset_loop returns %i (Is daemon exiting?)", slots_ready + ); + } else if(slots_ready > 0) { + // A linear scan is not ideal, but this is a select() loop + // not one built for performance. + // - update connslot to have callbacks instead of scan + // - switch to a modern poll loop (and reimplement differently + // for each OS supported) + // This should only be a concern if we are doing a large + // number of slot connections + for(int i=0; imgmt_slots->nr_slots; i++) { + if(eee->mgmt_slots->conn[i].fd == -1) { + continue; + } + + if(eee->mgmt_slots->conn[i].state == CONN_READY) { + mgmt_api_handler(eee, &eee->mgmt_slots->conn[i]); + } + } + } + + return ready; } diff --git a/src/win32/edge_utils_win32.c b/src/win32/edge_utils_win32.c index 9e7227e..f7090b5 100644 --- a/src/win32/edge_utils_win32.c +++ b/src/win32/edge_utils_win32.c @@ -22,6 +22,7 @@ #include #include // for traceEvent +#include "../edge_utils.h" // for edge_read_from_tap #include "edge_utils_win32.h" /* ************************************** */ From 35b238f1202cfe9fdd607abc282f9eea5910ef79 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Sat, 26 Oct 2024 16:59:11 +1100 Subject: [PATCH 11/85] Make it easier to see the steps needed for the two packet paths by refactoring into two functions --- apps/n3n-edge.c | 22 ++++- include/n3n/edge.h | 15 +++- src/edge_utils.c | 203 ++++++++++++++++++++++++++------------------- src/edge_utils.h | 11 +++ 4 files changed, 157 insertions(+), 94 deletions(-) create mode 100644 src/edge_utils.h diff --git a/apps/n3n-edge.c b/apps/n3n-edge.c index 06cd7e8..2fa6edb 100644 --- a/apps/n3n-edge.c +++ b/apps/n3n-edge.c @@ -977,10 +977,24 @@ int main (int argc, char* argv[]) { if(select(eee->sock + 1, &socket_mask, NULL, NULL, &wait_time) > 0) { if(FD_ISSET(eee->sock, &socket_mask)) { - - fetch_and_eventually_process_data(eee, eee->sock, - pktbuf, &expected, &position, - now); + if(!eee->conf.connect_tcp) { + edge_read_proto3_udp( + eee, + eee->sock, + pktbuf, + sizeof(pktbuf), + now + ); + } else { + edge_read_proto3_tcp( + eee, + eee->sock, + pktbuf, + &expected, + &position, + now + ); + } } } } diff --git a/include/n3n/edge.h b/include/n3n/edge.h index bfba243..748655b 100644 --- a/include/n3n/edge.h +++ b/include/n3n/edge.h @@ -28,8 +28,17 @@ void edge_term_conf (n2n_edge_conf_t *conf); void send_register_super (struct n3n_runtime_data *eee); void send_query_peer (struct n3n_runtime_data *eee, const n2n_mac_t dst_mac); int supernode_connect (struct n3n_runtime_data *eee); -int fetch_and_eventually_process_data (struct n3n_runtime_data *eee, SOCKET sock, - uint8_t *pktbuf, uint16_t *expected, uint16_t *position, - time_t now); + +void edge_read_proto3_udp (struct n3n_runtime_data *eee, + SOCKET sock, + uint8_t *pktbuf, + ssize_t pktbuf_len, + time_t now); +void edge_read_proto3_tcp (struct n3n_runtime_data *eee, + SOCKET sock, + uint8_t *pktbuf, + uint16_t *expected, + uint16_t *position, + time_t now); #endif diff --git a/src/edge_utils.c b/src/edge_utils.c index 18c1b99..1f7494b 100644 --- a/src/edge_utils.c +++ b/src/edge_utils.c @@ -2878,92 +2878,117 @@ void process_udp (struct n3n_runtime_data *eee, const struct sockaddr *sender_so /* ************************************** */ +void edge_read_proto3_udp (struct n3n_runtime_data *eee, + SOCKET sock, + uint8_t *pktbuf, + ssize_t pktbuf_len, + time_t now) { + struct sockaddr_storage sas; + struct sockaddr *sender_sock = (struct sockaddr*)&sas; + socklen_t ss_size = sizeof(sas); + + ssize_t bread = recvfrom( + sock, + pktbuf, + pktbuf_len, + 0 /*flags*/, + sender_sock, + &ss_size + ); + + if(bread < 0) { +#ifdef _WIN32 + unsigned int wsaerr = WSAGetLastError(); + if(wsaerr == WSAECONNRESET) { + return; + } + traceEvent(TRACE_ERROR, "WSAGetLastError(): %u", wsaerr); +#endif + + /* For UDP bread of zero just means no data (unlike TCP). */ + /* The fd is no good now. Maybe we lost our interface. */ + traceEvent(TRACE_ERROR, "recvfrom() failed %d errno %d (%s)", bread, errno, strerror(errno)); + *eee->keep_running = false; + return; + } + if(bread == 0) { + return; + } -int fetch_and_eventually_process_data (struct n3n_runtime_data *eee, SOCKET sock, - uint8_t *pktbuf, uint16_t *expected, uint16_t *position, - time_t now) { + // TODO: + // - detect when pktbuf is too small for the packet and add that to stats + // (could switch to using recvmsg() for that) - ssize_t bread = 0; + // we have a datagram to process... + // ...and the datagram has data (not just a header) + // + process_udp(eee, sender_sock, sock, pktbuf, bread, now); + return; +} +void edge_read_proto3_tcp (struct n3n_runtime_data *eee, + SOCKET sock, + uint8_t *pktbuf, + uint16_t *expected, + uint16_t *position, + time_t now) { struct sockaddr_storage sas; struct sockaddr *sender_sock = (struct sockaddr*)&sas; socklen_t ss_size = sizeof(sas); - if((!eee->conf.connect_tcp) -#ifndef SKIP_MULTICAST_PEERS_DISCOVERY - || (sock == eee->udp_multicast_sock) -#endif - ) { - // udp - bread = recvfrom(sock, (void *)pktbuf, N2N_PKT_BUF_SIZE, 0 /*flags*/, - sender_sock, &ss_size); + // tcp + ssize_t bread = recvfrom( + sock, + (void *)(pktbuf + *position), + *expected - *position, + 0 /*flags*/, + sender_sock, + &ss_size + ); - if((bread < 0) -#ifdef _WIN32 - && (WSAGetLastError() != WSAECONNRESET) -#endif - ) { - /* For UDP bread of zero just means no data (unlike TCP). */ - /* The fd is no good now. Maybe we lost our interface. */ - traceEvent(TRACE_ERROR, "recvfrom() failed %d errno %d (%s)", bread, errno, strerror(errno)); + if((bread <= 0) && (errno)) { + traceEvent( + TRACE_ERROR, + "recvfrom() failed %d errno %d (%s)", + bread, + errno, + strerror(errno) + ); #ifdef _WIN32 - traceEvent(TRACE_ERROR, "WSAGetLastError(): %u", WSAGetLastError()); + traceEvent(TRACE_ERROR, "WSAGetLastError(): %u", WSAGetLastError()); #endif - return -1; - } + supernode_disconnect(eee); + eee->sn_wait = 1; + return; + } - // TODO: if bread > 64K, something is wrong - // but this case should not happen + *position = *position + bread; - // we have a datagram to process... - if(bread > 0) { - // ...and the datagram has data (not just a header) - process_udp(eee, sender_sock, sock, pktbuf, bread, now); - } + if(*position != *expected) { + // not enough bytes yet to process buffer + return; + } - } else { - // tcp - bread = recvfrom(sock, - (void *)(pktbuf + *position), *expected - *position, 0 /*flags*/, - sender_sock, &ss_size); - if((bread <= 0) && (errno)) { - traceEvent(TRACE_ERROR, "recvfrom() failed %d errno %d (%s)", bread, errno, strerror(errno)); -#ifdef _WIN32 - traceEvent(TRACE_ERROR, "WSAGetLastError(): %u", WSAGetLastError()); -#endif + if(*position == sizeof(uint16_t)) { + // the prepended length has been read, preparing for the packet + *expected = *expected + be16toh(*(uint16_t*)(pktbuf)); + if(*expected > N2N_PKT_BUF_SIZE) { supernode_disconnect(eee); eee->sn_wait = 1; - goto tcp_done; - } - *position = *position + bread; - - if(*position == *expected) { - if(*position == sizeof(uint16_t)) { - // the prepended length has been read, preparing for the packet - *expected = *expected + be16toh(*(uint16_t*)(pktbuf)); - if(*expected > N2N_PKT_BUF_SIZE) { - supernode_disconnect(eee); - eee->sn_wait = 1; - traceEvent(TRACE_DEBUG, "too many bytes expected"); - goto tcp_done; - } - } else { - // full packet read, handle it - process_udp(eee, sender_sock, sock, - pktbuf + sizeof(uint16_t), *position - sizeof(uint16_t), now); - // reset, await new prepended length - *expected = sizeof(uint16_t); - *position = 0; - } + traceEvent(TRACE_DEBUG, "too many bytes expected"); } + return; } -tcp_done: - ; - return 0; + // full packet read, handle it + process_udp(eee, sender_sock, sock, + pktbuf + sizeof(uint16_t), *position - sizeof(uint16_t), now); + // reset, await new prepended length + *expected = sizeof(uint16_t); + *position = 0; + return; } - void print_edge_stats (const struct n3n_runtime_data *eee) { const struct n2n_edge_stats *s = &eee->stats; @@ -3030,17 +3055,24 @@ int run_edge_loop (struct n3n_runtime_data *eee) { // external packets if((eee->sock != -1) && FD_ISSET(eee->sock, &readers)) { - if(0 != fetch_and_eventually_process_data( - eee, - eee->sock, - pktbuf, - &expected, - &position, - now - )) { - *eee->keep_running = false; - } - if(eee->conf.connect_tcp) { + if(!eee->conf.connect_tcp) { + edge_read_proto3_udp( + eee, + eee->sock, + pktbuf, + sizeof(pktbuf), + now + ); + } else { + edge_read_proto3_tcp( + eee, + eee->sock, + pktbuf, + &expected, + &position, + now + ); + if((expected >= N2N_PKT_BUF_SIZE) || (position >= N2N_PKT_BUF_SIZE)) { // something went wrong, possibly even before // e.g. connection failure/closure in the middle of transmission (between len & data) @@ -3055,16 +3087,13 @@ int run_edge_loop (struct n3n_runtime_data *eee) { #ifndef SKIP_MULTICAST_PEERS_DISCOVERY if((eee->udp_multicast_sock != -1) && FD_ISSET(eee->udp_multicast_sock, &readers)) { - if(0 != fetch_and_eventually_process_data( - eee, - eee->udp_multicast_sock, - pktbuf, - &expected, - &position, - now - )) { - *eee->keep_running = false; - } + edge_read_proto3_udp( + eee, + eee->udp_multicast_sock, + pktbuf, + sizeof(pktbuf), + now + ); } #endif diff --git a/src/edge_utils.h b/src/edge_utils.h new file mode 100644 index 0000000..a1c4f60 --- /dev/null +++ b/src/edge_utils.h @@ -0,0 +1,11 @@ +/** + * Copyright (C) Hamish Coleman + * SPDX-License-Identifier: GPL-3.0-only + */ + +#ifndef _EDGE_UTILS_H_ +#define _EDGE_UTILS_H_ + +void edge_read_from_tap (struct n3n_runtime_data *eee); + +#endif From 8e9f5b0d3eb3192a30d69241945db20dffc75b32 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Sat, 26 Oct 2024 17:47:35 +1100 Subject: [PATCH 12/85] Fix some builds - the iwyu's change is at fault --- include/speck.h | 1 - src/cc20.c | 1 - 2 files changed, 2 deletions(-) diff --git a/include/speck.h b/include/speck.h index 18399b4..9f9cb18 100644 --- a/include/speck.h +++ b/include/speck.h @@ -27,7 +27,6 @@ #include // for uint64_t, uint32_t -#include #define u32 uint32_t diff --git a/src/cc20.c b/src/cc20.c index a6317eb..180e492 100644 --- a/src/cc20.c +++ b/src/cc20.c @@ -22,7 +22,6 @@ #include // for traceEvent #include // for calloc, free, size_t #include // for memcpy -#include #include #include "cc20.h" From 09e81396080f012a20cabaafed7ef4cd4da55a3a Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Sat, 26 Oct 2024 17:51:55 +1100 Subject: [PATCH 13/85] Revert remainder of iwyu changes to cc20 - need to fix macos build --- src/cc20.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/cc20.c b/src/cc20.c index 180e492..8774610 100644 --- a/src/cc20.c +++ b/src/cc20.c @@ -22,9 +22,9 @@ #include // for traceEvent #include // for calloc, free, size_t #include // for memcpy -#include #include "cc20.h" +#include "config.h" // HAVE_LIBCRYPTO #include "portable_endian.h" // for htole32 @@ -92,6 +92,7 @@ int cc20_crypt (unsigned char *out, const unsigned char *in, size_t in_len, // https://github.com/Ginurx/chacha20-c (public domain) +#include // for _mm_xor_si128, _mm_add_epi32, _mm_slli_epi32 #include // for _MM_SHUFFLE From 692d2e2fcb3c4c030bf7ae53912fd62ef152f2d7 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Sat, 26 Oct 2024 17:57:55 +1100 Subject: [PATCH 14/85] Prep for using conn handlers to deal with TCP mode --- libs/connslot/connslot.c | 95 +++++++++++++++++++++++++--------------- libs/connslot/connslot.h | 21 ++++++--- 2 files changed, 73 insertions(+), 43 deletions(-) diff --git a/libs/connslot/connslot.c b/libs/connslot/connslot.c index 00149ed..67f4227 100644 --- a/libs/connslot/connslot.c +++ b/libs/connslot/connslot.c @@ -6,11 +6,15 @@ */ #define _GNU_SOURCE +#ifndef _WIN32 +#include // for ntohs +#endif #include // for errno, EAGAIN, ENOENT, EWOULDBLOCK #include // for fcntl, F_SETFL, O_NONBLOCK #ifndef _WIN32 #include // for htons, htonl, sockaddr_in, sock... #endif +#include // for uint16_t #include // for remove #include // for free, abort, malloc, strtoul #include // for memmem, memcpy, strlen, strncpy @@ -60,6 +64,7 @@ void *memmem(void *haystack, size_t haystack_len, void * needle, size_t needle_l void conn_zero(conn_t *conn) { conn->fd = -1; conn->state = CONN_EMPTY; + conn->proto = CONN_PROTO_UNK; conn->reply = NULL; conn->reply_sendpos = 0; conn->activity = 0; @@ -116,47 +121,62 @@ void conn_read(conn_t *conn) { // This will truncate the time to a int - usually 32bits conn->activity = time(NULL); + unsigned int expected_length; - // case protocol==HTTP - - if (sb_len(conn->request)<4) { - // Not enough bytes to match the end of header check - return; - } - - // retrieve the cached expected length, if any - unsigned int expected_length = conn->request->rd_pos; - - if (expected_length == 0) { - char *p = memmem(conn->request->str, sb_len(conn->request), "\r\n\r\n", 4); - if (!p) { - // As yet, we dont have an entire header - return; - } + switch (conn->proto) { + case CONN_PROTO_HTTP: + if (sb_len(conn->request)<4) { + // Not enough bytes to match the end of header check + return; + } - int body_pos = p - conn->request->str + 4; + // retrieve the cached expected length, if any + expected_length = conn->request->rd_pos; + + if (expected_length == 0) { + char *p = memmem(conn->request->str, sb_len(conn->request), "\r\n\r\n", 4); + if (!p) { + // As yet, we dont have an entire header + return; + } + + int body_pos = p - conn->request->str + 4; + + // Determine if we need to read a body + p = memmem( + conn->request->str, + body_pos, + "Content-Length:", + 15 + ); + + if (!p) { + // We have an end of header, and the header has no content length field + // so assume there is no body to read + conn->state = CONN_READY; + return; + } + + p+=15; // Skip the field name + unsigned int content_length = strtoul(p, NULL, 10); + expected_length = body_pos + content_length; + // FIXME: what if Content-Length: is larger than unsigned int? + } + break; + case CONN_PROTO_BE16LEN: + if (sb_len(conn->request)<2) { + // Not enough bytes to have the header + return; + } - // Determine if we need to read a body - p = memmem( - conn->request->str, - body_pos, - "Content-Length:", - 15 - ); + expected_length = ntohs(*(uint16_t *)&conn->request->str); + break; - if (!p) { - // We have an end of header, and the header has no content length field - // so assume there is no body to read - conn->state = CONN_READY; + default: return; - } - - p+=15; // Skip the field name - unsigned int content_length = strtoul(p, NULL, 10); - expected_length = body_pos + content_length; - // FIXME: what if Content-Length: is larger than unsigned int? } + // By this point we must have an expected_length // cache the calculated total length in the conn @@ -542,7 +562,7 @@ int slots_fdset(slots_t *slots, fd_set *readers, fd_set *writers) { return fdmax; } -int slots_accept(slots_t *slots, int listen_nr) { +int slots_accept(slots_t *slots, int listen_nr, enum conn_proto proto) { int i; // TODO: remember previous checked slot and dont start at zero @@ -573,6 +593,7 @@ int slots_accept(slots_t *slots, int listen_nr) { // This will truncate the time to a int - usually 32bits slots->conn[i].activity = time(NULL); slots->conn[i].fd = client; + slots->conn[i].proto = proto; return i; } @@ -608,7 +629,9 @@ int slots_fdset_loop(slots_t *slots, fd_set *readers, fd_set *writers) { } if (FD_ISSET(slots->listen[i], readers)) { // A new connection - int slotnr = slots_accept(slots, i); + // TODO: + // - allow each listen socket to have a protocol + int slotnr = slots_accept(slots, i, CONN_PROTO_HTTP); switch (slotnr) { case -1: diff --git a/libs/connslot/connslot.h b/libs/connslot/connslot.h index f1a341e..8773d4f 100644 --- a/libs/connslot/connslot.h +++ b/libs/connslot/connslot.h @@ -26,12 +26,18 @@ void *memmem(void *haystack, size_t haystack_len, void * needle, size_t needle_l #endif enum __attribute__((__packed__)) conn_state { - CONN_EMPTY, - CONN_READING, - CONN_READY, - CONN_SENDING, - CONN_CLOSED, - CONN_ERROR, + CONN_EMPTY = 0, + CONN_READING = 1, + CONN_READY = 2, + CONN_SENDING = 3, + CONN_CLOSED = 4, + CONN_ERROR = 5, +}; + +enum __attribute__((__packed__)) conn_proto { + CONN_PROTO_UNK = 0, + CONN_PROTO_HTTP = 1, + CONN_PROTO_BE16LEN = 2, }; typedef struct conn { @@ -42,6 +48,7 @@ typedef struct conn { int fd; unsigned int reply_sendpos; enum conn_state state; + enum conn_proto proto; } conn_t; #define SLOTS_LISTEN 2 @@ -66,7 +73,7 @@ int slots_listen_tcp(slots_t *, int, bool); int slots_listen_unix(slots_t *, char *, int, int, int); void slots_listen_close(slots_t *); int slots_fdset(slots_t *, fd_set *, fd_set *); -int slots_accept(slots_t *, int); +int slots_accept(slots_t *, int, enum conn_proto); int slots_closeidle(slots_t *); int slots_fdset_loop(slots_t *, fd_set *, fd_set *); void slots_dump(strbuf_t **, slots_t *); From 8ea0c6e012d39c1ad93080ef8c88a345255e3527 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Sat, 26 Oct 2024 18:40:15 +1100 Subject: [PATCH 15/85] Link the systemd unit to the documentation --- packages/lib/systemd/system/n3n-edge.service | 1 + packages/lib/systemd/system/n3n-edge@.service | 1 + packages/lib/systemd/system/n3n-supernode.service | 1 + 3 files changed, 3 insertions(+) diff --git a/packages/lib/systemd/system/n3n-edge.service b/packages/lib/systemd/system/n3n-edge.service index f0cea9f..3bd1511 100644 --- a/packages/lib/systemd/system/n3n-edge.service +++ b/packages/lib/systemd/system/n3n-edge.service @@ -1,5 +1,6 @@ [Unit] Description=n3n edge process +Documentation=man:n3n-edge(8) After=network-online.target nfw.target Wants=network-online.target diff --git a/packages/lib/systemd/system/n3n-edge@.service b/packages/lib/systemd/system/n3n-edge@.service index c660125..ba6b053 100644 --- a/packages/lib/systemd/system/n3n-edge@.service +++ b/packages/lib/systemd/system/n3n-edge@.service @@ -1,5 +1,6 @@ [Unit] Description=n3n edge process, on %I +Documentation=man:n3n-edge(8) After=network-online.target nfw.target Wants=network-online.target diff --git a/packages/lib/systemd/system/n3n-supernode.service b/packages/lib/systemd/system/n3n-supernode.service index fe304ba..5fb7dcc 100644 --- a/packages/lib/systemd/system/n3n-supernode.service +++ b/packages/lib/systemd/system/n3n-supernode.service @@ -1,5 +1,6 @@ [Unit] Description=n3n supernode process +Documentation=man:n3n-supernode(8) After=network-online.target Wants=network-online.target From db361f8a4e8a61c984f50485eb73825bba26a4b1 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Sat, 26 Oct 2024 20:12:00 +1100 Subject: [PATCH 16/85] Add TCP support to the test packet generator --- scripts/test_packets | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/scripts/test_packets b/scripts/test_packets index f04ecf5..6cd60d8 100755 --- a/scripts/test_packets +++ b/scripts/test_packets @@ -694,9 +694,16 @@ def main(): ) ap.add_argument( "--raw", + action="store_true", help="Dont stablise the results - show the real data", default=False ) + ap.add_argument( + "--tcp", + action="store_true", + help="Switch to TCP connection", + default=False, + ) ap.add_argument("scenario", help="What packet to send") args = ap.parse_args() @@ -707,8 +714,14 @@ def main(): community = args.community.encode("utf8") PacketBase.set_default("community", community) - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + if args.tcp: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect((host, port)) + else: + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.settimeout(args.timeout) + if args.bind: sock.bind(('', args.bind)) @@ -718,12 +731,23 @@ def main(): print("test:") hexdump(0, buf) + if args.tcp: + # add framing + header = struct.pack('>H', len(buf)) + sock.sendto(header, (host, port)) + sock.sendto(buf, (host, port)) if args.timeout == 0: return - data = sock.recv(1600) + if args.tcp: + # fetch framing + header = sock.recv(2, socket.MSG_WAITALL) + size = struct.unpack('>H', header)[0] + data = sock.recv(size, socket.MSG_WAITALL) + else: + data = sock.recv(1600) recv_pkt = PacketGeneric.from_buffer(data) recv_pkt.decode(data) From e7b983bc800e7a8805b004644888bbd961dc29d3 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Sat, 26 Oct 2024 20:19:38 +1100 Subject: [PATCH 17/85] Ensure that we do at least one TCP connect test --- scripts/test_integration_packets.sh | 13 ++++++++++ scripts/test_packets | 4 +++- tests/test_integration_packets.sh.expected | 28 ++++++++++++++++++++++ 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/scripts/test_integration_packets.sh b/scripts/test_integration_packets.sh index 68bd5ef..729564f 100755 --- a/scripts/test_integration_packets.sh +++ b/scripts/test_integration_packets.sh @@ -78,4 +78,17 @@ docmd "${BINDIR}"/scripts/test_packets \ docmd "${TOPDIR}"/scripts/n3nctl -s ci_sn1 get_edges --raw +# TCP +# +docmd "${BINDIR}"/scripts/test_packets \ + --tcp \ + --bind 7000 \ + -s localhost:7001 \ + test_QUERY_PEER_ping + +# TODO: +# - run all the same tests above but that needs a persistant TCP connection +# - remember that since we bound the port to TCP/7000 that becomes busy until +# the TIME-WAIT period ends + docmd "${TOPDIR}"/scripts/n3nctl -s ci_sn1 -k $AUTH stop diff --git a/scripts/test_packets b/scripts/test_packets index 6cd60d8..221af15 100755 --- a/scripts/test_packets +++ b/scripts/test_packets @@ -716,7 +716,6 @@ def main(): if args.tcp: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.connect((host, port)) else: sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) @@ -725,6 +724,9 @@ def main(): if args.bind: sock.bind(('', args.bind)) + if args.tcp: + sock.connect((host, port)) + if args.scenario != "listen": send_pkt = PacketGeneric.from_scenario(args.scenario) buf = send_pkt.encode() diff --git a/tests/test_integration_packets.sh.expected b/tests/test_integration_packets.sh.expected index 25f2968..9bffeab 100644 --- a/tests/test_integration_packets.sh.expected +++ b/tests/test_integration_packets.sh.expected @@ -181,6 +181,34 @@ test: ### test: ./scripts/n3nctl -s ci_sn1 get_edges --raw [] +### test: ./scripts/test_packets --tcp --bind 7000 -s localhost:7001 test_QUERY_PEER_ping +test: +000: 03 02 00 0b 74 65 73 74 00 00 00 00 00 00 00 00 | test | +010: 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 | | +020: 00 00 00 00 00 00 | | +recv: +000: 03 02 00 2a 74 65 73 74 00 00 00 00 00 00 00 00 | *test | +010: 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 01 | | +020: 00 00 00 00 00 00 00 00 1b 58 7f 00 00 01 ff ff | X | +030: ff ff ff ff ff ff 5f 73 74 61 62 69 6c 69 73 65 | _stabilise| +040: 64 5f 00 00 00 00 00 00 00 00 |d_ | +{'_name': 'PEER_INFO', + 'proto_version': 3, + 'ttl': 2, + 'type': 10, + 'flags': 32, + 'community': b'test\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00', + 'aflags': 0, + 'srcMac': b'\x02\x00\x00\x00\x00\x01', + 'mac': b'\x00\x00\x00\x00\x00\x00', + 'sock_family': , + 'sock_port': 7000, + 'sock_addr': b'\x7f\x00\x00\x01', + 'load': 4294967295, + 'uptime': 4294967295, + 'version': b'_stabilised_'} + ### test: ./scripts/n3nctl -s ci_sn1 -k n3n stop 0 From 6487e8bf92e2385468989922a7dccd928a0aae0e Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Sat, 26 Oct 2024 21:28:29 +1100 Subject: [PATCH 18/85] Attempt to plumb the TCP status into anything Nothing in the usual peer_info or n2n_sock_t indicates that a peer is to be connected via TCP - it is all done by a separate connection table. This makes it hard to get status output that shows that a given peer should be reachable via TCP. This patch was an attempt to connect some of the dots, sadly it was not enough. It is probably useful framework, but more still needs to be done. Of most annoyance is that the new TCP test in test_integration_packets.sh is sending and receiving over TCP, but gets a wire sock structure that is still flagged as UDP --- include/n2n_wire.h | 3 +- scripts/test_packets | 6 ++++ src/edge_utils.c | 31 +++++++++++++++++---- src/n2n.c | 32 ++++++++++++++-------- src/sn_utils.c | 23 ++++++++-------- src/wire.c | 3 +- tests/test_integration_packets.sh.expected | 6 ++++ 7 files changed, 73 insertions(+), 31 deletions(-) diff --git a/include/n2n_wire.h b/include/n2n_wire.h index 2a10a22..8ae8b01 100644 --- a/include/n2n_wire.h +++ b/include/n2n_wire.h @@ -139,7 +139,8 @@ int fill_sockaddr (struct sockaddr * addr, const n2n_sock_t * sock); int fill_n2nsock (n2n_sock_t* sock, - const struct sockaddr* sa); + const struct sockaddr* sa, + int type); int encode_PACKET (uint8_t * base, size_t * idx, diff --git a/scripts/test_packets b/scripts/test_packets index 221af15..7bd4c05 100755 --- a/scripts/test_packets +++ b/scripts/test_packets @@ -187,6 +187,7 @@ class PacketBase(): def decode_sock(self, buffer, prefix="sock"): name_family = f"{prefix}_family" + name_type = f"{prefix}_type" name_port = f"{prefix}_port" name_addr = f"{prefix}_addr" @@ -202,6 +203,11 @@ class PacketBase(): format = "!HH16s" else: raise ValueError(f"Unknown sock_family {data[0]}") + sock_type = data[0] &0x4000 + if sock_type == 0: + self.data[name_type] = socket.SOCK_DGRAM + else: + self.data[name_type] = socket.SOCK_STREAM calcsize = struct.calcsize(format) buffer = buffer[:calcsize] diff --git a/src/edge_utils.c b/src/edge_utils.c index 1f7494b..1de8fb6 100644 --- a/src/edge_utils.c +++ b/src/edge_utils.c @@ -1095,6 +1095,10 @@ static ssize_t sendto_fd (struct n3n_runtime_data *eee, const void *buf, level = TRACE_DEBUG; } + // TODO: + // - remove n2ndest param, as the only reason it is here is to + // stringify for errors. + // Better would be to stringify the dest sockaddr_t traceEvent(level, "sendto(%s) failed (%d) %s", sock_to_cstr(sockbuf, n2ndest), errno, errstr); @@ -1149,6 +1153,9 @@ static void sendto_sock (struct n3n_runtime_data *eee, const void * buf, peer_addr.sin_port = 0; + // TODO: + // - also check n2n_sock_t type == SOCK_STREAM as a TCP indicator + // network order socket fill_sockaddr((struct sockaddr *) &peer_addr, sizeof(peer_addr), dest); @@ -2277,8 +2284,13 @@ void edge_read_from_tap (struct n3n_runtime_data * eee) { /** handle a datagram from the main UDP socket to the internet. */ -void process_udp (struct n3n_runtime_data *eee, const struct sockaddr *sender_sock, const SOCKET in_sock, - uint8_t *udp_buf, size_t udp_size, time_t now) { +void process_udp (struct n3n_runtime_data *eee, + const struct sockaddr *sender_sock, + const SOCKET in_sock, + uint8_t *udp_buf, + size_t udp_size, + time_t now, + int type) { n2n_common_t cmn; /* common fields in the packet header */ n2n_sock_str_t sockbuf1; @@ -2309,7 +2321,7 @@ void process_udp (struct n3n_runtime_data *eee, const struct sockaddr *sender_so memset(&sender, 0, sizeof(sender)); // REVISIT: type conversion back and forth, choose a consistent approach throughout whole code, // i.e. stick with more general sockaddr as long as possible and narrow only if required - fill_n2nsock(&sender, sender_sock); + fill_n2nsock(&sender, sender_sock, type); } /* The packet may not have an orig_sender socket spec. So default to last * hop as sender. */ @@ -2922,7 +2934,7 @@ void edge_read_proto3_udp (struct n3n_runtime_data *eee, // we have a datagram to process... // ...and the datagram has data (not just a header) // - process_udp(eee, sender_sock, sock, pktbuf, bread, now); + process_udp(eee, sender_sock, sock, pktbuf, bread, now, SOCK_DGRAM); return; } @@ -2981,8 +2993,15 @@ void edge_read_proto3_tcp (struct n3n_runtime_data *eee, } // full packet read, handle it - process_udp(eee, sender_sock, sock, - pktbuf + sizeof(uint16_t), *position - sizeof(uint16_t), now); + process_udp( + eee, + sender_sock, + sock, + pktbuf + sizeof(uint16_t), + *position - sizeof(uint16_t), + now, + SOCK_STREAM + ); // reset, await new prepended length *expected = sizeof(uint16_t); *position = 0; diff --git a/src/n2n.c b/src/n2n.c index ac6e09e..2101643 100644 --- a/src/n2n.c +++ b/src/n2n.c @@ -275,24 +275,34 @@ extern char * sock_to_cstr (n2n_sock_str_t out, } memset(out, 0, N2N_SOCKBUF_SIZE); + bool is_tcp = (sock->type == SOCK_STREAM); + if(AF_INET6 == sock->family) { char tmp[INET6_ADDRSTRLEN+1]; tmp[0] = '\0'; inet_ntop(AF_INET6, sock->addr.v6, tmp, sizeof(n2n_sock_str_t)); - snprintf(out, N2N_SOCKBUF_SIZE, "[%s]:%hu", tmp[0] ? tmp : "", sock->port); - return out; - } else { - const uint8_t * a = sock->addr.v4; - - snprintf(out, N2N_SOCKBUF_SIZE, "%hu.%hu.%hu.%hu:%hu", - (unsigned short)(a[0] & 0xff), - (unsigned short)(a[1] & 0xff), - (unsigned short)(a[2] & 0xff), - (unsigned short)(a[3] & 0xff), - (unsigned short)sock->port); + snprintf( + out, + N2N_SOCKBUF_SIZE, + "%s[%s]:%hu", + is_tcp ? "TCP/" : "", + tmp[0] ? tmp : "", + sock->port + ); return out; } + + const uint8_t * a = sock->addr.v4; + + snprintf(out, N2N_SOCKBUF_SIZE, "%s%hu.%hu.%hu.%hu:%hu", + is_tcp ? "TCP/" : "", + (unsigned short)(a[0] & 0xff), + (unsigned short)(a[1] & 0xff), + (unsigned short)(a[2] & 0xff), + (unsigned short)(a[3] & 0xff), + (unsigned short)sock->port); + return out; } // TODO: move to a strings helper source file diff --git a/src/sn_utils.c b/src/sn_utils.c index db40b0d..e3eae37 100644 --- a/src/sn_utils.c +++ b/src/sn_utils.c @@ -98,14 +98,6 @@ static int sort_communities (struct n3n_runtime_data *sss, time_t* p_last_sort, time_t now); -static int process_udp (struct n3n_runtime_data *sss, - const struct sockaddr *sender_sock, socklen_t sock_size, - const SOCKET socket_fd, - uint8_t *udp_buf, - size_t udp_size, - time_t now); - - /* ************************************** */ @@ -1642,7 +1634,8 @@ static int process_udp (struct n3n_runtime_data * sss, const SOCKET socket_fd, uint8_t * udp_buf, size_t udp_size, - time_t now) { + time_t now, + int type) { n2n_common_t cmn; /* common fields in the packet header */ size_t rem; @@ -1664,7 +1657,7 @@ static int process_udp (struct n3n_runtime_data * sss, time_t any_time = 0; memset(&sender, 0, sizeof(n2n_sock_t)); - fill_n2nsock(&sender, sender_sock); + fill_n2nsock(&sender, sender_sock, SOCK_DGRAM); orig_sender = &sender; traceEvent(TRACE_DEBUG, "processing incoming UDP packet [len: %lu][sender: %s]", @@ -2619,6 +2612,10 @@ static int process_udp (struct n3n_runtime_data * sss, pi.preferred_sock = scan->preferred_sock; } + // FIXME: + // If we get the request on TCP, the reply should indicate + // our prefered sock is TCP ?? + encode_PEER_INFO(encbuf, &encx, &cmn2, &pi); if(comm->header_encryption == HEADER_ENCRYPTION_ENABLED) { @@ -2844,7 +2841,8 @@ int run_sn_loop (struct n3n_runtime_data *sss) { sss->sock, pktbuf, bread, - now + now, + SOCK_DGRAM ); } } @@ -2908,7 +2906,8 @@ int run_sn_loop (struct n3n_runtime_data *sss) { conn->socket_fd, conn->buffer + sizeof(uint16_t), conn->position - sizeof(uint16_t), - now + now, + SOCK_STREAM ); // reset, await new prepended length diff --git a/src/wire.c b/src/wire.c index 7e70d5b..538db79 100644 --- a/src/wire.c +++ b/src/wire.c @@ -690,9 +690,10 @@ int fill_sockaddr (struct sockaddr * addr, // fills struct sockaddr's data into n2n_sock -int fill_n2nsock (n2n_sock_t* sock, const struct sockaddr* sa) { +int fill_n2nsock (n2n_sock_t* sock, const struct sockaddr* sa, int type) { sock->family = sa->sa_family; + sock->type = type; // SOCK_DGRAM or SOCK_STREAM switch(sock->family) { case AF_INET: { diff --git a/tests/test_integration_packets.sh.expected b/tests/test_integration_packets.sh.expected index 9bffeab..af257ca 100644 --- a/tests/test_integration_packets.sh.expected +++ b/tests/test_integration_packets.sh.expected @@ -22,6 +22,7 @@ recv: 'srcMac': b'\x02\x00\x00\x00\x00\x01', 'mac': b'\x00\x00\x00\x00\x00\x00', 'sock_family': , + 'sock_type': , 'sock_port': 7000, 'sock_addr': b'\x7f\x00\x00\x01', 'load': 4294967295, @@ -52,6 +53,7 @@ recv: 'masklen': 24, 'lifetime': 15, 'sock_family': , + 'sock_type': , 'sock_port': 7000, 'sock_addr': b'\x7f\x00\x00\x01', 'auth_scheme': 0, @@ -101,6 +103,7 @@ recv: 'srcMac': b'\x00\x00\x00\x00\x00\x01', 'mac': b'\x00\x00\x00\x00\x00\x03', 'sock_family': , + 'sock_type': , 'sock_port': 7000, 'sock_addr': b'\x7f\x00\x00\x01', 'load': 4294967295, @@ -130,6 +133,7 @@ recv: 'srcMac': b'\x00\x00\x00\x00\x00\x01', 'edgeMac': b'\x00\x00\x00\x00\x00\x03', 'sock_family': , + 'sock_type': , 'sock_port': 7000, 'sock_addr': b'\x7f\x00\x00\x01', 'ipv4': b'\x00\x00\x00\x00', @@ -163,6 +167,7 @@ recv: 'srcMac': b'\x00\x00\x00\x00\x00\x01', 'edgeMac': b'\x00\x00\x00\x00\x00\x03', 'sock_family': , + 'sock_type': , 'sock_port': 7000, 'sock_addr': b'\x7f\x00\x00\x01', 'compression': 1, @@ -203,6 +208,7 @@ recv: 'srcMac': b'\x02\x00\x00\x00\x00\x01', 'mac': b'\x00\x00\x00\x00\x00\x00', 'sock_family': , + 'sock_type': , 'sock_port': 7000, 'sock_addr': b'\x7f\x00\x00\x01', 'load': 4294967295, From 2d97ca8e7a813f61cbf97775a89e3b0c1c3b1461 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Sat, 26 Oct 2024 21:57:16 +1100 Subject: [PATCH 19/85] Fix breaking change in the TCP status tracking --- src/wire.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/wire.c b/src/wire.c index 538db79..3447f44 100644 --- a/src/wire.c +++ b/src/wire.c @@ -693,7 +693,8 @@ int fill_sockaddr (struct sockaddr * addr, int fill_n2nsock (n2n_sock_t* sock, const struct sockaddr* sa, int type) { sock->family = sa->sa_family; - sock->type = type; // SOCK_DGRAM or SOCK_STREAM + // TODO: re enable this when it doesnt break things + // sock->type = type; // SOCK_DGRAM or SOCK_STREAM switch(sock->family) { case AF_INET: { From 69786e07d2a465680691b6fba6a942377bb586df Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Sat, 26 Oct 2024 21:58:23 +1100 Subject: [PATCH 20/85] Add some TCP integration tests and testing stability improvements --- scripts/test_integration_edge_tcp.sh | 65 ++++++++++++++++ scripts/test_integration_supernode.sh | 6 +- scripts/test_packets | 1 + tests/test_integration_edge_tcp.sh.expected | 80 ++++++++++++++++++++ tests/test_integration_supernode.sh.expected | 6 +- tests/tests_integration.list | 1 + 6 files changed, 155 insertions(+), 4 deletions(-) create mode 100755 scripts/test_integration_edge_tcp.sh create mode 100644 tests/test_integration_edge_tcp.sh.expected diff --git a/scripts/test_integration_edge_tcp.sh b/scripts/test_integration_edge_tcp.sh new file mode 100755 index 0000000..7176b2c --- /dev/null +++ b/scripts/test_integration_edge_tcp.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# +# Copyright (C) Hamish Coleman +# SPDX-License-Identifier: GPL-3.0-only +# +# Do some quick tests via the Json API against the edge +# + +AUTH=n3n + +# boilerplate so we can support whaky cmake dirs +[ -z "$TOPDIR" ] && TOPDIR=. +[ -z "$BINDIR" ] && BINDIR=. + +docmd() { + echo "### test: $*" + "$@" + local S=$? + echo + return $S +} + +# We dont have perms for writing to the /run dir, TODO: improve this +sudo mkdir -p /run/n3n +sudo chown "$USER" /run/n3n + +# start a supernode +docmd "${BINDIR}"/apps/n3n-supernode start ci_sn \ + -v \ + --daemon \ + -Osupernode.macaddr=02:00:00:55:00:00 + +# Start the edge in the background +docmd sudo "${BINDIR}"/apps/n3n-edge start ci_edge1 \ + --daemon \ + -l localhost:7654 \ + -c test \ + -Oconnection.bind=:7700 \ + -Oconnection.connect_tcp=true \ + -Odaemon.userid="$USER" \ + -Otuntap.macaddr=02:00:00:77:00:00 \ + 1>&2 + +# TODO: probe the api endpoint, waiting for both the supernode and edge to be +# available? +sleep 0.1 + +docmd "${TOPDIR}"/scripts/n3nctl -s ci_edge1 get_communities + +echo "### test: ${TOPDIR}/scripts/n3nctl -s ci_edge1 get_packetstats" +"${TOPDIR}"/scripts/n3nctl -s ci_edge1 get_packetstats |jq '.[0:5]' +# this is filtering out the type=multicast_drop line as that has a changing +# number of packets counted +echo + +docmd "${TOPDIR}"/scripts/n3nctl -s ci_edge1 get_edges --raw |grep -v "last_seen" +docmd "${TOPDIR}"/scripts/n3nctl -s ci_sn get_edges --raw |grep -v -E "last_seen|time_alloc" + + +docmd ${TOPDIR}/scripts/n3nctl -s ci_edge1 get_supernodes --raw + +# stop them both +docmd "${TOPDIR}"/scripts/n3nctl -s ci_edge1 -k $AUTH stop +docmd "${TOPDIR}"/scripts/n3nctl -s ci_sn -k $AUTH stop + diff --git a/scripts/test_integration_supernode.sh b/scripts/test_integration_supernode.sh index 398f76a..97eba88 100755 --- a/scripts/test_integration_supernode.sh +++ b/scripts/test_integration_supernode.sh @@ -26,10 +26,12 @@ sudo chown "$USER" /run/n3n docmd "${BINDIR}"/apps/n3n-supernode start ci_sn1 \ --daemon \ -Oconnection.bind=7001 \ + -Osupernode.macaddr=02:00:00:00:70:01 \ -Osupernode.peer=localhost:7002 docmd "${BINDIR}"/apps/n3n-supernode start ci_sn2 \ --daemon \ -Oconnection.bind=7002 \ + -Osupernode.macaddr=02:00:00:00:70:02 \ -Osupernode.peer=localhost:7001 # TODO: probe the api endpoint, waiting for the supernode to be available? @@ -41,8 +43,8 @@ docmd "${TOPDIR}"/scripts/n3nctl -s ci_sn2 get_communities docmd "${TOPDIR}"/scripts/n3nctl -s ci_sn1 get_packetstats docmd "${TOPDIR}"/scripts/n3nctl -s ci_sn2 get_packetstats -docmd "${TOPDIR}"/scripts/n3nctl -s ci_sn1 get_edges --raw | grep -vE "last_seen|macaddr|time_alloc" -docmd "${TOPDIR}"/scripts/n3nctl -s ci_sn2 get_edges --raw | grep -vE "last_seen|macaddr|time_alloc" +docmd "${TOPDIR}"/scripts/n3nctl -s ci_sn1 get_edges --raw | grep -vE "last_seen|time_alloc" +docmd "${TOPDIR}"/scripts/n3nctl -s ci_sn2 get_edges --raw | grep -vE "last_seen|time_alloc" # Test with bad auth docmd "${TOPDIR}"/scripts/n3nctl -s ci_sn1 set_verbose 1 diff --git a/scripts/test_packets b/scripts/test_packets index 7bd4c05..a86a22f 100755 --- a/scripts/test_packets +++ b/scripts/test_packets @@ -722,6 +722,7 @@ def main(): if args.tcp: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR,1) else: sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) diff --git a/tests/test_integration_edge_tcp.sh.expected b/tests/test_integration_edge_tcp.sh.expected new file mode 100644 index 0000000..b6aae38 --- /dev/null +++ b/tests/test_integration_edge_tcp.sh.expected @@ -0,0 +1,80 @@ +### test: ./apps/n3n-supernode start ci_sn -v --daemon -Osupernode.macaddr=02:00:00:55:00:00 + +### test: ./scripts/n3nctl -s ci_edge1 get_communities +[ + { + "community": "test" + } +] + +### test: ./scripts/n3nctl -s ci_edge1 get_packetstats +[ + { + "rx_pkt": 0, + "tx_pkt": 2, + "type": "transop" + }, + { + "rx_pkt": 0, + "tx_pkt": 0, + "type": "p2p" + }, + { + "rx_pkt": 0, + "tx_pkt": 2, + "type": "super" + }, + { + "rx_pkt": 0, + "tx_pkt": 2, + "type": "super_broadcast" + }, + { + "tx_pkt": 0, + "type": "tuntap_error" + } +] + +### test: ./scripts/n3nctl -s ci_edge1 get_edges --raw +[] + +### test: ./scripts/n3nctl -s ci_sn get_edges --raw +[ + { + "community": "test", + "desc": "archer", + "ip4addr": "10.200.175.63/24", + "last_p2p": 0, + "last_sent_query": 0, + "local": 0, + "macaddr": "02:00:00:77:00:00", + "mode": "sn", + "prefered_sockaddr": "0.0.0.0:0", + "purgeable": 1, + "sockaddr": "127.0.0.1:7700", + "timeout": 0, + "uptime": 0, + "version": "" + } +] + +### test: ./scripts/n3nctl -s ci_edge1 get_supernodes --raw +[ + { + "current": 1, + "last_seen": 0, + "macaddr": "02:00:00:55:00:00", + "purgeable": 0, + "selection": "", + "sockaddr": "127.0.0.1:7654", + "uptime": 0, + "version": "" + } +] + +### test: ./scripts/n3nctl -s ci_edge1 -k n3n stop +0 + +### test: ./scripts/n3nctl -s ci_sn -k n3n stop +0 + diff --git a/tests/test_integration_supernode.sh.expected b/tests/test_integration_supernode.sh.expected index 8b442e7..31011a0 100644 --- a/tests/test_integration_supernode.sh.expected +++ b/tests/test_integration_supernode.sh.expected @@ -1,6 +1,6 @@ -### test: ./apps/n3n-supernode start ci_sn1 --daemon -Oconnection.bind=7001 -Osupernode.peer=localhost:7002 +### test: ./apps/n3n-supernode start ci_sn1 --daemon -Oconnection.bind=7001 -Osupernode.macaddr=02:00:00:00:70:01 -Osupernode.peer=localhost:7002 -### test: ./apps/n3n-supernode start ci_sn2 --daemon -Oconnection.bind=7002 -Osupernode.peer=localhost:7001 +### test: ./apps/n3n-supernode start ci_sn2 --daemon -Oconnection.bind=7002 -Osupernode.macaddr=02:00:00:00:70:02 -Osupernode.peer=localhost:7001 ### test: ./scripts/n3nctl -s ci_sn1 get_communities [ @@ -131,6 +131,7 @@ "last_p2p": 0, "last_sent_query": 0, "local": 0, + "macaddr": "02:00:00:00:70:02", "mode": "sn", "prefered_sockaddr": "0.0.0.0:0", "purgeable": 0, @@ -150,6 +151,7 @@ "last_p2p": 0, "last_sent_query": 0, "local": 0, + "macaddr": "02:00:00:00:70:01", "mode": "sn", "prefered_sockaddr": "0.0.0.0:0", "purgeable": 0, diff --git a/tests/tests_integration.list b/tests/tests_integration.list index 4bf3bc8..29ff94c 100644 --- a/tests/tests_integration.list +++ b/tests/tests_integration.list @@ -4,3 +4,4 @@ test_integration_packets.sh test_integration_supernode.sh test_integration_edge.sh +test_integration_edge_tcp.sh From 5e0092941d25ef2e030e3860ad12baa559ca6f8f Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Sat, 26 Oct 2024 22:20:00 +1100 Subject: [PATCH 21/85] Address lint issue --- scripts/test_integration_edge_tcp.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test_integration_edge_tcp.sh b/scripts/test_integration_edge_tcp.sh index 7176b2c..661f95b 100755 --- a/scripts/test_integration_edge_tcp.sh +++ b/scripts/test_integration_edge_tcp.sh @@ -57,7 +57,7 @@ docmd "${TOPDIR}"/scripts/n3nctl -s ci_edge1 get_edges --raw |grep -v "last_seen docmd "${TOPDIR}"/scripts/n3nctl -s ci_sn get_edges --raw |grep -v -E "last_seen|time_alloc" -docmd ${TOPDIR}/scripts/n3nctl -s ci_edge1 get_supernodes --raw +docmd "${TOPDIR}"/scripts/n3nctl -s ci_edge1 get_supernodes --raw # stop them both docmd "${TOPDIR}"/scripts/n3nctl -s ci_edge1 -k $AUTH stop From cda8870f671c12fb808ec37f75cd897dc57965f8 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Sat, 26 Oct 2024 22:28:52 +1100 Subject: [PATCH 22/85] Using a stable name should help with stable ip4addr --- scripts/test_integration_edge_tcp.sh | 1 + tests/test_integration_edge_tcp.sh.expected | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/test_integration_edge_tcp.sh b/scripts/test_integration_edge_tcp.sh index 661f95b..84c3b18 100755 --- a/scripts/test_integration_edge_tcp.sh +++ b/scripts/test_integration_edge_tcp.sh @@ -37,6 +37,7 @@ docmd sudo "${BINDIR}"/apps/n3n-edge start ci_edge1 \ -c test \ -Oconnection.bind=:7700 \ -Oconnection.connect_tcp=true \ + -Oconnection.description=ci_edge1 \ -Odaemon.userid="$USER" \ -Otuntap.macaddr=02:00:00:77:00:00 \ 1>&2 diff --git a/tests/test_integration_edge_tcp.sh.expected b/tests/test_integration_edge_tcp.sh.expected index b6aae38..f7641bc 100644 --- a/tests/test_integration_edge_tcp.sh.expected +++ b/tests/test_integration_edge_tcp.sh.expected @@ -42,8 +42,8 @@ [ { "community": "test", - "desc": "archer", - "ip4addr": "10.200.175.63/24", + "desc": "ci_edge1", + "ip4addr": "10.200.175.139/24", "last_p2p": 0, "last_sent_query": 0, "local": 0, From ced1c935b822288ee2466b5a7c59a85bfe2261eb Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Sun, 17 Nov 2024 14:39:22 +1100 Subject: [PATCH 23/85] Ensure that multiple running sesssions have distinct metrics and thus they can easily be merged --- include/n3n/metrics.h | 3 +++ src/edge_utils.c | 1 + src/metrics.c | 12 +++++++++--- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/include/n3n/metrics.h b/include/n3n/metrics.h index 5217521..4b2a3bd 100644 --- a/include/n3n/metrics.h +++ b/include/n3n/metrics.h @@ -62,4 +62,7 @@ void n3n_metrics_register (struct n3n_metrics_module *); // Render all the metrics into a strbuf void n3n_metrics_render (strbuf_t **reply); +// Set the session name for all metrics +void n3n_metrics_set_session (const char *); + #endif diff --git a/src/edge_utils.c b/src/edge_utils.c index 1de8fb6..1ae34ad 100644 --- a/src/edge_utils.c +++ b/src/edge_utils.c @@ -3367,6 +3367,7 @@ void edge_init_conf_defaults (n2n_edge_conf_t *conf, char *sessionname) { } else { conf->sessionname = "NULL"; } + n3n_metrics_set_session(conf->sessionname); conf->is_edge = true; diff --git a/src/metrics.c b/src/metrics.c index e564f55..b3663ef 100644 --- a/src/metrics.c +++ b/src/metrics.c @@ -10,6 +10,8 @@ #include #include +static char *sessionname; + static struct n3n_metrics_module *registered_metrics; void n3n_metrics_register (struct n3n_metrics_module *module) { @@ -48,7 +50,7 @@ static void metrics_render_uint32 (strbuf_t **reply, struct n3n_metrics_module * // - " UNIT name type\n" // - " HELP name type\n" metrics_name(reply, module->name, module->items_uint32[i].name); - sb_reprintf(reply, " "); + sb_reprintf(reply, "{session=\"%s\"} ", sessionname); metric_stringify_uint32( reply, module->items_uint32[i].offset, @@ -74,7 +76,8 @@ static void metrics_render_llu32 (strbuf_t **reply, struct n3n_metrics_module *m metrics_name(reply, module->name, info->name); sb_reprintf( reply, - "{%s=\"%s\",%s=\"%s\"} ", + "{session=\"%s\",%s=\"%s\",%s=\"%s\"} ", + sessionname, info->name1, info->items[i].val1, info->name2, @@ -161,7 +164,10 @@ static struct n3n_metrics_module strbuf_metrics_module = { }; /**********************************************************/ - void n3n_initfuncs_metrics () { n3n_metrics_register(&strbuf_metrics_module); } + +void n3n_metrics_set_session (const char *name) { + sessionname = name; +} From 4cc6a57d942908da8858a863b4d825aa917e7e82 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Sun, 17 Nov 2024 14:52:01 +1100 Subject: [PATCH 24/85] work around const discarding warning --- include/n3n/metrics.h | 2 +- src/metrics.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/n3n/metrics.h b/include/n3n/metrics.h index 4b2a3bd..285f32b 100644 --- a/include/n3n/metrics.h +++ b/include/n3n/metrics.h @@ -63,6 +63,6 @@ void n3n_metrics_register (struct n3n_metrics_module *); void n3n_metrics_render (strbuf_t **reply); // Set the session name for all metrics -void n3n_metrics_set_session (const char *); +void n3n_metrics_set_session (char *); #endif diff --git a/src/metrics.c b/src/metrics.c index b3663ef..5d28672 100644 --- a/src/metrics.c +++ b/src/metrics.c @@ -168,6 +168,6 @@ void n3n_initfuncs_metrics () { n3n_metrics_register(&strbuf_metrics_module); } -void n3n_metrics_set_session (const char *name) { +void n3n_metrics_set_session (char *name) { sessionname = name; } From 816c31158b13f3a0f9e2c9b092b88e31330c2371 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Sun, 17 Nov 2024 15:00:57 +1100 Subject: [PATCH 25/85] Refactor min and max to use their usual names Note that this not ideal, and we should use an OS pre-define macro, which may have some type safety in it. But this is another case where Windows did something subtly different to basically everyone else. --- include/n2n_define.h | 7 ------- src/edge_utils.c | 7 ++++--- src/mainloop.c | 17 +++++------------ src/minmax.h | 17 +++++++++++++++++ src/sn_utils.c | 11 ++++++----- tools/crypto_helper.c | 15 +++++++++++++-- 6 files changed, 45 insertions(+), 29 deletions(-) create mode 100644 src/minmax.h diff --git a/include/n2n_define.h b/include/n2n_define.h index 23f2e16..bd6ce6c 100644 --- a/include/n2n_define.h +++ b/include/n2n_define.h @@ -173,12 +173,5 @@ enum n3n_event_topic { #define N2N_TRANSFORM_ID_USER_START 64 #define N2N_TRANSFORM_ID_MAX 65535 -#ifndef max -#define max(a, b) (((a) < (b)) ? (b) : (a)) -#endif - -#ifndef min -#define min(a, b) (((a) >(b)) ? (b) : (a)) -#endif #endif diff --git a/src/edge_utils.c b/src/edge_utils.c index 1ae34ad..59d3df2 100644 --- a/src/edge_utils.c +++ b/src/edge_utils.c @@ -48,6 +48,7 @@ #include "header_encryption.h" // for packet_header_encrypt, packet_he... #include "mainloop.h" // for mainloop_runonce #include "management.h" // for mgmt_event_post +#include "minmax.h" // for MIN, MAX #include "n2n.h" // for n3n_runtime_data, n2n_edge_... #include "n2n_wire.h" // for fill_sockaddr, decod... #include "pearson.h" // for pearson_hash_128, pearson_hash_64 @@ -2185,7 +2186,7 @@ void edge_send_packet2net (struct n3n_runtime_data * eee, if(eee->conf.header_encryption == HEADER_ENCRYPTION_ENABLED) // in case of user-password auth, also encrypt the iv of payload assuming ChaCha20 and SPECK having the same iv size - packet_header_encrypt(pktbuf, headerIdx + (NULL != eee->conf.shared_secret) * min(idx - headerIdx, N2N_SPECK_IVEC_SIZE), idx, + packet_header_encrypt(pktbuf, headerIdx + (NULL != eee->conf.shared_secret) * MIN(idx - headerIdx, N2N_SPECK_IVEC_SIZE), idx, eee->conf.header_encryption_ctx_dynamic, eee->conf.header_iv_ctx_dynamic, time_stamp()); @@ -2349,9 +2350,9 @@ void process_udp (struct n3n_runtime_data *eee, // check static now (very likely to be REGISTER_SUPER_ACK, REGISTER_SUPER_NAK or invalid) if(eee->conf.shared_secret) { // hash the still encrypted packet to eventually be able to check it later (required for REGISTER_SUPER_ACK with user/pw auth) - pearson_hash_128(hash_buf, udp_buf, max(0, (int)udp_size - (int)N2N_REG_SUP_HASH_CHECK_LEN)); + pearson_hash_128(hash_buf, udp_buf, MAX(0, (int)udp_size - (int)N2N_REG_SUP_HASH_CHECK_LEN)); } - header_enc = packet_header_decrypt(udp_buf, max(0, (int)udp_size - (int)N2N_REG_SUP_HASH_CHECK_LEN), + header_enc = packet_header_decrypt(udp_buf, MAX(0, (int)udp_size - (int)N2N_REG_SUP_HASH_CHECK_LEN), (char *)eee->conf.community_name, eee->conf.header_encryption_ctx_static, eee->conf.header_iv_ctx_static, &stamp); diff --git a/src/mainloop.c b/src/mainloop.c index 33b0661..0e6649f 100644 --- a/src/mainloop.c +++ b/src/mainloop.c @@ -16,16 +16,9 @@ #include "edge_utils.h" // for edge_read_from_tap #include "management.h" // for readFromMgmtSocket +#include "minmax.h" // for min, max #include "n2n_define.h" -#ifndef max -#define max(a, b) (((a) < (b)) ? (b) : (a)) -#endif - -#ifndef min -#define min(a, b) (((a) >(b)) ? (b) : (a)) -#endif - static int setup_select (fd_set *rd, fd_set *wr, struct n3n_runtime_data *eee) { FD_ZERO(rd); FD_ZERO(wr); @@ -33,22 +26,22 @@ static int setup_select (fd_set *rd, fd_set *wr, struct n3n_runtime_data *eee) { if(eee->sock >= 0) { FD_SET(eee->sock, rd); - max_sock = max(max_sock, eee->sock); + max_sock = MAX(max_sock, eee->sock); } #ifndef SKIP_MULTICAST_PEERS_DISCOVERY if((eee->conf.allow_p2p) && (eee->conf.preferred_sock.family == (uint8_t)AF_INVALID)) { FD_SET(eee->udp_multicast_sock, rd); - max_sock = max(max_sock, eee->udp_multicast_sock); + max_sock = MAX(max_sock, eee->udp_multicast_sock); } #endif #ifndef _WIN32 FD_SET(eee->device.fd, rd); - max_sock = max(max_sock, eee->device.fd); + max_sock = MAX(max_sock, eee->device.fd); #endif - max_sock = max( + max_sock = MAX( max_sock, slots_fdset( eee->mgmt_slots, diff --git a/src/minmax.h b/src/minmax.h new file mode 100644 index 0000000..313b30d --- /dev/null +++ b/src/minmax.h @@ -0,0 +1,17 @@ +/* + * Copyright (C) 2024 Hamish Coleman + * SPDX-License-Identifier: GPL-3.0-only + * + */ + +// TODO: +// - on linux there are headers with these predefined +// - on windows, there are different predefines +// - use them! +#ifndef MAX +#define MAX(a, b) (((a) < (b)) ? (b) : (a)) +#endif + +#ifndef MIN +#define MIN(a, b) (((a) >(b)) ? (b) : (a)) +#endif diff --git a/src/sn_utils.c b/src/sn_utils.c index e3eae37..33acf1f 100644 --- a/src/sn_utils.c +++ b/src/sn_utils.c @@ -38,6 +38,7 @@ #include "auth.h" // for ascii_to_bin, calculate_dynamic_key #include "header_encryption.h" // for packet_header_encrypt, packet_header_... #include "management.h" // for process_mgmt +#include "minmax.h" // for MIN, MAX #include "n2n.h" // for sn_community, n3n_runtime_data #include "n2n_define.h" #include "n2n_regex.h" // for re_matchp, re_compile @@ -1717,8 +1718,8 @@ static int process_udp (struct n3n_runtime_data * sss, header_enc = 2; } if(!header_enc) { - pearson_hash_128(hash_buf, udp_buf, max(0, (int)udp_size - (int)N2N_REG_SUP_HASH_CHECK_LEN)); - header_enc = packet_header_decrypt(udp_buf, max(0, (int)udp_size - (int)N2N_REG_SUP_HASH_CHECK_LEN), comm->community, + pearson_hash_128(hash_buf, udp_buf, MAX(0, (int)udp_size - (int)N2N_REG_SUP_HASH_CHECK_LEN)); + header_enc = packet_header_decrypt(udp_buf, MAX(0, (int)udp_size - (int)N2N_REG_SUP_HASH_CHECK_LEN), comm->community, comm->header_encryption_ctx_static, comm->header_iv_ctx_static, &stamp); } @@ -1861,7 +1862,7 @@ static int process_udp (struct n3n_runtime_data * sss, if(comm->header_encryption == HEADER_ENCRYPTION_ENABLED) { // in case of user-password auth, also encrypt the iv of payload assuming ChaCha20 and SPECK having the same iv size - packet_header_encrypt(rec_buf, oldEncx + (NULL != comm->allowed_users) * min(encx - oldEncx, N2N_SPECK_IVEC_SIZE), encx, + packet_header_encrypt(rec_buf, oldEncx + (NULL != comm->allowed_users) * MIN(encx - oldEncx, N2N_SPECK_IVEC_SIZE), encx, comm->header_encryption_ctx_dynamic, comm->header_iv_ctx_dynamic, time_stamp()); } @@ -1876,7 +1877,7 @@ static int process_udp (struct n3n_runtime_data * sss, if(comm->header_encryption == HEADER_ENCRYPTION_ENABLED) { // in case of user-password auth, also encrypt the iv of payload assuming ChaCha20 and SPECK having the same iv size - packet_header_encrypt(rec_buf, idx + (NULL != comm->allowed_users) * min(encx - idx, N2N_SPECK_IVEC_SIZE), encx, + packet_header_encrypt(rec_buf, idx + (NULL != comm->allowed_users) * MIN(encx - idx, N2N_SPECK_IVEC_SIZE), encx, comm->header_encryption_ctx_dynamic, comm->header_iv_ctx_dynamic, time_stamp()); } @@ -2765,7 +2766,7 @@ int run_sn_loop (struct n3n_runtime_data *sss) { #endif slots_t *slots = sss->mgmt_slots; - max_sock = max( + max_sock = MAX( max_sock, slots_fdset( slots, diff --git a/tools/crypto_helper.c b/tools/crypto_helper.c index 05b3e87..f1a47e3 100644 --- a/tools/crypto_helper.c +++ b/tools/crypto_helper.c @@ -16,6 +16,17 @@ #include #include // for read, write +// TODO: +// - on linux there are headers with these predefined +// - on windows, there are different predefines +// - use them! +#ifndef MAX +#define MAX(a, b) (((a) < (b)) ? (b) : (a)) +#endif + +#ifndef MIN +#define MIN(a, b) (((a) >(b)) ? (b) : (a)) +#endif #define GETOPTS "Vhv" @@ -69,11 +80,11 @@ static void cmd_header_decrypt (int argc, char **argv, void *conf) { pearson_hash_128( hash_buf, buf, - max(0, (int)size - (int)N2N_REG_SUP_HASH_CHECK_LEN) + MAX(0, (int)size - (int)N2N_REG_SUP_HASH_CHECK_LEN) ); ok = packet_header_decrypt( buf, - max(0, (int)size - (int)N2N_REG_SUP_HASH_CHECK_LEN), + MAX(0, (int)size - (int)N2N_REG_SUP_HASH_CHECK_LEN), community, ctx_static, ctx_iv_static, From 9621b42f6b3daf73dd7ced4edfad73329066c561 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Sun, 17 Nov 2024 16:50:58 +1100 Subject: [PATCH 26/85] Start using mainloop registration Demonstrate that all the features are working by converting the tun fd read to use the new mainloop process. --- apps/example_edge_embed.c | 6 ++ apps/n3n-edge.c | 5 ++ include/n2n_typedefs.h | 4 +- include/n3n/mainloop.h | 28 ++++++++ src/edge_utils.c | 12 +++- src/initfuncs.c | 2 + src/mainloop.c | 132 ++++++++++++++++++++++++++++++++++---- src/mainloop.h | 14 ---- 8 files changed, 175 insertions(+), 28 deletions(-) create mode 100644 include/n3n/mainloop.h delete mode 100644 src/mainloop.h diff --git a/apps/example_edge_embed.c b/apps/example_edge_embed.c index 166e1c6..917de6c 100644 --- a/apps/example_edge_embed.c +++ b/apps/example_edge_embed.c @@ -20,6 +20,7 @@ #include #include // for edge_init_conf_defaults, edge_verify_conf +#include // for mainloop_register_fd, fd_info_proto #include // for n3n_peer_add_by_hostname #include #include // for snprintf, NULL @@ -68,6 +69,11 @@ int main () { return -1; } +#ifndef _WIN32 + // TODO: this internal fn should not be called publicly + mainloop_register_fd(tuntap.fd, fd_info_proto_tuntap); +#endif + eee = edge_init(&conf, &rc); if(eee == NULL) { exit(1); diff --git a/apps/n3n-edge.c b/apps/n3n-edge.c index 2fa6edb..7fbee5d 100644 --- a/apps/n3n-edge.c +++ b/apps/n3n-edge.c @@ -29,6 +29,7 @@ #include // for macaddr_str, macstr_t #include // for n3n_initfuncs() #include // for traceEvent +#include // for mainloop_register_fd #include // for test_hashing #include // for n3n_rand_seeds, n3n_rand_seeds_s... #include // for n3n_transform_lookup_id @@ -957,6 +958,10 @@ int main (int argc, char* argv[]) { eee->conf.mtu, eee->conf.metric) < 0) exit(1); +#ifndef _WIN32 + // TODO: this internal fn should not be called publicly + mainloop_register_fd(eee->device.fd, fd_info_proto_tuntap); +#endif in_addr_t addr = eee->conf.tuntap_v4.net_addr; struct in_addr *tmp = (struct in_addr *)&addr; traceEvent(TRACE_NORMAL, "created local tap device IPv4: %s/%u, MAC: %s", diff --git a/include/n2n_typedefs.h b/include/n2n_typedefs.h index f29563e..b82f7fe 100644 --- a/include/n2n_typedefs.h +++ b/include/n2n_typedefs.h @@ -132,11 +132,13 @@ typedef char devstr_t[N2N_IFNAMSIZ]; typedef struct tuntap_dev { +#ifndef _WIN32 int fd; + devstr_t dev_name; +#endif in_addr_t ip_addr; n2n_mac_t mac_addr; uint16_t mtu; - devstr_t dev_name; #ifdef _WIN32 HANDLE device_handle; char *device_name; diff --git a/include/n3n/mainloop.h b/include/n3n/mainloop.h new file mode 100644 index 0000000..ca34e3e --- /dev/null +++ b/include/n3n/mainloop.h @@ -0,0 +1,28 @@ +/** + * Copyright (C) Hamish Coleman + * + * non public structure and function definitions + * + * TODO: + * - fix the layering confusion in the calling code + * (apps/example_edge_embed.c apps/n3n-edge.c) + * and move this header back to the non-public src/ location + */ + +#ifndef _MAINLOOP_H_ +#define _MAINLOOP_H_ + +#include // for n3n_runtime_data + +enum __attribute__((__packed__)) fd_info_proto { + fd_info_proto_unknown = 0, + fd_info_proto_tuntap, +}; + +int mainloop_runonce (fd_set *, fd_set *, struct n3n_runtime_data *); + +void mainloop_register_fd (int, enum fd_info_proto); +void mainloop_unregister_fd (int); + + +#endif diff --git a/src/edge_utils.c b/src/edge_utils.c index 59d3df2..4ae05ca 100644 --- a/src/edge_utils.c +++ b/src/edge_utils.c @@ -29,6 +29,7 @@ #include // for n3n_peer_add_by_hostname #include // for is_null_mac #include // for traceEvent +#include // for mainloop_runonce, mainloop_regis... #include #include // for create_network_traffic_filte... #include // for n3n_rand, n3n_rand_sqr @@ -46,7 +47,6 @@ #include "edge_utils.h" #include "header_encryption.h" // for packet_header_encrypt, packet_he... -#include "mainloop.h" // for mainloop_runonce #include "management.h" // for mgmt_event_post #include "minmax.h" // for MIN, MAX #include "n2n.h" // for n3n_runtime_data, n2n_edge_... @@ -2218,6 +2218,10 @@ void edge_read_from_tap (struct n3n_runtime_data * eee) { len = tuntap_read( &(eee->device), eth_pkt, N2N_PKT_BUF_SIZE ); if((len <= 0) || (len > N2N_PKT_BUF_SIZE)) { + // TODO: + // - how often does this actually happen + // - why does it happen + // - can we just remove this special case? traceEvent( TRACE_WARNING, "read()=%d [%d/%s]", @@ -2229,6 +2233,9 @@ void edge_read_from_tap (struct n3n_runtime_data * eee) { eee->stats.tx_tuntap_error++; sleep(3); +#ifndef _WIN32 + mainloop_unregister_fd(eee->device.fd); +#endif tuntap_close(&(eee->device)); tuntap_open(&(eee->device), eee->conf.tuntap_dev_name, @@ -2238,6 +2245,9 @@ void edge_read_from_tap (struct n3n_runtime_data * eee) { eee->conf.mtu, eee->conf.metric ); +#ifndef _WIN32 + mainloop_register_fd(eee->device.fd, fd_info_proto_tuntap); +#endif return; } diff --git a/src/initfuncs.c b/src/initfuncs.c index 1621b0a..0a78762 100644 --- a/src/initfuncs.c +++ b/src/initfuncs.c @@ -7,6 +7,7 @@ // prototype any internal (non-public) initfuncs (always sorted!) void n3n_initfuncs_conffile_defs (); +void n3n_initfuncs_mainloop (); void n3n_initfuncs_metrics (); void n3n_initfuncs_pearson (); void n3n_initfuncs_peer_info (); @@ -28,6 +29,7 @@ void n3n_initfuncs () { // (sorted list) n3n_initfuncs_conffile_defs(); + n3n_initfuncs_mainloop(); n3n_initfuncs_metrics(); n3n_initfuncs_pearson(); n3n_initfuncs_peer_info(); diff --git a/src/mainloop.c b/src/mainloop.c index 0e6649f..3ff1103 100644 --- a/src/mainloop.c +++ b/src/mainloop.c @@ -4,8 +4,10 @@ * */ +#include #include // for slots_fdset #include // for n3n_runtime_data +#include // for fd_info_proto #include // for traceEvent #include #include @@ -19,10 +21,110 @@ #include "minmax.h" // for min, max #include "n2n_define.h" +static void handle_fd (int fd, enum fd_info_proto proto, struct n3n_runtime_data *eee) { + switch(proto) { + case fd_info_proto_unknown: + // should not happen! + assert(false); + break; + case fd_info_proto_tuntap: + // read an ethernet frame from the TAP socket; write on the IP + // socket + edge_read_from_tap(eee); + break; + } +} + +struct fd_info { + int fd; + enum fd_info_proto proto; +}; + +// A static array of known file descriptors will not scale once full TCP +// connection support is added, but will work for now +#define MAX_HANDLES 16 +static struct fd_info fdlist[MAX_HANDLES]; +static int fdlist_next_search; + +// Used only to initialise the array at startup +static void fdlist_zero () { + int slot = 0; + while(slot < MAX_HANDLES) { + fdlist[slot].fd = -1; + fdlist[slot].proto = fd_info_proto_unknown; + slot++; + } + fdlist_next_search = 0; +} + +static int fdlist_allocslot (int fd, enum fd_info_proto proto) { + int slot = fdlist_next_search % MAX_HANDLES; + int count = MAX_HANDLES; + while(count) { + if(fdlist[slot].fd == -1) { + fdlist[slot].fd = fd; + fdlist[slot].proto = proto; + fdlist_next_search = slot + 1; + return slot; + } + slot = (slot + 1) % MAX_HANDLES; + count--; + } + return -1; +} + +static void fdlist_freefd (int fd) { + int slot = 0; + while(slot < MAX_HANDLES) { + if(fdlist[slot].fd != fd) { + continue; + } + fdlist[slot].fd = -1; + fdlist[slot].proto = fd_info_proto_unknown; + fdlist_next_search = slot; + return; + } + + // TODO: + // - could assert or similar +} + +static int fdlist_read_fd_set (fd_set *rd) { + int max_sock = 0; + int slot = 0; + while(slot < MAX_HANDLES) { + if(fdlist[slot].fd != -1) { + FD_SET(fdlist[slot].fd, rd); + max_sock = MAX(max_sock, fdlist[slot].fd); + } + slot++; + } + return max_sock; +} + +static void fdlist_check_ready (fd_set *rd, struct n3n_runtime_data *eee) { + int slot = 0; + // A linear scan is not ideal, but until we support things other than + // select() it will need to suffice + while(slot < MAX_HANDLES) { + if(fdlist[slot].fd == -1) { + slot++; + continue; + } + if(!FD_ISSET(fdlist[slot].fd, rd)) { + slot++; + continue; + } + + handle_fd(fdlist[slot].fd, fdlist[slot].proto, eee); + slot++; + } +} + static int setup_select (fd_set *rd, fd_set *wr, struct n3n_runtime_data *eee) { FD_ZERO(rd); FD_ZERO(wr); - int max_sock = 0; + int max_sock = fdlist_read_fd_set(rd); if(eee->sock >= 0) { FD_SET(eee->sock, rd); @@ -36,11 +138,6 @@ static int setup_select (fd_set *rd, fd_set *wr, struct n3n_runtime_data *eee) { } #endif -#ifndef _WIN32 - FD_SET(eee->device.fd, rd); - max_sock = MAX(max_sock, eee->device.fd); -#endif - max_sock = MAX( max_sock, slots_fdset( @@ -81,12 +178,7 @@ int mainloop_runonce (fd_set *rd, fd_set *wr, struct n3n_runtime_data *eee) { // One timestamp to use for this entire loop iteration // time_t now = time(NULL); -#ifndef _WIN32 - if((eee->device.fd != -1) && FD_ISSET(eee->device.fd, rd)) { - // read an ethernet frame from the TAP socket; write on the IP socket - edge_read_from_tap(eee); - } -#endif + fdlist_check_ready(rd, eee); int slots_ready = slots_fdset_loop( eee->mgmt_slots, @@ -120,3 +212,19 @@ int mainloop_runonce (fd_set *rd, fd_set *wr, struct n3n_runtime_data *eee) { return ready; } + +void mainloop_register_fd (int fd, enum fd_info_proto proto) { + int slot = fdlist_allocslot(fd, proto); + + // TODO: the moment this starts to fire, we need to revamp the + // implementation of the fdlist table + assert(slot != -1); +} + +void mainloop_unregister_fd (int fd) { + fdlist_freefd(fd); +} + +void n3n_initfuncs_mainloop () { + fdlist_zero(); +} diff --git a/src/mainloop.h b/src/mainloop.h deleted file mode 100644 index 159b762..0000000 --- a/src/mainloop.h +++ /dev/null @@ -1,14 +0,0 @@ -/** - * Copyright (C) Hamish Coleman - * - * non public structure and function definitions - */ - -#ifndef _MAINLOOP_H_ -#define _MAINLOOP_H_ - -#include // for n3n_runtime_data - -int mainloop_runonce (fd_set *, fd_set *, struct n3n_runtime_data *); - -#endif From 103c203cc8f95f2bb360c20bde08721bab2e6bb4 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Sun, 17 Nov 2024 17:12:09 +1100 Subject: [PATCH 27/85] Attempt to work around a startup segfault in the windows client --- src/sn_selection.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/sn_selection.c b/src/sn_selection.c index be7d749..77c3da8 100644 --- a/src/sn_selection.c +++ b/src/sn_selection.c @@ -136,6 +136,12 @@ int sn_selection_criterion_common_data_default (struct n3n_runtime_data *eee) { switch(eee->conf.sn_selection_strategy) { case SN_SELECTION_STRATEGY_LOAD: { + // something something Windows, something something Complete + if(!eee->pending_peers) { + eee->sn_selection_criterion_common_data = 0; + return 0; + } + SN_SELECTION_CRITERION_DATA_TYPE tmp = 0; tmp = HASH_COUNT(eee->pending_peers); From cb9baab36779735c7ace2779d457a577a800c0f1 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Sun, 17 Nov 2024 17:58:19 +1100 Subject: [PATCH 28/85] Open up the possiblity of using slots_accept on a foreign socket --- libs/connslot/connslot.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libs/connslot/connslot.c b/libs/connslot/connslot.c index 67f4227..108adfe 100644 --- a/libs/connslot/connslot.c +++ b/libs/connslot/connslot.c @@ -562,7 +562,7 @@ int slots_fdset(slots_t *slots, fd_set *readers, fd_set *writers) { return fdmax; } -int slots_accept(slots_t *slots, int listen_nr, enum conn_proto proto) { +int slots_accept(slots_t *slots, int fd, enum conn_proto proto) { int i; // TODO: remember previous checked slot and dont start at zero @@ -577,7 +577,7 @@ int slots_accept(slots_t *slots, int listen_nr, enum conn_proto proto) { return -2; } - int client = accept(slots->listen[listen_nr], NULL, 0); + int client = accept(fd, NULL, 0); if (client == -1) { return -1; } @@ -631,7 +631,7 @@ int slots_fdset_loop(slots_t *slots, fd_set *readers, fd_set *writers) { // A new connection // TODO: // - allow each listen socket to have a protocol - int slotnr = slots_accept(slots, i, CONN_PROTO_HTTP); + int slotnr = slots_accept(slots, slots->listen[i], CONN_PROTO_HTTP); switch (slotnr) { case -1: From 89516e3cc5f4d86d3298bec419ded3f9c4a6293b Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Sun, 17 Nov 2024 18:07:05 +1100 Subject: [PATCH 29/85] Allow reusing the slots listen logic without populating the connslot listen table --- libs/connslot/connslot.c | 43 +++++++++++++++++++++++++++++----------- libs/connslot/connslot.h | 2 ++ 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/libs/connslot/connslot.c b/libs/connslot/connslot.c index 108adfe..607f0a9 100644 --- a/libs/connslot/connslot.c +++ b/libs/connslot/connslot.c @@ -400,12 +400,7 @@ static int _slots_listen_find_empty(slots_t *slots) { return listen_nr; } -int slots_listen_tcp(slots_t *slots, int port, bool allow_remote) { - int listen_nr = _slots_listen_find_empty(slots); - if (listen_nr <0) { - return -2; - } - +int slots_create_listen_tcp(int port, bool allow_remote) { int server; #ifndef _WIN32 int on = 1; @@ -450,17 +445,26 @@ int slots_listen_tcp(slots_t *slots, int port, bool allow_remote) { return -1; } - slots->listen[listen_nr] = server; - return 0; + return server; } -#ifndef _WIN32 -int slots_listen_unix(slots_t *slots, char *path, int mode, int uid, int gid) { +int slots_listen_tcp(slots_t *slots, int port, bool allow_remote) { int listen_nr = _slots_listen_find_empty(slots); if (listen_nr <0) { return -2; } + int fd = slots_create_listen_tcp(port, allow_remote); + if(fd == -1) { + return fd; + } + + slots->listen[listen_nr] = fd; + return 0; +} + +#ifndef _WIN32 +int slots_create_listen_unix(char *path, int mode, int uid, int gid) { struct sockaddr_un addr; if (strlen(path) > sizeof(addr.sun_path) -1) { @@ -505,9 +509,24 @@ int slots_listen_unix(slots_t *slots, char *path, int mode, int uid, int gid) { return -1; } - slots->listen[listen_nr] = server; - return result; + return server; +} + +int slots_listen_unix(slots_t *slots, char *path, int mode, int uid, int gid) { + int listen_nr = _slots_listen_find_empty(slots); + if (listen_nr <0) { + return -2; + } + + int fd = slots_create_listen_unix(path, mode, uid, gid); + if(fd == -1) { + return fd; + } + + slots->listen[listen_nr] = fd; + return 0; } + #endif /* diff --git a/libs/connslot/connslot.h b/libs/connslot/connslot.h index 8773d4f..838b854 100644 --- a/libs/connslot/connslot.h +++ b/libs/connslot/connslot.h @@ -69,7 +69,9 @@ void conn_close(conn_t *); void slots_free(slots_t *slots); slots_t *slots_malloc(int nr_slots, size_t, size_t); +int slots_create_listen_tcp(int, bool); int slots_listen_tcp(slots_t *, int, bool); +int slots_create_listen_unix(char *, int, int, int); int slots_listen_unix(slots_t *, char *, int, int, int); void slots_listen_close(slots_t *); int slots_fdset(slots_t *, fd_set *, fd_set *); From c4e50cda1198f13e73b8697f13805bfe370bee7b Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Sun, 17 Nov 2024 18:22:59 +1100 Subject: [PATCH 30/85] Convert management port listening sockets to use mainloop registration --- apps/n3n-edge.c | 6 ++---- include/n3n/mainloop.h | 1 + src/edge_utils.c | 15 +++++++++++---- src/mainloop.c | 15 +++++++++++++-- 4 files changed, 27 insertions(+), 10 deletions(-) diff --git a/apps/n3n-edge.c b/apps/n3n-edge.c index 7fbee5d..fd19e7d 100644 --- a/apps/n3n-edge.c +++ b/apps/n3n-edge.c @@ -20,7 +20,6 @@ #include // for isspace -#include // for slots_listen_close #include // for errno #include // for required_argument, no_argument #include // for PRIu64 @@ -698,7 +697,7 @@ static void term_handler (int sig) { #endif #ifdef _WIN32 -struct n3n_runtime_data *windows_stop_eee; +int windows_stop_fd; // Note well, this gets called from a brand new thread, thus is completely // different to how signals work in POSIX @@ -719,7 +718,7 @@ BOOL WINAPI ConsoleCtrlHandler (DWORD sig) { // mainloop to notice that we are no longer wanting to run. // // something something, darkside - slots_listen_close(windows_stop_eee->mgmt_slots); + closehandle(windows_stop_fd); switch(sig) { case CTRL_CLOSE_EVENT: @@ -1080,7 +1079,6 @@ int main (int argc, char* argv[]) { signal(SIGINT, term_handler); #endif #ifdef _WIN32 - windows_stop_eee = eee; SetConsoleCtrlHandler(ConsoleCtrlHandler, TRUE); #endif diff --git a/include/n3n/mainloop.h b/include/n3n/mainloop.h index ca34e3e..ba73afe 100644 --- a/include/n3n/mainloop.h +++ b/include/n3n/mainloop.h @@ -17,6 +17,7 @@ enum __attribute__((__packed__)) fd_info_proto { fd_info_proto_unknown = 0, fd_info_proto_tuntap, + fd_info_proto_listen_http, }; int mainloop_runonce (fd_set *, fd_set *, struct n3n_runtime_data *); diff --git a/src/edge_utils.c b/src/edge_utils.c index 4ae05ca..89ac398 100644 --- a/src/edge_utils.c +++ b/src/edge_utils.c @@ -3296,10 +3296,17 @@ static int edge_init_sockets (struct n3n_runtime_data *eee) { } if(eee->conf.mgmt_port) { - if(slots_listen_tcp(eee->mgmt_slots, eee->conf.mgmt_port, false)!=0) { + int fd = slots_create_listen_tcp(eee->conf.mgmt_port, false); + if(fd < 0) { perror("slots_listen_tcp"); exit(1); } + mainloop_register_fd(fd, fd_info_proto_listen_http); +#ifdef _WIN32 + // HACK! + extern int windows_stop_fd; + windows_stop_fd = fd; +#endif } n3n_config_setup_sessiondir(&eee->conf); @@ -3308,8 +3315,7 @@ static int edge_init_sockets (struct n3n_runtime_data *eee) { char unixsock[1024]; snprintf(unixsock, sizeof(unixsock), "%s/mgmt", eee->conf.sessiondir); - int e = slots_listen_unix( - eee->mgmt_slots, + int fd = slots_create_listen_unix( unixsock, eee->conf.mgmt_sock_perms, eee->conf.userid, @@ -3318,10 +3324,11 @@ static int edge_init_sockets (struct n3n_runtime_data *eee) { // TODO: // - do we actually want to tie the user/group to the running pid? - if(e!=0) { + if(fd < 0) { perror("slots_listen_tcp"); exit(1); } + mainloop_register_fd(fd, fd_info_proto_listen_http); #endif #ifndef SKIP_MULTICAST_PEERS_DISCOVERY diff --git a/src/mainloop.c b/src/mainloop.c index 3ff1103..f8f9865 100644 --- a/src/mainloop.c +++ b/src/mainloop.c @@ -26,12 +26,23 @@ static void handle_fd (int fd, enum fd_info_proto proto, struct n3n_runtime_data case fd_info_proto_unknown: // should not happen! assert(false); - break; + return; + case fd_info_proto_tuntap: // read an ethernet frame from the TAP socket; write on the IP // socket edge_read_from_tap(eee); - break; + return; + + case fd_info_proto_listen_http: + int slotnr = slots_accept(eee->mgmt_slots, fd, CONN_PROTO_HTTP); + if(slotnr < 0) { + // TODO: increment error stats + return; + } + // TODO: Schedule slot for immediately reading + // FD_SET(eee->mgmt_slots->conn[slotnr].fd, rd); + return; } } From 9531fd914a8e13af5a07c39ecbb23d72300b7723 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Sun, 17 Nov 2024 18:28:55 +1100 Subject: [PATCH 31/85] Follow stricter variable creation rules for older compilers --- src/mainloop.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/mainloop.c b/src/mainloop.c index f8f9865..ce518fa 100644 --- a/src/mainloop.c +++ b/src/mainloop.c @@ -34,7 +34,7 @@ static void handle_fd (int fd, enum fd_info_proto proto, struct n3n_runtime_data edge_read_from_tap(eee); return; - case fd_info_proto_listen_http: + case fd_info_proto_listen_http: { int slotnr = slots_accept(eee->mgmt_slots, fd, CONN_PROTO_HTTP); if(slotnr < 0) { // TODO: increment error stats @@ -43,6 +43,7 @@ static void handle_fd (int fd, enum fd_info_proto proto, struct n3n_runtime_data // TODO: Schedule slot for immediately reading // FD_SET(eee->mgmt_slots->conn[slotnr].fd, rd); return; + } } } From a3651a05a6035116c9ffc0a4fa17371f1072a537 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Sun, 17 Nov 2024 18:30:09 +1100 Subject: [PATCH 32/85] Remember to include what you use --- include/n3n/mainloop.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/n3n/mainloop.h b/include/n3n/mainloop.h index ba73afe..0ae45ef 100644 --- a/include/n3n/mainloop.h +++ b/include/n3n/mainloop.h @@ -13,6 +13,7 @@ #define _MAINLOOP_H_ #include // for n3n_runtime_data +#include // for fd_set enum __attribute__((__packed__)) fd_info_proto { fd_info_proto_unknown = 0, From 055bf36ad108922965a9a3aa3a92b5a59aefd936 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Sun, 17 Nov 2024 18:49:04 +1100 Subject: [PATCH 33/85] Address compile errors from CI checks --- include/n3n/mainloop.h | 2 ++ libs/connslot/connslot.c | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/include/n3n/mainloop.h b/include/n3n/mainloop.h index 0ae45ef..8623aaf 100644 --- a/include/n3n/mainloop.h +++ b/include/n3n/mainloop.h @@ -13,7 +13,9 @@ #define _MAINLOOP_H_ #include // for n3n_runtime_data +#ifndef _WIN32 #include // for fd_set +#endif enum __attribute__((__packed__)) fd_info_proto { fd_info_proto_unknown = 0, diff --git a/libs/connslot/connslot.c b/libs/connslot/connslot.c index 607f0a9..8fa08eb 100644 --- a/libs/connslot/connslot.c +++ b/libs/connslot/connslot.c @@ -504,6 +504,10 @@ int slots_create_listen_unix(char *path, int mode, int uid, int gid) { result += chown(path, uid, gid); } + if(result != 0) { + return -1; + } + // backlog of 1 - low, but sheds load quickly when we run out of slots if (listen(server, 1) < 0) { return -1; From a35f1e5d9c1429c5aa4e94d9cf356b9b41e73090 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Sun, 17 Nov 2024 19:13:47 +1100 Subject: [PATCH 34/85] Use correct strange windows close function - remember nothing is a file --- apps/n3n-edge.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/n3n-edge.c b/apps/n3n-edge.c index fd19e7d..b442994 100644 --- a/apps/n3n-edge.c +++ b/apps/n3n-edge.c @@ -718,7 +718,7 @@ BOOL WINAPI ConsoleCtrlHandler (DWORD sig) { // mainloop to notice that we are no longer wanting to run. // // something something, darkside - closehandle(windows_stop_fd); + closesocket(windows_stop_fd); switch(sig) { case CTRL_CLOSE_EVENT: From 8b2c0ed868593d4abbefa93af3d4ac0419798f5a Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Tue, 19 Nov 2024 19:14:57 +1100 Subject: [PATCH 35/85] Attempt to fix windows CI build issue --- apps/n3n-edge.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/n3n-edge.c b/apps/n3n-edge.c index b442994..5b156d7 100644 --- a/apps/n3n-edge.c +++ b/apps/n3n-edge.c @@ -697,7 +697,7 @@ static void term_handler (int sig) { #endif #ifdef _WIN32 -int windows_stop_fd; +extern int windows_stop_fd; // Note well, this gets called from a brand new thread, thus is completely // different to how signals work in POSIX From 359a21f8a980fe48508a81d85f8a391c36d91301 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Tue, 19 Nov 2024 19:34:26 +1100 Subject: [PATCH 36/85] Convert supernode to use the same console handler hack that edge uses --- apps/n3n-supernode.c | 55 +++++++++++++++++++++++++++++++++----------- 1 file changed, 41 insertions(+), 14 deletions(-) diff --git a/apps/n3n-supernode.c b/apps/n3n-supernode.c index 94409b6..fcafb83 100644 --- a/apps/n3n-supernode.c +++ b/apps/n3n-supernode.c @@ -351,15 +351,10 @@ static void n3n_sn_config (int argc, char **argv, char *defname, struct n3n_runt /* *************************************************** */ -static bool keep_running = true; +static bool keep_on_running = true; -#if defined(__linux__) || defined(_WIN32) -#ifdef _WIN32 -BOOL WINAPI term_handler (DWORD sig) -#else -static void term_handler (int sig) -#endif -{ +#ifndef _WIN32 +static void term_handler (int sig) { static int called = 0; if(called) { @@ -370,12 +365,44 @@ static void term_handler (int sig) called = 1; } - keep_running = false; + keep_on_running = false; +} +#endif + #ifdef _WIN32 +extern int windows_stop_fd; + +// Note well, this gets called from a brand new thread, thus is completely +// different to how signals work in POSIX +BOOL WINAPI ConsoleCtrlHandler (DWORD sig) { + // Tell the mainloop to exit next time it wakes + keep_on_running = false; + + traceEvent(TRACE_INFO, "starting stopping"); + // The windows environment claims to support signals, but they dont + // interrupt a running select() statement. Also, this console handler + // is run in its own thread, so it is also not interrupting the select() + // This is clearly contrary to how select was designed to be used and it + // makes process termination annoying, so we need a workaround. + // + // Since windows usually has a managment TCP port listening in the + // select fdset, we can close that - this immediately causes the select + // to return with activity on that file descriptor and allows the + // mainloop to notice that we are no longer wanting to run. + // + // something something, darkside + closesocket(windows_stop_fd); + + switch(sig) { + case CTRL_CLOSE_EVENT: + case CTRL_LOGOFF_EVENT: + case CTRL_SHUTDOWN_EVENT: + // Will terminate us after we return, blocking it to cleanup + Sleep(INFINITE); + } return(TRUE); -#endif } -#endif /* defined(__linux__) || defined(_WIN32) */ +#endif /* *************************************************** */ @@ -606,15 +633,15 @@ int main (int argc, char * argv[]) { traceEvent(TRACE_NORMAL, "supernode started"); -#ifdef __linux__ +#ifndef _WIN32 signal(SIGPIPE, SIG_IGN); signal(SIGTERM, term_handler); signal(SIGINT, term_handler); #endif #ifdef _WIN32 - SetConsoleCtrlHandler(term_handler, TRUE); + SetConsoleCtrlHandler(ConsoleCtrlHandler, TRUE); #endif - sss_node.keep_running = &keep_running; + sss_node.keep_running = &keep_on_running; return run_sn_loop(&sss_node); } From e319a5b576d14981d34e464f797f35f5dcd6f797 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Tue, 19 Nov 2024 19:34:55 +1100 Subject: [PATCH 37/85] Revert "Attempt to fix windows CI build issue" This reverts commit 8b2c0ed868593d4abbefa93af3d4ac0419798f5a. --- apps/n3n-edge.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/n3n-edge.c b/apps/n3n-edge.c index 5b156d7..b442994 100644 --- a/apps/n3n-edge.c +++ b/apps/n3n-edge.c @@ -697,7 +697,7 @@ static void term_handler (int sig) { #endif #ifdef _WIN32 -extern int windows_stop_fd; +int windows_stop_fd; // Note well, this gets called from a brand new thread, thus is completely // different to how signals work in POSIX From 21eea53c88907e151d21516e2d6e65f9fb299193 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Tue, 19 Nov 2024 19:41:31 +1100 Subject: [PATCH 38/85] Continue hacking at the windows stop hack. Wouldnt it be nice to simply use a POSIX compliant system.. oh, wait.. --- apps/n3n-edge.c | 2 +- apps/n3n-supernode.c | 6 ++++++ src/edge_utils.c | 7 ++++++- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/apps/n3n-edge.c b/apps/n3n-edge.c index b442994..5b156d7 100644 --- a/apps/n3n-edge.c +++ b/apps/n3n-edge.c @@ -697,7 +697,7 @@ static void term_handler (int sig) { #endif #ifdef _WIN32 -int windows_stop_fd; +extern int windows_stop_fd; // Note well, this gets called from a brand new thread, thus is completely // different to how signals work in POSIX diff --git a/apps/n3n-supernode.c b/apps/n3n-supernode.c index fcafb83..b39a629 100644 --- a/apps/n3n-supernode.c +++ b/apps/n3n-supernode.c @@ -573,6 +573,12 @@ int main (int argc, char * argv[]) { } traceEvent(TRACE_NORMAL, "supernode is listening on TCP %u (management)", sss_node.conf.mgmt_port); } +#ifdef _WIN32 + // HACK! + // Remove this once the supernode users mainloop and it also supports + // stopping on windows + windows_stop_fd = sss_node.mgmt_slots->listen[0]; +#endif n3n_config_setup_sessiondir(&sss_node.conf); diff --git a/src/edge_utils.c b/src/edge_utils.c index 89ac398..9e608ad 100644 --- a/src/edge_utils.c +++ b/src/edge_utils.c @@ -3288,6 +3288,12 @@ void edge_term (struct n3n_runtime_data * eee) { /* ************************************** */ +#ifdef _WIN32 +// HACK! +// Remove this once the mainloop supports stopping on windows +int windows_stop_fd; +#endif + static int edge_init_sockets (struct n3n_runtime_data *eee) { eee->mgmt_slots = slots_malloc(5, 5000, 500); @@ -3304,7 +3310,6 @@ static int edge_init_sockets (struct n3n_runtime_data *eee) { mainloop_register_fd(fd, fd_info_proto_listen_http); #ifdef _WIN32 // HACK! - extern int windows_stop_fd; windows_stop_fd = fd; #endif } From 85f31c3f8a05ae0c7bcd92a4bd2cadeb5326450d Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Tue, 19 Nov 2024 20:05:25 +1100 Subject: [PATCH 39/85] Fix spacing bug with metrics generated --- src/metrics.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/metrics.c b/src/metrics.c index 5d28672..747ef96 100644 --- a/src/metrics.c +++ b/src/metrics.c @@ -66,7 +66,7 @@ static void metrics_render_llu32 (strbuf_t **reply, struct n3n_metrics_module *m if(info->desc) { sb_reprintf(reply, "# HELP "); metrics_name(reply, module->name, info->name); - sb_reprintf(reply, "%s\n", info->desc); + sb_reprintf(reply, " %s\n", info->desc); } for(int i = 0; info->items[i].val1; i++) { From 873a6016f6d9214d5656ee889bca0ae40dbc31fb Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Tue, 19 Nov 2024 20:30:27 +1100 Subject: [PATCH 40/85] Add a metrics callback type for dynamic lists of metrics --- include/n3n/metrics.h | 12 ++++++++++ src/metrics.c | 55 +++++++++++++++++++++++++++++++++++-------- 2 files changed, 57 insertions(+), 10 deletions(-) diff --git a/include/n3n/metrics.h b/include/n3n/metrics.h index 285f32b..80d7216 100644 --- a/include/n3n/metrics.h +++ b/include/n3n/metrics.h @@ -14,6 +14,7 @@ enum __attribute__((__packed__)) n3n_metrics_items_type { n3n_metrics_type_invalid = 0, n3n_metrics_type_uint32, // items_uint32 is valid n3n_metrics_type_llu32, + n3n_metrics_type_cb, }; // The simplest type of metrics: everything is the same storage type, there are @@ -52,6 +53,7 @@ struct n3n_metrics_module { union { const struct n3n_metrics_items_uint32 *items_uint32; const struct n3n_metrics_items_llu32 *items_llu32; + const void (*cb)(strbuf_t **, const struct n3n_metrics_module *); }; const enum n3n_metrics_items_type type; }; @@ -59,6 +61,16 @@ struct n3n_metrics_module { // Register a block of metrics void n3n_metrics_register (struct n3n_metrics_module *); +// Helper to assist with rendering during n3n_metrics_type_cb +void n3n_metrics_render_u32tags ( + strbuf_t **reply, + const struct n3n_metrics_module *module, + const char *name, + const int offset, + const int tags, // The number of following tag+val pairs + ... + ); + // Render all the metrics into a strbuf void n3n_metrics_render (strbuf_t **reply); diff --git a/src/metrics.c b/src/metrics.c index 747ef96..0b3ab71 100644 --- a/src/metrics.c +++ b/src/metrics.c @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -60,6 +61,42 @@ static void metrics_render_uint32 (strbuf_t **reply, struct n3n_metrics_module * } } +void n3n_metrics_render_u32tags ( + strbuf_t **reply, + const struct n3n_metrics_module *module, + const char *name, + const int offset, + const int tags, + ...) { + va_list ap; + + metrics_name(reply, module->name, name); + sb_reprintf(reply, "{session=\"%s\"", sessionname); + + int count = tags; + va_start(ap, tags); + while(count) { + if(count) { + sb_reprintf(reply,","); + } + char *tag = va_arg(ap, char *); + char *val = va_arg(ap, char *); + sb_reprintf(reply,"%s=\"%s\"", tag, val); + count--; + } + va_end(ap); + + sb_reprintf(reply,"} "); + + metric_stringify_uint32( + reply, + offset, + module->data + ); + sb_reprintf(reply, "\n"); +} + + static void metrics_render_llu32 (strbuf_t **reply, struct n3n_metrics_module *module) { const struct n3n_metrics_items_llu32 *info = module->items_llu32; @@ -73,22 +110,17 @@ static void metrics_render_llu32 (strbuf_t **reply, struct n3n_metrics_module *m // TODO: // - " TYPE name type\n" // - " UNIT name type\n" - metrics_name(reply, module->name, info->name); - sb_reprintf( + n3n_metrics_render_u32tags( reply, - "{session=\"%s\",%s=\"%s\",%s=\"%s\"} ", - sessionname, + module, + info->name, + info->items[i].offset, + 2, // number of tag+val pairs info->name1, info->items[i].val1, info->name2, info->items[i].val2 ); - metric_stringify_uint32( - reply, - info->items[i].offset, - module->data - ); - sb_reprintf(reply, "\n"); } } @@ -109,6 +141,9 @@ void n3n_metrics_render (strbuf_t **reply) { case n3n_metrics_type_llu32: metrics_render_llu32(reply, module); break; + case n3n_metrics_type_cb: + module->cb(reply, module); + break; } } } From 4779628105bdf7b8eee2b22568745f7ddcd97692 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Tue, 19 Nov 2024 20:30:57 +1100 Subject: [PATCH 41/85] Track and output some metrics for the mainloop fd list --- src/mainloop.c | 51 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/src/mainloop.c b/src/mainloop.c index ce518fa..14aa0a8 100644 --- a/src/mainloop.c +++ b/src/mainloop.c @@ -8,6 +8,7 @@ #include // for slots_fdset #include // for n3n_runtime_data #include // for fd_info_proto +#include #include // for traceEvent #include #include @@ -47,9 +48,16 @@ static void handle_fd (int fd, enum fd_info_proto proto, struct n3n_runtime_data } } +static char *proto_str[] = { + "?", + "tuntap", + "listen_http", +}; + struct fd_info { - int fd; - enum fd_info_proto proto; + int fd; // The file descriptor for this connection + int stats_reads; // The number of read to read events + enum fd_info_proto proto; // What protocol to use on a read event }; // A static array of known file descriptors will not scale once full TCP @@ -58,6 +66,42 @@ struct fd_info { static struct fd_info fdlist[MAX_HANDLES]; static int fdlist_next_search; +static void metrics_callback (strbuf_t **reply, const struct n3n_metrics_module *module) { + int slot = 0; + char buf[16]; + while(slot < MAX_HANDLES) { + if(fdlist[slot].fd == -1) { + slot++; + continue; + } + + snprintf(buf, sizeof(buf), "%i", fdlist[slot].fd); + + n3n_metrics_render_u32tags( + reply, + module, + "fd_reads", + (char *)&fdlist[slot].stats_reads - (char *)&fdlist, + 2, // number of tag+val pairs + "fd", + buf, + "proto", + proto_str[fdlist[slot].proto] + ); + // TODO: + // - do we need to keep each fd lifecycle clear by tracking and + // outputting the open timestamp? + slot++; + } +} + +static struct n3n_metrics_module metrics_module = { + .name = "mainloop", + .data = &fdlist, + .cb = &metrics_callback, + .type = n3n_metrics_type_cb, +}; + // Used only to initialise the array at startup static void fdlist_zero () { int slot = 0; @@ -76,6 +120,7 @@ static int fdlist_allocslot (int fd, enum fd_info_proto proto) { if(fdlist[slot].fd == -1) { fdlist[slot].fd = fd; fdlist[slot].proto = proto; + fdlist[slot].stats_reads = 0; fdlist_next_search = slot + 1; return slot; } @@ -128,6 +173,7 @@ static void fdlist_check_ready (fd_set *rd, struct n3n_runtime_data *eee) { continue; } + fdlist[slot].stats_reads++; handle_fd(fdlist[slot].fd, fdlist[slot].proto, eee); slot++; } @@ -239,4 +285,5 @@ void mainloop_unregister_fd (int fd) { void n3n_initfuncs_mainloop () { fdlist_zero(); + n3n_metrics_register(&metrics_module); } From 445067c6ec33de7ecddbdace2cf25881cab42ac3 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Tue, 19 Nov 2024 20:37:37 +1100 Subject: [PATCH 42/85] Address lint concerns --- include/n3n/metrics.h | 14 +++++++------- src/metrics.c | 13 +++++++------ 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/include/n3n/metrics.h b/include/n3n/metrics.h index 80d7216..3b20a90 100644 --- a/include/n3n/metrics.h +++ b/include/n3n/metrics.h @@ -63,13 +63,13 @@ void n3n_metrics_register (struct n3n_metrics_module *); // Helper to assist with rendering during n3n_metrics_type_cb void n3n_metrics_render_u32tags ( - strbuf_t **reply, - const struct n3n_metrics_module *module, - const char *name, - const int offset, - const int tags, // The number of following tag+val pairs - ... - ); + strbuf_t **reply, + const struct n3n_metrics_module *module, + const char *name, + const int offset, + const int tags, // The number of following tag+val pairs + ... +); // Render all the metrics into a strbuf void n3n_metrics_render (strbuf_t **reply); diff --git a/src/metrics.c b/src/metrics.c index 0b3ab71..1696621 100644 --- a/src/metrics.c +++ b/src/metrics.c @@ -62,12 +62,13 @@ static void metrics_render_uint32 (strbuf_t **reply, struct n3n_metrics_module * } void n3n_metrics_render_u32tags ( - strbuf_t **reply, - const struct n3n_metrics_module *module, - const char *name, - const int offset, - const int tags, - ...) { + strbuf_t **reply, + const struct n3n_metrics_module *module, + const char *name, + const int offset, + const int tags, + ...) { + va_list ap; metrics_name(reply, module->name, name); From ff1e97353d4cad79f34afde7d64777985aefad76 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Tue, 19 Nov 2024 21:09:47 +1100 Subject: [PATCH 43/85] Start handling multicast VPN proto RX in the mainloop --- include/n3n/mainloop.h | 1 + src/edge_utils.c | 22 +++++++--------------- src/mainloop.c | 30 ++++++++++++++++++++++-------- src/management.c | 2 +- 4 files changed, 31 insertions(+), 24 deletions(-) diff --git a/include/n3n/mainloop.h b/include/n3n/mainloop.h index 8623aaf..4b7aed8 100644 --- a/include/n3n/mainloop.h +++ b/include/n3n/mainloop.h @@ -21,6 +21,7 @@ enum __attribute__((__packed__)) fd_info_proto { fd_info_proto_unknown = 0, fd_info_proto_tuntap, fd_info_proto_listen_http, + fd_info_proto_v3udp, }; int mainloop_runonce (fd_set *, fd_set *, struct n3n_runtime_data *); diff --git a/src/edge_utils.c b/src/edge_utils.c index 9e608ad..bc1258e 100644 --- a/src/edge_utils.c +++ b/src/edge_utils.c @@ -3114,19 +3114,6 @@ int run_edge_loop (struct n3n_runtime_data *eee) { } } } - -#ifndef SKIP_MULTICAST_PEERS_DISCOVERY - if((eee->udp_multicast_sock != -1) && FD_ISSET(eee->udp_multicast_sock, &readers)) { - edge_read_proto3_udp( - eee, - eee->udp_multicast_sock, - pktbuf, - sizeof(pktbuf), - now - ); - } -#endif - } // check for timed out slots @@ -3232,8 +3219,10 @@ void edge_term (struct n3n_runtime_data * eee) { closesocket(eee->sock); #ifndef SKIP_MULTICAST_PEERS_DISCOVERY - if(eee->udp_multicast_sock >= 0) + if(eee->udp_multicast_sock >= 0) { closesocket(eee->udp_multicast_sock); + mainloop_unregister_fd(eee->udp_multicast_sock); + } #endif clear_peer_list(&eee->pending_peers); @@ -3337,8 +3326,10 @@ static int edge_init_sockets (struct n3n_runtime_data *eee) { #endif #ifndef SKIP_MULTICAST_PEERS_DISCOVERY - if(eee->udp_multicast_sock >= 0) + if(eee->udp_multicast_sock >= 0) { closesocket(eee->udp_multicast_sock); + mainloop_unregister_fd(eee->udp_multicast_sock); + } /* Populate the multicast group for local edge */ eee->multicast_peer.family = AF_INET; @@ -3371,6 +3362,7 @@ static int edge_init_sockets (struct n3n_runtime_data *eee) { setsockopt(eee->udp_multicast_sock, SOL_SOCKET, SO_REUSEPORT, &enable_reuse, sizeof(enable_reuse)); #endif + mainloop_register_fd(eee->udp_multicast_sock, fd_info_proto_v3udp); #endif return(0); diff --git a/src/mainloop.c b/src/mainloop.c index 14aa0a8..11323c6 100644 --- a/src/mainloop.c +++ b/src/mainloop.c @@ -7,6 +7,7 @@ #include #include // for slots_fdset #include // for n3n_runtime_data +#include // for edge_read_proto3_udp #include // for fd_info_proto #include #include // for traceEvent @@ -22,7 +23,7 @@ #include "minmax.h" // for min, max #include "n2n_define.h" -static void handle_fd (int fd, enum fd_info_proto proto, struct n3n_runtime_data *eee) { +static void handle_fd (const time_t now, int fd, enum fd_info_proto proto, struct n3n_runtime_data *eee) { switch(proto) { case fd_info_proto_unknown: // should not happen! @@ -45,13 +46,26 @@ static void handle_fd (int fd, enum fd_info_proto proto, struct n3n_runtime_data // FD_SET(eee->mgmt_slots->conn[slotnr].fd, rd); return; } + + case fd_info_proto_v3udp: { + uint8_t pktbuf[N2N_PKT_BUF_SIZE]; + edge_read_proto3_udp( + eee, + fd, + pktbuf, + sizeof(pktbuf), + now + ); + return; + } } } static char *proto_str[] = { - "?", - "tuntap", - "listen_http", + [fd_info_proto_unknown] = "?", + [fd_info_proto_tuntap] = "tuntap", + [fd_info_proto_listen_http] = "listen_http", + [fd_info_proto_v3udp] = "v3udp", }; struct fd_info { @@ -159,7 +173,7 @@ static int fdlist_read_fd_set (fd_set *rd) { return max_sock; } -static void fdlist_check_ready (fd_set *rd, struct n3n_runtime_data *eee) { +static void fdlist_check_ready (fd_set *rd, const time_t now, struct n3n_runtime_data *eee) { int slot = 0; // A linear scan is not ideal, but until we support things other than // select() it will need to suffice @@ -174,7 +188,7 @@ static void fdlist_check_ready (fd_set *rd, struct n3n_runtime_data *eee) { } fdlist[slot].stats_reads++; - handle_fd(fdlist[slot].fd, fdlist[slot].proto, eee); + handle_fd(now, fdlist[slot].fd, fdlist[slot].proto, eee); slot++; } } @@ -234,9 +248,9 @@ int mainloop_runonce (fd_set *rd, fd_set *wr, struct n3n_runtime_data *eee) { } // One timestamp to use for this entire loop iteration - // time_t now = time(NULL); + time_t now = time(NULL); - fdlist_check_ready(rd, eee); + fdlist_check_ready(rd, now, eee); int slots_ready = slots_fdset_loop( eee->mgmt_slots, diff --git a/src/management.c b/src/management.c index 967bf43..040cd57 100644 --- a/src/management.c +++ b/src/management.c @@ -1086,7 +1086,7 @@ static void render_metrics_page (struct n3n_runtime_data *eee, conn_t *conn) { // Update the reply buffer after last potential realloc conn->reply = conn->request; - generate_http_headers(conn, "text/plain", 501); + generate_http_headers(conn, "text/plain", 200); } #include "management_index.html.h" From ae42e0c1c27e28e3165cc5659aab143104fb517d Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Tue, 19 Nov 2024 21:20:23 +1100 Subject: [PATCH 44/85] Multicast listening is now unconditional Unsure if this should be the case, so have added the note in the path to opening the socket --- src/edge_utils.c | 6 ++++++ src/mainloop.c | 7 ------- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/edge_utils.c b/src/edge_utils.c index bc1258e..b7cc6d4 100644 --- a/src/edge_utils.c +++ b/src/edge_utils.c @@ -3326,6 +3326,12 @@ static int edge_init_sockets (struct n3n_runtime_data *eee) { #endif #ifndef SKIP_MULTICAST_PEERS_DISCOVERY + // TODO: + // We used to gate multicast listening on: + // if((eee->conf.allow_p2p) + // && (eee->conf.preferred_sock.family == (uint8_t)AF_INVALID)) + // So, perhaps we should do that here? + if(eee->udp_multicast_sock >= 0) { closesocket(eee->udp_multicast_sock); mainloop_unregister_fd(eee->udp_multicast_sock); diff --git a/src/mainloop.c b/src/mainloop.c index 11323c6..01c83d0 100644 --- a/src/mainloop.c +++ b/src/mainloop.c @@ -202,13 +202,6 @@ static int setup_select (fd_set *rd, fd_set *wr, struct n3n_runtime_data *eee) { FD_SET(eee->sock, rd); max_sock = MAX(max_sock, eee->sock); } -#ifndef SKIP_MULTICAST_PEERS_DISCOVERY - if((eee->conf.allow_p2p) - && (eee->conf.preferred_sock.family == (uint8_t)AF_INVALID)) { - FD_SET(eee->udp_multicast_sock, rd); - max_sock = MAX(max_sock, eee->udp_multicast_sock); - } -#endif max_sock = MAX( max_sock, From 9646b677eb715fa823196dd6252249509d3d8e40 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Tue, 19 Nov 2024 21:23:16 +1100 Subject: [PATCH 45/85] Remove callback until an example user shows how to use and test --- include/n2n_typedefs.h | 2 -- src/edge_utils.c | 3 --- 2 files changed, 5 deletions(-) diff --git a/include/n2n_typedefs.h b/include/n2n_typedefs.h index b82f7fe..6add434 100644 --- a/include/n2n_typedefs.h +++ b/include/n2n_typedefs.h @@ -399,8 +399,6 @@ typedef struct n2n_edge_callbacks { /* Called periodically in the main loop. */ void (*main_loop_period)(struct n3n_runtime_data *eee, time_t now); - /* Called when a new socket to supernode is created. */ - void (*sock_opened)(struct n3n_runtime_data *eee); } n2n_edge_callbacks_t; /* *************************************************** */ diff --git a/src/edge_utils.c b/src/edge_utils.c index b7cc6d4..d497e52 100644 --- a/src/edge_utils.c +++ b/src/edge_utils.c @@ -426,9 +426,6 @@ void supernode_connect (struct n3n_runtime_data *eee) { sock_to_cstr(sockbuf, &local_sock)); } } - - if(eee->cb.sock_opened) - eee->cb.sock_opened(eee); } // REVISIT: add mgmt port notification to listener for better mgmt port From 69ae25596707a858cfee483000b05adafe00aa3a Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Tue, 19 Nov 2024 21:33:39 +1100 Subject: [PATCH 46/85] Fix infinite loop when freeing fdlist --- src/mainloop.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mainloop.c b/src/mainloop.c index 01c83d0..80a61fa 100644 --- a/src/mainloop.c +++ b/src/mainloop.c @@ -148,6 +148,7 @@ static void fdlist_freefd (int fd) { int slot = 0; while(slot < MAX_HANDLES) { if(fdlist[slot].fd != fd) { + slot++; continue; } fdlist[slot].fd = -1; From 323f6c0981c343a21882ccd0d8d8ed56b430177c Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Tue, 19 Nov 2024 21:38:22 +1100 Subject: [PATCH 47/85] Handle UDP VPN proto RX in the mainloop --- src/edge_utils.c | 53 +++++++++++++++++++++++------------------------- src/mainloop.c | 5 ----- 2 files changed, 25 insertions(+), 33 deletions(-) diff --git a/src/edge_utils.c b/src/edge_utils.c index d497e52..026e9ad 100644 --- a/src/edge_utils.c +++ b/src/edge_utils.c @@ -426,6 +426,10 @@ void supernode_connect (struct n3n_runtime_data *eee) { sock_to_cstr(sockbuf, &local_sock)); } } + + if(!eee->conf.connect_tcp) { + mainloop_register_fd(eee->sock, fd_info_proto_v3udp); + } } // REVISIT: add mgmt port notification to listener for better mgmt port @@ -442,6 +446,7 @@ void supernode_disconnect (struct n3n_runtime_data *eee) { } if(eee->sock >= 0) { closesocket(eee->sock); + mainloop_unregister_fd(eee->sock); eee->sock = -1; traceEvent(TRACE_DEBUG, "closed"); } @@ -3080,35 +3085,25 @@ int run_edge_loop (struct n3n_runtime_data *eee) { if(rc > 0) { // any or all of the FDs could have input; check them all - // external packets - if((eee->sock != -1) && FD_ISSET(eee->sock, &readers)) { - if(!eee->conf.connect_tcp) { - edge_read_proto3_udp( - eee, - eee->sock, - pktbuf, - sizeof(pktbuf), - now - ); - } else { - edge_read_proto3_tcp( - eee, - eee->sock, - pktbuf, - &expected, - &position, - now - ); + // external TCP packets + if((eee->conf.connect_tcp) && (eee->sock != -1) && FD_ISSET(eee->sock, &readers)) { + edge_read_proto3_tcp( + eee, + eee->sock, + pktbuf, + &expected, + &position, + now + ); - if((expected >= N2N_PKT_BUF_SIZE) || (position >= N2N_PKT_BUF_SIZE)) { - // something went wrong, possibly even before - // e.g. connection failure/closure in the middle of transmission (between len & data) - supernode_disconnect(eee); - eee->sn_wait = 1; + if((expected >= N2N_PKT_BUF_SIZE) || (position >= N2N_PKT_BUF_SIZE)) { + // something went wrong, possibly even before + // e.g. connection failure/closure in the middle of transmission (between len & data) + supernode_disconnect(eee); + eee->sn_wait = 1; - expected = sizeof(uint16_t); - position = 0; - } + expected = sizeof(uint16_t); + position = 0; } } } @@ -3212,8 +3207,10 @@ void edge_term (struct n3n_runtime_data * eee) { resolve_cancel_thread(eee->resolve_parameter); - if(eee->sock >= 0) + if(eee->sock >= 0) { closesocket(eee->sock); + mainloop_unregister_fd(eee->sock); + } #ifndef SKIP_MULTICAST_PEERS_DISCOVERY if(eee->udp_multicast_sock >= 0) { diff --git a/src/mainloop.c b/src/mainloop.c index 80a61fa..4941014 100644 --- a/src/mainloop.c +++ b/src/mainloop.c @@ -199,11 +199,6 @@ static int setup_select (fd_set *rd, fd_set *wr, struct n3n_runtime_data *eee) { FD_ZERO(wr); int max_sock = fdlist_read_fd_set(rd); - if(eee->sock >= 0) { - FD_SET(eee->sock, rd); - max_sock = MAX(max_sock, eee->sock); - } - max_sock = MAX( max_sock, slots_fdset( From 30b832fa8876f12a34de62513768fd8ba92d7ede Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Tue, 19 Nov 2024 22:06:41 +1100 Subject: [PATCH 48/85] Fix TCP RX path after UDP mainloop conversion --- src/mainloop.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/mainloop.c b/src/mainloop.c index 4941014..1fd052a 100644 --- a/src/mainloop.c +++ b/src/mainloop.c @@ -199,6 +199,12 @@ static int setup_select (fd_set *rd, fd_set *wr, struct n3n_runtime_data *eee) { FD_ZERO(wr); int max_sock = fdlist_read_fd_set(rd); + // HACK - until this mainloop supports v3tcp proto sockets + if((eee->conf.connect_tcp) && (eee->sock != -1)) { + FD_SET(eee->sock, rd); + max_sock = MAX(max_sock, eee->sock); + } + max_sock = MAX( max_sock, slots_fdset( From 4c10b7a3bcade36298a8eb9a98537de6eb17b266 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Tue, 19 Nov 2024 22:06:58 +1100 Subject: [PATCH 49/85] Attempt to fix compile issue with macos src/mainloop.c:115:11: error: incompatible function pointer types initializing 'const void (*)(strbuf_t **, const struct n3n_metrics_module *)' (aka 'const void (*)(struct strbuf **, const struct n3n_metrics_module *)') with an expression of type 'void (*)(strbuf_t **, const struct n3n_metrics_module *)' (aka 'void (*)(struct strbuf **, const struct n3n_metrics_module *)') [-Wincompatible-function-pointer-types] .cb = &metrics_callback, --- include/n3n/metrics.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/n3n/metrics.h b/include/n3n/metrics.h index 3b20a90..dd6d242 100644 --- a/include/n3n/metrics.h +++ b/include/n3n/metrics.h @@ -53,7 +53,7 @@ struct n3n_metrics_module { union { const struct n3n_metrics_items_uint32 *items_uint32; const struct n3n_metrics_items_llu32 *items_llu32; - const void (*cb)(strbuf_t **, const struct n3n_metrics_module *); + void (*cb)(strbuf_t **, const struct n3n_metrics_module *); }; const enum n3n_metrics_items_type type; }; From c4ff2bf65132ac0e9fddd9bc33aef029d0e4c405 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Thu, 21 Nov 2024 19:43:08 +1100 Subject: [PATCH 50/85] Generate more mainloop metrics --- src/mainloop.c | 116 ++++++++++++++++++++++++++++++++----------------- src/metrics.c | 11 ++++- 2 files changed, 86 insertions(+), 41 deletions(-) diff --git a/src/mainloop.c b/src/mainloop.c index 1fd052a..01c9a70 100644 --- a/src/mainloop.c +++ b/src/mainloop.c @@ -23,43 +23,32 @@ #include "minmax.h" // for min, max #include "n2n_define.h" -static void handle_fd (const time_t now, int fd, enum fd_info_proto proto, struct n3n_runtime_data *eee) { - switch(proto) { - case fd_info_proto_unknown: - // should not happen! - assert(false); - return; - - case fd_info_proto_tuntap: - // read an ethernet frame from the TAP socket; write on the IP - // socket - edge_read_from_tap(eee); - return; - - case fd_info_proto_listen_http: { - int slotnr = slots_accept(eee->mgmt_slots, fd, CONN_PROTO_HTTP); - if(slotnr < 0) { - // TODO: increment error stats - return; - } - // TODO: Schedule slot for immediately reading - // FD_SET(eee->mgmt_slots->conn[slotnr].fd, rd); - return; - } - - case fd_info_proto_v3udp: { - uint8_t pktbuf[N2N_PKT_BUF_SIZE]; - edge_read_proto3_udp( - eee, - fd, - pktbuf, - sizeof(pktbuf), - now - ); - return; - } - } -} +static struct metrics { + uint32_t mainloop; // mainloop_runonce() is called + uint32_t register_fd; // mainloop_register_fd() is called + uint32_t unregister_fd; // mainloop_unregister_fd() is called +} metrics; + +static struct n3n_metrics_items_llu32 metrics_items = { + .name = "count", + .desc = "Track the events in the lifecycle of mainloop objects", + .name1 = "event", + .items = { + { + .val1 = "mainloop", + .offset = offsetof(struct metrics, mainloop), + }, + { + .val1 = "register_fd", + .offset = offsetof(struct metrics, register_fd), + }, + { + .val1 = "unregister_fd", + .offset = offsetof(struct metrics, unregister_fd), + }, + { }, + }, +}; static char *proto_str[] = { [fd_info_proto_unknown] = "?", @@ -109,13 +98,20 @@ static void metrics_callback (strbuf_t **reply, const struct n3n_metrics_module } } -static struct n3n_metrics_module metrics_module = { +static struct n3n_metrics_module metrics_module_dynamic = { .name = "mainloop", .data = &fdlist, .cb = &metrics_callback, .type = n3n_metrics_type_cb, }; +static struct n3n_metrics_module metrics_module_static = { + .name = "mainloop", + .data = &metrics, + .items_llu32 = &metrics_items, + .type = n3n_metrics_type_llu32, +}; + // Used only to initialise the array at startup static void fdlist_zero () { int slot = 0; @@ -174,6 +170,44 @@ static int fdlist_read_fd_set (fd_set *rd) { return max_sock; } +static void handle_fd (const time_t now, int fd, enum fd_info_proto proto, struct n3n_runtime_data *eee) { + switch(proto) { + case fd_info_proto_unknown: + // should not happen! + assert(false); + return; + + case fd_info_proto_tuntap: + // read an ethernet frame from the TAP socket; write on the IP + // socket + edge_read_from_tap(eee); + return; + + case fd_info_proto_listen_http: { + int slotnr = slots_accept(eee->mgmt_slots, fd, CONN_PROTO_HTTP); + if(slotnr < 0) { + // TODO: increment error stats + return; + } + // TODO: Schedule slot for immediately reading + // FD_SET(eee->mgmt_slots->conn[slotnr].fd, rd); + return; + } + + case fd_info_proto_v3udp: { + uint8_t pktbuf[N2N_PKT_BUF_SIZE]; + edge_read_proto3_udp( + eee, + fd, + pktbuf, + sizeof(pktbuf), + now + ); + return; + } + } +} + static void fdlist_check_ready (fd_set *rd, const time_t now, struct n3n_runtime_data *eee) { int slot = 0; // A linear scan is not ideal, but until we support things other than @@ -218,6 +252,7 @@ static int setup_select (fd_set *rd, fd_set *wr, struct n3n_runtime_data *eee) { } int mainloop_runonce (fd_set *rd, fd_set *wr, struct n3n_runtime_data *eee) { + metrics.mainloop ++; int maxfd = setup_select(rd, wr, eee); @@ -281,6 +316,7 @@ int mainloop_runonce (fd_set *rd, fd_set *wr, struct n3n_runtime_data *eee) { } void mainloop_register_fd (int fd, enum fd_info_proto proto) { + metrics.register_fd++; int slot = fdlist_allocslot(fd, proto); // TODO: the moment this starts to fire, we need to revamp the @@ -289,10 +325,12 @@ void mainloop_register_fd (int fd, enum fd_info_proto proto) { } void mainloop_unregister_fd (int fd) { + metrics.unregister_fd++; fdlist_freefd(fd); } void n3n_initfuncs_mainloop () { fdlist_zero(); - n3n_metrics_register(&metrics_module); + n3n_metrics_register(&metrics_module_dynamic); + n3n_metrics_register(&metrics_module_static); } diff --git a/src/metrics.c b/src/metrics.c index 1696621..b2dc216 100644 --- a/src/metrics.c +++ b/src/metrics.c @@ -77,11 +77,18 @@ void n3n_metrics_render_u32tags ( int count = tags; va_start(ap, tags); while(count) { + char *tag = va_arg(ap, char *); + char *val = va_arg(ap, char *); + + // Skip empty tags + if(!tag || !val) { + count--; + continue; + } + if(count) { sb_reprintf(reply,","); } - char *tag = va_arg(ap, char *); - char *val = va_arg(ap, char *); sb_reprintf(reply,"%s=\"%s\"", tag, val); count--; } From ea6bb4206cf5202a9f54de19fb6af6ebab074b69 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Sat, 23 Nov 2024 11:38:47 +1100 Subject: [PATCH 51/85] Allow the fd handler to see the fd info as it may have connection buffers in the future --- src/mainloop.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/mainloop.c b/src/mainloop.c index 01c9a70..63effaa 100644 --- a/src/mainloop.c +++ b/src/mainloop.c @@ -170,8 +170,8 @@ static int fdlist_read_fd_set (fd_set *rd) { return max_sock; } -static void handle_fd (const time_t now, int fd, enum fd_info_proto proto, struct n3n_runtime_data *eee) { - switch(proto) { +static void handle_fd (const time_t now, const struct fd_info info, struct n3n_runtime_data *eee) { + switch(info.proto) { case fd_info_proto_unknown: // should not happen! assert(false); @@ -180,11 +180,12 @@ static void handle_fd (const time_t now, int fd, enum fd_info_proto proto, struc case fd_info_proto_tuntap: // read an ethernet frame from the TAP socket; write on the IP // socket + // TODO: change API to tell it which fd edge_read_from_tap(eee); return; case fd_info_proto_listen_http: { - int slotnr = slots_accept(eee->mgmt_slots, fd, CONN_PROTO_HTTP); + int slotnr = slots_accept(eee->mgmt_slots, info.fd, CONN_PROTO_HTTP); if(slotnr < 0) { // TODO: increment error stats return; @@ -198,7 +199,7 @@ static void handle_fd (const time_t now, int fd, enum fd_info_proto proto, struc uint8_t pktbuf[N2N_PKT_BUF_SIZE]; edge_read_proto3_udp( eee, - fd, + info.fd, pktbuf, sizeof(pktbuf), now @@ -223,7 +224,7 @@ static void fdlist_check_ready (fd_set *rd, const time_t now, struct n3n_runtime } fdlist[slot].stats_reads++; - handle_fd(now, fdlist[slot].fd, fdlist[slot].proto, eee); + handle_fd(now, fdlist[slot], eee); slot++; } } From 3c60d405bd1e41d2e05aa6572ffd4b09e7476596 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Sat, 23 Nov 2024 11:48:49 +1100 Subject: [PATCH 52/85] Refactor connslot to allow the filehandle to be managed separately from the buffers --- libs/connslot/connslot.c | 26 +++++++++++++------------- libs/connslot/connslot.h | 6 +++--- libs/connslot/httpd-test.c | 2 +- src/management.c | 2 +- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/libs/connslot/connslot.c b/libs/connslot/connslot.c index 8fa08eb..8d6aca8 100644 --- a/libs/connslot/connslot.c +++ b/libs/connslot/connslot.c @@ -89,7 +89,7 @@ int conn_init(conn_t *conn, size_t request_max, size_t reply_header_max) { return 0; } -void conn_read(conn_t *conn) { +void conn_read(conn_t *conn, int fd) { conn->state = CONN_READING; // If no space available, try increasing our capacity @@ -100,7 +100,7 @@ void conn_read(conn_t *conn) { } } - ssize_t size = sb_read(conn->fd, conn->request); + ssize_t size = sb_read(fd, conn->request); if (size == 0) { // As we are dealing with non blocking sockets, and have made a non @@ -193,12 +193,12 @@ void conn_read(conn_t *conn) { return; } -ssize_t conn_write(conn_t *conn) { +ssize_t conn_write(conn_t *conn, int fd) { ssize_t sent; conn->state = CONN_SENDING; - if (conn->fd == -1) { + if (fd == -1) { return 0; } #ifndef _WIN32 @@ -226,20 +226,20 @@ ssize_t conn_write(conn_t *conn) { nr++; } - sent = writev(conn->fd, &vecs[0], nr); + sent = writev(fd, &vecs[0], nr); #else // no iovec // if (conn->reply_sendpos < sb_len(conn->reply_header)) { sent = sb_write( - conn->fd, + fd, conn->reply_header, conn->reply_sendpos, -1 ); } else { sent = sb_write( - conn->fd, + fd, conn->reply, conn->reply_sendpos - sb_len(conn->reply_header), -1 @@ -272,8 +272,8 @@ int conn_iswriter(conn_t *conn) { } } -void conn_close(conn_t *conn) { - closesocket(conn->fd); +void conn_close(conn_t *conn, int fd) { + closesocket(fd); conn_zero(conn); // TODO: could shrink the size here, maybe in certain circumstances? } @@ -632,7 +632,7 @@ int slots_closeidle(slots_t *slots) { int delta_t = now - slots->conn[i].activity; if (delta_t > slots->timeout) { // TODO: metrics timeouts ++ - conn_close(&slots->conn[i]); + conn_close(&slots->conn[i], slots->conn[i].fd); nr_closed++; } } @@ -679,7 +679,7 @@ int slots_fdset_loop(slots_t *slots, fd_set *readers, fd_set *writers) { nr_open++; if (FD_ISSET(slots->conn[i].fd, readers)) { - conn_read(&slots->conn[i]); + conn_read(&slots->conn[i], slots->conn[i].fd); // possibly sets state to CONN_READY } @@ -697,14 +697,14 @@ int slots_fdset_loop(slots_t *slots, fd_set *readers, fd_set *writers) { /* fallsthrough */ case CONN_CLOSED: slots->nr_open--; - conn_close(&slots->conn[i]); + conn_close(&slots->conn[i], slots->conn[i].fd); continue; default: break; } if (FD_ISSET(slots->conn[i].fd, writers)) { - conn_write(&slots->conn[i]); + conn_write(&slots->conn[i], slots->conn[i].fd); } } diff --git a/libs/connslot/connslot.h b/libs/connslot/connslot.h index 838b854..59b213d 100644 --- a/libs/connslot/connslot.h +++ b/libs/connslot/connslot.h @@ -62,10 +62,10 @@ typedef struct slots { void conn_zero(conn_t *); int conn_init(conn_t *, size_t, size_t); -void conn_read(conn_t *); -ssize_t conn_write(conn_t *); +void conn_read(conn_t *, int); +ssize_t conn_write(conn_t *, int); int conn_iswriter(conn_t *); -void conn_close(conn_t *); +void conn_close(conn_t *, int); void slots_free(slots_t *slots); slots_t *slots_malloc(int nr_slots, size_t, size_t); diff --git a/libs/connslot/httpd-test.c b/libs/connslot/httpd-test.c index 6f2a9d4..a30b5ff 100644 --- a/libs/connslot/httpd-test.c +++ b/libs/connslot/httpd-test.c @@ -166,7 +166,7 @@ void httpd_test(int port) { // continue; // Try to immediately start sending the reply - conn_write(&slots->conn[i]); + conn_write(&slots->conn[i], slots->conn[i].fd); } } } diff --git a/src/management.c b/src/management.c index 040cd57..c6a2b31 100644 --- a/src/management.c +++ b/src/management.c @@ -1181,5 +1181,5 @@ void mgmt_api_handler (struct n3n_runtime_data *eee, conn_t *conn) { } // Try to immediately start sending the reply - conn_write(conn); + conn_write(conn, conn->fd); } From c84fc038f74e35dfa3c11199b35629b2ea68cff1 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Thu, 28 Nov 2024 15:35:23 +1100 Subject: [PATCH 53/85] Move quirk definition out of core header Only define it where it is actually used. This might get refactored into its own compat header later. --- include/n2n.h | 1 - src/edge_utils.c | 4 ++++ src/peer_info.c | 5 +++++ src/sn_utils.c | 5 +++++ tools/n3n-portfwd.c | 5 +++++ tools/n3n-route.c | 5 +++++ 6 files changed, 24 insertions(+), 1 deletion(-) diff --git a/include/n2n.h b/include/n2n.h index aa21253..1e33927 100644 --- a/include/n2n.h +++ b/include/n2n.h @@ -66,7 +66,6 @@ #include // for uint8_t, uint64_t, uint32_t, uint16_t #include // for time_t #include // for close -#define closesocket(a) close(a) #ifdef __linux__ #define N2N_CAN_NAME_IFACE 1 diff --git a/src/edge_utils.c b/src/edge_utils.c index 026e9ad..e27b915 100644 --- a/src/edge_utils.c +++ b/src/edge_utils.c @@ -74,6 +74,10 @@ #include // for setsockopt, AF_INET, connect #endif +#ifndef _WIN32 +// Another wonderful gift from the world of POSIX compliance is not worth much +#define closesocket(a) close(a) +#endif /* ************************************** */ diff --git a/src/peer_info.c b/src/peer_info.c index 98c2f4c..29a656b 100644 --- a/src/peer_info.c +++ b/src/peer_info.c @@ -23,6 +23,11 @@ #include "peer_info.h" #include "resolve.h" // for supernode2sock +#ifndef _WIN32 +// Another wonderful gift from the world of POSIX compliance is not worth much +#define closesocket(a) close(a) +#endif + static struct metrics { uint32_t init; // peer_info_init() is called uint32_t alloc; // peer_info_malloc() is called diff --git a/src/sn_utils.c b/src/sn_utils.c index 33acf1f..2504e20 100644 --- a/src/sn_utils.c +++ b/src/sn_utils.c @@ -65,6 +65,11 @@ #include // for recvfrom, shutdown, sockaddr_storage #endif +#ifndef _WIN32 +// Another wonderful gift from the world of POSIX compliance is not worth much +#define closesocket(a) close(a) +#endif + #define HASH_FIND_COMMUNITY(head, name, out) HASH_FIND_STR(head, name, out) diff --git a/tools/n3n-portfwd.c b/tools/n3n-portfwd.c index d90bca5..55f1bcd 100644 --- a/tools/n3n-portfwd.c +++ b/tools/n3n-portfwd.c @@ -45,6 +45,11 @@ #include // for connect, recv, send, socket, AF_INET #endif +#ifndef _WIN32 +// Another wonderful gift from the world of POSIX compliance is not worth much +#define closesocket(a) close(a) +#endif + #define WITH_PORT 1 #define CORRECT_TAG 2 diff --git a/tools/n3n-route.c b/tools/n3n-route.c index 8b1ad9b..009ccab 100644 --- a/tools/n3n-route.c +++ b/tools/n3n-route.c @@ -56,6 +56,11 @@ #include // for send, socket, AF_INET, recv, connect #endif +#ifndef _WIN32 +// Another wonderful gift from the world of POSIX compliance is not worth much +#define closesocket(a) close(a) +#endif + // FIXME, this tool needs porting to JsonRPC #define N2N_EDGE_MGMT_PORT 5644 From e2da0b6f33ab1494e2f34f45aae3fb9084d44301 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Thu, 28 Nov 2024 17:06:07 +1100 Subject: [PATCH 54/85] Refactor some connslot functions to make it easier to supply your own mainloop --- libs/connslot/connslot.c | 42 ++++++++++++++++++++++++++-------------- libs/connslot/connslot.h | 2 ++ 2 files changed, 29 insertions(+), 15 deletions(-) diff --git a/libs/connslot/connslot.c b/libs/connslot/connslot.c index 8d6aca8..390c806 100644 --- a/libs/connslot/connslot.c +++ b/libs/connslot/connslot.c @@ -89,6 +89,21 @@ int conn_init(conn_t *conn, size_t request_max, size_t reply_header_max) { return 0; } +void conn_accept(conn_t *conn, int fd, enum conn_proto proto) { + +#ifndef _WIN32 + fcntl(fd, F_SETFL, O_NONBLOCK); +#else + u_long arg = 1; + ioctlsocket(fd, FIONBIO, &arg); +#endif + + // This will truncate the time to a int - usually 32bits + conn->activity = time(NULL); + conn->fd = fd; + conn->proto = proto; +} + void conn_read(conn_t *conn, int fd) { conn->state = CONN_READING; @@ -278,6 +293,16 @@ void conn_close(conn_t *conn, int fd) { // TODO: could shrink the size here, maybe in certain circumstances? } +bool conn_closeidle(conn_t *conn, int now, int timeout) { + int delta_t = now - conn->activity; + if (delta_t > timeout) { + // TODO: metrics timeouts ++ + conn_close(conn, conn->fd); + return true; + } + return false; +} + void conn_dump(strbuf_t **buf, conn_t *conn) { sb_reprintf( buf, @@ -605,18 +630,8 @@ int slots_accept(slots_t *slots, int fd, enum conn_proto proto) { return -1; } -#ifndef _WIN32 - fcntl(client, F_SETFL, O_NONBLOCK); -#else - u_long arg = 1; - ioctlsocket(client, FIONBIO, &arg); -#endif - + conn_accept(&slots->conn[i], client, proto); slots->nr_open++; - // This will truncate the time to a int - usually 32bits - slots->conn[i].activity = time(NULL); - slots->conn[i].fd = client; - slots->conn[i].proto = proto; return i; } @@ -629,10 +644,7 @@ int slots_closeidle(slots_t *slots) { if (slots->conn[i].fd == -1) { continue; } - int delta_t = now - slots->conn[i].activity; - if (delta_t > slots->timeout) { - // TODO: metrics timeouts ++ - conn_close(&slots->conn[i], slots->conn[i].fd); + if (conn_closeidle(&slots->conn[i], now, slots->timeout)) { nr_closed++; } } diff --git a/libs/connslot/connslot.h b/libs/connslot/connslot.h index 59b213d..cf24f9f 100644 --- a/libs/connslot/connslot.h +++ b/libs/connslot/connslot.h @@ -62,10 +62,12 @@ typedef struct slots { void conn_zero(conn_t *); int conn_init(conn_t *, size_t, size_t); +void conn_accept(conn_t *, int, enum conn_proto); void conn_read(conn_t *, int); ssize_t conn_write(conn_t *, int); int conn_iswriter(conn_t *); void conn_close(conn_t *, int); +bool conn_closeidle(conn_t *, int, int); void slots_free(slots_t *slots); slots_t *slots_malloc(int nr_slots, size_t, size_t); From 29b2fd5793592885c49947a0ef10cdbbd37e14c3 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Thu, 28 Nov 2024 17:09:00 +1100 Subject: [PATCH 55/85] Migrate http client file handles into mainloop proper --- include/n3n/mainloop.h | 1 + src/edge_utils.c | 9 -- src/mainloop.c | 215 +++++++++++++++++++++++++++++------------ 3 files changed, 155 insertions(+), 70 deletions(-) diff --git a/include/n3n/mainloop.h b/include/n3n/mainloop.h index 4b7aed8..eeea9aa 100644 --- a/include/n3n/mainloop.h +++ b/include/n3n/mainloop.h @@ -22,6 +22,7 @@ enum __attribute__((__packed__)) fd_info_proto { fd_info_proto_tuntap, fd_info_proto_listen_http, fd_info_proto_v3udp, + fd_info_proto_http, }; int mainloop_runonce (fd_set *, fd_set *, struct n3n_runtime_data *); diff --git a/src/edge_utils.c b/src/edge_utils.c index e27b915..d12ddc5 100644 --- a/src/edge_utils.c +++ b/src/edge_utils.c @@ -3112,9 +3112,6 @@ int run_edge_loop (struct n3n_runtime_data *eee) { } } - // check for timed out slots - slots_closeidle(eee->mgmt_slots); - // If anything we recieved caused us to stop.. if(!(*eee->keep_running)) break; @@ -3263,7 +3260,6 @@ void edge_term (struct n3n_runtime_data * eee) { closeTraceFile(); - slots_free(eee->mgmt_slots); free(eee); #ifdef _WIN32 @@ -3283,11 +3279,6 @@ int windows_stop_fd; static int edge_init_sockets (struct n3n_runtime_data *eee) { - eee->mgmt_slots = slots_malloc(5, 5000, 500); - if(!eee->mgmt_slots) { - abort(); - } - if(eee->conf.mgmt_port) { int fd = slots_create_listen_tcp(eee->conf.mgmt_port, false); if(fd < 0) { diff --git a/src/mainloop.c b/src/mainloop.c index 63effaa..9ff84c1 100644 --- a/src/mainloop.c +++ b/src/mainloop.c @@ -8,6 +8,7 @@ #include // for slots_fdset #include // for n3n_runtime_data #include // for edge_read_proto3_udp +#include // for traceEvent #include // for fd_info_proto #include #include // for traceEvent @@ -16,17 +17,24 @@ #ifndef _WIN32 #include // for select, FD_ZERO, +#include // for close #endif #include "edge_utils.h" // for edge_read_from_tap #include "management.h" // for readFromMgmtSocket #include "minmax.h" // for min, max -#include "n2n_define.h" + +#ifndef _WIN32 +// Another wonderful gift from the world of POSIX compliance is not worth much +#define closesocket(a) close(a) +#endif static struct metrics { uint32_t mainloop; // mainloop_runonce() is called uint32_t register_fd; // mainloop_register_fd() is called uint32_t unregister_fd; // mainloop_unregister_fd() is called + uint32_t connlist_alloc; + uint32_t connlist_free; } metrics; static struct n3n_metrics_items_llu32 metrics_items = { @@ -46,6 +54,14 @@ static struct n3n_metrics_items_llu32 metrics_items = { .val1 = "unregister_fd", .offset = offsetof(struct metrics, unregister_fd), }, + { + .val1 = "connlist_alloc", + .offset = offsetof(struct metrics, connlist_alloc), + }, + { + .val1 = "connlist_free", + .offset = offsetof(struct metrics, connlist_free), + }, { }, }, }; @@ -55,12 +71,14 @@ static char *proto_str[] = { [fd_info_proto_tuntap] = "tuntap", [fd_info_proto_listen_http] = "listen_http", [fd_info_proto_v3udp] = "v3udp", + [fd_info_proto_http] = "http", }; struct fd_info { int fd; // The file descriptor for this connection - int stats_reads; // The number of read to read events + int stats_reads; // The number of ready to read events enum fd_info_proto proto; // What protocol to use on a read event + int8_t connnr; // which connlist[] is being used as buffer }; // A static array of known file descriptors will not scale once full TCP @@ -69,6 +87,11 @@ struct fd_info { static struct fd_info fdlist[MAX_HANDLES]; static int fdlist_next_search; +#define MAX_CONN 8 +// TODO: need pools of struct conn, for each expected buffer size +static struct conn connlist[MAX_CONN]; +static int connlist_next_search; + static void metrics_callback (strbuf_t **reply, const struct n3n_metrics_module *module) { int slot = 0; char buf[16]; @@ -112,10 +135,46 @@ static struct n3n_metrics_module metrics_module_static = { .type = n3n_metrics_type_llu32, }; +static void connlist_init () { + int conn = 0; + while(conn < MAX_CONN) { + conn_init(&connlist[conn], 4000, 1000); + conn++; + } + connlist_next_search = 0; +} + +static int connlist_alloc (enum conn_proto proto) { + int conn = connlist_next_search % MAX_CONN; + int count = MAX_CONN; + while(count) { + if(connlist[conn].proto == CONN_PROTO_UNK) { + connlist[conn].proto = proto; + connlist_next_search = conn + 1; + metrics.connlist_alloc++; + return conn; + } + conn = (conn + 1) % MAX_CONN; + count--; + } + return -1; +} + +static void connlist_free (int connnr) { + if(connnr > MAX_CONN) { + // TODO: error! + return; + } + connlist[connnr].proto = CONN_PROTO_UNK; + fdlist_next_search = connnr; + metrics.connlist_free++; +} + // Used only to initialise the array at startup static void fdlist_zero () { int slot = 0; while(slot < MAX_HANDLES) { + fdlist[slot].connnr = -1; fdlist[slot].fd = -1; fdlist[slot].proto = fd_info_proto_unknown; slot++; @@ -128,6 +187,7 @@ static int fdlist_allocslot (int fd, enum fd_info_proto proto) { int count = MAX_HANDLES; while(count) { if(fdlist[slot].fd == -1) { + metrics.register_fd++; fdlist[slot].fd = fd; fdlist[slot].proto = proto; fdlist[slot].stats_reads = 0; @@ -147,9 +207,13 @@ static void fdlist_freefd (int fd) { slot++; continue; } + metrics.unregister_fd++; fdlist[slot].fd = -1; fdlist[slot].proto = fd_info_proto_unknown; fdlist_next_search = slot; + if(fdlist[slot].connnr != -1) { + connlist_free(fdlist[slot].connnr); + } return; } @@ -157,14 +221,30 @@ static void fdlist_freefd (int fd) { // - could assert or similar } -static int fdlist_read_fd_set (fd_set *rd) { +static int fdlist_fd_set (fd_set *rd, fd_set *wr) { int max_sock = 0; int slot = 0; while(slot < MAX_HANDLES) { - if(fdlist[slot].fd != -1) { - FD_SET(fdlist[slot].fd, rd); - max_sock = MAX(max_sock, fdlist[slot].fd); + if(fdlist[slot].fd == -1) { + slot++; + continue; } + + // TODO: + // - if no empty conn, dont FD_SET on proto TCP listen + + FD_SET(fdlist[slot].fd, rd); + max_sock = MAX(max_sock, fdlist[slot].fd); + + if(fdlist[slot].connnr == -1) { + slot++; + continue; + } + + if(conn_iswriter(&connlist[fdlist[slot].connnr])) { + FD_SET(fdlist[slot].fd, wr); + } + slot++; } return max_sock; @@ -185,13 +265,25 @@ static void handle_fd (const time_t now, const struct fd_info info, struct n3n_r return; case fd_info_proto_listen_http: { - int slotnr = slots_accept(eee->mgmt_slots, info.fd, CONN_PROTO_HTTP); - if(slotnr < 0) { - // TODO: increment error stats + int client = accept(info.fd, NULL, 0); + if(client == -1) { + // TODO: + // - increment error stats + return; + } + + int slotnr = fdlist_allocslot(client, fd_info_proto_http); + int connnr = connlist_alloc(CONN_PROTO_HTTP); + if(slotnr < 0 || connnr < 0) { + // TODO: + // - increment error stats + // - send static text + closesocket(client); return; } - // TODO: Schedule slot for immediately reading - // FD_SET(eee->mgmt_slots->conn[slotnr].fd, rd); + fdlist[slotnr].connnr = connnr; + conn_accept(&connlist[connnr], client, CONN_PROTO_HTTP); + return; } @@ -206,25 +298,66 @@ static void handle_fd (const time_t now, const struct fd_info info, struct n3n_r ); return; } + + case fd_info_proto_http: { + struct conn *conn = &connlist[info.connnr]; + conn_read(conn, info.fd); + + switch(conn->state) { + case CONN_EMPTY: + case CONN_READING: + case CONN_SENDING: + // These states dont require us to do anything + // TODO: + // - handle reading/sending simultaneous? + return; + + case CONN_READY: + mgmt_api_handler(eee, conn); + return; + + case CONN_ERROR: + case CONN_CLOSED: + conn_close(conn, info.fd); + // TODO: freefd() is doing a fd search, we could optimise + fdlist_freefd(info.fd); + } + return; + } } } -static void fdlist_check_ready (fd_set *rd, const time_t now, struct n3n_runtime_data *eee) { +static void fdlist_check_ready (fd_set *rd, fd_set *wr, const time_t now, struct n3n_runtime_data *eee) { int slot = 0; // A linear scan is not ideal, but until we support things other than // select() it will need to suffice while(slot < MAX_HANDLES) { - if(fdlist[slot].fd == -1) { + int fd = fdlist[slot].fd; + if(fd == -1) { slot++; continue; } - if(!FD_ISSET(fdlist[slot].fd, rd)) { - slot++; - continue; + if(FD_ISSET(fd, rd)) { + fdlist[slot].stats_reads++; + handle_fd(now, fdlist[slot], eee); + } + if(FD_ISSET(fd, wr)) { + // We should not be listening on this socket if there is no + // connnr assigned, but paranoia.. + if(fdlist[slot].connnr == -1) { + traceEvent(TRACE_DEBUG, "writer bad connnr"); + slot++; + continue; + } + + // TODO: track the stats on writes? + conn_write(&connlist[fdlist[slot].connnr], fd); } - fdlist[slot].stats_reads++; - handle_fd(now, fdlist[slot], eee); + if(fdlist[slot].connnr != -1) { + int timeout = 60; + conn_closeidle(&connlist[fdlist[slot].connnr], now, timeout); + } slot++; } } @@ -232,7 +365,7 @@ static void fdlist_check_ready (fd_set *rd, const time_t now, struct n3n_runtime static int setup_select (fd_set *rd, fd_set *wr, struct n3n_runtime_data *eee) { FD_ZERO(rd); FD_ZERO(wr); - int max_sock = fdlist_read_fd_set(rd); + int max_sock = fdlist_fd_set(rd, wr); // HACK - until this mainloop supports v3tcp proto sockets if((eee->conf.connect_tcp) && (eee->sock != -1)) { @@ -240,15 +373,6 @@ static int setup_select (fd_set *rd, fd_set *wr, struct n3n_runtime_data *eee) { max_sock = MAX(max_sock, eee->sock); } - max_sock = MAX( - max_sock, - slots_fdset( - eee->mgmt_slots, - rd, - wr - ) - ); - return max_sock; } @@ -281,43 +405,12 @@ int mainloop_runonce (fd_set *rd, fd_set *wr, struct n3n_runtime_data *eee) { // One timestamp to use for this entire loop iteration time_t now = time(NULL); - fdlist_check_ready(rd, now, eee); - - int slots_ready = slots_fdset_loop( - eee->mgmt_slots, - rd, - wr - ); - - if(slots_ready < 0) { - traceEvent( - TRACE_ERROR, - "slots_fdset_loop returns %i (Is daemon exiting?)", slots_ready - ); - } else if(slots_ready > 0) { - // A linear scan is not ideal, but this is a select() loop - // not one built for performance. - // - update connslot to have callbacks instead of scan - // - switch to a modern poll loop (and reimplement differently - // for each OS supported) - // This should only be a concern if we are doing a large - // number of slot connections - for(int i=0; imgmt_slots->nr_slots; i++) { - if(eee->mgmt_slots->conn[i].fd == -1) { - continue; - } - - if(eee->mgmt_slots->conn[i].state == CONN_READY) { - mgmt_api_handler(eee, &eee->mgmt_slots->conn[i]); - } - } - } + fdlist_check_ready(rd, wr, now, eee); return ready; } void mainloop_register_fd (int fd, enum fd_info_proto proto) { - metrics.register_fd++; int slot = fdlist_allocslot(fd, proto); // TODO: the moment this starts to fire, we need to revamp the @@ -326,11 +419,11 @@ void mainloop_register_fd (int fd, enum fd_info_proto proto) { } void mainloop_unregister_fd (int fd) { - metrics.unregister_fd++; fdlist_freefd(fd); } void n3n_initfuncs_mainloop () { + connlist_init(); fdlist_zero(); n3n_metrics_register(&metrics_module_dynamic); n3n_metrics_register(&metrics_module_static); From c9661bb59ba220e5a30d2e80048824e3d4fdb4ef Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Thu, 28 Nov 2024 17:41:17 +1100 Subject: [PATCH 56/85] Update event handles to not break mainloop --- src/management.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/management.c b/src/management.c index c6a2b31..c74718f 100644 --- a/src/management.c +++ b/src/management.c @@ -11,6 +11,7 @@ #include // for jsonrpc_t, jsonrpc_parse #include // for is_null_mac #include // for traceEvent +#include // for mainloop_unregister_fd #include // for n3n_metrics_render #include // for ip_subnet_to_str, sock_to_cstr #include // for load_allowed_sn_community @@ -254,7 +255,10 @@ static void event_subscribe (struct n3n_runtime_data *eee, conn_t *conn) { // Take the filehandle away from the connslots. mgmt_event_subscribers[topicid] = conn->fd; - conn_zero(conn); + // TODO: + // - Keep these filehandles in the mainloop + // - change the mainloop proto mark it as "event" + mainloop_unregister_fd(conn->fd); // TODO: shutdown(fd, SHUT_RD) - but that does nothing for unix domain From be77eb7df65ceb138bf0d7813e7541cdd5d9a1e3 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Thu, 28 Nov 2024 17:41:55 +1100 Subject: [PATCH 57/85] Fix the idle file closing behaviour to stop breaking the mainloop --- libs/connslot/connslot.c | 7 ++++--- libs/connslot/connslot.h | 2 +- src/mainloop.c | 6 +++++- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/libs/connslot/connslot.c b/libs/connslot/connslot.c index 390c806..dd7f248 100644 --- a/libs/connslot/connslot.c +++ b/libs/connslot/connslot.c @@ -293,11 +293,11 @@ void conn_close(conn_t *conn, int fd) { // TODO: could shrink the size here, maybe in certain circumstances? } -bool conn_closeidle(conn_t *conn, int now, int timeout) { +bool conn_closeidle(conn_t *conn, int fd, int now, int timeout) { int delta_t = now - conn->activity; if (delta_t > timeout) { // TODO: metrics timeouts ++ - conn_close(conn, conn->fd); + conn_close(conn, fd); return true; } return false; @@ -644,7 +644,8 @@ int slots_closeidle(slots_t *slots) { if (slots->conn[i].fd == -1) { continue; } - if (conn_closeidle(&slots->conn[i], now, slots->timeout)) { + int fd = slots->conn[i].fd; + if (conn_closeidle(&slots->conn[i], fd, now, slots->timeout)) { nr_closed++; } } diff --git a/libs/connslot/connslot.h b/libs/connslot/connslot.h index cf24f9f..cbd5a8c 100644 --- a/libs/connslot/connslot.h +++ b/libs/connslot/connslot.h @@ -67,7 +67,7 @@ void conn_read(conn_t *, int); ssize_t conn_write(conn_t *, int); int conn_iswriter(conn_t *); void conn_close(conn_t *, int); -bool conn_closeidle(conn_t *, int, int); +bool conn_closeidle(conn_t *, int, int, int); void slots_free(slots_t *slots); slots_t *slots_malloc(int nr_slots, size_t, size_t); diff --git a/src/mainloop.c b/src/mainloop.c index 9ff84c1..a9f3b7e 100644 --- a/src/mainloop.c +++ b/src/mainloop.c @@ -356,7 +356,11 @@ static void fdlist_check_ready (fd_set *rd, fd_set *wr, const time_t now, struct if(fdlist[slot].connnr != -1) { int timeout = 60; - conn_closeidle(&connlist[fdlist[slot].connnr], now, timeout); + struct conn *conn = &connlist[fdlist[slot].connnr]; + bool closed = conn_closeidle(conn, fd, now, timeout); + if(closed) { + fdlist_freefd(fd); + } } slot++; } From f5174b30fc295699843f98f295ec989626ce8bb9 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Thu, 28 Nov 2024 18:31:58 +1100 Subject: [PATCH 58/85] Address lint concern --- src/mainloop.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mainloop.c b/src/mainloop.c index a9f3b7e..7d5d6d6 100644 --- a/src/mainloop.c +++ b/src/mainloop.c @@ -381,7 +381,7 @@ static int setup_select (fd_set *rd, fd_set *wr, struct n3n_runtime_data *eee) { } int mainloop_runonce (fd_set *rd, fd_set *wr, struct n3n_runtime_data *eee) { - metrics.mainloop ++; + metrics.mainloop++; int maxfd = setup_select(rd, wr, eee); From 22552222f2175423178797af25701e05ea09712c Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Thu, 28 Nov 2024 18:47:23 +1100 Subject: [PATCH 59/85] Remove old embedded callbacks There are no code examples in the codebase for these callbacks and no documentation written for them either. Without a clear use case, they mainly serve as an extra set of things to confuse people. Once a user is found, they can easily be added back in. --- include/n2n_typedefs.h | 27 --------------------------- src/edge_utils.c | 29 ----------------------------- 2 files changed, 56 deletions(-) diff --git a/include/n2n_typedefs.h b/include/n2n_typedefs.h index 6add434..9c4b5b4 100644 --- a/include/n2n_typedefs.h +++ b/include/n2n_typedefs.h @@ -377,32 +377,6 @@ struct network_traffic_filter { /* *************************************************** */ -/* Callbacks allow external programs to attach functions in response to - * N2N events. */ -typedef struct n2n_edge_callbacks { - /* The supernode registration has been updated */ - void (*sn_registration_updated)(struct n3n_runtime_data *eee, time_t now, const n2n_sock_t *sn); - - /* A packet has been received from a peer. N2N_DROP can be returned to - * drop the packet. The packet payload can be modified. This only allows - * the packet size to be reduced */ - n2n_verdict (*packet_from_peer)(struct n3n_runtime_data *eee, const n2n_sock_t *peer, uint8_t *payload, uint16_t *payload_size); - - /* A packet has been received from the TAP interface. N2N_DROP can be - * returned to drop the packet. The packet payload can be modified. - * This only allows the packet size to be reduced */ - n2n_verdict (*packet_from_tap)(struct n3n_runtime_data *eee, uint8_t *payload, uint16_t *payload_size); - - /* Called whenever the IP address of the TAP interface changes. */ - void (*ip_address_changed)(struct n3n_runtime_data *eee, uint32_t old_ip, uint32_t new_ip); - - /* Called periodically in the main loop. */ - void (*main_loop_period)(struct n3n_runtime_data *eee, time_t now); - -} n2n_edge_callbacks_t; - -/* *************************************************** */ - typedef enum n2n_transform { N2N_TRANSFORM_ID_INVAL = 0, N2N_TRANSFORM_ID_NULL = 1, @@ -554,7 +528,6 @@ struct n3n_runtime_data { n2n_trans_op_t transop; /**< The transop to use when encoding */ n2n_trans_op_t transop_lzo; /**< The transop for LZO compression */ n2n_trans_op_t transop_zstd; /**< The transop for ZSTD compression */ - n2n_edge_callbacks_t cb; /**< API callbacks */ SN_SELECTION_CRITERION_DATA_TYPE sn_selection_criterion_common_data; /* Sockets */ diff --git a/src/edge_utils.c b/src/edge_utils.c index d12ddc5..ebf5bad 100644 --- a/src/edge_utils.c +++ b/src/edge_utils.c @@ -1876,15 +1876,6 @@ static int handle_PACKET (struct n3n_runtime_data * eee, return(0); } - if(eee->cb.packet_from_peer) { - uint16_t tmp_eth_size = eth_size; - if(eee->cb.packet_from_peer(eee, orig_sender, eth_payload, &tmp_eth_size) == N2N_DROP) { - traceEvent(TRACE_DEBUG, "DROP packet of size %u", (unsigned int)eth_size); - return(0); - } - eth_size = tmp_eth_size; - } - /* Write ethernet packet to tap device. */ traceEvent(TRACE_DEBUG, "sending data of size %u to TAP", (unsigned int)eth_size); data_sent_len = tuntap_write(&(eee->device), eth_payload, eth_size); @@ -2284,15 +2275,6 @@ void edge_read_from_tap (struct n3n_runtime_data * eee) { } } - if(eee->cb.packet_from_tap) { - uint16_t tmp_len = len; - if(eee->cb.packet_from_tap(eee, eth_pkt, &tmp_len) == N2N_DROP) { - traceEvent(TRACE_DEBUG, "DROP packet of size %u", (unsigned int)len); - return; - } - len = tmp_len; - } - edge_send_packet2net(eee, eth_pkt, len); } @@ -2716,9 +2698,6 @@ void process_udp (struct n3n_runtime_data *eee, // NOTE: the register_interval should be chosen by the edge node based on its NAT configuration. // eee->conf.register_interval = ra.lifetime; - if(eee->cb.sn_registration_updated && !is_null_mac(eee->device.mac_addr)) - eee->cb.sn_registration_updated(eee, now, &sender); - break; } @@ -3161,14 +3140,9 @@ int run_edge_loop (struct n3n_runtime_data *eee) { // - multi-homing support if((eee->conf.tuntap_ip_mode == TUNTAP_IP_MODE_DHCP) && ((now - lastIfaceCheck) > IFACE_UPDATE_INTERVAL)) { - uint32_t old_ip = eee->device.ip_addr; - traceEvent(TRACE_INFO, "re-checking dynamic IP address"); tuntap_get_address(&(eee->device)); lastIfaceCheck = now; - - if((old_ip != eee->device.ip_addr) && eee->cb.ip_address_changed) - eee->cb.ip_address_changed(eee, old_ip, eee->device.ip_addr); } sort_supernodes(eee, now); @@ -3179,9 +3153,6 @@ int run_edge_loop (struct n3n_runtime_data *eee) { now ); - if(eee->cb.main_loop_period) - eee->cb.main_loop_period(eee, now); - } /* while */ send_unregister_super(eee); From a7b2ab5e4a7e935f9be392270ad22225c2c45c5a Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Thu, 28 Nov 2024 19:02:38 +1100 Subject: [PATCH 60/85] Refactor to remove ternary for ease of reading --- src/edge_utils.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/edge_utils.c b/src/edge_utils.c index ebf5bad..2f253c1 100644 --- a/src/edge_utils.c +++ b/src/edge_utils.c @@ -249,8 +249,11 @@ static int is_ip6_discovery (const void * buf, size_t bufsize) { // reset number of supernode connection attempts: try only once for already more realiable tcp connections void reset_sup_attempts (struct n3n_runtime_data *eee) { - - eee->sup_attempts = (eee->conf.connect_tcp) ? 1 : N2N_EDGE_SUP_ATTEMPTS; + if(eee->conf.connect_tcp) { + eee->sup_attempts = 1; + } else { + eee->sup_attempts = N2N_EDGE_SUP_ATTEMPTS; + } } From 3f1a6f66d820a6508584185edaad884e346d5cf9 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Thu, 28 Nov 2024 19:11:16 +1100 Subject: [PATCH 61/85] Refactor code with an early exit to make it easier to read --- src/edge_utils.c | 164 ++++++++++++++++++++++++----------------------- 1 file changed, 84 insertions(+), 80 deletions(-) diff --git a/src/edge_utils.c b/src/edge_utils.c index 2f253c1..09341c9 100644 --- a/src/edge_utils.c +++ b/src/edge_utils.c @@ -337,110 +337,114 @@ void supernode_connect (struct n3n_runtime_data *eee) { n2n_sock_t local_sock; n2n_sock_str_t sockbuf; - if((eee->conf.connect_tcp) && (eee->sock >= 0)) { + if(eee->conf.connect_tcp) { + // It might be already closed, but we can simply ignore errors and + // carry on closesocket(eee->sock); eee->sock = -1; } - if(eee->sock < 0) { + if(eee->sock >= 0) { + return; + } - eee->sock = open_socket( - eee->conf.bind_address, - sizeof(struct sockaddr_in), // FIXME this forces only IPv4 bindings - eee->conf.connect_tcp - ); + eee->sock = open_socket( + eee->conf.bind_address, + sizeof(struct sockaddr_in), // FIXME this forces only IPv4 bindings + eee->conf.connect_tcp + ); - if(eee->sock < 0) { - traceEvent(TRACE_ERROR, "failed to bind main UDP port"); - return; - } + if(eee->sock < 0) { + traceEvent(TRACE_ERROR, "failed to bind main UDP port"); + return; + } - fill_sockaddr((struct sockaddr*)&sn_sock, sizeof(sn_sock), &eee->curr_sn->sock); + fill_sockaddr((struct sockaddr*)&sn_sock, sizeof(sn_sock), &eee->curr_sn->sock); - // set tcp socket to O_NONBLOCK so connect does not hang - // requires checking the socket for readiness before sending and receving - if(eee->conf.connect_tcp) { + // set tcp socket to O_NONBLOCK so connect does not hang + // requires checking the socket for readiness before sending and receving + if(eee->conf.connect_tcp) { #ifdef _WIN32 - u_long value = 1; - ioctlsocket(eee->sock, FIONBIO, &value); + u_long value = 1; + ioctlsocket(eee->sock, FIONBIO, &value); #else - fcntl(eee->sock, F_SETFL, O_NONBLOCK); + fcntl(eee->sock, F_SETFL, O_NONBLOCK); #endif - if((connect(eee->sock, (struct sockaddr*)&(sn_sock), sizeof(struct sockaddr)) < 0) - && (errno != EINPROGRESS)) { - traceEvent(TRACE_INFO, "Error connecting TCP: %i", errno); - eee->sock = -1; - return; - } + if((connect(eee->sock, (struct sockaddr*)&(sn_sock), sizeof(struct sockaddr)) < 0) + && (errno != EINPROGRESS)) { + traceEvent(TRACE_INFO, "Error connecting TCP: %i", errno); + eee->sock = -1; + return; } + } - if(eee->conf.tos) { - /* - * See https://www.tucny.com/Home/dscp-tos for a quick table of - * the intended functions of each TOS value - * - * Note that the tos value is a byte and the manpage for IP_TOS - * defines it as a byte, but we hand setsockopt() an int value. - * This does work on linux, but - TODO, check this on other OS - */ - sockopt = eee->conf.tos; + if(eee->conf.tos) { + /* + * See https://www.tucny.com/Home/dscp-tos for a quick table of + * the intended functions of each TOS value + * + * Note that the tos value is a byte and the manpage for IP_TOS + * defines it as a byte, but we hand setsockopt() an int value. + * This does work on linux, but - TODO, check this on other OS + */ + sockopt = eee->conf.tos; - if(setsockopt(eee->sock, IPPROTO_IP, IP_TOS, (char *)&sockopt, sizeof(sockopt)) == 0) - traceEvent(TRACE_INFO, "TOS set to 0x%x", eee->conf.tos); - else - traceEvent(TRACE_WARNING, "could not set TOS 0x%x[%d]: %s", eee->conf.tos, errno, strerror(errno)); - } + if(setsockopt(eee->sock, IPPROTO_IP, IP_TOS, (char *)&sockopt, sizeof(sockopt)) == 0) + traceEvent(TRACE_INFO, "TOS set to 0x%x", eee->conf.tos); + else + traceEvent(TRACE_WARNING, "could not set TOS 0x%x[%d]: %s", eee->conf.tos, errno, strerror(errno)); + } #ifdef IP_PMTUDISC_DO - if(eee->conf.pmtu_discovery) { - sockopt = IP_PMTUDISC_DO; - } else { - sockopt = IP_PMTUDISC_DONT; - } - traceEvent( - TRACE_INFO, - "Setting pmtu_discovery %s", - (eee->conf.pmtu_discovery) ? "true" : "false" - ); + if(eee->conf.pmtu_discovery) { + sockopt = IP_PMTUDISC_DO; + } else { + sockopt = IP_PMTUDISC_DONT; + } + traceEvent( + TRACE_INFO, + "Setting pmtu_discovery %s", + (eee->conf.pmtu_discovery) ? "true" : "false" + ); - int i = setsockopt( - eee->sock, - IPPROTO_IP, - IP_MTU_DISCOVER, - &sockopt, - sizeof(sockopt) - ); + int i = setsockopt( + eee->sock, + IPPROTO_IP, + IP_MTU_DISCOVER, + &sockopt, + sizeof(sockopt) + ); - if(i < 0) { - traceEvent( - TRACE_WARNING, - "Setting pmtu_discovery failed: %s(%d)", - strerror(errno), - errno - ); - } + if(i < 0) { + traceEvent( + TRACE_WARNING, + "Setting pmtu_discovery failed: %s(%d)", + strerror(errno), + errno + ); + } #else - traceEvent(TRACE_INFO, "No platform support for setting pmtu_discovery"); + traceEvent(TRACE_INFO, "No platform support for setting pmtu_discovery"); #endif - if(detect_local_ip_address(&local_sock, eee) == 0) { - // always overwrite local port even/especially if chosen by OS... - eee->conf.preferred_sock.port = local_sock.port; - // only if auto-detection mode, ... - if(eee->conf.preferred_sock.family != AF_INVALID) { - // ... overwrite IP address, too (whole socket struct here) - memcpy(&eee->conf.preferred_sock, &local_sock, sizeof(n2n_sock_t)); - traceEvent(TRACE_INFO, "determined local socket [%s]", - sock_to_cstr(sockbuf, &local_sock)); - } + if(detect_local_ip_address(&local_sock, eee) == 0) { + // always overwrite local port even/especially if chosen by OS... + eee->conf.preferred_sock.port = local_sock.port; + // only if auto-detection mode, ... + if(eee->conf.preferred_sock.family != AF_INVALID) { + // ... overwrite IP address, too (whole socket struct here) + memcpy(&eee->conf.preferred_sock, &local_sock, sizeof(n2n_sock_t)); + traceEvent(TRACE_INFO, "determined local socket [%s]", + sock_to_cstr(sockbuf, &local_sock)); } + } - if(!eee->conf.connect_tcp) { - mainloop_register_fd(eee->sock, fd_info_proto_v3udp); - } + if(!eee->conf.connect_tcp) { + mainloop_register_fd(eee->sock, fd_info_proto_v3udp); } - // REVISIT: add mgmt port notification to listener for better mgmt port - // subscription support + // REVISIT: TODO: + // - add a management event for "new supernode socket" to make it simpler + // to track subscriptions externally return; } From cb7943900e863157b174e70fdbefb35349cb78d4 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Thu, 28 Nov 2024 19:22:47 +1100 Subject: [PATCH 62/85] Refactor function for readability and early exits --- src/edge_utils.c | 41 +++++++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/src/edge_utils.c b/src/edge_utils.c index 09341c9..34690a2 100644 --- a/src/edge_utils.c +++ b/src/edge_utils.c @@ -359,7 +359,13 @@ void supernode_connect (struct n3n_runtime_data *eee) { return; } - fill_sockaddr((struct sockaddr*)&sn_sock, sizeof(sn_sock), &eee->curr_sn->sock); + if(!eee->conf.connect_tcp) { + mainloop_register_fd(eee->sock, fd_info_proto_v3udp); + } + + // REVISIT: TODO: + // - add a management event for "new supernode socket" to make it simpler + // to track subscriptions externally // set tcp socket to O_NONBLOCK so connect does not hang // requires checking the socket for readiness before sending and receving @@ -370,6 +376,9 @@ void supernode_connect (struct n3n_runtime_data *eee) { #else fcntl(eee->sock, F_SETFL, O_NONBLOCK); #endif + + fill_sockaddr((struct sockaddr*)&sn_sock, sizeof(sn_sock), &eee->curr_sn->sock); + if((connect(eee->sock, (struct sockaddr*)&(sn_sock), sizeof(struct sockaddr)) < 0) && (errno != EINPROGRESS)) { traceEvent(TRACE_INFO, "Error connecting TCP: %i", errno); @@ -394,6 +403,7 @@ void supernode_connect (struct n3n_runtime_data *eee) { else traceEvent(TRACE_WARNING, "could not set TOS 0x%x[%d]: %s", eee->conf.tos, errno, strerror(errno)); } + #ifdef IP_PMTUDISC_DO if(eee->conf.pmtu_discovery) { sockopt = IP_PMTUDISC_DO; @@ -426,27 +436,22 @@ void supernode_connect (struct n3n_runtime_data *eee) { traceEvent(TRACE_INFO, "No platform support for setting pmtu_discovery"); #endif - if(detect_local_ip_address(&local_sock, eee) == 0) { - // always overwrite local port even/especially if chosen by OS... - eee->conf.preferred_sock.port = local_sock.port; - // only if auto-detection mode, ... - if(eee->conf.preferred_sock.family != AF_INVALID) { - // ... overwrite IP address, too (whole socket struct here) - memcpy(&eee->conf.preferred_sock, &local_sock, sizeof(n2n_sock_t)); - traceEvent(TRACE_INFO, "determined local socket [%s]", - sock_to_cstr(sockbuf, &local_sock)); - } + if(detect_local_ip_address(&local_sock, eee) != 0) { + return; } - if(!eee->conf.connect_tcp) { - mainloop_register_fd(eee->sock, fd_info_proto_v3udp); - } + // always overwrite local port even/especially if chosen by OS... + eee->conf.preferred_sock.port = local_sock.port; - // REVISIT: TODO: - // - add a management event for "new supernode socket" to make it simpler - // to track subscriptions externally + // only if auto-detection mode, ... + if(eee->conf.preferred_sock.family == AF_INVALID) { + return; + } - return; + // ... overwrite IP address, too (whole socket struct here) + memcpy(&eee->conf.preferred_sock, &local_sock, sizeof(n2n_sock_t)); + traceEvent(TRACE_INFO, "determined local socket [%s]", + sock_to_cstr(sockbuf, &local_sock)); } From 0c92614fe762f462a9907f503f4d370a04bfe289 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Thu, 28 Nov 2024 19:23:50 +1100 Subject: [PATCH 63/85] Fix spelling --- src/edge_utils.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/edge_utils.c b/src/edge_utils.c index 34690a2..9427993 100644 --- a/src/edge_utils.c +++ b/src/edge_utils.c @@ -1066,7 +1066,7 @@ static bool check_sock_ready (struct n3n_runtime_data *eee) { return false; } - // if required (tcp), wait until writeable as soket is set to + // if required (tcp), wait until writeable as socket is set to // O_NONBLOCK, could require some wait time directly after re-opening fd_set socket_mask; struct timeval wait_time; From f2a122653f05120f6fed25ab754ab32e4ea3bb40 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Thu, 28 Nov 2024 19:57:00 +1100 Subject: [PATCH 64/85] Refactor function for early exit and readability --- src/edge_utils.c | 78 ++++++++++++++++++++++++------------------------ 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/src/edge_utils.c b/src/edge_utils.c index 9427993..16d1cc8 100644 --- a/src/edge_utils.c +++ b/src/edge_utils.c @@ -1423,56 +1423,56 @@ static void send_unregister_super (struct n3n_runtime_data *eee) { } -static int sort_supernodes (struct n3n_runtime_data *eee, time_t now) { +static void sort_supernodes (struct n3n_runtime_data *eee, time_t now) { struct peer_info *scan, *tmp; - if(now - eee->last_sweep > SWEEP_TIME) { - // this routine gets periodically called - - if(!eee->sn_wait) { - // sort supernodes in ascending order of their selection_criterion fields - sn_selection_sort(&(eee->conf.supernodes)); - } + if(now - eee->last_sweep <= SWEEP_TIME) { + return; + } - if(eee->curr_sn != eee->conf.supernodes) { - // we have not been connected to the best/top one - send_unregister_super(eee); - eee->curr_sn = eee->conf.supernodes; - reset_sup_attempts(eee); - supernode_connect(eee); + // this routine gets periodically called - traceEvent( - TRACE_INFO, - "registering with supernode [%s][number of supernodes %d][attempts left %u]", - peer_info_get_hostname(eee->curr_sn), - HASH_COUNT(eee->conf.supernodes), - (unsigned int)eee->sup_attempts - ); + if(!eee->sn_wait) { + // sort supernodes in ascending order of their selection_criterion fields + sn_selection_sort(&(eee->conf.supernodes)); + } - send_register_super(eee); - eee->last_register_req = now; - eee->sn_wait = 1; - } + if(eee->curr_sn != eee->conf.supernodes) { + // we have not been connected to the best/top one + send_unregister_super(eee); + eee->curr_sn = eee->conf.supernodes; + reset_sup_attempts(eee); + supernode_connect(eee); - HASH_ITER(hh, eee->conf.supernodes, scan, tmp) { - if(scan == eee->curr_sn) - sn_selection_criterion_good(&(scan->selection_criterion)); - else - sn_selection_criterion_default(&(scan->selection_criterion)); - } - sn_selection_criterion_common_data_default(eee); + traceEvent( + TRACE_INFO, + "registering with supernode [%s][number of supernodes %d][attempts left %u]", + peer_info_get_hostname(eee->curr_sn), + HASH_COUNT(eee->conf.supernodes), + (unsigned int)eee->sup_attempts + ); - // send PING to all the supernodes - if(!eee->conf.connect_tcp) - send_query_peer(eee, null_mac); - eee->last_sweep = now; + send_register_super(eee); + eee->last_register_req = now; + eee->sn_wait = 1; + } - // no answer yet (so far, unused in regular edge code; mainly used during bootstrap loading) - eee->sn_pong = 0; + HASH_ITER(hh, eee->conf.supernodes, scan, tmp) { + if(scan == eee->curr_sn) + sn_selection_criterion_good(&(scan->selection_criterion)); + else + sn_selection_criterion_default(&(scan->selection_criterion)); } + sn_selection_criterion_common_data_default(eee); + + // send PING to all the supernodes + if(!eee->conf.connect_tcp) + send_query_peer(eee, null_mac); + eee->last_sweep = now; - return 0; /* OK */ + // no answer yet (so far, unused in regular edge code; mainly used during bootstrap loading) + eee->sn_pong = 0; } /** Send a REGISTER packet to another edge. */ From 3a7324db4a75d29e7b4e06185105e20b5e89e887 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Thu, 28 Nov 2024 20:53:53 +1100 Subject: [PATCH 65/85] Ensure wire function fully initialises its return. It also reminds us that this n2nsock structure is used in memcmp operations for some of the hash lookups - which could possibly be optimised in the IPv4 case. --- src/edge_utils.c | 2 -- src/sn_utils.c | 1 - src/wire.c | 3 +++ 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/edge_utils.c b/src/edge_utils.c index 16d1cc8..28862c6 100644 --- a/src/edge_utils.c +++ b/src/edge_utils.c @@ -2328,8 +2328,6 @@ void process_udp (struct n3n_runtime_data *eee, // TCP expects that we know our comm partner and does not deliver the sender memcpy(&sender, &(eee->curr_sn->sock), sizeof(sender)); else { - // FIXME: do not do random memset on the packet processing path - memset(&sender, 0, sizeof(sender)); // REVISIT: type conversion back and forth, choose a consistent approach throughout whole code, // i.e. stick with more general sockaddr as long as possible and narrow only if required fill_n2nsock(&sender, sender_sock, type); diff --git a/src/sn_utils.c b/src/sn_utils.c index 2504e20..3af5197 100644 --- a/src/sn_utils.c +++ b/src/sn_utils.c @@ -1662,7 +1662,6 @@ static int process_udp (struct n3n_runtime_data * sss, int skip_add; time_t any_time = 0; - memset(&sender, 0, sizeof(n2n_sock_t)); fill_n2nsock(&sender, sender_sock, SOCK_DGRAM); orig_sender = &sender; diff --git a/src/wire.c b/src/wire.c index 3447f44..b7160e2 100644 --- a/src/wire.c +++ b/src/wire.c @@ -691,6 +691,9 @@ int fill_sockaddr (struct sockaddr * addr, // fills struct sockaddr's data into n2n_sock int fill_n2nsock (n2n_sock_t* sock, const struct sockaddr* sa, int type) { + // Ensure the return struct is fully initialised + // TODO: could be optimised + memset(sock, 0, sizeof(n2n_sock_t)); sock->family = sa->sa_family; // TODO: re enable this when it doesnt break things From bdce818ff966898c973c1046054bc6f25e1f0f0a Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Mon, 2 Dec 2024 18:38:15 +1100 Subject: [PATCH 66/85] Factor out the part of conn_read that checks if a buffer is ready as it may be used on streams that are not reading --- libs/connslot/connslot.c | 68 +++++++++++++++++++++------------------- libs/connslot/connslot.h | 1 + 2 files changed, 37 insertions(+), 32 deletions(-) diff --git a/libs/connslot/connslot.c b/libs/connslot/connslot.c index dd7f248..f52e29e 100644 --- a/libs/connslot/connslot.c +++ b/libs/connslot/connslot.c @@ -104,38 +104,7 @@ void conn_accept(conn_t *conn, int fd, enum conn_proto proto) { conn->proto = proto; } -void conn_read(conn_t *conn, int fd) { - conn->state = CONN_READING; - - // If no space available, try increasing our capacity - if (!sb_avail(conn->request)) { - strbuf_t *p = sb_realloc(&conn->request, conn->request->capacity + 16); - if (!p) { - abort(); // FIXME: do something smarter? - } - } - - ssize_t size = sb_read(fd, conn->request); - - if (size == 0) { - // As we are dealing with non blocking sockets, and have made a non - // zero-sized read request, the only time we get a zero back is if the - // far end has closed - conn->state = CONN_CLOSED; - return; - } - - if (size == -1) { - if (errno == EWOULDBLOCK || errno == EAGAIN) { - conn->state = CONN_EMPTY; - return; - } - conn->state = CONN_ERROR; - return; - } - - // This will truncate the time to a int - usually 32bits - conn->activity = time(NULL); +void conn_check_ready(conn_t *conn) { unsigned int expected_length; switch (conn->proto) { @@ -208,6 +177,41 @@ void conn_read(conn_t *conn, int fd) { return; } +void conn_read(conn_t *conn, int fd) { + conn->state = CONN_READING; + + // If no space available, try increasing our capacity + if (!sb_avail(conn->request)) { + strbuf_t *p = sb_realloc(&conn->request, conn->request->capacity + 16); + if (!p) { + abort(); // FIXME: do something smarter? + } + } + + ssize_t size = sb_read(fd, conn->request); + + if (size == 0) { + // As we are dealing with non blocking sockets, and have made a non + // zero-sized read request, the only time we get a zero back is if the + // far end has closed + conn->state = CONN_CLOSED; + return; + } + + if (size == -1) { + if (errno == EWOULDBLOCK || errno == EAGAIN) { + conn->state = CONN_EMPTY; + return; + } + conn->state = CONN_ERROR; + return; + } + + // This will truncate the time to a int - usually 32bits + conn->activity = time(NULL); + conn_check_ready(conn); +} + ssize_t conn_write(conn_t *conn, int fd) { ssize_t sent; diff --git a/libs/connslot/connslot.h b/libs/connslot/connslot.h index cbd5a8c..64d1760 100644 --- a/libs/connslot/connslot.h +++ b/libs/connslot/connslot.h @@ -63,6 +63,7 @@ typedef struct slots { void conn_zero(conn_t *); int conn_init(conn_t *, size_t, size_t); void conn_accept(conn_t *, int, enum conn_proto); +void conn_check_ready(conn_t *); void conn_read(conn_t *, int); ssize_t conn_write(conn_t *, int); int conn_iswriter(conn_t *); From bd069646a0db543c6eb0887104b2aed2ec0b5466 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Mon, 2 Dec 2024 18:38:44 +1100 Subject: [PATCH 67/85] Fix expected size calculation --- libs/connslot/connslot.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/connslot/connslot.c b/libs/connslot/connslot.c index f52e29e..4366b0a 100644 --- a/libs/connslot/connslot.c +++ b/libs/connslot/connslot.c @@ -153,7 +153,7 @@ void conn_check_ready(conn_t *conn) { return; } - expected_length = ntohs(*(uint16_t *)&conn->request->str); + expected_length = ntohs(*(uint16_t *)&conn->request->str) + 2; break; default: From 4b138afc784a34e6e78f08e822d9597aadf9d22c Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Mon, 2 Dec 2024 18:39:09 +1100 Subject: [PATCH 68/85] Export the debug text generator for use by library users --- libs/connslot/connslot.h | 1 + 1 file changed, 1 insertion(+) diff --git a/libs/connslot/connslot.h b/libs/connslot/connslot.h index 64d1760..fc24325 100644 --- a/libs/connslot/connslot.h +++ b/libs/connslot/connslot.h @@ -69,6 +69,7 @@ ssize_t conn_write(conn_t *, int); int conn_iswriter(conn_t *); void conn_close(conn_t *, int); bool conn_closeidle(conn_t *, int, int, int); +void conn_dump(strbuf_t **, conn_t *); void slots_free(slots_t *slots); slots_t *slots_malloc(int nr_slots, size_t, size_t); From d61a50929161790f0224ea597fa071a18e314c05 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Mon, 2 Dec 2024 18:39:40 +1100 Subject: [PATCH 69/85] An include what you use fixup --- include/hexdump.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/hexdump.h b/include/hexdump.h index b4edd83..15ea79a 100644 --- a/include/hexdump.h +++ b/include/hexdump.h @@ -6,6 +6,8 @@ #ifndef HEXDUMP_H #define HEXDUMP_H +#include + void fhexdump(unsigned int display_addr, void *in, int size, FILE *stream); #endif From 28496f92ca70adc4a11e0332ef6cb9bcf17b74ee Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Mon, 2 Dec 2024 18:44:59 +1100 Subject: [PATCH 70/85] Refactor edge TCP mode receive path A send path that queues via the mainloop still needs to be built. --- apps/n3n-edge.c | 41 +++----------- include/n3n/edge.h | 3 +- include/n3n/mainloop.h | 4 ++ src/edge_utils.c | 96 +++++++------------------------- src/mainloop.c | 122 ++++++++++++++++++++++++++++++++++++++--- src/management.c | 17 ++++-- 6 files changed, 157 insertions(+), 126 deletions(-) diff --git a/apps/n3n-edge.c b/apps/n3n-edge.c index 5b156d7..7b60b40 100644 --- a/apps/n3n-edge.c +++ b/apps/n3n-edge.c @@ -743,14 +743,8 @@ int main (int argc, char* argv[]) { uint8_t seek_answer = 1; /* expecting answer from supernode */ time_t now, last_action = 0; /* timeout */ macstr_t mac_buf; /* output mac address */ - fd_set socket_mask; /* for supernode answer */ - struct timeval wait_time; /* timeout for sn answer */ peer_info_t *scan, *scan_tmp; /* supernode iteration */ - uint16_t expected = sizeof(uint16_t); - uint16_t position = 0; - uint8_t pktbuf[N2N_SN_PKTBUF_SIZE + sizeof(uint16_t)]; /* buffer + prepended buffer length in case of tcp */ - #ifdef HAVE_LIBCAP cap_t caps; #endif @@ -830,6 +824,7 @@ int main (int argc, char* argv[]) { traceEvent(TRACE_ERROR, "failed in edge_init"); exit(1); } + eee->keep_running = &keep_on_running; switch(eee->conf.tuntap_ip_mode) { case TUNTAP_IP_MODE_SN_ASSIGN: @@ -974,33 +969,12 @@ int main (int argc, char* argv[]) { // we usually wait for some answer, there however are exceptions when going back to a previous runlevel if(seek_answer) { - FD_ZERO(&socket_mask); - FD_SET(eee->sock, &socket_mask); - wait_time.tv_sec = BOOTSTRAP_TIMEOUT; - wait_time.tv_usec = 0; - - if(select(eee->sock + 1, &socket_mask, NULL, NULL, &wait_time) > 0) { - if(FD_ISSET(eee->sock, &socket_mask)) { - if(!eee->conf.connect_tcp) { - edge_read_proto3_udp( - eee, - eee->sock, - pktbuf, - sizeof(pktbuf), - now - ); - } else { - edge_read_proto3_tcp( - eee, - eee->sock, - pktbuf, - &expected, - &position, - now - ); - } - } - } + fd_set readers; + fd_set writers; + mainloop_runonce(&readers, &writers, eee); + + // FIXME: the mainloop could wait for BOOTSTRAP_TIMEOUT, not its + // usual timeout ?!? } seek_answer = 1; @@ -1082,7 +1056,6 @@ int main (int argc, char* argv[]) { SetConsoleCtrlHandler(ConsoleCtrlHandler, TRUE); #endif - eee->keep_running = &keep_on_running; traceEvent(TRACE_NORMAL, "edge started"); rc = run_edge_loop(eee); print_edge_stats(eee); diff --git a/include/n3n/edge.h b/include/n3n/edge.h index 748655b..5e7311a 100644 --- a/include/n3n/edge.h +++ b/include/n3n/edge.h @@ -37,8 +37,7 @@ void edge_read_proto3_udp (struct n3n_runtime_data *eee, void edge_read_proto3_tcp (struct n3n_runtime_data *eee, SOCKET sock, uint8_t *pktbuf, - uint16_t *expected, - uint16_t *position, + ssize_t pktbuf_len, time_t now); #endif diff --git a/include/n3n/mainloop.h b/include/n3n/mainloop.h index eeea9aa..3d4d0c0 100644 --- a/include/n3n/mainloop.h +++ b/include/n3n/mainloop.h @@ -22,9 +22,13 @@ enum __attribute__((__packed__)) fd_info_proto { fd_info_proto_tuntap, fd_info_proto_listen_http, fd_info_proto_v3udp, + fd_info_proto_v3tcp, fd_info_proto_http, }; +// Place debug info from the slots into the strbuf +void mainloop_dump (strbuf_t **); + int mainloop_runonce (fd_set *, fd_set *, struct n3n_runtime_data *); void mainloop_register_fd (int, enum fd_info_proto); diff --git a/src/edge_utils.c b/src/edge_utils.c index 28862c6..a62fc4d 100644 --- a/src/edge_utils.c +++ b/src/edge_utils.c @@ -340,6 +340,7 @@ void supernode_connect (struct n3n_runtime_data *eee) { if(eee->conf.connect_tcp) { // It might be already closed, but we can simply ignore errors and // carry on + mainloop_unregister_fd(eee->sock); closesocket(eee->sock); eee->sock = -1; } @@ -359,7 +360,9 @@ void supernode_connect (struct n3n_runtime_data *eee) { return; } - if(!eee->conf.connect_tcp) { + if(eee->conf.connect_tcp) { + mainloop_register_fd(eee->sock, fd_info_proto_v3tcp); + } else { mainloop_register_fd(eee->sock, fd_info_proto_v3udp); } @@ -2324,6 +2327,7 @@ void process_udp (struct n3n_runtime_data *eee, /* REVISIT: when UDP/IPv6 is supported we will need a flag to indicate which * IP transport version the packet arrived on. May need to UDP sockets. */ + // TODO: pass the sender to process_udp, dont calculate it here if(eee->conf.connect_tcp) // TCP expects that we know our comm partner and does not deliver the sender memcpy(&sender, &(eee->curr_sn->sock), sizeof(sender)); @@ -2947,31 +2951,14 @@ void edge_read_proto3_udp (struct n3n_runtime_data *eee, void edge_read_proto3_tcp (struct n3n_runtime_data *eee, SOCKET sock, uint8_t *pktbuf, - uint16_t *expected, - uint16_t *position, + ssize_t pktbuf_len, time_t now) { - struct sockaddr_storage sas; - struct sockaddr *sender_sock = (struct sockaddr*)&sas; - socklen_t ss_size = sizeof(sas); - // tcp - ssize_t bread = recvfrom( - sock, - (void *)(pktbuf + *position), - *expected - *position, - 0 /*flags*/, - sender_sock, - &ss_size - ); + // tcp gets handed a pre filled pktbuf - if((bread <= 0) && (errno)) { - traceEvent( - TRACE_ERROR, - "recvfrom() failed %d errno %d (%s)", - bread, - errno, - strerror(errno) - ); + // zero contents means an error + if(pktbuf_len <= 0) { + traceEvent(TRACE_ERROR, "tcp conn read error %i", pktbuf_len); #ifdef _WIN32 traceEvent(TRACE_ERROR, "WSAGetLastError(): %u", WSAGetLastError()); #endif @@ -2980,37 +2967,23 @@ void edge_read_proto3_tcp (struct n3n_runtime_data *eee, return; } - *position = *position + bread; - - if(*position != *expected) { - // not enough bytes yet to process buffer - return; - } - - if(*position == sizeof(uint16_t)) { - // the prepended length has been read, preparing for the packet - *expected = *expected + be16toh(*(uint16_t*)(pktbuf)); - if(*expected > N2N_PKT_BUF_SIZE) { - supernode_disconnect(eee); - eee->sn_wait = 1; - traceEvent(TRACE_DEBUG, "too many bytes expected"); - } + if(pktbuf_len > N2N_PKT_BUF_SIZE + 2) { + supernode_disconnect(eee); + eee->sn_wait = 1; + traceEvent(TRACE_DEBUG, "too many bytes expected"); return; } - // full packet read, handle it + // have a valid packet read, handle it process_udp( eee, - sender_sock, + NULL, sock, - pktbuf + sizeof(uint16_t), - *position - sizeof(uint16_t), + pktbuf, + pktbuf_len, now, SOCK_STREAM ); - // reset, await new prepended length - *expected = sizeof(uint16_t); - *position = 0; return; } @@ -3041,10 +3014,6 @@ int run_edge_loop (struct n3n_runtime_data *eee) { time_t last_purge_host = 0; #endif - uint16_t expected = sizeof(uint16_t); - uint16_t position = 0; - uint8_t pktbuf[N2N_PKT_BUF_SIZE + sizeof(uint16_t)]; /* buffer + prepended buffer length in case of tcp */ - #ifdef _WIN32 struct tunread_arg arg; arg.eee = eee; @@ -3068,39 +3037,12 @@ int run_edge_loop (struct n3n_runtime_data *eee) { while(*eee->keep_running) { - int rc; fd_set readers; fd_set writers; - rc = mainloop_runonce(&readers, &writers, eee); + mainloop_runonce(&readers, &writers, eee); time_t now = time(NULL); - if(rc > 0) { - // any or all of the FDs could have input; check them all - - // external TCP packets - if((eee->conf.connect_tcp) && (eee->sock != -1) && FD_ISSET(eee->sock, &readers)) { - edge_read_proto3_tcp( - eee, - eee->sock, - pktbuf, - &expected, - &position, - now - ); - - if((expected >= N2N_PKT_BUF_SIZE) || (position >= N2N_PKT_BUF_SIZE)) { - // something went wrong, possibly even before - // e.g. connection failure/closure in the middle of transmission (between len & data) - supernode_disconnect(eee); - eee->sn_wait = 1; - - expected = sizeof(uint16_t); - position = 0; - } - } - } - // If anything we recieved caused us to stop.. if(!(*eee->keep_running)) break; diff --git a/src/mainloop.c b/src/mainloop.c index 7d5d6d6..5628599 100644 --- a/src/mainloop.c +++ b/src/mainloop.c @@ -71,6 +71,7 @@ static char *proto_str[] = { [fd_info_proto_tuntap] = "tuntap", [fd_info_proto_listen_http] = "listen_http", [fd_info_proto_v3udp] = "v3udp", + [fd_info_proto_v3tcp] = "v3tcp", [fd_info_proto_http] = "http", }; @@ -165,8 +166,10 @@ static void connlist_free (int connnr) { // TODO: error! return; } + connlist[connnr].fd = -1; connlist[connnr].proto = CONN_PROTO_UNK; - fdlist_next_search = connnr; + connlist[connnr].state = CONN_EMPTY; + connlist_next_search = connnr; metrics.connlist_free++; } @@ -191,12 +194,27 @@ static int fdlist_allocslot (int fd, enum fd_info_proto proto) { fdlist[slot].fd = fd; fdlist[slot].proto = proto; fdlist[slot].stats_reads = 0; + + if(proto == fd_info_proto_v3tcp) { + int connnr = connlist_alloc(CONN_PROTO_BE16LEN); + assert(connnr != -1); + + fdlist[slot].connnr = connnr; + conn_accept(&connlist[connnr], fd, CONN_PROTO_BE16LEN); + } else { + fdlist[slot].connnr = -1; + } + fdlist_next_search = slot + 1; return slot; } slot = (slot + 1) % MAX_HANDLES; count--; } + + // TODO: the moment this starts to fire, we need to revamp the + // implementation of the fdlist table + assert(slot != -1); return -1; } @@ -208,12 +226,13 @@ static void fdlist_freefd (int fd) { continue; } metrics.unregister_fd++; - fdlist[slot].fd = -1; - fdlist[slot].proto = fd_info_proto_unknown; - fdlist_next_search = slot; if(fdlist[slot].connnr != -1) { connlist_free(fdlist[slot].connnr); + fdlist[slot].connnr = -1; } + fdlist[slot].fd = -1; + fdlist[slot].proto = fd_info_proto_unknown; + fdlist_next_search = slot; return; } @@ -299,6 +318,74 @@ static void handle_fd (const time_t now, const struct fd_info info, struct n3n_r return; } + case fd_info_proto_v3tcp: { + struct conn *conn = &connlist[info.connnr]; + conn_read(conn, info.fd); + + switch(conn->state) { + case CONN_EMPTY: + case CONN_READING: + case CONN_SENDING: + // These states dont require us to do anything + // TODO: + // - handle reading/sending simultaneous? + return; + + case CONN_ERROR: + case CONN_CLOSED: + conn_close(conn, info.fd); + sb_zero(conn->request); + // Let the upper layer realise its connection is gone by + // showing it a zero sized request + + // TODO: if the upper layer doesnt react properly, we leak + // slots and conns here + + /* fall through */ + + case CONN_READY: + int size = ntohs(*(uint16_t *)&conn->request->str); + + edge_read_proto3_tcp( + eee, + info.fd, + (uint8_t *)&conn->request->str[2], + size, + now + ); + + if(sb_len(conn->request) == (size + 2)) { + // We read exactly one packet + // TODO: this crosses layers by reaching inside the + // conn object + sb_zero(conn->request); + conn->state = CONN_EMPTY; + return; + } + + // Our buffer contains data beyond the single packet + + // TODO: this crosses layers by reaching inside the + // conn object + int more = sb_len(conn->request) - (size + 2); + traceEvent(TRACE_DEBUG, "packet has %i more bytes", more); + memmove( + conn->request->str, + &conn->request->str[size + 2], + more + ); + conn->request->rd_pos = 0; + conn->request->wr_pos = more; + conn->state = CONN_READING; + + // FIXME: sometimes we will have an entire next packet in + // the buffer, which means we should not wait for the FD + // to be read ready again + return; + } + return; + } + case fd_info_proto_http: { struct conn *conn = &connlist[info.connnr]; conn_read(conn, info.fd); @@ -414,12 +501,29 @@ int mainloop_runonce (fd_set *rd, fd_set *wr, struct n3n_runtime_data *eee) { return ready; } -void mainloop_register_fd (int fd, enum fd_info_proto proto) { - int slot = fdlist_allocslot(fd, proto); +void mainloop_dump (strbuf_t **buf) { + int i; + sb_reprintf(buf, "i : fd(read) pr connnr\n"); + for(i=0; irequest); - if(eee->conf.enable_debug_pages) { - slots_dump(&conn->request, eee->mgmt_slots); - status = 200; - } else { + if(!eee->conf.enable_debug_pages) { sb_printf(conn->request, "enable_debug_pages is false\n"); status = 403; + // Update the reply buffer after last potential realloc + conn->reply = conn->request; + generate_http_headers(conn, "text/plain", status); + return; } + + if(eee->mgmt_slots) { + slots_dump(&conn->request, eee->mgmt_slots); + sb_reprintf(&conn->request, "\n"); + } + mainloop_dump(&conn->request); + + status = 200; // Update the reply buffer after last potential realloc conn->reply = conn->request; generate_http_headers(conn, "text/plain", status); From 4702469350c38f19848e8e2dd09a1b402d5deb08 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Mon, 2 Dec 2024 18:52:14 +1100 Subject: [PATCH 71/85] Refactor the error path to be simpler - also addressing a lint concern --- src/mainloop.c | 8 +++++--- src/management.c | 5 +++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/mainloop.c b/src/mainloop.c index 5628599..3967c80 100644 --- a/src/mainloop.c +++ b/src/mainloop.c @@ -338,10 +338,12 @@ static void handle_fd (const time_t now, const struct fd_info info, struct n3n_r // Let the upper layer realise its connection is gone by // showing it a zero sized request - // TODO: if the upper layer doesnt react properly, we leak - // slots and conns here + // TODO: if the upper layer doesnt react properly by + // unregistering the dead filehandle, we leak slots and + // conns here - /* fall through */ + edge_read_proto3_tcp(eee, -1, NULL, -1, now); + return; case CONN_READY: int size = ntohs(*(uint16_t *)&conn->request->str); diff --git a/src/management.c b/src/management.c index 44ab576..7b60dcf 100644 --- a/src/management.c +++ b/src/management.c @@ -1188,6 +1188,11 @@ void mgmt_api_handler (struct n3n_runtime_data *eee, conn_t *conn) { } } if( i >= nr_handlers ) { + traceEvent( + TRACE_DEBUG, + "unknown endpoint %s", + conn->request->str + ); render_error(conn, "unknown endpoint"); } else { api_endpoints[i].func(eee, conn); From f3a0cd8e9191b1ceda1ac503f46972dfedf3041c Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Mon, 2 Dec 2024 18:59:13 +1100 Subject: [PATCH 72/85] Fix the missing required include --- include/n3n/mainloop.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/n3n/mainloop.h b/include/n3n/mainloop.h index 3d4d0c0..534306e 100644 --- a/include/n3n/mainloop.h +++ b/include/n3n/mainloop.h @@ -12,7 +12,9 @@ #ifndef _MAINLOOP_H_ #define _MAINLOOP_H_ +#include // for strbuf_t #include // for n3n_runtime_data + #ifndef _WIN32 #include // for fd_set #endif From 7eddf935f884a2123fa5211cb9f0d6b6ab0abd7c Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Mon, 2 Dec 2024 19:05:44 +1100 Subject: [PATCH 73/85] Fix compile on older standard compilers --- src/mainloop.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/mainloop.c b/src/mainloop.c index 3967c80..dbf878f 100644 --- a/src/mainloop.c +++ b/src/mainloop.c @@ -345,7 +345,7 @@ static void handle_fd (const time_t now, const struct fd_info info, struct n3n_r edge_read_proto3_tcp(eee, -1, NULL, -1, now); return; - case CONN_READY: + case CONN_READY: { int size = ntohs(*(uint16_t *)&conn->request->str); edge_read_proto3_tcp( @@ -384,6 +384,7 @@ static void handle_fd (const time_t now, const struct fd_info info, struct n3n_r // the buffer, which means we should not wait for the FD // to be read ready again return; + } } return; } From 7d5fb89d85d054f38d6663da8069e218d63f3726 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Sat, 7 Dec 2024 14:34:50 +1100 Subject: [PATCH 74/85] Refactor tests to make logic easier to explain --- apps/n3n-edge.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/apps/n3n-edge.c b/apps/n3n-edge.c index 7b60b40..f752400 100644 --- a/apps/n3n-edge.c +++ b/apps/n3n-edge.c @@ -845,11 +845,18 @@ int main (int argc, char* argv[]) { // for the sake of quickly establishing connection. REVISIT when a more elegant way to re-use main loop code // is found - // find at least one supernode alive to faster establish connection + // find at least one supernode alive to faster establish connection. // exceptions: - if((HASH_COUNT(eee->conf.supernodes) <= 1) || (eee->conf.connect_tcp) || (eee->conf.shared_secret)) { - // skip the initial supernode ping - traceEvent(TRACE_DEBUG, "skip PING to supernode"); + if(eee->conf.connect_tcp) { + traceEvent(TRACE_DEBUG, "skip PING to supernode: TCP mode"); + runlevel = 2; + } + if(eee->conf.shared_secret) { + traceEvent(TRACE_DEBUG, "skip PING to supernode: shared secret"); + runlevel = 2; + } + if(HASH_COUNT(eee->conf.supernodes) <= 1) { + traceEvent(TRACE_DEBUG, "skip PING to supernode: only one supernode"); runlevel = 2; } From 9ee4eee52421bd020da13ee1846643ad429bc98e Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Sat, 7 Dec 2024 14:41:44 +1100 Subject: [PATCH 75/85] Migrate the fd sets into the mainloop function --- apps/n3n-edge.c | 4 +--- include/n3n/mainloop.h | 2 +- src/edge_utils.c | 7 ++++--- src/mainloop.c | 25 ++++++++----------------- 4 files changed, 14 insertions(+), 24 deletions(-) diff --git a/apps/n3n-edge.c b/apps/n3n-edge.c index f752400..7867fdd 100644 --- a/apps/n3n-edge.c +++ b/apps/n3n-edge.c @@ -976,9 +976,7 @@ int main (int argc, char* argv[]) { // we usually wait for some answer, there however are exceptions when going back to a previous runlevel if(seek_answer) { - fd_set readers; - fd_set writers; - mainloop_runonce(&readers, &writers, eee); + mainloop_runonce(eee); // FIXME: the mainloop could wait for BOOTSTRAP_TIMEOUT, not its // usual timeout ?!? diff --git a/include/n3n/mainloop.h b/include/n3n/mainloop.h index 534306e..594f305 100644 --- a/include/n3n/mainloop.h +++ b/include/n3n/mainloop.h @@ -31,7 +31,7 @@ enum __attribute__((__packed__)) fd_info_proto { // Place debug info from the slots into the strbuf void mainloop_dump (strbuf_t **); -int mainloop_runonce (fd_set *, fd_set *, struct n3n_runtime_data *); +int mainloop_runonce (struct n3n_runtime_data *); void mainloop_register_fd (int, enum fd_info_proto); void mainloop_unregister_fd (int); diff --git a/src/edge_utils.c b/src/edge_utils.c index a62fc4d..8d81626 100644 --- a/src/edge_utils.c +++ b/src/edge_utils.c @@ -3036,11 +3036,12 @@ int run_edge_loop (struct n3n_runtime_data *eee) { */ while(*eee->keep_running) { + mainloop_runonce(eee); - fd_set readers; - fd_set writers; + // TODO: + // - migrate all the following regular actions into the + // mainloop_runonce() function - mainloop_runonce(&readers, &writers, eee); time_t now = time(NULL); // If anything we recieved caused us to stop.. diff --git a/src/mainloop.c b/src/mainloop.c index dbf878f..1aec226 100644 --- a/src/mainloop.c +++ b/src/mainloop.c @@ -456,24 +456,15 @@ static void fdlist_check_ready (fd_set *rd, fd_set *wr, const time_t now, struct } } -static int setup_select (fd_set *rd, fd_set *wr, struct n3n_runtime_data *eee) { - FD_ZERO(rd); - FD_ZERO(wr); - int max_sock = fdlist_fd_set(rd, wr); - - // HACK - until this mainloop supports v3tcp proto sockets - if((eee->conf.connect_tcp) && (eee->sock != -1)) { - FD_SET(eee->sock, rd); - max_sock = MAX(max_sock, eee->sock); - } - - return max_sock; -} +int mainloop_runonce (struct n3n_runtime_data *eee) { + fd_set rd; + fd_set wr; -int mainloop_runonce (fd_set *rd, fd_set *wr, struct n3n_runtime_data *eee) { metrics.mainloop++; - int maxfd = setup_select(rd, wr, eee); + FD_ZERO(&rd); + FD_ZERO(&wr); + int maxfd = fdlist_fd_set(&rd, &wr); // FIXME: // unlock the windows tun reader thread before select() and lock it @@ -489,7 +480,7 @@ int mainloop_runonce (fd_set *rd, fd_set *wr, struct n3n_runtime_data *eee) { } wait_time.tv_usec = 0; - int ready = select(maxfd + 1, rd, wr, NULL, &wait_time); + int ready = select(maxfd + 1, &rd, &wr, NULL, &wait_time); if(ready < 1) { // Nothing ready or an error @@ -499,7 +490,7 @@ int mainloop_runonce (fd_set *rd, fd_set *wr, struct n3n_runtime_data *eee) { // One timestamp to use for this entire loop iteration time_t now = time(NULL); - fdlist_check_ready(rd, wr, now, eee); + fdlist_check_ready(&rd, &wr, now, eee); return ready; } From 3d8b126851430da28a70d723b8f657bbda0b004f Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Sat, 7 Dec 2024 14:42:29 +1100 Subject: [PATCH 76/85] Be sure that we dont keep old file handles after a close --- src/edge_utils.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/edge_utils.c b/src/edge_utils.c index 8d81626..14ee963 100644 --- a/src/edge_utils.c +++ b/src/edge_utils.c @@ -3135,12 +3135,14 @@ void edge_term (struct n3n_runtime_data * eee) { if(eee->sock >= 0) { closesocket(eee->sock); mainloop_unregister_fd(eee->sock); + eee->sock = -1; } #ifndef SKIP_MULTICAST_PEERS_DISCOVERY if(eee->udp_multicast_sock >= 0) { closesocket(eee->udp_multicast_sock); mainloop_unregister_fd(eee->udp_multicast_sock); + eee->udp_multicast_sock = -1; } #endif @@ -3248,6 +3250,7 @@ static int edge_init_sockets (struct n3n_runtime_data *eee) { if(eee->udp_multicast_sock >= 0) { closesocket(eee->udp_multicast_sock); mainloop_unregister_fd(eee->udp_multicast_sock); + eee->udp_multicast_sock = -1; } /* Populate the multicast group for local edge */ From 00dfd3833988c9906aa79e387aad797758234f12 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Sat, 7 Dec 2024 15:23:15 +1100 Subject: [PATCH 77/85] Every caller has already dereferenced eee before calling sendto_sock, so we can skip the null check --- src/edge_utils.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/edge_utils.c b/src/edge_utils.c index 14ee963..773e2ad 100644 --- a/src/edge_utils.c +++ b/src/edge_utils.c @@ -1159,12 +1159,6 @@ static void sendto_sock (struct n3n_runtime_data *eee, const void * buf, ssize_t sent; int value = 0; - // TODO: audit callers and confirm if this can ever happen - if(!eee) { - traceEvent(TRACE_WARNING, "bad eee"); - return; - } - if(!dest->family) // invalid socket return; From 4c054a10816ddb6d3acd758e4c17fc21ba383b08 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Sat, 7 Dec 2024 15:24:28 +1100 Subject: [PATCH 78/85] Fix visual indent for ITER code block --- src/edge_utils.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/edge_utils.c b/src/edge_utils.c index 773e2ad..46083b5 100644 --- a/src/edge_utils.c +++ b/src/edge_utils.c @@ -2053,7 +2053,7 @@ static int send_packet (struct n3n_runtime_data * eee, // if no supernode around, foward the broadcast to all known peers if(eee->sn_wait) { HASH_ITER(hh, eee->known_peers, peer, tmp_peer) - sendto_sock(eee, pktbuf, pktlen, &peer->sock); + sendto_sock(eee, pktbuf, pktlen, &peer->sock); return 0; } // fall through otherwise From 301cb1c91f786218655710bcbac1e79cc0cf01c9 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Sat, 7 Dec 2024 23:12:55 +1100 Subject: [PATCH 79/85] Fix which buffer we clear after finishing sending --- libs/connslot/connslot.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/connslot/connslot.c b/libs/connslot/connslot.c index 4366b0a..b0382c7 100644 --- a/libs/connslot/connslot.c +++ b/libs/connslot/connslot.c @@ -274,7 +274,7 @@ ssize_t conn_write(conn_t *conn, int fd) { conn->state = CONN_EMPTY; conn->reply_sendpos = 0; sb_zero(conn->reply_header); - sb_zero(conn->request); + sb_zero(conn->reply); } // This will truncate the time to a int - usually 32bits From c1b42c57bc9b2ee9bcb265b93f3af6b45f40a5ab Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Sat, 7 Dec 2024 23:13:10 +1100 Subject: [PATCH 80/85] Mark appenders as not touching the source data --- libs/connslot/strbuf.c | 4 ++-- libs/connslot/strbuf.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/libs/connslot/strbuf.c b/libs/connslot/strbuf.c index 7e0ef87..11778b5 100644 --- a/libs/connslot/strbuf.c +++ b/libs/connslot/strbuf.c @@ -143,7 +143,7 @@ bool sb_overflowed(strbuf_t *p) { * @param bufsize is the length of the new data to copy * @return the total length of the stored data */ -size_t sb_append(strbuf_t *p, void *buf, ssize_t bufsize) { +size_t sb_append(strbuf_t *p, const void *buf, ssize_t bufsize) { ssize_t avail = sb_avail(p); if (avail <= 0) { // Cannot append to a full buffer @@ -177,7 +177,7 @@ size_t sb_append(strbuf_t *p, void *buf, ssize_t bufsize) { * @param bufsize is the length of the new data to copy * @return the total length of the stored data */ -strbuf_t *sb_reappend(strbuf_t **pp, void *buf, size_t bufsize) { +strbuf_t *sb_reappend(strbuf_t **pp, const void *buf, size_t bufsize) { strbuf_t *p = *pp; size_t needed = p->wr_pos + bufsize; if (needed > p->capacity) { diff --git a/libs/connslot/strbuf.h b/libs/connslot/strbuf.h index 7250f54..d3ed13b 100644 --- a/libs/connslot/strbuf.h +++ b/libs/connslot/strbuf.h @@ -48,8 +48,8 @@ size_t sb_len(strbuf_t *); ssize_t sb_avail(strbuf_t *); bool sb_full(strbuf_t *); bool sb_overflowed(strbuf_t *); -size_t sb_append(strbuf_t *, void *, ssize_t); -strbuf_t *sb_reappend(strbuf_t **, void *, size_t); +size_t sb_append(strbuf_t *, const void *, ssize_t); +strbuf_t *sb_reappend(strbuf_t **, const void *, size_t); size_t sb_vprintf(strbuf_t *, const char *, va_list); size_t sb_printf(strbuf_t *, const char *, ...) __attribute__ ((format (printf, 2, 3))); From c765bbe322ea41ee8b51e48987098b941cc839b4 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Sat, 7 Dec 2024 23:24:15 +1100 Subject: [PATCH 81/85] Wire up TCP tx path to the mainloop This introduces a memcpy() operation on the TCP send path, but the intent is that a packet buffer allocation system will be used in the future that will allow removing that. --- include/n3n/mainloop.h | 7 +++ src/edge_utils.c | 102 ++++++----------------------------------- src/mainloop.c | 54 ++++++++++++++++++++++ 3 files changed, 76 insertions(+), 87 deletions(-) diff --git a/include/n3n/mainloop.h b/include/n3n/mainloop.h index 594f305..00ce77c 100644 --- a/include/n3n/mainloop.h +++ b/include/n3n/mainloop.h @@ -31,6 +31,13 @@ enum __attribute__((__packed__)) fd_info_proto { // Place debug info from the slots into the strbuf void mainloop_dump (strbuf_t **); +// Starts sending a packet, queuing if needed +// returns false when: +// - no queue slot is available +// - the file handle is not registered with the mainloop +// - the file handle is not registered as a v3tcp proto +bool mainloop_send_v3tcp (int, const void *, int); + int mainloop_runonce (struct n3n_runtime_data *); void mainloop_register_fd (int, enum fd_info_proto); diff --git a/src/edge_utils.c b/src/edge_utils.c index 46083b5..d6f252b 100644 --- a/src/edge_utils.c +++ b/src/edge_utils.c @@ -1051,54 +1051,20 @@ static void check_known_peer_sock_change (struct n3n_runtime_data *eee, /* ************************************** */ -/* - * Confirm that we can send to this edge. - * TODO: for the TCP case, this could cause a stall in the packet - * send path, so this probably should be reworked to use a queue - * (and non blocking IO) - */ -static bool check_sock_ready (struct n3n_runtime_data *eee) { - if(!eee->conf.connect_tcp) { - // Just show udp sockets as ready - // TODO: this is may not be always true - return true; - } - - if(eee->sock == -1) { - // If we have no sock, dont attempt to FD_SET() it - return false; - } - - // if required (tcp), wait until writeable as socket is set to - // O_NONBLOCK, could require some wait time directly after re-opening - fd_set socket_mask; - struct timeval wait_time; - - FD_ZERO(&socket_mask); - FD_SET(eee->sock, &socket_mask); - wait_time.tv_sec = 0; - wait_time.tv_usec = 500000; - return select(eee->sock + 1, NULL, &socket_mask, NULL, &wait_time); -} - /** Send a datagram to a socket file descriptor */ -static ssize_t sendto_fd (struct n3n_runtime_data *eee, const void *buf, +static void sendto_fd (struct n3n_runtime_data *eee, const void *buf, size_t len, struct sockaddr_in *dest, const n2n_sock_t * n2ndest) { ssize_t sent = 0; - if(!check_sock_ready(eee)) { - goto err_out; - } - sent = sendto(eee->sock, buf, len, 0 /*flags*/, (struct sockaddr *)dest, sizeof(struct sockaddr_in)); if(sent != -1) { // sendto success traceEvent(TRACE_DEBUG, "sent=%d", (signed int)sent); - return sent; + return; } // We only get here if sendto failed, so errno must be valid @@ -1129,25 +1095,9 @@ static ssize_t sendto_fd (struct n3n_runtime_data *eee, const void *buf, #endif /* - * we get here if the sock is not ready or - * if the sendto had an error + * TODO: metrics for errors */ -err_out: - if(eee->conf.connect_tcp) { - supernode_disconnect(eee); - eee->sn_wait = 1; - // Not true if eee->sock == -1 - traceEvent(TRACE_DEBUG, "error in sendto_fd"); - } - - /* - * If we got an error and are using UDP, this is still an error - * case. The only caller of sendto_fd() checks the return only - * in the TCP case. - * - * Thus, we can safely return an error code for any error. - */ - return -1; + return; } @@ -1156,8 +1106,6 @@ static void sendto_sock (struct n3n_runtime_data *eee, const void * buf, size_t len, const n2n_sock_t * dest) { struct sockaddr_in peer_addr; - ssize_t sent; - int value = 0; if(!dest->family) // invalid socket @@ -1167,44 +1115,24 @@ static void sendto_sock (struct n3n_runtime_data *eee, const void * buf, // invalid socket file descriptor, e.g. TCP unconnected has fd of '-1' return; - peer_addr.sin_port = 0; - // TODO: - // - also check n2n_sock_t type == SOCK_STREAM as a TCP indicator - - // network order socket - fill_sockaddr((struct sockaddr *) &peer_addr, sizeof(peer_addr), dest); + // - also check n2n_sock_t type == SOCK_STREAM as a TCP indicator? // if the connection is tcp, i.e. not the regular sock... if(eee->conf.connect_tcp) { - - setsockopt(eee->sock, IPPROTO_TCP, TCP_NODELAY, (void *)&value, sizeof(value)); - value = 1; -#ifdef LINUX - setsockopt(eee->sock, IPPROTO_TCP, TCP_CORK, &value, sizeof(value)); -#endif - - // prepend packet length... - uint16_t pktsize16 = htobe16(len); - sent = sendto_fd(eee, (uint8_t*)&pktsize16, sizeof(pktsize16), &peer_addr, dest); - - if(sent <= 0) - return; - // ...before sending the actual data + mainloop_send_v3tcp(eee->sock, buf, len); + /* + * TODO: metrics for errors + */ + return; } - sent = sendto_fd(eee, buf, len, &peer_addr, dest); - // if the connection is tcp, i.e. not the regular sock... - if(eee->conf.connect_tcp) { - value = 1; /* value should still be set to 1 */ - setsockopt(eee->sock, IPPROTO_TCP, TCP_NODELAY, (void *)&value, sizeof(value)); -#ifdef LINUX - value = 0; - setsockopt(eee->sock, IPPROTO_TCP, TCP_CORK, &value, sizeof(value)); -#endif - } + peer_addr.sin_port = 0; - return; + // network order socket + fill_sockaddr((struct sockaddr *) &peer_addr, sizeof(peer_addr), dest); + + sendto_fd(eee, buf, len, &peer_addr, dest); } diff --git a/src/mainloop.c b/src/mainloop.c index 1aec226..82f64a4 100644 --- a/src/mainloop.c +++ b/src/mainloop.c @@ -35,6 +35,7 @@ static struct metrics { uint32_t unregister_fd; // mainloop_unregister_fd() is called uint32_t connlist_alloc; uint32_t connlist_free; + uint32_t send_queue_fail; // Attempted to send v3tcp but buffer in use } metrics; static struct n3n_metrics_items_llu32 metrics_items = { @@ -62,6 +63,10 @@ static struct n3n_metrics_items_llu32 metrics_items = { .val1 = "connlist_free", .offset = offsetof(struct metrics, connlist_free), }, + { + .val1 = "send_queue_fail", + .offset = offsetof(struct metrics, send_queue_fail), + }, { }, }, }; @@ -516,6 +521,55 @@ void mainloop_dump (strbuf_t **buf) { } } +bool mainloop_send_v3tcp (int fd, const void *buf, int bufsize) { + // TODO: + // - avoid the linear scan by changing the params to pass a fdlist slottnr + // instead of a filehandle + int slot = 0; + while(slot < MAX_HANDLES) { + if(fdlist[slot].fd == fd) { + break; + } + slot++; + } + if(fdlist[slot].fd != fd) { + // Couldnt find this fd + return false; + } + + if(fdlist[slot].connnr == -1) { + // No buffer associated with this fd + return false; + } + + struct conn *conn = &connlist[fdlist[slot].connnr]; + + if(!conn->reply) { + conn->reply = sb_malloc(N2N_PKT_BUF_SIZE + 2, N2N_PKT_BUF_SIZE + 2); + } else { + if(sb_len(conn->reply)) { + // send buffer already in use + // TODO: + // - metrics! + return false; + } + sb_zero(conn->reply); + } + + uint16_t pktsize16 = htobe16(bufsize); + sb_append(conn->reply, &pktsize16, sizeof(pktsize16)); + + // TODO: + // - avoid memcpy by using a global buffer pool and transferring ownership + sb_append(conn->reply, buf, bufsize); + + // TODO: + // - check bufsize for N2N_PKT_BUF_SIZE overflow + + conn_write(conn, fd); + return true; +} + void mainloop_register_fd (int fd, enum fd_info_proto proto) { fdlist_allocslot(fd, proto); } From 73ee1fb49cbc0007fda05dbf07f77986cc00b72c Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Sat, 7 Dec 2024 23:33:27 +1100 Subject: [PATCH 82/85] Include what you use --- src/edge_utils.c | 1 - src/mainloop.c | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/edge_utils.c b/src/edge_utils.c index d6f252b..dda1716 100644 --- a/src/edge_utils.c +++ b/src/edge_utils.c @@ -53,7 +53,6 @@ #include "n2n_wire.h" // for fill_sockaddr, decod... #include "pearson.h" // for pearson_hash_128, pearson_hash_64 #include "peer_info.h" // for peer_info, clear_peer_list, ... -#include "portable_endian.h" // for be16toh, htobe16 #include "resolve.h" // for resolve_create_thread, resolve_c... #include "sn_selection.h" // for sn_selection_criterion_common_da... #include "speck.h" // for speck_128_decrypt, speck_128_enc... diff --git a/src/mainloop.c b/src/mainloop.c index 82f64a4..5bc7ab9 100644 --- a/src/mainloop.c +++ b/src/mainloop.c @@ -23,6 +23,7 @@ #include "edge_utils.h" // for edge_read_from_tap #include "management.h" // for readFromMgmtSocket #include "minmax.h" // for min, max +#include "portable_endian.h" // for htobe16 #ifndef _WIN32 // Another wonderful gift from the world of POSIX compliance is not worth much From 3e72976091f0a8f2d2491351b82e8d6fc8813697 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Sat, 7 Dec 2024 23:35:26 +1100 Subject: [PATCH 83/85] Address linter concerns --- src/edge_utils.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/edge_utils.c b/src/edge_utils.c index dda1716..14bcf5a 100644 --- a/src/edge_utils.c +++ b/src/edge_utils.c @@ -1052,8 +1052,8 @@ static void check_known_peer_sock_change (struct n3n_runtime_data *eee, /** Send a datagram to a socket file descriptor */ static void sendto_fd (struct n3n_runtime_data *eee, const void *buf, - size_t len, struct sockaddr_in *dest, - const n2n_sock_t * n2ndest) { + size_t len, struct sockaddr_in *dest, + const n2n_sock_t * n2ndest) { ssize_t sent = 0; @@ -1979,8 +1979,9 @@ static int send_packet (struct n3n_runtime_data * eee, // if no supernode around, foward the broadcast to all known peers if(eee->sn_wait) { - HASH_ITER(hh, eee->known_peers, peer, tmp_peer) + HASH_ITER(hh, eee->known_peers, peer, tmp_peer) { sendto_sock(eee, pktbuf, pktlen, &peer->sock); + } return 0; } // fall through otherwise From 54e3687178623ac0a155d075fd8e82107b537e54 Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Mon, 9 Dec 2024 18:08:39 +1100 Subject: [PATCH 84/85] Handle non-blocking tcp connections on windows --- src/edge_utils.c | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/src/edge_utils.c b/src/edge_utils.c index 14bcf5a..d7865a4 100644 --- a/src/edge_utils.c +++ b/src/edge_utils.c @@ -381,12 +381,35 @@ void supernode_connect (struct n3n_runtime_data *eee) { fill_sockaddr((struct sockaddr*)&sn_sock, sizeof(sn_sock), &eee->curr_sn->sock); - if((connect(eee->sock, (struct sockaddr*)&(sn_sock), sizeof(struct sockaddr)) < 0) - && (errno != EINPROGRESS)) { + int result = connect( + eee->sock, + (struct sockaddr*)&(sn_sock), + sizeof(struct sockaddr) + ); + +#ifndef _WIN32 + if((result == -1) && (errno != EINPROGRESS)) { traceEvent(TRACE_INFO, "Error connecting TCP: %i", errno); + closesocket(eee->sock); + mainloop_unregister_fd(eee->sock); eee->sock = -1; return; } +#else + // Oh Windows, this just seems needlessly incompatible + int wsaerr = WSAGetLastError(); + if((result == -1) && (wsaerr != WSAEWOULDBLOCK)) { + traceEvent( + TRACE_INFO, + "Error connecting TCP: WSAGetLastError %i", + wsaerr + ); + closesocket(eee->sock); + mainloop_unregister_fd(eee->sock); + eee->sock = -1; + return; + } +#endif } if(eee->conf.tos) { From 74be511511d2cd423d0a4eb0123f6dc1a122265c Mon Sep 17 00:00:00 2001 From: Hamish Coleman Date: Mon, 9 Dec 2024 18:10:36 +1100 Subject: [PATCH 85/85] Update config help, since TCP mode is functioning on Windows --- src/conffile_defs.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/conffile_defs.c b/src/conffile_defs.c index fd2b564..e8a4900 100644 --- a/src/conffile_defs.c +++ b/src/conffile_defs.c @@ -131,8 +131,7 @@ static struct n3n_conf_option section_connection[] = { .desc = "Control use of TCP connections to supernode", .help = "Defaulting to false, this is used to enable the use of a " "TCP connection to the supernode. If set, the allow_p2p " - "setting should usually also be set to false. This feature " - "is not currently available on Windows edge nodes.", + "setting should usually also be set to false.", }, { .name = "description",