Merge pull request #785 from MichaelBrim/scalable-svcmgr

Scalability improvements and a few bug fixes

.gitlab-ci.yml

@@ -107,7 +107,7 @@ stages:
   script:
     - cd unifyfs-build/t && $JOB_LAUNCH_COMMAND make check
   after_script:
-    - rm -rf /tmp/unify* /tmp/tmp.* /tmp/mdhim* /tmp/na_sm | true
+    - rm -rf /tmp/unify* /tmp/tmp.* /tmp/na_sm | true
 
 # Run the integration test suite with the options provided from the specific
 # job.

Makefile.am

@@ -1,4 +1,4 @@
-SUBDIRS = extras meta client server util examples t
+SUBDIRS = extras client server util examples t
 
 CONFIG = ordered
 

client/src/unifyfs_api.c

@@ -265,13 +265,13 @@ unifyfs_rc unifyfs_initialize(const char* mountpoint,
     /* add mount point as a new directory in the file list */
     if (unifyfs_fid_from_path(client, mountpoint) < 0) {
         /* no entry exists for mount point, so create one */
-        int fid = unifyfs_fid_create_directory(client, mountpoint);
-        if (fid < 0) {
+        rc = unifyfs_fid_create_directory(client, mountpoint);
+        if (rc != UNIFYFS_SUCCESS) {
             /* if there was an error, return it */
             LOGERR("failed to create directory entry for mount point: `%s'",
                    mountpoint);
             unifyfs_client_fini(client);
-            return UNIFYFS_FAILURE;
+            return rc;
         }
     }
 

common/src/Makefile.mk

@@ -6,6 +6,8 @@ UNIFYFS_COMMON_INSTALL_HDRS = \
 UNIFYFS_COMMON_BASE_SRCS = \
   %reldir%/arraylist.h \
   %reldir%/arraylist.c \
+  %reldir%/compare_fn.h \
+  %reldir%/compare_fn.c \
   %reldir%/ini.h \
   %reldir%/ini.c \
   %reldir%/rm_enumerator.h \

common/src/compare_fn.c

@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2023, Lawrence Livermore National Security, LLC.
+ * Produced at the Lawrence Livermore National Laboratory.
+ *
+ * Copyright 2023, UT-Battelle, LLC.
+ *
+ * LLNL-CODE-741539
+ * All rights reserved.
+ *
+ * This is the license for UnifyFS.
+ * For details, see https://github.com/LLNL/UnifyFS.
+ * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text.
+ */
+
+#include "compare_fn.h"
+
+int int_compare_fn(const void* a, const void* b)
+{
+    int ai = *(int*)a;
+    int bi = *(int*)b;
+    if (ai == bi) {
+        return 0;
+    } else if (ai > bi) {
+        return 1;
+    } else {
+        return -1;
+    }
+}
+
+int uint_compare_fn(const void* a, const void* b)
+{
+    unsigned int ai = *(unsigned int*)a;
+    unsigned int bi = *(unsigned int*)b;
+    if (ai == bi) {
+        return 0;
+    } else if (ai > bi) {
+        return 1;
+    } else {
+        return -1;
+    }
+}
+
+int float_compare_fn(const void* a, const void* b)
+{
+    float af = *(float*)a;
+    float bf = *(float*)b;
+    if (af == bf) {
+        return 0;
+    } else if (af > bf) {
+        return 1;
+    } else {
+        return -1;
+    }
+}
+
+int double_compare_fn(const void* a, const void* b)
+{
+    double ad = *(double*)a;
+    double bd = *(double*)b;
+    if (ad == bd) {
+        return 0;
+    } else if (ad > bd) {
+        return 1;
+    } else {
+        return -1;
+    }
+}

common/src/compare_fn.h

@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) 2023, Lawrence Livermore National Security, LLC.
+ * Produced at the Lawrence Livermore National Laboratory.
+ *
+ * Copyright 2023, UT-Battelle, LLC.
+ *
+ * LLNL-CODE-741539
+ * All rights reserved.
+ *
+ * This is the license for UnifyFS.
+ * For details, see https://github.com/LLNL/UnifyFS.
+ * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text.
+ */
+
+#ifndef COMPARE_FUNC_H
+#define COMPARE_FUNC_H
+
+typedef int (*compare_fn)(const void *, const void *);
+
+int int_compare_fn(const void* a, const void* b);
+int uint_compare_fn(const void* a, const void* b);
+
+int float_compare_fn(const void* a, const void* b);
+int double_compare_fn(const void* a, const void* b);
+
+#endif /* COMPARE_FN_H */

common/src/seg_tree.c

@@ -38,7 +38,8 @@
 #endif
 
 static int
-compare_func(struct seg_tree_node* node1, struct seg_tree_node* node2)
+stn_compare_func(struct seg_tree_node* node1,
+                 struct seg_tree_node* node2)
 {
     if (node1->start > node2->end) {
         return 1;
@@ -49,14 +50,14 @@ compare_func(struct seg_tree_node* node1, struct seg_tree_node* node2)
     }
 }
 
-RB_PROTOTYPE(inttree, seg_tree_node, entry, compare_func)
-RB_GENERATE(inttree, seg_tree_node, entry, compare_func)
+RB_PROTOTYPE(inttree, seg_tree_node, entry, stn_compare_func)
+RB_GENERATE(inttree, seg_tree_node, entry, stn_compare_func)
 
 /* Returns 0 on success, positive non-zero error code otherwise */
 int seg_tree_init(struct seg_tree* seg_tree)
 {
     memset(seg_tree, 0, sizeof(*seg_tree));
-    pthread_rwlock_init(&seg_tree->rwlock, NULL);
+    ABT_rwlock_create(&(seg_tree->rwlock));
     RB_INIT(&seg_tree->head);
 
     return 0;
@@ -68,6 +69,7 @@ int seg_tree_init(struct seg_tree* seg_tree)
 void seg_tree_destroy(struct seg_tree* seg_tree)
 {
     seg_tree_clear(seg_tree);
+    ABT_rwlock_free(&(seg_tree->rwlock));
 }
 
 /* Allocate a node for the range tree.  Free node with free() when finished */
@@ -523,9 +525,9 @@ seg_tree_iter(struct seg_tree* seg_tree, struct seg_tree_node* start)
 void
 seg_tree_rdlock(struct seg_tree* seg_tree)
 {
-    int rc = pthread_rwlock_rdlock(&seg_tree->rwlock);
+    int rc = ABT_rwlock_rdlock(seg_tree->rwlock);
     if (rc) {
-        LOGERR("pthread_rwlock_rdlock() failed - rc=%d", rc);
+        LOGERR("ABT_rwlock_rdlock() failed - rc=%d", rc);
     }
 }
 
@@ -537,9 +539,9 @@ seg_tree_rdlock(struct seg_tree* seg_tree)
 void
 seg_tree_wrlock(struct seg_tree* seg_tree)
 {
-    int rc = pthread_rwlock_wrlock(&seg_tree->rwlock);
+    int rc = ABT_rwlock_wrlock(seg_tree->rwlock);
     if (rc) {
-        LOGERR("pthread_rwlock_wrlock() failed - rc=%d", rc);
+        LOGERR("ABT_rwlock_wrlock() failed - rc=%d", rc);
     }
 }
 
@@ -551,9 +553,9 @@ seg_tree_wrlock(struct seg_tree* seg_tree)
 void
 seg_tree_unlock(struct seg_tree* seg_tree)
 {
-    int rc = pthread_rwlock_unlock(&seg_tree->rwlock);
+    int rc = ABT_rwlock_unlock(seg_tree->rwlock);
     if (rc) {
-        LOGERR("pthread_rwlock_unlock() failed - rc=%d", rc);
+        LOGERR("ABT_rwlock_unlock() failed - rc=%d", rc);
     }
 }
 
@@ -589,6 +591,7 @@ void seg_tree_clear(struct seg_tree* seg_tree)
 
     seg_tree->count = 0;
     seg_tree->max = 0;
+
     seg_tree_unlock(seg_tree);
 }
 

common/src/seg_tree.h

@@ -15,7 +15,7 @@
 #ifndef __SEG_TREE_H__
 #define __SEG_TREE_H__
 
-#include <pthread.h>
+#include <abt.h>
 #include "tree.h"
 
 struct seg_tree_node {
@@ -28,7 +28,7 @@ struct seg_tree_node {
 
 struct seg_tree {
     RB_HEAD(inttree, seg_tree_node) head;
-    pthread_rwlock_t rwlock;
+    ABT_rwlock rwlock;
     unsigned long count;     /* number of segments stored in tree */
     unsigned long max;       /* maximum logical offset value in the tree */
 };

common/src/unifyfs_client_rpcs.h

@@ -20,6 +20,7 @@
  */
 
 #include <time.h>
+#include <abt.h>
 #include <margo.h>
 #include <mercury.h>
 #include <mercury_proc_string.h>
@@ -56,6 +57,14 @@ typedef enum {
     UNIFYFS_CLIENT_CALLBACK_UNLINK
 } client_callback_e;
 
+typedef struct {
+    client_rpc_e req_type;
+    hg_handle_t handle;
+    void* input;
+    void* bulk_buf;
+    size_t bulk_sz;
+} client_rpc_req_t;
+
 /* unifyfs_attach_rpc (client => server)
  *
  * initialize server access to client's shared memory and file state */

common/src/unifyfs_meta.h

@@ -36,6 +36,8 @@ extern "C" {
 # define UNIFYFS_METADATA_CACHE_SECONDS 5
 #endif
 
+/* a valid gfid generated via MD5 hash will never be zero */
+#define INVALID_GFID (0)
 
 /* extent slice size used for metadata */
 extern size_t meta_slice_sz;
@@ -109,6 +111,9 @@ typedef struct {
     struct timespec atime;
     struct timespec mtime;
     struct timespec ctime;
+
+    /* metadata caching timestamp */
+    time_t last_update;
 } unifyfs_file_attr_t;
 
 enum {
@@ -123,7 +128,7 @@ void unifyfs_file_attr_set_invalid(unifyfs_file_attr_t* attr)
 {
     memset(attr, 0, sizeof(*attr));
     attr->filename     = NULL;
-    attr->gfid         = -1;
+    attr->gfid         = INVALID_GFID;
     attr->is_laminated = -1;
     attr->is_shared    = -1;
     attr->mode         = (uint32_t) -1;
@@ -173,10 +178,18 @@ int unifyfs_file_attr_update(int attr_op,
 {
     if (!dst || !src
         || (attr_op == UNIFYFS_FILE_ATTR_OP_INVALID)
-        || (dst->gfid != src->gfid)) {
+        || (src->gfid == INVALID_GFID)) {
         return EINVAL;
     }
 
+    if (attr_op == UNIFYFS_FILE_ATTR_OP_CREATE) {
+        dst->gfid = src->gfid;
+    }
+
+    struct timespec tp = {0};
+    clock_gettime(CLOCK_REALTIME, &tp);
+    dst->last_update = tp.tv_sec;
+
     LOGDBG("updating attributes for gfid=%d", dst->gfid);
 
     /* Update fields only with valid values and associated operation.
@@ -336,16 +349,6 @@ int compare_name_rank_pair(const void* a, const void* b)
     return cmp;
 }
 
-/* qsort comparison function for int */
-static inline
-int compare_int(const void* a, const void* b)
-{
-    int aval = *(const int*)a;
-    int bval = *(const int*)b;
-    return aval - bval;
-}
-
-
 /*
  * Hash a file path to a uint64_t using MD5
  * @param path absolute file path
@@ -354,7 +357,7 @@ int compare_int(const void* a, const void* b)
 uint64_t compute_path_md5(const char* path);
 
 /*
- * Hash a file path to an integer gfid
+ * Hash a file path to a positive integer gfid
  * @param path absolute file path
  * @return gfid
  */
@@ -365,12 +368,8 @@ int unifyfs_generate_gfid(const char* path)
     uint64_t hash64 = compute_path_md5(path);
     uint32_t hash32 = (uint32_t)(hash64 >> 32);
 
-    /* TODO: Remove next statement once we get rid of MDHIM.
-     *
-     * MDHIM requires positive values for integer keys, due to the way
-     * slice servers are calculated. We use an integer key for the
-     * gfid -> file attributes index. To guarantee a positive value, we
-     * shift right one bit to make sure the top bit is zero. */
+    /* To guarantee a positive value, we shift right one bit
+     * to make sure the top bit is zero. */
     hash32 = hash32 >> 1;
 
     return (int)hash32;

common/src/unifyfs_rc.h

@@ -43,8 +43,6 @@
     ENUMITEM(GOTCHA, "Gotcha operation error")                         \
     ENUMITEM(KEYVAL, "Key-value store operation error")                \
     ENUMITEM(MARGO, "Mercury/Argobots operation error")                \
-    ENUMITEM(MDHIM, "MDHIM operation error")                           \
-    ENUMITEM(META, "Metadata store operation error")                   \
     ENUMITEM(NYI, "Not yet implemented")                               \
     ENUMITEM(PMI, "PMI2/PMIx error")                                   \
     ENUMITEM(SHMEM, "Shared memory region init/access error")          \

common/src/unifyfs_server_rpcs.h

@@ -20,6 +20,7 @@
  */
 
 #include <time.h>
+#include <abt.h>
 #include <margo.h>
 #include <mercury.h>
 #include <mercury_proc_string.h>
@@ -47,10 +48,11 @@ typedef enum {
     UNIFYFS_SERVER_BCAST_RPC_EXTENTS,
     UNIFYFS_SERVER_BCAST_RPC_FILEATTR,
     UNIFYFS_SERVER_BCAST_RPC_LAMINATE,
+    UNIFYFS_SERVER_BCAST_RPC_METAGET,
     UNIFYFS_SERVER_BCAST_RPC_TRANSFER,
     UNIFYFS_SERVER_BCAST_RPC_TRUNCATE,
     UNIFYFS_SERVER_BCAST_RPC_UNLINK,
-    UNIFYFS_SERVER_BCAST_RPC_METAGET
+    UNIFYFS_SERVER_PENDING_SYNC
 } server_rpc_e;
 
 /* structure to track server-to-server rpc request state */