Version 2.6.5 Exposed functions to construct suffix array of a given …

…integer array + miscellaneous fixes.

CHANGES

@@ -1,3 +1,8 @@
+Changes in 2.6.5 (January 1, 2022)
+- Exposed functions to construct suffix array of a given integer array.
+- Improved detection of various compiler intrinsics.
+- Capped free space parameter to avoid crashing due to 32-bit integer overflow.
+
 Changes in 2.6.0 (October 21, 2021)
 - libsais16 for 16-bit inputs.
 

Makefile

@@ -1,6 +1,6 @@
 PROJECT=sais
 PLIBNAME=lib$(PROJECT)
-PVER=2.6.0
+PVER=2.6.5
 PSOVER=2
 ifeq ($(OS),Windows_NT)
   PLIBSTATIC=$(PROJECT).a

README.md

@@ -5,7 +5,7 @@ The libsais is a library for fast (see [Benchmarks](#benchmarks) below) linear t
 * Nataliya Timoshevskaya, Wu-chun Feng *SAIS-OPT: On the characterization and optimization of the SA-IS algorithm for suffix array construction*, 2014
 * Jing Yi Xie, Ge Nong, Bin Lao, Wentao Xu *Scalable Suffix Sorting on a Multicore Machine*, 2020
 
-Copyright (c) 2021 Ilya Grebnov <[email protected]>
+Copyright (c) 2021-2022 Ilya Grebnov <[email protected]>
 
 >The libsais is inspired by [libdivsufsort](https://github.com/y-256/libdivsufsort), [sais](https://sites.google.com/site/yuta256/sais) libraries by Yuta Mori and [msufsort](https://github.com/michaelmaniscalco/msufsort) by Michael Maniscalco.
 
@@ -19,6 +19,10 @@ The libsais provides simple C99 API to construct suffix array and Burrows-Wheele
 The libsais is released under the [Apache License Version 2.0](LICENSE "Apache license")
 
 ## Changes
+* January 1, 2022 (2.6.5)
+  * Exposed functions to construct suffix array of a given integer array.
+  * Improved detection of various compiler intrinsics.
+  * Capped free space parameter to avoid crashing due to 32-bit integer overflow.
 * October 21, 2021 (2.6.0)
   * libsais16 for 16-bit inputs.
 * October 15, 2021 (2.5.0)
@@ -36,82 +40,60 @@ The libsais is released under the [Apache License Version 2.0](LICENSE "Apache l
 * February 23, 2021 (1.0.0)
   * Initial release.
 
-## API
+## Versions of the libsais library
+* [libsais.c](src/libsais.c) (and corresponding [libsais.h](src/libsais.h)) is for suffix array, forward BWT and reverse BWT construction over 8-bit inputs smaller than 2GB (2147483648 bytes).
+  * This version of the library could also be used to construct suffix array of an integer array (with a caveat that input array must be mutable).
+* [libsais64.c](src/libsais64.c) (and corresponding [libsais64.h](src/libsais64.h)) is optional extension of the library for inputs larger or equlas to 2GB (2147483648 bytes).
+* [libsais16.c](src/libsais16.c) (and corresponding [libsais16.h](src/libsais16.h)) is independent version of the library for 16-bit inputs.
+
+## Examples of APIs (see [libsais.h](src/libsais.h), [libsais16.h](src/libsais16.h) and [libsais64.h](src/libsais64.h) for complete APIs list)
 ```c
     /**
     * Constructs the suffix array of a given string.
     * @param T [0..n-1] The input string.
     * @param SA [0..n-1+fs] The output array of suffixes.
     * @param n The length of the given string.
-    * @param fs Extra space available at the end of SA array (can be 0).
+    * @param fs Extra space available at the end of SA array (0 should be enough for most cases).
     * @param freq [0..255] The output symbol frequency table (can be NULL).
     * @return 0 if no error occurred, -1 or -2 otherwise.
     */
     int32_t libsais(const uint8_t * T, int32_t * SA, int32_t n, int32_t fs, int32_t * freq);
 
     /**
-    * Constructs the burrows-wheeler transformed string of a given string.
-    * @param T [0..n-1] The input string.
-    * @param U [0..n-1] The output string (can be T).
-    * @param A [0..n-1+fs] The temporary array.
-    * @param n The length of the given string.
-    * @param fs Extra space available at the end of A array (can be 0).
-    * @param freq [0..255] The output symbol frequency table (can be NULL).
-    * @return The primary index if no error occurred, -1 or -2 otherwise.
-    */
-    int32_t libsais_bwt(const uint8_t * T, uint8_t * U, int32_t * A, int32_t n, int32_t fs, int32_t * freq);
-
-    /**
-    * Constructs the original string from a given burrows-wheeler transformed string with primary index.
-    * @param T [0..n-1] The input string.
-    * @param U [0..n-1] The output string (can be T).
-    * @param A [0..n] The temporary array (NOTE, temporary array must be n + 1 size).
-    * @param n The length of the given string.
-    * @param freq [0..255] The input symbol frequency table (can be NULL).
-    * @param i The primary index.
-    * @return 0 if no error occurred, -1 or -2 otherwise.
-    */
-    int32_t libsais_unbwt(const uint8_t * T, uint8_t * U, int32_t * A, int32_t n, const int32_t * freq, int32_t i);
-
-#if defined(_OPENMP)
-    /**
-    * Constructs the suffix array of a given string in parallel using OpenMP.
-    * @param T [0..n-1] The input string.
+    * Constructs the suffix array of a given integer array.
+    * Note, during construction input array will be modified, but restored at the end if no errors occurred.
+    * @param T [0..n-1] The input integer array.
     * @param SA [0..n-1+fs] The output array of suffixes.
-    * @param n The length of the given string.
-    * @param fs Extra space available at the end of SA array (can be 0).
-    * @param freq [0..255] The output symbol frequency table (can be NULL).
-    * @param threads The number of OpenMP threads to use (can be 0 for OpenMP default).
+    * @param n The length of the integer array.
+    * @param k The alphabet size of the input integer array.
+    * @param fs Extra space available at the end of SA array (can be 0, but 4k or better 6k is recommended for optimal performance).
     * @return 0 if no error occurred, -1 or -2 otherwise.
     */
-    int32_t libsais_omp(const uint8_t * T, int32_t * SA, int32_t n, int32_t fs, int32_t * freq, int32_t threads);
+    int32_t libsais_int(int32_t * T, int32_t * SA, int32_t n, int32_t k, int32_t fs);
 
     /**
-    * Constructs the burrows-wheeler transformed string of a given string in parallel using OpenMP.
+    * Constructs the burrows-wheeler transformed string of a given string.
     * @param T [0..n-1] The input string.
     * @param U [0..n-1] The output string (can be T).
     * @param A [0..n-1+fs] The temporary array.
     * @param n The length of the given string.
-    * @param fs Extra space available at the end of A array (can be 0).
+    * @param fs Extra space available at the end of A array (0 should be enough for most cases).
     * @param freq [0..255] The output symbol frequency table (can be NULL).
-    * @param threads The number of OpenMP threads to use (can be 0 for OpenMP default).
     * @return The primary index if no error occurred, -1 or -2 otherwise.
     */
-    int32_t libsais_bwt_omp(const uint8_t * T, uint8_t * U, int32_t * A, int32_t n, int32_t fs, int32_t * freq, int32_t threads);
+    int32_t libsais_bwt(const uint8_t * T, uint8_t * U, int32_t * A, int32_t n, int32_t fs, int32_t * freq);
 
     /**
-    * Constructs the original string from a given burrows-wheeler transformed string with primary index in parallel using OpenMP.
+    * Constructs the original string from a given burrows-wheeler transformed string with primary index.
     * @param T [0..n-1] The input string.
     * @param U [0..n-1] The output string (can be T).
     * @param A [0..n] The temporary array (NOTE, temporary array must be n + 1 size).
     * @param n The length of the given string.
-    * @param freq [0..255] The input symbol frequency table (can be NULL).
+    * @param freq [0..255] The input symbol frequency table (can be NULL).                	
     * @param i The primary index.
-    * @param threads The number of OpenMP threads to use (can be 0 for OpenMP default).
     * @return 0 if no error occurred, -1 or -2 otherwise.
     */
-    int32_t libsais_unbwt_omp(const uint8_t * T, uint8_t * U, int32_t * A, int32_t n, const int32_t * freq, int32_t i, int32_t threads);
-#endif
+    int32_t libsais_unbwt(const uint8_t * T, uint8_t * U, int32_t * A, int32_t n, const int32_t * freq, int32_t i);
 ```
 
 ---
@@ -154,9 +136,7 @@ The times below are the minimum of five runs measuring **multi-threaded (MT)** p
 
 ## Additional memory
 
-The libsais reuses free space allocated for suffix array during induction. Sometimes this space might not be sufficient for most efficient algorithm and libsais will need to fallback to less efficient one (libsais has 4 algorithms at different break points). You could avoid this fallbacks and improve performance by allocating additional space at the end of suffix array.
-
-> * All other files from [Benchmarks](#benchmarks) above do not suffer from this fallbacks.
+The libsais reuses space allocated for suffix array during construction. Sometimes this free space is not sufficient for most optimal algorithm (this is uncommon) and libsais will need to fallback to less efficient one (libsais has 4 algorithms at different break-points point: 6k, 4k, 2k and 1k; where k is alphabet size). To improve performance for those cases you could allocating additional space at the end of suffix array.
 
 |  file           |    size     |     libsais + O(n)  (ST)   |     libsais + O(1) (ST)    |speedup (ST)|    libsais + O(n)  (MT)    |     libsais + O(1) (ST)    |speedup (MT)|
 |:---------------:|:-----------:|:--------------------------:|:--------------------------:|:----------:|:--------------------------:|:--------------------------:|:----------:|
@@ -166,3 +146,5 @@ The libsais reuses free space allocated for suffix array during induction. Somet
 |         ooffice |     6152192 |   0.113 sec (  54.55 MB/s) |   0.117 sec (  52.45 MB/s) |  **+4.01%**|   0.081 sec (  76.38 MB/s) |   0.088 sec (  70.30 MB/s) |  **+8.65%**|
 |            abac |      200000 |   0.002 sec (  84.36 MB/s) |   0.003 sec (  73.63 MB/s) | **+14.56%**|   0.002 sec ( 105.08 MB/s) |   0.002 sec (  86.64 MB/s) | **+21.27%**|
 |           test3 |     2097088 |   0.034 sec (  61.54 MB/s) |   0.037 sec (  56.45 MB/s) |  **+9.03%**|   0.028 sec (  75.76 MB/s) |   0.032 sec (  64.93 MB/s) | **+16.68%**|
+
+> * All other files from [Benchmarks](#benchmarks) above do not suffer from this fallbacks.

VERSION

@@ -1 +1 @@
-2.6.0
+2.6.5

src/libsais.c

@@ -3,7 +3,7 @@
 This file is a part of libsais, a library for linear time
 suffix array and burrows wheeler transform construction.
 
-   Copyright (c) 2021 Ilya Grebnov <[email protected]>
+   Copyright (c) 2021-2022 Ilya Grebnov <[email protected]>
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
@@ -21,8 +21,6 @@ Please see the file LICENSE for full copyright information.
 
 --*/
 
-#include "libsais_internal.h"
-
 #include "libsais.h"
 
 #include <stddef.h>
@@ -107,9 +105,17 @@ typedef struct LIBSAIS_UNBWT_CONTEXT
     #if __has_builtin(__builtin_prefetch)
         #define HAS_BUILTIN_PREFECTCH
     #endif
-#elif defined(__GNUC__) && __GNUC__ > 3
+#elif defined(__GNUC__) && ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 2)) || (__GNUC__ >= 4)
     #define HAS_BUILTIN_PREFECTCH
-#endif 
+#endif
+
+#if defined(__has_builtin)
+    #if __has_builtin(__builtin_bswap16)
+        #define HAS_BUILTIN_BSWAP16
+    #endif
+#elif defined(__GNUC__) && ((__GNUC__ == 4) && (__GNUC_MINOR__ >= 8)) || (__GNUC__ >= 5)
+    #define HAS_BUILTIN_BSWAP16
+#endif
 
 #if defined(HAS_BUILTIN_PREFECTCH)
     #define libsais_prefetch(address) __builtin_prefetch((const void *)(address), 0, 0)
@@ -149,12 +155,8 @@ typedef struct LIBSAIS_UNBWT_CONTEXT
 #endif
 
 #if defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__)
-    #if defined(__GNUC__) || defined(__clang__)
-        #if defined(__builtin_bswap16)
-            #define libsais_bswap16(x) (__builtin_bswap16(x))
-        #else
-            #define libsais_bswap16(x) ((uint16_t)(x >> 8) | (uint16_t)(x << 8))
-        #endif
+    #if defined(HAS_BUILTIN_BSWAP16)
+        #define libsais_bswap16(x) (__builtin_bswap16(x))
     #elif defined(_MSC_VER) && !defined(__INTEL_COMPILER)
         #define libsais_bswap16(x) (_byteswap_ushort(x))
     #else
@@ -6259,6 +6261,8 @@ static void libsais_reconstruct_compacted_lms_suffixes_32s_1k_omp(sa_sint_t * RE
 
 static sa_sint_t libsais_main_32s(sa_sint_t * RESTRICT T, sa_sint_t * RESTRICT SA, sa_sint_t n, sa_sint_t k, sa_sint_t fs, sa_sint_t threads, LIBSAIS_THREAD_STATE * RESTRICT thread_state)
 {
+    fs = fs < (SAINT_MAX - n) ? fs : (SAINT_MAX - n);
+
     if (k > 0 && fs / k >= 6)
     {
         sa_sint_t alignment = (fs - 1024) / k >= 6 ? 1024 : 16;
@@ -6451,21 +6455,10 @@ static sa_sint_t libsais_main_32s(sa_sint_t * RESTRICT T, sa_sint_t * RESTRICT S
     }
 }
 
-int32_t libsais_main_32s_internal(int32_t * T, int32_t * SA, int32_t n, int32_t k, int32_t fs, int32_t threads)
-{
-    LIBSAIS_THREAD_STATE * RESTRICT thread_state = threads > 1 ? libsais_alloc_thread_state(threads) : NULL;
-
-    sa_sint_t index = thread_state != NULL || threads == 1
-        ? libsais_main_32s(T, SA, n, k, fs, threads, thread_state)
-        : -2;
-
-    libsais_free_thread_state(thread_state);
-
-    return index;
-}
-
 static sa_sint_t libsais_main_8u(const uint8_t * T, sa_sint_t * SA, sa_sint_t n, sa_sint_t * RESTRICT buckets, sa_sint_t bwt, sa_sint_t r, sa_sint_t * RESTRICT I, sa_sint_t fs, sa_sint_t * freq, sa_sint_t threads, LIBSAIS_THREAD_STATE * RESTRICT thread_state)
 {
+    fs = fs < (SAINT_MAX - n) ? fs : (SAINT_MAX - n);
+
     sa_sint_t m = libsais_count_and_gather_lms_suffixes_8u_omp(T, SA, n, buckets, threads, thread_state);
 
     libsais_initialize_buckets_start_and_end_8u(buckets, freq);
@@ -6519,6 +6512,19 @@ static sa_sint_t libsais_main(const uint8_t * T, sa_sint_t * SA, sa_sint_t n, sa
     return index;
 }
 
+static int32_t libsais_main_int(sa_sint_t * T, sa_sint_t * SA, sa_sint_t n, sa_sint_t k, sa_sint_t fs, sa_sint_t threads)
+{
+    LIBSAIS_THREAD_STATE * RESTRICT thread_state = threads > 1 ? libsais_alloc_thread_state(threads) : NULL;
+
+    sa_sint_t index = thread_state != NULL || threads == 1
+        ? libsais_main_32s(T, SA, n, k, fs, threads, thread_state)
+        : -2;
+
+    libsais_free_thread_state(thread_state);
+
+    return index;
+}
+
 static sa_sint_t libsais_main_ctx(const LIBSAIS_CONTEXT * ctx, const uint8_t * T, sa_sint_t * SA, sa_sint_t n, sa_sint_t bwt, sa_sint_t r, sa_sint_t * I, sa_sint_t fs, sa_sint_t * freq)
 {
     return ctx != NULL && (ctx->buckets != NULL && (ctx->thread_state != NULL || ctx->threads == 1))
@@ -6603,6 +6609,21 @@ int32_t libsais(const uint8_t * T, int32_t * SA, int32_t n, int32_t fs, int32_t
     return libsais_main(T, SA, n, 0, 0, NULL, fs, freq, 1);
 }
 
+int32_t libsais_int(int32_t * T, int32_t * SA, int32_t n, int32_t k, int32_t fs)
+{
+    if ((T == NULL) || (SA == NULL) || (n < 0) || (fs < 0))
+    {
+        return -1;
+    }
+    else if (n < 2)
+    {
+        if (n == 1) { SA[0] = 0; }
+        return 0;
+    }
+
+    return libsais_main_int(T, SA, n, k, fs, 1);
+}
+
 int32_t libsais_ctx(const void * ctx, const uint8_t * T, int32_t * SA, int32_t n, int32_t fs, int32_t * freq)
 {
     if ((ctx == NULL) || (T == NULL) || (SA == NULL) || (n < 0) || (fs < 0))
@@ -6759,6 +6780,23 @@ int32_t libsais_omp(const uint8_t * T, int32_t * SA, int32_t n, int32_t fs, int3
     return libsais_main(T, SA, n, 0, 0, NULL, fs, freq, threads);
 }
 
+int32_t libsais_int_omp(int32_t * T, int32_t * SA, int32_t n, int32_t k, int32_t fs, int32_t threads)
+{
+    if ((T == NULL) || (SA == NULL) || (n < 0) || (fs < 0) || (threads < 0))
+    {
+        return -1;
+    }
+    else if (n < 2)
+    {
+        if (n == 1) { SA[0] = 0; }
+        return 0;
+    }
+
+    threads = threads > 0 ? threads : omp_get_max_threads();
+
+    return libsais_main_int(T, SA, n, k, fs, threads);
+}
+
 int32_t libsais_bwt_omp(const uint8_t * T, uint8_t * U, int32_t * A, int32_t n, int32_t fs, int32_t * freq, int32_t threads)
 {
     if ((T == NULL) || (U == NULL) || (A == NULL) || (n < 0) || (fs < 0) || (threads < 0))