Skip to content

Commit

Permalink
Use Number datatype for NumericTypeHandler to retain Numeric type
Browse files Browse the repository at this point in the history 
Signed-off-by: Dinu John <[email protected]>
  • Loading branch information
@dinujoh
dinujoh committed Nov 4, 2024
1 parent 66076a9 commit 764e36c
Show file tree
Hide file tree
Showing 11 changed files with 76 additions and 58 deletions.
2 changes: 1 addition & 1 deletion ...src/main/java/org/opensearch/dataprepper/plugins/source/rds/datatype/DataTypeHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,5 +17,5 @@ public interface DataTypeHandler {
* @param metadata Additional metadata about the table structure and properties
* @return A string representation of the converted value
*/
String handle(MySQLDataType columnType, String columnName, Object value, TableMetadata metadata);
Object handle(MySQLDataType columnType, String columnName, Object value, TableMetadata metadata);
}
2 changes: 1 addition & 1 deletion .../src/main/java/org/opensearch/dataprepper/plugins/source/rds/datatype/DataTypeHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ MySQLDataType.DataCategory.JSON, new JsonTypeHandler(),
MySQLDataType.DataCategory.SPATIAL, new SpatialTypeHandler()
);

public static String getDataByColumnType(final MySQLDataType columnType, final String columnName, final Object value,
public static Object getDataByColumnType(final MySQLDataType columnType, final String columnName, final Object value,
final TableMetadata metadata) {
if (value == null) {
return null;
Expand Down
2 changes: 1 addition & 1 deletion ...ain/java/org/opensearch/dataprepper/plugins/source/rds/datatype/impl/JsonTypeHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ public String handle(final MySQLDataType columnType, final String columnName, fi
return convertToJson((byte[]) value);
}

private static String convertToJson(final byte[] jsonBytes) {
private String convertToJson(final byte[] jsonBytes) {
try {
return JsonBinary.parseAsString(jsonBytes);
} catch (IOException e) {
Expand Down
19 changes: 9 additions & 10 deletions .../java/org/opensearch/dataprepper/plugins/source/rds/datatype/impl/NumericTypeHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
public class NumericTypeHandler implements DataTypeHandler {

@Override
public String handle(final MySQLDataType columnType, final String columnName, final Object value,
public Number handle(final MySQLDataType columnType, final String columnName, final Object value,
final TableMetadata metadata) {
if (value == null) {
return null;
Expand All @@ -26,30 +26,29 @@ public String handle(final MySQLDataType columnType, final String columnName, fi
return handleNumericType(columnType, (Number) value);
}

private String handleNumericType(final MySQLDataType columnType, final Number value) {
private Number handleNumericType(final MySQLDataType columnType, final Number value) {
if (columnType.isUnsigned()) {
if(columnType == MySQLDataType.BIGINT_UNSIGNED) {
if (columnType == MySQLDataType.BIGINT_UNSIGNED) {
return handleUnsignedDouble(value);
} else {
return handleUnsignedNumber(value, columnType.getUnsignedMask());
}
}
return value.toString();
return value;
}

private String handleUnsignedNumber(final Number value, final long mask) {
private Number handleUnsignedNumber(final Number value, final long mask) {
final long longVal = value.longValue();
return String.valueOf(longVal < 0 ? longVal & mask : longVal);
return longVal < 0 ? longVal & mask : longVal;
}

private String handleUnsignedDouble(final Number value) {
private Number handleUnsignedDouble(final Number value) {
long longVal = value.longValue();
if (longVal < 0) {
return BigInteger.valueOf(longVal & Long.MAX_VALUE)
.add(BigInteger.valueOf(Long.MAX_VALUE))
.add(BigInteger.ONE)
.toString();
.add(BigInteger.ONE);
}
return String.valueOf(longVal);
return value;
}
}
4 changes: 2 additions & 2 deletions ...n/java/org/opensearch/dataprepper/plugins/source/rds/datatype/impl/StringTypeHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ public String handle(final MySQLDataType columnType, final String columnName, fi
}
}

private static List<String> getSetValues(final long numericValue, final String[] setStrValues) {
private List<String> getSetValues(final long numericValue, final String[] setStrValues) {
final List<String> setValues = new ArrayList<>();
for (int i = 0; i < setStrValues.length; i++) {
if ((numericValue & (1L << i)) != 0) {
Expand All @@ -34,7 +34,7 @@ private static List<String> getSetValues(final long numericValue, final String[]
return setValues;
}

private static String getEnumValue(final int numericValue, final String[] enumStrValues) {
private String getEnumValue(final int numericValue, final String[] enumStrValues) {
return enumStrValues[numericValue - 1];
}
}
4 changes: 3 additions & 1 deletion ...va/org/opensearch/dataprepper/plugins/source/rds/datatype/impl/BinaryTypeHandlerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import org.opensearch.dataprepper.plugins.source.rds.datatype.MySQLDataType;
import org.opensearch.dataprepper.plugins.source.rds.model.TableMetadata;

import static org.hamcrest.CoreMatchers.instanceOf;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.MatcherAssert.assertThat;

Expand All @@ -24,8 +25,9 @@ public void test_handle() {
final TableMetadata metadata = new TableMetadata(
UUID.randomUUID().toString(), UUID.randomUUID().toString(), List.of(columnName), List.of(columnName),
Collections.emptyMap(), Collections.emptyMap());
String result = handler.handle(columnType, columnName, testData, metadata);
Object result = handler.handle(columnType, columnName, testData, metadata);

assertThat(result, is(instanceOf(String.class)));
assertThat(result, is(Base64.getEncoder().encodeToString(testData)));
}
}
4 changes: 3 additions & 1 deletion ...java/org/opensearch/dataprepper/plugins/source/rds/datatype/impl/JsonTypeHandlerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import java.util.List;
import java.util.UUID;

import static org.hamcrest.CoreMatchers.instanceOf;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.MatcherAssert.assertThat;

Expand All @@ -24,8 +25,9 @@ public void test_handle() {
final TableMetadata metadata = new TableMetadata(
UUID.randomUUID().toString(), UUID.randomUUID().toString(), List.of(columnName), List.of(columnName),
Collections.emptyMap(), Collections.emptyMap());
String result = handler.handle(columnType, columnName, testData, metadata);
Object result = handler.handle(columnType, columnName, testData, metadata);

assertThat(result, is(instanceOf(String.class)));
assertThat(result, is(jsonValue));
}
}
82 changes: 45 additions & 37 deletions ...a/org/opensearch/dataprepper/plugins/source/rds/datatype/impl/NumericTypeHandlerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import java.util.UUID;
import java.util.stream.Stream;

import static org.hamcrest.CoreMatchers.instanceOf;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.jupiter.api.Assertions.assertThrows;
Expand All @@ -27,71 +28,78 @@ public void test_handle(final MySQLDataType mySQLDataType, final String columnNa
UUID.randomUUID().toString(), UUID.randomUUID().toString(), List.of(columnName), List.of(columnName),
Collections.emptyMap(), Collections.emptyMap());
final DataTypeHandler numericTypeHandler = new NumericTypeHandler();
String result = numericTypeHandler.handle(mySQLDataType, columnName, value, metadata);
Object result = numericTypeHandler.handle(mySQLDataType, columnName, value, metadata);

if (result != null) {
assertThat(result, instanceOf(expectedValue.getClass()));
}
assertThat(result, is(expectedValue));
}

private static Stream<Arguments> provideNumericTypeData() {
return Stream.of(
// TINYINT tests (signed: -128 to 127)
Arguments.of(MySQLDataType.TINYINT, "tinyint_col", (byte)1, "1"),
Arguments.of(MySQLDataType.TINYINT, "tinyint_col", (byte)-128, "-128"),
Arguments.of(MySQLDataType.TINYINT, "tinyint_col", (byte)127, "127"),
Arguments.of(MySQLDataType.TINYINT, "tinyint_col", (byte)1, (byte)1),
Arguments.of(MySQLDataType.TINYINT, "tinyint_col", (byte)-128, (byte)-128),
Arguments.of(MySQLDataType.TINYINT, "tinyint_col", (byte)127, (byte)127),
Arguments.of(MySQLDataType.TINYINT, "tinyint_col", null, null),

// TINYINT UNSIGNED tests (0 to 255)
Arguments.of(MySQLDataType.TINYINT_UNSIGNED, "tinyint_unsigned_col", (short)0, "0"),
Arguments.of(MySQLDataType.TINYINT_UNSIGNED, "tinyint_unsigned_col", (short)255, "255"),
Arguments.of(MySQLDataType.TINYINT_UNSIGNED, "tinyint_unsigned_col", (short)128, "128"),
Arguments.of(MySQLDataType.TINYINT_UNSIGNED, "tinyint_unsigned_col", (short)0, 0L),
Arguments.of(MySQLDataType.TINYINT_UNSIGNED, "tinyint_unsigned_col", (short)255, 255L),
Arguments.of(MySQLDataType.TINYINT_UNSIGNED, "tinyint_unsigned_col", (short)128, 128L),
Arguments.of(MySQLDataType.TINYINT_UNSIGNED, "tinyint_unsigned_col", (short)-1, 255L),

// SMALLINT tests (signed: -32,768 to 32,767)
Arguments.of(MySQLDataType.SMALLINT, "smallint_col", (short)32767, "32767"),
Arguments.of(MySQLDataType.SMALLINT, "smallint_col", (short)-32768, "-32768"),
Arguments.of(MySQLDataType.SMALLINT, "smallint_col", (short)0, "0"),
Arguments.of(MySQLDataType.SMALLINT, "smallint_col", (short)32767, (short)32767),
Arguments.of(MySQLDataType.SMALLINT, "smallint_col", (short)-32768, (short)-32768),
Arguments.of(MySQLDataType.SMALLINT, "smallint_col", (short)0, (short)0),

// SMALLINT UNSIGNED tests (0 to 65,535)
Arguments.of(MySQLDataType.SMALLINT_UNSIGNED, "smallint_unsigned_col", 0, "0"),
Arguments.of(MySQLDataType.SMALLINT_UNSIGNED, "smallint_unsigned_col", 65535, "65535"),
Arguments.of(MySQLDataType.SMALLINT_UNSIGNED, "smallint_unsigned_col", 32768, "32768"),
Arguments.of(MySQLDataType.SMALLINT_UNSIGNED, "smallint_unsigned_col", 0, 0L),
Arguments.of(MySQLDataType.SMALLINT_UNSIGNED, "smallint_unsigned_col", 65535, 65535L),
Arguments.of(MySQLDataType.SMALLINT_UNSIGNED, "smallint_unsigned_col", 32768, 32768L),
Arguments.of(MySQLDataType.SMALLINT_UNSIGNED, "smallint_unsigned_col", -1, 65535L),

// INTEGER/INT tests (signed: -2,147,483,648 to 2,147,483,647)
Arguments.of(MySQLDataType.INT, "int_col", 2147483647, "2147483647"),
Arguments.of(MySQLDataType.INT, "int_col", -2147483648, "-2147483648"),
Arguments.of(MySQLDataType.INT, "int_col", 0, "0"),
Arguments.of(MySQLDataType.INT, "int_col", 2147483647, 2147483647),
Arguments.of(MySQLDataType.INT, "int_col", -2147483648, -2147483648),
Arguments.of(MySQLDataType.INT, "int_col", 0, 0),

// INTEGER/INT UNSIGNED tests (0 to 4,294,967,295)
Arguments.of(MySQLDataType.INT_UNSIGNED, "int_unsigned_col", 4294967295L, "4294967295"),
Arguments.of(MySQLDataType.INT_UNSIGNED, "int_unsigned_col", 0L, "0"),
Arguments.of(MySQLDataType.INT_UNSIGNED, "int_unsigned_col", 2147483648L, "2147483648"),
Arguments.of(MySQLDataType.INT_UNSIGNED, "int_unsigned_col", 4294967295L, 4294967295L),
Arguments.of(MySQLDataType.INT_UNSIGNED, "int_unsigned_col", 0L, 0L),
Arguments.of(MySQLDataType.INT_UNSIGNED, "int_unsigned_col", 2147483648L, 2147483648L),
Arguments.of(MySQLDataType.INT_UNSIGNED, "int_unsigned_col", -1, 4294967295L),

// BIGINT tests (signed: -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807)
Arguments.of(MySQLDataType.BIGINT, "bigint_col", 9223372036854775807L, "9223372036854775807"),
Arguments.of(MySQLDataType.BIGINT, "bigint_col", -9223372036854775808L, "-9223372036854775808"),
Arguments.of(MySQLDataType.BIGINT, "bigint_col", 0L, "0"),
Arguments.of(MySQLDataType.BIGINT, "bigint_col", 9223372036854775807L, 9223372036854775807L),
Arguments.of(MySQLDataType.BIGINT, "bigint_col", -9223372036854775808L, -9223372036854775808L),
Arguments.of(MySQLDataType.BIGINT, "bigint_col", 0L, 0L),

// BIGINT UNSIGNED tests (0 to 18,446,744,073,709,551,615)
Arguments.of(MySQLDataType.BIGINT_UNSIGNED, "bigint_unsigned_col", new BigInteger("18446744073709551615"), "18446744073709551615"),
Arguments.of(MySQLDataType.BIGINT_UNSIGNED, "bigint_unsigned_col", BigInteger.ZERO, "0"),
Arguments.of(MySQLDataType.BIGINT_UNSIGNED, "bigint_unsigned_col", new BigInteger("9223372036854775808"), "9223372036854775808"),
Arguments.of(MySQLDataType.BIGINT_UNSIGNED, "bigint_unsigned_col", new BigInteger("18446744073709551615"), new BigInteger("18446744073709551615")),
Arguments.of(MySQLDataType.BIGINT_UNSIGNED, "bigint_unsigned_col", BigInteger.ZERO, new BigInteger("0")),
Arguments.of(MySQLDataType.BIGINT_UNSIGNED, "bigint_unsigned_col", new BigInteger("9223372036854775808"), new BigInteger("9223372036854775808")),
Arguments.of(MySQLDataType.BIGINT_UNSIGNED, "bigint_unsigned_col", new BigInteger("-1"), new BigInteger("18446744073709551615")),

// DECIMAL/NUMERIC tests
Arguments.of(MySQLDataType.DECIMAL, "decimal_col", new BigDecimal("123.45"), "123.45"),
Arguments.of(MySQLDataType.DECIMAL, "decimal_col", new BigDecimal("-123.45"), "-123.45"),
Arguments.of(MySQLDataType.DECIMAL, "decimal_col", new BigDecimal("0.0"), "0.0"),
Arguments.of(MySQLDataType.DECIMAL, "decimal_col", new BigDecimal("999999.99"), "999999.99"),
Arguments.of(MySQLDataType.DECIMAL, "decimal_col", new BigDecimal("123.45"), new BigDecimal("123.45")),
Arguments.of(MySQLDataType.DECIMAL, "decimal_col", new BigDecimal("-123.45"), new BigDecimal("-123.45")),
Arguments.of(MySQLDataType.DECIMAL, "decimal_col", new BigDecimal("0.0"), new BigDecimal("0.0")),
Arguments.of(MySQLDataType.DECIMAL, "decimal_col", new BigDecimal("999999.99"), new BigDecimal("999999.99")),

// FLOAT tests
Arguments.of(MySQLDataType.FLOAT, "float_col", 123.45f, "123.45"),
Arguments.of(MySQLDataType.FLOAT, "float_col", -123.45f, "-123.45"),
Arguments.of(MySQLDataType.FLOAT, "float_col", 0.0f, "0.0"),
Arguments.of(MySQLDataType.FLOAT, "float_col", Float.MAX_VALUE, String.valueOf(Float.MAX_VALUE)),
Arguments.of(MySQLDataType.FLOAT, "float_col", 123.45f, 123.45f),
Arguments.of(MySQLDataType.FLOAT, "float_col", -123.45f, -123.45f),
Arguments.of(MySQLDataType.FLOAT, "float_col", 0.0f, 0.0f),
Arguments.of(MySQLDataType.FLOAT, "float_col", Float.MAX_VALUE, Float.MAX_VALUE),

// DOUBLE tests
Arguments.of(MySQLDataType.DOUBLE, "double_col", 123.45678901234, "123.45678901234"),
Arguments.of(MySQLDataType.DOUBLE, "double_col", -123.45678901234, "-123.45678901234"),
Arguments.of(MySQLDataType.DOUBLE, "double_col", 0.0, "0.0"),
Arguments.of(MySQLDataType.DOUBLE, "double_col", Double.MAX_VALUE, String.valueOf(Double.MAX_VALUE))
Arguments.of(MySQLDataType.DOUBLE, "double_col", 123.45678901234, 123.45678901234),
Arguments.of(MySQLDataType.DOUBLE, "double_col", -123.45678901234, -123.45678901234),
Arguments.of(MySQLDataType.DOUBLE, "double_col", 0.0, 0.0),
Arguments.of(MySQLDataType.DOUBLE, "double_col", Double.MAX_VALUE, Double.MAX_VALUE)
);
}

Expand Down
4 changes: 3 additions & 1 deletion ...a/org/opensearch/dataprepper/plugins/source/rds/datatype/impl/SpatialTypeHandlerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import java.util.List;
import java.util.UUID;

import static org.hamcrest.CoreMatchers.instanceOf;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.MatcherAssert.assertThat;

Expand All @@ -23,8 +24,9 @@ public void test_handle() {
final TableMetadata metadata = new TableMetadata(
UUID.randomUUID().toString(), UUID.randomUUID().toString(), List.of(columnName), List.of(columnName),
Collections.emptyMap(), Collections.emptyMap());
String result = handler.handle(columnType, columnName, value.getBytes(), metadata);
Object result = handler.handle(columnType, columnName, value.getBytes(), metadata);

assertThat(result, is(instanceOf(String.class)));
assertThat(result, is(value));
}
}
7 changes: 5 additions & 2 deletions ...va/org/opensearch/dataprepper/plugins/source/rds/datatype/impl/StringTypeHandlerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import java.util.Map;
import java.util.UUID;

import static org.hamcrest.CoreMatchers.instanceOf;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
Expand All @@ -26,8 +27,9 @@ public void test_handle_char_string() {
UUID.randomUUID().toString(), UUID.randomUUID().toString(), List.of(columnName), List.of(columnName),
Collections.emptyMap(), Collections.emptyMap());

String result = handler.handle(columnType, columnName, value, metadata);
Object result = handler.handle(columnType, columnName, value, metadata);

assertThat(result, is(instanceOf(String.class)));
assertThat(result, is(value));
}

Expand All @@ -42,8 +44,9 @@ public void test_handle_byte_string() {
UUID.randomUUID().toString(), UUID.randomUUID().toString(), List.of(columnName), List.of(columnName),
Collections.emptyMap(), Collections.emptyMap());

String result = handler.handle(columnType, columnName, testBytes, metadata);
Object result = handler.handle(columnType, columnName, testBytes, metadata);

assertThat(result, is(instanceOf(String.class)));
assertThat(result, is(value));
}

Expand Down
4 changes: 3 additions & 1 deletion .../org/opensearch/dataprepper/plugins/source/rds/datatype/impl/TemporalTypeHandlerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import java.util.List;
import java.util.UUID;

import static org.hamcrest.CoreMatchers.instanceOf;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.MatcherAssert.assertThat;

Expand All @@ -23,8 +24,9 @@ public void test_handle() {
final TableMetadata metadata = new TableMetadata(
UUID.randomUUID().toString(), UUID.randomUUID().toString(), List.of(columnName), List.of(columnName),
Collections.emptyMap(), Collections.emptyMap());
String result = handler.handle(columnType, columnName, value, metadata);
Object result = handler.handle(columnType, columnName, value, metadata);

assertThat(result, is(instanceOf(String.class)));
assertThat(result, is(value));
}
}

0 comments on commit 764e36c

Please sign in to comment.