Skip to content

Commit

Permalink
Add extra tests for Pandas export
Browse files Browse the repository at this point in the history
  • Loading branch information
adamreeve committed Nov 5, 2023
1 parent c0744c4 commit 18b8f08
Show file tree
Hide file tree
Showing 2 changed files with 81 additions and 1 deletion.
2 changes: 1 addition & 1 deletion nptdms/export/pandas_export.py
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ def _channels_to_dataframe(
def _array_for_pd(array):
""" Convert data array to a format suitable for a Pandas dataframe
"""
if np.issubdtype(array.dtype, np.dtype('void')):
if np.issubdtype(array.dtype, np.dtype('void')) and len(array.dtype) == 0:
# If dtype is void then the array must also be empty.
# Pandas doesn't like void data types, so these are converted to empty float64 arrays
# and Pandas will fill values with NaN
Expand Down
80 changes: 80 additions & 0 deletions nptdms/test/test_pandas.py
Original file line number Diff line number Diff line change
Expand Up @@ -352,6 +352,86 @@ def check_series(series):
check_series(channel_df["/'Group'/'Channel1'"])


@pytest.mark.parametrize('arrow_dtypes', [False, True])
def test_bool_data_to_pandas(arrow_dtypes):
test_file, expected_data = scenarios.bool_data().values
df = test_file.load()['group'].as_dataframe(arrow_dtypes=arrow_dtypes)
np.testing.assert_allclose(df['bool_channel'], expected_data[('group', 'bool_channel')])


@pytest.mark.parametrize('arrow_dtypes', [False, True])
def test_string_data_to_pandas(arrow_dtypes):
strings = ["abcdefg", "qwertyuiop"]

test_file = GeneratedFile()
toc = ("kTocMetaData", "kTocRawData", "kTocNewObjList")
metadata = (
# Number of objects
"01 00 00 00"
# Length of the object path
"18 00 00 00")
metadata += string_hexlify("/'Group'/'StringChannel'")
metadata += (
# Length of index information
"1C 00 00 00"
# Raw data data type
"20 00 00 00"
# Dimension
"01 00 00 00"
# Number of raw data values
"02 00 00 00"
"00 00 00 00"
# Number of bytes in data
"19 00 00 00"
"00 00 00 00"
# Number of properties (0)
"00 00 00 00")
data = (
"07 00 00 00" # index to after first string
"11 00 00 00" # index to after second string
)
for string in strings:
data += string_hexlify(string)
test_file.add_segment(toc, metadata, data)
tdms_data = test_file.load()

series = tdms_data["Group"].as_dataframe(arrow_dtypes=arrow_dtypes)["StringChannel"]

assert len(series) == len(strings)
for expected, read in zip(strings, series):
assert expected == read


def test_dataframe_with_complex_data():
test_file, expected_data = scenarios.complex_data().values
df = test_file.load()['group'].as_dataframe()
np.testing.assert_allclose(df['complex_single_channel'], expected_data[('group', 'complex_single_channel')])
np.testing.assert_allclose(df['complex_double_channel'], expected_data[('group', 'complex_double_channel')])


def test_dataframe_with_raw_timestamp_data():
test_file = GeneratedFile()
seconds = 3672033330
second_fractions = 1234567890 * 10 ** 10
test_file.add_segment(
("kTocMetaData", "kTocRawData", "kTocNewObjList"),
segment_objects_metadata(
channel_metadata("/'group'/'channel1'", 0x44, 4),
),
hexlify_value("<Q", 0) + hexlify_value("<q", seconds) +
hexlify_value("<Q", second_fractions) + hexlify_value("<q", seconds) +
hexlify_value("<Q", 0) + hexlify_value("<q", seconds + 1) +
hexlify_value("<Q", second_fractions) + hexlify_value("<q", seconds + 1)
)

with test_file.get_tempfile() as temp_file:
tdms_data = TdmsFile.read(temp_file.file, raw_timestamps=True)
with pytest.raises(ValueError) as exc_info:
tdms_data['group'].as_dataframe()
message = str(exc_info.value)
assert "compound dtype" in message


def test_export_with_empty_channels():
"""Convert a group to dataframe when a channel has empty data and void data type"""

Expand Down

0 comments on commit 18b8f08

Please sign in to comment.