Skip to content

Commit

Permalink
only after iterator
Browse files Browse the repository at this point in the history
  • Loading branch information
ritchie46 committed Jun 10, 2024
1 parent c344704 commit e823762
Show file tree
Hide file tree
Showing 4 changed files with 14 additions and 23 deletions.
14 changes: 7 additions & 7 deletions crates/polars-core/src/chunked_array/from_iterator_par.rs
Original file line number Diff line number Diff line change
Expand Up @@ -82,21 +82,21 @@ where
{
fn from_par_iter<I: IntoParallelIterator<Item = Option<T::Native>>>(iter: I) -> Self {
let chunks = collect_into_linked_list(iter, MutablePrimitiveArray::new);
Self::from_chunk_iter("", chunks)
Self::from_chunk_iter("", chunks).optional_rechunk()
}
}

impl FromParallelIterator<bool> for BooleanChunked {
fn from_par_iter<I: IntoParallelIterator<Item = bool>>(iter: I) -> Self {
let chunks = collect_into_linked_list(iter, MutableBooleanArray::new);
Self::from_chunk_iter("", chunks)
Self::from_chunk_iter("", chunks).optional_rechunk()
}
}

impl FromParallelIterator<Option<bool>> for BooleanChunked {
fn from_par_iter<I: IntoParallelIterator<Item = Option<bool>>>(iter: I) -> Self {
let chunks = collect_into_linked_list(iter, MutableBooleanArray::new);
Self::from_chunk_iter("", chunks)
Self::from_chunk_iter("", chunks).optional_rechunk()
}
}

Expand All @@ -106,7 +106,7 @@ where
{
fn from_par_iter<I: IntoParallelIterator<Item = Ptr>>(iter: I) -> Self {
let chunks = collect_into_linked_list(iter, MutableBinaryViewArray::new);
Self::from_chunk_iter("", chunks)
Self::from_chunk_iter("", chunks).optional_rechunk()
}
}

Expand All @@ -116,7 +116,7 @@ where
{
fn from_par_iter<I: IntoParallelIterator<Item = Ptr>>(iter: I) -> Self {
let chunks = collect_into_linked_list(iter, MutableBinaryViewArray::new);
Self::from_chunk_iter("", chunks)
Self::from_chunk_iter("", chunks).optional_rechunk()
}
}

Expand All @@ -126,7 +126,7 @@ where
{
fn from_par_iter<I: IntoParallelIterator<Item = Option<Ptr>>>(iter: I) -> Self {
let chunks = collect_into_linked_list(iter, MutableBinaryViewArray::new);
Self::from_chunk_iter("", chunks)
Self::from_chunk_iter("", chunks).optional_rechunk()
}
}

Expand All @@ -136,7 +136,7 @@ where
{
fn from_par_iter<I: IntoParallelIterator<Item = Option<Ptr>>>(iter: I) -> Self {
let chunks = collect_into_linked_list(iter, MutableBinaryViewArray::new);
Self::from_chunk_iter("", chunks)
Self::from_chunk_iter("", chunks).optional_rechunk()
}
}

Expand Down
11 changes: 7 additions & 4 deletions crates/polars-core/src/chunked_array/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -144,9 +144,13 @@ pub struct ChunkedArray<T: PolarsDataType> {
}

impl<T: PolarsDataType> ChunkedArray<T> {
fn check_rechunk(self) -> Self {
fn should_rechunk(&self) -> bool {
self.chunks.len() > 1 && self.chunks.len() > self.len() / 3
}

fn optional_rechunk(self) -> Self {
// Rechunk if we have many small chunks.
if self.chunks.len() > 1 && self.chunks.len() > self.len() / 3 {
if self.should_rechunk() {
self.rechunk()
} else {
self
Expand All @@ -161,7 +165,7 @@ impl<T: PolarsDataType> ChunkedArray<T> {
unsafe {
let mut chunked_arr = Self::new_with_dims(field, chunks, 0, 0);
chunked_arr.compute_len();
chunked_arr.check_rechunk()
chunked_arr
}
}

Expand All @@ -186,7 +190,6 @@ impl<T: PolarsDataType> ChunkedArray<T> {
length,
null_count,
}
.check_rechunk()
}

/// Get a reference to the used [`Metadata`]
Expand Down
11 changes: 0 additions & 11 deletions crates/polars/tests/it/core/constructor.rs

This file was deleted.

1 change: 0 additions & 1 deletion crates/polars/tests/it/core/mod.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
mod constructor;
mod date_like;
mod group_by;
mod joins;
Expand Down

0 comments on commit e823762

Please sign in to comment.