refactored compaction_session.py, addressed comments, pytests pass

ray-project · Jul 19, 2024 · 207ecba · 207ecba
1 parent 56f0aa6
commit 207ecba
Show file tree

Hide file tree

Showing 11 changed files with 928 additions and 575 deletions.
diff --git a/deltacat/compute/compactor/model/compact_partition_params.py b/deltacat/compute/compactor/model/compact_partition_params.py
@@ -185,6 +185,11 @@ def compacted_file_content_type(self, content_type: ContentType) -> None:
 
     @property
     def task_max_parallelism(self) -> int:
+        if self.pg_config:
+            cluster_resources = self["pg_config"].resource
+            cluster_cpus = cluster_resources["CPU"]
+            self.task_max_parallelism = cluster_cpus
+            self["task_max_parallelism"] = self.task_max_parallelism
         return self["task_max_parallelism"]
 
     @task_max_parallelism.setter

diff --git a/deltacat/compute/compactor_v2/compaction_session.py b/deltacat/compute/compactor_v2/compaction_session.py