refactor: change spark testing (add spark_session as pytest fixture)

ydataai · Sep 17, 2023 · d6e6a2e · d6e6a2e
1 parent 1f64447
commit d6e6a2e
Show file tree

Hide file tree

Showing 7 changed files with 18 additions and 25 deletions.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -220,7 +220,6 @@ jobs:
         run: echo "ARROW_PRE_0_15_IPC_FORMAT=1" >> $GITHUB_ENV
       - run: echo "SPARK_LOCAL_IP=127.0.0.1" >> $GITHUB_ENV
       - run: make install
-      - run: make install-spark-ci
       - run: pip install -r requirements-spark.txt # Make sure the proper version of pandas is install after everything
       - run: make test_spark
 
diff --git a/Makefile b/Makefile
@@ -17,7 +17,7 @@ test:
 	ydata_profiling -h
 
 test_spark:
-	pytest --spark_home=${SPARK_HOME} tests/backends/spark_backend/
+	pytest tests/backends/spark_backend/
 	ydata_profiling -h
 
 test_cov:
@@ -52,13 +52,6 @@ install_dev_spark:
 install:
 	pip install -e .[notebook]
 
-install-spark-ci:
-	sudo apt-get update
-	sudo apt-get -y install openjdk-8-jdk
-	curl https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz \
-	--output ${SPARK_DIRECTORY}/spark.tgz
-	cd ${SPARK_DIRECTORY} && tar -xvzf spark.tgz && mv spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION} spark
-
 # compile requirements files
 compile:
 	$(PYTHON) -m piptools compile -o requirements.txt pyproject.toml

diff --git a/pyproject.toml b/pyproject.toml
@@ -76,7 +76,6 @@ dev = [
     "pre-commit>=2.8.2",
     "pytest",
     "pytest-cov",
-    "pytest-spark",
     "pyarrow",
     "twine>=3.1.1",
     "wheel",

diff --git a/requirements-dev.txt b/requirements-dev.txt
@@ -78,8 +78,6 @@ fastjsonschema==2.18.0
     # via nbformat
 filelock==3.12.4
     # via virtualenv
-findspark==2.0.1
-    # via pytest-spark
 fonttools==4.42.1
     # via matplotlib
 htmlmin==0.1.12
@@ -276,12 +274,9 @@ pytest==7.4.2
     # via
     #   nbval
     #   pytest-cov
-    #   pytest-spark
     #   ydata-profiling (pyproject.toml)
 pytest-cov==4.1.0
     # via ydata-profiling (pyproject.toml)
-pytest-spark==0.6.0
-    # via ydata-profiling (pyproject.toml)
 python-dateutil==2.8.2
     # via
     #   jupyter-client

diff --git a/requirements-spark.txt b/requirements-spark.txt
@@ -69,8 +69,6 @@ fastjsonschema==2.18.0
     # via nbformat
 filelock==3.12.4
     # via virtualenv
-findspark==2.0.1
-    # via pytest-spark
 fonttools==4.42.1
     # via matplotlib
 htmlmin==0.1.12
@@ -255,12 +253,9 @@ pytest==7.4.2
     # via
     #   nbval
     #   pytest-cov
-    #   pytest-spark
     #   ydata-profiling (pyproject.toml)
 pytest-cov==4.1.0
     # via ydata-profiling (pyproject.toml)
-pytest-spark==0.6.0
-    # via ydata-profiling (pyproject.toml)
 python-dateutil==2.8.2
     # via
     #   jupyter-client

diff --git a/requirements-test.txt b/requirements-test.txt
@@ -69,8 +69,6 @@ fastjsonschema==2.18.0
     # via nbformat
 filelock==3.12.4
     # via virtualenv
-findspark==2.0.1
-    # via pytest-spark
 fonttools==4.42.1
     # via matplotlib
 htmlmin==0.1.12
@@ -251,12 +249,9 @@ pytest==7.4.2
     # via
     #   nbval
     #   pytest-cov
-    #   pytest-spark
     #   ydata-profiling (pyproject.toml)
 pytest-cov==4.1.0
     # via ydata-profiling (pyproject.toml)
-pytest-spark==0.6.0
-    # via ydata-profiling (pyproject.toml)
 python-dateutil==2.8.2
     # via
     #   jupyter-client

diff --git a/tests/backends/spark_backend/conftest.py b/tests/backends/spark_backend/conftest.py
@@ -0,0 +1,17 @@
+import pytest
+from pyspark.sql import SparkSession
+
+
+@pytest.fixture(scope="session")
+def spark_session():
+    spark = (
+        SparkSession.builder.master("local[1]")
+        .appName("local-tests")
+        .config("spark.executor.cores", "1")
+        .config("spark.executor.instances", "1")
+        .config("spark.sql.shuffle.partitions", "1")
+        .config("spark.driver.bindAddress", "127.0.0.1")
+        .getOrCreate()
+    )
+    yield spark
+    spark.stop()