From b257bdd9493e227b16345d9aeef9222104bc3aef Mon Sep 17 00:00:00 2001 From: Parth Bansal Date: Fri, 5 Jul 2024 16:29:16 +0200 Subject: [PATCH 1/9] update --- tests/integration/test_dbutils.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/integration/test_dbutils.py b/tests/integration/test_dbutils.py index a7e780eb3..25db4a838 100644 --- a/tests/integration/test_dbutils.py +++ b/tests/integration/test_dbutils.py @@ -8,6 +8,13 @@ from databricks.sdk.errors import NotFound +def test_put_local_path(w, random, tmp_path): + to_write = random(1024 * 1024 * 2.5).encode() + w.dbutils.fs.put(f'file:{tmp_path}', to_write, True) + assert w.dbutils.fs.head(f'file:{tmp_path}') == to_write + assert False + + def test_rest_dbfs_ls(w, env_or_skip): from databricks.sdk.runtime import dbutils From b2e0841a5c18701d6baea99ad9c88ca47e2e7bf5 Mon Sep 17 00:00:00 2001 From: Parth Bansal Date: Fri, 5 Jul 2024 16:53:29 +0200 Subject: [PATCH 2/9] update --- Makefile | 2 +- tests/integration/test_dbutils.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index eb8fe8397..8d388eb8a 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,7 @@ test: pytest -m 'not integration and not benchmark' --cov=databricks --cov-report html tests integration: - pytest -n auto -m 'integration and not benchmark' --reruns 2 --dist loadgroup --cov=databricks --cov-report html tests + pytest -n auto -m 'integration and not benchmark' --reruns 2 --dist loadgroup --cov=databricks --cov-report html tests/integration/test_dbutils.py::test_put_local_path benchmark: pytest -m 'benchmark' tests diff --git a/tests/integration/test_dbutils.py b/tests/integration/test_dbutils.py index 25db4a838..3c96019fc 100644 --- a/tests/integration/test_dbutils.py +++ b/tests/integration/test_dbutils.py @@ -10,6 +10,7 @@ def test_put_local_path(w, random, tmp_path): to_write = random(1024 * 1024 * 2.5).encode() + tmp_path = tmp_path / "tmp_file" w.dbutils.fs.put(f'file:{tmp_path}', to_write, True) assert w.dbutils.fs.head(f'file:{tmp_path}') == to_write assert False From 519d8c2deee0501f01f848cd1e3706801b60ce26 Mon Sep 17 00:00:00 2001 From: Parth Bansal Date: Fri, 5 Jul 2024 16:56:17 +0200 Subject: [PATCH 3/9] update --- tests/integration/test_dbutils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_dbutils.py b/tests/integration/test_dbutils.py index 3c96019fc..596a854db 100644 --- a/tests/integration/test_dbutils.py +++ b/tests/integration/test_dbutils.py @@ -9,7 +9,7 @@ def test_put_local_path(w, random, tmp_path): - to_write = random(1024 * 1024 * 2.5).encode() + to_write = random(1024 * 1024 * 2.5) tmp_path = tmp_path / "tmp_file" w.dbutils.fs.put(f'file:{tmp_path}', to_write, True) assert w.dbutils.fs.head(f'file:{tmp_path}') == to_write From 65dfaec4bc6d06ae7f527d2a5b38d75fb3ca08a4 Mon Sep 17 00:00:00 2001 From: Parth Bansal Date: Fri, 5 Jul 2024 17:00:48 +0200 Subject: [PATCH 4/9] update --- tests/integration/test_dbutils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_dbutils.py b/tests/integration/test_dbutils.py index 596a854db..6c31ae504 100644 --- a/tests/integration/test_dbutils.py +++ b/tests/integration/test_dbutils.py @@ -12,7 +12,7 @@ def test_put_local_path(w, random, tmp_path): to_write = random(1024 * 1024 * 2.5) tmp_path = tmp_path / "tmp_file" w.dbutils.fs.put(f'file:{tmp_path}', to_write, True) - assert w.dbutils.fs.head(f'file:{tmp_path}') == to_write + assert w.dbutils.fs.head(f'file:{tmp_path}', 1024*1024*2.5) == to_write assert False From 8521b4695dfc2b4267686d8e7f113003947758df Mon Sep 17 00:00:00 2001 From: Parth Bansal Date: Fri, 5 Jul 2024 17:04:16 +0200 Subject: [PATCH 5/9] update --- tests/integration/test_dbutils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_dbutils.py b/tests/integration/test_dbutils.py index 6c31ae504..6309bbcb9 100644 --- a/tests/integration/test_dbutils.py +++ b/tests/integration/test_dbutils.py @@ -9,10 +9,10 @@ def test_put_local_path(w, random, tmp_path): - to_write = random(1024 * 1024 * 2.5) + to_write = random(1024 * 1024 * 2) tmp_path = tmp_path / "tmp_file" w.dbutils.fs.put(f'file:{tmp_path}', to_write, True) - assert w.dbutils.fs.head(f'file:{tmp_path}', 1024*1024*2.5) == to_write + assert w.dbutils.fs.head(f'file:{tmp_path}', 1024*1024*2) == to_write assert False From 1cb0a4bfc8a30da7e3cf7afeea105395ce6eb71d Mon Sep 17 00:00:00 2001 From: Parth Bansal Date: Fri, 5 Jul 2024 17:06:39 +0200 Subject: [PATCH 6/9] update --- Makefile | 2 +- tests/integration/test_dbutils.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 8d388eb8a..eb8fe8397 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,7 @@ test: pytest -m 'not integration and not benchmark' --cov=databricks --cov-report html tests integration: - pytest -n auto -m 'integration and not benchmark' --reruns 2 --dist loadgroup --cov=databricks --cov-report html tests/integration/test_dbutils.py::test_put_local_path + pytest -n auto -m 'integration and not benchmark' --reruns 2 --dist loadgroup --cov=databricks --cov-report html tests benchmark: pytest -m 'benchmark' tests diff --git a/tests/integration/test_dbutils.py b/tests/integration/test_dbutils.py index 6309bbcb9..667899df0 100644 --- a/tests/integration/test_dbutils.py +++ b/tests/integration/test_dbutils.py @@ -13,7 +13,6 @@ def test_put_local_path(w, random, tmp_path): tmp_path = tmp_path / "tmp_file" w.dbutils.fs.put(f'file:{tmp_path}', to_write, True) assert w.dbutils.fs.head(f'file:{tmp_path}', 1024*1024*2) == to_write - assert False def test_rest_dbfs_ls(w, env_or_skip): From b5e70769a929283cb09da4f83d94c2513eb8ea50 Mon Sep 17 00:00:00 2001 From: Parth Bansal Date: Mon, 8 Jul 2024 10:25:01 +0200 Subject: [PATCH 7/9] update --- Makefile | 2 +- tests/integration/test_dbutils.py | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index eb8fe8397..8d388eb8a 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,7 @@ test: pytest -m 'not integration and not benchmark' --cov=databricks --cov-report html tests integration: - pytest -n auto -m 'integration and not benchmark' --reruns 2 --dist loadgroup --cov=databricks --cov-report html tests + pytest -n auto -m 'integration and not benchmark' --reruns 2 --dist loadgroup --cov=databricks --cov-report html tests/integration/test_dbutils.py::test_put_local_path benchmark: pytest -m 'benchmark' tests diff --git a/tests/integration/test_dbutils.py b/tests/integration/test_dbutils.py index 667899df0..bc6dc764a 100644 --- a/tests/integration/test_dbutils.py +++ b/tests/integration/test_dbutils.py @@ -8,11 +8,13 @@ from databricks.sdk.errors import NotFound -def test_put_local_path(w, random, tmp_path): +def test_put_local_path(fs_and_base_path, random): + fs, base_path = fs_and_base_path to_write = random(1024 * 1024 * 2) - tmp_path = tmp_path / "tmp_file" - w.dbutils.fs.put(f'file:{tmp_path}', to_write, True) - assert w.dbutils.fs.head(f'file:{tmp_path}', 1024*1024*2) == to_write + base_path = base_path / "tmp_file" + fs.put(base_path, to_write, True) + assert fs.head(base_path, 1024 * 1024 * 2) == to_write + def test_rest_dbfs_ls(w, env_or_skip): From 58b488c6eab82cdedb242c6288557a4ed69cc913 Mon Sep 17 00:00:00 2001 From: Parth Bansal Date: Mon, 8 Jul 2024 10:54:16 +0200 Subject: [PATCH 8/9] update --- tests/integration/test_dbutils.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_dbutils.py b/tests/integration/test_dbutils.py index bc6dc764a..29f5cc387 100644 --- a/tests/integration/test_dbutils.py +++ b/tests/integration/test_dbutils.py @@ -3,17 +3,21 @@ import os import pytest - +import platform from databricks.sdk.core import DatabricksError from databricks.sdk.errors import NotFound def test_put_local_path(fs_and_base_path, random): fs, base_path = fs_and_base_path + if platform.system() == 'Windows': + dummy_file = f'C:\\Windows\\Temp\\{random()}' + else: + dummy_file = f'/tmp/{random()}' to_write = random(1024 * 1024 * 2) - base_path = base_path / "tmp_file" - fs.put(base_path, to_write, True) - assert fs.head(base_path, 1024 * 1024 * 2) == to_write + # tmp_path = tmp_path / "tmp_file" + fs.put(dummy_file, to_write, True) + assert fs.head(dummy_file, 1024 * 1024 * 2) == to_write From e27a2df88158f10813e873df087b8d3bfe4b2677 Mon Sep 17 00:00:00 2001 From: Parth Bansal Date: Mon, 8 Jul 2024 10:59:06 +0200 Subject: [PATCH 9/9] update --- tests/integration/test_dbutils.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/tests/integration/test_dbutils.py b/tests/integration/test_dbutils.py index 29f5cc387..8ec60d689 100644 --- a/tests/integration/test_dbutils.py +++ b/tests/integration/test_dbutils.py @@ -8,17 +8,11 @@ from databricks.sdk.errors import NotFound -def test_put_local_path(fs_and_base_path, random): - fs, base_path = fs_and_base_path - if platform.system() == 'Windows': - dummy_file = f'C:\\Windows\\Temp\\{random()}' - else: - dummy_file = f'/tmp/{random()}' +def test_put_local_path(w, random, tmp_path): to_write = random(1024 * 1024 * 2) - # tmp_path = tmp_path / "tmp_file" - fs.put(dummy_file, to_write, True) - assert fs.head(dummy_file, 1024 * 1024 * 2) == to_write - + tmp_path = tmp_path / "tmp_file" + w.dbutils.fs.put(tmp_path, to_write, True) + assert w.dbutils.fs.head(tmp_path, 1024*1024*2) == to_write def test_rest_dbfs_ls(w, env_or_skip):