diff --git a/Dockerfile b/Dockerfile index 190ad6ef..d9d1b93e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,11 +4,15 @@ WORKDIR /usr/src/databricks-cli COPY . . -RUN pip install --upgrade pip && \ - pip install -r dev-requirements.txt && \ +RUN pip install --upgrade --no-cache-dir pip && \ + pip install --no-cache-dir \ + -r dev-requirements.txt \ + -r tox-requirements.txt && \ pip list && \ ./lint.sh && \ - pip install . && \ - pytest tests + pip install --no-cache-dir . && \ + pytest tests && \ + pip uninstall -y \ + -r tox-requirements.txt ENTRYPOINT [ "databricks" ] diff --git a/databricks_cli/workspace/api.py b/databricks_cli/workspace/api.py index a9889d2b..a4dcbdbb 100644 --- a/databricks_cli/workspace/api.py +++ b/databricks_cli/workspace/api.py @@ -152,7 +152,7 @@ def import_workspace_dir(self, source_path, target_path, overwrite, exclude_hidd click.echo(('{} does not have a valid extension of {}. Skip this file and ' + 'continue.').format(cur_src, extensions)) - def export_workspace_dir(self, source_path, target_path, overwrite): + def export_workspace_dir(self, source_path, target_path, format=WorkspaceFormat.SOURCE, overwrite): if os.path.isfile(target_path): click.echo('{} exists as a file. Skipping this subtree {}' .format(target_path, source_path)) @@ -167,7 +167,7 @@ def export_workspace_dir(self, source_path, target_path, overwrite): elif obj.is_notebook: cur_dst = cur_dst + WorkspaceLanguage.to_extension(obj.language) try: - self.export_workspace(cur_src, cur_dst, WorkspaceFormat.SOURCE, overwrite) + self.export_workspace(cur_src, cur_dst, format, overwrite) click.echo('{} -> {}'.format(cur_src, cur_dst)) except LocalFileExistsException: click.echo('{} already exists locally as {}. Skip.'.format(cur_src, cur_dst)) diff --git a/databricks_cli/workspace/cli.py b/databricks_cli/workspace/cli.py index 19997e9a..ceb5925b 100644 --- a/databricks_cli/workspace/cli.py +++ b/databricks_cli/workspace/cli.py @@ -147,22 +147,24 @@ def delete_cli(api_client, workspace_path, recursive): @click.argument('source_path') @click.argument('target_path') @click.option('--overwrite', '-o', is_flag=True, default=False) +@click.option('--format', '-f', default=WorkspaceFormat.SOURCE, type=FormatClickType()) @debug_option @profile_option @eat_exceptions @provide_api_client -def export_dir_cli(api_client, source_path, target_path, overwrite): +def export_dir_cli(api_client, source_path, target_path, format, overwrite): """ Recursively exports a directory from the Databricks workspace. Only directories and notebooks are exported. Notebooks are always exported in the SOURCE format. Notebooks will also have the extension of .scala, .py, .sql, or .r appended depending on the language type. + ADDED format -- jeffreybreen """ workspace_api = WorkspaceApi(api_client) assert workspace_api.get_status(source_path).is_dir, 'The source path must be a directory. {}' \ .format(source_path) - workspace_api.export_workspace_dir(source_path, target_path, overwrite) + workspace_api.export_workspace_dir(source_path, target_path, format, overwrite) @click.command(context_settings=CONTEXT_SETTINGS, diff --git a/dev-requirements.txt b/dev-requirements.txt index 8ef288ec..dc009910 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1 +1,3 @@ -tox==2.9.1 +tox==3.2.1 +click==6.7 +tabulate==0.8.2 diff --git a/setup.py b/setup.py index 5c338cb1..e01bdc8d 100644 --- a/setup.py +++ b/setup.py @@ -56,6 +56,7 @@ 'Intended Audience :: System Administrators', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3.6', + 'License :: OSI Approved :: Apache Software License' ], keywords='databricks cli', url='https://github.com/databricks/databricks-cli' diff --git a/tox-requirements.txt b/tox-requirements.txt index 5d37cc4c..00569934 100644 --- a/tox-requirements.txt +++ b/tox-requirements.txt @@ -1,10 +1,10 @@ # Test reqs -prospector[with_pyroma]==0.12.7 -pylint==1.8.2 -pep8-naming==0.5.0 -pytest==3.2.1 +prospector[with_pyroma]==1.1.1 +pylint==1.9.3 +pep8-naming==0.7.0 +pytest==3.7.1 mock==2.0.0 -decorator==4.2.1 -rstcheck==3.2 +decorator==4.3.0 +rstcheck==3.3 pytest-cov codecov