diff --git a/README.md b/README.md index 2ca09a5..5b0e3c5 100644 --- a/README.md +++ b/README.md @@ -225,3 +225,8 @@ Detecting drift concept and get analyses and explainability of this drift. An is Adapting Eurybia for models consumed in API mode. An issue is open: [Adapt Eurybia to API mode](https://github.com/MAIF/eurybia/issues/9) If you want to contribute, you can contact us in the [discussion tab](https://github.com/MAIF/eurybia/discussions) + + +## Note + +Eurybia uses [datapane](https://github.com/datapane/datapane) to generate its reports. The support of datapane being dropped since 2023, the package has been embedded as a module of Eurybia. diff --git a/eurybia/core/smartplotter.py b/eurybia/core/smartplotter.py index aba87d7..67d16b1 100644 --- a/eurybia/core/smartplotter.py +++ b/eurybia/core/smartplotter.py @@ -46,7 +46,9 @@ class SmartPlotter: def __init__(self, smartdrift): self._palette_name = list(colors_loading().keys())[0] - self._style_dict = define_style(select_palette(colors_loading(), self._palette_name)) + self._style_dict = define_style( + select_palette(colors_loading(), self._palette_name) + ) self.smartdrift = smartdrift def generate_fig_univariate( @@ -86,15 +88,24 @@ def generate_fig_univariate( hue = self.smartdrift._datadrift_target if df_all is None: df_all = self.smartdrift._df_concat - df_all.loc[df_all[hue] == 0, hue] = list(self.smartdrift.dataset_names.keys())[1] - df_all.loc[df_all[hue] == 1, hue] = list(self.smartdrift.dataset_names.keys())[0] + df_all.loc[df_all[hue] == 0, hue] = list( + self.smartdrift.dataset_names.keys() + )[1] + df_all.loc[df_all[hue] == 1, hue] = list( + self.smartdrift.dataset_names.keys() + )[0] if dict_color_palette is None: dict_color_palette = self._style_dict col_types = compute_col_types(df_all=df_all) + if col_types[col] == VarType.TYPE_NUM: - fig = self.generate_fig_univariate_continuous(df_all, col, hue=hue, dict_color_palette=dict_color_palette) + fig = self.generate_fig_univariate_continuous( + df_all, col, hue=hue, dict_color_palette=dict_color_palette + ) elif col_types[col] == VarType.TYPE_CAT: - fig = self.generate_fig_univariate_categorical(df_all, col, hue=hue, dict_color_palette=dict_color_palette) + fig = self.generate_fig_univariate_categorical( + df_all, col, hue=hue, dict_color_palette=dict_color_palette + ) else: raise NotImplementedError("Series dtype not supported") return fig @@ -114,7 +125,6 @@ def generate_fig_univariate_continuous( width: Optional[str] = None, hovermode: Optional[str] = None, ) -> plotly.graph_objs._figure.Figure: - """ Returns a plotly figure containing the distribution of a continuous feature. @@ -147,7 +157,10 @@ def generate_fig_univariate_continuous( plotly.graph_objs._figure.Figure """ df_all.loc[:, col].fillna(0, inplace=True) - datasets = [df_all[df_all[hue] == val][col].values.tolist() for val in df_all[hue].unique()] + datasets = [ + df_all[df_all[hue] == val][col].values.tolist() + for val in df_all[hue].unique() + ] fig = ff.create_distplot( datasets, @@ -249,20 +262,33 @@ def generate_fig_univariate_categorical( ------- plotly.graph_objs._figure.Figure """ - df_cat = df_all.groupby([col, hue]).agg({col: "count"}).rename(columns={col: "count"}).reset_index() - df_cat["Percent"] = df_cat["count"] * 100 / df_cat.groupby(hue)["count"].transform("sum") + df_cat = ( + df_all.groupby([col, hue]) + .agg({col: "count"}) + .rename(columns={col: "count"}) + .reset_index() + ) + df_cat["Percent"] = ( + df_cat["count"] * 100 / df_cat.groupby(hue)["count"].transform("sum") + ) if pd.api.types.is_numeric_dtype(df_cat[col].dtype): df_cat = df_cat.sort_values(col, ascending=True) df_cat[col] = df_cat[col].astype(str) - nb_cat = df_cat.groupby([col]).agg({"count": "sum"}).reset_index()[col].nunique() + nb_cat = ( + df_cat.groupby([col]).agg({"count": "sum"}).reset_index()[col].nunique() + ) if nb_cat > nb_cat_max: - df_cat = self._merge_small_categories(df_cat=df_cat, col=col, hue=hue, nb_cat_max=nb_cat_max) + df_cat = self._merge_small_categories( + df_cat=df_cat, col=col, hue=hue, nb_cat_max=nb_cat_max + ) df_to_sort = df_cat.copy().reset_index(drop=True) - df_to_sort["Sorted_indicator"] = df_to_sort.sort_values([col]).groupby([col])["Percent"].diff() + df_to_sort["Sorted_indicator"] = ( + df_to_sort.sort_values([col]).groupby([col])["Percent"].diff() + ) df_to_sort["Sorted_indicator"] = np.abs(df_to_sort["Sorted_indicator"]) df_sorted = df_to_sort.dropna()[[col, "Sorted_indicator"]] @@ -272,7 +298,9 @@ def generate_fig_univariate_categorical( .drop("Sorted_indicator", axis=1) ) - df_cat["Percent_displayed"] = df_cat["Percent"].apply(lambda row: str(round(row, 2)) + " %") + df_cat["Percent_displayed"] = df_cat["Percent"].apply( + lambda row: str(round(row, 2)) + " %" + ) modalities = df_cat[hue].unique().tolist() @@ -285,7 +313,10 @@ def generate_fig_univariate_categorical( color=hue, text="Percent_displayed", ) - fig1.update_traces(marker_color=list(self._style_dict["univariate_cat_bar"].values())[0], showlegend=True) + fig1.update_traces( + marker_color=list(self._style_dict["univariate_cat_bar"].values())[0], + showlegend=True, + ) fig2 = px.bar( df_cat[df_cat[hue] == modalities[1]], @@ -296,7 +327,10 @@ def generate_fig_univariate_categorical( color=hue, text="Percent_displayed", ) - fig2.update_traces(marker_color=list(self._style_dict["univariate_cat_bar"].values())[1], showlegend=True) + fig2.update_traces( + marker_color=list(self._style_dict["univariate_cat_bar"].values())[1], + showlegend=True, + ) fig = fig1.add_trace(fig2.data[0]) @@ -336,21 +370,31 @@ def generate_fig_univariate_categorical( return fig - def _merge_small_categories(self, df_cat: pd.DataFrame, col: str, hue: str, nb_cat_max: int) -> pd.DataFrame: + def _merge_small_categories( + self, df_cat: pd.DataFrame, col: str, hue: str, nb_cat_max: int + ) -> pd.DataFrame: """ Merges categories of column 'col' of df_cat into 'Other' category so that the number of categories is less than nb_cat_max. """ df_cat_sum_hue = df_cat.groupby([col]).agg({"count": "sum"}).reset_index() - list_cat_to_merge = df_cat_sum_hue.sort_values("count", ascending=False)[col].to_list()[nb_cat_max - 1 :] + list_cat_to_merge = df_cat_sum_hue.sort_values("count", ascending=False)[ + col + ].to_list()[nb_cat_max - 1 :] df_cat_other = ( - df_cat.loc[df_cat[col].isin(list_cat_to_merge)].groupby(hue, as_index=False)[["count", "Percent"]].sum() + df_cat.loc[df_cat[col].isin(list_cat_to_merge)] + .groupby(hue, as_index=False)[["count", "Percent"]] + .sum() ) df_cat_other[col] = "Other" - return pd.concat([df_cat.loc[~df_cat[col].isin(list_cat_to_merge)], df_cat_other]) + return pd.concat( + [df_cat.loc[~df_cat[col].isin(list_cat_to_merge)], df_cat_other] + ) def scatter_feature_importance( - self, feature_importance: pd.DataFrame = None, datadrift_stat_test: pd.DataFrame = None + self, + feature_importance: pd.DataFrame = None, + datadrift_stat_test: pd.DataFrame = None, ) -> plotly.graph_objs._figure.Figure: """ Displays scatter of feature importance between drift @@ -392,7 +436,16 @@ def scatter_feature_importance( + f"Datadrift test: {t} - pvalue: {pv:.5f}
" + f"Datadrift model Importance: {ddrimp*100:.1f}" for feat, depimp, t, pv, ddrimp in zip( - *map(data.get, ["features", "deployed_model", "testname", "pvalue", "datadrift_classifier"]) + *map( + data.get, + [ + "features", + "deployed_model", + "testname", + "pvalue", + "datadrift_classifier", + ], + ) ) ] @@ -401,7 +454,9 @@ def scatter_feature_importance( go.Scatter( x=data["datadrift_classifier"], y=data["deployed_model"], - marker_symbol=datadrift_stat_test["testname"].apply(lambda x: symbol_dict[x]), + marker_symbol=datadrift_stat_test["testname"].apply( + lambda x: symbol_dict[x] + ), mode="markers", showlegend=False, hovertext=hv_text, @@ -409,12 +464,20 @@ def scatter_feature_importance( ) ) - fig.update_traces(marker={"size": 15, "opacity": 0.8, "line": {"width": 0.8, "color": "white"}}) + fig.update_traces( + marker={ + "size": 15, + "opacity": 0.8, + "line": {"width": 0.8, "color": "white"}, + } + ) fig.data[0].marker.color = data["pvalue"] fig.data[0].marker.coloraxis = "coloraxis" fig.layout.coloraxis.colorscale = self._style_dict["featimportance_colorscale"] - fig.layout.coloraxis.colorbar = {"title": {"text": "Univariate
DataDrift Test
Pvalue"}} + fig.layout.coloraxis.colorbar = { + "title": {"text": "Univariate
DataDrift Test
Pvalue"} + } height = self._style_dict["height"] width = self._style_dict["width"] @@ -476,24 +539,31 @@ def generate_historical_datadrift_metric( datadrift_historical = self.smartdrift.historical_auc if datadrift_historical is not None: if self.smartdrift.deployed_model is not None: - datadrift_historical = datadrift_historical[["date", "auc", "JS_predict"]] + datadrift_historical = datadrift_historical[ + ["date", "auc", "JS_predict"] + ] datadrift_historical = ( - datadrift_historical.groupby(["date"])[["auc", "JS_predict"]].mean().reset_index() + datadrift_historical.groupby(["date"])[["auc", "JS_predict"]] + .mean() + .reset_index() ) datadrift_historical.sort_values(by="date", inplace=True) else: datadrift_historical = datadrift_historical[["date", "auc"]] - datadrift_historical = datadrift_historical.groupby("date")["auc"].mean().reset_index() + datadrift_historical = ( + datadrift_historical.groupby("date")["auc"].mean().reset_index() + ) datadrift_historical.sort_values(by="date", inplace=True) datadrift_historical["auc_displayed"] = datadrift_historical["auc"].round(2) if self.smartdrift.deployed_model is not None: - fig = make_subplots(specs=[[{"secondary_y": True}]]) fig.add_trace( go.Scatter( - x=datadrift_historical["date"], y=datadrift_historical["auc"], name="Datadrift classifier AUC" + x=datadrift_historical["date"], + y=datadrift_historical["auc"], + name="Datadrift classifier AUC", ), secondary_y=False, ) @@ -508,8 +578,13 @@ def generate_historical_datadrift_metric( ) fig.update_layout(title_text="Evolution of data drift") - fig.update_yaxes(title_text="Datadrift classifier AUC ", secondary_y=False) - fig.update_yaxes(title_text="Jensen_Shannon Prediction Divergence ", secondary_y=True) + fig.update_yaxes( + title_text="Datadrift classifier AUC ", secondary_y=False + ) + fig.update_yaxes( + title_text="Jensen_Shannon Prediction Divergence ", + secondary_y=True, + ) fig.update_yaxes(range=[0.5, 1], secondary_y=False) fig.update_yaxes(range=[0, 0.3], secondary_y=True) else: @@ -600,7 +675,9 @@ def generate_modeldrift_data( For more information see the documentation""" ) data_modeldrift[metric] = data_modeldrift[metric].apply( - lambda row: round(row, len([char for char in str(row).split(".")[1] if char == "0"]) + 3) + lambda row: round( + row, len([char for char in str(row).split(".")[1] if char == "0"]) + 3 + ) ) fig = px.line( @@ -688,7 +765,12 @@ def generate_indicator( color = sns.blend_palette(["green", "yellow", "orange", "red"], 100) color = color.as_hex() list_color_glob = list() - threshold = [i for i in np.arange(min_gauge, max_gauge, (max_gauge - min_gauge) / len(color))] + threshold = [ + i + for i in np.arange( + min_gauge, max_gauge, (max_gauge - min_gauge) / len(color) + ) + ] for i in range(1, len(threshold) + 1): dict_color = dict() if i == len(threshold): @@ -705,7 +787,11 @@ def generate_indicator( domain={"x": [0, 1], "y": [0, 1]}, title={"text": title, "align": "center", "font": {"size": 20}}, gauge={ - "axis": {"range": [min_gauge, max_gauge], "ticktext": ["No Drift", "High Drift"], "tickwidth": 1}, + "axis": { + "range": [min_gauge, max_gauge], + "ticktext": ["No Drift", "High Drift"], + "tickwidth": 1, + }, "bar": {"color": "black"}, "borderwidth": 0, "steps": list_color_glob, diff --git a/eurybia/report/common.py b/eurybia/report/common.py index 460c61d..d4f44a9 100644 --- a/eurybia/report/common.py +++ b/eurybia/report/common.py @@ -1,13 +1,17 @@ """ Common functions used in report """ + import os from enum import Enum from numbers import Number from typing import Callable, Dict, Optional, Union import pandas as pd -from pandas.api.types import is_bool_dtype, is_categorical_dtype, is_numeric_dtype, is_string_dtype +from pandas.api.types import ( + is_numeric_dtype, + infer_dtype, +) class VarType(Enum): @@ -23,7 +27,9 @@ def __str__(self): return str(self.value) -def display_value(value: float, thousands_separator: str = ",", decimal_separator: str = ".") -> str: +def display_value( + value: float, thousands_separator: str = ",", decimal_separator: str = "." +) -> str: """ Display a value as a string with specific format. Parameters @@ -43,7 +49,9 @@ def display_value(value: float, thousands_separator: str = ",", decimal_separato '1,255,000' """ value_str = f"{value:,}".replace(",", "/thousands/").replace(".", "/decimal/") - return value_str.replace("/thousands/", thousands_separator).replace("/decimal/", decimal_separator) + return value_str.replace("/thousands/", thousands_separator).replace( + "/decimal/", decimal_separator + ) def replace_dict_values(obj: Dict, replace_fn: Callable, *args) -> dict: @@ -76,11 +84,11 @@ def series_dtype(s: pd.Series) -> VarType: ------- VarType """ - if is_bool_dtype(s): + if infer_dtype(s) == "boolean": return VarType.TYPE_CAT - elif is_string_dtype(s): + elif infer_dtype(s, skipna=True) == "string": return VarType.TYPE_CAT - elif is_categorical_dtype(s): + elif isinstance(s.dtype, pd.CategoricalDtype): return VarType.TYPE_CAT elif is_numeric_dtype(s): if numeric_is_continuous(s): @@ -139,7 +147,9 @@ def get_callable(path: str): try: import_module(mod) except Exception as e: - raise ImportError(f"Encountered error: `{e}` when loading module '{path}'") from e + raise ImportError( + f"Encountered error: `{e}` when loading module '{path}'" + ) from e obj = getattr(obj, part) if isinstance(obj, type): obj_type: type = obj diff --git a/eurybia/report/datapane/__init__.py b/eurybia/report/datapane/__init__.py new file mode 100644 index 0000000..f435acb --- /dev/null +++ b/eurybia/report/datapane/__init__.py @@ -0,0 +1,127 @@ +# Copyright 2020 StackHut Limited (trading as Datapane) +# SPDX-License-Identifier: Apache-2.0 +import sys +from pathlib import Path + +try: + from . import _version +except ImportError: + # NOTE - could use subprocess to get from git? + __rev__ = "local" + __is_dev_build__ = True +else: + __rev__ = _version.__rev__ + __is_dev_build__ = getattr(_version, "__is_dev_build__", False) + del _version + +__version__ = "0.17.0" + + +# Public API re-exports +from .client import ( # isort:skip otherwise circular import issue + IN_PYTEST, + DPClientError, + DPMode, + enable_logging, + print_debug_info, + get_dp_mode, + set_dp_mode, +) # isort:skip otherwise circular import issue + +from .blocks import ( + HTML, + Attachment, + BigNumber, + Block, + Code, + DataTable, + Embed, + Empty, + Formula, + Group, + Media, + Page, + Plot, + Select, + SelectType, + Table, + Text, + Toggle, + VAlign, + wrap_block, +) +from .processors import ( + FontChoice, + Formatting, + TextAlignment, + Width, + build_report, + save_report, + stringify_report, + upload_report, +) +from .view import App, Blocks, Report, View + +# Other useful re-exports +# ruff: noqa: I001 +from . import builtins # isort:skip otherwise circular import issue + +X = wrap_block + +__all__ = [ + "App", + "Report", + "DPClientError", + "builtins", + "enable_logging", + "print_debug_info", + "Block", + "Attachment", + "BigNumber", + "Empty", + "DataTable", + "Media", + "Plot", + "Table", + "Select", + "SelectType", + "Formula", + "HTML", + "Code", + "Embed", + "Group", + "Text", + "Toggle", + "VAlign", + "Blocks", + "upload_report", + "save_report", + "build_report", + "stringify_report", + "X", + "Page", + "View", + "Width", + "FontChoice", + "Formatting", + "TextAlignment", +] + + +script_name = sys.argv[0] +script_exe = Path(script_name).stem +by_datapane = False # hardcode for now as not using legacy runner +if script_exe == "datapane" or script_name == "-m": # or "pytest" in script_name: + # argv[0] will be "-m" as client module as submodule of this module + set_dp_mode(DPMode.SCRIPT) +elif by_datapane or script_exe == "dp-runner": + set_dp_mode(DPMode.FRAMEWORK) +else: + set_dp_mode(DPMode.LIBRARY) + +# TODO - do we want to init only in jupyter / interactive / etc. +# only init fully in library-mode, as framework and app init explicitly +if get_dp_mode() == DPMode.LIBRARY and not IN_PYTEST: + from .client.config import init + + init() diff --git a/eurybia/report/datapane/_vendor/base64io/LICENSE b/eurybia/report/datapane/_vendor/base64io/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/eurybia/report/datapane/_vendor/base64io/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/eurybia/report/datapane/_vendor/base64io/__init__.py b/eurybia/report/datapane/_vendor/base64io/__init__.py new file mode 100644 index 0000000..d5f4a99 --- /dev/null +++ b/eurybia/report/datapane/_vendor/base64io/__init__.py @@ -0,0 +1,378 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Base64 stream with context manager support.""" +from __future__ import division + +import base64 +import io +import logging +import string +import sys + +LOGGER_NAME = "base64io" + +try: # Python 3.5.0 and 3.5.1 have incompatible typing modules + from types import TracebackType # noqa pylint: disable=unused-import + from typing import ( # type: ignore[attr-defined] # noqa pylint: disable=unused-import + IO, + AnyStr, + Iterable, + List, + Literal, + Optional, + Type, + Union, + ) +except ImportError: # pragma: no cover + # We only actually need these imports when running the mypy checks + pass + +__all__ = ("Base64IO",) +__version__ = "1.0.3" +_LOGGER = logging.getLogger(LOGGER_NAME) + + +def _py2(): + # type: () -> bool + """Determine if runtime is Python 2. + + :returns: decision: + :rtype: bool + """ + return sys.version_info[0] == 2 + + +if not _py2(): + # The "file" object does not exist in Python 3, but we need to reference + # it in Python 2 code paths. Defining this here accomplishes two things: + # First, it allows linters to accept "file" as a defined object in Python 3. + # Second, it will serve as a canary to ensure that there are no references + # to "file" in Python 3 code paths. + file = NotImplemented # pylint: disable=invalid-name + + +def _to_bytes(data): + # type: (AnyStr) -> bytes + """Convert input data from either string or bytes to bytes. + + :param data: Data to convert + :returns: ``data`` converted to bytes + :rtype: bytes + """ + if isinstance(data, bytes): + return data + return data.encode("utf-8") + + +class Base64IO(io.IOBase): + """Base64 stream with context manager support. + + Wraps a stream, base64-decoding read results before returning them and base64-encoding + written bytes before writing them to the stream. Instances + of this class are not reusable in order maintain consistency with the :class:`io.IOBase` + behavior on ``close()``. + + .. note:: + + Provides iterator and context manager interfaces. + + .. warning:: + + Because up to two bytes of data must be buffered to ensure correct base64 encoding + of all data written, this object **must** be closed after you are done writing to + avoid data loss. If used as a context manager, we take care of that for you. + + :param wrapped: Stream to wrap + """ + + closed = False + + def __init__(self, wrapped): + # type: (Base64IO, IO) -> None + """Check for required methods on wrapped stream and set up read buffer. + + :raises TypeError: if ``wrapped`` does not have attributes needed to determine the stream's state + """ + required_attrs = ("read", "write", "close", "closed", "flush") + if not all(hasattr(wrapped, attr) for attr in required_attrs): + raise TypeError("Base64IO wrapped object must have attributes: %s" % (repr(sorted(required_attrs)),)) + super(Base64IO, self).__init__() + self.__wrapped = wrapped + self.__read_buffer = b"" + self.__write_buffer = b"" + + def __enter__(self): + # type: () -> Base64IO + """Return self on enter.""" + return self + + def __exit__(self, exc_type, exc_value, traceback): + # type: (Optional[Type[BaseException]], Optional[BaseException], Optional[TracebackType]) -> Literal[False] + """Properly close self on exit.""" + self.close() + return False + + def close(self): + # type: () -> None + """Close this stream, encoding and writing any buffered bytes is present. + + .. note:: + + This does **not** close the wrapped stream. + """ + if self.__write_buffer: + self.__wrapped.write(base64.b64encode(self.__write_buffer)) + self.__write_buffer = b"" + self.closed = True + + def _passthrough_interactive_check(self, method_name, mode): + # type: (str, str) -> bool + """Attempt to call the specified method on the wrapped stream and return the result. + + If the method is not found on the wrapped stream, return False. + + .. note:: + + Special Case: If wrapped stream is a Python 2 file, inspect the file mode. + + :param str method_name: Name of method to call + :param str mode: Python 2 mode character + :rtype: bool + """ + try: + method = getattr(self.__wrapped, method_name) + except AttributeError: + if ( + _py2() + and isinstance(self.__wrapped, file) # pylint: disable=isinstance-second-argument-not-valid-type + and mode in self.__wrapped.mode + ): + return True + return False + else: + return method() + + def writable(self): + # type: () -> bool + """Determine if the stream can be written to. + + Delegates to wrapped stream when possible. + Otherwise returns False. + + :rtype: bool + """ + return self._passthrough_interactive_check("writable", "w") + + def readable(self): + # type: () -> bool + """Determine if the stream can be read from. + + Delegates to wrapped stream when possible. + Otherwise returns False. + + :rtype: bool + """ + return self._passthrough_interactive_check("readable", "r") + + def flush(self): + # type: () -> None + """Flush the write buffer of the wrapped stream.""" + return self.__wrapped.flush() + + def write(self, b): + # type: (bytes) -> int + """Base64-encode the bytes and write them to the wrapped stream. + + Any bytes that would require padding for the next write call are buffered until the + next write or close. + + .. warning:: + + Because up to two bytes of data must be buffered to ensure correct base64 encoding + of all data written, this object **must** be closed after you are done writing to + avoid data loss. If used as a context manager, we take care of that for you. + + :param bytes b: Bytes to write to wrapped stream + :raises ValueError: if called on closed Base64IO object + :raises IOError: if underlying stream is not writable + """ + if self.closed: + raise ValueError("I/O operation on closed file.") + + if not self.writable(): + raise IOError("Stream is not writable") + + # Load any stashed bytes and clear the buffer + _bytes_to_write = self.__write_buffer + b + self.__write_buffer = b"" + + # If an even base64 chunk or finalizing the stream, write through. + if len(_bytes_to_write) % 3 == 0: + return self.__wrapped.write(base64.b64encode(_bytes_to_write)) + + # We're not finalizing the stream, so stash the trailing bytes and encode the rest. + trailing_byte_pos = -1 * (len(_bytes_to_write) % 3) + self.__write_buffer = _bytes_to_write[trailing_byte_pos:] + return self.__wrapped.write(base64.b64encode(_bytes_to_write[:trailing_byte_pos])) + + def writelines(self, lines): + # type: (Iterable[bytes]) -> None + """Write a list of lines. + + :param list lines: Lines to write + """ + for line in lines: + self.write(line) + + def _read_additional_data_removing_whitespace(self, data, total_bytes_to_read): + # type: (bytes, int) -> bytes + """Read additional data from wrapped stream until we reach the desired number of bytes. + + .. note:: + + All whitespace is ignored. + + :param bytes data: Data that has already been read from wrapped stream + :param int total_bytes_to_read: Number of total non-whitespace bytes to read from wrapped stream + :returns: ``total_bytes_to_read`` bytes from wrapped stream with no whitespace + :rtype: bytes + """ + if total_bytes_to_read is None: + # If the requested number of bytes is None, we read the entire message, in which + # case the base64 module happily removes any whitespace. + return data + + _data_buffer = io.BytesIO() + + _data_buffer.write(b"".join(data.split())) + _remaining_bytes_to_read = total_bytes_to_read - _data_buffer.tell() + + while _remaining_bytes_to_read > 0: + _raw_additional_data = _to_bytes(self.__wrapped.read(_remaining_bytes_to_read)) + if not _raw_additional_data: + # No more data to read from wrapped stream. + break + + _data_buffer.write(b"".join(_raw_additional_data.split())) + _remaining_bytes_to_read = total_bytes_to_read - _data_buffer.tell() + return _data_buffer.getvalue() + + def read(self, b=-1): + # type: (int) -> bytes + """Read bytes from wrapped stream, base64-decoding before return. + + .. note:: + + The number of bytes requested from the wrapped stream is adjusted to return the + requested number of bytes after decoding returned bytes. + + :param int b: Number of bytes to read + :returns: Decoded bytes from wrapped stream + :rtype: bytes + """ + if self.closed: + raise ValueError("I/O operation on closed file.") + + if not self.readable(): + raise IOError("Stream is not readable") + + if b is None or b < 0: + b = -1 + _bytes_to_read = -1 + elif b == 0: + _bytes_to_read = 0 + elif b > 0: + # Calculate number of encoded bytes that must be read to get b raw bytes. + _bytes_to_read = int((b - len(self.__read_buffer)) * 4 / 3) + _bytes_to_read += 4 - _bytes_to_read % 4 + + # Read encoded bytes from wrapped stream. + data = _to_bytes(self.__wrapped.read(_bytes_to_read)) + # Remove whitespace from read data and attempt to read more data to get the desired + # number of bytes. + + if any(char in data for char in string.whitespace.encode("utf-8")): + data = self._read_additional_data_removing_whitespace(data, _bytes_to_read) + + results = io.BytesIO() + # First, load any stashed bytes + results.write(self.__read_buffer) + # Decode encoded bytes. + results.write(base64.b64decode(data)) + + results.seek(0) + output_data = results.read(b) + # Stash any extra bytes for the next run. + self.__read_buffer = results.read() + + return output_data + + def __iter__(self): + # Until https://github.com/python/typing/issues/11 + # there's no good way to tell mypy about custom + # iterators that subclass io.IOBase. + """Let this class act as an iterator.""" + return self + + def readline(self, limit=-1): + # type: (int) -> bytes + """Read and return one line from the stream. + + If limit is specified, at most limit bytes will be read. + + .. note:: + + Because the source that this reads from may not contain any OEL characters, we + read "lines" in chunks of length ``io.DEFAULT_BUFFER_SIZE``. + + :type limit: int + :rtype: bytes + """ + return self.read(limit if limit > 0 else io.DEFAULT_BUFFER_SIZE) + + def readlines(self, hint=-1): + # type: (int) -> List[bytes] + """Read and return a list of lines from the stream. + + ``hint`` can be specified to control the number of lines read: no more lines will + be read if the total size (in bytes/characters) of all lines so far exceeds hint. + + :type hint: int + :returns: Lines of data + :rtype: list of bytes + """ + lines = [] + total_len = 0 + hint_defined = hint > 0 + + for line in self: # type: ignore + lines.append(line) + total_len += len(line) + + hint_satisfied = total_len > hint + if hint_defined and hint_satisfied: + break + return lines + + def __next__(self): + # type: () -> bytes + """Python 3 iterator hook.""" + line = self.readline() + if line: + return line + raise StopIteration() + + def next(self): + # type: () -> bytes + """Python 2 iterator hook.""" + return self.__next__() diff --git a/eurybia/report/datapane/_vendor/bottle.LICENSE b/eurybia/report/datapane/_vendor/bottle.LICENSE new file mode 100644 index 0000000..a4ea15f --- /dev/null +++ b/eurybia/report/datapane/_vendor/bottle.LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2009-2022, Marcel Hellkamp. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/eurybia/report/datapane/_vendor/bottle.py b/eurybia/report/datapane/_vendor/bottle.py new file mode 100644 index 0000000..916e260 --- /dev/null +++ b/eurybia/report/datapane/_vendor/bottle.py @@ -0,0 +1,4417 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +Bottle is a fast and simple micro-framework for small web applications. It +offers request dispatching (Routes) with URL parameter support, templates, +a built-in HTTP Server and adapters for many third party WSGI/HTTP-server and +template engines - all in a single file and with no dependencies other than the +Python Standard Library. + +Homepage and documentation: http://bottlepy.org/ + +Copyright (c) 2009-2018, Marcel Hellkamp. +License: MIT (see LICENSE for details) +""" + +from __future__ import print_function +import sys + +__author__ = 'Marcel Hellkamp' +__version__ = '0.13-dev' +__license__ = 'MIT' + +############################################################################### +# Command-line interface ###################################################### +############################################################################### +# INFO: Some server adapters need to monkey-patch std-lib modules before they +# are imported. This is why some of the command-line handling is done here, but +# the actual call to _main() is at the end of the file. + + +def _cli_parse(args): # pragma: no coverage + from argparse import ArgumentParser + + parser = ArgumentParser(prog=args[0], usage="%(prog)s [options] package.module:app") + opt = parser.add_argument + opt("--version", action="store_true", help="show version number.") + opt("-b", "--bind", metavar="ADDRESS", help="bind socket to ADDRESS.") + opt("-s", "--server", default='wsgiref', help="use SERVER as backend.") + opt("-p", "--plugin", action="append", help="install additional plugin/s.") + opt("-c", "--conf", action="append", metavar="FILE", + help="load config values from FILE.") + opt("-C", "--param", action="append", metavar="NAME=VALUE", + help="override config values.") + opt("--debug", action="store_true", help="start server in debug mode.") + opt("--reload", action="store_true", help="auto-reload on file changes.") + opt('app', help='WSGI app entry point.', nargs='?') + + cli_args = parser.parse_args(args[1:]) + + return cli_args, parser + + +def _cli_patch(cli_args): # pragma: no coverage + parsed_args, _ = _cli_parse(cli_args) + opts = parsed_args + if opts.server: + if opts.server.startswith('gevent'): + import gevent.monkey + gevent.monkey.patch_all() + elif opts.server.startswith('eventlet'): + import eventlet + eventlet.monkey_patch() + + +if __name__ == '__main__': + _cli_patch(sys.argv) + +############################################################################### +# Imports and Python 2/3 unification ########################################## +############################################################################### + +import base64, calendar, cgi, email.utils, functools, hmac, itertools,\ + mimetypes, os, re, tempfile, threading, time, warnings, weakref, hashlib + +from types import FunctionType +from datetime import date as datedate, datetime, timedelta +from tempfile import NamedTemporaryFile +from traceback import format_exc, print_exc +from unicodedata import normalize + +try: + from ujson import dumps as json_dumps, loads as json_lds +except ImportError: + from json import dumps as json_dumps, loads as json_lds + +py = sys.version_info +py3k = py.major > 2 + +# Lots of stdlib and builtin differences. +if py3k: + import http.client as httplib + import _thread as thread + from urllib.parse import urljoin, SplitResult as UrlSplitResult + from urllib.parse import urlencode, quote as urlquote, unquote as urlunquote + urlunquote = functools.partial(urlunquote, encoding='latin1') + from http.cookies import SimpleCookie, Morsel, CookieError + from collections.abc import MutableMapping as DictMixin + from types import ModuleType as new_module + import pickle + from io import BytesIO + import configparser + # getfullargspec was deprecated in 3.5 and un-deprecated in 3.6 + # getargspec was deprecated in 3.0 and removed in 3.11 + from inspect import getfullargspec + def getargspec(func): + spec = getfullargspec(func) + kwargs = makelist(spec[0]) + makelist(spec.kwonlyargs) + return kwargs, spec[1], spec[2], spec[3] + + basestring = str + unicode = str + json_loads = lambda s: json_lds(touni(s)) + callable = lambda x: hasattr(x, '__call__') + imap = map + + def _raise(*a): + raise a[0](a[1]).with_traceback(a[2]) +else: # 2.x + import httplib + import thread + from urlparse import urljoin, SplitResult as UrlSplitResult + from urllib import urlencode, quote as urlquote, unquote as urlunquote + from Cookie import SimpleCookie, Morsel, CookieError + from itertools import imap + import cPickle as pickle + from imp import new_module + from StringIO import StringIO as BytesIO + import ConfigParser as configparser + from collections import MutableMapping as DictMixin + from inspect import getargspec + + unicode = unicode + json_loads = json_lds + exec(compile('def _raise(*a): raise a[0], a[1], a[2]', '', 'exec')) + +# Some helpers for string/byte handling +def tob(s, enc='utf8'): + if isinstance(s, unicode): + return s.encode(enc) + return b'' if s is None else bytes(s) + + +def touni(s, enc='utf8', err='strict'): + if isinstance(s, bytes): + return s.decode(enc, err) + return unicode("" if s is None else s) + + +tonat = touni if py3k else tob + + +def _stderr(*args): + try: + print(*args, file=sys.stderr) + except (IOError, AttributeError): + pass # Some environments do not allow printing (mod_wsgi) + + +# A bug in functools causes it to break if the wrapper is an instance method +def update_wrapper(wrapper, wrapped, *a, **ka): + try: + functools.update_wrapper(wrapper, wrapped, *a, **ka) + except AttributeError: + pass + +# These helpers are used at module level and need to be defined first. +# And yes, I know PEP-8, but sometimes a lower-case classname makes more sense. + + +def depr(major, minor, cause, fix): + text = "Warning: Use of deprecated feature or API. (Deprecated in Bottle-%d.%d)\n"\ + "Cause: %s\n"\ + "Fix: %s\n" % (major, minor, cause, fix) + if DEBUG == 'strict': + raise DeprecationWarning(text) + warnings.warn(text, DeprecationWarning, stacklevel=3) + return DeprecationWarning(text) + + +def makelist(data): # This is just too handy + if isinstance(data, (tuple, list, set, dict)): + return list(data) + elif data: + return [data] + else: + return [] + + +class DictProperty(object): + """ Property that maps to a key in a local dict-like attribute. """ + + def __init__(self, attr, key=None, read_only=False): + self.attr, self.key, self.read_only = attr, key, read_only + + def __call__(self, func): + functools.update_wrapper(self, func, updated=[]) + self.getter, self.key = func, self.key or func.__name__ + return self + + def __get__(self, obj, cls): + if obj is None: return self + key, storage = self.key, getattr(obj, self.attr) + if key not in storage: storage[key] = self.getter(obj) + return storage[key] + + def __set__(self, obj, value): + if self.read_only: raise AttributeError("Read-Only property.") + getattr(obj, self.attr)[self.key] = value + + def __delete__(self, obj): + if self.read_only: raise AttributeError("Read-Only property.") + del getattr(obj, self.attr)[self.key] + + +class cached_property(object): + """ A property that is only computed once per instance and then replaces + itself with an ordinary attribute. Deleting the attribute resets the + property. """ + + def __init__(self, func): + update_wrapper(self, func) + self.func = func + + def __get__(self, obj, cls): + if obj is None: return self + value = obj.__dict__[self.func.__name__] = self.func(obj) + return value + + +class lazy_attribute(object): + """ A property that caches itself to the class object. """ + + def __init__(self, func): + functools.update_wrapper(self, func, updated=[]) + self.getter = func + + def __get__(self, obj, cls): + value = self.getter(cls) + setattr(cls, self.__name__, value) + return value + + +############################################################################### +# Exceptions and Events ####################################################### +############################################################################### + + +class BottleException(Exception): + """ A base class for exceptions used by bottle. """ + pass + +############################################################################### +# Routing ###################################################################### +############################################################################### + + +class RouteError(BottleException): + """ This is a base class for all routing related exceptions """ + + +class RouteReset(BottleException): + """ If raised by a plugin or request handler, the route is reset and all + plugins are re-applied. """ + + +class RouterUnknownModeError(RouteError): + + pass + + +class RouteSyntaxError(RouteError): + """ The route parser found something not supported by this router. """ + + +class RouteBuildError(RouteError): + """ The route could not be built. """ + + +def _re_flatten(p): + """ Turn all capturing groups in a regular expression pattern into + non-capturing groups. """ + if '(' not in p: + return p + return re.sub(r'(\\*)(\(\?P<[^>]+>|\((?!\?))', lambda m: m.group(0) if + len(m.group(1)) % 2 else m.group(1) + '(?:', p) + + +class Router(object): + """ A Router is an ordered collection of route->target pairs. It is used to + efficiently match WSGI requests against a number of routes and return + the first target that satisfies the request. The target may be anything, + usually a string, ID or callable object. A route consists of a path-rule + and a HTTP method. + + The path-rule is either a static path (e.g. `/contact`) or a dynamic + path that contains wildcards (e.g. `/wiki/`). The wildcard syntax + and details on the matching order are described in docs:`routing`. + """ + + default_pattern = '[^/]+' + default_filter = 're' + + #: The current CPython regexp implementation does not allow more + #: than 99 matching groups per regular expression. + _MAX_GROUPS_PER_PATTERN = 99 + + def __init__(self, strict=False): + self.rules = [] # All rules in order + self._groups = {} # index of regexes to find them in dyna_routes + self.builder = {} # Data structure for the url builder + self.static = {} # Search structure for static routes + self.dyna_routes = {} + self.dyna_regexes = {} # Search structure for dynamic routes + #: If true, static routes are no longer checked first. + self.strict_order = strict + self.filters = { + 're': lambda conf: (_re_flatten(conf or self.default_pattern), + None, None), + 'int': lambda conf: (r'-?\d+', int, lambda x: str(int(x))), + 'float': lambda conf: (r'-?[\d.]+', float, lambda x: str(float(x))), + 'path': lambda conf: (r'.+?', None, None) + } + + def add_filter(self, name, func): + """ Add a filter. The provided function is called with the configuration + string as parameter and must return a (regexp, to_python, to_url) tuple. + The first element is a string, the last two are callables or None. """ + self.filters[name] = func + + rule_syntax = re.compile('(\\\\*)' + '(?:(?::([a-zA-Z_][a-zA-Z_0-9]*)?()(?:#(.*?)#)?)' + '|(?:<([a-zA-Z_][a-zA-Z_0-9]*)?(?::([a-zA-Z_]*)' + '(?::((?:\\\\.|[^\\\\>])+)?)?)?>))') + + def _itertokens(self, rule): + offset, prefix = 0, '' + for match in self.rule_syntax.finditer(rule): + prefix += rule[offset:match.start()] + g = match.groups() + if g[2] is not None: + depr(0, 13, "Use of old route syntax.", + "Use instead of :name in routes.") + if len(g[0]) % 2: # Escaped wildcard + prefix += match.group(0)[len(g[0]):] + offset = match.end() + continue + if prefix: + yield prefix, None, None + name, filtr, conf = g[4:7] if g[2] is None else g[1:4] + yield name, filtr or 'default', conf or None + offset, prefix = match.end(), '' + if offset <= len(rule) or prefix: + yield prefix + rule[offset:], None, None + + def add(self, rule, method, target, name=None): + """ Add a new rule or replace the target for an existing rule. """ + anons = 0 # Number of anonymous wildcards found + keys = [] # Names of keys + pattern = '' # Regular expression pattern with named groups + filters = [] # Lists of wildcard input filters + builder = [] # Data structure for the URL builder + is_static = True + + for key, mode, conf in self._itertokens(rule): + if mode: + is_static = False + if mode == 'default': mode = self.default_filter + mask, in_filter, out_filter = self.filters[mode](conf) + if not key: + pattern += '(?:%s)' % mask + key = 'anon%d' % anons + anons += 1 + else: + pattern += '(?P<%s>%s)' % (key, mask) + keys.append(key) + if in_filter: filters.append((key, in_filter)) + builder.append((key, out_filter or str)) + elif key: + pattern += re.escape(key) + builder.append((None, key)) + + self.builder[rule] = builder + if name: self.builder[name] = builder + + if is_static and not self.strict_order: + self.static.setdefault(method, {}) + self.static[method][self.build(rule)] = (target, None) + return + + try: + re_pattern = re.compile('^(%s)$' % pattern) + re_match = re_pattern.match + except re.error as e: + raise RouteSyntaxError("Could not add Route: %s (%s)" % (rule, e)) + + if filters: + + def getargs(path): + url_args = re_match(path).groupdict() + for name, wildcard_filter in filters: + try: + url_args[name] = wildcard_filter(url_args[name]) + except ValueError: + raise HTTPError(400, 'Path has wrong format.') + return url_args + elif re_pattern.groupindex: + + def getargs(path): + return re_match(path).groupdict() + else: + getargs = None + + flatpat = _re_flatten(pattern) + whole_rule = (rule, flatpat, target, getargs) + + if (flatpat, method) in self._groups: + if DEBUG: + msg = 'Route <%s %s> overwrites a previously defined route' + warnings.warn(msg % (method, rule), RuntimeWarning) + self.dyna_routes[method][ + self._groups[flatpat, method]] = whole_rule + else: + self.dyna_routes.setdefault(method, []).append(whole_rule) + self._groups[flatpat, method] = len(self.dyna_routes[method]) - 1 + + self._compile(method) + + def _compile(self, method): + all_rules = self.dyna_routes[method] + comborules = self.dyna_regexes[method] = [] + maxgroups = self._MAX_GROUPS_PER_PATTERN + for x in range(0, len(all_rules), maxgroups): + some = all_rules[x:x + maxgroups] + combined = (flatpat for (_, flatpat, _, _) in some) + combined = '|'.join('(^%s$)' % flatpat for flatpat in combined) + combined = re.compile(combined).match + rules = [(target, getargs) for (_, _, target, getargs) in some] + comborules.append((combined, rules)) + + def build(self, _name, *anons, **query): + """ Build an URL by filling the wildcards in a rule. """ + builder = self.builder.get(_name) + if not builder: + raise RouteBuildError("No route with that name.", _name) + try: + for i, value in enumerate(anons): + query['anon%d' % i] = value + url = ''.join([f(query.pop(n)) if n else f for (n, f) in builder]) + return url if not query else url + '?' + urlencode(query) + except KeyError as E: + raise RouteBuildError('Missing URL argument: %r' % E.args[0]) + + def match(self, environ): + """ Return a (target, url_args) tuple or raise HTTPError(400/404/405). """ + verb = environ['REQUEST_METHOD'].upper() + path = environ['PATH_INFO'] or '/' + + methods = ('PROXY', 'HEAD', 'GET', 'ANY') if verb == 'HEAD' else ('PROXY', verb, 'ANY') + + for method in methods: + if method in self.static and path in self.static[method]: + target, getargs = self.static[method][path] + return target, getargs(path) if getargs else {} + elif method in self.dyna_regexes: + for combined, rules in self.dyna_regexes[method]: + match = combined(path) + if match: + target, getargs = rules[match.lastindex - 1] + return target, getargs(path) if getargs else {} + + # No matching route found. Collect alternative methods for 405 response + allowed = set([]) + nocheck = set(methods) + for method in set(self.static) - nocheck: + if path in self.static[method]: + allowed.add(method) + for method in set(self.dyna_regexes) - allowed - nocheck: + for combined, rules in self.dyna_regexes[method]: + match = combined(path) + if match: + allowed.add(method) + if allowed: + allow_header = ",".join(sorted(allowed)) + raise HTTPError(405, "Method not allowed.", Allow=allow_header) + + # No matching route and no alternative method found. We give up + raise HTTPError(404, "Not found: " + repr(path)) + + +class Route(object): + """ This class wraps a route callback along with route specific metadata and + configuration and applies Plugins on demand. It is also responsible for + turning an URL path rule into a regular expression usable by the Router. + """ + + def __init__(self, app, rule, method, callback, + name=None, + plugins=None, + skiplist=None, **config): + #: The application this route is installed to. + self.app = app + #: The path-rule string (e.g. ``/wiki/``). + self.rule = rule + #: The HTTP method as a string (e.g. ``GET``). + self.method = method + #: The original callback with no plugins applied. Useful for introspection. + self.callback = callback + #: The name of the route (if specified) or ``None``. + self.name = name or None + #: A list of route-specific plugins (see :meth:`Bottle.route`). + self.plugins = plugins or [] + #: A list of plugins to not apply to this route (see :meth:`Bottle.route`). + self.skiplist = skiplist or [] + #: Additional keyword arguments passed to the :meth:`Bottle.route` + #: decorator are stored in this dictionary. Used for route-specific + #: plugin configuration and meta-data. + self.config = app.config._make_overlay() + self.config.load_dict(config) + + @cached_property + def call(self): + """ The route callback with all plugins applied. This property is + created on demand and then cached to speed up subsequent requests.""" + return self._make_callback() + + def reset(self): + """ Forget any cached values. The next time :attr:`call` is accessed, + all plugins are re-applied. """ + self.__dict__.pop('call', None) + + def prepare(self): + """ Do all on-demand work immediately (useful for debugging).""" + self.call + + def all_plugins(self): + """ Yield all Plugins affecting this route. """ + unique = set() + for p in reversed(self.app.plugins + self.plugins): + if True in self.skiplist: break + name = getattr(p, 'name', False) + if name and (name in self.skiplist or name in unique): continue + if p in self.skiplist or type(p) in self.skiplist: continue + if name: unique.add(name) + yield p + + def _make_callback(self): + callback = self.callback + for plugin in self.all_plugins(): + try: + if hasattr(plugin, 'apply'): + callback = plugin.apply(callback, self) + else: + callback = plugin(callback) + except RouteReset: # Try again with changed configuration. + return self._make_callback() + if callback is not self.callback: + update_wrapper(callback, self.callback) + return callback + + def get_undecorated_callback(self): + """ Return the callback. If the callback is a decorated function, try to + recover the original function. """ + func = self.callback + func = getattr(func, '__func__' if py3k else 'im_func', func) + closure_attr = '__closure__' if py3k else 'func_closure' + while hasattr(func, closure_attr) and getattr(func, closure_attr): + attributes = getattr(func, closure_attr) + func = attributes[0].cell_contents + + # in case of decorators with multiple arguments + if not isinstance(func, FunctionType): + # pick first FunctionType instance from multiple arguments + func = filter(lambda x: isinstance(x, FunctionType), + map(lambda x: x.cell_contents, attributes)) + func = list(func)[0] # py3 support + return func + + def get_callback_args(self): + """ Return a list of argument names the callback (most likely) accepts + as keyword arguments. If the callback is a decorated function, try + to recover the original function before inspection. """ + return getargspec(self.get_undecorated_callback())[0] + + def get_config(self, key, default=None): + """ Lookup a config field and return its value, first checking the + route.config, then route.app.config.""" + depr(0, 13, "Route.get_config() is deprecated.", + "The Route.config property already includes values from the" + " application config for missing keys. Access it directly.") + return self.config.get(key, default) + + def __repr__(self): + cb = self.get_undecorated_callback() + return '<%s %s -> %s:%s>' % (self.method, self.rule, cb.__module__, cb.__name__) + +############################################################################### +# Application Object ########################################################### +############################################################################### + + +class Bottle(object): + """ Each Bottle object represents a single, distinct web application and + consists of routes, callbacks, plugins, resources and configuration. + Instances are callable WSGI applications. + + :param catchall: If true (default), handle all exceptions. Turn off to + let debugging middleware handle exceptions. + """ + + @lazy_attribute + def _global_config(cls): + cfg = ConfigDict() + cfg.meta_set('catchall', 'validate', bool) + return cfg + + def __init__(self, **kwargs): + #: A :class:`ConfigDict` for app specific configuration. + self.config = self._global_config._make_overlay() + self.config._add_change_listener( + functools.partial(self.trigger_hook, 'config')) + + self.config.update({ + "catchall": True + }) + + if kwargs.get('catchall') is False: + depr(0, 13, "Bottle(catchall) keyword argument.", + "The 'catchall' setting is now part of the app " + "configuration. Fix: `app.config['catchall'] = False`") + self.config['catchall'] = False + if kwargs.get('autojson') is False: + depr(0, 13, "Bottle(autojson) keyword argument.", + "The 'autojson' setting is now part of the app " + "configuration. Fix: `app.config['json.enable'] = False`") + self.config['json.disable'] = True + + self._mounts = [] + + #: A :class:`ResourceManager` for application files + self.resources = ResourceManager() + + self.routes = [] # List of installed :class:`Route` instances. + self.router = Router() # Maps requests to :class:`Route` instances. + self.error_handler = {} + + # Core plugins + self.plugins = [] # List of installed plugins. + self.install(JSONPlugin()) + self.install(TemplatePlugin()) + + #: If true, most exceptions are caught and returned as :exc:`HTTPError` + catchall = DictProperty('config', 'catchall') + + __hook_names = 'before_request', 'after_request', 'app_reset', 'config' + __hook_reversed = {'after_request'} + + @cached_property + def _hooks(self): + return dict((name, []) for name in self.__hook_names) + + def add_hook(self, name, func): + """ Attach a callback to a hook. Three hooks are currently implemented: + + before_request + Executed once before each request. The request context is + available, but no routing has happened yet. + after_request + Executed once after each request regardless of its outcome. + app_reset + Called whenever :meth:`Bottle.reset` is called. + """ + if name in self.__hook_reversed: + self._hooks[name].insert(0, func) + else: + self._hooks[name].append(func) + + def remove_hook(self, name, func): + """ Remove a callback from a hook. """ + if name in self._hooks and func in self._hooks[name]: + self._hooks[name].remove(func) + return True + + def trigger_hook(self, __name, *args, **kwargs): + """ Trigger a hook and return a list of results. """ + return [hook(*args, **kwargs) for hook in self._hooks[__name][:]] + + def hook(self, name): + """ Return a decorator that attaches a callback to a hook. See + :meth:`add_hook` for details.""" + + def decorator(func): + self.add_hook(name, func) + return func + + return decorator + + def _mount_wsgi(self, prefix, app, **options): + segments = [p for p in prefix.split('/') if p] + if not segments: + raise ValueError('WSGI applications cannot be mounted to "/".') + path_depth = len(segments) + + def mountpoint_wrapper(): + try: + request.path_shift(path_depth) + rs = HTTPResponse([]) + + def start_response(status, headerlist, exc_info=None): + if exc_info: + _raise(*exc_info) + if py3k: + # Errors here mean that the mounted WSGI app did not + # follow PEP-3333 (which requires latin1) or used a + # pre-encoding other than utf8 :/ + status = status.encode('latin1').decode('utf8') + headerlist = [(k, v.encode('latin1').decode('utf8')) + for (k, v) in headerlist] + rs.status = status + for name, value in headerlist: + rs.add_header(name, value) + return rs.body.append + + body = app(request.environ, start_response) + rs.body = itertools.chain(rs.body, body) if rs.body else body + return rs + finally: + request.path_shift(-path_depth) + + options.setdefault('skip', True) + options.setdefault('method', 'PROXY') + options.setdefault('mountpoint', {'prefix': prefix, 'target': app}) + options['callback'] = mountpoint_wrapper + + self.route('/%s/<:re:.*>' % '/'.join(segments), **options) + if not prefix.endswith('/'): + self.route('/' + '/'.join(segments), **options) + + def _mount_app(self, prefix, app, **options): + if app in self._mounts or '_mount.app' in app.config: + depr(0, 13, "Application mounted multiple times. Falling back to WSGI mount.", + "Clone application before mounting to a different location.") + return self._mount_wsgi(prefix, app, **options) + + if options: + depr(0, 13, "Unsupported mount options. Falling back to WSGI mount.", + "Do not specify any route options when mounting bottle application.") + return self._mount_wsgi(prefix, app, **options) + + if not prefix.endswith("/"): + depr(0, 13, "Prefix must end in '/'. Falling back to WSGI mount.", + "Consider adding an explicit redirect from '/prefix' to '/prefix/' in the parent application.") + return self._mount_wsgi(prefix, app, **options) + + self._mounts.append(app) + app.config['_mount.prefix'] = prefix + app.config['_mount.app'] = self + for route in app.routes: + route.rule = prefix + route.rule.lstrip('/') + self.add_route(route) + + def mount(self, prefix, app, **options): + """ Mount an application (:class:`Bottle` or plain WSGI) to a specific + URL prefix. Example:: + + parent_app.mount('/prefix/', child_app) + + :param prefix: path prefix or `mount-point`. + :param app: an instance of :class:`Bottle` or a WSGI application. + + Plugins from the parent application are not applied to the routes + of the mounted child application. If you need plugins in the child + application, install them separately. + + While it is possible to use path wildcards within the prefix path + (:class:`Bottle` childs only), it is highly discouraged. + + The prefix path must end with a slash. If you want to access the + root of the child application via `/prefix` in addition to + `/prefix/`, consider adding a route with a 307 redirect to the + parent application. + """ + + if not prefix.startswith('/'): + raise ValueError("Prefix must start with '/'") + + if isinstance(app, Bottle): + return self._mount_app(prefix, app, **options) + else: + return self._mount_wsgi(prefix, app, **options) + + def merge(self, routes): + """ Merge the routes of another :class:`Bottle` application or a list of + :class:`Route` objects into this application. The routes keep their + 'owner', meaning that the :data:`Route.app` attribute is not + changed. """ + if isinstance(routes, Bottle): + routes = routes.routes + for route in routes: + self.add_route(route) + + def install(self, plugin): + """ Add a plugin to the list of plugins and prepare it for being + applied to all routes of this application. A plugin may be a simple + decorator or an object that implements the :class:`Plugin` API. + """ + if hasattr(plugin, 'setup'): plugin.setup(self) + if not callable(plugin) and not hasattr(plugin, 'apply'): + raise TypeError("Plugins must be callable or implement .apply()") + self.plugins.append(plugin) + self.reset() + return plugin + + def uninstall(self, plugin): + """ Uninstall plugins. Pass an instance to remove a specific plugin, a type + object to remove all plugins that match that type, a string to remove + all plugins with a matching ``name`` attribute or ``True`` to remove all + plugins. Return the list of removed plugins. """ + removed, remove = [], plugin + for i, plugin in list(enumerate(self.plugins))[::-1]: + if remove is True or remove is plugin or remove is type(plugin) \ + or getattr(plugin, 'name', True) == remove: + removed.append(plugin) + del self.plugins[i] + if hasattr(plugin, 'close'): plugin.close() + if removed: self.reset() + return removed + + def reset(self, route=None): + """ Reset all routes (force plugins to be re-applied) and clear all + caches. If an ID or route object is given, only that specific route + is affected. """ + if route is None: routes = self.routes + elif isinstance(route, Route): routes = [route] + else: routes = [self.routes[route]] + for route in routes: + route.reset() + if DEBUG: + for route in routes: + route.prepare() + self.trigger_hook('app_reset') + + def close(self): + """ Close the application and all installed plugins. """ + for plugin in self.plugins: + if hasattr(plugin, 'close'): plugin.close() + + def run(self, **kwargs): + """ Calls :func:`run` with the same parameters. """ + run(self, **kwargs) + + def match(self, environ): + """ Search for a matching route and return a (:class:`Route`, urlargs) + tuple. The second value is a dictionary with parameters extracted + from the URL. Raise :exc:`HTTPError` (404/405) on a non-match.""" + return self.router.match(environ) + + def get_url(self, routename, **kargs): + """ Return a string that matches a named route """ + scriptname = request.environ.get('SCRIPT_NAME', '').strip('/') + '/' + location = self.router.build(routename, **kargs).lstrip('/') + return urljoin(urljoin('/', scriptname), location) + + def add_route(self, route): + """ Add a route object, but do not change the :data:`Route.app` + attribute.""" + self.routes.append(route) + self.router.add(route.rule, route.method, route, name=route.name) + if DEBUG: route.prepare() + + def route(self, + path=None, + method='GET', + callback=None, + name=None, + apply=None, + skip=None, **config): + """ A decorator to bind a function to a request URL. Example:: + + @app.route('/hello/') + def hello(name): + return 'Hello %s' % name + + The ```` part is a wildcard. See :class:`Router` for syntax + details. + + :param path: Request path or a list of paths to listen to. If no + path is specified, it is automatically generated from the + signature of the function. + :param method: HTTP method (`GET`, `POST`, `PUT`, ...) or a list of + methods to listen to. (default: `GET`) + :param callback: An optional shortcut to avoid the decorator + syntax. ``route(..., callback=func)`` equals ``route(...)(func)`` + :param name: The name for this route. (default: None) + :param apply: A decorator or plugin or a list of plugins. These are + applied to the route callback in addition to installed plugins. + :param skip: A list of plugins, plugin classes or names. Matching + plugins are not installed to this route. ``True`` skips all. + + Any additional keyword arguments are stored as route-specific + configuration and passed to plugins (see :meth:`Plugin.apply`). + """ + if callable(path): path, callback = None, path + plugins = makelist(apply) + skiplist = makelist(skip) + + def decorator(callback): + if isinstance(callback, basestring): callback = load(callback) + for rule in makelist(path) or yieldroutes(callback): + for verb in makelist(method): + verb = verb.upper() + route = Route(self, rule, verb, callback, + name=name, + plugins=plugins, + skiplist=skiplist, **config) + self.add_route(route) + return callback + + return decorator(callback) if callback else decorator + + def get(self, path=None, method='GET', **options): + """ Equals :meth:`route`. """ + return self.route(path, method, **options) + + def post(self, path=None, method='POST', **options): + """ Equals :meth:`route` with a ``POST`` method parameter. """ + return self.route(path, method, **options) + + def put(self, path=None, method='PUT', **options): + """ Equals :meth:`route` with a ``PUT`` method parameter. """ + return self.route(path, method, **options) + + def delete(self, path=None, method='DELETE', **options): + """ Equals :meth:`route` with a ``DELETE`` method parameter. """ + return self.route(path, method, **options) + + def patch(self, path=None, method='PATCH', **options): + """ Equals :meth:`route` with a ``PATCH`` method parameter. """ + return self.route(path, method, **options) + + def error(self, code=500, callback=None): + """ Register an output handler for a HTTP error code. Can + be used as a decorator or called directly :: + + def error_handler_500(error): + return 'error_handler_500' + + app.error(code=500, callback=error_handler_500) + + @app.error(404) + def error_handler_404(error): + return 'error_handler_404' + + """ + + def decorator(callback): + if isinstance(callback, basestring): callback = load(callback) + self.error_handler[int(code)] = callback + return callback + + return decorator(callback) if callback else decorator + + def default_error_handler(self, res): + return tob(template(ERROR_PAGE_TEMPLATE, e=res, template_settings=dict(name='__ERROR_PAGE_TEMPLATE'))) + + def _handle(self, environ): + path = environ['bottle.raw_path'] = environ['PATH_INFO'] + if py3k: + environ['PATH_INFO'] = path.encode('latin1').decode('utf8', 'ignore') + + environ['bottle.app'] = self + request.bind(environ) + response.bind() + + try: + while True: # Remove in 0.14 together with RouteReset + out = None + try: + self.trigger_hook('before_request') + route, args = self.router.match(environ) + environ['route.handle'] = route + environ['bottle.route'] = route + environ['route.url_args'] = args + out = route.call(**args) + break + except HTTPResponse as E: + out = E + break + except RouteReset: + depr(0, 13, "RouteReset exception deprecated", + "Call route.call() after route.reset() and " + "return the result.") + route.reset() + continue + finally: + if isinstance(out, HTTPResponse): + out.apply(response) + try: + self.trigger_hook('after_request') + except HTTPResponse as E: + out = E + out.apply(response) + except (KeyboardInterrupt, SystemExit, MemoryError): + raise + except Exception as E: + if not self.catchall: raise + stacktrace = format_exc() + environ['wsgi.errors'].write(stacktrace) + environ['wsgi.errors'].flush() + environ['bottle.exc_info'] = sys.exc_info() + out = HTTPError(500, "Internal Server Error", E, stacktrace) + out.apply(response) + + return out + + def _cast(self, out, peek=None): + """ Try to convert the parameter into something WSGI compatible and set + correct HTTP headers when possible. + Support: False, str, unicode, dict, HTTPResponse, HTTPError, file-like, + iterable of strings and iterable of unicodes + """ + + # Empty output is done here + if not out: + if 'Content-Length' not in response: + response['Content-Length'] = 0 + return [] + # Join lists of byte or unicode strings. Mixed lists are NOT supported + if isinstance(out, (tuple, list))\ + and isinstance(out[0], (bytes, unicode)): + out = out[0][0:0].join(out) # b'abc'[0:0] -> b'' + # Encode unicode strings + if isinstance(out, unicode): + out = out.encode(response.charset) + # Byte Strings are just returned + if isinstance(out, bytes): + if 'Content-Length' not in response: + response['Content-Length'] = len(out) + return [out] + # HTTPError or HTTPException (recursive, because they may wrap anything) + # TODO: Handle these explicitly in handle() or make them iterable. + if isinstance(out, HTTPError): + out.apply(response) + out = self.error_handler.get(out.status_code, + self.default_error_handler)(out) + return self._cast(out) + if isinstance(out, HTTPResponse): + out.apply(response) + return self._cast(out.body) + + # File-like objects. + if hasattr(out, 'read'): + if 'wsgi.file_wrapper' in request.environ: + return request.environ['wsgi.file_wrapper'](out) + elif hasattr(out, 'close') or not hasattr(out, '__iter__'): + return WSGIFileWrapper(out) + + # Handle Iterables. We peek into them to detect their inner type. + try: + iout = iter(out) + first = next(iout) + while not first: + first = next(iout) + except StopIteration: + return self._cast('') + except HTTPResponse as E: + first = E + except (KeyboardInterrupt, SystemExit, MemoryError): + raise + except Exception as error: + if not self.catchall: raise + first = HTTPError(500, 'Unhandled exception', error, format_exc()) + + # These are the inner types allowed in iterator or generator objects. + if isinstance(first, HTTPResponse): + return self._cast(first) + elif isinstance(first, bytes): + new_iter = itertools.chain([first], iout) + elif isinstance(first, unicode): + encoder = lambda x: x.encode(response.charset) + new_iter = imap(encoder, itertools.chain([first], iout)) + else: + msg = 'Unsupported response type: %s' % type(first) + return self._cast(HTTPError(500, msg)) + if hasattr(out, 'close'): + new_iter = _closeiter(new_iter, out.close) + return new_iter + + def wsgi(self, environ, start_response): + """ The bottle WSGI-interface. """ + try: + out = self._cast(self._handle(environ)) + # rfc2616 section 4.3 + if response._status_code in (100, 101, 204, 304)\ + or environ['REQUEST_METHOD'] == 'HEAD': + if hasattr(out, 'close'): out.close() + out = [] + exc_info = environ.get('bottle.exc_info') + if exc_info is not None: + del environ['bottle.exc_info'] + start_response(response._wsgi_status_line(), response.headerlist, exc_info) + return out + except (KeyboardInterrupt, SystemExit, MemoryError): + raise + except Exception as E: + if not self.catchall: raise + err = '

Critical error while processing request: %s

' \ + % html_escape(environ.get('PATH_INFO', '/')) + if DEBUG: + err += '

Error:

\n
\n%s\n
\n' \ + '

Traceback:

\n
\n%s\n
\n' \ + % (html_escape(repr(E)), html_escape(format_exc())) + environ['wsgi.errors'].write(err) + environ['wsgi.errors'].flush() + headers = [('Content-Type', 'text/html; charset=UTF-8')] + start_response('500 INTERNAL SERVER ERROR', headers, sys.exc_info()) + return [tob(err)] + + def __call__(self, environ, start_response): + """ Each instance of :class:'Bottle' is a WSGI application. """ + return self.wsgi(environ, start_response) + + def __enter__(self): + """ Use this application as default for all module-level shortcuts. """ + default_app.push(self) + return self + + def __exit__(self, exc_type, exc_value, traceback): + default_app.pop() + + def __setattr__(self, name, value): + if name in self.__dict__: + raise AttributeError("Attribute %s already defined. Plugin conflict?" % name) + self.__dict__[name] = value + + +############################################################################### +# HTTP and WSGI Tools ########################################################## +############################################################################### + + +class BaseRequest(object): + """ A wrapper for WSGI environment dictionaries that adds a lot of + convenient access methods and properties. Most of them are read-only. + + Adding new attributes to a request actually adds them to the environ + dictionary (as 'bottle.request.ext.'). This is the recommended + way to store and access request-specific data. + """ + + __slots__ = ('environ', ) + + #: Maximum size of memory buffer for :attr:`body` in bytes. + MEMFILE_MAX = 102400 + + def __init__(self, environ=None): + """ Wrap a WSGI environ dictionary. """ + #: The wrapped WSGI environ dictionary. This is the only real attribute. + #: All other attributes actually are read-only properties. + self.environ = {} if environ is None else environ + self.environ['bottle.request'] = self + + @DictProperty('environ', 'bottle.app', read_only=True) + def app(self): + """ Bottle application handling this request. """ + raise RuntimeError('This request is not connected to an application.') + + @DictProperty('environ', 'bottle.route', read_only=True) + def route(self): + """ The bottle :class:`Route` object that matches this request. """ + raise RuntimeError('This request is not connected to a route.') + + @DictProperty('environ', 'route.url_args', read_only=True) + def url_args(self): + """ The arguments extracted from the URL. """ + raise RuntimeError('This request is not connected to a route.') + + @property + def path(self): + """ The value of ``PATH_INFO`` with exactly one prefixed slash (to fix + broken clients and avoid the "empty path" edge case). """ + return '/' + self.environ.get('PATH_INFO', '').lstrip('/') + + @property + def method(self): + """ The ``REQUEST_METHOD`` value as an uppercase string. """ + return self.environ.get('REQUEST_METHOD', 'GET').upper() + + @DictProperty('environ', 'bottle.request.headers', read_only=True) + def headers(self): + """ A :class:`WSGIHeaderDict` that provides case-insensitive access to + HTTP request headers. """ + return WSGIHeaderDict(self.environ) + + def get_header(self, name, default=None): + """ Return the value of a request header, or a given default value. """ + return self.headers.get(name, default) + + @DictProperty('environ', 'bottle.request.cookies', read_only=True) + def cookies(self): + """ Cookies parsed into a :class:`FormsDict`. Signed cookies are NOT + decoded. Use :meth:`get_cookie` if you expect signed cookies. """ + cookies = SimpleCookie(self.environ.get('HTTP_COOKIE', '')).values() + return FormsDict((c.key, c.value) for c in cookies) + + def get_cookie(self, key, default=None, secret=None, digestmod=hashlib.sha256): + """ Return the content of a cookie. To read a `Signed Cookie`, the + `secret` must match the one used to create the cookie (see + :meth:`BaseResponse.set_cookie`). If anything goes wrong (missing + cookie or wrong signature), return a default value. """ + value = self.cookies.get(key) + if secret: + # See BaseResponse.set_cookie for details on signed cookies. + if value and value.startswith('!') and '?' in value: + sig, msg = map(tob, value[1:].split('?', 1)) + hash = hmac.new(tob(secret), msg, digestmod=digestmod).digest() + if _lscmp(sig, base64.b64encode(hash)): + dst = pickle.loads(base64.b64decode(msg)) + if dst and dst[0] == key: + return dst[1] + return default + return value or default + + @DictProperty('environ', 'bottle.request.query', read_only=True) + def query(self): + """ The :attr:`query_string` parsed into a :class:`FormsDict`. These + values are sometimes called "URL arguments" or "GET parameters", but + not to be confused with "URL wildcards" as they are provided by the + :class:`Router`. """ + get = self.environ['bottle.get'] = FormsDict() + pairs = _parse_qsl(self.environ.get('QUERY_STRING', '')) + for key, value in pairs: + get[key] = value + return get + + @DictProperty('environ', 'bottle.request.forms', read_only=True) + def forms(self): + """ Form values parsed from an `url-encoded` or `multipart/form-data` + encoded POST or PUT request body. The result is returned as a + :class:`FormsDict`. All keys and values are strings. File uploads + are stored separately in :attr:`files`. """ + forms = FormsDict() + forms.recode_unicode = self.POST.recode_unicode + for name, item in self.POST.allitems(): + if not isinstance(item, FileUpload): + forms[name] = item + return forms + + @DictProperty('environ', 'bottle.request.params', read_only=True) + def params(self): + """ A :class:`FormsDict` with the combined values of :attr:`query` and + :attr:`forms`. File uploads are stored in :attr:`files`. """ + params = FormsDict() + for key, value in self.query.allitems(): + params[key] = value + for key, value in self.forms.allitems(): + params[key] = value + return params + + @DictProperty('environ', 'bottle.request.files', read_only=True) + def files(self): + """ File uploads parsed from `multipart/form-data` encoded POST or PUT + request body. The values are instances of :class:`FileUpload`. + + """ + files = FormsDict() + files.recode_unicode = self.POST.recode_unicode + for name, item in self.POST.allitems(): + if isinstance(item, FileUpload): + files[name] = item + return files + + @DictProperty('environ', 'bottle.request.json', read_only=True) + def json(self): + """ If the ``Content-Type`` header is ``application/json`` or + ``application/json-rpc``, this property holds the parsed content + of the request body. Only requests smaller than :attr:`MEMFILE_MAX` + are processed to avoid memory exhaustion. + Invalid JSON raises a 400 error response. + """ + ctype = self.environ.get('CONTENT_TYPE', '').lower().split(';')[0] + if ctype in ('application/json', 'application/json-rpc'): + b = self._get_body_string(self.MEMFILE_MAX) + if not b: + return None + try: + return json_loads(b) + except (ValueError, TypeError): + raise HTTPError(400, 'Invalid JSON') + return None + + def _iter_body(self, read, bufsize): + maxread = max(0, self.content_length) + while maxread: + part = read(min(maxread, bufsize)) + if not part: break + yield part + maxread -= len(part) + + @staticmethod + def _iter_chunked(read, bufsize): + err = HTTPError(400, 'Error while parsing chunked transfer body.') + rn, sem, bs = tob('\r\n'), tob(';'), tob('') + while True: + header = read(1) + while header[-2:] != rn: + c = read(1) + header += c + if not c: raise err + if len(header) > bufsize: raise err + size, _, _ = header.partition(sem) + try: + maxread = int(tonat(size.strip()), 16) + except ValueError: + raise err + if maxread == 0: break + buff = bs + while maxread > 0: + if not buff: + buff = read(min(maxread, bufsize)) + part, buff = buff[:maxread], buff[maxread:] + if not part: raise err + yield part + maxread -= len(part) + if read(2) != rn: + raise err + + @DictProperty('environ', 'bottle.request.body', read_only=True) + def _body(self): + try: + read_func = self.environ['wsgi.input'].read + except KeyError: + self.environ['wsgi.input'] = BytesIO() + return self.environ['wsgi.input'] + body_iter = self._iter_chunked if self.chunked else self._iter_body + body, body_size, is_temp_file = BytesIO(), 0, False + for part in body_iter(read_func, self.MEMFILE_MAX): + body.write(part) + body_size += len(part) + if not is_temp_file and body_size > self.MEMFILE_MAX: + body, tmp = NamedTemporaryFile(mode='w+b'), body + body.write(tmp.getvalue()) + del tmp + is_temp_file = True + self.environ['wsgi.input'] = body + body.seek(0) + return body + + def _get_body_string(self, maxread): + """ Read body into a string. Raise HTTPError(413) on requests that are + too large. """ + if self.content_length > maxread: + raise HTTPError(413, 'Request entity too large') + data = self.body.read(maxread + 1) + if len(data) > maxread: + raise HTTPError(413, 'Request entity too large') + return data + + @property + def body(self): + """ The HTTP request body as a seek-able file-like object. Depending on + :attr:`MEMFILE_MAX`, this is either a temporary file or a + :class:`io.BytesIO` instance. Accessing this property for the first + time reads and replaces the ``wsgi.input`` environ variable. + Subsequent accesses just do a `seek(0)` on the file object. """ + self._body.seek(0) + return self._body + + @property + def chunked(self): + """ True if Chunked transfer encoding was. """ + return 'chunked' in self.environ.get( + 'HTTP_TRANSFER_ENCODING', '').lower() + + #: An alias for :attr:`query`. + GET = query + + @DictProperty('environ', 'bottle.request.post', read_only=True) + def POST(self): + """ The values of :attr:`forms` and :attr:`files` combined into a single + :class:`FormsDict`. Values are either strings (form values) or + instances of :class:`cgi.FieldStorage` (file uploads). + """ + post = FormsDict() + # We default to application/x-www-form-urlencoded for everything that + # is not multipart and take the fast path (also: 3.1 workaround) + if not self.content_type.startswith('multipart/'): + body = tonat(self._get_body_string(self.MEMFILE_MAX), 'latin1') + for key, value in _parse_qsl(body): + post[key] = value + return post + + safe_env = {'QUERY_STRING': ''} # Build a safe environment for cgi + for key in ('REQUEST_METHOD', 'CONTENT_TYPE', 'CONTENT_LENGTH'): + if key in self.environ: safe_env[key] = self.environ[key] + args = dict(fp=self.body, environ=safe_env, keep_blank_values=True) + + if py3k: + args['encoding'] = 'utf8' + post.recode_unicode = False + data = cgi.FieldStorage(**args) + self['_cgi.FieldStorage'] = data #http://bugs.python.org/issue18394 + data = data.list or [] + for item in data: + if item.filename is None: + post[item.name] = item.value + else: + post[item.name] = FileUpload(item.file, item.name, + item.filename, item.headers) + return post + + @property + def url(self): + """ The full request URI including hostname and scheme. If your app + lives behind a reverse proxy or load balancer and you get confusing + results, make sure that the ``X-Forwarded-Host`` header is set + correctly. """ + return self.urlparts.geturl() + + @DictProperty('environ', 'bottle.request.urlparts', read_only=True) + def urlparts(self): + """ The :attr:`url` string as an :class:`urlparse.SplitResult` tuple. + The tuple contains (scheme, host, path, query_string and fragment), + but the fragment is always empty because it is not visible to the + server. """ + env = self.environ + http = env.get('HTTP_X_FORWARDED_PROTO') \ + or env.get('wsgi.url_scheme', 'http') + host = env.get('HTTP_X_FORWARDED_HOST') or env.get('HTTP_HOST') + if not host: + # HTTP 1.1 requires a Host-header. This is for HTTP/1.0 clients. + host = env.get('SERVER_NAME', '127.0.0.1') + port = env.get('SERVER_PORT') + if port and port != ('80' if http == 'http' else '443'): + host += ':' + port + path = urlquote(self.fullpath) + return UrlSplitResult(http, host, path, env.get('QUERY_STRING'), '') + + @property + def fullpath(self): + """ Request path including :attr:`script_name` (if present). """ + return urljoin(self.script_name, self.path.lstrip('/')) + + @property + def query_string(self): + """ The raw :attr:`query` part of the URL (everything in between ``?`` + and ``#``) as a string. """ + return self.environ.get('QUERY_STRING', '') + + @property + def script_name(self): + """ The initial portion of the URL's `path` that was removed by a higher + level (server or routing middleware) before the application was + called. This script path is returned with leading and tailing + slashes. """ + script_name = self.environ.get('SCRIPT_NAME', '').strip('/') + return '/' + script_name + '/' if script_name else '/' + + def path_shift(self, shift=1): + """ Shift path segments from :attr:`path` to :attr:`script_name` and + vice versa. + + :param shift: The number of path segments to shift. May be negative + to change the shift direction. (default: 1) + """ + script, path = path_shift(self.environ.get('SCRIPT_NAME', '/'), self.path, shift) + self['SCRIPT_NAME'], self['PATH_INFO'] = script, path + + @property + def content_length(self): + """ The request body length as an integer. The client is responsible to + set this header. Otherwise, the real length of the body is unknown + and -1 is returned. In this case, :attr:`body` will be empty. """ + return int(self.environ.get('CONTENT_LENGTH') or -1) + + @property + def content_type(self): + """ The Content-Type header as a lowercase-string (default: empty). """ + return self.environ.get('CONTENT_TYPE', '').lower() + + @property + def is_xhr(self): + """ True if the request was triggered by a XMLHttpRequest. This only + works with JavaScript libraries that support the `X-Requested-With` + header (most of the popular libraries do). """ + requested_with = self.environ.get('HTTP_X_REQUESTED_WITH', '') + return requested_with.lower() == 'xmlhttprequest' + + @property + def is_ajax(self): + """ Alias for :attr:`is_xhr`. "Ajax" is not the right term. """ + return self.is_xhr + + @property + def auth(self): + """ HTTP authentication data as a (user, password) tuple. This + implementation currently supports basic (not digest) authentication + only. If the authentication happened at a higher level (e.g. in the + front web-server or a middleware), the password field is None, but + the user field is looked up from the ``REMOTE_USER`` environ + variable. On any errors, None is returned. """ + basic = parse_auth(self.environ.get('HTTP_AUTHORIZATION', '')) + if basic: return basic + ruser = self.environ.get('REMOTE_USER') + if ruser: return (ruser, None) + return None + + @property + def remote_route(self): + """ A list of all IPs that were involved in this request, starting with + the client IP and followed by zero or more proxies. This does only + work if all proxies support the ```X-Forwarded-For`` header. Note + that this information can be forged by malicious clients. """ + proxy = self.environ.get('HTTP_X_FORWARDED_FOR') + if proxy: return [ip.strip() for ip in proxy.split(',')] + remote = self.environ.get('REMOTE_ADDR') + return [remote] if remote else [] + + @property + def remote_addr(self): + """ The client IP as a string. Note that this information can be forged + by malicious clients. """ + route = self.remote_route + return route[0] if route else None + + def copy(self): + """ Return a new :class:`Request` with a shallow :attr:`environ` copy. """ + return Request(self.environ.copy()) + + def get(self, value, default=None): + return self.environ.get(value, default) + + def __getitem__(self, key): + return self.environ[key] + + def __delitem__(self, key): + self[key] = "" + del (self.environ[key]) + + def __iter__(self): + return iter(self.environ) + + def __len__(self): + return len(self.environ) + + def keys(self): + return self.environ.keys() + + def __setitem__(self, key, value): + """ Change an environ value and clear all caches that depend on it. """ + + if self.environ.get('bottle.request.readonly'): + raise KeyError('The environ dictionary is read-only.') + + self.environ[key] = value + todelete = () + + if key == 'wsgi.input': + todelete = ('body', 'forms', 'files', 'params', 'post', 'json') + elif key == 'QUERY_STRING': + todelete = ('query', 'params') + elif key.startswith('HTTP_'): + todelete = ('headers', 'cookies') + + for key in todelete: + self.environ.pop('bottle.request.' + key, None) + + def __repr__(self): + return '<%s: %s %s>' % (self.__class__.__name__, self.method, self.url) + + def __getattr__(self, name): + """ Search in self.environ for additional user defined attributes. """ + try: + var = self.environ['bottle.request.ext.%s' % name] + return var.__get__(self) if hasattr(var, '__get__') else var + except KeyError: + raise AttributeError('Attribute %r not defined.' % name) + + def __setattr__(self, name, value): + if name == 'environ': return object.__setattr__(self, name, value) + key = 'bottle.request.ext.%s' % name + if hasattr(self, name): + raise AttributeError("Attribute already defined: %s" % name) + self.environ[key] = value + + def __delattr__(self, name): + try: + del self.environ['bottle.request.ext.%s' % name] + except KeyError: + raise AttributeError("Attribute not defined: %s" % name) + + +def _hkey(key): + if '\n' in key or '\r' in key or '\0' in key: + raise ValueError("Header names must not contain control characters: %r" % key) + return key.title().replace('_', '-') + + +def _hval(value): + value = tonat(value) + if '\n' in value or '\r' in value or '\0' in value: + raise ValueError("Header value must not contain control characters: %r" % value) + return value + + +class HeaderProperty(object): + def __init__(self, name, reader=None, writer=None, default=''): + self.name, self.default = name, default + self.reader, self.writer = reader, writer + self.__doc__ = 'Current value of the %r header.' % name.title() + + def __get__(self, obj, _): + if obj is None: return self + value = obj.get_header(self.name, self.default) + return self.reader(value) if self.reader else value + + def __set__(self, obj, value): + obj[self.name] = self.writer(value) if self.writer else value + + def __delete__(self, obj): + del obj[self.name] + + +class BaseResponse(object): + """ Storage class for a response body as well as headers and cookies. + + This class does support dict-like case-insensitive item-access to + headers, but is NOT a dict. Most notably, iterating over a response + yields parts of the body and not the headers. + + :param body: The response body as one of the supported types. + :param status: Either an HTTP status code (e.g. 200) or a status line + including the reason phrase (e.g. '200 OK'). + :param headers: A dictionary or a list of name-value pairs. + + Additional keyword arguments are added to the list of headers. + Underscores in the header name are replaced with dashes. + """ + + default_status = 200 + default_content_type = 'text/html; charset=UTF-8' + + # Header denylist for specific response codes + # (rfc2616 section 10.2.3 and 10.3.5) + bad_headers = { + 204: frozenset(('Content-Type', 'Content-Length')), + 304: frozenset(('Allow', 'Content-Encoding', 'Content-Language', + 'Content-Length', 'Content-Range', 'Content-Type', + 'Content-Md5', 'Last-Modified')) + } + + def __init__(self, body='', status=None, headers=None, **more_headers): + self._cookies = None + self._headers = {} + self.body = body + self.status = status or self.default_status + if headers: + if isinstance(headers, dict): + headers = headers.items() + for name, value in headers: + self.add_header(name, value) + if more_headers: + for name, value in more_headers.items(): + self.add_header(name, value) + + def copy(self, cls=None): + """ Returns a copy of self. """ + cls = cls or BaseResponse + assert issubclass(cls, BaseResponse) + copy = cls() + copy.status = self.status + copy._headers = dict((k, v[:]) for (k, v) in self._headers.items()) + if self._cookies: + cookies = copy._cookies = SimpleCookie() + for k,v in self._cookies.items(): + cookies[k] = v.value + cookies[k].update(v) # also copy cookie attributes + return copy + + def __iter__(self): + return iter(self.body) + + def close(self): + if hasattr(self.body, 'close'): + self.body.close() + + @property + def status_line(self): + """ The HTTP status line as a string (e.g. ``404 Not Found``).""" + return self._status_line + + @property + def status_code(self): + """ The HTTP status code as an integer (e.g. 404).""" + return self._status_code + + def _set_status(self, status): + if isinstance(status, int): + code, status = status, _HTTP_STATUS_LINES.get(status) + elif ' ' in status: + if '\n' in status or '\r' in status or '\0' in status: + raise ValueError('Status line must not include control chars.') + status = status.strip() + code = int(status.split()[0]) + else: + raise ValueError('String status line without a reason phrase.') + if not 100 <= code <= 999: + raise ValueError('Status code out of range.') + self._status_code = code + self._status_line = str(status or ('%d Unknown' % code)) + + def _get_status(self): + return self._status_line + + status = property( + _get_status, _set_status, None, + ''' A writeable property to change the HTTP response status. It accepts + either a numeric code (100-999) or a string with a custom reason + phrase (e.g. "404 Brain not found"). Both :data:`status_line` and + :data:`status_code` are updated accordingly. The return value is + always a status string. ''') + del _get_status, _set_status + + @property + def headers(self): + """ An instance of :class:`HeaderDict`, a case-insensitive dict-like + view on the response headers. """ + hdict = HeaderDict() + hdict.dict = self._headers + return hdict + + def __contains__(self, name): + return _hkey(name) in self._headers + + def __delitem__(self, name): + del self._headers[_hkey(name)] + + def __getitem__(self, name): + return self._headers[_hkey(name)][-1] + + def __setitem__(self, name, value): + self._headers[_hkey(name)] = [_hval(value)] + + def get_header(self, name, default=None): + """ Return the value of a previously defined header. If there is no + header with that name, return a default value. """ + return self._headers.get(_hkey(name), [default])[-1] + + def set_header(self, name, value): + """ Create a new response header, replacing any previously defined + headers with the same name. """ + self._headers[_hkey(name)] = [_hval(value)] + + def add_header(self, name, value): + """ Add an additional response header, not removing duplicates. """ + self._headers.setdefault(_hkey(name), []).append(_hval(value)) + + def iter_headers(self): + """ Yield (header, value) tuples, skipping headers that are not + allowed with the current response status code. """ + return self.headerlist + + def _wsgi_status_line(self): + """ WSGI conform status line (latin1-encodeable) """ + if py3k: + return self._status_line.encode('utf8').decode('latin1') + return self._status_line + + @property + def headerlist(self): + """ WSGI conform list of (header, value) tuples. """ + out = [] + headers = list(self._headers.items()) + if 'Content-Type' not in self._headers: + headers.append(('Content-Type', [self.default_content_type])) + if self._status_code in self.bad_headers: + bad_headers = self.bad_headers[self._status_code] + headers = [h for h in headers if h[0] not in bad_headers] + out += [(name, val) for (name, vals) in headers for val in vals] + if self._cookies: + for c in self._cookies.values(): + out.append(('Set-Cookie', _hval(c.OutputString()))) + if py3k: + out = [(k, v.encode('utf8').decode('latin1')) for (k, v) in out] + return out + + content_type = HeaderProperty('Content-Type') + content_length = HeaderProperty('Content-Length', reader=int, default=-1) + expires = HeaderProperty( + 'Expires', + reader=lambda x: datetime.utcfromtimestamp(parse_date(x)), + writer=lambda x: http_date(x)) + + @property + def charset(self, default='UTF-8'): + """ Return the charset specified in the content-type header (default: utf8). """ + if 'charset=' in self.content_type: + return self.content_type.split('charset=')[-1].split(';')[0].strip() + return default + + def set_cookie(self, name, value, secret=None, digestmod=hashlib.sha256, **options): + """ Create a new cookie or replace an old one. If the `secret` parameter is + set, create a `Signed Cookie` (described below). + + :param name: the name of the cookie. + :param value: the value of the cookie. + :param secret: a signature key required for signed cookies. + + Additionally, this method accepts all RFC 2109 attributes that are + supported by :class:`cookie.Morsel`, including: + + :param maxage: maximum age in seconds. (default: None) + :param expires: a datetime object or UNIX timestamp. (default: None) + :param domain: the domain that is allowed to read the cookie. + (default: current domain) + :param path: limits the cookie to a given path (default: current path) + :param secure: limit the cookie to HTTPS connections (default: off). + :param httponly: prevents client-side javascript to read this cookie + (default: off, requires Python 2.6 or newer). + :param samesite: Control or disable third-party use for this cookie. + Possible values: `lax`, `strict` or `none` (default). + + If neither `expires` nor `maxage` is set (default), the cookie will + expire at the end of the browser session (as soon as the browser + window is closed). + + Signed cookies may store any pickle-able object and are + cryptographically signed to prevent manipulation. Keep in mind that + cookies are limited to 4kb in most browsers. + + Warning: Pickle is a potentially dangerous format. If an attacker + gains access to the secret key, he could forge cookies that execute + code on server side if unpickled. Using pickle is discouraged and + support for it will be removed in later versions of bottle. + + Warning: Signed cookies are not encrypted (the client can still see + the content) and not copy-protected (the client can restore an old + cookie). The main intention is to make pickling and unpickling + save, not to store secret information at client side. + """ + if not self._cookies: + self._cookies = SimpleCookie() + + # Monkey-patch Cookie lib to support 'SameSite' parameter + # https://tools.ietf.org/html/draft-west-first-party-cookies-07#section-4.1 + if py < (3, 8, 0): + Morsel._reserved.setdefault('samesite', 'SameSite') + + if secret: + if not isinstance(value, basestring): + depr(0, 13, "Pickling of arbitrary objects into cookies is " + "deprecated.", "Only store strings in cookies. " + "JSON strings are fine, too.") + encoded = base64.b64encode(pickle.dumps([name, value], -1)) + sig = base64.b64encode(hmac.new(tob(secret), encoded, + digestmod=digestmod).digest()) + value = touni(tob('!') + sig + tob('?') + encoded) + elif not isinstance(value, basestring): + raise TypeError('Secret key required for non-string cookies.') + + # Cookie size plus options must not exceed 4kb. + if len(name) + len(value) > 3800: + raise ValueError('Content does not fit into a cookie.') + + self._cookies[name] = value + + for key, value in options.items(): + if key in ('max_age', 'maxage'): # 'maxage' variant added in 0.13 + key = 'max-age' + if isinstance(value, timedelta): + value = value.seconds + value.days * 24 * 3600 + if key == 'expires': + value = http_date(value) + if key in ('same_site', 'samesite'): # 'samesite' variant added in 0.13 + key, value = 'samesite', (value or "none").lower() + if value not in ('lax', 'strict', 'none'): + raise CookieError("Invalid value for SameSite") + if key in ('secure', 'httponly') and not value: + continue + self._cookies[name][key] = value + + def delete_cookie(self, key, **kwargs): + """ Delete a cookie. Be sure to use the same `domain` and `path` + settings as used to create the cookie. """ + kwargs['max_age'] = -1 + kwargs['expires'] = 0 + self.set_cookie(key, '', **kwargs) + + def __repr__(self): + out = '' + for name, value in self.headerlist: + out += '%s: %s\n' % (name.title(), value.strip()) + return out + + +def _local_property(): + ls = threading.local() + + def fget(_): + try: + return ls.var + except AttributeError: + raise RuntimeError("Request context not initialized.") + + def fset(_, value): + ls.var = value + + def fdel(_): + del ls.var + + return property(fget, fset, fdel, 'Thread-local property') + + +class LocalRequest(BaseRequest): + """ A thread-local subclass of :class:`BaseRequest` with a different + set of attributes for each thread. There is usually only one global + instance of this class (:data:`request`). If accessed during a + request/response cycle, this instance always refers to the *current* + request (even on a multithreaded server). """ + bind = BaseRequest.__init__ + environ = _local_property() + + +class LocalResponse(BaseResponse): + """ A thread-local subclass of :class:`BaseResponse` with a different + set of attributes for each thread. There is usually only one global + instance of this class (:data:`response`). Its attributes are used + to build the HTTP response at the end of the request/response cycle. + """ + bind = BaseResponse.__init__ + _status_line = _local_property() + _status_code = _local_property() + _cookies = _local_property() + _headers = _local_property() + body = _local_property() + + +Request = BaseRequest +Response = BaseResponse + + +class HTTPResponse(Response, BottleException): + def __init__(self, body='', status=None, headers=None, **more_headers): + super(HTTPResponse, self).__init__(body, status, headers, **more_headers) + + def apply(self, other): + other._status_code = self._status_code + other._status_line = self._status_line + other._headers = self._headers + other._cookies = self._cookies + other.body = self.body + + +class HTTPError(HTTPResponse): + default_status = 500 + + def __init__(self, + status=None, + body=None, + exception=None, + traceback=None, **more_headers): + self.exception = exception + self.traceback = traceback + super(HTTPError, self).__init__(body, status, **more_headers) + +############################################################################### +# Plugins ###################################################################### +############################################################################### + + +class PluginError(BottleException): + pass + + +class JSONPlugin(object): + name = 'json' + api = 2 + + def __init__(self, json_dumps=json_dumps): + self.json_dumps = json_dumps + + def setup(self, app): + app.config._define('json.enable', default=True, validate=bool, + help="Enable or disable automatic dict->json filter.") + app.config._define('json.ascii', default=False, validate=bool, + help="Use only 7-bit ASCII characters in output.") + app.config._define('json.indent', default=True, validate=bool, + help="Add whitespace to make json more readable.") + app.config._define('json.dump_func', default=None, + help="If defined, use this function to transform" + " dict into json. The other options no longer" + " apply.") + + def apply(self, callback, route): + dumps = self.json_dumps + if not self.json_dumps: return callback + + @functools.wraps(callback) + def wrapper(*a, **ka): + try: + rv = callback(*a, **ka) + except HTTPResponse as resp: + rv = resp + + if isinstance(rv, dict): + #Attempt to serialize, raises exception on failure + json_response = dumps(rv) + #Set content type only if serialization successful + response.content_type = 'application/json' + return json_response + elif isinstance(rv, HTTPResponse) and isinstance(rv.body, dict): + rv.body = dumps(rv.body) + rv.content_type = 'application/json' + return rv + + return wrapper + + +class TemplatePlugin(object): + """ This plugin applies the :func:`view` decorator to all routes with a + `template` config parameter. If the parameter is a tuple, the second + element must be a dict with additional options (e.g. `template_engine`) + or default variables for the template. """ + name = 'template' + api = 2 + + def setup(self, app): + app.tpl = self + + def apply(self, callback, route): + conf = route.config.get('template') + if isinstance(conf, (tuple, list)) and len(conf) == 2: + return view(conf[0], **conf[1])(callback) + elif isinstance(conf, str): + return view(conf)(callback) + else: + return callback + + +#: Not a plugin, but part of the plugin API. TODO: Find a better place. +class _ImportRedirect(object): + def __init__(self, name, impmask): + """ Create a virtual package that redirects imports (see PEP 302). """ + self.name = name + self.impmask = impmask + self.module = sys.modules.setdefault(name, new_module(name)) + self.module.__dict__.update({ + '__file__': __file__, + '__path__': [], + '__all__': [], + '__loader__': self + }) + sys.meta_path.append(self) + + def find_spec(self, fullname, path, target=None): + if '.' not in fullname: return + if fullname.rsplit('.', 1)[0] != self.name: return + from importlib.util import spec_from_loader + return spec_from_loader(fullname, self) + + def find_module(self, fullname, path=None): + if '.' not in fullname: return + if fullname.rsplit('.', 1)[0] != self.name: return + return self + + def load_module(self, fullname): + if fullname in sys.modules: return sys.modules[fullname] + modname = fullname.rsplit('.', 1)[1] + realname = self.impmask % modname + __import__(realname) + module = sys.modules[fullname] = sys.modules[realname] + setattr(self.module, modname, module) + module.__loader__ = self + return module + +############################################################################### +# Common Utilities ############################################################# +############################################################################### + + +class MultiDict(DictMixin): + """ This dict stores multiple values per key, but behaves exactly like a + normal dict in that it returns only the newest value for any given key. + There are special methods available to access the full list of values. + """ + + def __init__(self, *a, **k): + self.dict = dict((k, [v]) for (k, v) in dict(*a, **k).items()) + + def __len__(self): + return len(self.dict) + + def __iter__(self): + return iter(self.dict) + + def __contains__(self, key): + return key in self.dict + + def __delitem__(self, key): + del self.dict[key] + + def __getitem__(self, key): + return self.dict[key][-1] + + def __setitem__(self, key, value): + self.append(key, value) + + def keys(self): + return self.dict.keys() + + if py3k: + + def values(self): + return (v[-1] for v in self.dict.values()) + + def items(self): + return ((k, v[-1]) for k, v in self.dict.items()) + + def allitems(self): + return ((k, v) for k, vl in self.dict.items() for v in vl) + + iterkeys = keys + itervalues = values + iteritems = items + iterallitems = allitems + + else: + + def values(self): + return [v[-1] for v in self.dict.values()] + + def items(self): + return [(k, v[-1]) for k, v in self.dict.items()] + + def iterkeys(self): + return self.dict.iterkeys() + + def itervalues(self): + return (v[-1] for v in self.dict.itervalues()) + + def iteritems(self): + return ((k, v[-1]) for k, v in self.dict.iteritems()) + + def iterallitems(self): + return ((k, v) for k, vl in self.dict.iteritems() for v in vl) + + def allitems(self): + return [(k, v) for k, vl in self.dict.iteritems() for v in vl] + + def get(self, key, default=None, index=-1, type=None): + """ Return the most recent value for a key. + + :param default: The default value to be returned if the key is not + present or the type conversion fails. + :param index: An index for the list of available values. + :param type: If defined, this callable is used to cast the value + into a specific type. Exception are suppressed and result in + the default value to be returned. + """ + try: + val = self.dict[key][index] + return type(val) if type else val + except Exception: + pass + return default + + def append(self, key, value): + """ Add a new value to the list of values for this key. """ + self.dict.setdefault(key, []).append(value) + + def replace(self, key, value): + """ Replace the list of values with a single value. """ + self.dict[key] = [value] + + def getall(self, key): + """ Return a (possibly empty) list of values for a key. """ + return self.dict.get(key) or [] + + #: Aliases for WTForms to mimic other multi-dict APIs (Django) + getone = get + getlist = getall + + +class FormsDict(MultiDict): + """ This :class:`MultiDict` subclass is used to store request form data. + Additionally to the normal dict-like item access methods (which return + unmodified data as native strings), this container also supports + attribute-like access to its values. Attributes are automatically de- + or recoded to match :attr:`input_encoding` (default: 'utf8'). Missing + attributes default to an empty string. """ + + #: Encoding used for attribute values. + input_encoding = 'utf8' + #: If true (default), unicode strings are first encoded with `latin1` + #: and then decoded to match :attr:`input_encoding`. + recode_unicode = True + + def _fix(self, s, encoding=None): + if isinstance(s, unicode) and self.recode_unicode: # Python 3 WSGI + return s.encode('latin1').decode(encoding or self.input_encoding) + elif isinstance(s, bytes): # Python 2 WSGI + return s.decode(encoding or self.input_encoding) + else: + return s + + def decode(self, encoding=None): + """ Returns a copy with all keys and values de- or recoded to match + :attr:`input_encoding`. Some libraries (e.g. WTForms) want a + unicode dictionary. """ + copy = FormsDict() + enc = copy.input_encoding = encoding or self.input_encoding + copy.recode_unicode = False + for key, value in self.allitems(): + copy.append(self._fix(key, enc), self._fix(value, enc)) + return copy + + def getunicode(self, name, default=None, encoding=None): + """ Return the value as a unicode string, or the default. """ + try: + return self._fix(self[name], encoding) + except (UnicodeError, KeyError): + return default + + def __getattr__(self, name, default=unicode()): + # Without this guard, pickle generates a cryptic TypeError: + if name.startswith('__') and name.endswith('__'): + return super(FormsDict, self).__getattr__(name) + return self.getunicode(name, default=default) + +class HeaderDict(MultiDict): + """ A case-insensitive version of :class:`MultiDict` that defaults to + replace the old value instead of appending it. """ + + def __init__(self, *a, **ka): + self.dict = {} + if a or ka: self.update(*a, **ka) + + def __contains__(self, key): + return _hkey(key) in self.dict + + def __delitem__(self, key): + del self.dict[_hkey(key)] + + def __getitem__(self, key): + return self.dict[_hkey(key)][-1] + + def __setitem__(self, key, value): + self.dict[_hkey(key)] = [_hval(value)] + + def append(self, key, value): + self.dict.setdefault(_hkey(key), []).append(_hval(value)) + + def replace(self, key, value): + self.dict[_hkey(key)] = [_hval(value)] + + def getall(self, key): + return self.dict.get(_hkey(key)) or [] + + def get(self, key, default=None, index=-1): + return MultiDict.get(self, _hkey(key), default, index) + + def filter(self, names): + for name in (_hkey(n) for n in names): + if name in self.dict: + del self.dict[name] + + +class WSGIHeaderDict(DictMixin): + """ This dict-like class wraps a WSGI environ dict and provides convenient + access to HTTP_* fields. Keys and values are native strings + (2.x bytes or 3.x unicode) and keys are case-insensitive. If the WSGI + environment contains non-native string values, these are de- or encoded + using a lossless 'latin1' character set. + + The API will remain stable even on changes to the relevant PEPs. + Currently PEP 333, 444 and 3333 are supported. (PEP 444 is the only one + that uses non-native strings.) + """ + #: List of keys that do not have a ``HTTP_`` prefix. + cgikeys = ('CONTENT_TYPE', 'CONTENT_LENGTH') + + def __init__(self, environ): + self.environ = environ + + def _ekey(self, key): + """ Translate header field name to CGI/WSGI environ key. """ + key = key.replace('-', '_').upper() + if key in self.cgikeys: + return key + return 'HTTP_' + key + + def raw(self, key, default=None): + """ Return the header value as is (may be bytes or unicode). """ + return self.environ.get(self._ekey(key), default) + + def __getitem__(self, key): + val = self.environ[self._ekey(key)] + if py3k: + if isinstance(val, unicode): + val = val.encode('latin1').decode('utf8') + else: + val = val.decode('utf8') + return val + + def __setitem__(self, key, value): + raise TypeError("%s is read-only." % self.__class__) + + def __delitem__(self, key): + raise TypeError("%s is read-only." % self.__class__) + + def __iter__(self): + for key in self.environ: + if key[:5] == 'HTTP_': + yield _hkey(key[5:]) + elif key in self.cgikeys: + yield _hkey(key) + + def keys(self): + return [x for x in self] + + def __len__(self): + return len(self.keys()) + + def __contains__(self, key): + return self._ekey(key) in self.environ + +_UNSET = object() + +class ConfigDict(dict): + """ A dict-like configuration storage with additional support for + namespaces, validators, meta-data, overlays and more. + + This dict-like class is heavily optimized for read access. All read-only + methods as well as item access should be as fast as the built-in dict. + """ + + __slots__ = ('_meta', '_change_listener', '_overlays', '_virtual_keys', '_source', '__weakref__') + + def __init__(self): + self._meta = {} + self._change_listener = [] + #: Weak references of overlays that need to be kept in sync. + self._overlays = [] + #: Config that is the source for this overlay. + self._source = None + #: Keys of values copied from the source (values we do not own) + self._virtual_keys = set() + + def load_module(self, path, squash=True): + """Load values from a Python module. + + Example modue ``config.py``:: + + DEBUG = True + SQLITE = { + "db": ":memory:" + } + + + >>> c = ConfigDict() + >>> c.load_module('config') + {DEBUG: True, 'SQLITE.DB': 'memory'} + >>> c.load_module("config", False) + {'DEBUG': True, 'SQLITE': {'DB': 'memory'}} + + :param squash: If true (default), dictionary values are assumed to + represent namespaces (see :meth:`load_dict`). + """ + config_obj = load(path) + obj = {key: getattr(config_obj, key) for key in dir(config_obj) + if key.isupper()} + + if squash: + self.load_dict(obj) + else: + self.update(obj) + return self + + def load_config(self, filename, **options): + """ Load values from an ``*.ini`` style config file. + + A configuration file consists of sections, each led by a + ``[section]`` header, followed by key/value entries separated by + either ``=`` or ``:``. Section names and keys are case-insensitive. + Leading and trailing whitespace is removed from keys and values. + Values can be omitted, in which case the key/value delimiter may + also be left out. Values can also span multiple lines, as long as + they are indented deeper than the first line of the value. Commands + are prefixed by ``#`` or ``;`` and may only appear on their own on + an otherwise empty line. + + Both section and key names may contain dots (``.``) as namespace + separators. The actual configuration parameter name is constructed + by joining section name and key name together and converting to + lower case. + + The special sections ``bottle`` and ``ROOT`` refer to the root + namespace and the ``DEFAULT`` section defines default values for all + other sections. + + With Python 3, extended string interpolation is enabled. + + :param filename: The path of a config file, or a list of paths. + :param options: All keyword parameters are passed to the underlying + :class:`python:configparser.ConfigParser` constructor call. + + """ + options.setdefault('allow_no_value', True) + if py3k: + options.setdefault('interpolation', + configparser.ExtendedInterpolation()) + conf = configparser.ConfigParser(**options) + conf.read(filename) + for section in conf.sections(): + for key in conf.options(section): + value = conf.get(section, key) + if section not in ('bottle', 'ROOT'): + key = section + '.' + key + self[key.lower()] = value + return self + + def load_dict(self, source, namespace=''): + """ Load values from a dictionary structure. Nesting can be used to + represent namespaces. + + >>> c = ConfigDict() + >>> c.load_dict({'some': {'namespace': {'key': 'value'} } }) + {'some.namespace.key': 'value'} + """ + for key, value in source.items(): + if isinstance(key, basestring): + nskey = (namespace + '.' + key).strip('.') + if isinstance(value, dict): + self.load_dict(value, namespace=nskey) + else: + self[nskey] = value + else: + raise TypeError('Key has type %r (not a string)' % type(key)) + return self + + def update(self, *a, **ka): + """ If the first parameter is a string, all keys are prefixed with this + namespace. Apart from that it works just as the usual dict.update(). + + >>> c = ConfigDict() + >>> c.update('some.namespace', key='value') + """ + prefix = '' + if a and isinstance(a[0], basestring): + prefix = a[0].strip('.') + '.' + a = a[1:] + for key, value in dict(*a, **ka).items(): + self[prefix + key] = value + + def setdefault(self, key, value): + if key not in self: + self[key] = value + return self[key] + + def __setitem__(self, key, value): + if not isinstance(key, basestring): + raise TypeError('Key has type %r (not a string)' % type(key)) + + self._virtual_keys.discard(key) + + value = self.meta_get(key, 'filter', lambda x: x)(value) + if key in self and self[key] is value: + return + + self._on_change(key, value) + dict.__setitem__(self, key, value) + + for overlay in self._iter_overlays(): + overlay._set_virtual(key, value) + + def __delitem__(self, key): + if key not in self: + raise KeyError(key) + if key in self._virtual_keys: + raise KeyError("Virtual keys cannot be deleted: %s" % key) + + if self._source and key in self._source: + # Not virtual, but present in source -> Restore virtual value + dict.__delitem__(self, key) + self._set_virtual(key, self._source[key]) + else: # not virtual, not present in source. This is OUR value + self._on_change(key, None) + dict.__delitem__(self, key) + for overlay in self._iter_overlays(): + overlay._delete_virtual(key) + + def _set_virtual(self, key, value): + """ Recursively set or update virtual keys. Do nothing if non-virtual + value is present. """ + if key in self and key not in self._virtual_keys: + return # Do nothing for non-virtual keys. + + self._virtual_keys.add(key) + if key in self and self[key] is not value: + self._on_change(key, value) + dict.__setitem__(self, key, value) + for overlay in self._iter_overlays(): + overlay._set_virtual(key, value) + + def _delete_virtual(self, key): + """ Recursively delete virtual entry. Do nothing if key is not virtual. + """ + if key not in self._virtual_keys: + return # Do nothing for non-virtual keys. + + if key in self: + self._on_change(key, None) + dict.__delitem__(self, key) + self._virtual_keys.discard(key) + for overlay in self._iter_overlays(): + overlay._delete_virtual(key) + + def _on_change(self, key, value): + for cb in self._change_listener: + if cb(self, key, value): + return True + + def _add_change_listener(self, func): + self._change_listener.append(func) + return func + + def meta_get(self, key, metafield, default=None): + """ Return the value of a meta field for a key. """ + return self._meta.get(key, {}).get(metafield, default) + + def meta_set(self, key, metafield, value): + """ Set the meta field for a key to a new value. """ + self._meta.setdefault(key, {})[metafield] = value + + def meta_list(self, key): + """ Return an iterable of meta field names defined for a key. """ + return self._meta.get(key, {}).keys() + + def _define(self, key, default=_UNSET, help=_UNSET, validate=_UNSET): + """ (Unstable) Shortcut for plugins to define own config parameters. """ + if default is not _UNSET: + self.setdefault(key, default) + if help is not _UNSET: + self.meta_set(key, 'help', help) + if validate is not _UNSET: + self.meta_set(key, 'validate', validate) + + def _iter_overlays(self): + for ref in self._overlays: + overlay = ref() + if overlay is not None: + yield overlay + + def _make_overlay(self): + """ (Unstable) Create a new overlay that acts like a chained map: Values + missing in the overlay are copied from the source map. Both maps + share the same meta entries. + + Entries that were copied from the source are called 'virtual'. You + can not delete virtual keys, but overwrite them, which turns them + into non-virtual entries. Setting keys on an overlay never affects + its source, but may affect any number of child overlays. + + Other than collections.ChainMap or most other implementations, this + approach does not resolve missing keys on demand, but instead + actively copies all values from the source to the overlay and keeps + track of virtual and non-virtual keys internally. This removes any + lookup-overhead. Read-access is as fast as a build-in dict for both + virtual and non-virtual keys. + + Changes are propagated recursively and depth-first. A failing + on-change handler in an overlay stops the propagation of virtual + values and may result in an partly updated tree. Take extra care + here and make sure that on-change handlers never fail. + + Used by Route.config + """ + # Cleanup dead references + self._overlays[:] = [ref for ref in self._overlays if ref() is not None] + + overlay = ConfigDict() + overlay._meta = self._meta + overlay._source = self + self._overlays.append(weakref.ref(overlay)) + for key in self: + overlay._set_virtual(key, self[key]) + return overlay + + + + +class AppStack(list): + """ A stack-like list. Calling it returns the head of the stack. """ + + def __call__(self): + """ Return the current default application. """ + return self.default + + def push(self, value=None): + """ Add a new :class:`Bottle` instance to the stack """ + if not isinstance(value, Bottle): + value = Bottle() + self.append(value) + return value + new_app = push + + @property + def default(self): + try: + return self[-1] + except IndexError: + return self.push() + + +class WSGIFileWrapper(object): + def __init__(self, fp, buffer_size=1024 * 64): + self.fp, self.buffer_size = fp, buffer_size + for attr in 'fileno', 'close', 'read', 'readlines', 'tell', 'seek': + if hasattr(fp, attr): setattr(self, attr, getattr(fp, attr)) + + def __iter__(self): + buff, read = self.buffer_size, self.read + part = read(buff) + while part: + yield part + part = read(buff) + + +class _closeiter(object): + """ This only exists to be able to attach a .close method to iterators that + do not support attribute assignment (most of itertools). """ + + def __init__(self, iterator, close=None): + self.iterator = iterator + self.close_callbacks = makelist(close) + + def __iter__(self): + return iter(self.iterator) + + def close(self): + for func in self.close_callbacks: + func() + + +class ResourceManager(object): + """ This class manages a list of search paths and helps to find and open + application-bound resources (files). + + :param base: default value for :meth:`add_path` calls. + :param opener: callable used to open resources. + :param cachemode: controls which lookups are cached. One of 'all', + 'found' or 'none'. + """ + + def __init__(self, base='./', opener=open, cachemode='all'): + self.opener = opener + self.base = base + self.cachemode = cachemode + + #: A list of search paths. See :meth:`add_path` for details. + self.path = [] + #: A cache for resolved paths. ``res.cache.clear()`` clears the cache. + self.cache = {} + + def add_path(self, path, base=None, index=None, create=False): + """ Add a new path to the list of search paths. Return False if the + path does not exist. + + :param path: The new search path. Relative paths are turned into + an absolute and normalized form. If the path looks like a file + (not ending in `/`), the filename is stripped off. + :param base: Path used to absolutize relative search paths. + Defaults to :attr:`base` which defaults to ``os.getcwd()``. + :param index: Position within the list of search paths. Defaults + to last index (appends to the list). + + The `base` parameter makes it easy to reference files installed + along with a python module or package:: + + res.add_path('./resources/', __file__) + """ + base = os.path.abspath(os.path.dirname(base or self.base)) + path = os.path.abspath(os.path.join(base, os.path.dirname(path))) + path += os.sep + if path in self.path: + self.path.remove(path) + if create and not os.path.isdir(path): + os.makedirs(path) + if index is None: + self.path.append(path) + else: + self.path.insert(index, path) + self.cache.clear() + return os.path.exists(path) + + def __iter__(self): + """ Iterate over all existing files in all registered paths. """ + search = self.path[:] + while search: + path = search.pop() + if not os.path.isdir(path): continue + for name in os.listdir(path): + full = os.path.join(path, name) + if os.path.isdir(full): search.append(full) + else: yield full + + def lookup(self, name): + """ Search for a resource and return an absolute file path, or `None`. + + The :attr:`path` list is searched in order. The first match is + returned. Symlinks are followed. The result is cached to speed up + future lookups. """ + if name not in self.cache or DEBUG: + for path in self.path: + fpath = os.path.join(path, name) + if os.path.isfile(fpath): + if self.cachemode in ('all', 'found'): + self.cache[name] = fpath + return fpath + if self.cachemode == 'all': + self.cache[name] = None + return self.cache[name] + + def open(self, name, mode='r', *args, **kwargs): + """ Find a resource and return a file object, or raise IOError. """ + fname = self.lookup(name) + if not fname: raise IOError("Resource %r not found." % name) + return self.opener(fname, mode=mode, *args, **kwargs) + + +class FileUpload(object): + def __init__(self, fileobj, name, filename, headers=None): + """ Wrapper for file uploads. """ + #: Open file(-like) object (BytesIO buffer or temporary file) + self.file = fileobj + #: Name of the upload form field + self.name = name + #: Raw filename as sent by the client (may contain unsafe characters) + self.raw_filename = filename + #: A :class:`HeaderDict` with additional headers (e.g. content-type) + self.headers = HeaderDict(headers) if headers else HeaderDict() + + content_type = HeaderProperty('Content-Type') + content_length = HeaderProperty('Content-Length', reader=int, default=-1) + + def get_header(self, name, default=None): + """ Return the value of a header within the multipart part. """ + return self.headers.get(name, default) + + @cached_property + def filename(self): + """ Name of the file on the client file system, but normalized to ensure + file system compatibility. An empty filename is returned as 'empty'. + + Only ASCII letters, digits, dashes, underscores and dots are + allowed in the final filename. Accents are removed, if possible. + Whitespace is replaced by a single dash. Leading or tailing dots + or dashes are removed. The filename is limited to 255 characters. + """ + fname = self.raw_filename + if not isinstance(fname, unicode): + fname = fname.decode('utf8', 'ignore') + fname = normalize('NFKD', fname) + fname = fname.encode('ASCII', 'ignore').decode('ASCII') + fname = os.path.basename(fname.replace('\\', os.path.sep)) + fname = re.sub(r'[^a-zA-Z0-9-_.\s]', '', fname).strip() + fname = re.sub(r'[-\s]+', '-', fname).strip('.-') + return fname[:255] or 'empty' + + def _copy_file(self, fp, chunk_size=2 ** 16): + read, write, offset = self.file.read, fp.write, self.file.tell() + while 1: + buf = read(chunk_size) + if not buf: break + write(buf) + self.file.seek(offset) + + def save(self, destination, overwrite=False, chunk_size=2 ** 16): + """ Save file to disk or copy its content to an open file(-like) object. + If *destination* is a directory, :attr:`filename` is added to the + path. Existing files are not overwritten by default (IOError). + + :param destination: File path, directory or file(-like) object. + :param overwrite: If True, replace existing files. (default: False) + :param chunk_size: Bytes to read at a time. (default: 64kb) + """ + if isinstance(destination, basestring): # Except file-likes here + if os.path.isdir(destination): + destination = os.path.join(destination, self.filename) + if not overwrite and os.path.exists(destination): + raise IOError('File exists.') + with open(destination, 'wb') as fp: + self._copy_file(fp, chunk_size) + else: + self._copy_file(destination, chunk_size) + +############################################################################### +# Application Helper ########################################################### +############################################################################### + + +def abort(code=500, text='Unknown Error.'): + """ Aborts execution and causes a HTTP error. """ + raise HTTPError(code, text) + + +def redirect(url, code=None): + """ Aborts execution and causes a 303 or 302 redirect, depending on + the HTTP protocol version. """ + if not code: + code = 303 if request.get('SERVER_PROTOCOL') == "HTTP/1.1" else 302 + res = response.copy(cls=HTTPResponse) + res.status = code + res.body = "" + res.set_header('Location', urljoin(request.url, url)) + raise res + + +def _rangeiter(fp, offset, limit, bufsize=1024 * 1024): + """ Yield chunks from a range in a file. """ + fp.seek(offset) + while limit > 0: + part = fp.read(min(limit, bufsize)) + if not part: + break + limit -= len(part) + yield part + + +def static_file(filename, root, + mimetype=True, + download=False, + charset='UTF-8', + etag=None, + headers=None): + """ Open a file in a safe way and return an instance of :exc:`HTTPResponse` + that can be sent back to the client. + + :param filename: Name or path of the file to send, relative to ``root``. + :param root: Root path for file lookups. Should be an absolute directory + path. + :param mimetype: Provide the content-type header (default: guess from + file extension) + :param download: If True, ask the browser to open a `Save as...` dialog + instead of opening the file with the associated program. You can + specify a custom filename as a string. If not specified, the + original filename is used (default: False). + :param charset: The charset for files with a ``text/*`` mime-type. + (default: UTF-8) + :param etag: Provide a pre-computed ETag header. If set to ``False``, + ETag handling is disabled. (default: auto-generate ETag header) + :param headers: Additional headers dict to add to the response. + + While checking user input is always a good idea, this function provides + additional protection against malicious ``filename`` parameters from + breaking out of the ``root`` directory and leaking sensitive information + to an attacker. + + Read-protected files or files outside of the ``root`` directory are + answered with ``403 Access Denied``. Missing files result in a + ``404 Not Found`` response. Conditional requests (``If-Modified-Since``, + ``If-None-Match``) are answered with ``304 Not Modified`` whenever + possible. ``HEAD`` and ``Range`` requests (used by download managers to + check or continue partial downloads) are also handled automatically. + + """ + + root = os.path.join(os.path.abspath(root), '') + filename = os.path.abspath(os.path.join(root, filename.strip('/\\'))) + headers = headers.copy() if headers else {} + + if not filename.startswith(root): + return HTTPError(403, "Access denied.") + if not os.path.exists(filename) or not os.path.isfile(filename): + return HTTPError(404, "File does not exist.") + if not os.access(filename, os.R_OK): + return HTTPError(403, "You do not have permission to access this file.") + + if mimetype is True: + if download and download is not True: + mimetype, encoding = mimetypes.guess_type(download) + else: + mimetype, encoding = mimetypes.guess_type(filename) + if encoding: + headers['Content-Encoding'] = encoding + + if mimetype: + if (mimetype[:5] == 'text/' or mimetype == 'application/javascript')\ + and charset and 'charset' not in mimetype: + mimetype += '; charset=%s' % charset + headers['Content-Type'] = mimetype + + if download: + download = os.path.basename(filename if download is True else download) + headers['Content-Disposition'] = 'attachment; filename="%s"' % download + + stats = os.stat(filename) + headers['Content-Length'] = clen = stats.st_size + headers['Last-Modified'] = email.utils.formatdate(stats.st_mtime, + usegmt=True) + headers['Date'] = email.utils.formatdate(time.time(), usegmt=True) + + getenv = request.environ.get + + if etag is None: + etag = '%d:%d:%d:%d:%s' % (stats.st_dev, stats.st_ino, stats.st_mtime, + clen, filename) + etag = hashlib.sha1(tob(etag)).hexdigest() + + if etag: + headers['ETag'] = etag + check = getenv('HTTP_IF_NONE_MATCH') + if check and check == etag: + return HTTPResponse(status=304, **headers) + + ims = getenv('HTTP_IF_MODIFIED_SINCE') + if ims: + ims = parse_date(ims.split(";")[0].strip()) + if ims is not None and ims >= int(stats.st_mtime): + return HTTPResponse(status=304, **headers) + + body = '' if request.method == 'HEAD' else open(filename, 'rb') + + headers["Accept-Ranges"] = "bytes" + range_header = getenv('HTTP_RANGE') + if range_header: + ranges = list(parse_range_header(range_header, clen)) + if not ranges: + return HTTPError(416, "Requested Range Not Satisfiable") + offset, end = ranges[0] + rlen = end - offset + headers["Content-Range"] = "bytes %d-%d/%d" % (offset, end - 1, clen) + headers["Content-Length"] = str(rlen) + if body: body = _closeiter(_rangeiter(body, offset, rlen), body.close) + return HTTPResponse(body, status=206, **headers) + return HTTPResponse(body, **headers) + +############################################################################### +# HTTP Utilities and MISC (TODO) ############################################### +############################################################################### + + +def debug(mode=True): + """ Change the debug level. + There is only one debug level supported at the moment.""" + global DEBUG + if mode: warnings.simplefilter('default') + DEBUG = bool(mode) + + +def http_date(value): + if isinstance(value, basestring): + return value + if isinstance(value, datetime): + # aware datetime.datetime is converted to UTC time + # naive datetime.datetime is treated as UTC time + value = value.utctimetuple() + elif isinstance(value, datedate): + # datetime.date is naive, and is treated as UTC time + value = value.timetuple() + if not isinstance(value, (int, float)): + # convert struct_time in UTC to UNIX timestamp + value = calendar.timegm(value) + return email.utils.formatdate(value, usegmt=True) + + +def parse_date(ims): + """ Parse rfc1123, rfc850 and asctime timestamps and return UTC epoch. """ + try: + ts = email.utils.parsedate_tz(ims) + return calendar.timegm(ts[:8] + (0, )) - (ts[9] or 0) + except (TypeError, ValueError, IndexError, OverflowError): + return None + + +def parse_auth(header): + """ Parse rfc2617 HTTP authentication header string (basic) and return (user,pass) tuple or None""" + try: + method, data = header.split(None, 1) + if method.lower() == 'basic': + user, pwd = touni(base64.b64decode(tob(data))).split(':', 1) + return user, pwd + except (KeyError, ValueError): + return None + + +def parse_range_header(header, maxlen=0): + """ Yield (start, end) ranges parsed from a HTTP Range header. Skip + unsatisfiable ranges. The end index is non-inclusive.""" + if not header or header[:6] != 'bytes=': return + ranges = [r.split('-', 1) for r in header[6:].split(',') if '-' in r] + for start, end in ranges: + try: + if not start: # bytes=-100 -> last 100 bytes + start, end = max(0, maxlen - int(end)), maxlen + elif not end: # bytes=100- -> all but the first 99 bytes + start, end = int(start), maxlen + else: # bytes=100-200 -> bytes 100-200 (inclusive) + start, end = int(start), min(int(end) + 1, maxlen) + if 0 <= start < end <= maxlen: + yield start, end + except ValueError: + pass + + +#: Header tokenizer used by _parse_http_header() +_hsplit = re.compile('(?:(?:"((?:[^"\\\\]|\\\\.)*)")|([^;,=]+))([;,=]?)').findall + +def _parse_http_header(h): + """ Parses a typical multi-valued and parametrised HTTP header (e.g. Accept headers) and returns a list of values + and parameters. For non-standard or broken input, this implementation may return partial results. + :param h: A header string (e.g. ``text/html,text/plain;q=0.9,*/*;q=0.8``) + :return: List of (value, params) tuples. The second element is a (possibly empty) dict. + """ + values = [] + if '"' not in h: # INFO: Fast path without regexp (~2x faster) + for value in h.split(','): + parts = value.split(';') + values.append((parts[0].strip(), {})) + for attr in parts[1:]: + name, value = attr.split('=', 1) + values[-1][1][name.strip()] = value.strip() + else: + lop, key, attrs = ',', None, {} + for quoted, plain, tok in _hsplit(h): + value = plain.strip() if plain else quoted.replace('\\"', '"') + if lop == ',': + attrs = {} + values.append((value, attrs)) + elif lop == ';': + if tok == '=': + key = value + else: + attrs[value] = '' + elif lop == '=' and key: + attrs[key] = value + key = None + lop = tok + return values + + +def _parse_qsl(qs): + r = [] + for pair in qs.split('&'): + if not pair: continue + nv = pair.split('=', 1) + if len(nv) != 2: nv.append('') + key = urlunquote(nv[0].replace('+', ' ')) + value = urlunquote(nv[1].replace('+', ' ')) + r.append((key, value)) + return r + + +def _lscmp(a, b): + """ Compares two strings in a cryptographically safe way: + Runtime is not affected by length of common prefix. """ + return not sum(0 if x == y else 1 + for x, y in zip(a, b)) and len(a) == len(b) + + +def cookie_encode(data, key, digestmod=None): + """ Encode and sign a pickle-able object. Return a (byte) string """ + depr(0, 13, "cookie_encode() will be removed soon.", + "Do not use this API directly.") + digestmod = digestmod or hashlib.sha256 + msg = base64.b64encode(pickle.dumps(data, -1)) + sig = base64.b64encode(hmac.new(tob(key), msg, digestmod=digestmod).digest()) + return tob('!') + sig + tob('?') + msg + + +def cookie_decode(data, key, digestmod=None): + """ Verify and decode an encoded string. Return an object or None.""" + depr(0, 13, "cookie_decode() will be removed soon.", + "Do not use this API directly.") + data = tob(data) + if cookie_is_encoded(data): + sig, msg = data.split(tob('?'), 1) + digestmod = digestmod or hashlib.sha256 + hashed = hmac.new(tob(key), msg, digestmod=digestmod).digest() + if _lscmp(sig[1:], base64.b64encode(hashed)): + return pickle.loads(base64.b64decode(msg)) + return None + + +def cookie_is_encoded(data): + """ Return True if the argument looks like a encoded cookie.""" + depr(0, 13, "cookie_is_encoded() will be removed soon.", + "Do not use this API directly.") + return bool(data.startswith(tob('!')) and tob('?') in data) + + +def html_escape(string): + """ Escape HTML special characters ``&<>`` and quotes ``'"``. """ + return string.replace('&', '&').replace('<', '<').replace('>', '>')\ + .replace('"', '"').replace("'", ''') + + +def html_quote(string): + """ Escape and quote a string to be used as an HTTP attribute.""" + return '"%s"' % html_escape(string).replace('\n', ' ')\ + .replace('\r', ' ').replace('\t', ' ') + + +def yieldroutes(func): + """ Return a generator for routes that match the signature (name, args) + of the func parameter. This may yield more than one route if the function + takes optional keyword arguments. The output is best described by example:: + + a() -> '/a' + b(x, y) -> '/b//' + c(x, y=5) -> '/c/' and '/c//' + d(x=5, y=6) -> '/d' and '/d/' and '/d//' + """ + path = '/' + func.__name__.replace('__', '/').lstrip('/') + spec = getargspec(func) + argc = len(spec[0]) - len(spec[3] or []) + path += ('/<%s>' * argc) % tuple(spec[0][:argc]) + yield path + for arg in spec[0][argc:]: + path += '/<%s>' % arg + yield path + + +def path_shift(script_name, path_info, shift=1): + """ Shift path fragments from PATH_INFO to SCRIPT_NAME and vice versa. + + :return: The modified paths. + :param script_name: The SCRIPT_NAME path. + :param script_name: The PATH_INFO path. + :param shift: The number of path fragments to shift. May be negative to + change the shift direction. (default: 1) + """ + if shift == 0: return script_name, path_info + pathlist = path_info.strip('/').split('/') + scriptlist = script_name.strip('/').split('/') + if pathlist and pathlist[0] == '': pathlist = [] + if scriptlist and scriptlist[0] == '': scriptlist = [] + if 0 < shift <= len(pathlist): + moved = pathlist[:shift] + scriptlist = scriptlist + moved + pathlist = pathlist[shift:] + elif 0 > shift >= -len(scriptlist): + moved = scriptlist[shift:] + pathlist = moved + pathlist + scriptlist = scriptlist[:shift] + else: + empty = 'SCRIPT_NAME' if shift < 0 else 'PATH_INFO' + raise AssertionError("Cannot shift. Nothing left from %s" % empty) + new_script_name = '/' + '/'.join(scriptlist) + new_path_info = '/' + '/'.join(pathlist) + if path_info.endswith('/') and pathlist: new_path_info += '/' + return new_script_name, new_path_info + + +def auth_basic(check, realm="private", text="Access denied"): + """ Callback decorator to require HTTP auth (basic). + TODO: Add route(check_auth=...) parameter. """ + + def decorator(func): + + @functools.wraps(func) + def wrapper(*a, **ka): + user, password = request.auth or (None, None) + if user is None or not check(user, password): + err = HTTPError(401, text) + err.add_header('WWW-Authenticate', 'Basic realm="%s"' % realm) + return err + return func(*a, **ka) + + return wrapper + + return decorator + +# Shortcuts for common Bottle methods. +# They all refer to the current default application. + + +def make_default_app_wrapper(name): + """ Return a callable that relays calls to the current default app. """ + + @functools.wraps(getattr(Bottle, name)) + def wrapper(*a, **ka): + return getattr(app(), name)(*a, **ka) + + return wrapper + + +route = make_default_app_wrapper('route') +get = make_default_app_wrapper('get') +post = make_default_app_wrapper('post') +put = make_default_app_wrapper('put') +delete = make_default_app_wrapper('delete') +patch = make_default_app_wrapper('patch') +error = make_default_app_wrapper('error') +mount = make_default_app_wrapper('mount') +hook = make_default_app_wrapper('hook') +install = make_default_app_wrapper('install') +uninstall = make_default_app_wrapper('uninstall') +url = make_default_app_wrapper('get_url') + +############################################################################### +# Server Adapter ############################################################### +############################################################################### + +# Before you edit or add a server adapter, please read: +# - https://github.com/bottlepy/bottle/pull/647#issuecomment-60152870 +# - https://github.com/bottlepy/bottle/pull/865#issuecomment-242795341 + +class ServerAdapter(object): + quiet = False + + def __init__(self, host='127.0.0.1', port=8080, **options): + self.options = options + self.host = host + self.port = int(port) + + def run(self, handler): # pragma: no cover + pass + + def __repr__(self): + args = ', '.join('%s=%s' % (k, repr(v)) + for k, v in self.options.items()) + return "%s(%s)" % (self.__class__.__name__, args) + + +class CGIServer(ServerAdapter): + quiet = True + + def run(self, handler): # pragma: no cover + from wsgiref.handlers import CGIHandler + + def fixed_environ(environ, start_response): + environ.setdefault('PATH_INFO', '') + return handler(environ, start_response) + + CGIHandler().run(fixed_environ) + + +class FlupFCGIServer(ServerAdapter): + def run(self, handler): # pragma: no cover + import flup.server.fcgi + self.options.setdefault('bindAddress', (self.host, self.port)) + flup.server.fcgi.WSGIServer(handler, **self.options).run() + + +class WSGIRefServer(ServerAdapter): + def run(self, app): # pragma: no cover + from wsgiref.simple_server import make_server + from wsgiref.simple_server import WSGIRequestHandler, WSGIServer + import socket + + class FixedHandler(WSGIRequestHandler): + def address_string(self): # Prevent reverse DNS lookups please. + return self.client_address[0] + + def log_request(*args, **kw): + if not self.quiet: + return WSGIRequestHandler.log_request(*args, **kw) + + handler_cls = self.options.get('handler_class', FixedHandler) + server_cls = self.options.get('server_class', WSGIServer) + + if ':' in self.host: # Fix wsgiref for IPv6 addresses. + if getattr(server_cls, 'address_family') == socket.AF_INET: + + class server_cls(server_cls): + address_family = socket.AF_INET6 + + self.srv = make_server(self.host, self.port, app, server_cls, + handler_cls) + self.port = self.srv.server_port # update port actual port (0 means random) + try: + self.srv.serve_forever() + except KeyboardInterrupt: + self.srv.server_close() # Prevent ResourceWarning: unclosed socket + raise + + +class CherryPyServer(ServerAdapter): + def run(self, handler): # pragma: no cover + depr(0, 13, "The wsgi server part of cherrypy was split into a new " + "project called 'cheroot'.", "Use the 'cheroot' server " + "adapter instead of cherrypy.") + from cherrypy import wsgiserver # This will fail for CherryPy >= 9 + + self.options['bind_addr'] = (self.host, self.port) + self.options['wsgi_app'] = handler + + certfile = self.options.get('certfile') + if certfile: + del self.options['certfile'] + keyfile = self.options.get('keyfile') + if keyfile: + del self.options['keyfile'] + + server = wsgiserver.CherryPyWSGIServer(**self.options) + if certfile: + server.ssl_certificate = certfile + if keyfile: + server.ssl_private_key = keyfile + + try: + server.start() + finally: + server.stop() + + +class CherootServer(ServerAdapter): + def run(self, handler): # pragma: no cover + from cheroot import wsgi + from cheroot.ssl import builtin + self.options['bind_addr'] = (self.host, self.port) + self.options['wsgi_app'] = handler + certfile = self.options.pop('certfile', None) + keyfile = self.options.pop('keyfile', None) + chainfile = self.options.pop('chainfile', None) + server = wsgi.Server(**self.options) + if certfile and keyfile: + server.ssl_adapter = builtin.BuiltinSSLAdapter( + certfile, keyfile, chainfile) + try: + server.start() + finally: + server.stop() + + +class WaitressServer(ServerAdapter): + def run(self, handler): + from waitress import serve + serve(handler, host=self.host, port=self.port, _quiet=self.quiet, **self.options) + + +class PasteServer(ServerAdapter): + def run(self, handler): # pragma: no cover + from paste import httpserver + from paste.translogger import TransLogger + handler = TransLogger(handler, setup_console_handler=(not self.quiet)) + httpserver.serve(handler, + host=self.host, + port=str(self.port), **self.options) + + +class MeinheldServer(ServerAdapter): + def run(self, handler): + from meinheld import server + server.listen((self.host, self.port)) + server.run(handler) + + +class FapwsServer(ServerAdapter): + """ Extremely fast webserver using libev. See https://github.com/william-os4y/fapws3 """ + + def run(self, handler): # pragma: no cover + depr(0, 13, "fapws3 is not maintained and support will be dropped.") + import fapws._evwsgi as evwsgi + from fapws import base, config + port = self.port + if float(config.SERVER_IDENT[-2:]) > 0.4: + # fapws3 silently changed its API in 0.5 + port = str(port) + evwsgi.start(self.host, port) + # fapws3 never releases the GIL. Complain upstream. I tried. No luck. + if 'BOTTLE_CHILD' in os.environ and not self.quiet: + _stderr("WARNING: Auto-reloading does not work with Fapws3.") + _stderr(" (Fapws3 breaks python thread support)") + evwsgi.set_base_module(base) + + def app(environ, start_response): + environ['wsgi.multiprocess'] = False + return handler(environ, start_response) + + evwsgi.wsgi_cb(('', app)) + evwsgi.run() + + +class TornadoServer(ServerAdapter): + """ The super hyped asynchronous server by facebook. Untested. """ + + def run(self, handler): # pragma: no cover + import tornado.wsgi, tornado.httpserver, tornado.ioloop + container = tornado.wsgi.WSGIContainer(handler) + server = tornado.httpserver.HTTPServer(container) + server.listen(port=self.port, address=self.host) + tornado.ioloop.IOLoop.instance().start() + + +class AppEngineServer(ServerAdapter): + """ Adapter for Google App Engine. """ + quiet = True + + def run(self, handler): + depr(0, 13, "AppEngineServer no longer required", + "Configure your application directly in your app.yaml") + from google.appengine.ext.webapp import util + # A main() function in the handler script enables 'App Caching'. + # Lets makes sure it is there. This _really_ improves performance. + module = sys.modules.get('__main__') + if module and not hasattr(module, 'main'): + module.main = lambda: util.run_wsgi_app(handler) + util.run_wsgi_app(handler) + + +class TwistedServer(ServerAdapter): + """ Untested. """ + + def run(self, handler): + from twisted.web import server, wsgi + from twisted.python.threadpool import ThreadPool + from twisted.internet import reactor + thread_pool = ThreadPool() + thread_pool.start() + reactor.addSystemEventTrigger('after', 'shutdown', thread_pool.stop) + factory = server.Site(wsgi.WSGIResource(reactor, thread_pool, handler)) + reactor.listenTCP(self.port, factory, interface=self.host) + if not reactor.running: + reactor.run() + + +class DieselServer(ServerAdapter): + """ Untested. """ + + def run(self, handler): + depr(0, 13, "Diesel is not tested or supported and will be removed.") + from diesel.protocols.wsgi import WSGIApplication + app = WSGIApplication(handler, port=self.port) + app.run() + + +class GeventServer(ServerAdapter): + """ Untested. Options: + + * See gevent.wsgi.WSGIServer() documentation for more options. + """ + + def run(self, handler): + from gevent import pywsgi, local + if not isinstance(threading.local(), local.local): + msg = "Bottle requires gevent.monkey.patch_all() (before import)" + raise RuntimeError(msg) + if self.quiet: + self.options['log'] = None + address = (self.host, self.port) + server = pywsgi.WSGIServer(address, handler, **self.options) + if 'BOTTLE_CHILD' in os.environ: + import signal + signal.signal(signal.SIGINT, lambda s, f: server.stop()) + server.serve_forever() + + +class GunicornServer(ServerAdapter): + """ Untested. See http://gunicorn.org/configure.html for options. """ + + def run(self, handler): + from gunicorn.app.base import BaseApplication + + if self.host.startswith("unix:"): + config = {'bind': self.host} + else: + config = {'bind': "%s:%d" % (self.host, self.port)} + + config.update(self.options) + + class GunicornApplication(BaseApplication): + def load_config(self): + for key, value in config.items(): + self.cfg.set(key, value) + + def load(self): + return handler + + GunicornApplication().run() + + +class EventletServer(ServerAdapter): + """ Untested. Options: + + * `backlog` adjust the eventlet backlog parameter which is the maximum + number of queued connections. Should be at least 1; the maximum + value is system-dependent. + * `family`: (default is 2) socket family, optional. See socket + documentation for available families. + """ + + def run(self, handler): + from eventlet import wsgi, listen, patcher + if not patcher.is_monkey_patched(os): + msg = "Bottle requires eventlet.monkey_patch() (before import)" + raise RuntimeError(msg) + socket_args = {} + for arg in ('backlog', 'family'): + try: + socket_args[arg] = self.options.pop(arg) + except KeyError: + pass + address = (self.host, self.port) + try: + wsgi.server(listen(address, **socket_args), handler, + log_output=(not self.quiet)) + except TypeError: + # Fallback, if we have old version of eventlet + wsgi.server(listen(address), handler) + + +class BjoernServer(ServerAdapter): + """ Fast server written in C: https://github.com/jonashaag/bjoern """ + + def run(self, handler): + from bjoern import run + run(handler, self.host, self.port, reuse_port=True) + +class AsyncioServerAdapter(ServerAdapter): + """ Extend ServerAdapter for adding custom event loop """ + def get_event_loop(self): + pass + +class AiohttpServer(AsyncioServerAdapter): + """ Asynchronous HTTP client/server framework for asyncio + https://pypi.python.org/pypi/aiohttp/ + https://pypi.org/project/aiohttp-wsgi/ + """ + + def get_event_loop(self): + import asyncio + return asyncio.new_event_loop() + + def run(self, handler): + import asyncio + from aiohttp_wsgi.wsgi import serve + self.loop = self.get_event_loop() + asyncio.set_event_loop(self.loop) + + if 'BOTTLE_CHILD' in os.environ: + import signal + signal.signal(signal.SIGINT, lambda s, f: self.loop.stop()) + + serve(handler, host=self.host, port=self.port) + + +class AiohttpUVLoopServer(AiohttpServer): + """uvloop + https://github.com/MagicStack/uvloop + """ + def get_event_loop(self): + import uvloop + return uvloop.new_event_loop() + +class AutoServer(ServerAdapter): + """ Untested. """ + adapters = [WaitressServer, PasteServer, TwistedServer, CherryPyServer, + CherootServer, WSGIRefServer] + + def run(self, handler): + for sa in self.adapters: + try: + return sa(self.host, self.port, **self.options).run(handler) + except ImportError: + pass + + +server_names = { + 'cgi': CGIServer, + 'flup': FlupFCGIServer, + 'wsgiref': WSGIRefServer, + 'waitress': WaitressServer, + 'cherrypy': CherryPyServer, + 'cheroot': CherootServer, + 'paste': PasteServer, + 'fapws3': FapwsServer, + 'tornado': TornadoServer, + 'gae': AppEngineServer, + 'twisted': TwistedServer, + 'diesel': DieselServer, + 'meinheld': MeinheldServer, + 'gunicorn': GunicornServer, + 'eventlet': EventletServer, + 'gevent': GeventServer, + 'bjoern': BjoernServer, + 'aiohttp': AiohttpServer, + 'uvloop': AiohttpUVLoopServer, + 'auto': AutoServer, +} + +############################################################################### +# Application Control ########################################################## +############################################################################### + + +def load(target, **namespace): + """ Import a module or fetch an object from a module. + + * ``package.module`` returns `module` as a module object. + * ``pack.mod:name`` returns the module variable `name` from `pack.mod`. + * ``pack.mod:func()`` calls `pack.mod.func()` and returns the result. + + The last form accepts not only function calls, but any type of + expression. Keyword arguments passed to this function are available as + local variables. Example: ``import_string('re:compile(x)', x='[a-z]')`` + """ + module, target = target.split(":", 1) if ':' in target else (target, None) + if module not in sys.modules: __import__(module) + if not target: return sys.modules[module] + if target.isalnum(): return getattr(sys.modules[module], target) + package_name = module.split('.')[0] + namespace[package_name] = sys.modules[package_name] + return eval('%s.%s' % (module, target), namespace) + + +def load_app(target): + """ Load a bottle application from a module and make sure that the import + does not affect the current default application, but returns a separate + application object. See :func:`load` for the target parameter. """ + global NORUN + NORUN, nr_old = True, NORUN + tmp = default_app.push() # Create a new "default application" + try: + rv = load(target) # Import the target module + return rv if callable(rv) else tmp + finally: + default_app.remove(tmp) # Remove the temporary added default application + NORUN = nr_old + + +_debug = debug + + +def run(app=None, + server='wsgiref', + host='127.0.0.1', + port=8080, + interval=1, + reloader=False, + quiet=False, + plugins=None, + debug=None, + config=None, **kargs): + """ Start a server instance. This method blocks until the server terminates. + + :param app: WSGI application or target string supported by + :func:`load_app`. (default: :func:`default_app`) + :param server: Server adapter to use. See :data:`server_names` keys + for valid names or pass a :class:`ServerAdapter` subclass. + (default: `wsgiref`) + :param host: Server address to bind to. Pass ``0.0.0.0`` to listens on + all interfaces including the external one. (default: 127.0.0.1) + :param port: Server port to bind to. Values below 1024 require root + privileges. (default: 8080) + :param reloader: Start auto-reloading server? (default: False) + :param interval: Auto-reloader interval in seconds (default: 1) + :param quiet: Suppress output to stdout and stderr? (default: False) + :param options: Options passed to the server adapter. + """ + if NORUN: return + if reloader and not os.environ.get('BOTTLE_CHILD'): + import subprocess + fd, lockfile = tempfile.mkstemp(prefix='bottle.', suffix='.lock') + environ = os.environ.copy() + environ['BOTTLE_CHILD'] = 'true' + environ['BOTTLE_LOCKFILE'] = lockfile + args = [sys.executable] + sys.argv + # If a package was loaded with `python -m`, then `sys.argv` needs to be + # restored to the original value, or imports might break. See #1336 + if getattr(sys.modules.get('__main__'), '__package__', None): + args[1:1] = ["-m", sys.modules['__main__'].__package__] + + try: + os.close(fd) # We never write to this file + while os.path.exists(lockfile): + p = subprocess.Popen(args, env=environ) + while p.poll() is None: + os.utime(lockfile, None) # Tell child we are still alive + time.sleep(interval) + if p.returncode == 3: # Child wants to be restarted + continue + sys.exit(p.returncode) + except KeyboardInterrupt: + pass + finally: + if os.path.exists(lockfile): + os.unlink(lockfile) + return + + try: + if debug is not None: _debug(debug) + app = app or default_app() + if isinstance(app, basestring): + app = load_app(app) + if not callable(app): + raise ValueError("Application is not callable: %r" % app) + + for plugin in plugins or []: + if isinstance(plugin, basestring): + plugin = load(plugin) + app.install(plugin) + + if config: + app.config.update(config) + + if server in server_names: + server = server_names.get(server) + if isinstance(server, basestring): + server = load(server) + if isinstance(server, type): + server = server(host=host, port=port, **kargs) + if not isinstance(server, ServerAdapter): + raise ValueError("Unknown or unsupported server: %r" % server) + + server.quiet = server.quiet or quiet + if not server.quiet: + _stderr("Bottle v%s server starting up (using %s)..." % + (__version__, repr(server))) + if server.host.startswith("unix:"): + _stderr("Listening on %s" % server.host) + else: + _stderr("Listening on http://%s:%d/" % + (server.host, server.port)) + _stderr("Hit Ctrl-C to quit.\n") + + if reloader: + lockfile = os.environ.get('BOTTLE_LOCKFILE') + bgcheck = FileCheckerThread(lockfile, interval) + with bgcheck: + server.run(app) + if bgcheck.status == 'reload': + sys.exit(3) + else: + server.run(app) + except KeyboardInterrupt: + pass + except (SystemExit, MemoryError): + raise + except: + if not reloader: raise + if not getattr(server, 'quiet', quiet): + print_exc() + time.sleep(interval) + sys.exit(3) + + +class FileCheckerThread(threading.Thread): + """ Interrupt main-thread as soon as a changed module file is detected, + the lockfile gets deleted or gets too old. """ + + def __init__(self, lockfile, interval): + threading.Thread.__init__(self) + self.daemon = True + self.lockfile, self.interval = lockfile, interval + #: Is one of 'reload', 'error' or 'exit' + self.status = None + + def run(self): + exists = os.path.exists + mtime = lambda p: os.stat(p).st_mtime + files = dict() + + for module in list(sys.modules.values()): + path = getattr(module, '__file__', '') or '' + if path[-4:] in ('.pyo', '.pyc'): path = path[:-1] + if path and exists(path): files[path] = mtime(path) + + while not self.status: + if not exists(self.lockfile)\ + or mtime(self.lockfile) < time.time() - self.interval - 5: + self.status = 'error' + thread.interrupt_main() + for path, lmtime in list(files.items()): + if not exists(path) or mtime(path) > lmtime: + self.status = 'reload' + thread.interrupt_main() + break + time.sleep(self.interval) + + def __enter__(self): + self.start() + + def __exit__(self, exc_type, *_): + if not self.status: self.status = 'exit' # silent exit + self.join() + return exc_type is not None and issubclass(exc_type, KeyboardInterrupt) + +############################################################################### +# Template Adapters ############################################################ +############################################################################### + + +class TemplateError(BottleException): + pass + + +class BaseTemplate(object): + """ Base class and minimal API for template adapters """ + extensions = ['tpl', 'html', 'thtml', 'stpl'] + settings = {} #used in prepare() + defaults = {} #used in render() + + def __init__(self, + source=None, + name=None, + lookup=None, + encoding='utf8', **settings): + """ Create a new template. + If the source parameter (str or buffer) is missing, the name argument + is used to guess a template filename. Subclasses can assume that + self.source and/or self.filename are set. Both are strings. + The lookup, encoding and settings parameters are stored as instance + variables. + The lookup parameter stores a list containing directory paths. + The encoding parameter should be used to decode byte strings or files. + The settings parameter contains a dict for engine-specific settings. + """ + self.name = name + self.source = source.read() if hasattr(source, 'read') else source + self.filename = source.filename if hasattr(source, 'filename') else None + self.lookup = [os.path.abspath(x) for x in lookup] if lookup else [] + self.encoding = encoding + self.settings = self.settings.copy() # Copy from class variable + self.settings.update(settings) # Apply + if not self.source and self.name: + self.filename = self.search(self.name, self.lookup) + if not self.filename: + raise TemplateError('Template %s not found.' % repr(name)) + if not self.source and not self.filename: + raise TemplateError('No template specified.') + self.prepare(**self.settings) + + @classmethod + def search(cls, name, lookup=None): + """ Search name in all directories specified in lookup. + First without, then with common extensions. Return first hit. """ + if not lookup: + raise depr(0, 12, "Empty template lookup path.", "Configure a template lookup path.") + + if os.path.isabs(name): + raise depr(0, 12, "Use of absolute path for template name.", + "Refer to templates with names or paths relative to the lookup path.") + + for spath in lookup: + spath = os.path.abspath(spath) + os.sep + fname = os.path.abspath(os.path.join(spath, name)) + if not fname.startswith(spath): continue + if os.path.isfile(fname): return fname + for ext in cls.extensions: + if os.path.isfile('%s.%s' % (fname, ext)): + return '%s.%s' % (fname, ext) + + @classmethod + def global_config(cls, key, *args): + """ This reads or sets the global settings stored in class.settings. """ + if args: + cls.settings = cls.settings.copy() # Make settings local to class + cls.settings[key] = args[0] + else: + return cls.settings[key] + + def prepare(self, **options): + """ Run preparations (parsing, caching, ...). + It should be possible to call this again to refresh a template or to + update settings. + """ + raise NotImplementedError + + def render(self, *args, **kwargs): + """ Render the template with the specified local variables and return + a single byte or unicode string. If it is a byte string, the encoding + must match self.encoding. This method must be thread-safe! + Local variables may be provided in dictionaries (args) + or directly, as keywords (kwargs). + """ + raise NotImplementedError + + +class MakoTemplate(BaseTemplate): + def prepare(self, **options): + from mako.template import Template + from mako.lookup import TemplateLookup + options.update({'input_encoding': self.encoding}) + options.setdefault('format_exceptions', bool(DEBUG)) + lookup = TemplateLookup(directories=self.lookup, **options) + if self.source: + self.tpl = Template(self.source, lookup=lookup, **options) + else: + self.tpl = Template(uri=self.name, + filename=self.filename, + lookup=lookup, **options) + + def render(self, *args, **kwargs): + for dictarg in args: + kwargs.update(dictarg) + _defaults = self.defaults.copy() + _defaults.update(kwargs) + return self.tpl.render(**_defaults) + + +class CheetahTemplate(BaseTemplate): + def prepare(self, **options): + from Cheetah.Template import Template + self.context = threading.local() + self.context.vars = {} + options['searchList'] = [self.context.vars] + if self.source: + self.tpl = Template(source=self.source, **options) + else: + self.tpl = Template(file=self.filename, **options) + + def render(self, *args, **kwargs): + for dictarg in args: + kwargs.update(dictarg) + self.context.vars.update(self.defaults) + self.context.vars.update(kwargs) + out = str(self.tpl) + self.context.vars.clear() + return out + + +class Jinja2Template(BaseTemplate): + def prepare(self, filters=None, tests=None, globals={}, **kwargs): + from jinja2 import Environment, FunctionLoader + self.env = Environment(loader=FunctionLoader(self.loader), **kwargs) + if filters: self.env.filters.update(filters) + if tests: self.env.tests.update(tests) + if globals: self.env.globals.update(globals) + if self.source: + self.tpl = self.env.from_string(self.source) + else: + self.tpl = self.env.get_template(self.name) + + def render(self, *args, **kwargs): + for dictarg in args: + kwargs.update(dictarg) + _defaults = self.defaults.copy() + _defaults.update(kwargs) + return self.tpl.render(**_defaults) + + def loader(self, name): + if name == self.filename: + fname = name + else: + fname = self.search(name, self.lookup) + if not fname: return + with open(fname, "rb") as f: + return (f.read().decode(self.encoding), fname, lambda: False) + + +class SimpleTemplate(BaseTemplate): + def prepare(self, + escape_func=html_escape, + noescape=False, + syntax=None, **ka): + self.cache = {} + enc = self.encoding + self._str = lambda x: touni(x, enc) + self._escape = lambda x: escape_func(touni(x, enc)) + self.syntax = syntax + if noescape: + self._str, self._escape = self._escape, self._str + + @cached_property + def co(self): + return compile(self.code, self.filename or '', 'exec') + + @cached_property + def code(self): + source = self.source + if not source: + with open(self.filename, 'rb') as f: + source = f.read() + try: + source, encoding = touni(source), 'utf8' + except UnicodeError: + raise depr(0, 11, 'Unsupported template encodings.', 'Use utf-8 for templates.') + parser = StplParser(source, encoding=encoding, syntax=self.syntax) + code = parser.translate() + self.encoding = parser.encoding + return code + + def _rebase(self, _env, _name=None, **kwargs): + _env['_rebase'] = (_name, kwargs) + + def _include(self, _env, _name=None, **kwargs): + env = _env.copy() + env.update(kwargs) + if _name not in self.cache: + self.cache[_name] = self.__class__(name=_name, lookup=self.lookup, syntax=self.syntax) + return self.cache[_name].execute(env['_stdout'], env) + + def execute(self, _stdout, kwargs): + env = self.defaults.copy() + env.update(kwargs) + env.update({ + '_stdout': _stdout, + '_printlist': _stdout.extend, + 'include': functools.partial(self._include, env), + 'rebase': functools.partial(self._rebase, env), + '_rebase': None, + '_str': self._str, + '_escape': self._escape, + 'get': env.get, + 'setdefault': env.setdefault, + 'defined': env.__contains__ + }) + exec(self.co, env) + if env.get('_rebase'): + subtpl, rargs = env.pop('_rebase') + rargs['base'] = ''.join(_stdout) #copy stdout + del _stdout[:] # clear stdout + return self._include(env, subtpl, **rargs) + return env + + def render(self, *args, **kwargs): + """ Render the template using keyword arguments as local variables. """ + env = {} + stdout = [] + for dictarg in args: + env.update(dictarg) + env.update(kwargs) + self.execute(stdout, env) + return ''.join(stdout) + + +class StplSyntaxError(TemplateError): + pass + + +class StplParser(object): + """ Parser for stpl templates. """ + _re_cache = {} #: Cache for compiled re patterns + + # This huge pile of voodoo magic splits python code into 8 different tokens. + # We use the verbose (?x) regex mode to make this more manageable + + _re_tok = r'''( + [urbURB]* + (?: ''(?!') + |""(?!") + |'{6} + |"{6} + |'(?:[^\\']|\\.)+?' + |"(?:[^\\"]|\\.)+?" + |'{3}(?:[^\\]|\\.|\n)+?'{3} + |"{3}(?:[^\\]|\\.|\n)+?"{3} + ) + )''' + + _re_inl = _re_tok.replace(r'|\n', '') # We re-use this string pattern later + + _re_tok += r''' + # 2: Comments (until end of line, but not the newline itself) + |(\#.*) + + # 3: Open and close (4) grouping tokens + |([\[\{\(]) + |([\]\}\)]) + + # 5,6: Keywords that start or continue a python block (only start of line) + |^([\ \t]*(?:if|for|while|with|try|def|class)\b) + |^([\ \t]*(?:elif|else|except|finally)\b) + + # 7: Our special 'end' keyword (but only if it stands alone) + |((?:^|;)[\ \t]*end[\ \t]*(?=(?:%(block_close)s[\ \t]*)?\r?$|;|\#)) + + # 8: A customizable end-of-code-block template token (only end of line) + |(%(block_close)s[\ \t]*(?=\r?$)) + + # 9: And finally, a single newline. The 10th token is 'everything else' + |(\r?\n) + ''' + + # Match the start tokens of code areas in a template + _re_split = r'''(?m)^[ \t]*(\\?)((%(line_start)s)|(%(block_start)s))''' + # Match inline statements (may contain python strings) + _re_inl = r'''%%(inline_start)s((?:%s|[^'"\n])*?)%%(inline_end)s''' % _re_inl + + # add the flag in front of the regexp to avoid Deprecation warning (see Issue #949) + # verbose and dot-matches-newline mode + _re_tok = '(?mx)' + _re_tok + _re_inl = '(?mx)' + _re_inl + + + default_syntax = '<% %> % {{ }}' + + def __init__(self, source, syntax=None, encoding='utf8'): + self.source, self.encoding = touni(source, encoding), encoding + self.set_syntax(syntax or self.default_syntax) + self.code_buffer, self.text_buffer = [], [] + self.lineno, self.offset = 1, 0 + self.indent, self.indent_mod = 0, 0 + self.paren_depth = 0 + + def get_syntax(self): + """ Tokens as a space separated string (default: <% %> % {{ }}) """ + return self._syntax + + def set_syntax(self, syntax): + self._syntax = syntax + self._tokens = syntax.split() + if syntax not in self._re_cache: + names = 'block_start block_close line_start inline_start inline_end' + etokens = map(re.escape, self._tokens) + pattern_vars = dict(zip(names.split(), etokens)) + patterns = (self._re_split, self._re_tok, self._re_inl) + patterns = [re.compile(p % pattern_vars) for p in patterns] + self._re_cache[syntax] = patterns + self.re_split, self.re_tok, self.re_inl = self._re_cache[syntax] + + syntax = property(get_syntax, set_syntax) + + def translate(self): + if self.offset: raise RuntimeError('Parser is a one time instance.') + while True: + m = self.re_split.search(self.source, pos=self.offset) + if m: + text = self.source[self.offset:m.start()] + self.text_buffer.append(text) + self.offset = m.end() + if m.group(1): # Escape syntax + line, sep, _ = self.source[self.offset:].partition('\n') + self.text_buffer.append(self.source[m.start():m.start(1)] + + m.group(2) + line + sep) + self.offset += len(line + sep) + continue + self.flush_text() + self.offset += self.read_code(self.source[self.offset:], + multiline=bool(m.group(4))) + else: + break + self.text_buffer.append(self.source[self.offset:]) + self.flush_text() + return ''.join(self.code_buffer) + + def read_code(self, pysource, multiline): + code_line, comment = '', '' + offset = 0 + while True: + m = self.re_tok.search(pysource, pos=offset) + if not m: + code_line += pysource[offset:] + offset = len(pysource) + self.write_code(code_line.strip(), comment) + break + code_line += pysource[offset:m.start()] + offset = m.end() + _str, _com, _po, _pc, _blk1, _blk2, _end, _cend, _nl = m.groups() + if self.paren_depth > 0 and (_blk1 or _blk2): # a if b else c + code_line += _blk1 or _blk2 + continue + if _str: # Python string + code_line += _str + elif _com: # Python comment (up to EOL) + comment = _com + if multiline and _com.strip().endswith(self._tokens[1]): + multiline = False # Allow end-of-block in comments + elif _po: # open parenthesis + self.paren_depth += 1 + code_line += _po + elif _pc: # close parenthesis + if self.paren_depth > 0: + # we could check for matching parentheses here, but it's + # easier to leave that to python - just check counts + self.paren_depth -= 1 + code_line += _pc + elif _blk1: # Start-block keyword (if/for/while/def/try/...) + code_line = _blk1 + self.indent += 1 + self.indent_mod -= 1 + elif _blk2: # Continue-block keyword (else/elif/except/...) + code_line = _blk2 + self.indent_mod -= 1 + elif _cend: # The end-code-block template token (usually '%>') + if multiline: multiline = False + else: code_line += _cend + elif _end: + self.indent -= 1 + self.indent_mod += 1 + else: # \n + self.write_code(code_line.strip(), comment) + self.lineno += 1 + code_line, comment, self.indent_mod = '', '', 0 + if not multiline: + break + + return offset + + def flush_text(self): + text = ''.join(self.text_buffer) + del self.text_buffer[:] + if not text: return + parts, pos, nl = [], 0, '\\\n' + ' ' * self.indent + for m in self.re_inl.finditer(text): + prefix, pos = text[pos:m.start()], m.end() + if prefix: + parts.append(nl.join(map(repr, prefix.splitlines(True)))) + if prefix.endswith('\n'): parts[-1] += nl + parts.append(self.process_inline(m.group(1).strip())) + if pos < len(text): + prefix = text[pos:] + lines = prefix.splitlines(True) + if lines[-1].endswith('\\\\\n'): lines[-1] = lines[-1][:-3] + elif lines[-1].endswith('\\\\\r\n'): lines[-1] = lines[-1][:-4] + parts.append(nl.join(map(repr, lines))) + code = '_printlist((%s,))' % ', '.join(parts) + self.lineno += code.count('\n') + 1 + self.write_code(code) + + @staticmethod + def process_inline(chunk): + if chunk[0] == '!': return '_str(%s)' % chunk[1:] + return '_escape(%s)' % chunk + + def write_code(self, line, comment=''): + code = ' ' * (self.indent + self.indent_mod) + code += line.lstrip() + comment + '\n' + self.code_buffer.append(code) + + +def template(*args, **kwargs): + """ + Get a rendered template as a string iterator. + You can use a name, a filename or a template string as first parameter. + Template rendering arguments can be passed as dictionaries + or directly (as keyword arguments). + """ + tpl = args[0] if args else None + for dictarg in args[1:]: + kwargs.update(dictarg) + adapter = kwargs.pop('template_adapter', SimpleTemplate) + lookup = kwargs.pop('template_lookup', TEMPLATE_PATH) + tplid = (id(lookup), tpl) + if tplid not in TEMPLATES or DEBUG: + settings = kwargs.pop('template_settings', {}) + if isinstance(tpl, adapter): + TEMPLATES[tplid] = tpl + if settings: TEMPLATES[tplid].prepare(**settings) + elif "\n" in tpl or "{" in tpl or "%" in tpl or '$' in tpl: + TEMPLATES[tplid] = adapter(source=tpl, lookup=lookup, **settings) + else: + TEMPLATES[tplid] = adapter(name=tpl, lookup=lookup, **settings) + if not TEMPLATES[tplid]: + abort(500, 'Template (%s) not found' % tpl) + return TEMPLATES[tplid].render(kwargs) + + +mako_template = functools.partial(template, template_adapter=MakoTemplate) +cheetah_template = functools.partial(template, + template_adapter=CheetahTemplate) +jinja2_template = functools.partial(template, template_adapter=Jinja2Template) + + +def view(tpl_name, **defaults): + """ Decorator: renders a template for a handler. + The handler can control its behavior like that: + + - return a dict of template vars to fill out the template + - return something other than a dict and the view decorator will not + process the template, but return the handler result as is. + This includes returning a HTTPResponse(dict) to get, + for instance, JSON with autojson or other castfilters. + """ + + def decorator(func): + + @functools.wraps(func) + def wrapper(*args, **kwargs): + result = func(*args, **kwargs) + if isinstance(result, (dict, DictMixin)): + tplvars = defaults.copy() + tplvars.update(result) + return template(tpl_name, **tplvars) + elif result is None: + return template(tpl_name, **defaults) + return result + + return wrapper + + return decorator + + +mako_view = functools.partial(view, template_adapter=MakoTemplate) +cheetah_view = functools.partial(view, template_adapter=CheetahTemplate) +jinja2_view = functools.partial(view, template_adapter=Jinja2Template) + +############################################################################### +# Constants and Globals ######################################################## +############################################################################### + +TEMPLATE_PATH = ['./', './views/'] +TEMPLATES = {} +DEBUG = False +NORUN = False # If set, run() does nothing. Used by load_app() + +#: A dict to map HTTP status codes (e.g. 404) to phrases (e.g. 'Not Found') +HTTP_CODES = httplib.responses.copy() +HTTP_CODES[418] = "I'm a teapot" # RFC 2324 +HTTP_CODES[428] = "Precondition Required" +HTTP_CODES[429] = "Too Many Requests" +HTTP_CODES[431] = "Request Header Fields Too Large" +HTTP_CODES[451] = "Unavailable For Legal Reasons" # RFC 7725 +HTTP_CODES[511] = "Network Authentication Required" +_HTTP_STATUS_LINES = dict((k, '%d %s' % (k, v)) + for (k, v) in HTTP_CODES.items()) + +#: The default template used for error pages. Override with @error() +ERROR_PAGE_TEMPLATE = """ +%%try: + %%from %s import DEBUG, request + + + + Error: {{e.status}} + + + +

Error: {{e.status}}

+

Sorry, the requested URL {{repr(request.url)}} + caused an error:

+ 
{{e.body}}
+ %%if DEBUG and e.exception: +

Exception:

+ %%try: + %%exc = repr(e.exception) + %%except: + %%exc = '' %% type(e.exception).__name__ + %%end + 
{{exc}}
+ %%end + %%if DEBUG and e.traceback: +

Traceback:

+ 
{{e.traceback}}
+ %%end + + +%%except ImportError: + ImportError: Could not generate the error page. Please add bottle to + the import path. +%%end +""" % __name__ + +#: A thread-safe instance of :class:`LocalRequest`. If accessed from within a +#: request callback, this instance always refers to the *current* request +#: (even on a multi-threaded server). +request = LocalRequest() + +#: A thread-safe instance of :class:`LocalResponse`. It is used to change the +#: HTTP response for the *current* request. +response = LocalResponse() + +#: A thread-safe namespace. Not used by Bottle. +local = threading.local() + +# Initialize app stack (create first empty Bottle app now deferred until needed) +# BC: 0.6.4 and needed for run() +apps = app = default_app = AppStack() + +#: A virtual package that redirects import statements. +#: Example: ``import bottle.ext.sqlite`` actually imports `bottle_sqlite`. +ext = _ImportRedirect('bottle.ext' if __name__ == '__main__' else + __name__ + ".ext", 'bottle_%s').module + + +def _main(argv): # pragma: no coverage + args, parser = _cli_parse(argv) + + def _cli_error(cli_msg): + parser.print_help() + _stderr('\nError: %s\n' % cli_msg) + sys.exit(1) + + if args.version: + print('Bottle %s' % __version__) + sys.exit(0) + if not args.app: + _cli_error("No application entry point specified.") + + sys.path.insert(0, '.') + sys.modules.setdefault('bottle', sys.modules['__main__']) + + host, port = (args.bind or 'localhost'), 8080 + if ':' in host and host.rfind(']') < host.rfind(':'): + host, port = host.rsplit(':', 1) + host = host.strip('[]') + + config = ConfigDict() + + for cfile in args.conf or []: + try: + if cfile.endswith('.json'): + with open(cfile, 'rb') as fp: + config.load_dict(json_loads(fp.read())) + else: + config.load_config(cfile) + except configparser.Error as parse_error: + _cli_error(parse_error) + except IOError: + _cli_error("Unable to read config file %r" % cfile) + except (UnicodeError, TypeError, ValueError) as error: + _cli_error("Unable to parse config file %r: %s" % (cfile, error)) + + for cval in args.param or []: + if '=' in cval: + config.update((cval.split('=', 1),)) + else: + config[cval] = True + + run(args.app, + host=host, + port=int(port), + server=args.server, + reloader=args.reload, + plugins=args.plugin, + debug=args.debug, + config=config) + + +if __name__ == '__main__': # pragma: no coverage + _main(sys.argv) diff --git a/eurybia/report/datapane/blocks/__init__.py b/eurybia/report/datapane/blocks/__init__.py new file mode 100644 index 0000000..3dc8cd9 --- /dev/null +++ b/eurybia/report/datapane/blocks/__init__.py @@ -0,0 +1,12 @@ +# flake8: noqa:F401 +# import typing as t + +from .asset import Attachment, DataTable, Media, Plot, Table +from .base import BaseBlock, BlockList, BlockOrPrimitive, DataBlock, wrap_block +from .empty import Empty +from .layout import Group, Page, Select, SelectType, Toggle, VAlign +from .misc_blocks import BigNumber +from .text import HTML, Code, Embed, Formula, Text + +# Block = t.Union["Group", "Select", "DataBlock", "Empty", "Function"] +Block = BaseBlock diff --git a/eurybia/report/datapane/blocks/asset.py b/eurybia/report/datapane/blocks/asset.py new file mode 100644 index 0000000..0bfcfc3 --- /dev/null +++ b/eurybia/report/datapane/blocks/asset.py @@ -0,0 +1,230 @@ +"""Asset-based blocks""" + +from __future__ import annotations + +import typing as t +from pathlib import Path + +import pandas as pd +from pandas.io.formats.style import Styler + +from eurybia.report.datapane.common import NPath, SSDict +from eurybia.report.datapane.common.df_processor import to_df +from eurybia.report.datapane.common.viewxml_utils import mk_attribs + +from .base import BlockId, DataBlock + +if t.TYPE_CHECKING: + from eurybia.report.datapane.processors.file_store import FileEntry + + +class AssetBlock(DataBlock): + """ + AssetBlock objects form basis of all File-related blocks (abstract class, not exported) + """ + + _prev_entry: t.Optional[FileEntry] = None + + # TODO - we may need to support file here as well to handle media, etc. + def __init__( + self, + data: t.Optional[t.Any] = None, + file: t.Optional[Path] = None, + caption: str = "", + name: t.Optional[BlockId] = None, + label: t.Optional[str] = None, + **kwargs, + ): + # storing objects for delayed upload + super().__init__(name=name, label=label, **kwargs) + self.data = data + self.file = file + self.caption = caption + self.file_attribs: SSDict = dict() + + def get_file_attribs(self) -> SSDict: + return self.file_attribs + + +class Media(AssetBlock): + """ + The Media block allows you to include images, GIFs, video and audio in your apps. If the file is in a supported format, it will be displayed inline in your app. + + To include an image, you can use `dp.Media` and pass the path. + + !!! note + Supported video, audio and image formats depend on the browser used to view the report. MP3, MP4, and all common image formats are generally supported by modern browsers + """ + + _tag = "Media" + + def __init__( + self, + file: NPath, + name: BlockId = None, + label: str = None, + caption: t.Optional[str] = None, + ): + """ + Args: + file: Path to a file to attach to the report (e.g. a JPEG image) + name: A unique name for the block to reference when adding text or embedding (optional) + caption: A caption to display below the file (optional) + label: A label used when displaying the block (optional) + """ + file = Path(file).expanduser() + super().__init__(file=file, name=name, caption=caption, label=label) + + +class Attachment(AssetBlock): + """ + If you want to include static files like PDFs or Excel docs in your app, use the `dp.Attachment` block. + + You can also pass in a Python object directly. Once you upload the app, your users will be able to explore and download these attachments. + + !!! tip + To attach streamable / viewable video, audio or images, use the `dp.Media` block instead + """ + + _tag = "Attachment" + + def __init__( + self, + data: t.Optional[t.Any] = None, + file: t.Optional[NPath] = None, + filename: t.Optional[str] = None, + caption: t.Optional[str] = None, + name: BlockId = None, + label: str = None, + ): + """ + Args: + data: A python object to attach to the report (e.g. a dictionary) + file: Path to a file to attach to the report (e.g. a csv file) + filename: Name to be used when downloading the file (optional) + caption: A caption to display below the file (optional) + name: A unique name for the block to reference when adding text or embedding (optional) + label: A label used when displaying the block (optional) + + !!! note + + Either `data` or `file` must be provided + """ + if file: + file = Path(file).expanduser() + filename = filename or file.name + elif data: + filename = filename or "test.data" + + super().__init__( + data=data, + file=file, + filename=filename, + name=name, + caption=caption, + label=label, + ) + + +class Plot(AssetBlock): + """ + Datapane supports all major Python visualization libraries, allowing you to add interactive plots and visualizations to your app. + + The `dp.Plot` block takes a plot object from one of the supported Python visualization libraries and renders it in your app. + + !!! info + Datapane will automatically wrap your visualization or plot in a `dp.Plot` block if you pass it into your app directly. + """ + + _tag = "Plot" + + def __init__( + self, + data: t.Any, + caption: t.Optional[str] = None, + responsive: bool = True, + scale: float = 1.0, + name: BlockId = None, + label: str = None, + ): + """ + Args: + data: The `plot` object to attach + caption: A caption to display below the plot (optional) + responsive: Whether the plot should automatically be resized to fit, set to False if your plot looks odd (optional, default: True) + scale: Set the scaling factor for the plt (optional, default = 1.0) + name: A unique name for the block to reference when adding text or embedding (optional) + label: A label used when displaying the block (optional) + """ + super().__init__( + data=data, + caption=caption, + responsive=responsive, + scale=scale, + name=name, + label=label, + ) + + +class Table(AssetBlock): + """ + Table blocks store the contents of a DataFrame as a HTML `table` whose style can be customised using + pandas' `Styler` API. + + !!! tip + `Table` is the best option for displaying multidimensional DataFrames, as `DataTable` will flatten your data. + """ + + # NOTE - Tables are stored as HTML fragment files rather than inline within the Report document + + _tag = "Table" + + def __init__( + self, + data: t.Union[pd.DataFrame, Styler], + caption: t.Optional[str] = None, + name: BlockId = None, + label: str = None, + ): + """ + Args: + data: The pandas `Styler` instance or dataframe to generate the table from + caption: A caption to display below the table (optional) + name: A unique name for the block to reference when adding text or embedding (optional) + label: A label used when displaying the block (optional) + """ + super().__init__(data=data, caption=caption, name=name, label=label) + + +class DataTable(AssetBlock): + """ + The DataTable block takes a pandas DataFrame and renders an interactive, sortable, searchable table in your app, along with advanced analysis options such as exploring data through [SandDance](https://www.microsoft.com/en-us/research/project/sanddance/). + + It supports large datasets and viewers can also download the table from the website as a CSV or Excel file. + + !!! tip + `Table` is the best option for displaying multidimensional DataFrames, as `DataTable` will flatten your data. + """ + + _tag = "DataTable" + + def __init__( + self, + df: pd.DataFrame, + caption: t.Optional[str] = None, + name: BlockId = None, + label: str = None, + ): + """ + Args: + df: The pandas dataframe to attach to the report + caption: A caption to display below the plot (optional) + name: A unique name for the block to reference when adding text or embedding (optional) + label: A label used when displaying the block (optional) + """ + # create a copy of the df to process + df = to_df(df) + super().__init__(data=df, caption=caption, name=name, label=label) + # TODO - support pyarrow schema for local reports + (rows, columns) = df.shape + self.file_attribs = mk_attribs(rows=rows, columns=columns, schema="[]") diff --git a/eurybia/report/datapane/blocks/base.py b/eurybia/report/datapane/blocks/base.py new file mode 100644 index 0000000..f676335 --- /dev/null +++ b/eurybia/report/datapane/blocks/base.py @@ -0,0 +1,124 @@ +""" +Datapane Blocks API + +Describes the collection of `Block` objects that can be combined together to make a `datapane.client.api.report.core.Report`. +""" + +from __future__ import annotations + +import typing as t +from abc import ABC + +from lxml.builder import ElementMaker + +from eurybia.report.datapane.client import DPClientError, log +from eurybia.report.datapane.common.viewxml_utils import is_valid_id, mk_attribs + +if t.TYPE_CHECKING: + from typing_extensions import Self + + from eurybia.report.datapane.blocks import Block + from eurybia.report.datapane.view import ViewVisitor + +E = ElementMaker() # XML Tag Factory + + +BlockId = str + +VV = t.TypeVar("VV", bound="ViewVisitor") + + +class BaseBlock(ABC): + """Base Block class - subclassed by all Block types + + ..note:: The class is not used directly. + """ + + _tag: str + + def __init__(self, name: t.Optional[BlockId] = None, **kwargs: t.Any): + """ + Args: + name: A unique name to reference the block, used when referencing blocks via the report editor and when embedding + """ + self._block_name: str = self._tag.lower() + self.name = name + + # validate name + if name and not is_valid_id(name): + raise DPClientError(f"Invalid name '{name}' for block") + + self._attributes: t.Dict[str, str] = dict() + self._add_attributes(name=name, **kwargs) + + self._truncate_strings(kwargs, "caption", 512) + self._truncate_strings(kwargs, "label", 256) + + @staticmethod + def _truncate_strings(kwargs: dict, key: str, max_length: int): + if key in kwargs: + x: str = kwargs[key] + if x and len(x) > max_length: + kwargs[key] = f"{x[:max_length-3]}..." + log.warning(f"{key} currently '{x}'") + log.warning( + f"{key} must be less than {max_length} characters, truncating" + ) + # raise DPClientError(f"{key} must be less than {max_length} characters, '{x}'") + + def _add_attributes(self, **kwargs): + self._attributes.update(mk_attribs(**kwargs)) + + def _ipython_display_(self): + """Display the block as a side effect within a Jupyter notebook""" + from IPython.display import HTML, display + + from eurybia.report.datapane.ipython.environment import get_environment + from eurybia.report.datapane.processors.api import stringify_report + from eurybia.report.datapane.view import Blocks + + if get_environment().support_rich_display: + html_str = stringify_report(Blocks(self)) + display(HTML(html_str)) + else: + display(self.__str__()) + + def accept(self, visitor: VV) -> VV: + visitor.visit(self) + return visitor + + def __str__(self) -> str: + return f"<{self._tag} attribs={self._attributes}>" + + def __copy__(self) -> Self: + """custom copy that deep copies attributes""" + inst = self.__class__.__new__(self.__class__) + inst.__dict__.update(self.__dict__) + inst._attributes = self._attributes.copy() + + return inst + + +class DataBlock(BaseBlock): + """Abstract block that represents a leaf-node in the tree, e.g. a Plot or Table + + ..note:: This class is not used directly. + """ + + +BlockOrPrimitive = t.Union["BaseBlock", t.Any] # TODO - expand +BlockList = t.List["BaseBlock"] +# Block = BaseBlock + + +def wrap_block(b: BlockOrPrimitive) -> Block: + from .wrappers import convert_to_block + + # if isinstance(b, Page): + # raise DPClientError("Page objects can only be at the top-level") + if not isinstance(b, BaseBlock): + # import here as a very slow module due to nested imports + # from ..files import convert + + return convert_to_block(b) + return t.cast("Block", b) diff --git a/eurybia/report/datapane/blocks/empty.py b/eurybia/report/datapane/blocks/empty.py new file mode 100644 index 0000000..a926d12 --- /dev/null +++ b/eurybia/report/datapane/blocks/empty.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +import secrets + +from .base import BaseBlock, BlockId + + +def gen_name() -> str: + """Return a (safe) name for use in a Block""" + return f"id-{secrets.token_urlsafe(8)}" + + +class Empty(BaseBlock): + """ + An empty block that can be updated / replaced later + + Args: + name: A unique name for the block to reference when updating the report + """ + + _tag = "Empty" + + def __init__(self, name: BlockId): + super().__init__(name=name) diff --git a/eurybia/report/datapane/blocks/layout.py b/eurybia/report/datapane/blocks/layout.py new file mode 100644 index 0000000..7a52915 --- /dev/null +++ b/eurybia/report/datapane/blocks/layout.py @@ -0,0 +1,266 @@ +from __future__ import annotations + +import typing as t +from collections import deque +from functools import reduce + +from eurybia.report.datapane.client import DPClientError, log +from eurybia.report.datapane.common.dp_types import StrEnum + +from .base import BaseBlock, BlockId, BlockList, BlockOrPrimitive, wrap_block +from .empty import Empty, gen_name + +if t.TYPE_CHECKING: + from typing_extensions import Self + + from eurybia.report.datapane.blocks import Block + + from .base import VV + + +class SelectType(StrEnum): + DROPDOWN = "dropdown" + TABS = "tabs" + + +class VAlign(StrEnum): + TOP = "top" + CENTER = "center" + BOTTOM = "bottom" + + +class ContainerBlock(BaseBlock): + """ + Abstract Block that supports nested/contained blocks + - represents a subtree in the document + """ + + blocks: BlockList + # how many blocks must there be in the container + report_minimum_blocks: int = 1 + + def __init__( + self, + *arg_blocks: BlockOrPrimitive, + blocks: t.List[BlockOrPrimitive] = None, + **kwargs, + ): + self.blocks = blocks or list(arg_blocks) + self.blocks = [wrap_block(b) for b in self.blocks] + + super().__init__(**kwargs) + + def __iter__(self): + return BlockListIterator(self.blocks.__iter__()) + + def __add__(self, other: Self) -> Self: + self.blocks.extend(other.blocks) + return self + + def __and__(self, other: Self) -> Self: + self.blocks.extend(other.blocks) + return self + + def __copy__(self) -> Self: + inst = super().__copy__() + inst.blocks = self.blocks.copy() + return inst + + @classmethod + def empty(cls) -> Self: + return cls(blocks=[Empty(gen_name())]) + + def traverse(self, visitor: VV) -> VV: + # perform a depth-first traversal of the contained blocks + return reduce( + lambda _visitor, block: block.accept(_visitor), self.blocks, visitor + ) + + +class Page(ContainerBlock): + """ + Apps on Datapane can have multiple pages, which are presented to users as tabs at the top of your app. These can be used similarly to sheets in an Excel document. + + To add a page, use the `dp.Page` block at the top-level of your app, and give it a title with the `title` parameter. + + !!! info + Pages cannot be nested, and can only exist at the root level of your `dp.App` object. If you're using pages, all other blocks must be contained inside a Page block. + + !!! note + This is included for backwards-compatability, and can be replaced by using Selects going forwards. + """ + + # BC-only helper - converted into a Select + Group within the post-XML processor + _tag = "_Page" + + def __init__( + self, + *arg_blocks: BlockOrPrimitive, + blocks: t.List[BlockOrPrimitive] = None, + title: str = None, + name: BlockId = None, + ): + """ + Args: + *arg_blocks: Blocks to add to Page + blocks: Allows providing the report blocks as a single list + title: The page title (optional) + name: A unique id for the Page to aid querying (optional) + + !!! tip + Page can be passed using either arg parameters or the `blocks` kwarg, e.g. `dp.Page(group, select)` or `dp.Group(blocks=[group, select])` + """ + self.title = title + super().__init__(*arg_blocks, blocks=blocks, label=title, name=name) + + if any(isinstance(b, Page) for b in self.blocks): + raise DPClientError( + "Nested pages not supported, please use Selects and Groups" + ) + + +class Select(ContainerBlock): + """ + Selects act as a container that holds a list of nested Blocks objects, such + as Tables, Plots, etc.. - but only one may be __visible__, or "selected", at once. + + The user can choose which nested object to view dynamically using either tabs or a dropdown. + + !!! note + Select expects a list of Blocks, e.g. a Plot or Table, but also includes Select or Groups themselves, but if a Python object is passed, e.g. a Dataframe, Datapane will attempt to convert it automatically. + + """ + + _tag = "Select" + report_minimum_blocks = 2 + + def __init__( + self, + *arg_blocks: BlockOrPrimitive, + blocks: t.List[BlockOrPrimitive] = None, + type: SelectType = SelectType.TABS, + name: BlockId = None, + label: str = None, + ): + """ + Args: + *arg_blocks: Page to add to report + blocks: Allows providing the report blocks as a single list + type: An instance of SelectType that indicates if the select should use tabs or a dropdown (default: Tabs) + name: A unique id for the blocks to aid querying (optional) + label: A label used when displaying the block (optional) + + !!! tip + Select can be passed using either arg parameters or the `blocks` kwarg, e.g. `dp.Select(table, plot, type=dp.SelectType.TABS)` or `dp.Select(blocks=[table, plot])` + """ + super().__init__(*arg_blocks, blocks=blocks, name=name, label=label, type=type) + if len(self.blocks) < 2: + log.info("Creating a Select with less than 2 objects") + + +class Group(ContainerBlock): + """ + If you pass a list of blocks (such as `Plot` and `Table`) to an app, they are -- by default -- laid out in a single column with a row per block. + + If you would like to customize the rows and columns, Datapane provides a `Group` block which takes a list of blocks and a number of columns and lays them out in a grid. + + !!! tip + As `Group` blocks are blocks themselves, they are composable, and you can create more custom layers of nested blocks, for instance nesting 2 rows in the left column of a 2 column layout + """ + + _tag = "Group" + + def __init__( + self, + *arg_blocks: BlockOrPrimitive, + blocks: t.List[BlockOrPrimitive] = None, + name: BlockId = None, + label: t.Optional[str] = None, + widths: t.Optional[t.List[t.Union[int, float]]] = None, + valign: VAlign = VAlign.TOP, + columns: int = 1, + ): + """ + Args: + *arg_blocks: Group to add to report + blocks: Allows providing the report blocks as a single list + name: A unique id for the blocks to aid querying (optional) + label: A label used when displaying the block (optional) + widths: A list of numbers representing the proportion of vertical space given to each column (optional) + valign: The vertical alignment of blocks in the Group (default = VAlign.TOP) + columns: Display the contained blocks, e.g. Plots, using _n_ columns (default = 1), setting to 0 auto-wraps the columns + + !!! note + Group can be passed using either arg parameters or the `blocks` kwarg, e.g. `dp.Group(plot, table, columns=2)` or `dp.Group(blocks=[plot, table], columns=2)`. + """ + + if widths is not None and len(widths) != columns: + raise DPClientError( + "Group 'widths' list length does not match number of columns" + ) + + # columns = columns or len(self.blocks) + self.columns = columns + super().__init__( + *arg_blocks, + blocks=blocks, + name=name, + label=label, + columns=columns, + widths=widths, + valign=valign, + ) + + +class Toggle(ContainerBlock): + """ + Toggles act as a container that holds a list of nested Block objects, whose visibility can be toggled on or off by the report viewer + """ + + _tag = "Toggle" + report_minimum_blocks = 1 + + def __init__( + self, + *arg_blocks: BlockOrPrimitive, + blocks: t.List[BlockOrPrimitive] = None, + name: BlockId = None, + label: str = None, + ): + """ + Args: + *arg_blocks: Group to add to report + blocks: Allows providing the report blocks as a single list + name: A unique id for the blocks to aid querying (optional) + label: A label used when displaying the block (optional) + """ + super().__init__(*arg_blocks, blocks=blocks, name=name, label=label) + self._wrap_blocks() + + def _wrap_blocks(self) -> None: + """Wrap the list of blocks in a top-level block element if needed""" + if len(self.blocks) > 1: + # only wrap if not all blocks are a Group object + self.blocks = [Group(blocks=self.blocks)] + + +class BlockListIterator: + """Wrapper around default list iterator that supports depth-first traversal of blocks""" + + def __init__(self, _iter): + # linearise all blocks into a deque as we traverse + self.nested = deque(_iter) + + def __next__(self) -> Block: + try: + b: Block = self.nested.popleft() + except IndexError: + raise StopIteration() + + if isinstance(b, ContainerBlock): + # add the nested iter contents for next "next" call, hence traversing the tree + self.nested.extendleft(reversed(b.blocks)) + return b + + def __iter__(self): + return self diff --git a/eurybia/report/datapane/blocks/misc_blocks.py b/eurybia/report/datapane/blocks/misc_blocks.py new file mode 100644 index 0000000..e102038 --- /dev/null +++ b/eurybia/report/datapane/blocks/misc_blocks.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +import typing as t + +from .base import BlockId, DataBlock + +NumberValue = t.Union[str, int, float] + + +class BigNumber(DataBlock): + """ + A single number or change can often be the most important thing in an app. + + The `BigNumber`block allows you to present KPIs, changes, and statistics in a friendly way to your viewers. + + You can optionally set intent, and pass in numbers or text. + """ + + _tag = "BigNumber" + + def __init__( + self, + heading: str, + value: NumberValue, + change: t.Optional[NumberValue] = None, + prev_value: t.Optional[NumberValue] = None, + is_positive_intent: t.Optional[bool] = None, + is_upward_change: t.Optional[bool] = None, + name: BlockId = None, + label: str = None, + ): + """ + Args: + heading: A title that gives context to the displayed number + value: The value of the number + prev_value: The previous value to display as comparison (optional) + change: The amount changed between the value and previous value (optional) + is_positive_intent: Displays the change on a green background if `True`, and red otherwise. Follows `is_upward_change` if not set (optional) + is_upward_change: Whether the change is upward or downward (required when `change` is set) + name: A unique name for the block to reference when adding text or embedding (optional) + label: A label used when displaying the block (optional) + """ + if change: + if is_upward_change is None: + # We can't reliably infer the direction of change from the change string + raise ValueError('Argument "is_upward_change" is required when "change" is set') + if is_positive_intent is None: + # Set the intent to be the direction of change if not specified (up = green, down = red) + is_positive_intent = is_upward_change + + super().__init__( + heading=heading, + value=value, + change=change, + prev_value=prev_value, + is_positive_intent=bool(is_positive_intent), + is_upward_change=bool(is_upward_change), + name=name, + label=label, + ) diff --git a/eurybia/report/datapane/blocks/text.py b/eurybia/report/datapane/blocks/text.py new file mode 100644 index 0000000..7395d4a --- /dev/null +++ b/eurybia/report/datapane/blocks/text.py @@ -0,0 +1,230 @@ +from __future__ import annotations + +import re +import textwrap +import typing as t +from collections import deque +from pathlib import Path + +from dominate.dom_tag import dom_tag + +from eurybia.report.datapane.client import DPClientError +from eurybia.report.datapane.common import NPath, utf_read_text +from eurybia.report.datapane.common.viewxml_utils import get_embed_url + +from .base import BlockId, BlockOrPrimitive, DataBlock, wrap_block +from .layout import Group + + +class EmbeddedTextBlock(DataBlock): + """ + Abstract Block for embedded text formats that are stored directly in the + document (rather than external references) + """ + + content: str + + def __init__(self, content: str, name: BlockId = None, **kwargs): + super().__init__(name, **kwargs) + self.content = content.strip() + + +class Text(EmbeddedTextBlock): + """ + You can add short or long-form Markdown content to your app with the `Text` block. + + !!! info + Markdown is a lightweight markup language that allows you to include formatted text in your app, and can be accessed through `dp.Text`, or by passing in a string directly. + + Check [here](https://github.com/adam-p/markdown-here/wiki/Markdown-Cheatsheet) for more information on how to format your text with markdown. + """ + + _tag = "Text" + + def __init__( + self, + text: str = None, + file: NPath = None, + name: BlockId = None, + label: str = None, + ): + """ + Args: + text: The markdown formatted text, use triple-quotes, (`\"\"\"# My Title\"\"\"`) to create multi-line markdown text + file: Path to a file containing markdown text + name: A unique name for the block to reference when adding text or embedding (optional) + label: A label used when displaying the block (optional) + + !!! note + File encodings are auto-detected, if this fails please read the file manually with an explicit encoding and use the text parameter on dp.Attachment + """ + if text: + text = textwrap.dedent(text).strip() + + assert text or file + content = text or utf_read_text(Path(file).expanduser()) + super().__init__(content=content, name=name, label=label) + + def format(self, *args: BlockOrPrimitive, **kwargs: BlockOrPrimitive) -> Group: + """ + Format the markdown text template, using the supplied context to insert blocks into `{{}}` markers in the template. + + `{}` markers can be empty, hence positional, or have a name, e.g. `{{plot}}`, which is used to lookup the value from the keyword context. + + Args: + *args: positional template context arguments + **kwargs: keyword template context arguments + + !!! tip + Either Python objects, e.g. dataframes, and plots, or Datapane blocks as context + + Returns: + A datapane Group object containing the list of text and embedded objects + """ + + splits = re.split(r"\{\{(\w*)\}\}", self.content) + deque_args = deque(args) + blocks = [] + + for i, x in enumerate(splits): + is_block = bool(i % 2) + + if is_block: + try: + if x: + blocks.append(wrap_block(kwargs[x])) + else: + blocks.append(wrap_block(deque_args.popleft())) + except (IndexError, KeyError): + raise DPClientError( + f"Unknown/missing object '{x}' referenced in Markdown format" + ) + + else: + x = x.strip() + if x: + blocks.append(Text(x)) + + return Group(blocks=blocks) + + +class Code(EmbeddedTextBlock): + """ + The code block allows you to embed syntax-highlighted source code into your app. + + !!! note + This block currently supports Python and JavaScript. + """ + + _tag = "Code" + + def __init__( + self, + code: str, + language: str = "python", + caption: str = None, + name: BlockId = None, + label: str = None, + ): + """ + Args: + code: The source code + language: The language of the code, most common languages are supported (optional - defaults to Python) + caption: A caption to display below the Code (optional) + name: A unique name for the block to reference when adding text or embedding (optional) + label: A label used when displaying the block (optional) + """ + super().__init__( + content=code, language=language, caption=caption, name=name, label=label + ) + + +class HTML(EmbeddedTextBlock): + """ + The HTML block allows you to add raw HTML to your app, allowing for highly customized components, such as your company's brand, logo, and more. + + !!! info + The HTML block is sandboxed and cannot execute JavaScript. + """ + + _tag = "HTML" + + def __init__( + self, html: t.Union[str, dom_tag], name: BlockId = None, label: str = None + ): + """ + Args: + html: The HTML fragment to embed - can be a string or a [dominate](https://github.com/Knio/dominate/) tag + name: A unique name for the block to reference when adding text or embedding (optional) + label: A label used when displaying the block (optional) + """ + super().__init__(content=str(html), name=name, label=label) + + +class Formula(EmbeddedTextBlock): + """ + The formula block allows you easily to add [_LaTeX_](https://en.wikipedia.org/wiki/LaTeX)-formatted equations to your app, with an optional caption. + + !!! tip + A brief intro into _LaTeX_ formulas can be found [here](https://en.wikibooks.org/wiki/LaTeX/Mathematics). + """ + + _tag = "Formula" + + def __init__( + self, formula: str, caption: str = None, name: BlockId = None, label: str = None + ): + r""" + Args: + formula: The formula to embed, using LaTeX format (use raw strings) + caption: A caption to display below the Formula (optional) + name: A unique name for the block to reference when adding text or embedding (optional) + label: A label used when displaying the block (optional) + + !!! note + LaTeX commonly uses special characters, hence prefix your formulas with `r` to make them raw strings, e.g. `r"\frac{1}{\sqrt{x^2 + 1}}"` + + Under the hood we use MathJAX to render the equations in the browser and not a full TeX engine. This means that some of your TeX input may not be rendered correctly on our system - read the MathJAX documentation for more info. + """ + super().__init__(content=formula, caption=caption, name=name, label=label) + + +class Embed(EmbeddedTextBlock): + """ + The Embed block lets you embed content from other platforms e.g. Youtube, Spotify. + + !!! tip + If you're trying to embed an `iframe`, you can wrap it in an `HTML` block. + """ + + _tag = "Embed" + + def __init__( + self, + url: str, + width: int = 960, + height: int = 540, + name: BlockId = None, + label: str = None, + ): + """ + Args: + url: The URL of the resource to be embedded + width: The width of the embedded object (optional) + height: The height of the embedded object (optional) + name: A unique name for the block to reference when adding text or embedding (optional) + label: A label used when displaying the block (optional) + """ + + result = get_embed_url(url, width=width, height=height) + + # if "html" not in result: + # raise DPClientError(f"Can't embed result from provider for URL '{url}'") + super().__init__( + content=result.html, + name=name, + label=label, + url=url, + title=result.title, + provider_name=result.provider, + ) diff --git a/eurybia/report/datapane/blocks/wrappers.py b/eurybia/report/datapane/blocks/wrappers.py new file mode 100644 index 0000000..8ab599e --- /dev/null +++ b/eurybia/report/datapane/blocks/wrappers.py @@ -0,0 +1,81 @@ +"""Type-driven block wrapping""" + +# flake8: noqa:F811 +from __future__ import annotations + +import typing as t +from pathlib import Path + +import pandas as pd +from altair.utils import SchemaBase +from multimethod import multimethod + +from eurybia.report.datapane import blocks as b +from eurybia.report.datapane import optional_libs as opt +from eurybia.report.datapane.client import DPClientError + +from .base import DataBlock + + +@multimethod +def convert_to_block(x: object) -> DataBlock: + raise DPClientError( + f"{type(x)} not supported directly, please pass into in the appropriate dp object (including dp.Attachment if want to upload as a pickle)" + ) + + +# NOTE - this is currently disabled to avoid confusing users when they +# try to embed any Python object, instead they must use dp.Attachment +# @multimethod +# def convert_to_block(x: t.Any) -> DataBlock: +# return b.Attachment(x) + + +@multimethod +def convert_to_block(x: str) -> DataBlock: + return b.Text(x) + + +@multimethod +def convert_to_block(x: Path) -> DataBlock: + return b.Attachment(file=x) + + +@multimethod +def convert_to_block(x: pd.DataFrame) -> DataBlock: + n_cells = x.shape[0] * x.shape[1] + return b.Table(x) if n_cells <= 250 else b.DataTable(x) + + +# Plots +@multimethod +def convert_to_block(x: SchemaBase) -> DataBlock: + return b.Plot(x) + + +if opt.HAVE_BOKEH: + + @multimethod + def convert_to_block(x: t.Union[opt.BFigure, opt.BLayout]) -> DataBlock: + return b.Plot(x) + + +if opt.HAVE_PLOTLY: + + @multimethod + def convert_to_block(x: opt.PFigure) -> DataBlock: + return b.Plot(x) + + +if opt.HAVE_FOLIUM: + + @multimethod + def convert_to_block(x: opt.Map) -> DataBlock: + return b.Plot(x) + + +if opt.HAVE_MATPLOTLIB: + + @multimethod + def convert_to_block(x: t.Union[opt.Figure, opt.Axes, opt.ndarray]) -> DataBlock: + return b.Plot(x) diff --git a/eurybia/report/datapane/builtins.py b/eurybia/report/datapane/builtins.py new file mode 100644 index 0000000..cafd854 --- /dev/null +++ b/eurybia/report/datapane/builtins.py @@ -0,0 +1,395 @@ +""" +Datapane built-in helper functions to make creating your reports a bit simpler and reduce common tasks +""" + +import random +import typing as t +from pathlib import Path +import sys + +import altair as alt + +if sys.version_info < (3, 10): + import importlib_resources as ir +else: + from importlib import resources as ir +import numpy as np +import pandas as pd + +from eurybia.report.datapane import Blocks +from eurybia.report.datapane import blocks as b +from eurybia.report.datapane.common import NPath + +__all__ = [ + "add_code", + "build_md_view", + "demo", + "gen_df", + "gen_table_df", + "gen_plot", +] + + +def add_code( + block: b.BlockOrPrimitive, code: str, language: str = "python" +) -> b.Select: + """ + Attach code fragment to an existing plot/figure/dataframe for use within a report + + Args: + block: The existing object to add code to - can be either an existing dp Block or an Python object + code: The code fragment to add + language: The language of the code fragment (optional) + + Returns: + A Select block that provides the figure and the code in tabs that can be selected by the user + """ + + w_block = b.wrap_block(block) + w_block._add_attributes(label="Figure") + return b.Select( + w_block, b.Code(code, language, label="Code"), type=b.SelectType.TABS + ) + + +def build_md_view( + text_or_file: t.Union[str, NPath], + *args: b.BlockOrPrimitive, + **kwargs: b.BlockOrPrimitive, +) -> Blocks: + """ + An easy way to build a complete report from a single top-level markdown text / file template. + Any additional context can be passed in and will be inserted into the Markdown template. + + Args: + text_or_file: The markdown text, or path to a markdown file, using {{}} for templating + *args: positional template context arguments + **kwargs: keyword template context arguments + + Returns: + A datapane App object for saving or uploading + + ..tip:: Either text or file is required as input + ..tip:: Context, via args/kwargs can be plain Python objects, e.g. dataframes, and plots, or Datapane blocks, e.g. dp.Plot, etc. + + """ + try: + b_text = ( + b.Text(file=text_or_file) + if Path(text_or_file).exists() + else b.Text(text=t.cast(str, text_or_file)) + ) + except OSError: + b_text = b.Text(text=t.cast(str, text_or_file)) + + group = b_text.format(*args, **kwargs) + return Blocks(group) + + +def gen_df(dim: int = 4) -> pd.DataFrame: + """Return a test simple df""" + axis = [i for i in range(0, dim)] + data = {"x": axis, "y": axis} + return pd.DataFrame.from_dict(data) + + +def gen_table_df(rows: int = 4, alphabet: str = "ABCDEF") -> pd.DataFrame: + """Return a test complex df for adding to a DataTable""" + data = [{x: random.randint(0, 1000) for x in alphabet} for _ in range(0, rows)] + return pd.DataFrame.from_dict(data) + + +def gen_plot() -> alt.Chart: + """Generate a sample Altair plot""" + return alt.Chart(gen_df()).mark_line().encode(x="x", y="y") + + +def demo() -> Blocks: + """ + Generate a sample demo view + + Returns: + A datapane View object for saving or uploading + """ + + import altair as alt # noqa + import folium # noqa + import matplotlib.pyplot as plt # noqa + import plotly.graph_objects as go # noqa + from bokeh.plotting import figure # noqa + + def _gen_bokeh(**kw): + p = figure( + title="simple line example", x_axis_label="x", y_axis_label="y", **kw + ) + p.line([1, 2, 3, 4, 5], [6, 7, 2, 4, 5], legend_label="Temp.", line_width=2) + return p + + def _gen_plotly(): + fig = go.Figure() + fig.add_trace(go.Scatter(x=[0, 1, 2, 3, 4, 5], y=[1.5, 1, 1.3, 0.7, 0.8, 0.9])) + fig.add_trace(go.Bar(x=[0, 1, 2, 3, 4, 5], y=[1, 0.5, 0.7, -1.2, 0.3, 0.4])) + return fig + + def _gen_matplotlib(): + # pd.set_option("plotting.backend", "matplotlib") + fig, ax = plt.subplots() + df = gen_df() + ax.plot(df["x"], df["y"]) + # gen_df().plot.scatter("x", "y", ax=ax) + return fig + + def _gen_html(w: int = 30, h: int = 30): + return b.HTML( + f""" +
+
+ HTML Block +
+
""" + ) + + def _color_large_vals(s: t.Any): + return [ + "background-color: rgba(255, 0, 0, 0.3)" if val > 800 else "" for val in s + ] + + def _gen_folium(): + return folium.Map( + location=[45.372, -121.6972], zoom_start=12, tiles="Stamen Terrain" + ) + + df1 = gen_table_df(10) + styler1 = df1.style.apply(_color_large_vals, axis=1).hide(axis="index") + + def _vega_sine(): + x = np.arange(100) + source = pd.DataFrame({"x": x, "f(x)": np.sin(x / 5)}) + + return alt.Chart(source).mark_line().encode(x="x", y="f(x)") + + vega_sine = _vega_sine() + + def _vega_bar(): + source = pd.DataFrame( + { + "a": ["A", "B", "C", "D", "E", "F", "G", "H", "I"], + "b": [28, 55, 43, 91, 81, 53, 19, 87, 52], + } + ) + + return alt.Chart(source).mark_bar().encode(x="a", y="b") + + vega_bar = _vega_bar() + + basics = """ +This page describes Datapane, an API for creating data-driven reports from Python. +Datapane reports are comprised of blocks, which can be collected together and laid-out to form multiple-pages reports. +Some of the basic blocks include tables and plots. + +## Tables + +The Table block displays a tabular set of data, and takes either a dataframe or a pandas Styler "styled" object, + +```python +dp.Table(df, caption="A Table") +``` + +{{table}} + +The DataTable block also takes a dataframe and allows the user to search and filter the data when viewing the report + + +## Plots + +The Plot block supports Altair, Bokeh, Plotly, Matplotlib, and Folium plots, + +```python +dp.Plot(altair_plot, caption="A Plot") +``` + +{{plot}} + +## Other Blocks + +Datapane has many other block types, including formulas, files, embeds, images, and big numbers - see the Blocks page for more info. + +Additionally layout blocks provide the ability nest blocks to create groups of columns and user selections - see the Layout page for more info. + +{{other}} + + + """ + logo = ir.files("datapane.resources") / "datapane-icon-192x192.png" + other = b.Group( + b.Media(file=str(logo)), + b.BigNumber( + heading="Datapane Blocks", value=11, prev_value=6, is_upward_change=True + ), + b.Formula(r"\frac{1}{\sqrt{x^2 + 1}}", caption="Simple formula"), + columns=0, + ) + page_1 = b.Text(basics, label="Intro").format( + table=b.Table(gen_table_df(), caption="A table"), plot=vega_sine, other=other + ) + + layout = """ +Blocks on a page can be laid-out in Datapane using a flexible row and column system. furthermore, multiple blocks can be +nested into a single block where the user can select between which block, e.g. a plot, to view. +See https://docs.datapane.com/reports/layout-and-customization for more info. + +## Group Blocks + +Group blocks allow you to take a list of blocks, and lay-them out over a number of `rows` and `columns`, allowing you to create 2-column layouts, grids, and more, + +```python +dp.Group(plot1, plot2, columns=2) +cells = [dp.Text(f"### Cell {x}") for x in range(6)] +dp.Group(*cells, columns=0) # 0 implies auto +``` + +{{group1}} + +{{group2}} + +## Select Blocks + +Select blocks allow you to collect a list of blocks, e.g. plots, and allow the user to select between them, either via tabs or a dropdown list. + +```python +dp.Select(plot1, plot2, type=dp.SelectType.TABS) +dp.Select(plot1, plot2, type=dp.SelectType.DROPDOWN) +``` + +{{select1}} + +{{select2}} + + +Both Group and Select blocks can be nested within one another, in any order to create, for instance dropdowns with 2 columns inside, as below + +```python +group1 = dp.Group(plot1, plot2, columns=2) +dp.Select(group1, df) +``` + +{{nested}} +""" + + group1 = b.Group(vega_bar, vega_sine, columns=2) + group2 = b.Group(*[f"### Cell {x}" for x in range(6)], columns=3) + select1 = b.Select(vega_bar, vega_sine, type=b.SelectType.TABS, name="vega_select") + select2 = b.Select(vega_bar, vega_sine, type=b.SelectType.DROPDOWN) + + nested = b.Select(group1, b.Table(gen_table_df())) + page_2 = b.Text(layout, label="Layout").format( + group1=group1, group2=group2, select1=select1, select2=select2, nested=nested + ) + + adv_blocks = r""" +A list and demonstration of all the blocks supported by Datapane - see https://docs.datapane.com/reports/blocks for more info. + +## Plot Blocks + +```python +dp.Group(dp.Plot(altair_plot, caption="Altair Plot"), + dp.Plot(bokeh_plot, caption="Bokeh Plot"), + dp.Plot(matplotlib_plot, caption="Matplotlib Plot"), + dp.Plot(plotly_plot, caption="Plotly Plot"), + dp.Plot(folium_plot, caption="Folium Plot"), + columns=2) +``` + +{{plots}} + +## Table Blocks + +```python +dp.Table(df, caption="Basic Table") +dp.Table(styled_df, caption="Styled Table") +dp.DataTable(df, caption="Interactive DataTable") +``` + +{{tables}} + +## Text Blocks + +```python +dp.Text("Hello, __world__!") +dp.Code("print('Hello, world!')") +dp.Formula(r"\frac{1}{\sqrt{x^2 + 1}}") +dp.HTML("

Hello World

") +dp.BigNumber(heading="Datapane Blocks", value=11, prev_value=6, is_upward_change=True) +``` + +{{text}} + +## Embedding + +You can embed any URLs that spport the OEmbed protocol, including YouTube and Twitter. + +```python +dp.Embed("https://www.youtube.com/watch?v=JDe14ulcfLA") +dp.Embed("https://twitter.com/datapaneapp/status/1300831345413890050") +``` + +{{embed}} + +## Media and Attachments + +Files and Python objects can be added to a Datapane report, and be viewed (depending on browser support) and downloaded. + +```python +dp.Media(file="./logo.png") +dp.Attachment(data=[1,2,3]) +``` + +{{media}} +""" + + plots = b.Group( + b.Plot(vega_sine, name="vega", caption="Altair Plot"), + b.Plot(_gen_bokeh(), name="bokeh", caption="Bokeh Plot"), + b.Plot(_gen_matplotlib(), name="matplotlib", caption="Matplotlib Plot"), + b.Plot(_gen_plotly(), name="plotly", caption="Plotly Plot"), + b.Plot(_gen_folium(), name="folium", caption="Folium Plot"), + name="plots_group", + columns=2, + ) + tables = b.Group( + b.Table(df1, name="table1", caption="Basic Table"), + b.Table(styler1, name="styled-table", caption="Styled Table"), + b.DataTable( + gen_table_df(1000), name="data_table", caption="Interactive DataTable" + ), + ) + text = b.Group( + b.Text("Hello, __world__!", name="markdown"), + b.Code("print('Hello, world!'", name="code"), + b.Formula(r"\frac{1}{\sqrt{x^2 + 1}}"), + b.HTML("

Hello World

", name="HTML"), + b.BigNumber( + heading="Datapane Blocks", + value=11, + prev_value=6, + is_upward_change=True, + name="big_num", + ), + columns=0, + ) + embed = b.Group( + b.Embed("https://www.youtube.com/watch?v=JDe14ulcfLA", name="youtube_embed"), + b.Embed("https://twitter.com/datapaneapp/status/1300831345413890050"), + columns=2, + ) + media = b.Group( + b.Media(file=str(logo), name="logo_img"), + b.Attachment(data=[1, 2, 3]), + columns=2, + ) + + page_3 = b.Text(adv_blocks, label="Blocks").format( + plots=plots, tables=tables, text=text, embed=embed, media=media + ) + + return Blocks(b.Select(page_1, page_2, page_3)) diff --git a/eurybia/report/datapane/client/__init__.py b/eurybia/report/datapane/client/__init__.py new file mode 100644 index 0000000..d95fadb --- /dev/null +++ b/eurybia/report/datapane/client/__init__.py @@ -0,0 +1,7 @@ +# Copyright 2020 StackHut Limited (trading as Datapane) +# SPDX-License-Identifier: Apache-2.0 +# flake8: noqa:F401 +from .exceptions import DPClientError +from .utils import IN_PYTEST, DPMode, display_msg, enable_logging, get_dp_mode, log, print_debug_info, set_dp_mode + +# from .config import init # isort:skip otherwise circular import issue diff --git a/eurybia/report/datapane/client/config.py b/eurybia/report/datapane/client/config.py new file mode 100644 index 0000000..b666622 --- /dev/null +++ b/eurybia/report/datapane/client/config.py @@ -0,0 +1,62 @@ +""" +Config subsystem +We use a module-level singleton pattern here, where top-level functions and globals +act like the state for a "Manager" class around the internal Config object +""" +import typing as t +from typing import Optional + +from .utils import log + + +class Config: + @property + def is_public(self) -> bool: + return True + + @property + def is_org(self) -> bool: + return not self.is_public + + @property + def is_authenticated(self) -> bool: + return False + + @property + def is_anonymous(self) -> bool: + return True + + +config: Optional[Config] = None + + +################################################################################ +# MODULE LEVEL INTERFACE +def init(config: t.Optional[Config] = None) -> Config: + """ + Init an API config + - this MUST handle being called multiple times and only from the main-thread + """ + if globals().get("config") is not None: + log.debug("Reinitialising client config") + + if config: + set_config(config) + else: + config = Config() + + return config + + +def set_config(c: Optional[Config]): + global config + config = c + + +def get_config() -> Config: + """Get the current config object, doesn't attempt to re-init the API token""" + global config + if config is None: + raise RuntimeError("Config must be initialised before it can be used") + + return config diff --git a/eurybia/report/datapane/client/exceptions.py b/eurybia/report/datapane/client/exceptions.py new file mode 100644 index 0000000..6594d04 --- /dev/null +++ b/eurybia/report/datapane/client/exceptions.py @@ -0,0 +1,47 @@ +from eurybia.report.datapane.common import DPError + + +def add_help_text(x: str) -> str: + return f"{x}\nPlease run with `dp.enable_logging()`, restart your Jupyter kernel/Python instance, and/or visit https://www.github.com/datapane/datapane" + + +class DPClientError(DPError): + def __str__(self): + # update the error message with help text + return add_help_text(super().__str__()) + + +class IncompatibleVersionError(DPClientError): + pass + + +class UnsupportedResourceError(DPClientError): + pass + + +class ReportTooLargeError(DPClientError): + pass + + +class InvalidTokenError(DPClientError): + pass + + +class UnsupportedFeatureError(DPClientError): + pass + + +class InvalidReportError(DPClientError): + pass + + +class ViewError(DPClientError): + pass + + +class MissingCloudPackagesError(DPClientError): + def __init__(self, *a, **kw): + # quick hack until we setup a conda meta-package for cloud + self.args = ( + "Cloud packages not found, please run `pip install datapane[cloud]` or `conda install -c conda-forge nbconvert flit-core`", + ) diff --git a/eurybia/report/datapane/client/utils.py b/eurybia/report/datapane/client/utils.py new file mode 100644 index 0000000..03f807a --- /dev/null +++ b/eurybia/report/datapane/client/utils.py @@ -0,0 +1,188 @@ +from __future__ import annotations + +import enum +import logging +import logging.config +import os +import string +import sys +import typing as t + +############## +# Client constants +TEST_ENV = bool(os.environ.get("DP_TEST_ENV", "")) +IN_PYTEST = "pytest" in sys.modules # and TEST_ENV +# we're running on datapane platform +ON_DATAPANE: bool = "DATAPANE_ON_DATAPANE" in os.environ + + +################################################################################ +# Logging +# export the application logger at WARNING level by default +log: logging.Logger = logging.getLogger("datapane") +if log.level == logging.NOTSET: + log.setLevel(logging.WARNING) + + +_have_setup_logging: bool = False + + +def _setup_dp_logging(verbosity: int = 0, logs_stream: t.TextIO = None) -> None: + global _have_setup_logging + + log_level = "WARNING" + if verbosity == 1: + log_level = "INFO" + elif verbosity > 1: + log_level = "DEBUG" + + # don't configure global logging config when running as a library + if get_dp_mode() == DPMode.LIBRARY: + log.warning("Configuring datapane logging in library mode") + # return None + + # TODO - only allow setting once? + if _have_setup_logging: + log.warning( + f"Reconfiguring datapane logger when running as {get_dp_mode().name}" + ) + # raise AssertionError("Attempting to reconfigure datapane logger") + return None + + # initial setup via dict-config + _have_setup_logging = True + log_config = { + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "colored": { + "()": "colorlog.ColoredFormatter", + "format": "[%(blue)s%(asctime)s%(reset)s] [%(log_color)s%(levelname)-5s%(reset)s] %(message)s", + "datefmt": "%H:%M:%S", + "reset": True, + "log_colors": { + "DEBUG": "cyan", + "INFO": "green", + "WARNING": "yellow", + "ERROR": "red", + "CRITICAL": "red,bg_white", + }, + "style": "%", + } + }, + "handlers": { + "console": { + "class": "logging.StreamHandler", + "level": log_level, + "formatter": "colored", + "stream": logs_stream or sys.stderr, + } + }, + "loggers": {"datapane": {"level": log_level, "propagate": True}}, + # only show INFO for anything else + "root": {"handlers": ["console"], "level": "INFO"}, + } + logging.config.dictConfig(log_config) + + +def enable_logging(): + """Enable logging for debug purposes""" + _setup_dp_logging(verbosity=2) + + +def print_debug_info(): + """Print useful debugging information""" + fields = dict() + + # Known dependencies + import numpy as np + import pandas as pd + import pyarrow as pa + + fields["pandas_version"] = pd.__version__ + fields["numpy_version"] = np.__version__ + fields["pyarrow_version"] = pa.__version__ + + print("Datapane Debugging Info") + for k, v in fields.items(): + print(f"{k}: {v}") + + +################################################################################ +# Output +def open_in_browser(url: str): + """Open the given URL in the user's browser""" + from eurybia.report.datapane.ipython.environment import get_environment + + environment = get_environment() + + # TODO - this is a bit of a hack, but works for now. JupyterLab (Codespaces) doesn't support webbrowser.open. + if ( + environment.is_notebook_environment + and not environment.can_open_links_from_python + ): + ip = environment.get_ipython() + ip.run_cell_magic("javascript", "", f'window.open("{url}", "_blank")') + else: + import webbrowser + + webbrowser.open(url, new=1) + + +class MarkdownFormatter(string.Formatter): + """Support {:l} and {:cmd} format fields""" + + in_jupyter: bool + + def __init__(self, in_jupyter: bool): + self.in_jupyter = in_jupyter + super().__init__() + + def format_field(self, value: t.Any, format_spec: str) -> t.Any: + if format_spec.endswith("l"): + if self.in_jupyter: + value = f"here" + else: + value = f"at {value}" + format_spec = format_spec[:-1] + elif format_spec.endswith("cmd"): + value = f"!{value}" if self.in_jupyter else value + format_spec = format_spec[:-3] + return super().format_field(value, format_spec) + + +def display_msg(text: str, **params: str): + from eurybia.report.datapane.ipython.environment import get_environment + + environment = get_environment() + + msg = MarkdownFormatter(environment.is_notebook_environment).format(text, **params) + if environment.is_notebook_environment: + from IPython.display import Markdown, display + + display(Markdown(msg)) + else: + print(msg) + + +############################################################ +class DPMode(enum.Enum): + """DP can operate in multiple modes as specified by this Enum""" + + SCRIPT = enum.auto() # run from the cmd-line + LIBRARY = enum.auto() # imported into a process + FRAMEWORK = enum.auto() # running dp-runner + + +# default in Library mode +__dp_mode: DPMode = DPMode.LIBRARY + + +def get_dp_mode() -> DPMode: + global __dp_mode + return __dp_mode + + +def set_dp_mode(dp_mode: DPMode) -> None: + global __dp_mode + __dp_mode = dp_mode diff --git a/eurybia/report/datapane/cloud_api/__init__.py b/eurybia/report/datapane/cloud_api/__init__.py new file mode 100644 index 0000000..14a789a --- /dev/null +++ b/eurybia/report/datapane/cloud_api/__init__.py @@ -0,0 +1,9 @@ +""" +Module to working with Datapane Cloud +""" +# flake8: noqa:F401 +from .common import Resource +from .dp_object import DPObjectRef +from .file import File +from .report import CloudReport +from .user import hello_world, login, logout, ping, signup, template diff --git a/eurybia/report/datapane/cloud_api/common.py b/eurybia/report/datapane/cloud_api/common.py new file mode 100644 index 0000000..e7fc2f5 --- /dev/null +++ b/eurybia/report/datapane/cloud_api/common.py @@ -0,0 +1,84 @@ +"""## Common objects + +Common/shared objects and types used throughout the client API + +..note:: This module is not used directly +""" + +import atexit +import os +import shutil +import time +from datetime import timedelta +from pathlib import Path +from tempfile import gettempdir, mkdtemp, mkstemp + +from typing_extensions import Self + +from eurybia.report.datapane.client import log +from eurybia.report.datapane.common import MIME, guess_type + +################################################################################ +# Tmpfile handling +# We create a tmp-dir per Python execution that stores all working files, +# we attempt to delete where possible, but where not, we allow the atexit handler +# to cleanup for us on shutdown +cache_dir = Path(gettempdir()) + +# Remove any old dp-tmp-* dirs over 24hrs old which might not have been cleaned up due to unexpected exit +one_day_ago = time.time() - timedelta(days=1).total_seconds() +prev_tmp_dirs = ( + p + for p in cache_dir.glob("dp-tmp-*") + if p.is_dir() and p.stat().st_mtime < one_day_ago +) +for p in prev_tmp_dirs: + log.debug(f"Removing stale temp dir {p}") + shutil.rmtree(p, ignore_errors=True) + +# create new dp-tmp for this session +tmp_dir = Path(mkdtemp(prefix="dp-tmp-", dir=cache_dir)).absolute() + + +class DPTmpFile: + """ + Generate a tempfile in dp temp dir + when used as a contextmanager, deleted on removing scope + otherwise, removed by atexit hook + """ + + def __init__(self, ext: str): + fd, name = mkstemp(suffix=ext, prefix="dp-tmp-", dir=tmp_dir) + os.close(fd) + self.file = Path(name) + + def __enter__(self) -> Self: + return self + + def __exit__(self, exc_type, exc_value, exc_traceback): # noqa: ANN001 + log.debug(f"Removing {self.name}") + if self.file.exists(): + self.file.unlink() # (missing_ok=True) + + @property + def name(self) -> str: + return str(self.file) + + @property + def full_name(self) -> str: + return str(self.file.absolute()) + + @property + def mime(self) -> MIME: + return guess_type(self.file) + + def __str__(self) -> str: + return self.name + + +@atexit.register +def cleanup_tmp(): + """Ensure we cleanup the tmp_dir on Python VM exit""" + # breaks tests + # log.debug(f"Removing current session DP tmp work dir {tmp_dir}") + shutil.rmtree(tmp_dir, ignore_errors=True) diff --git a/eurybia/report/datapane/common/__init__.py b/eurybia/report/datapane/common/__init__.py new file mode 100644 index 0000000..d5a7c5c --- /dev/null +++ b/eurybia/report/datapane/common/__init__.py @@ -0,0 +1,35 @@ +""" +Shared code used by client and dp-server +NOTE - this module should not depend on any client or server specific code and is imported first +""" +# Copyright 2020 StackHut Limited (trading as Datapane) +# SPDX-License-Identifier: Apache-2.0 +# flake8: noqa:F401 +from .datafiles import ArrowFormat +from .dp_types import ( + ARROW_EXT, + ARROW_MIMETYPE, + HTML, + JSON, + MIME, + PKL_MIMETYPE, + SECS_1_HOUR, + SECS_1_WEEK, + SIZE_1_MB, + TD_1_DAY, + TD_1_HOUR, + URL, + DPError, + EnumType, + Hash, + JDict, + JList, + NPath, + SDict, + SList, + SSDict, + log, +) +from .ops_utils import pushd, timestamp +from .utils import dict_drop_empty, guess_type, utf_read_text +from .viewxml_utils import ViewXML, load_doc, validate_view_doc diff --git a/eurybia/report/datapane/common/datafiles.py b/eurybia/report/datapane/common/datafiles.py new file mode 100644 index 0000000..e77e06c --- /dev/null +++ b/eurybia/report/datapane/common/datafiles.py @@ -0,0 +1,119 @@ +"""Dataset Format handling""" +import abc +import enum +from typing import IO, Dict, Type, Union + +import pandas as pd +import pyarrow as pa +from pandas.errors import ParserError +from pyarrow import RecordBatchFileWriter + +from .df_processor import obj_to_str, process_df, str_to_arrow_str +from .dp_types import ARROW_EXT, ARROW_MIMETYPE, MIME, log +from .utils import guess_encoding + + +def write_table(table: pa.Table, sink: Union[str, IO[bytes]]): + """Write an arrow table to a file""" + writer = RecordBatchFileWriter(sink, table.schema) + writer.write(table) + writer.close() + + +PathOrFile = Union[str, IO] + + +class DFFormatter(abc.ABC): + # TODO - tie to mimetypes lib + content_type: MIME + ext: str + enum: str + + @staticmethod + @abc.abstractmethod + def load_file(fn: PathOrFile) -> pd.DataFrame: + pass + + @staticmethod + @abc.abstractmethod + def save_file(fn: PathOrFile, df: pd.DataFrame): + pass + + +DFFormatterCls = Type[DFFormatter] + + +class ArrowFormat(DFFormatter): + content_type = ARROW_MIMETYPE + ext = ARROW_EXT + enum = "ARROW" + + @staticmethod + def load_file(fn: PathOrFile) -> pd.DataFrame: + df = pa.ipc.open_file(fn).read_pandas() + # NOTE - need to convert categories from object to string https://github.com/apache/arrow/issues/33070 + obj_to_str(df) + str_to_arrow_str(df) + return df + + @staticmethod + def save_file(fn: PathOrFile, df: pd.DataFrame): + df = process_df(df) + # NOTE - can pass expected schema and columns for output df here + table: pa.Table = pa.Table.from_pandas(df, preserve_index=False) + write_table(table, fn) + + +class CSVFormat(DFFormatter): + content_type = MIME("text/csv") + ext = ".csv" + enum = "CSV" + + @staticmethod + def load_file(fn: PathOrFile) -> pd.DataFrame: + # TODO - fix + if not isinstance(fn, str): + raise ValueError("FObj not yet supported") + + try: + return pd.read_csv(fn, engine="c", sep=",") + except UnicodeDecodeError: + encoding = guess_encoding(fn) + return pd.read_csv(fn, engine="c", sep=",", encoding=encoding) + except ParserError as e: + log.warning(f"Error parsing CSV file ({e}), trying python fallback") + try: + return pd.read_csv(fn, engine="python", sep=None) + except UnicodeDecodeError: + encoding = guess_encoding(fn) + return pd.read_csv(fn, engine="python", sep=None, encoding=encoding) + + @staticmethod + def save_file(fn: PathOrFile, df: pd.DataFrame): + df.to_csv(fn, index=False) + + +class ExcelFormat(DFFormatter): + content_type = MIME("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet") + ext = ".xlsx" + enum = "EXCEL" + + @staticmethod + def load_file(fn: PathOrFile) -> pd.DataFrame: + return pd.read_excel(fn, engine="openpyxl") + + @staticmethod + def save_file(fn: PathOrFile, df: pd.DataFrame): + df.to_excel(fn, index=False, engine="openpyxl") + + +class DatasetFormats(enum.Enum): + """Used to switch between the different format handlers""" + + CSV = CSVFormat + EXCEL = ExcelFormat + ARROW = ArrowFormat + + +# TODO - make into enums? +df_ext_map: Dict[str, DFFormatterCls] = {x.value.ext: x.value for x in DatasetFormats} diff --git a/eurybia/report/datapane/common/df_processor.py b/eurybia/report/datapane/common/df_processor.py new file mode 100644 index 0000000..164768b --- /dev/null +++ b/eurybia/report/datapane/common/df_processor.py @@ -0,0 +1,228 @@ +import datetime +from numbers import Number +from typing import Any + +import numpy as np +import pandas as pd +from packaging.specifiers import SpecifierSet +from packaging.version import Version + +PD_VERSION = Version(pd.__version__) +PD_1_3_GREATER = SpecifierSet(">=1.3.0") + + +def convert_axis(df: pd.DataFrame): + """flatten both columns and indexes""" + + # flatten hierarchical columns and convert to strings + if df.columns.nlevels > 1: + df.columns = ["/".join(a) for a in df.columns.to_flat_index()] + + df.columns = df.columns.astype("string") + + # flatten/reset indexes + if isinstance(df.index, pd.RangeIndex): + pass # allow RangeIndex - reset numbers? + elif isinstance(df.index, pd.Int64Index): + df.reset_index(inplace=True, drop=True) + else: + # reset if any other index type, e.g. MultiIndex, custom Index + # all new column dtypes will converted in latter functions + df.reset_index(inplace=True) + + +def downcast_numbers(data: pd.DataFrame): + """Downcast numerics""" + + def downcast_ints(ser: pd.Series) -> pd.Series: + try: + ser = pd.to_numeric(ser, downcast="signed") + ser = pd.to_numeric(ser, downcast="unsigned") + except Exception: + pass # catch failure on Int64Dtype + return ser + + # A result of downcast(timedelta64[ns]) is int and hard to understand. + # e.g.) 0 days 00:54:38.777572 -> 3278777572000 [ns] + df_num = data.select_dtypes("integer", exclude=["timedelta"]) # , pd.Int64Dtype]) + data[df_num.columns] = df_num.apply(downcast_ints) + + def downcast_floats(ser: pd.Series) -> pd.Series: + ser = pd.to_numeric(ser, downcast="float", errors="ignore") + return ser + + # float downcasting currently disabled - alters values (both float64 and int64) and rounds to 'inf' instead of erroring + # see https://github.com/pandas-dev/pandas/issues/19729 + # df_num = data.select_dtypes("floating") + # data[df_num.columns] = df_num.apply(downcast_floats) + + +def timedelta_to_str(df: pd.DataFrame): + """ + convert timedelta to str + NOTE - only until arrow.js supports Duration type + """ + df_td = df.select_dtypes("timedelta") + df[df_td.columns] = np.where(pd.isnull(df_td), pd.NA, df_td.astype("string")) + + +def parse_categories(data: pd.DataFrame): + """Detect and converts categories""" + + def criteria(ser: pd.Series) -> bool: + """Decides whether to convert into categorical""" + nunique: int = ser.nunique() + + if nunique <= 20 and (nunique != ser.size): + # few unique values => make it a category regardless of the proportion + return True + + prop_unique = (nunique + 1) / (ser.size + 1) # + 1 for nan + + if prop_unique <= 0.05: + # a lot of redundant information => categories are more compact + return True + + return False + + def try_to_category(ser: pd.Series) -> pd.Series: + return ser.astype("category") if criteria(ser) else ser + + potential_cats = data.select_dtypes(["string", "object"]) + data[potential_cats.columns] = potential_cats.apply(try_to_category) + + +def obj_to_str(df: pd.DataFrame): + """Converts remaining objects columns to str""" + # convert objects to str / NA + df_str = df.select_dtypes("object") + df[df_str.columns] = df_str.astype("string") + + # convert categorical values (as strings if object) + def to_str_cat_vals(x: pd.Series) -> pd.Series: + if x.cat.categories.dtype == np.dtype("object"): + x.cat.categories = x.cat.categories.astype("string") + return x + + df_cat = df.select_dtypes("category") + df[df_cat.columns] = df_cat.apply(to_str_cat_vals) + + +def bipartite_to_bool(df: pd.DataFrame): + # This was removed from our processing steps as some users required the numerical representation of binary columns. + """Converts biperatite numeric {0, 1} columns to bool.""" + # Get names of numeric columns with only 2 unique values. + df_num = df.select_dtypes("integer", exclude=["timedelta"]) + bipartite_columns = df_num.columns[df_num.dropna().nunique() == 2] + + for column in bipartite_columns: + series = df[column] + + # Only apply the type change to {0, 1} columns + val_range = series.min(), series.max() + val_min, val_max = val_range[0], val_range[1] + + if val_min == 0 and val_max == 1: + df[column] = df[column].astype(bool) + + +def str_to_arrow_str(df: pd.DataFrame): + """Use the memory-efficient pyarrow string dtype (pandas >= 1.3 only)""" + # convert objects to str / NA + df_str = df.select_dtypes("string") + df[df_str.columns] = df_str.astype("string[pyarrow]") + + +def process_df(df: pd.DataFrame, copy: bool = False) -> pd.DataFrame: + """ + Processing steps needed before writing / after reading + We only modify the dataframe to optimise size, + rather than convert/infer types, e.g. no longer parsing dates from strings + + NOTE - this mutates the dataframe by default but returns it - use the returned copy! + """ + if copy: + df = df.copy(deep=True) + + convert_axis(df) + + # convert timedelta + timedelta_to_str(df) + + # NOTE - pandas >= 1.3 handles downcasting of nullable values correctly + df = df.convert_dtypes() + downcast_numbers(df) + + # save timedeltas cols (unneeded whilst timedelta_to_str used) + # td_col = df.select_dtypes("timedelta") + # df[td_col.columns] = td_col + obj_to_str(df) + parse_categories(df) + + # convert all strings to use the arrow dtype + str_to_arrow_str(df) + + return df + + +def to_df(value: Any) -> pd.DataFrame: + """ + Converts a python object, i.e. a app's output, to a dataframe + NOTE - this returns a new DF each time + """ + if value is None: + # This return the empty dataframe, which atm is the same as + # the empty file object in the CAS. + # However this is not ensured as pyarrow changes + return pd.DataFrame() + + if isinstance(value, pd.DataFrame): + return value.copy(deep=True) + + if isinstance(value, (pd.Series, pd.Index)): + if value.name is not None: + return pd.DataFrame(value) + + return pd.DataFrame({"Result": value}) + + if isinstance(value, (Number, str, bool, datetime.datetime, datetime.timedelta)): + return pd.DataFrame({"Result": value}, index=[0]) + + if isinstance(value, np.ndarray): + try: + out_df = pd.DataFrame(value) + except ValueError: + squeezed = np.squeeze(value) + if squeezed.shape == (): + # must be a scalar + out_df = pd.DataFrame({"Result": squeezed}, index=[0]) + else: + out_df = pd.DataFrame(squeezed) + + if out_df.columns.tolist() == [0]: + out_df.columns = ["Result"] + + return out_df + + raise ValueError("Must return a primitive, pd.DataFrame, pd.Series or numpy array.") + + +TRUNCATE_CELLS = 10000 +TRUNCATE_ROWS = 1000 + + +def truncate_dataframe( + df: pd.DataFrame, max_rows: int = TRUNCATE_ROWS, max_cells: int = TRUNCATE_CELLS +) -> pd.DataFrame: + """Truncate a pandas dataframe if needed""" + rows, cols = df.shape + # determine max rows to truncate df to based on max cells and df cols + cols = cols or 1 # handle empty df + max_rows = min(max_rows, int(max_cells / cols)) + # return non-truncated preview if df smaller than max rows allowed + if rows <= max_rows: + return df + # truncate df to fit max cells + if not isinstance(df.index, pd.RangeIndex): + raise ValueError("Dataframe has unsupported index type") + return df.truncate(before=0, after=max_rows - 1, copy=False) diff --git a/eurybia/report/datapane/common/dp_types.py b/eurybia/report/datapane/common/dp_types.py new file mode 100644 index 0000000..d33e849 --- /dev/null +++ b/eurybia/report/datapane/common/dp_types.py @@ -0,0 +1,49 @@ +import logging +import typing as t +from datetime import timedelta +from enum import Enum +from os import PathLike +from pathlib import Path + +# Typedefs +# A JSON-serialisable config object +SDict = t.Dict[str, t.Any] +SSDict = t.Dict[str, str] +SList = t.List[str] +# NOTE - mypy cannot handle recursive types like this currently. Will review in the future +JSON = t.Union[str, int, float, bool, None, t.Mapping[str, "JSON"], t.List["JSON"]] # type: ignore +JDict = SDict # should be JSON +JList = t.List[JSON] # type: ignore +MIME = t.NewType("MIME", str) +URL = t.NewType("URL", str) +HTML = t.NewType("HTML", str) +NPath = t.Union[Path, PathLike, str] +Hash = t.NewType("Hash", str) +EnumType = int # alias for enum values + +# Constants +# NOTE - PKL_MIMETYPE and ARROW_MIMETYPE are custom mimetypes +PKL_MIMETYPE = MIME("application/vnd.pickle+binary") +ARROW_MIMETYPE = MIME("application/vnd.apache.arrow+binary") +ARROW_EXT = ".arrow" +TD_1_HOUR = timedelta(hours=1) +TD_1_DAY = timedelta(days=1) +SECS_1_HOUR: int = int(TD_1_HOUR.total_seconds()) +SECS_1_WEEK: int = int(timedelta(weeks=1).total_seconds()) +SIZE_1_MB: int = 1024 * 1024 + + +class StrEnum(str, Enum): + # TODO - replace with StrEnum in py3.11 stdlib + def __str__(self): + return str(self.value) + + +# Errors +class DPError(Exception): + """Base DP Error""" + + pass + + +log = logging.getLogger("datapane") diff --git a/eurybia/report/datapane/common/ops_utils.py b/eurybia/report/datapane/common/ops_utils.py new file mode 100644 index 0000000..f3ff679 --- /dev/null +++ b/eurybia/report/datapane/common/ops_utils.py @@ -0,0 +1,143 @@ +import datetime +import gzip +import io +import os +import shutil +import subprocess +import time +import typing as t +from contextlib import contextmanager +from pathlib import Path +from tempfile import NamedTemporaryFile, TemporaryDirectory, mkstemp + +from .dp_types import NPath, log +from .utils import ON_WINDOWS + + +@contextmanager +def log_command(command: str) -> t.Generator[None, None, None]: + """Log an internal process""" + log.info(f"Starting {command}") + yield + log.info(f"Finished {command}") + + +@contextmanager +def create_temp_file( + suffix: str, prefix: str = "datapane-temp-", mode: str = "w+b" +) -> t.Generator[NamedTemporaryFile, None, None]: + """Creates a NamedTemporaryFile that doesn't disappear on .close()""" + temp_file = NamedTemporaryFile(suffix=suffix, prefix=prefix, mode=mode, delete=False) + try: + yield temp_file + finally: + os.unlink(temp_file.name) + + +@contextmanager +def temp_fname(suffix: str, prefix: str = "datapane-temp-", keep: bool = False) -> t.Generator[str, None, None]: + """Wrapper to generate a temporary filename only that is deleted on leaving context""" + # TODO - return Path + (in_f, in_f_name) = mkstemp(suffix=suffix, prefix=prefix) + try: + os.close(in_f) + yield in_f_name + finally: + if os.path.exists(in_f_name) and not keep: + os.unlink(in_f_name) + + +@contextmanager +def unix_compress_file(f_name: NPath, level: int = 6) -> t.Generator[str, None, None]: + """(UNIX only) Return path to a compressed version of the input filename""" + subprocess.run(["gzip", "-kf", f"-{level}", f_name], check=True) + f_name_gz = f"{f_name}.gz" + try: + yield f_name_gz + finally: + os.unlink(f_name_gz) + + +@contextmanager +def unix_decompress_file(f_name: NPath) -> t.Generator[str, None, None]: + """(UNIX only) Return path to a compressed version of the input filename""" + subprocess.run(["gunzip", "-kf", f_name], check=True) + f_name_gz = f"{f_name}.gz" + try: + yield f_name_gz + finally: + os.unlink(f_name_gz) + + +@contextmanager +def compress_file(f_name: NPath, level: int = 6) -> t.Generator[str, None, None]: + """(X-Plat) Return path to a compressed version of the input filename""" + f_name_gz = f"{f_name}.gz" + with open(f_name, "rb") as f_in, gzip.open(f_name_gz, "wb", compresslevel=level) as f_out: + shutil.copyfileobj(f_in, f_out) + try: + yield f_name_gz + finally: + # NOTE - disable on windows temporarily + if not ON_WINDOWS: + os.unlink(f_name_gz) + + +def inmemory_compress(content: t.BinaryIO) -> t.BinaryIO: + """(x-plat) Memory-based gzip compression""" + content.seek(0) + zbuf = io.BytesIO() + with gzip.GzipFile(mode="wb", fileobj=zbuf, mtime=0.0) as zfile: + zfile.write(content.read()) + zbuf.seek(0) + return zbuf + + +@contextmanager +def temp_workdir() -> t.Generator[str, None, None]: + """Set working dir to a tempdir for duration of context""" + with TemporaryDirectory() as tmp_dir: + curdir = os.getcwd() + os.chdir(tmp_dir) + try: + yield None + finally: + os.chdir(curdir) + + +@contextmanager +def pushd(directory: NPath, pre_create: bool = False, post_remove: bool = False) -> t.Generator[None, None, None]: + """Switch dir and push it onto the (call-)stack""" + directory = Path(directory) + cwd = os.getcwd() + log.debug(f"[cd] {cwd} -> {directory}") + if not directory.exists() and pre_create: + Path(directory).mkdir(parents=True) + os.chdir(directory) + try: + yield + finally: + log.debug(f"[cd] {cwd} <- {directory}") + os.chdir(cwd) + if post_remove: + shutil.rmtree(directory, ignore_errors=True) + + +def get_filesize(filename: Path) -> int: + return filename.stat().st_size + + +def walk_path(path: Path) -> t.Iterable[Path]: + for p in path.rglob("*"): + if not p.is_dir(): + yield p + + +def unixtime() -> int: + return int(time.time()) + + +def timestamp(x: t.Optional[datetime.datetime] = None) -> str: + """Return ISO timestamp for a datetime""" + x = x or datetime.datetime.utcnow() + return f'{x.isoformat(timespec="seconds")}{"" if x.tzinfo else "Z"}' diff --git a/eurybia/report/datapane/common/utils.py b/eurybia/report/datapane/common/utils.py new file mode 100644 index 0000000..13fdff0 --- /dev/null +++ b/eurybia/report/datapane/common/utils.py @@ -0,0 +1,121 @@ +import locale +import logging +import mimetypes +import re +import sys +import typing as t +from pathlib import Path + +import chardet + +if sys.version_info < (3, 10): + import importlib_resources as ir +else: + from importlib import resources as ir +from chardet.universaldetector import UniversalDetector + +from .dp_types import MIME + +log = logging.getLogger("datapane") + +################################################################################ +# CONSTANTS +ON_WINDOWS = sys.platform == "win32" + +################################################################################ +# MIME-type handling +mimetypes.init( + files=[str(ir.files("eurybia.report.datapane.resources") / "mime.types")] +) + +# TODO - hardcode as temporary fix until mimetypes double extension issue is sorted +_double_ext_map = { + ".vl.json": "application/vnd.vegalite.v5+json", + ".vl2.json": "application/vnd.vegalite.v2+json", + ".vl3.json": "application/vnd.vegalite.v3+json", + ".vl4.json": "application/vnd.vegalite.v4+json", + ".vl5.json": "application/vnd.vegalite.v5+json", + ".bokeh.json": "application/vnd.bokeh.show+json", + ".pl.json": "application/vnd.plotly.v1+json", + ".fl.html": "application/vnd.folium+html", + ".tbl.html": "application/vnd.datapane.table+html", + ".tar.gz": "application/x-tgz", +} +double_ext_map: t.Dict[str, MIME] = {k: MIME(v) for k, v in _double_ext_map.items()} + + +def guess_type(filename: Path) -> MIME: + ext = "".join(filename.suffixes) + if ext in double_ext_map.keys(): + return double_ext_map[ext] + mtype: str + mtype, _ = mimetypes.guess_type(str(filename)) + return MIME(mtype or "application/octet-stream") + + +def guess_encoding(fn: str) -> str: + with open(fn, "rb") as f: + detector = UniversalDetector() + for line in f.readlines(): + detector.feed(line) + if detector.done: + break + detector.close() + return detector.result["encoding"] + + +def utf_read_text(file: Path) -> str: + """Encoding-aware text reader + - handles cases like on Windows where a file is UTF-8, but default locale is windows-1252 + """ + if ON_WINDOWS: + f_bytes = file.read_bytes() + f_enc: str = chardet.detect(f_bytes)["encoding"] + # NOTE - can just special case utf-8 files here? + def_enc = locale.getpreferredencoding() + log.debug(f"Default encoding is {def_enc}, file encoded as {f_enc}") + if def_enc.upper() != f_enc.upper(): + log.warning(f"Text file {file} encoded as {f_enc}, auto-converting") + return f_bytes.decode(encoding=f_enc) + else: + # for linux/macOS assume utf-8 + return file.read_text() + + +def dict_drop_empty( + xs: t.Optional[t.Dict] = None, none_only: bool = False, **kwargs +) -> t.Dict: + """Return a new dict with the empty/falsey values removed""" + xs = {**(xs or {}), **kwargs} + + if none_only: + return {k: v for (k, v) in xs.items() if v is not None} + else: + return {k: v for (k, v) in xs.items() if v or isinstance(v, bool)} + + +def should_compress_mime_type_for_upload(mime_type: str) -> bool: + # This strategy is based on: + # - looking at mime type databases used by `mimetypes` module + # - our custom mime types in double_ext_map + # - some other online sources that capture real-world usage: + # - https://letstalkaboutwebperf.com/en/gzip-brotli-server-config/ + # - https://github.com/h5bp/server-configs-nginx/blob/main/mime.types + return any( + pattern.search(mime_type) for pattern in _SHOULD_COMPRESS_MIME_TYPE_REGEXPS + ) + + +_SHOULD_COMPRESS_MIME_TYPE_REGEXPS = [ + re.compile(p) + for p in [ + r"^text/", + r"\+json$", + r"\+xml$", + r"\+html$", + r"^application/json$", + r"^application/vnd\.pickle\+binary$", + r"^application/vnd\.apache\.arrow\+binary$", + r"^application/xml$", + ] +] diff --git a/eurybia/report/datapane/common/versioning.py b/eurybia/report/datapane/common/versioning.py new file mode 100644 index 0000000..2edee3f --- /dev/null +++ b/eurybia/report/datapane/common/versioning.py @@ -0,0 +1,35 @@ +from typing import Union + +from packaging import version as v +from packaging.specifiers import SpecifierSet + +from .dp_types import log + + +class VersionMismatch(Exception): + pass + + +def is_version_compatible( + provider_v_in: Union[str, v.Version], + consumer_v_in: Union[str, v.Version], + raise_exception: bool = True, +) -> bool: + """ + Check provider supports consumer and throws exception if not + + Set the spec so that the consumer has to be within a micro/patch release of the provider + NOTE - this isn't semver - breaks when have > v1 release as then treats minor as breaking, + e.g. 2.2.5 is not compat with 2.1.5 + """ + consumer_v = v.Version(consumer_v_in) if isinstance(consumer_v_in, str) else consumer_v_in + provider_v = v.Version(provider_v_in) if isinstance(provider_v_in, str) else provider_v_in + + provider_spec = SpecifierSet(f"~={provider_v.major}.{provider_v.minor}.0") + + log.debug(f"Provider spec {provider_spec}, Consumer version {consumer_v}") + if consumer_v not in provider_spec: + if raise_exception: + raise VersionMismatch(f"Consumer ({consumer_v}) and Provider ({provider_spec}) API versions not compatible") + return False + return True diff --git a/eurybia/report/datapane/common/viewxml_utils.py b/eurybia/report/datapane/common/viewxml_utils.py new file mode 100644 index 0000000..3d20921 --- /dev/null +++ b/eurybia/report/datapane/common/viewxml_utils.py @@ -0,0 +1,127 @@ +import dataclasses as dc +import json +import math +import re +import typing as t +import sys +from collections.abc import Sized +from numbers import Number + +if sys.version_info < (3, 10): + import importlib_resources as ir +else: + from importlib import resources as ir +from lxml import etree +from lxml.etree import DocumentInvalid +from lxml.etree import _Element as ElementT +from micawber import ProviderException, bootstrap_basic, bootstrap_noembed, cache + +from .dp_types import HTML, DPError, SSDict, log + +local_view_resources = ir.files("eurybia.report.datapane.resources.view_resources") +rng_validator = etree.RelaxNG(file=str(local_view_resources / "full_schema.rng")) + +dp_namespace: str = "https://datapane.com/schemas/report/1/" +ViewXML = str + + +def load_doc(x: str) -> ElementT: + parser = etree.XMLParser( + strip_cdata=False, recover=True, remove_blank_text=True, remove_comments=True + ) + return etree.fromstring(x, parser=parser) + + +def is_valid_id(id: str) -> bool: + """(cached) regex to check for a xsd:ID name""" + return re.fullmatch(r"^[a-zA-Z_][\w.-]*$", id) is not None + + +def validate_view_doc( + xml_str: t.Optional[str] = None, + xml_doc: t.Optional[ElementT] = None, + quiet: bool = False, +) -> bool: + """Validate the model against the schema, throws an etree.DocumentInvalid if not""" + assert xml_str or (xml_doc is not None) + if xml_str: + xml_doc = etree.fromstring(xml_str) + + try: + rng_validator.assertValid(xml_doc) + return True + except DocumentInvalid: + if not quiet: + xml_str = ( + xml_str if xml_str else etree.tounicode(xml_doc, pretty_print=True) + ) + log.error( + f"Error validating report document:\n\n{xml_str}\n{rng_validator.error_log}\n" + ) + raise + + +def conv_attrib(v: t.Any) -> t.Optional[str]: + """ + Convert a value to a str for use as an ElementBuilder attribute + - also handles None to a string for optional field values + """ + # TODO - use a proper serialisation framework here / lxml features + if v is None: + return v + elif isinstance(v, Sized) and len(v) == 0: + return None + elif isinstance(v, str): + return v + elif isinstance(v, Number) and type(v) != bool: + if math.isinf(v) and v > 0: + return "INF" + elif math.isinf(v) and v < 0: + return "-INF" + elif math.isnan(v): + return "NaN" + else: + return str(v) + else: + return json.dumps(v) + + +def mk_attribs(**attribs: t.Any) -> SSDict: + """convert attributes, dropping None and empty values""" + return { + str(k): v1 for (k, v) in attribs.items() if (v1 := conv_attrib(v)) is not None + } + + +##################################################################### +# Embed Asset Helpers +providers = bootstrap_basic(cache=cache.Cache()) + + +@dc.dataclass(frozen=True) +class Embedded: + html: HTML + title: str + provider: str + + +def get_embed_url(url: str, width: int = 960, height: int = 540) -> Embedded: + """Return html for an embeddable URL""" + try: + r = providers.request(url, maxwidth=width, maxheight=height) + except ProviderException: + # add noembed to the list and try again + try: + log.debug("Initialising NoEmbed OEmbed provider") + bootstrap_noembed(registry=providers) + r = providers.request(url, maxwidth=width, maxheight=height) + except ProviderException: + raise DPError( + f"No embed provider found for URL '{url}' - is there an active internet connection?" + ) + + return Embedded( + html=r["html"], + title=r.get("title", "Title"), + provider=r.get("provider_name", "Embedding"), + ) diff --git a/eurybia/report/datapane/ipython/__init__.py b/eurybia/report/datapane/ipython/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/eurybia/report/datapane/ipython/environment.py b/eurybia/report/datapane/ipython/environment.py new file mode 100644 index 0000000..a471f73 --- /dev/null +++ b/eurybia/report/datapane/ipython/environment.py @@ -0,0 +1,264 @@ +from __future__ import annotations + +import io +import json +import os +import sys +import typing as t +from functools import cached_property +from pathlib import Path + +from eurybia.report.datapane.client import log +from eurybia.report.datapane.client.exceptions import DPClientError + +from .exceptions import NotebookException + +if t.TYPE_CHECKING: + from IPython.core.interactiveshell import InteractiveShell + +__all__ = ("get_environment",) + + +_env = None + + +def get_environment() -> PythonEnvironment: + """Returns the current IPython environment""" + global _env + if _env is None: + _env = _get_environment() + log.info("Detected IPython environment: %s", _env.name) + return _env + + +def _get_ipython() -> t.Optional[InteractiveShell]: + try: + return get_ipython() # type: ignore + except NameError: + from IPython import get_ipython + + return get_ipython() + + +def is_zqm_interactive_shell() -> bool: + """Check if we are running in a ZMQ interactive shell""" + return _get_ipython().__class__.__name__ == "ZMQInteractiveShell" + + +def is_terminal_interactive_shell() -> bool: + """Check if we are running in a terminal interactive shell""" + return _get_ipython().__class__.__name__ == "TerminalInteractiveShell" + + +def get_ipython_user_ns() -> dict: + return _get_ipython().user_ns + + +def _get_environment() -> PythonEnvironment: + """Determines the current IPython environment and returns an instance of the appropriate class""" + + # TODO: change this to be a list, and put the logic in a `.supported()` method + if "google.colab" in sys.modules: + return ColabEnvironment() + elif {"CODESPACES", "JUPYTERLAB_PATH", "VSCODE_CWD"}.issubset(set(os.environ)): + return CodespacesVSCodeJupyterEnvironment() + elif {"CODESPACES", "JUPYTERLAB_PATH", "JPY_PARENT_PID"}.issubset(set(os.environ)): + return CodespacesJupyterLabEnvironment() + elif ( + is_zqm_interactive_shell() and "PAPERMILL_OUTPUT_PATH" in get_ipython_user_ns() + ): + return PapermillEnvironment() + elif is_zqm_interactive_shell() and "VSCODE_PID" in os.environ: + return VSCodeJupyterEnvironment() + elif is_zqm_interactive_shell() and {"JPY_SESSION_NAME", "JPY_PARENT_PID"}.issubset( + set(os.environ) + ): + return JupyterLabEnvironment() + elif is_zqm_interactive_shell() and "JPY_PARENT_PID" in os.environ: + return JupyterNotebookEnvironment() + elif is_zqm_interactive_shell(): + return UnsupportedNotebookEnvironment() + elif "PYCHARM_HOSTED" in os.environ: + return PyCharmEnvironment() + elif "VSCODE_PID" in os.environ: + return VSCodeEnvironment() + elif is_terminal_interactive_shell(): + return IPythonTerminalEnvironment() + else: + return UnrecognizedEnvironment() + + +class PythonEnvironment: + name: str = "Python Environment" + can_open_links_from_python: bool = True + is_notebook_environment: bool = False + support_rich_display: bool = True + supports_ipywidgets: bool = False + + get_ipython = staticmethod(_get_ipython) + + def get_notebook_json(self) -> dict: + try: + path = self._get_notebook_path() + with open(path, encoding="utf-8") as f: + notebook_json = json.load(f) + except FileNotFoundError: + raise DPClientError( + "Notebook not found. This command must be executed from within a Jupyter notebook environment." + ) + return notebook_json + + def _get_notebook_path(self) -> Path: + """Get the path to the current notebook""" + raise FileNotFoundError("Could not discover notebook path") + + +class UnrecognizedEnvironment(PythonEnvironment): + name = "Unrecognized Environment" + is_notebook_environment = False + can_open_links_from_python = False + + +class IPythonTerminalEnvironment(PythonEnvironment): + name = "IPython Terminal Environment" + + +class IPythonZMQEnvironment(IPythonTerminalEnvironment): + name = "IPython ZMQ Environment" + is_notebook_environment = True + + def _get_notebook_path(self) -> Path: + # added in ipykernel v6.21 + # https://github.com/ipython/ipykernel/pull/1078 + user_ns = get_ipython_user_ns() + + # If it's an actual path, it'll always be absolute + if (path := user_ns.get("__session__", None)) and path.startswith("/"): + return Path(path) + + return super()._get_notebook_path() + + +class UnsupportedNotebookEnvironment(IPythonZMQEnvironment): + name = "Unsupported Notebook Environment" + + +class PyCharmEnvironment(PythonEnvironment): + name = "PyCharm Environment" + support_rich_display = False + + +class VSCodeEnvironment(PythonEnvironment): + name = "VSCode Environment" + + +class JupyterEnvironment(IPythonZMQEnvironment): + name = "Jupyter Environment" + supports_ipywidgets = True + + def _get_notebook_path(self): + """Get the path for the current Jupyter notebook""" + + # ipynbname plays with the session API, so see if ipykernel gave us a better option + try: + return super()._get_notebook_path() + except FileNotFoundError as e: + import ipynbname + + try: + return ipynbname.path() + except IndexError: + raise e + + +class JupyterLabEnvironment(JupyterEnvironment): + name = "JupyterLab Environment" + + +class JupyterNotebookEnvironment(JupyterEnvironment): + name = "Jupyter Notebook Environment" + + +class VSCodeJupyterEnvironment(JupyterEnvironment): + name = "VSCode Jupyter Environment" + + # Attempting to use IPyWidgetsControllerUI from within VSCode Jupyter extension + # seems extremely flakey: + # - server often fails to start or stop + # - once it has failed once, it can require restarting VSCode to fix things. + # So it's better to fall back to something that seems to work better. + # + supports_ipywidgets = False + + def _get_notebook_path(self) -> Path: + user_ns = get_ipython_user_ns() + if vsc_path := user_ns.get("__vsc_ipynb_file__", None): + return Path(vsc_path) + + # VSCode path name in sesssion may be suffixed with -jvsc-[identifier] + # https://github.com/microsoft/vscode-jupyter/blob/113c3e54ac1d3cb81ab6473d1a5fa4a20cce4755/src/kernels/helpers.ts#L149-L168 + sessioned_nb_path = super()._get_notebook_path() + suffix = sessioned_nb_path.suffix + sans_suffix = sessioned_nb_path.with_suffix("") + nb_path = Path(str(sans_suffix).split("-jvsc-")[0]).with_suffix(suffix) + return nb_path + + +class ColabEnvironment(IPythonZMQEnvironment): + name = "Google Colab Environment" + + def get_notebook_json(self) -> dict: + """Get the JSON for the current Colab notebook""" + import ipynbname + from google.colab import auth + from googleapiclient.discovery import build + from googleapiclient.http import MediaIoBaseDownload + + # Get the notebook's Google Drive file_id + file_id = ipynbname.name().replace("fileId=", "") + + try: + auth.authenticate_user() + except Exception as e: + raise NotebookException( + "Google Drive authentication failed. Please allow this notebook to access your Google Drive." + ) from e + + drive_service = build("drive", "v3") + + request = drive_service.files().get_media(fileId=file_id) + downloaded = io.BytesIO() + downloader = MediaIoBaseDownload(downloaded, request) + done = False + while done is False: + # _ is a placeholder for a progress object that we ignore. + # (Our file is small, so we skip reporting progress.) + _, done = downloader.next_chunk() + + downloaded.seek(0) + notebook_json = json.loads(downloaded.read().decode("utf-8")) + + return notebook_json + + +class CodespacesVSCodeJupyterEnvironment(VSCodeJupyterEnvironment): + name = "Codespaces VSCode Jupyter Environment" + + +class CodespacesJupyterLabEnvironment(JupyterLabEnvironment): + name = "Codespaces JupyterLab Environment" + can_open_links_from_python = False + + +class PapermillEnvironment(IPythonZMQEnvironment): + name = "Papermill Environment" + + def _get_notebook_path(self) -> Path: + user_ns = get_ipython_user_ns() + if path := user_ns.get("PAPERMILL_OUTPUT_PATH", None): + return Path(path) + return super()._get_notebook_path() + + @cached_property + def support_rich_display(self) -> bool: # type: ignore[override] + return get_ipython_user_ns().get("DP_SERVER_RUNNER", False) diff --git a/eurybia/report/datapane/ipython/exceptions.py b/eurybia/report/datapane/ipython/exceptions.py new file mode 100644 index 0000000..57f4a61 --- /dev/null +++ b/eurybia/report/datapane/ipython/exceptions.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from eurybia.report.datapane.client import display_msg + + +class NotebookException(Exception): + """Exception raised when a Notebook to Datapane conversion fails.""" + + def _render_traceback_(self): + display_msg( + f"""**Conversion failed** + +{str(self)}""" + ) + + +class NotebookParityException(NotebookException): + """Exception raised when IPython output cache is not in sync with the saved notebook""" + + +class BlocksNotFoundException(NotebookException): + """Exception raised when no blocks are found during conversion""" diff --git a/eurybia/report/datapane/ipython/templates.py b/eurybia/report/datapane/ipython/templates.py new file mode 100644 index 0000000..de6088b --- /dev/null +++ b/eurybia/report/datapane/ipython/templates.py @@ -0,0 +1,276 @@ +""" +Conversion templates for IPython notebooks to Datapane apps. + +..note: Early stage feature - a diverse set of templates that are subject to change. + Most implementations are currently low-level and will be replaced with higher-level abstractions over time. +""" + +from __future__ import annotations + +import typing as t +from abc import abstractmethod + +import eurybia.report.datapane.blocks as b +from eurybia.report.datapane.client.utils import display_msg +from eurybia.report.datapane.ipython.exceptions import BlocksNotFoundException + +BlockFilterF = t.Callable[[b.BaseBlock], bool] +BlockTypes = t.Union[t.Tuple[t.Type[b.BaseBlock], ...], t.Type] +BaseElementList = t.List[b.BaseBlock] + +_registry: t.Dict[str, t.Type[IPythonTemplate]] = {} + + +def partition_blocks_by_predicates( + blocks: BaseElementList, predicates: t.List[BlockFilterF] +) -> t.List[BaseElementList]: + """Partition blocks by predicates""" + partitions: t.List[BaseElementList] = [[] for _ in range(len(predicates))] + + for block in blocks: + for i, pred in enumerate(predicates): + if pred(block): + partitions[i].append(block) + break + + return partitions + + +def partition_blocks_by_types( + blocks: BaseElementList, partition_types: t.List[t.Type[b.BaseBlock]] +) -> t.List[BaseElementList]: + """Partition blocks by types""" + + predicates: t.List[BlockFilterF] = [ + lambda block, partition_type=partition_type: isinstance(block, partition_type) # type: ignore # https://github.com/python/mypy/issues/12557, https://github.com/python/mypy/issues/4226 + for partition_type in partition_types + ] + partitions: t.List[BaseElementList] = partition_blocks_by_predicates( + blocks, predicates + ) + + return partitions + + +def filter_blocks_by_predicate( + blocks: BaseElementList, predicate: BlockFilterF +) -> BaseElementList: + """Filter blocks by predicates""" + filtered_blocks, *_ = partition_blocks_by_predicates(blocks, [predicate]) + + return filtered_blocks + + +def filter_blocks_by_types( + blocks: BaseElementList, block_types: BlockTypes +) -> BaseElementList: + """Filter blocks by type""" + filtered_blocks, *_ = partition_blocks_by_predicates( + blocks, [lambda block: isinstance(block, block_types)] + ) + + return filtered_blocks + + +def guess_template(blocks: BaseElementList) -> t.Type[IPythonTemplate]: + """Guess the template to use based on the blocks provided""" + app_template: t.Type[IPythonTemplate] + + # DashboardTemplate: Contains only Plot, BigNumber, and DataTable blocks + if all([isinstance(block, (b.Plot, b.BigNumber, b.DataTable)) for block in blocks]): + app_template = DashboardTemplate + # TitledPagesTemplate: Contains text blocks with at least two headings + elif ( + len( + filter_blocks_by_predicate( + blocks, + lambda block: isinstance(block, b.Text) + and block.content.startswith("# "), + ) + ) + >= 2 + ): + app_template = TitledPagesTemplate + # DescriptivePagesTemplate: Contains at least two text blocks that are followed by a different block type + elif ( + "".join( + ["1" if isinstance(block, (b.Text)) else "0" for block in blocks] + ).count("10") + >= 2 + ): + app_template = DescriptivePagesTemplate + # AssetListTemplate: Does not contain any text or code blocks + elif not any(filter_blocks_by_types(blocks, (b.Text, b.Code))): + app_template = AssetListTemplate + # AssetCodeListTemplate: Does not contain any text blocks + elif not any(filter_blocks_by_types(blocks, b.Text)): + app_template = AssetCodeListTemplate + # ReportTemplate: The default template if we can't make a guess + else: + app_template = ReportTemplate + + display_msg( + f"Automatically selecting the `{app_template.name}` template. You can override this with `template=template_name` from the following templates: {', '.join(_registry.keys())}." + ) + + return app_template + + +class IPythonTemplate: + name: str = "IPythonTemplate" + + def __init__(self, blocks): + self.blocks = blocks + + @classmethod + def __init_subclass__(cls, template_name, **kwargs): + super().__init_subclass__(**kwargs) + cls.name = template_name + _registry[template_name] = cls + + @abstractmethod + def transform(self) -> None: + raise NotImplementedError + + def validate(self): + if not self.blocks: + raise BlocksNotFoundException("No blocks required by template were found.") + + +class ReportTemplate(IPythonTemplate, template_name="report"): + """Creates a report with blocks in the given order""" + + def transform(self) -> None: + pass + + +class DashboardTemplate(IPythonTemplate, template_name="dashboard"): + """Creates a dashboard with a 3 column group of BigNumbers, followed by a 2 column group of Plots""" + + def transform(self) -> None: + blocks: BaseElementList = self.blocks + + big_numbers, plots, tables = partition_blocks_by_types( + blocks, + [b.BigNumber, b.Plot, b.DataTable], + ) + + blocks = [] + + if big_numbers: + blocks.append(b.Group(blocks=big_numbers, columns=3)) + + tabs: BaseElementList = [] + + if plots: + tabs.append(b.Group(blocks=plots, columns=2, label="Figures")) + + if tables: + tabs.append(b.Group(blocks=tables, label="Tables")) + + # We may only have plots or tables, so we don't need to use a Select block + if tabs: + if len(tabs) > 1: + blocks.append(b.Select(blocks=tabs)) + else: + blocks.append(tabs[0]) + + self.blocks = blocks + + +class AssetListTemplate(IPythonTemplate, template_name="asset_list"): + """Starts a new page for every supported Datapane block""" + + def transform(self) -> None: + blocks = filter_blocks_by_predicate( + self.blocks, lambda block: not isinstance(block, (b.Code, b.Text)) + ) + pages: t.List[b.LayoutBlock] = [ + b.Page(blocks=[block], title=f"{idx + 1}. {block._tag}") + for idx, block in enumerate(blocks) + ] + self.blocks = pages + + +class AssetCodeListTemplate(IPythonTemplate, template_name="asset_code_list"): + """Starts a new page for every supported Datapane block, with the code block after the asset""" + + def transform(self) -> None: + blocks = self.blocks + pages: t.List[b.LayoutBlock] = [] + last_block: t.Optional[b.BaseBlock] = None + + for block in blocks: + # If the block is not a Code or Text block, add it to a new page + if not isinstance(block, (b.Text, b.Code)): + # If the last block was a code block, add it to the current page + if isinstance(last_block, b.Code): + pages.append( + b.Page( + blocks=[b.Group(block, last_block)], + title=f"{len(pages) + 1}. {block._tag}", + ) + ) + else: + pages.append( + b.Page(blocks=[block], title=f"{len(pages) + 1}. {block._tag}") + ) + + last_block = block + self.blocks = pages + + +class DescriptivePagesTemplate(IPythonTemplate, template_name="descriptive_pages"): + """Start a new page every time a contiguous block of text is encountered""" + + def transform(self) -> None: + blocks = self.blocks + pages: t.List[b.LayoutBlock] = [] + page_blocks: BaseElementList = [] + last_block: t.Optional[b.BaseBlock] = None + page_title: str = "Page 1" + + for block in blocks: + # If the block is a Text block, and the last block was not a Text block, start a new page + if isinstance(block, b.Text) and not isinstance(last_block, b.Text): + if page_blocks: + pages.append(b.Page(blocks=page_blocks, title=page_title)) + page_blocks = [] + + if block.content.startswith("# "): + page_title = block.content.partition("\n")[0].lstrip("# ").strip() + else: + page_title = f"Page {len(pages) + 1}" + + page_blocks.append(block) + last_block = block + + # add the last page + pages.append(b.Page(blocks=page_blocks, title=page_title)) + self.blocks = pages + + +class TitledPagesTemplate(IPythonTemplate, template_name="titled_pages"): + """Start a new page every time heading (#) text is encountered""" + + def transform(self) -> None: + blocks = self.blocks + pages: t.List[b.LayoutBlock] = [] + page_blocks: BaseElementList = [] + page_title: str = "Page 1" + + for block in blocks: + # If the block is a Text block, and it starts with a # heading, start a new page + if isinstance(block, b.Text): + if block.content.startswith("# "): + if page_blocks: + pages.append(b.Page(blocks=page_blocks, title=page_title)) + page_blocks = [] + + page_title = block.content.partition("\n")[0].lstrip("#").strip() + + page_blocks.append(block) + + # add the last page + pages.append(b.Page(blocks=page_blocks, title=page_title)) + self.blocks = pages diff --git a/eurybia/report/datapane/ipython/utils.py b/eurybia/report/datapane/ipython/utils.py new file mode 100644 index 0000000..d2ad7fe --- /dev/null +++ b/eurybia/report/datapane/ipython/utils.py @@ -0,0 +1,163 @@ +""" +Datapane helper functions to improve the Datapane UX in IPython notebooks +""" + +from __future__ import annotations + +import typing +from contextlib import suppress + +from eurybia.report.datapane.client.exceptions import DPClientError +from eurybia.report.datapane.client.utils import display_msg + +from .environment import get_environment +from .exceptions import BlocksNotFoundException, NotebookParityException + +if typing.TYPE_CHECKING: + from eurybia.report.datapane.blocks import BaseBlock + + +def output_cell_to_block( + cell: dict, ipython_output_cache: dict +) -> typing.Optional[BaseBlock]: + """Convert a IPython notebook output cell to a Datapane Block""" + from eurybia.report.datapane.blocks import wrap_block + + # Get the output object from the IPython output cache + cell_output_object = ipython_output_cache.get(cell["execution_count"], None) + + # If there's no corresponding output object, skip + if cell_output_object is not None: + with suppress(Exception): + return wrap_block(cell_output_object) + + return None + + +def check_notebook_cache_parity( + notebook_json: dict, ipython_input_cache: list +) -> typing.Tuple[bool, typing.List[int]]: + """Check that the IPython output cache is in sync with the saved notebook""" + is_dirty = False + dirty_cells: typing.List[int] = [] + + # inline !bang commands (get_ipython().system), %line magics, and %%cell magics are not cached + # exclude these from conversion + ignored_cell_functions = [ + "get_ipython().system", + "get_ipython().run_line_magic", + "get_ipython().run_cell_magic", + ] + + # broad check: check the execution count is the same + execution_counts = [ + cell.get("execution_count", 0) or 0 for cell in notebook_json["cells"] + ] + + latest_cell_execution_count = max(execution_counts, default=0) + + # -2 to account for zero-based indexing and the invoking cell not being saved + latest_cache_execution_count = len(ipython_input_cache) - 2 + if latest_cache_execution_count != latest_cell_execution_count: + is_dirty = True + return is_dirty, dirty_cells + + # narrow check: check the cell source is the same for executed cells + for cell in notebook_json["cells"]: + cell_execution_count = cell.get("execution_count", None) + if cell["cell_type"] == "code" and cell_execution_count: + if cell_execution_count < len(ipython_input_cache): + input_cache_source = ipython_input_cache[cell_execution_count].strip() + + # skip and mark cells containing ignored functions + if any( + ignored_function in input_cache_source + for ignored_function in ignored_cell_functions + ): + cell["contains_ignored_functions"] = True + # dirty because input has changed between execution and save. + elif "".join(cell["source"]).strip() != input_cache_source: + is_dirty = True + dirty_cells.append(cell_execution_count) + + return is_dirty, dirty_cells + + +def cells_to_blocks( + opt_out: bool = True, show_code: bool = False, show_markdown: bool = True +) -> typing.List[BaseBlock]: + """Convert IPython notebook cells to a list of Datapane Blocks + + Recognized cell tags: + - `dp-exclude` - Exclude this cell (when opt_out=True) + - `dp-include` - Include this cell (when opt_out=False) + - `dp-show-code` - Show the input code for this cell + - `dp-show-markdown` - Show the markdown for this cell + + ..note:: IPython output caching must be enabled for this function to work. It is enabled by default. + """ + environment = get_environment() + if not environment.is_notebook_environment: + raise DPClientError("This function can only be used in a notebook environment") + + ip = environment.get_ipython() + user_ns = ip.user_ns + ipython_output_cache = user_ns["_oh"] + ipython_input_cache = user_ns["_ih"] + + notebook_json = environment.get_notebook_json() + # TODO: debug message for Colab, remove after testing + + notebook_is_dirty, dirty_cells = check_notebook_cache_parity( + notebook_json, ipython_input_cache + ) + + if notebook_is_dirty: + notebook_parity_message = "Please ensure all cells in the notebook have been executed and saved before running the conversion." + + if dirty_cells: + notebook_parity_message += f""" + +The following cells have not been executed and saved: {', '.join(map(str, dirty_cells))}""" + + raise NotebookParityException(notebook_parity_message) + + blocks = [] + + for cell in notebook_json["cells"]: + tags = cell["metadata"].get("tags", []) + + if (opt_out and "dp-exclude" not in tags) or ( + not opt_out and "dp-include" in tags + ): + if (cell["cell_type"] == "markdown" and cell.get("source")) and ( + show_markdown or "dp-show-markdown" in tags + ): + from eurybia.report.datapane.blocks.text import Text + + markdown_block: BaseBlock = Text("".join(cell["source"])) + blocks.append(markdown_block) + elif cell["cell_type"] == "code" and not cell.get( + "contains_ignored_functions", False + ): + if "dp-show-code" in tags or show_code: + from eurybia.report.datapane.blocks.text import Code + + code_block: BaseBlock = Code("".join(cell["source"])) + blocks.append(code_block) + + output_block = output_cell_to_block(cell, ipython_output_cache) + + if output_block: + blocks.append(output_block) + elif "dp-include" in tags: + display_msg( + f'Cell output of type {type(ipython_output_cache.get(cell["execution_count"]))} not supported. Skipping.', + ) + + if not blocks: + raise BlocksNotFoundException("No blocks found.") + + display_msg("Notebook converted to blocks.") + + return blocks diff --git a/eurybia/report/datapane/optional_libs.py b/eurybia/report/datapane/optional_libs.py new file mode 100644 index 0000000..4b4c936 --- /dev/null +++ b/eurybia/report/datapane/optional_libs.py @@ -0,0 +1,69 @@ +""" +Dynamic handling for optional libraries - this module is imported on demand +""" + +# flake8: noqa:F811 isort:skip_file +from __future__ import annotations + +from packaging import version as v +from packaging.specifiers import SpecifierSet + +from eurybia.report.datapane.client import log + +# NOTE - need to update this and keep in sync with JS +BOKEH_V_SPECIFIER = SpecifierSet("~=2.4.2") +PLOTLY_V_SPECIFIER = SpecifierSet(">=4.0.0") +FOLIUM_V_SPECIFIER = SpecifierSet(">=0.12.0") + + +def _check_version(name: str, _v: v.Version, ss: SpecifierSet): + if _v not in ss: + log.warning( + f"{name} version {_v} is not supported, these plots may not display correctly, please install version {ss}" + ) + + +# Optional Plotting library import handling +# Matplotlib +try: + from matplotlib.figure import Axes, Figure + from numpy import ndarray + + HAVE_MATPLOTLIB = True +except ImportError: + log.debug("No matplotlib found") + HAVE_MATPLOTLIB = False + +# Folium +try: + import folium + from folium import Map + + _check_version("Folium", v.Version(folium.__version__), FOLIUM_V_SPECIFIER) + HAVE_FOLIUM = True +except ImportError: + HAVE_FOLIUM = False + log.debug("No folium found") + +# Bokeh +try: + import bokeh + from bokeh.layouts import LayoutDOM as BLayout + from bokeh.plotting.figure import Figure as BFigure + + _check_version("Bokeh", v.Version(bokeh.__version__), BOKEH_V_SPECIFIER) + HAVE_BOKEH = True +except ImportError: + HAVE_BOKEH = False + log.debug("No Bokeh Found") + +# Plotly +try: + import plotly + from plotly.graph_objects import Figure as PFigure + + _check_version("Plotly", v.Version(plotly.__version__), PLOTLY_V_SPECIFIER) + HAVE_PLOTLY = True +except ImportError: + HAVE_PLOTLY = False + log.debug("No Plotly Found") diff --git a/eurybia/report/datapane/processors/__init__.py b/eurybia/report/datapane/processors/__init__.py new file mode 100644 index 0000000..6782e1e --- /dev/null +++ b/eurybia/report/datapane/processors/__init__.py @@ -0,0 +1,5 @@ +# flake8: noqa:F401 +from .api import build_report, save_report, stringify_report, upload_report +from .file_store import FileEntry, FileStore +from .processors import ConvertXML, PreProcessView +from .types import FontChoice, Formatting, Pipeline, TextAlignment, ViewState, Width, mk_null_pipe diff --git a/eurybia/report/datapane/processors/api.py b/eurybia/report/datapane/processors/api.py new file mode 100644 index 0000000..cb70e52 --- /dev/null +++ b/eurybia/report/datapane/processors/api.py @@ -0,0 +1,149 @@ +""" +Datapane Processors + +API for processing Views, e.g. rendering it locally and publishing to a remote server +""" + +from __future__ import annotations + +import os +import typing as t +from pathlib import Path +from shutil import rmtree + +from eurybia.report.datapane.client import DPClientError +from eurybia.report.datapane.common import NPath +from eurybia.report.datapane.view import Blocks, BlocksT + +from .file_store import B64FileEntry, GzipTmpFileEntry +from .processors import ( + ConvertXML, + ExportHTMLFileAssets, + ExportHTMLInlineAssets, + ExportHTMLStringInlineAssets, + PreProcessView, +) +from .types import Formatting, Pipeline, ViewState + +__all__ = ["upload_report", "save_report", "build_report", "stringify_report"] + + +################################################################################ +# exported public API +def build_report( + blocks: BlocksT, + name: str = "Report", + dest: t.Optional[NPath] = None, + formatting: t.Optional[Formatting] = None, + overwrite: bool = False, +) -> None: + """Build an (static) app with a directory structure, which can be served by a local http server + + !!! note + This outputs compressed assets into the dir as well, may be an issue if self-hosting + + Args: + blocks: The `Blocks` object or a list of Blocks + name: The name of the app directory to be created + dest: File path to store the app directory + formatting: Sets the basic app styling + overwrite: Replace existing app with the same name and destination if already exists (default: False) + """ + # TODO(product) - unknown if we should keep this... + + # build the dest dir + app_dir: Path = Path(dest or os.getcwd()) / name + app_exists = app_dir.is_dir() + + if app_exists and overwrite: + rmtree(app_dir) + elif app_exists and not overwrite: + raise DPClientError( + f"Report exists at given path {str(app_dir)} -- set `overwrite=True` to allow overwrite" + ) + + assets_dir = app_dir / "assets" + assets_dir.mkdir(parents=True) + + # write the app html and assets + s = ViewState( + blocks=Blocks.wrap_blocks(blocks), + file_entry_klass=GzipTmpFileEntry, + dir_path=assets_dir, + ) + _: str = ( + Pipeline(s) + .pipe(PreProcessView(is_finalised=True)) + .pipe(ConvertXML()) + .pipe(ExportHTMLFileAssets(app_dir=app_dir, name=name, formatting=formatting)) + .result + ) + + +def save_report( + blocks: BlocksT, + path: str, + open: bool = False, + name: str = "Report", + formatting: t.Optional[Formatting] = None, +) -> None: + """Save the app document to a local HTML file + + Args: + blocks: The `Blocks` object or a list of Blocks + path: File path to store the document + open: Open in your browser after creating (default: False) + name: Name of the document (optional: uses path if not provided) + formatting: Sets the basic app styling + """ + + s = ViewState(blocks=Blocks.wrap_blocks(blocks), file_entry_klass=B64FileEntry) + _: str = ( + Pipeline(s) + .pipe(PreProcessView(is_finalised=True)) + .pipe(ConvertXML()) + .pipe( + ExportHTMLInlineAssets( + path=path, open=open, name=name, formatting=formatting + ) + ) + .result + ) + + +def stringify_report( + blocks: BlocksT, + name: t.Optional[str] = None, + formatting: t.Optional[Formatting] = None, +) -> str: + """Stringify the app document to a HTML string + + Args: + blocks: The `Blocks` object or a list of Blocks + name: Name of the document (optional: uses path if not provided) + formatting: Sets the basic app styling + """ + + s = ViewState(blocks=Blocks.wrap_blocks(blocks), file_entry_klass=B64FileEntry) + report_html: str = ( + Pipeline(s) + .pipe(PreProcessView(is_finalised=False)) + .pipe(ConvertXML()) + .pipe(ExportHTMLStringInlineAssets(name=name, formatting=formatting)) + .result + ) + + return report_html + + +def upload_report( + *args, + **kwargs, +) -> None: + """ + (No longer supported). + Upload as a report, including its attached assets, to the logged-in Datapane Server. + """ + raise DPClientError( + "Datapane Cloud is now read-only and does not support Report uploading, only local, saved HTML report output is supported" + ) diff --git a/eurybia/report/datapane/processors/file_store.py b/eurybia/report/datapane/processors/file_store.py new file mode 100644 index 0000000..f15ccf6 --- /dev/null +++ b/eurybia/report/datapane/processors/file_store.py @@ -0,0 +1,214 @@ +from __future__ import annotations + +import abc +import datetime +import gzip +import hashlib +import io +import tempfile +import typing as t +from pathlib import Path +from shutil import copyfileobj + +from typing_extensions import Self + +from eurybia.report.datapane._vendor import base64io +from eurybia.report.datapane.common import guess_type + +SERVED_REPORT_ASSETS_DIR = "assets" +GZIP_MTIME = datetime.datetime(year=2000, month=1, day=1).timestamp() + + +class FileEntry: + file: t.IO + _ext: str + _dir_path: t.Optional[Path] + + # post-freeze + frozen: bool = False + mime: str + hash: str + size: int + wrapped: t.BinaryIO + + def __init__( + self, ext: str, mime: t.Optional[str] = None, dir_path: t.Optional[Path] = None + ): + self.mime = mime or guess_type(Path(f"tmp{ext}")) + self._ext = ext + self._dir_path = dir_path + + @abc.abstractmethod + def freeze(self) -> None: + """Must be called after writing / adding to store + # TODO - add to contextmanager?? + """ + + @property + @abc.abstractmethod + def src(self) -> str: + pass + + def as_dict(self) -> dict: + assert self.frozen + return dict(src=self.src, hash=self.hash, size=self.size, mime=self.mime) + + def __eq__(self, other: FileEntry) -> bool: + if self.hash: + return self.hash == other.hash + raise NotImplementedError() + + +class NullWriter(io.BytesIO): + def write(self, s): + pass + + def writelines(self, *a, **kw) -> None: + pass + + +class DummyFileEntry(FileEntry): + """File entry that discards all data - for internal use""" + + def __init__(self, *a, **kw): + super().__init__(*a, **kw) + self.file = NullWriter() + + def freeze(self): + self.size = 0 + self.hash = "abcdef" + self.frozen = True + + def src(self) -> str: + return "/dev/null" + + +class B64FileEntry(FileEntry): + """Memory-based b64 file""" + + # requires b64io is bytes only and wraps to a bytes file only + file: base64io.Base64IO + wrapped: io.BytesIO + contents: bytes + + def __init__(self, ext: str, mime: t.Optional[str] = None, *a, **kw): + super().__init__(ext, mime, *a, **kw) + self.wrapped = io.BytesIO() + self.file = base64io.Base64IO(self.wrapped) + + def freeze(self) -> None: + if not self.frozen: + self.frozen = True + # get a reference to the buffer to splice later + self.file.close() + self.file.flush() + self.contents = self.wrapped.getvalue() + # calc other properties + self.hash = hashlib.sha256(self.contents).hexdigest()[:10] + self.size = self.wrapped.tell() + + @property + def src(self) -> str: + return f"data:{self.mime};base64,{self.contents.decode('ascii')}" + + +class GzipTmpFileEntry(FileEntry): + """Gzipped file, by default stored in /tmp""" + + # both file and wapper files are bytes-only + file: gzip.GzipFile + # TODO - this could actually be an in-memory file... + wrapped: tempfile.NamedTemporaryFile + has_output_dir: bool = False + + # Do we need DPTmpFile here, or just use namedtempfile?? + def __init__( + self, ext: str, mime: t.Optional[str] = None, dir_path: t.Optional[Path] = None + ): + super().__init__(ext, mime, dir_path) + + if dir_path: + # create as a permanent file within the given dir + self.has_output_dir = True + self.wrapped = tempfile.NamedTemporaryFile( + "w+b", suffix=ext, prefix="dp-", dir=dir_path, delete=False + ) + else: + self.wrapped = tempfile.NamedTemporaryFile("w+b", suffix=ext, prefix="dp-") + + self.file = gzip.GzipFile(fileobj=self.wrapped, mode="w+b", mtime=GZIP_MTIME) + + def calc_hash(self, f: t.IO) -> str: + f.seek(0) + file_hash = hashlib.sha256() + while chunk := f.read(8192): + file_hash.update(chunk) + return file_hash.hexdigest()[:10] + + @property + def src(self) -> str: + if self.has_output_dir: + return f"/{SERVED_REPORT_ASSETS_DIR}/{Path(self.wrapped.name).name}" + else: + return "NYI" + + def freeze(self) -> None: + if not self.frozen: + self.frozen = True + self.file.flush() + self.file.close() + self.wrapped.flush() + # size will be the compressed size... + self.size = self.wrapped.tell() + self.hash = self.calc_hash(self.wrapped) + + +class FileStore: + # TODO - make this a CAS (index by object hash itself?) + # NOTE - currently we pass dir_path via the FileStore, could move into the file themselves? + def __init__( + self, fw_klass: t.Type[FileEntry], assets_dir: t.Optional[Path] = None + ): + super().__init__() + self.fw_klass = fw_klass + self.files: t.List[FileEntry] = [] + self.dir_path = assets_dir + + def __add__(self, other: FileStore) -> Self: + # TODO - ensure factory is the same for both + self.files.extend(other.files) + return self + + @property + def store_count(self) -> int: + return len(self.files) + + @property + def file_list(self) -> t.List[t.BinaryIO]: + return [f.wrapped for f in self.files] + + def get_file(self, ext: str, mime: str) -> FileEntry: + return self.fw_klass(ext, mime, self.dir_path) + + def add_file(self, fw: FileEntry) -> None: + fw.freeze() + self.files.append(fw) + + def load_file(self, path: Path) -> FileEntry: + """load a file into the store (makes a copy)""" + # TODO - ideally lazily-link a path to the store (rather than copy it in) + ext = "".join(path.suffixes) + dest_obj = self.fw_klass(ext=ext, dir_path=self.dir_path) + with path.open("rb") as src_obj: + copyfileobj(src_obj, dest_obj.file) + self.add_file(dest_obj) + return dest_obj + + def as_dict(self) -> dict: + """Build a json structure suitable for embedding in a html file, json-rpc response, etc.""" + x: FileEntry # noqa: F842 + return {x.hash: x.as_dict() for x in self.files} + + def get_entry(self, hash: str) -> t.Optional[FileEntry]: + # TODO - change self.files to a dict[hash, FileEntry]? + return next((f for f in self.files if f.hash == hash), None) diff --git a/eurybia/report/datapane/processors/processors.py b/eurybia/report/datapane/processors/processors.py new file mode 100644 index 0000000..e924cbf --- /dev/null +++ b/eurybia/report/datapane/processors/processors.py @@ -0,0 +1,348 @@ +from __future__ import annotations + +import json +import logging +import os +import sys +import typing as t +from abc import ABC +from copy import copy +from itertools import count +from os import path as osp +from pathlib import Path +from uuid import uuid4 + +if sys.version_info < (3, 10): + import importlib_resources as ir +else: + from importlib import resources as ir + +from lxml import etree + +from eurybia.report.datapane import blocks as b +from eurybia.report.datapane._vendor.bottle import SimpleTemplate +from eurybia.report.datapane.client.exceptions import InvalidReportError +from eurybia.report.datapane.client.utils import display_msg, log, open_in_browser +from eurybia.report.datapane.common import HTML, NPath, timestamp, validate_view_doc +from eurybia.report.datapane.common.viewxml_utils import ElementT, local_view_resources +from eurybia.report.datapane.view import PreProcess, XMLBuilder + +from .file_store import FileEntry +from .types import BaseProcessor, Formatting + +if t.TYPE_CHECKING: + pass + + +class PreProcessView(BaseProcessor): + """Optimisations to improve the layout of the view using the Block-API""" + + def __init__(self, *, is_finalised: bool = True) -> None: + self.is_finalised = is_finalised + super().__init__() + + def __call__(self, _: t.Any) -> None: + # AST checks + if len(self.s.blocks.blocks) == 0: + raise InvalidReportError( + "Empty blocks object - must contain at least one block" + ) + + # convert Page -> Select + Group + v = copy(self.s.blocks) + if all(isinstance(blk, b.Page) for blk in v.blocks): + # convert to top-level Select + p: b.Page # noqa: F842 + v.blocks = [ + b.Select( + blocks=[ + b.Group(blocks=p.blocks, label=p.title, name=p.name) + for p in v.blocks + ], + type=b.SelectType.TABS, + ) + ] + + # Block-API visitors + pp = PreProcess(is_finalised=self.is_finalised) + v.accept(pp) + v1 = pp.root + # v1 = copy(v) + + # update the processor state + self.s.blocks = v1 + + return None + + +class ConvertXML(BaseProcessor): + """Convert the View AST into an XML fragment""" + + local_post_xslt = etree.parse(str(local_view_resources / "local_post_process.xslt")) + local_post_transform = etree.XSLT(local_post_xslt) + + def __init__(self, *, pretty_print: bool = False, fragment: bool = False) -> None: + self.pretty_print: bool = pretty_print + self.fragment: bool = fragment + super().__init__() + + def __call__(self, _: t.Any) -> ElementT: + initial_doc = self.convert_xml() + transformed_doc = self.post_transforms(initial_doc) + + # convert to string + view_xml_str: str = etree.tounicode( + transformed_doc, pretty_print=self.pretty_print + ) + # s1 = dc.replace(s, view_xml=view_xml_str) + self.s.view_xml = view_xml_str + + if log.isEnabledFor(logging.DEBUG): + log.debug(etree.tounicode(transformed_doc, pretty_print=True)) + + # return the doc for further processing (xml str stored in state) + return transformed_doc + + def convert_xml(self) -> ElementT: + # create initial state + builder_state = XMLBuilder(store=self.s.store) + self.s.blocks.accept(builder_state) + return builder_state.get_root(self.fragment) + + def post_transforms(self, view_doc: ElementT) -> ElementT: + # TODO - post-xml transformations, essentially xslt / lxml-based DOM operations + # post_process via xslt + processed_view_doc: ElementT = self.local_post_transform(view_doc) + + # TODO - custom lxml-based transforms go here... + + # validate post all transformations + validate_view_doc(xml_doc=processed_view_doc) + return processed_view_doc + + +class PreUploadProcessor(BaseProcessor): + def __call__(self, doc: ElementT) -> t.Tuple[str, t.List[t.BinaryIO]]: + """ + pre-upload pass of the XML doc so can be uploaded to DPCloud + modifies the document based on the FileStore + """ + + # NOTE - this currently relies on all assets existing linearly in document order + # in the asset store - if we move to a cas we will need to update the algorithm here + # replace ref -> attachment in view + # all blocks with a ref + refs: t.List[ElementT] = doc.xpath( + "/View//*[@src][starts-with(@src, 'ref://')]" + ) + for idx, ref, f_entry in zip(count(0), refs, self.s.store.files): + ref: ElementT + f_entry: FileEntry + _hash: str = ref.get("src").split("://")[1] + ref.set("src", f"attachment://{idx}") + assert _hash == f_entry.hash # sanity check + + self.s.view_xml = etree.tounicode(doc) + return (self.s.view_xml, self.s.store.file_list) + + +############################################################################### +# HTML Exporting Processors +class BaseExportHTML(BaseProcessor, ABC): + """Provides shared logic for writing an app to local disk""" + + # Type is `ir.abc.Traversable` which extends `Path`, + # but the former isn't compatible with `shutil` + template_dir: Path = t.cast( + Path, ir.files("eurybia.report.datapane.resources.html_templates") + ) + template: SimpleTemplate + template_name: str + + def __init_subclass__(cls, **kwargs): + super().__init_subclass__(**kwargs) + # TODO (JB) - why doesn't altering TEMPLATE_PATH work as described in docs? Need to pass dir to `lookup` kwarg instead + cls.template = SimpleTemplate( + name=cls.template_name, lookup=[str(cls.template_dir)] + ) + + def get_cdn(self) -> str: + from eurybia.report.datapane import __is_dev_build__, __version__ + + if cdn_base := os.getenv("DATAPANE_CDN_BASE"): + return cdn_base + elif __is_dev_build__: + return "https://datapane-cdn.com/dev" + else: + return f"https://datapane-cdn.com/v{__version__}" + + def escape_json_htmlsafe(self, obj: t.Any) -> str: + """Escape JSON object for embedding in bottle templates.""" + + # Taken from Jinja2's |tojson pipe function + # (https://github.com/pallets/jinja/blob/b7cb6ee6675b12a027c5e7518f832b2926dfe293/src/jinja2/utils.py#L628) + # Use of markupsafe is removed, as we use bottle's SimpleTemplate. + return ( + json.dumps(obj) + .replace("<", "\\u003c") + .replace(">", "\\u003e") + .replace("&", "\\u0026") + .replace("'", "\\u0027") + ) + + def _write_html_template( + self, + name: str, + formatting: t.Optional[Formatting] = None, + app_runner: bool = False, + ) -> t.Tuple[str, str]: + """Internal method to write the ViewXML and assets into a HTML container and associated files""" + name = name or "app" + formatting = formatting or Formatting() + + report_id: str = uuid4().hex + + # TODO - split this out? + vs = self.s + if vs: + assets = vs.store.as_dict() or {} + view_xml = vs.view_xml + else: + assets = {} + view_xml = "" + + app_data = dict(view_xml=view_xml, assets=assets) + html = self.template.render( + # Escape JS multi-line strings + app_data=self.escape_json_htmlsafe(app_data), + report_width_class=formatting.width.to_css(), + report_name=name, + report_date=timestamp(), + css_header=formatting.to_css(), + is_light_prose=json.dumps(formatting.light_prose), + events=False, + report_id=report_id, + cdn_static="https://datapane-cdn.com/static", + cdn_base=self.get_cdn(), + app_runner=app_runner, + ) + + return html, report_id + + +class ExportBaseHTMLOnly(BaseExportHTML): + """Export the base view used to render an App, containing no ViewXML nor Assets""" + + # TODO (JB) - Create base HTML-only template + template_name = "local_template.html" + + def __init__(self, debug: bool, formatting: t.Optional[Formatting] = None): + self.debug = debug + self.formatting = formatting + + def generate_chrome(self) -> HTML: + # TODO - this is a bit hacky + self.s = None + html, report_id = self._write_html_template( + "app", formatting=self.formatting, app_runner=True + ) + return HTML(html) + + def get_cdn(self) -> str: + return "/web-static" if self.debug else super().get_cdn() + + def __call__(self, _: t.Any) -> None: + return None + + +class ExportHTMLInlineAssets(BaseExportHTML): + """ + Export a view into a single HTML file containing: + - View XML - embedded + - Assetes - embedded as b64 data-uris + """ + + template_name = "local_template.html" + + def __init__( + self, + path: str, + open: bool = False, + name: str = "app", + formatting: t.Optional[Formatting] = None, + ): + self.path = path + self.open = open + self.name = name + self.formatting = formatting + + def __call__(self, _: t.Any) -> str: + html, report_id = self._write_html_template( + name=self.name, formatting=self.formatting + ) + + Path(self.path).write_text(html, encoding="utf-8") + + display_msg(f"App saved to ./{self.path}") + + if self.open: + path_uri = f"file://{osp.realpath(osp.expanduser(self.path))}" + open_in_browser(path_uri) + + return report_id + + +class ExportHTMLFileAssets(BaseExportHTML): + """ + Export a view into a single HTML file on disk, containing + - View XML - embedded + - Assets - referenced as remote resources + """ + + template_name = "local_template.html" + + def __init__( + self, + app_dir: Path, + name: str = "app", + formatting: t.Optional[Formatting] = None, + ): + self.app_dir = app_dir + self.name = name + self.formatting = formatting + + def __call__(self, dest: t.Optional[NPath] = None) -> Path: + html, report_id = self._write_html_template( + name=self.name, + formatting=self.formatting, + ) + + index_path = self.app_dir / "index.html" + index_path.write_text(html, encoding="utf-8") + display_msg(f"Built app in {self.app_dir}") + return self.app_dir + + +class ExportHTMLStringInlineAssets(BaseExportHTML): + """ + Export the View as an in-memory string representing a resizable HTML fragment, containing + - View XML - embedded + - Assetes - embedded as b64 data-uris + """ + + template_name = "ipython_template.html" + + def __init__( + self, + name: str = "Stringified App", + formatting: t.Optional[Formatting] = None, + ): + self.name = name + self.formatting = formatting + + def __call__(self, _: t.Any) -> HTML: + html, report_id = self._write_html_template( + name=self.name, formatting=self.formatting + ) + + return HTML(html) diff --git a/eurybia/report/datapane/processors/types.py b/eurybia/report/datapane/processors/types.py new file mode 100644 index 0000000..f1c973f --- /dev/null +++ b/eurybia/report/datapane/processors/types.py @@ -0,0 +1,132 @@ +from __future__ import annotations + +import dataclasses as dc +import typing as t +from enum import Enum +from pathlib import Path + +from eurybia.report.datapane.common import ViewXML +from eurybia.report.datapane.view import Blocks + +from .file_store import DummyFileEntry, FileEntry, FileStore + + +@dc.dataclass +class ViewState: + # maybe a FileHandler interface?? + blocks: Blocks + file_entry_klass: dc.InitVar[t.Type[FileEntry]] + store: FileStore = dc.field(init=False) + view_xml: ViewXML = "" + entries: t.Dict[str, str] = dc.field(default_factory=dict) + dir_path: dc.InitVar[t.Optional[Path]] = None + + def __post_init__(self, file_entry_klass, dir_path): + # TODO - should we use a lambda for file_entry_klass with dir_path captured? + self.store = FileStore(fw_klass=file_entry_klass, assets_dir=dir_path) + + +P_IN = t.TypeVar("P_IN") +P_OUT = t.TypeVar("P_OUT") + + +class BaseProcessor(t.Generic[P_IN, P_OUT]): + """Processor class that handles pipeline operations""" + + s: ViewState + + def __call__(self, x: P_IN) -> P_OUT: + raise NotImplementedError("Implement in subclass") + + +# TODO - type this properly +class Pipeline(t.Generic[P_IN]): + """ + A simple, programmable, eagerly-evaluated, pipeline specialised on ViewAST transformations + similar to f :: State s => s ViewState x -> s ViewState y + """ + + # NOTE - toolz has an untyped function for this + + def __init__(self, s: ViewState, x: P_IN = None): + self._state = s + self._x = x + + def pipe(self, p: BaseProcessor[P_IN, P_OUT]) -> Pipeline[P_OUT]: + p.s = self._state + y = p.__call__(self._x) # need to call as positional args + self._state = p.s + return Pipeline(self._state, y) + + @property + def state(self) -> ViewState: + return self._state + + @property + def result(self) -> P_IN: + return self._x + + +def mk_null_pipe(blocks: Blocks) -> Pipeline[None]: + s = ViewState(blocks, file_entry_klass=DummyFileEntry) + return Pipeline(s) + + +# Top-level API options / types +class Width(Enum): + NARROW = "narrow" + MEDIUM = "medium" + FULL = "full" + + def to_css(self) -> str: + if self == self.NARROW: + return "max-w-3xl" + elif self == self.MEDIUM: + return "max-w-screen-xl" + else: + return "max-w-full" + + +class TextAlignment(Enum): + JUSTIFY = "justify" + LEFT = "left" + RIGHT = "right" + CENTER = "center" + + +class FontChoice(Enum): + DEFAULT = "Inter, ui-sans-serif, system-ui" + SANS = "ui-sans-serif, sans-serif, system-ui" + SERIF = "ui-serif, serif, system-ui" + MONOSPACE = "ui-monospace, monospace, system-ui" + + +# Currently unused +# class PageLayout(Enum): +# TOP = "top" +# SIDE = "side" + + +@dc.dataclass +class Formatting: + """Configure styling and formatting""" + + bg_color: str = "#FFF" + accent_color: str = "#4E46E5" + font: t.Union[FontChoice, str] = FontChoice.DEFAULT + text_alignment: TextAlignment = TextAlignment.LEFT + width: Width = Width.MEDIUM + light_prose: bool = False + + def to_css(self) -> str: + if isinstance(self.font, FontChoice): + font = self.font.value + else: + font = self.font + + return f""":root {{ + --dp-accent-color: {self.accent_color}; + --dp-bg-color: {self.bg_color}; + --dp-text-align: {self.text_alignment.value}; + --dp-font-family: {font}; +}}""" diff --git a/eurybia/report/datapane/resources/__init__.py b/eurybia/report/datapane/resources/__init__.py new file mode 100644 index 0000000..c056c02 --- /dev/null +++ b/eurybia/report/datapane/resources/__init__.py @@ -0,0 +1,2 @@ +# Copyright 2020 StackHut Limited (trading as Datapane) +# SPDX-License-Identifier: Apache-2.0 diff --git a/eurybia/report/datapane/resources/datapane-icon-192x192.png b/eurybia/report/datapane/resources/datapane-icon-192x192.png new file mode 100644 index 0000000..a585ee8 Binary files /dev/null and b/eurybia/report/datapane/resources/datapane-icon-192x192.png differ diff --git a/eurybia/report/datapane/resources/html_templates/README.md b/eurybia/report/datapane/resources/html_templates/README.md new file mode 100644 index 0000000..2bbac12 --- /dev/null +++ b/eurybia/report/datapane/resources/html_templates/README.md @@ -0,0 +1,2 @@ + +## Notes diff --git a/eurybia/report/datapane/resources/html_templates/__init__.py b/eurybia/report/datapane/resources/html_templates/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/eurybia/report/datapane/resources/html_templates/head_chrome.html b/eurybia/report/datapane/resources/html_templates/head_chrome.html new file mode 100644 index 0000000..e090d47 --- /dev/null +++ b/eurybia/report/datapane/resources/html_templates/head_chrome.html @@ -0,0 +1,34 @@ +% # we use a single png for all icon types + + +% # we use a single png for all icon types + + + + + + + + + + diff --git a/eurybia/report/datapane/resources/html_templates/ipython_base.html b/eurybia/report/datapane/resources/html_templates/ipython_base.html new file mode 100644 index 0000000..60c433e --- /dev/null +++ b/eurybia/report/datapane/resources/html_templates/ipython_base.html @@ -0,0 +1,12 @@ + + + diff --git a/eurybia/report/datapane/resources/html_templates/ipython_template.html b/eurybia/report/datapane/resources/html_templates/ipython_template.html new file mode 100644 index 0000000..bed6b45 --- /dev/null +++ b/eurybia/report/datapane/resources/html_templates/ipython_template.html @@ -0,0 +1,16 @@ +% rebase("ipython_base.html") + + + + % include("head_chrome.html") + + + +
+
+
+ + + diff --git a/eurybia/report/datapane/resources/html_templates/local_template.html b/eurybia/report/datapane/resources/html_templates/local_template.html new file mode 100644 index 0000000..cb06c55 --- /dev/null +++ b/eurybia/report/datapane/resources/html_templates/local_template.html @@ -0,0 +1,9 @@ + + + + % include("head_chrome.html") + + +
+ + diff --git a/eurybia/report/datapane/resources/mime.types b/eurybia/report/datapane/resources/mime.types new file mode 100644 index 0000000..2d17ff0 --- /dev/null +++ b/eurybia/report/datapane/resources/mime.types @@ -0,0 +1,26 @@ +# Additional mime-type we support for assets +# MIME type Extension +# plots +application/vnd.bokeh.show+json bokeh.json +application/vnd.vegalite.v5+json vl.json +application/vnd.vegalite.v2+json vl2.json +application/vnd.vegalite.v3+json vl3.json +application/vnd.vegalite.v4+json vl4.json +application/vnd.vegalite.v5+json vl5.json +application/vnd.plotly.v1+json pl.json +application/vnd.folium+html fl.html + +# images - already handled + +# misc +# datafiles - handled directly as it's own type +application/vnd.apache.arrow+binary arrow +application/vnd.nstack.table+html tbl.html +# Jupyter notebooks +application/x-ipynb+json ipynb +# Pickled objects +application/vnd.pickle+binary pkl +application/x-pywheel+zip whl +application/x-tgz tgz +# base64'd objects +application/base64 b64 diff --git a/eurybia/report/datapane/resources/view_resources/__init__.py b/eurybia/report/datapane/resources/view_resources/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/eurybia/report/datapane/resources/view_resources/full_schema.rnc b/eurybia/report/datapane/resources/view_resources/full_schema.rnc new file mode 100644 index 0000000..385dda7 --- /dev/null +++ b/eurybia/report/datapane/resources/view_resources/full_schema.rnc @@ -0,0 +1,202 @@ +# This schema currently covers all view states - eventually to be split out to handle all substates +# (want to delay splitting out as long as possible for simplicities sake) +# NOTES: +# - we don't allow mixed-mode content right now, e.g. char data along with nested elements +# instead text nodes must be leaf nodes +# - using simply removes all white-space only nodes, including space between node and nested node +# which would be mixed-mode and isn't allowed by schema anyway +# - whitespace otherwise is preserved inside nodes, e.g. Code blocks, without requireing CDATA - CDATA is purely for escaping +# see https://docs.microsoft.com/en-us/previous-versions/dotnet/netframework-4.0/ms256144(v=vs.100) +# NOTE - the .rng file is autogenerated using the `trang` java tool +# run `java -jar /path/to/trang.jar full_schema.rnc full_schema.rng` +#default namespace = "https://datapane.com/view" + +namespace a = "http://relaxng.org/ns/compatibility/annotations/1.0" + +start = element View { + attribute fragment { xsd:boolean }, + attribute version { xsd:positiveInteger }, + # do we want Internal here just-in-case?? + Block+ +} + + +################################################################################ +# Internal fields, used when modifying the internal report structure +# TODO - do we need this?? +Internal = element Internal { + empty + # id_count used to track free variables count + # attribute id_count { xsd:nonNegativeInteger } +} + + +################################################################################ +# Main Report Tree +#Pages = element Pages { +# attribute layout { string "top" | string "side" }?, +# Page+ +#} + +# name is optional generally, can be user-defined, used for referencing +block_name = attribute name { xsd:ID } +block_label = attribute label { xsd:string { minLength = "1" maxLength = "256" } } +block_label_or_name = block_name?, block_label? + + +# Pages don't have id's or name's - as can't be extracted to be displayed elsewhere +# Pages have an implict single column +#Page = element Page { +# block_label_or_name, Block+ +#} + +Block = LayoutBlock | DataBlock | Empty + +# Used to describe a Placeholder element we can patch - like empty div in HTML +Empty = element Empty { + block_name +} + + +################################################################################ +# NOTE - we could add Grid, Columns, etc. here +LayoutBlock = Group | Select | Toggle + +Select = element Select { + block_label_or_name, + attribute type { string "dropdown" | string "tabs" }?, + # Selects used to require at least 2 elements (for Reports), with Apps this has been relaxed + Block* +} + +Group = element Group { + block_label_or_name, + + [ a:defaultValue = "1" ] + attribute columns { xsd:nonNegativeInteger }, + # widths is a json list of ints + attribute widths { xsd:string { minLength = "2" pattern = """\[\d+(,\s*\d+)*\]""" } }?, + attribute valign { string "top" | string "center" | string "bottom" }, + + Block* +} + +Toggle = element Toggle { + block_label_or_name, + Block +} + +################################################################################ +DataBlock = EmbeddedTextBlock | AssetBlock | BigNumber + +EmbeddedTextBlock = Text | HTML | Code | Embed | Formula + +AssetBlock = Attachment | Media | Plot | Table | DataTable + +################################################################################ +# misc +opt_caption = attribute caption { xsd:string { minLength = "1" maxLength = "512" } }? + + +################################################################################ +# EmbeddedTextBlocks +# Markdown Text +Text = element Text { + block_label_or_name, + xsd:string { minLength = "1" pattern = "(.|\s)*\S(.|\s)*" } +} + +HTML = element HTML { + block_label_or_name, + xsd:string { minLength = "1" } +} + +Code = element Code { + block_label_or_name, + attribute language { xsd:string { minLength = "1" maxLength = "127" } }, + opt_caption, + xsd:string { minLength = "1" } +} + +Embed = element Embed { + block_label_or_name, + attribute url { xsd:anyURI }, + attribute title { xsd:string { minLength = "1" maxLength = "255" } }, + attribute provider_name { xsd:string { minLength = "1" maxLength = "127" } }, + # TODO - make optional? + xsd:string { minLength = "1" } +} + +Formula = element Formula { + block_label_or_name, + opt_caption, + xsd:string { minLength = "1" } +} + +BigNumber = element BigNumber { + block_label_or_name, + attribute heading { xsd:string { minLength = "1" maxLength = "127" } }, + attribute value { xsd:string { minLength = "1" maxLength = "127" } }, + + # optional attributes + attribute change { xsd:string { minLength = "1" maxLength = "127" } }?, + attribute prev_value { xsd:string { minLength = "1" maxLength = "127" } }?, + attribute is_positive_intent { xsd:boolean }?, + attribute is_upward_change { xsd:boolean }? +} + +################################################################################ +# AssetBlocks + +# These are dervived from the attached asset file and not set by the user +staticAssetAttributes = + attribute cas_ref { xsd:string { pattern = "[0-9a-f]{64}" } } ?, + attribute type { xsd:string { pattern = '\w+/[\w.+\-]+' } }, + # attribute size { xsd:positiveInteger }, + # attribute hash { xsd:string { pattern = "[0-9a-f]{10}" } }, + attribute uploaded_filename { xsd:string { maxLength = "127" } } ? + +commonAttributes = + # Do we need ref also? + block_label_or_name, + opt_caption, + attribute src { xsd:anyURI { pattern = "((attachment|http|https|file|data|ref|cas):|/).+"} } + + +Media = element Media { + staticAssetAttributes?, + commonAttributes + } + +# Additional atrtibutes (and Type tag) per Asset type, these can be set by the user +Attachment = element Attachment { + staticAssetAttributes?, + commonAttributes, + attribute filename { xsd:string { minLength = "1" maxLength = "127" } } + } + +Plot = element Plot { + staticAssetAttributes?, + commonAttributes, + + [ a:defaultValue = "1.0" ] + attribute scale { xsd:decimal { minExclusive = "0" } }, + + [ a:defaultValue = "true" ] + attribute responsive { xsd:boolean} +} + +Table = element Table { + staticAssetAttributes?, + commonAttributes +} + +DataTable = element DataTable { + staticAssetAttributes?, + commonAttributes, + # these assets are applied during renderable + attribute rows { xsd:positiveInteger }?, + attribute columns { xsd:positiveInteger }?, + attribute schema { xsd:string { minLength = "1" } }? + # attribute cells { xsd:positiveInteger } +} diff --git a/eurybia/report/datapane/resources/view_resources/full_schema.rng b/eurybia/report/datapane/resources/view_resources/full_schema.rng new file mode 100644 index 0000000..b0793ad --- /dev/null +++ b/eurybia/report/datapane/resources/view_resources/full_schema.rng @@ -0,0 +1,420 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1 + 256 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + dropdown + tabs + + + + + + + + + + + + + + + + + + + + 2 + \[\d+(,\s*\d+)*\] + + + + + + top + center + bottom + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1 + 512 + + + + + + + + + + + 1 + (.|\s)*\S(.|\s)* + + + + + + + + 1 + + + + + + + + + 1 + 127 + + + + + 1 + + + + + + + + + + + + 1 + 255 + + + + + 1 + 127 + + + + + 1 + + + + + + + + + 1 + + + + + + + + + 1 + 127 + + + + + 1 + 127 + + + + + + + 1 + 127 + + + + + + + 1 + 127 + + + + + + + + + + + + + + + + + + + + + + + [0-9a-f]{64} + + + + + + \w+/[\w.+\-]+ + + + + + + + 127 + + + + + + + + + + + ((attachment|http|https|file|data|ref|cas):|/).+ + + + + + + + + + + + + + + + + + + + + + 1 + 127 + + + + + + + + + + + + + 0 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1 + + + + + + + + diff --git a/eurybia/report/datapane/resources/view_resources/local_post_process.xslt b/eurybia/report/datapane/resources/view_resources/local_post_process.xslt new file mode 100644 index 0000000..0016cd7 --- /dev/null +++ b/eurybia/report/datapane/resources/view_resources/local_post_process.xslt @@ -0,0 +1,47 @@ + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/eurybia/report/datapane/view/__init__.py b/eurybia/report/datapane/view/__init__.py new file mode 100644 index 0000000..075b23a --- /dev/null +++ b/eurybia/report/datapane/view/__init__.py @@ -0,0 +1,4 @@ +# flake8: noqa:F401 +from .view_blocks import App, Blocks, BlocksT, Report, View +from .visitors import PreProcess, PrettyPrinter, ViewVisitor +from .xml_visitor import XMLBuilder diff --git a/eurybia/report/datapane/view/asset_writers.py b/eurybia/report/datapane/view/asset_writers.py new file mode 100644 index 0000000..e4d4401 --- /dev/null +++ b/eurybia/report/datapane/view/asset_writers.py @@ -0,0 +1,175 @@ +""" +# TODO - optimise import handling here +# NOTE - flake8 disabled on this file, as is not a fan of multimethod overriding here +""" + +# flake8: noqa:F811 +from __future__ import annotations + +import json +import pickle +import typing as t +from contextlib import suppress +from io import TextIOWrapper + +import pandas as pd +from altair.utils import SchemaBase +from multimethod import multimethod +from packaging import version as v +from packaging.specifiers import SpecifierSet +from pandas.io.formats.style import Styler + +from eurybia.report.datapane import optional_libs as opt +from eurybia.report.datapane.client import DPClientError, log +from eurybia.report.datapane.common import ArrowFormat + +from .xml_visitor import AssetMeta + +# NOTE - need to update this and keep in sync with JS +BOKEH_V_SPECIFIER = SpecifierSet("~=2.4.2") +PLOTLY_V_SPECIFIER = SpecifierSet(">=4.0.0") +FOLIUM_V_SPECIFIER = SpecifierSet(">=0.12.0") + + +def _check_version(name: str, _v: v.Version, ss: SpecifierSet): + if _v not in ss: + log.warning( + f"{name} version {_v} is not supported, these plots may not display correctly, please install version {ss}" + ) + + +class DPTextIOWrapper(TextIOWrapper): + """Custom IO Wrapper that detaches before closing - see https://bugs.python.org/issue21363""" + + def __init__(self, f, *a, **kw): + super().__init__(f, encoding="utf-8", *a, **kw) + + def __del__(self): + # don't close the underlying stream + with suppress(Exception): + self.flush() + with suppress(Exception): + self.detach() + + +class AttachmentWriter: + # pickle + @multimethod + def get_meta(self, x: t.Any) -> AssetMeta: + return AssetMeta(ext=".pkl", mime="application/vnd.pickle+binary") + + @multimethod + def get_meta(self, x: str) -> AssetMeta: + return AssetMeta(ext=".json", mime="application/json") + + @multimethod + def write_file(self, x: t.Any, f) -> None: + pickle.dump(x, f) + + @multimethod + def write_file(self, x: str, f) -> None: + out: str = json.dumps(json.loads(x)) + f.write(out.encode()) + + +class DataTableWriter: + @multimethod + def get_meta(self, x: pd.DataFrame) -> AssetMeta: + return AssetMeta(mime=ArrowFormat.content_type, ext=ArrowFormat.ext) + + @multimethod + def write_file(self, x: pd.DataFrame, f) -> None: + if x.size == 0: + raise DPClientError("Empty DataFrame provided") + # process_df called in Arrow.save_file + ArrowFormat.save_file(f, x) + + +class HTMLTableWriter: + @multimethod + def get_meta(self, x: t.Union[pd.DataFrame, Styler]) -> AssetMeta: + return AssetMeta(mime="application/vnd.datapane.table+html", ext=".tbl.html") + + @multimethod + def write_file(self, x: pd.DataFrame, f) -> None: + self._check(x) + out = x.to_html().encode() + f.write(out) + + @multimethod + def write_file(self, x: Styler, f) -> None: + self._check(x.data) + out = x.to_html().encode() + f.write(out) + + def _check(self, df: pd.DataFrame) -> None: + n_cells = df.shape[0] * df.shape[1] + if n_cells > 500: + log.warning( + "Table is over recommended size, consider using dp.DataTable instead or aggregating the df first" + ) + + +class PlotWriter: + obj_type: t.Any + + # Altair (always installed) + @multimethod + def get_meta(self, x: SchemaBase) -> AssetMeta: + return AssetMeta(mime="application/vnd.vegalite.v5+json", ext=".vl.json") + + @multimethod + def write_file(self, x: SchemaBase, f) -> None: + json.dump(x.to_dict(), DPTextIOWrapper(f)) + + if opt.HAVE_FOLIUM: + + @multimethod + def get_meta(self, x: opt.Map) -> AssetMeta: + return AssetMeta(mime="application/vnd.folium+html", ext=".fl.html") + + @multimethod + def write_file(self, x: opt.Map, f) -> None: + html: str = x.get_root().render() + f.write(html.encode()) + + if opt.HAVE_BOKEH: + + @multimethod + def get_meta(self, x: t.Union[opt.BFigure, opt.BLayout]) -> AssetMeta: + return AssetMeta(mime="application/vnd.bokeh.show+json", ext=".bokeh.json") + + @multimethod + def write_file(self, x: t.Union[opt.BFigure, opt.BLayout], f): + from bokeh.embed import json_item + + json.dump(json_item(x), DPTextIOWrapper(f)) + + if opt.HAVE_PLOTLY: + + @multimethod + def get_meta(self, x: opt.PFigure) -> AssetMeta: + return AssetMeta(mime="application/vnd.plotly.v1+json", ext=".pl.json") + + @multimethod + def write_file(self, x: opt.PFigure, f): + json.dump(x.to_json(), DPTextIOWrapper(f)) + + if opt.HAVE_MATPLOTLIB: + + @multimethod + def get_meta(self, x: t.Union[opt.Axes, opt.Figure, opt.ndarray]) -> AssetMeta: + return AssetMeta(mime="image/svg+xml", ext=".svg") + + @multimethod + def write_file(self, x: opt.Figure, f) -> None: + x.savefig(DPTextIOWrapper(f), format="svg", bbox_inches="tight") + + @multimethod + def write_file(self, x: opt.Axes, f) -> None: + self.write_file(x.get_figure(), f) + + @multimethod + def write_file(self, x: opt.ndarray, f) -> None: + fig = x.flatten()[0].get_figure() + self.write_file(fig, f) diff --git a/eurybia/report/datapane/view/view_blocks.py b/eurybia/report/datapane/view/view_blocks.py new file mode 100644 index 0000000..03c151a --- /dev/null +++ b/eurybia/report/datapane/view/view_blocks.py @@ -0,0 +1,195 @@ +from __future__ import annotations + +import typing as t +import warnings +from copy import copy + +from lxml import etree +from lxml.etree import _Element as ElementT + +from eurybia.report.datapane.blocks import Group +from eurybia.report.datapane.blocks.base import BlockOrPrimitive +from eurybia.report.datapane.blocks.layout import ContainerBlock + +if t.TYPE_CHECKING: + from typing_extensions import Self + + from eurybia.report.datapane.processors.types import Formatting + + +class Blocks(ContainerBlock): + """Container that holds a collection of blocks""" + + # This is essentially an easy-to-use wrapper around a list of blocks + # that is composable. + # TODO - move to datapane.blocks ? + + _tag = "Blocks" + + def __init__( + self, + *arg_blocks: BlockOrPrimitive, + blocks: t.List[BlockOrPrimitive] = None, + **kwargs, + ): + # if passed a single View into a View object, pull out the contained blocks and use instead + if len(arg_blocks) == 1 and isinstance(arg_blocks[0], Blocks): + arg_blocks = tuple(arg_blocks[0].blocks) + + super().__init__(*arg_blocks, blocks=blocks, **kwargs) + + def __or__(self, other: Blocks) -> Blocks: + x = Group(blocks=self.blocks) if len(self.blocks) > 1 else self.blocks[0] + y = Group(blocks=other.blocks) if len(other.blocks) > 1 else other.blocks[0] + z = Group(x, y, columns=2) + return Blocks(z) + + @classmethod + def from_notebook( + cls, + opt_out: bool = True, + show_code: bool = False, + show_markdown: bool = True, + template: str = "auto", + ) -> Self: + from eurybia.report.datapane.ipython import templates as ip_t + from eurybia.report.datapane.ipython.utils import cells_to_blocks + + blocks = cells_to_blocks( + opt_out=opt_out, show_code=show_code, show_markdown=show_markdown + ) + app_template_cls = ip_t._registry.get(template) or ip_t.guess_template(blocks) + app_template = app_template_cls(blocks) + app_template.transform() + app_template.validate() + return cls(blocks=app_template.blocks) + + def get_dom(self) -> ElementT: + """Return the Document structure for the View""" + # internal debugging method + from eurybia.report.datapane.processors.file_store import ( + DummyFileEntry, + FileStore, + ) + + from .xml_visitor import XMLBuilder + + builder = XMLBuilder(FileStore(DummyFileEntry)) + self.accept(builder) + return builder.get_root() + + def get_dom_str(self) -> str: + dom = self.get_dom() + return etree.tounicode(dom, pretty_print=True) + + def pprint(self) -> None: + from .visitors import PrettyPrinter + + self.accept(PrettyPrinter()) + + @classmethod + def wrap_blocks( + cls, x: t.Union[Self, t.List[BlockOrPrimitive], BlockOrPrimitive] + ) -> Self: + blocks: Self + if isinstance(x, Blocks): + blocks = copy(x) + elif isinstance(x, list): + blocks = cls(*x) + else: + blocks = cls(x) + return blocks + + @property + def has_compute(self): + return False + + +class View(Blocks): + pass + + +BlocksT = t.Union[ + Blocks, t.List[BlockOrPrimitive], t.Mapping[str, BlockOrPrimitive], BlockOrPrimitive +] + + +class App(Blocks): + """ + App documents collate plots, text, tables, and files into an interactive document that + can be analysed and shared by users in their browser + """ + + # Backwards compatible interfaces/wrappers + + def __init__( + self, + *arg_blocks: BlockOrPrimitive, + blocks: t.List[BlockOrPrimitive] = None, + **kwargs, + ): + if "layout" in kwargs: + raise ValueError( + "App(layout=...) is no longer supported, please use `dp.Group(columns=...)` for horizontal layouts" + ) + warnings.warn( + "Instead of dp.App(), please see our newer API dp.Blocks(). " + + "Instead of App.upload(), App.save_report() etc., you can use dp.upload_report(blocks), dp.save_report(blocks)", + DeprecationWarning, + ) + super().__init__(*arg_blocks, blocks=blocks, **kwargs) + + def upload( + self, + *args, + **kwargs, + ) -> None: + from ..processors import upload_report + + upload_report(*args, **kwargs) + + def save( + self, + path: str, + open: bool = False, + standalone: bool = False, + name: t.Optional[str] = None, + author: t.Optional[str] = None, + formatting: t.Optional[Formatting] = None, + cdn_base: t.Optional[str] = None, + ) -> None: + from ..processors import save_report + + if standalone: + raise ValueError("save(standalone=True) is no longer supported, sorry!") + if author is not None: + raise ValueError('save(author="...") is no longer supported, sorry!') + if cdn_base is not None: + raise ValueError('save(cdn_base="...") is no longer supported, sorry!') + save_report(blocks=self, path=path, open=open, name=name, formatting=formatting) + + def stringify( + self, + standalone: bool = False, + name: t.Optional[str] = None, + author: t.Optional[str] = None, + formatting: t.Optional[Formatting] = None, + cdn_base: t.Optional[str] = None, + template_name: str = "template.html", + ) -> str: + from ..processors import stringify_report + + if standalone: + raise ValueError("save(standalone=True) is no longer supported, sorry!") + if author is not None: + raise ValueError('save(author="...") is no longer supported, sorry!') + if cdn_base is not None: + raise ValueError('save(cdn_base="...") is no longer supported, sorry!') + if template_name != "template.html": + raise ValueError('save(template_name="...") is no longer supported, sorry!') + + return stringify_report(blocks=self, name=name, formatting=formatting) + + +class Report(App): + pass diff --git a/eurybia/report/datapane/view/visitors.py b/eurybia/report/datapane/view/visitors.py new file mode 100644 index 0000000..f854029 --- /dev/null +++ b/eurybia/report/datapane/view/visitors.py @@ -0,0 +1,127 @@ +# flake8: noqa:F811 +from __future__ import annotations + +import abc +import dataclasses as dc +import typing as t +from contextlib import contextmanager +from copy import copy + +from multimethod import multimethod + +from eurybia.report.datapane import blocks as bk +from eurybia.report.datapane.blocks import BaseBlock +from eurybia.report.datapane.blocks.layout import ContainerBlock +from eurybia.report.datapane.client import DPClientError, log + +from .view_blocks import Blocks + +if t.TYPE_CHECKING: + from eurybia.report.datapane.app.runtime import FEntries + from eurybia.report.datapane.blocks.base import VV + + +@dc.dataclass +class ViewVisitor(abc.ABC): + @multimethod + def visit(self: VV, b: BaseBlock) -> VV: + return self + + +@dc.dataclass +class PrettyPrinter(ViewVisitor): + """Print out the view in an indented, XML-like tree form""" + + indent: int = 1 + + @multimethod + def visit(self, b: BaseBlock): + print("|", "-" * self.indent, str(b), sep="") + + @multimethod + def visit(self, b: ContainerBlock): + print("|", "-" * self.indent, str(b), sep="") + self.indent += 2 + _ = b.traverse(self) + self.indent -= 2 + + +# TODO - split out into BlockBuilder helper here +@dc.dataclass +class PreProcess(ViewVisitor): + """Block-level preprocessor, operations include, + - Inline consecutive unnamed Text blocks + - auto-column processing (TODO) + + We may need multiple passes here to make things easier + + We also transform the AST during the XMLBuilder visitor, and post-XML conversion in the ConvertXML processor. + Where the transform should be placed is still under consideration, and depends on the nature of the transform. + Some may be easier as a simple XSLT transform, others as more complex Python code operation either on the Blocks AST + or the XML DOM via lxml. + """ + + in_collapsible_group: bool = False + # is_finalised determines if we allow dynamic blocks or validate based on their static/current subblocks + is_finalised: bool = True + current_state: list[BaseBlock] = dc.field(default_factory=list) + current_text: list[bk.Text] = dc.field(default_factory=list) + + @multimethod + def visit(self, b: BaseBlock): + self.merge_text() + self.current_state.append(copy(b)) + + @multimethod + def visit(self, b: bk.Text): + if b.name is None: + self.current_text.append(b) + else: + self.merge_text() + self.current_state.append(copy(b)) + + @multimethod + def visit(self, b: ContainerBlock): + self.merge_text() + + if len(b.blocks) < b.report_minimum_blocks: + msg = f"{b.__class__.__name__} has less than {b.report_minimum_blocks} objects" + if self.is_finalised: + raise DPClientError(msg) + else: + log.warning(msg) + + with self.fresh_state(b): + self.in_collapsible_group = isinstance(b, Blocks) or ( + isinstance(b, bk.Group) and b.columns == 1 + ) + _ = b.traverse(self) + self.merge_text() + + def merge_text(self): + # log.debug("Merging text nodes") + if self.current_text: + t1: bk.Text + if self.in_collapsible_group: + new_text = "\n\n".join(t1.content for t1 in self.current_text) + self.current_state.append(bk.Text(new_text)) + else: + self.current_state.extend(copy(t1) for t1 in self.current_text) + self.current_text = [] + + @property + def root(self) -> BaseBlock: + return self.current_state[0] + + @contextmanager + def fresh_state(self, b: ContainerBlock) -> None: + x = self.current_state + self.current_state = [] + + yield None + + # build a new instance of the container block + b1 = copy(b) + b1.blocks = self.current_state + x.append(b1) + self.current_state = x diff --git a/eurybia/report/datapane/view/xml_visitor.py b/eurybia/report/datapane/view/xml_visitor.py new file mode 100644 index 0000000..a8d42eb --- /dev/null +++ b/eurybia/report/datapane/view/xml_visitor.py @@ -0,0 +1,187 @@ +# flake8: noqa:F811 +from __future__ import annotations + +import dataclasses as dc +import typing as t +from collections import namedtuple + +from lxml import etree +from lxml.builder import ElementMaker +from multimethod import DispatchError, multimethod + +from eurybia.report.datapane import DPClientError +from eurybia.report.datapane.blocks import BaseBlock +from eurybia.report.datapane.blocks.asset import AssetBlock +from eurybia.report.datapane.blocks.empty import gen_name +from eurybia.report.datapane.blocks.layout import ContainerBlock +from eurybia.report.datapane.blocks.text import EmbeddedTextBlock +from eurybia.report.datapane.client import log +from eurybia.report.datapane.common.viewxml_utils import ElementT, mk_attribs +from eurybia.report.datapane.view.view_blocks import Blocks +from eurybia.report.datapane.view.visitors import ViewVisitor + +if t.TYPE_CHECKING: + from eurybia.report.datapane.processors import FileEntry, FileStore + + # from typing_extensions import Self + +E = ElementMaker() # XML Tag Factory + + +@dc.dataclass +class XMLBuilder(ViewVisitor): + """Convert the Blocks into an XML document""" + + store: FileStore + # element: t.Optional[etree.Element] = None # Empty Group Element? + elements: t.List[ElementT] = dc.field(default_factory=list) + + def get_root(self, fragment: bool = False) -> ElementT: + """Return the top-level ViewXML""" + # create the top-level + + # get the top-level root + _top_group: ElementT = self.elements.pop() + assert _top_group.tag == "Group" + assert not self.elements + + # create top-level structure + return E.View( + # E.Internal(), + *_top_group.getchildren(), + **mk_attribs(version="1", fragment=fragment), + ) + + @property + def store_count(self) -> int: + return len(self.store.files) + + def add_element(self, _: BaseBlock, e: etree.Element) -> XMLBuilder: + """Add an element to the list of nodes at the current XML tree location""" + self.elements.append(e) + return self + + # xml convertors + @multimethod + def visit(self, b: BaseBlock) -> XMLBuilder: + """Base implementation - just created an empty tag including all the initial attributes""" + _E = getattr(E, b._tag) + return self.add_element(b, _E(**b._attributes)) + + def _visit_subnodes(self, b: ContainerBlock) -> t.List[ElementT]: + cur_elements = self.elements + self.elements = [] + b.traverse(self) # visit subnodes + res = self.elements + self.elements = cur_elements + return res + + @multimethod + def visit(self, b: ContainerBlock) -> XMLBuilder: + sub_elements = self._visit_subnodes(b) + # build the element + _E = getattr(E, b._tag) + element = _E(*sub_elements, **b._attributes) + return self.add_element(b, element) + + @multimethod + def visit(self, b: Blocks) -> XMLBuilder: + sub_elements = self._visit_subnodes(b) + + # Blocks are converted to Group internally + if label := b._attributes.get("label"): + log.info(f"Found label {label} in top-level Blocks/View") + element = E.Group(*sub_elements, columns="1", valign="top") + return self.add_element(b, element) + + @multimethod + def visit(self, b: EmbeddedTextBlock) -> XMLBuilder: + # NOTE - do we use etree.CDATA wrapper? + _E = getattr(E, b._tag) + return self.add_element(b, _E(etree.CDATA(b.content), **b._attributes)) + + @multimethod + def visit(self, b: AssetBlock): + """Main XMl creation method - visitor method""" + fe = self._add_asset_to_store(b) + + _E = getattr(E, b._tag) + + e: etree._Element = _E( + type=fe.mime, + # size=conv_attrib(fe.size), + # hash=fe.hash, + **{**b._attributes, **b.get_file_attribs()}, + # src=f"attachment://{self.store_count}", + src=f"ref://{fe.hash}", + ) + + if b.caption: + e.set("caption", b.caption) + return self.add_element(b, e) + + def _add_asset_to_store(self, b: AssetBlock) -> FileEntry: + """Default asset store handler that operates on native Python objects""" + # import here as a very slow module due to nested imports + # from .. import files + + # check if we already have stored this asset to the store + # TODO - do we just persist the asset store across the session?? + if b._prev_entry: + if type(b._prev_entry) == self.store.fw_klass: + self.store.add_file(b._prev_entry) + return b._prev_entry + else: + b._prev_entry = None + + if b.data is not None: + # fe = files.add_to_store(self.data, s.store) + try: + writer = get_writer(b) + meta: AssetMeta = writer.get_meta(b.data) + fe = self.store.get_file(meta.ext, meta.mime) + writer.write_file(b.data, fe.file) + self.store.add_file(fe) + except DispatchError: + raise DPClientError( + f"{type(b.data).__name__} not supported for {self.__class__.__name__}" + ) + elif b.file is not None: + fe = self.store.load_file(b.file) + else: + raise DPClientError("No asset to add") + + b._prev_entry = fe + return fe + + +AssetMeta = namedtuple("AssetMeta", "ext mime") + + +class AssetWriterP(t.Protocol): + """Implement these in any class to support asset writing + for a particular AssetBlock""" + + def get_meta(self, x: t.Any) -> AssetMeta: ... + + def write_file(self, x: t.Any, f) -> None: ... + + +asset_mapping: t.Dict[t.Type[AssetBlock], t.Type[AssetWriterP]] = dict() + + +def get_writer(b: AssetBlock) -> AssetWriterP: + import eurybia.report.datapane.blocks.asset as a + + from . import asset_writers as aw + + if not asset_mapping: + asset_mapping.update( + { + a.Plot: aw.PlotWriter, + a.Table: aw.HTMLTableWriter, + a.DataTable: aw.DataTableWriter, + a.Attachment: aw.AttachmentWriter, + } + ) + return asset_mapping[type(b)]() diff --git a/eurybia/report/generation.py b/eurybia/report/generation.py index cdecd0c..7d3fc8e 100644 --- a/eurybia/report/generation.py +++ b/eurybia/report/generation.py @@ -1,10 +1,11 @@ """ Report generation helper module. """ + from datetime import datetime from typing import Optional -import datapane as dp +import eurybia.report.datapane as dp import pandas as pd from shapash.explainer.smart_explainer import SmartExplainer @@ -12,7 +13,9 @@ from eurybia.report.project_report import DriftReport -def _get_index(dr: DriftReport, project_info_file: str, config_report: Optional[dict]) -> dp.Page: +def _get_index( + dr: DriftReport, project_info_file: str, config_report: Optional[dict] +) -> dp.Page: """ This function generates and returns a Datapane page containing the Eurybia report index @@ -39,7 +42,9 @@ def _get_index(dr: DriftReport, project_info_file: str, config_report: Optional[ index_block = [] # Title and logo - index_block += [dp.Group(dp.HTML(eurybia_logo), dp.Text(f"# {dr.title_story}"), columns=2)] + index_block += [ + dp.Group(dp.HTML(eurybia_logo), dp.Text(f"# {dr.title_story}"), columns=2) + ] if ( config_report is not None @@ -53,7 +58,9 @@ def _get_index(dr: DriftReport, project_info_file: str, config_report: Optional[ # Tabs index if project_info_file is not None: index_str += "- Project information: report context and information \n" - index_str += "- Consistency Analysis: highlighting differences between the two datasets \n" + index_str += ( + "- Consistency Analysis: highlighting differences between the two datasets \n" + ) index_str += "- Data drift: In-depth data drift analysis \n" if dr.smartdrift.data_modeldrift is not None: @@ -63,7 +70,10 @@ def _get_index(dr: DriftReport, project_info_file: str, config_report: Optional[ # AUC auc_block = dr.smartdrift.plot.generate_indicator( - fig_value=dr.smartdrift.auc, height=280, width=500, title="Datadrift classifier AUC" + fig_value=dr.smartdrift.auc, + height=280, + width=500, + title="Datadrift classifier AUC", ) # Jensen-Shannon @@ -110,7 +120,11 @@ def _dict_to_text_blocks(text_dict, level=1): blocks.append(dp.Text(text)) text = "" blocks.append( - dp.Group(dp.Text("#" * min(level, 6) + " " + str(k)), _dict_to_text_blocks(v, level + 1), columns=1) + dp.Group( + dp.Text("#" * min(level, 6) + " " + str(k)), + _dict_to_text_blocks(v, level + 1), + columns=1, + ) ) if text != "": blocks.append(dp.Text(text)) @@ -198,12 +212,20 @@ def _get_consistency_analysis(dr: DriftReport) -> dp.Page: blocks += [ dp.Table( data=pd.DataFrame(dr.smartdrift.err_mods) - .rename(columns={"err_mods": "Modalities present in one dataset and absent in the other :"}) + .rename( + columns={ + "err_mods": "Modalities present in one dataset and absent in the other :" + } + ) .transpose() ) ] else: - blocks += [dp.Text("- No modalities have been detected as present in one dataset and absent in the other.")] + blocks += [ + dp.Text( + "- No modalities have been detected as present in one dataset and absent in the other." + ) + ] page_consistency = dp.Page(title="Consistency Analysis", blocks=blocks) return page_consistency @@ -225,7 +247,9 @@ def _get_datadrift(dr: DriftReport) -> dp.Page: # Loop for save in list plots of display analysis plot_dataset_analysis = [] table_dataset_analysis = [] - fig_list, labels, table_list = dr.display_dataset_analysis(global_analysis=False)["univariate"] + fig_list, labels, table_list = dr.display_dataset_analysis(global_analysis=False)[ + "univariate" + ] for i in range(len(labels)): plot_dataset_analysis.append(dp.Plot(fig_list[i], label=labels[i])) table_dataset_analysis.append(dp.Table(table_list[i], label=labels[i])) @@ -254,7 +278,10 @@ def _get_datadrift(dr: DriftReport) -> dp.Page: ), dp.Plot( dr.smartdrift.plot.generate_indicator( - fig_value=dr.smartdrift.auc, height=300, width=500, title="Datadrift classifier AUC" + fig_value=dr.smartdrift.auc, + height=300, + width=500, + title="Datadrift classifier AUC", ) ), dp.Text("## Importance of features in data drift"), @@ -306,7 +333,9 @@ def _get_datadrift(dr: DriftReport) -> dp.Page: "Histogram density showing the distributions of the production model outputs on both baseline and current datasets." ), dp.Plot( - dr.smartdrift.plot.generate_fig_univariate(df_all=dr.smartdrift.df_predict, col="Score", hue="dataset") + dr.smartdrift.plot.generate_fig_univariate( + df_all=dr.smartdrift.df_predict, col="Score", hue="dataset" + ) ), dp.Text( """Jensen Shannon Divergence (JSD). The JSD measures the effect of a data drift on the deployed model performance. @@ -366,7 +395,11 @@ def _get_modeldrift(dr: DriftReport) -> dp.Page: else: for i in range(len(labels)): plot_modeldrift.append(dp.Plot(fig_list[i], label=labels[i])) - modeldrift_plot = dp.Select(blocks=plot_modeldrift, label="reference_columns", type=dp.SelectType.DROPDOWN) + modeldrift_plot = dp.Select( + blocks=plot_modeldrift, + label="reference_columns", + type=dp.SelectType.DROPDOWN, + ) else: modeldrift_plot = dp.Text("## Smartdrift.data_modeldrift is None") blocks = [ diff --git a/requirements.dev.txt b/requirements.dev.txt index 108db69..394f104 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -28,5 +28,11 @@ seaborn>=0.12.2 notebook>=6.0.0 Jinja2>=2.11.0 scipy>=1.1.0 -datapane>=0.16.7 pre-commit==2.18.1 +altair>=5.0.0 +pyarrow>=9.0.0 +chardet>=5.0.0,<6.0.0 +lxml>=4.7.1,<5.0.0 +micawber>=0.5.3 +dominate>=2.7.0,<3.0.0 +multimethod>=1.9.0,<2.0.0 \ No newline at end of file diff --git a/setup.py b/setup.py index b142d0a..93af566 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,7 @@ #!/usr/bin/env python """The setup script.""" + import os from setuptools import setup @@ -17,13 +18,20 @@ requirements = [ "catboost>=1.0.1", - "datapane>=0.16.7", "ipywidgets>=7.4.2", "jinja2>=2.11.0", "scipy>=1.4.0", "seaborn>=0.10.1", "shapash>=2.0.0", "jupyter", + # datapane dependencies + "altair>=5.0.0", + "pyarrow>=9.0.0", + "chardet>=5.0.0,<6.0.0", + "lxml>=4.7.1,<5.0.0", + "micawber>=0.5.3", + "dominate>=2.7.0,<3.0.0", + "multimethod>=1.9.0,<2.0.0", ] @@ -65,7 +73,14 @@ "eurybia.style": "eurybia/style", "eurybia.utils": "eurybia/utils", }, - packages=["eurybia", "eurybia.data", "eurybia.core", "eurybia.report", "eurybia.style", "eurybia.utils"], + packages=[ + "eurybia", + "eurybia.data", + "eurybia.core", + "eurybia.report", + "eurybia.style", + "eurybia.utils", + ], data_files=[ ("data", ["eurybia/data/house_prices_dataset.csv"]), ("data", ["eurybia/data/house_prices_labels.json"]), diff --git a/tests/integration_tests/test_report_generation.py b/tests/integration_tests/test_report_generation.py index d463c7f..19307d7 100644 --- a/tests/integration_tests/test_report_generation.py +++ b/tests/integration_tests/test_report_generation.py @@ -9,7 +9,7 @@ import pandas as pd from category_encoders import OrdinalEncoder -from datapane.client import config +from eurybia.report.datapane.client import config from sklearn.ensemble import RandomForestClassifier from sklearn.model_selection import train_test_split @@ -35,16 +35,24 @@ def setUp(self): y = titan_df["Survived"] X = titan_df.drop("Survived", axis=1).drop("Name", axis=1) varcat = ["Pclass", "Sex", "Embarked", "Title"] - categ_encoding = OrdinalEncoder(cols=varcat, handle_unknown="ignore", return_df=True).fit(X) + categ_encoding = OrdinalEncoder( + cols=varcat, handle_unknown="ignore", return_df=True + ).fit(X) X_encoded = categ_encoding.transform(X) - Xtrain, Xtest, ytrain, ytest = train_test_split(X, y, train_size=0.75, random_state=1) - Xtrain_e, Xtest_e, ytrain_e, ytest_e = train_test_split(X_encoded, y, train_size=0.75, random_state=1) + Xtrain, Xtest, ytrain, ytest = train_test_split( + X, y, train_size=0.75, random_state=1 + ) + Xtrain_e, Xtest_e, ytrain_e, ytest_e = train_test_split( + X_encoded, y, train_size=0.75, random_state=1 + ) rf = RandomForestClassifier(n_estimators=100, min_samples_leaf=3) rf.fit(Xtrain_e, ytrain_e) - smartdrift = SmartDrift(Xtrain, Xtest, deployed_model=rf, encoding=categ_encoding) + smartdrift = SmartDrift( + Xtrain, Xtest, deployed_model=rf, encoding=categ_encoding + ) smartdrift.compile(full_validation=True) self.smartdrift = smartdrift self.xpl = smartdrift.xpl @@ -92,7 +100,10 @@ def test_execute_report_3(self): explainer=self.xpl, project_info_file=os.path.join(current_path, "../data/project_info.yml"), output_file="./report.html", - config_report=dict(title_story="Test integration", title_description="Title of test integration"), + config_report=dict( + title_story="Test integration", + title_description="Title of test integration", + ), ) assert os.path.exists("./report.html") @@ -101,14 +112,23 @@ def test_execute_report_modeldrift_1(self): """ Test execute_report() method """ - df_perf = pd.DataFrame({"mois": [1, 2, 3], "annee": [2018, 2019, 2020], "performance": [2, 3.46, 2.5]}) + df_perf = pd.DataFrame( + { + "mois": [1, 2, 3], + "annee": [2018, 2019, 2020], + "performance": [2, 3.46, 2.5], + } + ) self.smartdrift.add_data_modeldrift(dataset=df_perf, metric="performance") execute_report( smartdrift=self.smartdrift, explainer=self.xpl, project_info_file=os.path.join(current_path, "../data/project_info.yml"), output_file="./report.html", - config_report=dict(title_story="Test integration", title_description="Title of test integration"), + config_report=dict( + title_story="Test integration", + title_description="Title of test integration", + ), ) assert os.path.exists("./report.html") @@ -127,14 +147,19 @@ def test_execute_report_modeldrift_2(self): } ) self.smartdrift.add_data_modeldrift( - dataset=df_perf2, metric="performance", reference_columns=["reference_column1", "reference_column2"] + dataset=df_perf2, + metric="performance", + reference_columns=["reference_column1", "reference_column2"], ) execute_report( smartdrift=self.smartdrift, explainer=self.xpl, project_info_file=os.path.join(current_path, "../data/project_info.yml"), output_file="./report.html", - config_report=dict(title_story="Test integration", title_description="Title of test integration"), + config_report=dict( + title_story="Test integration", + title_description="Title of test integration", + ), ) assert os.path.exists("./report.html") diff --git a/tests/unit_tests/core/test_smartplotter.py b/tests/unit_tests/core/test_smartplotter.py index c80a58b..b870518 100644 --- a/tests/unit_tests/core/test_smartplotter.py +++ b/tests/unit_tests/core/test_smartplotter.py @@ -31,7 +31,9 @@ def setUp(self): script_path = Path(path.abspath(__file__)).parent.parent.parent.parent titanic_csv = path.join(script_path, TITANIC_PATH) titanic_df = pd.read_csv(titanic_csv) - titanic_df_1, titanic_df_2 = train_test_split(titanic_df, test_size=0.5, random_state=42) + titanic_df_1, titanic_df_2 = train_test_split( + titanic_df, test_size=0.5, random_state=42 + ) self.smartdrift = SmartDrift(titanic_df_1, titanic_df_2) @patch("eurybia.core.smartplotter.SmartPlotter.generate_fig_univariate_continuous") @@ -70,7 +72,10 @@ def test_generate_fig_univariate_2(self, mock_plot_cat, mock_plot_cont): Unit test for generate_fig_univariate()'s method """ df = pd.DataFrame( - {"int_data": list(range(50)), "data_train_test": ["train", "train", "train", "train", "test"] * 10} + { + "int_data": list(range(50)), + "data_train_test": ["train", "train", "train", "train", "test"] * 10, + } ) dict_color_palette = { @@ -149,7 +154,10 @@ def test_generate_fig_univariate_categorical(self): Unit test for generate_fig_univariate_categorical()'s method """ df = pd.DataFrame( - {"int_data": [0, 0, 0, 1, 1, 0], "data_train_test": ["train", "train", "train", "train", "test", "test"]} + { + "int_data": [0, 0, 0, 1, 1, 0], + "data_train_test": ["train", "train", "train", "train", "test", "test"], + } ) dict_color_palette = { "df_Baseline": (74 / 255, 99 / 255, 138 / 255, 0.7), @@ -165,7 +173,9 @@ def test_generate_fig_univariate_categorical(self): fig = self.smartdrift.plot.generate_fig_univariate_categorical( df, "int_data", "data_train_test", dict_color_palette=dict_color_palette ) - nb_bars = len(fig.to_dict()["data"][0]["y"]) + len(fig.to_dict()["data"][1]["y"]) + nb_bars = len(fig.to_dict()["data"][0]["y"]) + len( + fig.to_dict()["data"][1]["y"] + ) assert nb_bars == 4 # Number of bars def test_scatter_feature_importance(self): @@ -173,13 +183,49 @@ def test_scatter_feature_importance(self): Unit test for scatter_feature_importance()'s method """ importances = { - "feature": ["Parch", "Embarked", "SibSp", "Age", "Fare", "Pclass", "Title", "Sex"], - "deployed_model": [0.014573, 0.046866, 0.054405, 0.068945, 0.102350, 0.155283, 0.255974, 0.263060], - "datadrift_classifier": [0.045921, 0.118159, 0.018614, 0.137789, 0.184038, 0.143481, 0.000000, 0.162925], + "feature": [ + "Parch", + "Embarked", + "SibSp", + "Age", + "Fare", + "Pclass", + "Title", + "Sex", + ], + "deployed_model": [ + 0.014573, + 0.046866, + 0.054405, + 0.068945, + 0.102350, + 0.155283, + 0.255974, + 0.263060, + ], + "datadrift_classifier": [ + 0.045921, + 0.118159, + 0.018614, + 0.137789, + 0.184038, + 0.143481, + 0.000000, + 0.162925, + ], } features_importances = pd.DataFrame(importances) stat_test = { - "feature": ["Parch", "Embarked", "SibSp", "Age", "Fare", "Pclass", "Title", "Sex"], + "feature": [ + "Parch", + "Embarked", + "SibSp", + "Age", + "Fare", + "Pclass", + "Title", + "Sex", + ], "testname": [ "Chi-Square", "Chi-Square", @@ -225,9 +271,14 @@ def test_generate_historical_datadrift_metric_1(self): """ Unit test for generate_historical_datadrift_metric()'s method """ - auc = {"date": ["23/09/2021", "23/08/2021", "23/07/2021"], "auc": [0.528107, 0.532106, 0.510653]} + auc = { + "date": ["23/09/2021", "23/08/2021", "23/07/2021"], + "auc": [0.528107, 0.532106, 0.510653], + } datadrift_historical = pd.DataFrame(auc) - fig = self.smartdrift.plot.generate_historical_datadrift_metric(datadrift_historical) + fig = self.smartdrift.plot.generate_historical_datadrift_metric( + datadrift_historical + ) assert isinstance(fig, plotly.graph_objs._figure.Figure) def test_generate_historical_datadrift_metric_2(self): @@ -240,7 +291,9 @@ def test_generate_historical_datadrift_metric_2(self): "JS_predict": [0.28107, 0.32106, 0.50653], } datadrift_historical = pd.DataFrame(auc) - fig = self.smartdrift.plot.generate_historical_datadrift_metric(datadrift_historical) + fig = self.smartdrift.plot.generate_historical_datadrift_metric( + datadrift_historical + ) assert isinstance(fig, plotly.graph_objs._figure.Figure) def test_generate_modeldrift_data_1(self): @@ -273,7 +326,9 @@ def test_generate_modeldrift_data_2(self): df_lift = pd.DataFrame(lift_info) fig = self.smartdrift.plot.generate_modeldrift_data( - df_lift, metric="lift", reference_columns=["plage_historique", "target", "type_lift"] + df_lift, + metric="lift", + reference_columns=["plage_historique", "target", "type_lift"], ) assert isinstance(fig, plotly.graph_objs._figure.Figure) @@ -292,5 +347,7 @@ def test_generate_indicator(self): Unit test for generate_indicator()'s method """ self.smartdrift.compile() - fig = self.smartdrift.plot.generate_indicator(fig_value=self.smartdrift.auc, height=300, width=500, title="AUC") + fig = self.smartdrift.plot.generate_indicator( + fig_value=self.smartdrift.auc, height=300, width=500, title="AUC" + ) assert isinstance(fig, plotly.graph_objs._figure.Figure) diff --git a/tests/unit_tests/report/test_data_analysis.py b/tests/unit_tests/report/test_data_analysis.py index 5d37927..30eea1f 100644 --- a/tests/unit_tests/report/test_data_analysis.py +++ b/tests/unit_tests/report/test_data_analysis.py @@ -9,7 +9,10 @@ import pandas as pd from eurybia.report.common import compute_col_types -from eurybia.report.data_analysis import perform_global_dataframe_analysis, perform_univariate_dataframe_analysis +from eurybia.report.data_analysis import ( + perform_global_dataframe_analysis, + perform_univariate_dataframe_analysis, +) class TestDataAnalysis(unittest.TestCase): @@ -60,7 +63,9 @@ def test_perform_univariate_dataframe_analysis_1(self): "float_cat_data": [0.2, 0.2, 0.2, 0.6, 0.6, 0.6] * 10, } ) - dico = perform_univariate_dataframe_analysis(df, col_types=compute_col_types(df)) + dico = perform_univariate_dataframe_analysis( + df, col_types=compute_col_types(df) + ) expected_d = { "int_continuous_data": { "count": "60", @@ -94,5 +99,7 @@ def test_perform_univariate_dataframe_analysis_2(self): Unit test 2 perform_univariate_dataframe_analysis method """ df = None - dico = perform_univariate_dataframe_analysis(df, col_types=compute_col_types(df)) + dico = perform_univariate_dataframe_analysis( + df, col_types=compute_col_types(df) + ) assert dico == {}