From 1bbc44e8e2c691c86e5e92ac21b93834e8f4d51d Mon Sep 17 00:00:00 2001 From: Cenz Wong <44856918+cenzwong@users.noreply.github.com> Date: Fri, 20 Sep 2024 11:44:17 +0100 Subject: [PATCH 1/8] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index f5961e4..92e6ae1 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ pytest # TODO - Change the text in docstring +- Change pytest test case - Change the visibility to Public - Build mkdocs - Build wheels for PyPi From 8646c00eab795893c19fc7889484695770ff72a7 Mon Sep 17 00:00:00 2001 From: Cenz Wong <44856918+cenzwong@users.noreply.github.com> Date: Fri, 20 Sep 2024 11:44:27 +0100 Subject: [PATCH 2/8] Update README.md --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 92e6ae1..069374b 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,6 @@ pytest ``` # TODO -- Change the text in docstring - Change pytest test case - Change the visibility to Public - Build mkdocs From 736e1b9476bd1178236c11362dddf937d21844ae Mon Sep 17 00:00:00 2001 From: Cenz Wong <44856918+cenzwong@users.noreply.github.com> Date: Fri, 20 Sep 2024 11:48:10 +0100 Subject: [PATCH 3/8] minor fix --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index b19c179..6e0e222 100644 --- a/main.py +++ b/main.py @@ -19,6 +19,6 @@ # pyspark.sql.SparkSession.convert_dict_to_dataframe = se.convert_dict_to_dataframe spark.convert_dict_to_dataframe(data_dict, column_names, explode=True).show() -dict_.items() +data_dict.items() my_list = [1, 2, 3, 4] spark.createDataFrame(my_list).show() From 9a7e668260bd38c2f9843049db8a64a35bec61b5 Mon Sep 17 00:00:00 2001 From: Cenz Wong <44856918+cenzwong@users.noreply.github.com> Date: Fri, 20 Sep 2024 11:51:28 +0100 Subject: [PATCH 4/8] adding autosave in vscode --- .vscode/settings.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.vscode/settings.json b/.vscode/settings.json index 9b38853..1c552d3 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,4 +1,11 @@ { + "files.autoSave": "onFocusChange", + "editor.defaultFormatter": "ms-python.black-formatter", + "editor.formatOnPaste": true, + "editor.formatOnSave": true, + "notebook.defaultFormatter": "ms-python.black-formatter", + "notebook.formatOnCellExecution": true, + "notebook.formatOnSave.enabled": true, "python.testing.pytestArgs": [ "tests" ], From 4ace4b75a409291e972d4b7122df46120f280a15 Mon Sep 17 00:00:00 2001 From: Cenz Wong <44856918+cenzwong@users.noreply.github.com> Date: Fri, 20 Sep 2024 13:11:56 +0100 Subject: [PATCH 5/8] update README.md --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 069374b..14a524c 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,11 @@ # pyspark-helper python package that assist you writing pyspark +This repository is designed to replicate the structure of PySpark, making it highly accessible for users. +- The `functions` folder contains all PySpark functions, where both the input and output are Columns. +- The `Spark_ext` houses functions that necessitate a Spark instance, such as creating a DataFrame. +- The `transformation_ext` includes functions associated with DataFrame transformations, where both the input and output are DataFrames # Lint From dd74a490b08d72462fc7f358c4da98b7289dd897 Mon Sep 17 00:00:00 2001 From: Cenz Wong <44856918+cenzwong@users.noreply.github.com> Date: Fri, 20 Sep 2024 13:27:02 +0100 Subject: [PATCH 6/8] update --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 14a524c..17c22de 100644 --- a/README.md +++ b/README.md @@ -23,3 +23,7 @@ pytest - Change the visibility to Public - Build mkdocs - Build wheels for PyPi + +# Reference: +> https://github.com/apache/spark/tree/master/python/pyspark +> https://github.com/mrpowers-io/quinn/tree/planning-1.0-release From d5b333e6a0cebe8d53c9202c9485a944bbbb0e36 Mon Sep 17 00:00:00 2001 From: Cenz Wong <44856918+cenzwong@users.noreply.github.com> Date: Fri, 20 Sep 2024 13:27:45 +0100 Subject: [PATCH 7/8] update README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 17c22de..65730d5 100644 --- a/README.md +++ b/README.md @@ -26,4 +26,4 @@ pytest # Reference: > https://github.com/apache/spark/tree/master/python/pyspark -> https://github.com/mrpowers-io/quinn/tree/planning-1.0-release +> https://github.com/mrpowers-io/quinn From d8192d4ad645a49fb24a4201c52e661707097eb7 Mon Sep 17 00:00:00 2001 From: Cenz Wong <44856918+cenzwong@users.noreply.github.com> Date: Fri, 20 Sep 2024 15:43:10 +0100 Subject: [PATCH 8/8] Create LICENSE --- LICENSE | 121 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..0e259d4 --- /dev/null +++ b/LICENSE @@ -0,0 +1,121 @@ +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work.