Adding pre commit hooks

.github/PULL_REQUEST_TEMPLATE.md

@@ -6,7 +6,7 @@ _Summary of changes in this PR or what it accomplishes._
 
 <!--
 
-Please title your PR as follows: `feature: fix foo bar`. 
+Please title your PR as follows: `feature: fix foo bar`.
 Always start with the thing you are fixing, then describe the fix.
 Don't use past tense (e.g. "fixed foo bar").
 

.pre-commit-config.yaml

@@ -1,13 +1,32 @@
 # See https://pre-commit.com for more information
 # See https://pre-commit.com/hooks.html for more hooks
 repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.3.0
+    hooks:
+      # - id: check-yaml
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+      - id: mixed-line-ending
+      - id: check-added-large-files
+  - repo: https://github.com/pycqa/flake8
+    rev: 6.1.0
+    hooks:
+      - id: flake8
+        additional_dependencies: [flake8-black, flake8-bugbear, flake8-isort]
+  - repo: https://github.com/pycqa/isort
+    rev: 5.13.2
+    hooks:
+      - id: isort
+        name: isort (python)
+      - id: isort
+        name: isort (cython)
+        types: [cython]
+      - id: isort
+        name: isort (pyi)
+        types: [pyi]
   - repo: https://github.com/ambv/black
     rev: 23.1.0
     hooks:
       - id: black
         language_version: python3
-  - repo: https://github.com/pycqa/flake8
-    rev: 6.0.0
-    hooks:
-      - id: flake8
-        additional_dependencies: [flake8-black, flake8-bugbear]

.python-version

@@ -1 +1 @@
-3.10.10
+3.10.13

Pipfile

@@ -17,6 +17,7 @@ flake8-bugbear = "*"
 black = "*"
 localstack = "*"
 awscli-local = "*"
+pre-commit = "*"
 
 [requires]
 python_version = "3.10"

jobs/etl/__init__.py

@@ -1,4 +1,4 @@
 from .extract import extract
-from .load_postgresql import load_to_postgresql_db
 from .load_documentdb import load_to_document_db
+from .load_postgresql import load_to_postgresql_db
 from .load_s3 import load_to_s3

jobs/etl/extract.py

@@ -1,9 +1,10 @@
 from typing import List
+
 from awsglue.context import GlueContext
 
 from jobs.io import read_from_options
-from libs.config import Config
 from libs.aws import AwsS3Client
+from libs.config import Config
 
 
 def extract(glueContext: GlueContext, config: Config):

jobs/etl/load_s3.py

@@ -1,4 +1,5 @@
-from awsglue.context import GlueContext, DynamicFrame
+from awsglue.context import DynamicFrame, GlueContext
+
 from jobs.io.writer import write_from_options
 from libs.config import Config
 

jobs/io/__init__.py

@@ -1,3 +1,3 @@
+from .helpers import get_connection_options
 from .reader import read_from_options
 from .writer import write_from_options
-from .helpers import get_connection_options

jobs/io/helpers.py

@@ -1,5 +1,6 @@
 from typing import Dict, Tuple, Union
 from urllib.parse import quote
+
 from libs.common import load_tls_ca_bundle
 
 

jobs/io/reader.py

@@ -1,4 +1,5 @@
-from awsglue.context import GlueContext, DynamicFrame
+from awsglue.context import DynamicFrame, GlueContext
+
 from .helpers import get_connection_options
 
 

jobs/io/writer.py

@@ -1,4 +1,5 @@
-from awsglue.context import GlueContext, DynamicFrame
+from awsglue.context import DynamicFrame, GlueContext
+
 from .helpers import get_connection_options
 
 

jobs/pyspark_hello_world.py

@@ -1,11 +1,13 @@
 import sys
 
-from awsglue.utils import getResolvedOptions
-from pyspark.context import SparkContext
 from awsglue.context import GlueContext
 from awsglue.job import Job
+from awsglue.utils import getResolvedOptions
+from pyspark.context import SparkContext
+
+from jobs.etl import extract, load_to_postgresql_db, load_to_s3
 
-from jobs.etl import extract, load_to_s3, load_to_postgresql_db, load_to_document_db
+# from jobs.etl import load_to_document_db # isort: skip_file
 from libs.config import get_config
 
 args = getResolvedOptions(sys.argv, ["JOB_NAME"])

libs/aws/s3.py

@@ -1,5 +1,7 @@
 from typing import Union
+
 import boto3
+
 from libs.config import get_config
 
 

libs/common/__init__.py

@@ -1,2 +1,2 @@
-from .util import cached_property
 from .perm import load_tls_ca_bundle
+from .util import cached_property

libs/config/__init__.py

@@ -1 +1 @@
-from .config import get_config, Config
+from .config import Config, get_config

libs/config/secrets.py

@@ -1,7 +1,9 @@
 from __future__ import annotations
-import boto3
+
 import base64
 import json
+
+import boto3
 from botocore.exceptions import ClientError
 
 secrets_resolver_instance = None

libs/db/docdb_mongo.py

@@ -1,5 +1,7 @@
-from libs.common import load_tls_ca_bundle
 from pymongo import MongoClient as PyMongoClient
+
+from libs.common import load_tls_ca_bundle
+
 from .mongo import MongoClient
 
 

libs/db/mongo.py

@@ -1,5 +1,7 @@
 from typing import Any, Dict, List, Tuple, Union
+
 from pymongo import MongoClient as PyMongoClient
+
 from libs.common.logconfig import LogConfig