diff --git a/.github/workflows/test-on-kube.yml b/.github/workflows/test-on-kube.yml index eaa350f2e..079f03f40 100644 --- a/.github/workflows/test-on-kube.yml +++ b/.github/workflows/test-on-kube.yml @@ -25,20 +25,19 @@ jobs: - name: Create k8s Kind Cluster uses: helm/kind-action@v1.4.0 with: - cluster_name: basegun-testing config: ./infra/kube/kind/kind-config.yml wait: 60s verbosity: 2 - - name: Set up Helm - uses: azure/setup-helm@v3 - with: - version: v3.11.2 + # - name: Set up Helm + # uses: azure/setup-helm@v3 + # with: + # version: v3.11.2 - name: Set up ingress controller run: | helm repo add traefik https://traefik.github.io/charts && helm repo update - helm install --namespace ingress-traefik --create-namespace traefik traefik/traefik --values ./infra/kube/kind/traefik-values.yml + helm install --namespace traefik --create-namespace traefik traefik/traefik --values ./infra/kube/kind/traefik-values.yml - name: Add hosts to /etc/hosts run: | @@ -50,9 +49,8 @@ jobs: run: | TAG=$(make get-current-tag) BUILD_TARGET=test docker-compose -f docker-compose-prod.yml build kind load docker-image \ - basegun-backend:$(make get-current-tag)-prod \ - basegun-frontend:$(make get-current-tag)-prod \ - --name basegun-testing + basegun-backend:$(make get-current-tag) \ + basegun-frontend:$(make get-current-tag) helm upgrade --install basegun ./infra/kube/helm/ \ --set ingress.hosts[0].host="$LOCAL_DOMAIN" \ --set ingress.hosts[0].paths[0].path="/" \ diff --git a/backend/src/main.py b/backend/src/main.py index 6c5ddfd30..7426efa9e 100644 --- a/backend/src/main.py +++ b/backend/src/main.py @@ -1,144 +1,18 @@ - -import os -import logging -import time -import json -from logging.handlers import TimedRotatingFileHandler -from datetime import datetime -from uuid import uuid4 -from typing import Union - -import boto3 -from botocore.client import ClientError -from fastapi import BackgroundTasks, Cookie, FastAPI, File, Form, HTTPException, Request, Response, UploadFile -from fastapi.responses import PlainTextResponse +from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware -from gelfformatter import GelfFormatter -from user_agents import parse -from src.model import load_model_inference, predict_image - - -CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) - - -def init_variable(key: str, value: str) -> str: - """Inits global variable for folder path - - Args: - key (str): variable key in environ - value (str): value to give if variable does not exist yet - - Returns: - str: final variable value - """ - if key in os.environ: - VAR = os.environ[key] - else: - VAR = value - print("WARNING: The variable "+key+" is not set. Using", VAR) - if os.path.isabs(VAR): - os.makedirs(VAR, exist_ok = True) - return VAR - - -def setup_logs(log_dir: str) -> logging.Logger: - """Setups environment for logs - - Args: - log_dir (str): folder for log storage - logging.Logger: logger object - """ - print(">>> Reload logs config") - # clear previous logs - for f in os.listdir(log_dir): - os.remove(os.path.join(log_dir, f)) - # configure new logs - formatter = GelfFormatter() - logger = logging.getLogger("Basegun") - # new log file at midnight - log_file = os.path.join(log_dir, "log.json") - handler = TimedRotatingFileHandler( - log_file, - when="midnight", - interval=1, - backupCount=7) - logger.setLevel(logging.INFO) - handler.setFormatter(formatter) - logger.addHandler(handler) - return logger - - -def get_device(user_agent) -> str: - """Explicitly gives the device of a user-agent object - - Args: - user_agent: info given by the user browser - - Returns: - str: mobile, pc, tablet or other - """ - if user_agent.is_mobile: - return "mobile" - elif user_agent.is_pc: - return "pc" - elif user_agent.is_tablet: - return "tablet" - else: - return "other" - - -def get_base_logs(user_agent, user_id: str) -> dict: - """Generates the common information for custom logs in basegun. - Each function can add some info specific to the current process, - then we insert these custom logs as extra - - Args: - user_agent: user agent object - user_id (str): UUID identifying a unique user - - Returns: - dict: the base custom information - """ - extras_logging = { - "bg_date": datetime.now().isoformat(), - "bg_user_id": user_id, - "bg_version": APP_VERSION, - "bg_model": MODEL_VERSION, - "bg_device": get_device(user_agent), - "bg_device_family": user_agent.device.family, - "bg_device_os": user_agent.os.family, - "bg_device_browser": user_agent.browser.family, - } - return extras_logging - - - -def upload_image(content: bytes, image_key: str): - """Uploads an image to s3 bucket - path uploaded-images/WORKSPACE/img_name - where WORKSPACE is dev, preprod or prod - - Args: - content (bytes): file content - image_key (str): path we want to have - """ - start = time.time() - object = s3.Object(S3_BUCKET_NAME, image_key) - object.put(Body=content) - extras_logging = { - "bg_date": datetime.now().isoformat(), - "bg_upload_time": time.time()-start, - "bg_image_url": image_key - } - logger.info("Upload successful", extra=extras_logging) +from .routes import router #################### # SETUP # #################### # FastAPI Setup -app = FastAPI() +app = FastAPI(docs_url="/api") +app.include_router( + router, + prefix="/api", +) origins = [ # allow requests from front-end "http://basegun.fr", "https://basegun.fr", @@ -156,170 +30,7 @@ def upload_image(content: bytes, image_key: str): allow_headers=["*"], ) -# Logs -PATH_LOGS = init_variable("PATH_LOGS", - os.path.abspath(os.path.join(CURRENT_DIR,"/tmp/logs"))) -logger = setup_logs(PATH_LOGS) - -# Load model -MODEL_PATH = os.path.join( - CURRENT_DIR, - "weights/model.pth") -model = None -if os.path.exists(MODEL_PATH): - model = load_model_inference(MODEL_PATH) -if not model: - raise RuntimeError("Model not found") - -# Object storage -S3_URL_ENDPOINT = init_variable("S3_URL_ENDPOINT", "https://s3.gra.io.cloud.ovh.net/") -S3_BUCKET_NAME = "basegun-s3" -S3_PREFIX = os.path.join("uploaded-images/", os.environ['WORKSPACE']) -s3 = boto3.resource("s3", endpoint_url=S3_URL_ENDPOINT) -""" TODO : check if connection successful -try: - s3.meta.client.head_bucket(Bucket=S3_BUCKET_NAME) -except ClientError: - logger.exception("Cannot find s3 bucket ! Are you sure your credentials are correct ?") -""" - -# Versions -if "versions.json" in os.listdir(os.path.dirname(CURRENT_DIR)): - with open("versions.json", "r") as f: - versions = json.load(f) - APP_VERSION = versions["app"] - MODEL_VERSION = versions["model"] -else: - logger.warn("File versions.json not found") - APP_VERSION = "-1" - MODEL_VERSION = "-1" - -#################### -# ROUTES # -#################### -@app.get("/", response_class=PlainTextResponse) +@app.get("/") def home(): - return "Basegun backend" - - -@app.get("/version", response_class=PlainTextResponse) -def version(): - return APP_VERSION - - -@app.get("/logs") -def logs(): - if "WORKSPACE" in os.environ and os.environ["WORKSPACE"] != "prod": - with open(os.path.join(PATH_LOGS, "log.json"), "r") as f: - lines = f.readlines() - res = [json.loads(l) for l in lines] - res.reverse() - return res - else: - return PlainTextResponse("Forbidden") - - -@app.post("/upload") -async def imageupload( - request: Request, - response: Response, - background_tasks: BackgroundTasks, - image: UploadFile = File(...), - date: float = Form(...), - geolocation: str = Form(...), - user_id: Union[str, None] = Cookie(None) ): - - # prepare content logs - user_agent = parse(request.headers.get("user-agent")) - extras_logging = get_base_logs(user_agent, user_id) - extras_logging["bg_geolocation"] = geolocation - extras_logging["bg_upload_time"] = round(time.time() - date, 2) - - try: - img_key = os.path.join(S3_PREFIX, - str(uuid4()) + os.path.splitext(image.filename)[1].lower()) - img_bytes = image.file.read() - - # upload image to OVH Cloud - background_tasks.add_task(upload_image, img_bytes, img_key) - extras_logging["bg_image_url"] = img_key - - # set user id - if not user_id: - user_id = uuid4() - response.set_cookie(key="user_id", value=user_id) - extras_logging["bg_user_id"] = user_id - - # send image to model for prediction - start = time.time() - label, confidence = predict_image(model, img_bytes) - extras_logging["bg_label"] = label - extras_logging["bg_confidence"] = confidence - extras_logging["bg_model_time"] = round(time.time()-start, 2) - if confidence < 46: - extras_logging["bg_confidence_level"] = "low" - elif confidence < 76: - extras_logging["bg_confidence_level"] = "medium" - else: - extras_logging["bg_confidence_level"] = "high" - - logger.info("Identification request", extra=extras_logging) - - return { - "path": img_key, - "label": label, - "confidence": confidence, - "confidence_level": extras_logging["bg_confidence_level"] - } - - except Exception as e: - extras_logging["bg_error_type"] = e.__class__.__name__ - logger.exception(e, extra=extras_logging) - raise HTTPException(status_code=500, detail=str(e)) - - -@app.post("/identification-feedback") -async def log_feedback(request: Request, user_id: Union[str, None] = Cookie(None)): - res = await request.json() - - user_agent = parse(request.headers.get("user-agent")) - extras_logging = get_base_logs(user_agent, user_id) - - extras_logging["bg_feedback_bool"] = res["feedback"] - for key in ["image_url", "label", "confidence", "confidence_level"]: - extras_logging["bg_"+key] = res[key] - - logger.info("Identification feedback", extra=extras_logging) - return - - -@app.post("/tutorial-feedback") -async def log_tutorial_feedback(request: Request, user_id: Union[str, None] = Cookie(None)): - res = await request.json() - - user_agent = parse(request.headers.get("user-agent")) - extras_logging = get_base_logs(user_agent, user_id) - - for key in ["image_url", "label", "confidence", "confidence_level", - "tutorial_feedback", "tutorial_option", "route_name"]: - extras_logging["bg_"+key] = res[key] - - logger.info("Tutorial feedback", extra=extras_logging) - return - - -@app.post("/identification-dummy") -async def log_identification_dummy(request: Request, user_id: Union[str, None] = Cookie(None)): - res = await request.json() - - user_agent = parse(request.headers.get("user-agent")) - extras_logging = get_base_logs(user_agent, user_id) - - # to know if the firearm is dummy or real - extras_logging["bg_dummy_bool"] = res["is_dummy"] - for key in ["image_url", "label", "confidence", "confidence_level", "tutorial_option"]: - extras_logging["bg_"+key] = res[key] - - logger.info("Identification dummy", extra=extras_logging) - return \ No newline at end of file + return "Basegun backend" \ No newline at end of file diff --git a/backend/src/routes.py b/backend/src/routes.py new file mode 100644 index 000000000..ec8cbad15 --- /dev/null +++ b/backend/src/routes.py @@ -0,0 +1,298 @@ + +import os +import logging +import time +import json +from logging.handlers import TimedRotatingFileHandler +from datetime import datetime +from uuid import uuid4 +from typing import Union + +import boto3 +from botocore.client import ClientError +from fastapi import BackgroundTasks, Cookie, FastAPI, File, Form, HTTPException, Request, Response, UploadFile, APIRouter +from fastapi.responses import PlainTextResponse +from fastapi.middleware.cors import CORSMiddleware +from gelfformatter import GelfFormatter +from user_agents import parse +from src.model import load_model_inference, predict_image + + +CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) + + +def init_variable(key: str, value: str) -> str: + """Inits global variable for folder path + + Args: + key (str): variable key in environ + value (str): value to give if variable does not exist yet + + Returns: + str: final variable value + """ + if key in os.environ: + VAR = os.environ[key] + else: + VAR = value + print("WARNING: The variable "+key+" is not set. Using", VAR) + if os.path.isabs(VAR): + os.makedirs(VAR, exist_ok = True) + return VAR + + +def setup_logs(log_dir: str) -> logging.Logger: + """Setups environment for logs + + Args: + log_dir (str): folder for log storage + logging.Logger: logger object + """ + print(">>> Reload logs config") + # clear previous logs + for f in os.listdir(log_dir): + os.remove(os.path.join(log_dir, f)) + # configure new logs + formatter = GelfFormatter() + logger = logging.getLogger("Basegun") + # new log file at midnight + log_file = os.path.join(log_dir, "log.json") + handler = TimedRotatingFileHandler( + log_file, + when="midnight", + interval=1, + backupCount=7) + logger.setLevel(logging.INFO) + handler.setFormatter(formatter) + logger.addHandler(handler) + return logger + + +def get_device(user_agent) -> str: + """Explicitly gives the device of a user-agent object + + Args: + user_agent: info given by the user browser + + Returns: + str: mobile, pc, tablet or other + """ + if user_agent.is_mobile: + return "mobile" + elif user_agent.is_pc: + return "pc" + elif user_agent.is_tablet: + return "tablet" + else: + return "other" + + +def get_base_logs(user_agent, user_id: str) -> dict: + """Generates the common information for custom logs in basegun. + Each function can add some info specific to the current process, + then we insert these custom logs as extra + + Args: + user_agent: user agent object + user_id (str): UUID identifying a unique user + + Returns: + dict: the base custom information + """ + extras_logging = { + "bg_date": datetime.now().isoformat(), + "bg_user_id": user_id, + "bg_version": APP_VERSION, + "bg_model": MODEL_VERSION, + "bg_device": get_device(user_agent), + "bg_device_family": user_agent.device.family, + "bg_device_os": user_agent.os.family, + "bg_device_browser": user_agent.browser.family, + } + return extras_logging + + + +def upload_image(content: bytes, image_key: str): + """Uploads an image to s3 bucket + path uploaded-images/WORKSPACE/img_name + where WORKSPACE is dev, preprod or prod + + Args: + content (bytes): file content + image_key (str): path we want to have + """ + start = time.time() + object = s3.Object(S3_BUCKET_NAME, image_key) + object.put(Body=content) + extras_logging = { + "bg_date": datetime.now().isoformat(), + "bg_upload_time": time.time()-start, + "bg_image_url": image_key + } + logger.info("Upload successful", extra=extras_logging) + +# Logs +PATH_LOGS = init_variable("PATH_LOGS", + os.path.abspath(os.path.join(CURRENT_DIR,"/tmp/logs"))) +logger = setup_logs(PATH_LOGS) + +# Load model +MODEL_PATH = os.path.join( + CURRENT_DIR, + "weights/model.pth") +model = None +if os.path.exists(MODEL_PATH): + model = load_model_inference(MODEL_PATH) +if not model: + raise RuntimeError("Model not found") + +# Object storage +S3_URL_ENDPOINT = init_variable("S3_URL_ENDPOINT", "https://s3.gra.io.cloud.ovh.net/") +S3_BUCKET_NAME = "basegun-s3" +S3_PREFIX = os.path.join("uploaded-images/", os.environ['WORKSPACE']) +s3 = boto3.resource("s3", endpoint_url=S3_URL_ENDPOINT) +""" TODO : check if connection successful +try: + s3.meta.client.head_bucket(Bucket=S3_BUCKET_NAME) +except ClientError: + logger.exception("Cannot find s3 bucket ! Are you sure your credentials are correct ?") +""" + +# Versions +if "versions.json" in os.listdir(os.path.dirname(CURRENT_DIR)): + with open("versions.json", "r") as f: + versions = json.load(f) + APP_VERSION = versions["app"] + MODEL_VERSION = versions["model"] +else: + logger.warn("File versions.json not found") + APP_VERSION = "-1" + MODEL_VERSION = "-1" + + +#################### +# ROUTES # +#################### +router = APIRouter() + +@router.get("/version", response_class=PlainTextResponse) +def version(): + return APP_VERSION + + +@router.get("/logs") +def logs(): + if "WORKSPACE" in os.environ and os.environ["WORKSPACE"] != "prod": + with open(os.path.join(PATH_LOGS, "log.json"), "r") as f: + lines = f.readlines() + res = [json.loads(l) for l in lines] + res.reverse() + return res + else: + return PlainTextResponse("Forbidden") + + +@router.post("/upload") +async def imageupload( + request: Request, + response: Response, + background_tasks: BackgroundTasks, + image: UploadFile = File(...), + date: float = Form(...), + geolocation: str = Form(...), + user_id: Union[str, None] = Cookie(None) ): + + # prepare content logs + user_agent = parse(request.headers.get("user-agent")) + extras_logging = get_base_logs(user_agent, user_id) + extras_logging["bg_geolocation"] = geolocation + extras_logging["bg_upload_time"] = round(time.time() - date, 2) + + try: + img_key = os.path.join(S3_PREFIX, + str(uuid4()) + os.path.splitext(image.filename)[1].lower()) + img_bytes = image.file.read() + + # upload image to OVH Cloud + background_tasks.add_task(upload_image, img_bytes, img_key) + extras_logging["bg_image_url"] = img_key + + # set user id + if not user_id: + user_id = uuid4() + response.set_cookie(key="user_id", value=user_id) + extras_logging["bg_user_id"] = user_id + + # send image to model for prediction + start = time.time() + label, confidence = predict_image(model, img_bytes) + extras_logging["bg_label"] = label + extras_logging["bg_confidence"] = confidence + extras_logging["bg_model_time"] = round(time.time()-start, 2) + if confidence < 46: + extras_logging["bg_confidence_level"] = "low" + elif confidence < 76: + extras_logging["bg_confidence_level"] = "medium" + else: + extras_logging["bg_confidence_level"] = "high" + + logger.info("Identification request", extra=extras_logging) + + return { + "path": img_key, + "label": label, + "confidence": confidence, + "confidence_level": extras_logging["bg_confidence_level"] + } + + except Exception as e: + extras_logging["bg_error_type"] = e.__class__.__name__ + logger.exception(e, extra=extras_logging) + raise HTTPException(status_code=500, detail=str(e)) + + +@router.post("/identification-feedback") +async def log_feedback(request: Request, user_id: Union[str, None] = Cookie(None)): + res = await request.json() + + user_agent = parse(request.headers.get("user-agent")) + extras_logging = get_base_logs(user_agent, user_id) + + extras_logging["bg_feedback_bool"] = res["feedback"] + for key in ["image_url", "label", "confidence", "confidence_level"]: + extras_logging["bg_"+key] = res[key] + + logger.info("Identification feedback", extra=extras_logging) + return + + +@router.post("/tutorial-feedback") +async def log_tutorial_feedback(request: Request, user_id: Union[str, None] = Cookie(None)): + res = await request.json() + + user_agent = parse(request.headers.get("user-agent")) + extras_logging = get_base_logs(user_agent, user_id) + + for key in ["image_url", "label", "confidence", "confidence_level", + "tutorial_feedback", "tutorial_option", "route_name"]: + extras_logging["bg_"+key] = res[key] + + logger.info("Tutorial feedback", extra=extras_logging) + return + + +@router.post("/identification-dummy") +async def log_identification_dummy(request: Request, user_id: Union[str, None] = Cookie(None)): + res = await request.json() + + user_agent = parse(request.headers.get("user-agent")) + extras_logging = get_base_logs(user_agent, user_id) + + # to know if the firearm is dummy or real + extras_logging["bg_dummy_bool"] = res["is_dummy"] + for key in ["image_url", "label", "confidence", "confidence_level", "tutorial_option"]: + extras_logging["bg_"+key] = res[key] + + logger.info("Identification dummy", extra=extras_logging) + return \ No newline at end of file diff --git a/docker-compose-prod.yml b/docker-compose-prod.yml index afd2f375d..f318531b2 100644 --- a/docker-compose-prod.yml +++ b/docker-compose-prod.yml @@ -14,7 +14,7 @@ services: - AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY - WORKSPACE=${WORKSPACE:-prod} - image: basegun-backend:${TAG}-prod + image: basegun-backend:${TAG} ports: - 5000:5000 @@ -23,6 +23,6 @@ services: context: ./frontend target: prod container_name: basegun-frontend - image: basegun-frontend:${TAG}-prod + image: basegun-frontend:${TAG} ports: - ${PORT_PROD:-80}:8080 \ No newline at end of file diff --git a/infra/kube/helm/templates/ingress.yaml b/infra/kube/helm/templates/ingress.yaml index b9fb0e97a..49689be99 100644 --- a/infra/kube/helm/templates/ingress.yaml +++ b/infra/kube/helm/templates/ingress.yaml @@ -1,4 +1,6 @@ {{- if .Values.ingress.enabled -}} +{{- $svcPortFrontend := .Values.frontend.service.port -}} +{{- $svcPortBackend := .Values.backend.service.port -}} apiVersion: networking.k8s.io/v1 kind: Ingress metadata: @@ -32,13 +34,13 @@ spec: service: name: basegun-frontend port: - number: 8080 + number: {{ $svcPortFrontend }} - path: /api/ pathType: Prefix backend: service: name: basegun-backend port: - number: 5000 + number: {{ $svcPortBackend }} {{- end }} {{- end }} \ No newline at end of file diff --git a/infra/kube/helm/templates/service.yaml b/infra/kube/helm/templates/service.yaml index 7bb28f7fb..0159f9c2d 100644 --- a/infra/kube/helm/templates/service.yaml +++ b/infra/kube/helm/templates/service.yaml @@ -9,8 +9,6 @@ spec: ports: - port: {{ .Values.frontend.service.port }} targetPort: {{ .Values.frontend.service.containerPort }} - protocol: TCP - name: http selector: {{- include "basegun.FrontSelectorLabels" . | nindent 4 }} --- @@ -25,7 +23,5 @@ spec: ports: - port: {{ .Values.backend.service.port }} targetPort: {{ .Values.backend.service.containerPort }} - protocol: TCP - name: http selector: {{- include "basegun.BackSelectorLabels" . | nindent 4 }} diff --git a/infra/kube/helm/values.yaml b/infra/kube/helm/values.yaml index c7d5a92d7..d94308661 100644 --- a/infra/kube/helm/values.yaml +++ b/infra/kube/helm/values.yaml @@ -59,7 +59,7 @@ backend: memory: 256Mi service: type: ClusterIP - port: 5000 + port: 80 containerPort: 5000 autoscaling: enabled: false @@ -111,7 +111,7 @@ frontend: service: type: ClusterIP - port: 8080 + port: 80 containerPort: 8080 resources: # We usually recommend not to specify default resources and to leave this as a conscious diff --git a/infra/scripts/run.sh b/infra/scripts/run.sh new file mode 100644 index 000000000..d34cf60bd --- /dev/null +++ b/infra/scripts/run.sh @@ -0,0 +1,31 @@ +# Launch a cluster like the one in CI + +TAG=$(make get-current-tag) BUILD_TARGET=test docker-compose -f docker-compose-prod.yml build +kind create cluster --config ./infra/kube/kind/kind-config.yml + +helm repo add traefik https://traefik.github.io/charts && helm repo update +helm upgrade \ + --install \ + --wait \ + --namespace traefik \ + --create-namespace \ + --values ./infra/kube/kind/traefik-values.yml \ + traefik traefik/traefik + +kind load docker-image \ + basegun-backend:$(make get-current-tag) \ + basegun-frontend:$(make get-current-tag) + +helm upgrade --install basegun ./infra/kube/helm/ \ + --set ingress.hosts[0].host=basegun.k8s.local \ + --set ingress.hosts[0].paths[0].path="/" \ + --set ingress.hosts[0].paths[0].pathType="Prefix" \ + --set backend.image.repository="basegun-backend" \ + --set backend.image.tag="$(make get-current-tag)" \ + --set frontend.image.repository="basegun-frontend" \ + --set frontend.image.tag="$(make get-current-tag)" \ + --set backend.secret.create="true" \ + --set-string backend.secret.values.AWS_ACCESS_KEY_ID="x" \ + --set-string backend.secret.values.AWS_SECRET_ACCESS_KEY="x" \ + --set-string backend.secret.values.X_OVH_TOKEN="x" \ + --set-string backend.secret.values.API_OVH_TOKEN="x"