Merge pull request #426 from tukcomCD2024/develop_back

feat : 백엔드 코어 서버 개발 master 합병

.github/workflows/animation-application-ci-cd-flow.yml

@@ -5,6 +5,9 @@ on:
     branches:
       - develop_back_animation
       - 'hotfix/[0-9a-zA-z]+-B-animation-#[0-9a-zA-z]+'
+  pull_request:
+    branches:
+      - develop_back_animation
 
 jobs:
   deploy-animation:
@@ -80,8 +83,8 @@ jobs:
             docker ps -q --filter "name=animated_drawings" | xargs -r docker stop
             docker ps -aq --filter "name=animated_drawings" | xargs -r docker rm
             aws ecr get-login-password --region ${{ secrets.AWS_REGION }} | docker login --username ${{ secrets.AWS_DOCKER_USER }} --password-stdin ${{ secrets.AWS_USER_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com
-            docker pull ${{ steps.meta.outputs.tags }}
             docker image prune -f
+            docker pull ${{ steps.meta.outputs.tags }}
             docker run -d -p 5555:5555 -p 9001:9001 --name animated_drawings -e ENVIRONMENT='dev' ${{ steps.meta.outputs.tags }}
 
       - name: Remove Github Actions IP from security group

.github/workflows/core-application-ci-cd-flow.yml

@@ -1,34 +1,51 @@
-name: ci/cd
+name: core-ci/cd
 
 on:
   push:
     branches:
-      - develop_back
-      - 'hotfix/[0-9a-zA-z]+-B-#[0-9a-zA-z]+'
+      - develop_back_core
+      - 'hotfix/[0-9a-zA-z]+-B-core-#[0-9a-zA-z]+'
   pull_request:
     branches:
-      - develop_back
+      - develop_back_core
 
 jobs:
-  formatting:
-    name: formatting
-    if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' }}
+  build_and_test:
+    name: build and test
+    if: ${{github.event_name == 'pull_request' }}
     runs-on: ubuntu-latest
+
+    defaults:
+      run:
+        working-directory: ./backend/core
+
     steps:
       - name: set up JDK 17
         uses: actions/setup-java@v3
         with:
           java-version: '17'
-          distribution: 'corretto'
-      - uses: actions/checkout@v3
-      - uses: axel-op/googlejavaformat-action@v3
+          distribution: 'liberica'
+
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+          token: ${{ secrets.CI_PAT }}
+
+      - name: formatting
+        uses: axel-op/googlejavaformat-action@v3
         with:
           args: "--replace"
           skip-commit: true
 
+      - name: grant execute permission for gradlew
+        run: chmod +x ./gradlew
+
+      - name: test
+        run: ./gradlew clean test
+
   deploy-core:
     name: core application deploy
-    needs: formatting
     if: ${{ github.event_name == 'push' }}
     runs-on: ubuntu-latest
 
@@ -115,8 +132,9 @@ jobs:
             docker ps -q --filter "name=core" | xargs -r docker stop
             docker ps -aq --filter "name=core" | xargs -r docker rm
             aws ecr get-login-password --region ${{ secrets.AWS_REGION }} | docker login --username ${{ secrets.AWS_DOCKER_USER }} --password-stdin ${{ secrets.AWS_USER_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com
+            docker image prune -f
             docker pull ${{ steps.meta.outputs.tags }}
-            docker run -d -p 8080:8080 --name core --network test_backend ${{ steps.meta.outputs.tags }}
+            docker run -d -p 8080:8080 -e ENVIRONMENT=dev --name core --network test_backend ${{ steps.meta.outputs.tags }}
 
       - name: Remove Github Actions IP from security group
         if: always()

.github/workflows/notification-application-ci-cd-flow.yml

@@ -1,12 +1,49 @@
-name: ci/cd
+name: notification-ci/cd
 
 on:
   push:
     branches:
       - develop_back_notification
       - 'hotfix/[0-9a-zA-z]+-B-notification-#[0-9a-zA-z]+'
+  pull_request:
+    branches:
+      - develop_back_notification
 
 jobs:
+  build_and_test:
+    name: build and test
+    if: ${{github.event_name == 'pull_request' }}
+    runs-on: ubuntu-latest
+
+    defaults:
+      run:
+        working-directory: ./backend/core
+
+    steps:
+      - name: set up JDK 17
+        uses: actions/setup-java@v3
+        with:
+          java-version: '17'
+          distribution: 'liberica'
+
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+          token: ${{ secrets.CI_PAT }}
+
+      - name: formatting
+        uses: axel-op/googlejavaformat-action@v3
+        with:
+          args: "--replace"
+          skip-commit: true
+
+      - name: grant execute permission for gradlew
+        run: chmod +x ./gradlew
+
+      - name: test
+        run: ./gradlew clean test
+
   deploy-notification:
     name: notification application deploy
     if: ${{ github.event_name == 'push' }}
@@ -80,28 +117,29 @@ jobs:
 
       - name: Add Github Actions IP to Security group
         run: |
-          aws ec2 authorize-security-group-ingress --group-id ${{ secrets.AWS_EC2_NOTIFICATION_SG_ID }} --group-name ${{secrets.AWS_EC2_NOTIFICATION_SG_NAME}} --protocol tcp --port 22 --cidr ${{ steps.ip.outputs.ipv4 }}/32
+          aws ec2 authorize-security-group-ingress --group-id ${{ secrets.AWS_EC2_CORE_SG_ID }} --group-name ${{secrets.AWS_EC2_CORE_SG_NAME}} --protocol tcp --port 22 --cidr ${{ steps.ip.outputs.ipv4 }}/32
 
       - name: Connect ec2 and Run Docker Container
         uses: appleboy/[email protected]
         env:
           AWS_ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
         with:
-          host: ${{ secrets.SSH_NOTIFICATION_HOST }}
+          host: ${{ secrets.SSH_CORE_HOST }}
           username: ${{ secrets.SSH_USERNAME }}
           key: ${{ secrets.SSH_CORE_PRIVATE_KEY }}
           port: ${{ secrets.SSH_PORT }}
           script: |
             docker ps -q --filter "name=notification" | xargs -r docker stop
             docker ps -aq --filter "name=notification" | xargs -r docker rm
             aws ecr get-login-password --region ${{ secrets.AWS_REGION }} | docker login --username ${{ secrets.AWS_DOCKER_USER }} --password-stdin ${{ secrets.AWS_USER_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com
+            docker image prune -f
             docker pull ${{ steps.meta.outputs.tags }}
-            docker run -d -p 8080:8080 --name notification --network test_backend ${{ steps.meta.outputs.tags }}
+            docker run -d -p 8081:8081 -e ENVIRONMENT=dev --name notification --network test_backend ${{ steps.meta.outputs.tags }}
 
       - name: Remove Github Actions IP from security group
         if: always()
         run: |
-          aws ec2 revoke-security-group-ingress --group-id ${{ secrets.AWS_EC2_NOTIFICATION_SG_ID }} --group-name ${{secrets.AWS_EC2_NOTIFICATION_SG_NAME}} --protocol tcp --port 22 --cidr ${{ steps.ip.outputs.ipv4 }}/32
+          aws ec2 revoke-security-group-ingress --group-id ${{ secrets.AWS_EC2_CORE_SG_ID }} --group-name ${{secrets.AWS_EC2_CORE_SG_NAME}} --protocol tcp --port 22 --cidr ${{ steps.ip.outputs.ipv4 }}/32
 
       - uses: sarisia/actions-status-discord@v1
         if: success()

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "backend/notification/src/main/resources/config"]
 	path = backend/notification/src/main/resources/config
 	url = https://github.com/seokho-1116/DroidBlossom-config-notification.git
+[submodule "backend/AnimatedDrawings/application/config/yml"]
+	path = backend/AnimatedDrawings/application/config/yml
+	url = https://github.com/seokho-1116/DroidBlosson-config-animation.git

backend/AnimatedDrawings/application/__init__.py

@@ -0,0 +1,2 @@
+import celery_app
+__all__ = ('celery_app',)

backend/AnimatedDrawings/application/animation_queue.py

@@ -1,122 +1,117 @@
-import json
-import logging
 import uuid
-from _xxsubinterpreters import ChannelClosedError
-from json.decoder import JSONDecodeError
+from ast import literal_eval
 
-import pika
 from celery import chain
-from pika.adapters.blocking_connection import BlockingChannel
-from pika.spec import Basic, BasicProperties
+from kombu import Exchange, Queue
 
 from application.config.queue_config import QueueConfig
+from application.logging.logger_factory import LoggerFactory
 from application.model.motion import Motion
 from application.model.retarget import Retarget
-from application.task.make_animation import MakeAnimation
-from application.task.save_capsule_skin import SaveCapsuleSkin
-from application.task.send_notification import SendNotification
-
-logger = logging.getLogger('animation_queue_controller')
+from application.task.tasks import create_animation, save_capsule_skin, \
+    send_notification
+from kombu_connection_pool import connection, connections
 
 
 class AnimationQueueController:
     def __init__(self):
         self.queue_config = QueueConfig()
         self.require_keys = ['memberId', 'memberName', 'skinName', 'imageUrl',
                              'retarget', 'motionName']
-        self.celery_work_queue_name = 'makeAnimation.queue'
-        self.celery_success_queue_name = 'saveCapsuleSkin.queue'
-        self.celery_send_notification_queue_name = 'sendNotification.queue'
-        self.make_animation_task = MakeAnimation()
-        self.save_capsule_skin_task = SaveCapsuleSkin()
-        self.send_notification_task = SendNotification()
+        self.celery_work_queue_name = 'task.makeAnimation.queue'
+        self.celery_success_queue_name = 'task.saveCapsuleSkin.queue'
+        self.celery_send_notification_queue_name = 'task.sendNotification.queue'
+        self.logger = LoggerFactory.get_logger(__name__)
 
     def run(self):
-        # rabbitmq 채널 연결
-        connection = pika.BlockingConnection(
-            pika.ConnectionParameters(host=self.queue_config.queue_host))
-        channel = connection.channel()
-
-        channel.queue_declare(queue=self.queue_config.queue_name, durable=True)
-
-        channel.basic_consume(queue=self.queue_config.queue_name,
-                              on_message_callback=self.callback,
-                              auto_ack=False)
-
-        try:
-            channel.start_consuming()
-        except ChannelClosedError as e:
-            logger.info("커넥션 연결 오류")
-            raise e
-        finally:
-            channel.close()
+        """
+        큐 메시지 리스닝
+        :return:
+        """
+        capsule_skin_exchange = Exchange(
+            name=QueueConfig.CAPSULE_SKIN_REQUEST_EXCHANGE_NAME,
+            type='direct',
+            durable=True)
+        capsule_skin_queue = Queue(
+            name=QueueConfig.CAPSULE_SKIN_REQUEST_QUEUE_NAME,
+            exchange=capsule_skin_exchange,
+            routing_key=QueueConfig.CAPSULE_SKIN_REQUEST_QUEUE_NAME)
+
+        with connections[connection].acquire(block=True) as conn:
+            with conn.Consumer(queues=[capsule_skin_queue],
+                               callbacks=[self.callback],
+                               accept=['json']):
+                self.logger.info('메시지 수신 시작')
+                while True:
+                    conn.drain_events()
 
     def callback(
         self,
-        channel: BlockingChannel,
-        method: Basic.Deliver,
-        header: BasicProperties,
-        body: bytes,
+        body: [dict, str, bytes],
+        message: any
     ) -> None:
         """
-        queue에서 message consume 시 동작하는 callback
-        celery worker한테 animation 생성 작업을 실행한다
-        :param channel: 큐와 연결된 채널
-        :param method: 메시지의 상태
-        :param header: 기본 정보
-        :param body: queue로부터 넘어온 데이터
+        큐에 메시지가 도착했을 때 동작하는 콜백 함수
+        :param body:
+        :param message:
+        :return:
         """
-        logger.info('큐 메시지 처리 시작 %s', header.message_id)
         try:
-            json_object = self.parse_json(body)
+            self.logger.debug('메시지 수신 완료, 콜백 동작')
+            parsed_data = self.parse_body(body)
 
-            filename = f"capsuleSkin/{json_object['memberId']}/{uuid.uuid4()}.gif"
+            filename = f"capsuleSkin/{parsed_data['memberId']}/{uuid.uuid4()}.gif"
 
             chain(
-                self.make_animation_task.s(input_data=json_object,
-                                           filename=filename)
+                create_animation.s(input_data=parsed_data,
+                                   filename=filename)
                 .set(queue=self.celery_work_queue_name),
 
-                self.save_capsule_skin_task.s(input_data=json_object,
-                                              filename=filename)
+                save_capsule_skin.s(input_data=parsed_data,
+                                    filename=filename)
                 .set(queue=self.celery_success_queue_name),
 
-                self.send_notification_task.s(input_data=json_object,
-                                              filename=filename)
+                send_notification.s(input_data=parsed_data,
+                                    filename=filename)
                 .set(queue=self.celery_send_notification_queue_name)
             ).apply_async(
                 ignore_result=True
             )
 
-            channel.basic_ack(delivery_tag=method.delivery_tag)
+            message.ack()
+            self.logger.debug('celery에 작업 전달 완료')
         except Exception as e:
-            logger.exception('메시지 처리 오류', e)
-            channel.basic_reject(delivery_tag=method.delivery_tag,
-                                 requeue=False)
+            self.logger.exception('작업 큐 메시지 처리 오류 %r', e)
+            message.reject()
 
-    def parse_json(self, body: bytes):
+    def parse_body(self, body: [dict, str, bytes]) -> dict:
         """
-        json bytes를 파싱해 dict로 반환하는 함수
-        :param body: queue로부터 넘어온 json bytes
-        :return: body에서 파싱된 dict
-
-        :raises JSONDecodeError: 유효하지 않은 json 형태인 경우
-        :raise TypeError: 잘못된 json 입력 타입인 경우
+        큐로부터 온 메시지 파싱 함수
+        :param body: 큐에서 온 메시지
+        :return: 파싱된 메시지 dict
         """
         try:
-            json_object = json.loads(body.decode(encoding='utf8'))
-            json_object['memberId'] = str(json_object['memberId'])
-            json_object['retarget'] = Retarget[json_object['retarget']].value
-            json_object['motionName'] = Motion[json_object['motionName']].value
+            if isinstance(body, str):
+                dict_data = literal_eval(body)
+            elif isinstance(body, dict):
+                dict_data = body
+            elif isinstance(body, bytes):
+                dict_data = literal_eval(body.decode('utf-8'))
+            else:
+                self.logger.error('처리할 수 없는 타입 오류')
+                raise TypeError('처리할 수 없는 타입입니다')
+
+            dict_data['retarget'] = Retarget[dict_data['retarget']].value
+            dict_data['motionName'] = Motion[dict_data['motionName']].value
 
             for key in self.require_keys:
-                if key not in json_object:
+                if key not in dict_data:
                     raise KeyError
 
-            return json_object
+            return dict_data
 
-        except (JSONDecodeError, KeyError, TypeError) as e:
-            logger.exception('json 파싱 오류', e)
+        except (KeyError, TypeError) as e:
+            self.logger.exception('작업 큐 메시지 파싱 오류 %r', e)
             raise e