Drobert fetch intra

.gitignore

@@ -1,3 +1,5 @@
+logs
+
 # PostgreSQL
 postgres_data
 .postgres_data

backend/portfolio42_api/admin.py

@@ -1,3 +1,8 @@
 from django.contrib import admin
+from .models import User, Project, Skill, Cursus
 
 # Register your models here.
+admin.site.register(User)
+admin.site.register(Project)
+admin.site.register(Skill)
+admin.site.register(Cursus)

backend/portfolio42_api/management/api42/__init__.py

@@ -0,0 +1,2 @@
+from .api42 import Api42
+from .api_error import ApiException

backend/portfolio42_api/management/api42/api42.py

@@ -0,0 +1,157 @@
+import os
+import requests
+from .api_error import ApiException
+import time
+from datetime import datetime, timedelta
+import logging
+
+class AuthApi42():
+    _TOKEN_URL = 'https://api.intra.42.fr/oauth/token'
+
+    def __init__(self,
+                 uid : str = os.environ.get('INTRA_UID'),
+                 secret : str = os.environ.get('INTRA_SECRET'),
+                 reqs_per_second : int = 2,
+                 wait_for_limit : bool = False):
+        self._uid = uid
+        self._secret = secret
+
+        self._token_expires = datetime(1,1,1)
+        self._access_token = None
+        self._reqs_per_second = reqs_per_second
+        self._await_limit = wait_for_limit
+        self._window = [] # Will store when a request expires (aka datetime.now() + 1 second)
+
+    @property
+    def uid(self):
+        return self._uid
+
+    @property
+    def secret(self):
+        return self._secret
+
+    @secret.setter
+    def secret(self, _secret : str):
+        self._secret = _secret
+
+    @property
+    def access_token(self):
+        return self._access_token
+
+
+    @property
+    def await_limit(self):
+        return self._await_limit
+
+    @await_limit.setter
+    def await_limit(self, flag : bool):
+        self._await_limit = flag
+
+    # This function should be called each time the user wants to make a request
+    @property
+    def token(self):
+        if (self._token_expires > datetime.now() and self._access_token is not None):
+            # removed timed-out requests from our window
+            self._window = [t for t in self._window if datetime.now() < t]
+
+            # If we hit the request limit wait until a slot in the window opens up
+            if (len(self._window) >= self._reqs_per_second):
+                if (self._await_limit):
+                    sleep_time = self._window[0] - datetime.now()
+                    time.sleep(sleep_time.total_seconds())
+                else:
+                    logging.error('Too Many Requests')
+                    raise ApiException('Too many requests')
+            return self._access_token
+
+        # If the token has expired or its not available try to refresh token
+        data = {
+            'grant_type': 'client_credentials',
+            'client_id': self._uid,
+            'client_secret': self._secret
+        }
+
+        # Fetch token
+        res = requests.post(AuthApi42._TOKEN_URL, data=data)
+        json = res.json()
+
+        # throw an exception if we encounter an error (non 200 response code)
+        if (res.status_code != 200):
+            error_str = f"Unknown Error ({res.status_code})"
+            try:
+                error_str = res.json()['error_description']
+            except:
+                pass
+            raise RuntimeError(error_str)
+
+        # Extract info from response
+        self._access_token = json['access_token']
+        self._token_expires = datetime.now() + timedelta(seconds=json['expires_in'])
+
+        logging.info(f"Fetched new token, expires at {self._token_expires}")
+
+        now = datetime.now()
+
+        # removed timed-out requests from our window
+        self._window = [t for t in self._window if now < t]
+
+        if (len(self._window) >= self._reqs_per_second):
+            if (self.await_limit):
+                sleep_time = self._window[0] - datetime.now()
+                time.sleep(sleep_time.total_seconds())
+            else:
+                raise ApiException('Too many requests')
+
+        return self._access_token
+
+    # Should be called after making a request,
+    # updates the window for more accurate rate limit checking
+    def report_request(self):
+        self._window.append(datetime.now() + timedelta(seconds=1))
+
+
+class Api42():
+    _API_URL = "https://api.intra.42.fr/v2"
+
+    """
+        Interface for interacting with the 42 api
+
+        uid - UID of application obtained from intra
+        secret - SECRET of application obtained from intra
+        req_limit - The amount of requests that can be made per second
+    """
+    def __init__(self, uid :str, secret : str, req_limit : int = 2):
+        self._auth = AuthApi42(uid, secret, reqs_per_second = req_limit, wait_for_limit= True)
+
+    def await_limit(self, should_wait : bool):
+        self._auth.await_limit(should_wait)
+
+    def get(self, endpoint : str, params : dict = { }):
+        # Make request
+        headers = {'Authorization': f"Bearer {self._auth.token}"}
+        res = requests.get(f"{Api42._API_URL}{endpoint}",
+                           headers=headers, params=params)
+        self._auth.report_request()
+
+        logging.info(f"Made request to 42 API at {endpoint} ({res.status_code})")
+
+        # Check the status code
+        if(res.status_code != 200):
+            error_reason = f"Error while fetching, status code: {res.status_code}"
+            try:
+                error_reason = res.json()['error']
+            except:
+                if (len(res.text) != 0):
+                    error_reason = res.text
+                pass
+            logging.error("error_reason")
+            raise ApiException(error_reason)
+
+        # Try parsing json
+        try:
+            return res.json()
+        except:
+            logging.error("Response was not in json format")
+            raise ApiException("Response was not in json format")
+
+

backend/portfolio42_api/management/api42/api_error.py

@@ -0,0 +1,6 @@
+class ApiException(Exception):
+    def __init__(self, message):
+        self.message = message
+
+    def __str__(self):
+        return f"Api Exception: {self.message}"

backend/portfolio42_api/management/commands/sync_db.py

@@ -0,0 +1,159 @@
+import os
+import sys
+import requests
+from pathlib import Path
+from datetime import datetime
+from django.core.management.base import BaseCommand, CommandError
+from portfolio42_api.models import Project, Skill, User, Cursus, ProjectCursus, CursusUser, CursusUserSkill, CursusSkill, ProjectUser
+from portfolio42_api.management.api42 import Api42, ApiException
+import logging
+
+def parser_add_db_command(cmd):
+    cmd.add_argument('intra_ids',
+                nargs='*',
+                action='store',
+                help='Which intra id(s) to update',
+                metavar='intra id',
+                type=int)
+
+def update_basic(api: Api42, endpoint : str, func : callable, ids : [] = []):
+
+    # Api settings
+    per_page = 100
+    page = 0 # This will increase in the while loop
+    projects_returned = per_page
+
+    params = {'per_page': per_page, 'page': page}
+    if (len(ids) > 0):
+        if (len(ids) > 100):
+            raise ValueError()
+        params['filter[id]'] = ','.join(map(str, ids))
+
+
+    while (projects_returned == per_page):
+        json = {}
+
+        try:
+            json = api.get(endpoint, params)
+            logging.info(f"Obtained ({len(json)}) objects from api request")
+        except ApiException as e:
+            logging.error(f"Error on 42 API request")
+            break
+
+        for obj in json:
+            func(obj)
+
+        page += 1
+        params['page'] = page
+        projects_returned = len(json)
+
+        # If we are looking for specific ids we do not need to make another request
+        if (len(ids) > 0):
+            break
+
+# endpoint should be formatted with `:id` which will be replaced with the actual id
+def update_from_db(api : Api42, table, endpoint : str, func : callable, is_basic : bool = False, ids : [] = []):
+    all = []
+    if (len(ids) == 0):
+        all = table.objects.all()
+    else:
+        all = table.objects.filter(intra_id__in=ids)
+    endpoint_start = endpoint[:endpoint.index(':id')]
+    endpoint_end = endpoint[endpoint.index(':id') + 3:]
+
+    for o in all:
+        ep = f"{endpoint_start}{o.intra_id}{endpoint_end}"
+        json = api.get(ep)
+        if (is_basic):
+            func(json)
+        else:
+            for e in json:
+                func(o, e)
+
+# Updates cursususer and cursususerskill
+def update_cursususer_skill(user, cursususer):
+    cu = CursusUser.update(user, cursususer)
+    if (cu == None):
+        return
+    for i in cursususer['skills']:
+        CursusUserSkill.update(cu, i)
+
+def update_relations(api : Api42):
+    update_from_db(api, Cursus, '/cursus/:id/projects', ProjectCursus.update)
+    update_from_db(api, Cursus, '/cursus/:id/skills', CursusSkill.update)
+    update_from_db(api, User, '/users/:id/cursus_users', update_cursususer_skill)
+    update_from_db(api, User, '/users/:id/projects_users', ProjectUser.update)
+
+def update_all(api : Api42):
+    update_basic(api, '/projects', Project.update)
+    update_basic(api, '/skills', Skill.update)
+    update_basic(api, '/cursus', Cursus.update)
+    update_from_db(api, User, '/users/:id', User.update, True)
+    update_relations(api)
+
+class Command(BaseCommand):
+    help = "Sync the database with the intra api"
+
+    def add_arguments(self, parser):
+        subparsers = parser.add_subparsers(dest='command', required=True, metavar='sub-command')
+
+        parser_add_db_command(subparsers.add_parser('user', help="Update user table"))
+        parser_add_db_command(subparsers.add_parser('cursus', help="Update cursus table"))
+        parser_add_db_command(subparsers.add_parser('project', help="Update project table"))
+        parser_add_db_command(subparsers.add_parser('skill', help="Update skill table"))
+
+        parser_add_db_command(subparsers.add_parser('relations', help="Update relations in the database"))
+        subparsers.add_parser('all', help="Runs user, cursus, project, skill & relations")
+
+        parser.add_argument('--no-logfile',
+                            action='store_true',
+                            dest="no_logfile",
+                            help='If it should not create a logfile',
+                            default=False)
+        parser.add_argument('--log-dir',
+                            action='store',
+                            dest='log_dir',
+                            help='Where to store the logs',
+                            default=Path("./logs"),
+                            type=Path)
+
+    def handle(self, *args, **options):
+        command = options['command']
+
+        # setup logging
+        log_level = options['verbosity'] * 10
+        log_format = "[%(asctime)s][%(levelname)s] %(message)s"
+        log_time_format = "%y%m%d%H%M%S"
+        log_handlers = []
+        if (log_level > 0):
+            if (not options['no_logfile']):
+                log_dir : Path = options['log_dir']
+                log_dir.mkdir(parents=True, exist_ok=True)
+                logfile_name = f"{datetime.now().strftime('%y%m%d%H%M%S')}_{command}.log"
+                handler = logging.FileHandler(f"{log_dir.absolute()}/{logfile_name}")
+                log_handlers.append(handler)
+
+        sh = logging.StreamHandler(sys.stdout)
+        log_handlers.append(sh)
+
+        logging.basicConfig(level=log_level,
+                            format=log_format,
+                            datefmt=log_time_format,
+                            handlers=log_handlers)
+
+        api = Api42(os.environ.get('INTRA_UID'), os.environ.get('INTRA_SECRET'))
+
+        match command:
+            case 'project':
+                update_basic(api, '/projects', Project.update, options['intra_ids'])
+            case 'skill':
+                update_basic(api, '/skills', Skill.update, options['intra_ids'])
+            case 'cursus':
+                update_basic(api, '/cursus', Cursus.update, options['intra_ids'])
+            case 'user':
+                update_from_db(api, User, '/users/:id', User.update, True, options['intra_ids'])
+            case 'relations':
+                update_relations(api)
+            case 'all':
+                update_all(api)
+