From 3c912172133a085542979758fccbdf8cf76e92ee Mon Sep 17 00:00:00 2001 From: selfisekai Date: Sat, 7 Dec 2019 22:50:55 +0100 Subject: [PATCH 01/19] Locust load testing file --- locustfile.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 locustfile.py diff --git a/locustfile.py b/locustfile.py new file mode 100644 index 00000000..b6cfc3ea --- /dev/null +++ b/locustfile.py @@ -0,0 +1,36 @@ +from locust import HttpLocust, TaskSet, task, between +from random import randint + +USERNAME = "rand" +PASSWORD = "pass" +SERVER_SECRET_KEY = "123456789" + +class UserBehavior(TaskSet): + headers = { + 'Host': 'localhost:5000', + 'Accept': 'application/json, text/plain, */*', + 'Accept-Language': 'en-US,en;q=0.5', + 'Accept-Encoding': 'gzip, deflate', + 'Content-Type': 'application/x-www-form-urlencoded', + 'Origin': 'http://localhost:3000', + } + + def on_start(self): + res = self.client.post("/api/login", "user_name={}&password={}".format(USERNAME, PASSWORD), headers=self.headers) + self.headers['token'] = res.headers['token'] + + @task(1) + def get_user_signup_requests(self): + self.client.get("/api/user/requests", headers=self.headers) + + @task(1) + def get_blocked_users_request(self): + self.client.get("/api/user/blocked", headers=self.headers) + + @task(1) + def get_downloads_user_request(self): + self.client.get("/api/user/downloads/" + str([50, 100, 150][randint(0, 2)]), headers=self.headers) + +class APIUser(HttpLocust): + task_set = UserBehavior + wait_time = between(2, 7) From 27924638e0b38fa127378a50eff99f4cc7c9cff9 Mon Sep 17 00:00:00 2001 From: selfisekai Date: Sun, 8 Dec 2019 13:18:49 +0100 Subject: [PATCH 02/19] moved locustfile to components/core/tests --- components/core/tests/locustfile.py | 39 +++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 components/core/tests/locustfile.py diff --git a/components/core/tests/locustfile.py b/components/core/tests/locustfile.py new file mode 100644 index 00000000..4dacebe0 --- /dev/null +++ b/components/core/tests/locustfile.py @@ -0,0 +1,39 @@ +# for use with the Locust framework +# https://locust.io/ + +from locust import HttpLocust, TaskSet, task, between +from random import randint + +USERNAME = "rand" +PASSWORD = "pass" +SERVER_SECRET_KEY = "123456789" + +class UserBehavior(TaskSet): + headers = { + 'Host': 'localhost:5000', + 'Accept': 'application/json, text/plain, */*', + 'Accept-Language': 'en-US,en;q=0.5', + 'Accept-Encoding': 'gzip, deflate', + 'Content-Type': 'application/x-www-form-urlencoded', + 'Origin': 'http://localhost:3000', + } + + def on_start(self): + res = self.client.post("/api/login", "user_name={}&password={}".format(USERNAME, PASSWORD), headers=self.headers) + self.headers['token'] = res.headers['token'] + + @task(1) + def get_user_signup_requests(self): + self.client.get("/api/user/requests", headers=self.headers) + + @task(1) + def get_blocked_users_request(self): + self.client.get("/api/user/blocked", headers=self.headers) + + @task(1) + def get_downloads_user_request(self): + self.client.get("/api/user/downloads/" + str([50, 100, 150][randint(0, 2)]), headers=self.headers) + +class APIUser(HttpLocust): + task_set = UserBehavior + wait_time = between(2, 7) From e0555dcc6170af6d3239618baece541ab304f39e Mon Sep 17 00:00:00 2001 From: kmehant Date: Sun, 8 Dec 2019 18:01:25 +0530 Subject: [PATCH 03/19] Remove duplicate configuration --- locustfile.py | 36 ------------------------------------ 1 file changed, 36 deletions(-) delete mode 100644 locustfile.py diff --git a/locustfile.py b/locustfile.py deleted file mode 100644 index b6cfc3ea..00000000 --- a/locustfile.py +++ /dev/null @@ -1,36 +0,0 @@ -from locust import HttpLocust, TaskSet, task, between -from random import randint - -USERNAME = "rand" -PASSWORD = "pass" -SERVER_SECRET_KEY = "123456789" - -class UserBehavior(TaskSet): - headers = { - 'Host': 'localhost:5000', - 'Accept': 'application/json, text/plain, */*', - 'Accept-Language': 'en-US,en;q=0.5', - 'Accept-Encoding': 'gzip, deflate', - 'Content-Type': 'application/x-www-form-urlencoded', - 'Origin': 'http://localhost:3000', - } - - def on_start(self): - res = self.client.post("/api/login", "user_name={}&password={}".format(USERNAME, PASSWORD), headers=self.headers) - self.headers['token'] = res.headers['token'] - - @task(1) - def get_user_signup_requests(self): - self.client.get("/api/user/requests", headers=self.headers) - - @task(1) - def get_blocked_users_request(self): - self.client.get("/api/user/blocked", headers=self.headers) - - @task(1) - def get_downloads_user_request(self): - self.client.get("/api/user/downloads/" + str([50, 100, 150][randint(0, 2)]), headers=self.headers) - -class APIUser(HttpLocust): - task_set = UserBehavior - wait_time = between(2, 7) From 633d47008403eb60aca7a1ab5f2ec28e991adf5b Mon Sep 17 00:00:00 2001 From: selfisekai Date: Sat, 4 Jan 2020 18:40:39 +0100 Subject: [PATCH 04/19] production dockerfile for frontend-react --- components/frontend-react/Dockerfile.prod | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 components/frontend-react/Dockerfile.prod diff --git a/components/frontend-react/Dockerfile.prod b/components/frontend-react/Dockerfile.prod new file mode 100644 index 00000000..074539f5 --- /dev/null +++ b/components/frontend-react/Dockerfile.prod @@ -0,0 +1,15 @@ +# frontend building stage +FROM node:12 +WORKDIR /bassa-ui +COPY package.json . +COPY package-lock.json . +RUN npm install +COPY . . +RUN npm run build + +# setting up the nginx server +FROM nginx:1.16 +WORKDIR /usr/share/nginx/html +COPY --from=0 /bassa-ui/build . +EXPOSE 80 +CMD nginx -g 'daemon off;' From 8b1f08b7ae31f1b99c657555085a1fd2bb64b9bf Mon Sep 17 00:00:00 2001 From: selfisekai Date: Sat, 4 Jan 2020 18:54:26 +0100 Subject: [PATCH 05/19] development dockerfile for frontend-react --- components/frontend-react/Dockerfile | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 components/frontend-react/Dockerfile diff --git a/components/frontend-react/Dockerfile b/components/frontend-react/Dockerfile new file mode 100644 index 00000000..97ed842a --- /dev/null +++ b/components/frontend-react/Dockerfile @@ -0,0 +1,13 @@ +FROM node:12 + +WORKDIR /bassa + +COPY package.json . +COPY package-lock.json . +RUN npm install + +COPY . . + +EXPOSE 3000 + +CMD ["npm", "start"] From 769cdbcc31b83dfe405b83ba2fa916a19fec707b Mon Sep 17 00:00:00 2001 From: the catalyst Date: Thu, 20 Feb 2020 13:20:30 +0530 Subject: [PATCH 06/19] FEAT: add blueprints to make the routes look more manageable and created a view directory to store each type of endpoints seperately --- components/core/Main.py | 3 +- components/core/Server.py | 53 +++---------------- .../core/routes/views/Download_endpoints.py | 42 +++++++++++++++ .../core/routes/views/FileSender_endpoints.py | 19 +++++++ 4 files changed, 70 insertions(+), 47 deletions(-) create mode 100644 components/core/routes/views/Download_endpoints.py create mode 100644 components/core/routes/views/FileSender_endpoints.py diff --git a/components/core/Main.py b/components/core/Main.py index 5039bc18..213784c0 100644 --- a/components/core/Main.py +++ b/components/core/Main.py @@ -1,11 +1,12 @@ #!flask/bin/python -from Server import socketio, server +from Server import socketio, server, register_blueprint from ConfReader import check_conf_availability import logging def main(): + register_blueprint(server) socketio.run(server, host='0.0.0.0', port=5000) # server.run(debug=True) diff --git a/components/core/Server.py b/components/core/Server.py index ca43be2f..73e6412b 100644 --- a/components/core/Server.py +++ b/components/core/Server.py @@ -3,7 +3,7 @@ from flask_socketio import join_room import os from file_sender import sender -from routes import User, Download +from routes.views import User_endpoints, Download_endpoints, FileSender_endpoints # socket connections @@ -26,49 +26,10 @@ def serve_ui1(): return send_file(os.path.dirname(os.path.realpath(__file__)) + "/ui/index.html") -# download endpoints -server.add_url_rule(rule='/download/start', endpoint='start', view_func=Download.start) -server.add_url_rule(rule='/download/kill', endpoint='kill', view_func=Download.kill) -server.add_url_rule(rule='/api/download', endpoint='add_download_request', view_func=Download.add_download_request, - methods=['POST']) -server.add_url_rule(rule='/api/download/', endpoint='remove_download_request', - view_func=Download.remove_download_request, methods=['DELETE']) -server.add_url_rule(rule='/api/download/rate/', endpoint='rate_download_request', - view_func=Download.rate_download_request, methods=['POST']) -server.add_url_rule(rule='/api/downloads/', endpoint='get_downloads_request', - view_func=Download.get_downloads_request, methods=['GET']) -server.add_url_rule(rule='/api/download/', endpoint='get_download', view_func=Download.get_download, - methods=['GET']) -server.add_url_rule(rule='/api/file_from_minio/', endpoint='get_file_from_minio', view_func=Download.get_file_from_minio, - methods=['GET']) -# user action api endpoints -server.add_url_rule(rule='/api/login', endpoint='login', view_func=User.login, methods=['POST']) -server.add_url_rule(rule='/api/regularuser', endpoint='regular_user_request', view_func=User.regular_user_request, - methods=['POST']) -server.add_url_rule(rule='/api/user', endpoint='add_user_request', view_func=User.add_user_request, methods=['POST']) -server.add_url_rule(rule='/api/user/', endpoint='remove_user_request', - view_func=User.remove_user_request, methods=['DELETE']) -server.add_url_rule(rule='/api/user/', endpoint='update_user_request', - view_func=User.update_user_request, methods=['PUT']) -server.add_url_rule(rule='/api/user', endpoint='get_users_request', view_func=User.get_users_request, methods=['GET']) -server.add_url_rule(rule='/api/user/requests', endpoint='get_user_signup_requests', - view_func=User.get_user_signup_requests, methods=['GET']) -server.add_url_rule(rule='/api/user/approve/', endpoint='approve_user_request', - view_func=User.approve_user_request, methods=['POST']) -server.add_url_rule(rule='/api/user/blocked', endpoint='get_blocked_users_request', - view_func=User.get_blocked_users_request, methods=['GET']) -server.add_url_rule(rule='/api/user/blocked/', endpoint='block_user_request', - view_func=User.block_user_request, methods=['POST']) -server.add_url_rule(rule='/api/user/blocked/', endpoint='unblock_user_request', - view_func=User.unblock_user_request, methods=['DELETE']) -server.add_url_rule(rule='/api/user/downloads/', endpoint='get_downloads_user_request', - view_func=User.get_downloads_user_request, methods=['GET']) -server.add_url_rule(rule='/api/user/heavy', endpoint='get_topten_heaviest_users', - view_func=User.get_topten_heaviest_users, methods=['GET']) - -# file sending endpoints -server.add_url_rule(rule='/api/file', endpoint='send_file_from_path', view_func=sender.send_file_from_path, - methods=['GET']) -server.add_url_rule(rule='/api/compress', endpoint='start_compression', view_func=sender.start_compression, methods=['POST']) -server.add_url_rule(rule='/api/compression-progress', endpoint='get_compression_progress', view_func=sender.check_compression_progress, methods=['GET']) +def register_blueprint(server): + """Register Flask blueprints.""" + server.register_blueprint(User_endpoints.userprint, url_prefix="/api") + server.register_blueprint(Download_endpoints.downloadprint) + server.register_blueprint(FileSender_endpoints.filesenderprint, url_prefix="/api") + return None diff --git a/components/core/routes/views/Download_endpoints.py b/components/core/routes/views/Download_endpoints.py new file mode 100644 index 00000000..b5d30f2e --- /dev/null +++ b/components/core/routes/views/Download_endpoints.py @@ -0,0 +1,42 @@ +from flask import Blueprint +from routes.Download import * + +downloadprint = Blueprint("download", __name__) + +downloadprint.add_url_rule(rule='/download/start', + endpoint='start', + view_func=start) + +downloadprint.add_url_rule(rule='/download/kill', + endpoint='kill', + view_func=kill) + +downloadprint.add_url_rule(rule='/api/download', + endpoint='add_download_request', + view_func=add_download_request, + methods=['POST']) + +downloadprint.add_url_rule(rule='/api/download/', + endpoint='remove_download_request', + view_func=remove_download_request, + methods=['DELETE']) + +downloadprint.add_url_rule(rule='/api/download/rate/', + endpoint='rate_download_request', + view_func=rate_download_request, + methods=['POST']) + +downloadprint.add_url_rule(rule='/api/downloads/', + endpoint='get_downloads_request', + view_func=get_downloads_request, + methods=['GET']) + +downloadprint.add_url_rule(rule='/api/download/', + endpoint='get_download', + view_func=get_download, + methods=['GET']) + +downloadprint.add_url_rule(rule='/api/file_from_minio/', + endpoint='get_file_from_minio', + view_func=get_file_from_minio, + methods=['GET']) diff --git a/components/core/routes/views/FileSender_endpoints.py b/components/core/routes/views/FileSender_endpoints.py new file mode 100644 index 00000000..75be0428 --- /dev/null +++ b/components/core/routes/views/FileSender_endpoints.py @@ -0,0 +1,19 @@ +from flask import Blueprint +from file_sender.sender import * + +filesenderprint = Blueprint("filesenderprint", __name__) + +filesenderprint.add_url_rule(rule='/file', + endpoint='send_file_from_path', + view_func=send_file_from_path, + methods=['GET']) + +filesenderprint.add_url_rule(rule='/compress', + endpoint='start_compression', + view_func=start_compression, + methods=['POST']) + +filesenderprint.add_url_rule(rule='/compression-progress', + endpoint='get_compression_progress', + view_func=check_compression_progress, + methods=['GET']) From ec07d0980b78b2acfed2d14dfa9b12978f33d7a7 Mon Sep 17 00:00:00 2001 From: the catalyst Date: Thu, 20 Feb 2020 18:14:58 +0530 Subject: [PATCH 07/19] add user endpoints --- .../core/routes/views/User_endpoints.py | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 components/core/routes/views/User_endpoints.py diff --git a/components/core/routes/views/User_endpoints.py b/components/core/routes/views/User_endpoints.py new file mode 100644 index 00000000..f9a63cfe --- /dev/null +++ b/components/core/routes/views/User_endpoints.py @@ -0,0 +1,70 @@ +from flask import Blueprint +from routes.User import * + +# print(dir(user)) +userprint = Blueprint("user", __name__) + +userprint.add_url_rule(rule='/login', + endpoint='login', + view_func=login, + methods=['POST']) + +userprint.add_url_rule(rule='/regularuser', + endpoint='regular_user_request', + view_func=regular_user_request, + methods=['POST']) + +userprint.add_url_rule(rule='/user', + endpoint='add_user_request', + view_func=add_user_request, + methods=['POST']) + +userprint.add_url_rule(rule='/user/', + endpoint='remove_user_request', + view_func=remove_user_request, + methods=['DELETE']) + +userprint.add_url_rule(rule='/user/', + endpoint='update_user_request', + view_func=update_user_request, + methods=['PUT']) + +userprint.add_url_rule(rule='/user', + endpoint='get_users_request', + view_func=get_users_request, + methods=['GET']) + +userprint.add_url_rule(rule='/user/requests', + endpoint='get_user_signup_requests', + view_func=get_user_signup_requests, + methods=['GET']) + +userprint.add_url_rule(rule='/user/approve/', + endpoint='approve_user_request', + view_func=approve_user_request, + methods=['POST']) + +userprint.add_url_rule(rule='/user/blocked', + endpoint='get_blocked_users_request', + view_func=get_blocked_users_request, + methods=['GET']) + +userprint.add_url_rule(rule='/user/blocked/', + endpoint='block_user_request', + view_func=block_user_request, + methods=['POST']) + +userprint.add_url_rule(rule='/user/blocked/', + endpoint='unblock_user_request', + view_func=unblock_user_request, + methods=['DELETE']) + +userprint.add_url_rule(rule='/user/downloads/', + endpoint='get_downloads_user_request', + view_func=get_downloads_user_request, + methods=['GET']) + +userprint.add_url_rule(rule='/user/heavy', + endpoint='get_topten_heaviest_users', + view_func=get_topten_heaviest_users, + methods=['GET']) From 5f01a76d26be005ee35edc9c038d3248b53f0ed8 Mon Sep 17 00:00:00 2001 From: Dewansh Rawat Date: Mon, 30 Mar 2020 07:57:47 +0530 Subject: [PATCH 08/19] Adding logo to Readme title --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7dc8175a..6e2ab185 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@
-![logo](http://gdurl.com/7XYK) +![Bassa logo](https://github.com/scorelab/Bassa/blob/master/ui/src/assets/images/bassa.png?raw=true) [![Build Status](https://travis-ci.org/scorelab/Bassa.svg?branch=master)](https://travis-ci.org/scorelab/Bassa) [![Codacy Badge](https://api.codacy.com/project/badge/Grade/7de63c7b9a69448787e8014a12a260b1)](https://www.codacy.com/app/SCoRe-Lab/Bassa?utm_source=github.com&utm_medium=referral&utm_content=scorelab/Bassa&utm_campaign=badger) From 77d052e7e274a6ac23e5de813421775daa4938d8 Mon Sep 17 00:00:00 2001 From: Dewansh Rawat Date: Mon, 30 Mar 2020 08:01:16 +0530 Subject: [PATCH 09/19] Shifting contributors section from HTML to Markdown The Contributors section was designed in HTML, which was inconsistent with the standard markdown. Updated the same to keep it consistent in markdown --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6e2ab185..13b4a614 100644 --- a/README.md +++ b/README.md @@ -54,9 +54,8 @@ You can also discuss about other projects on [SCoRe Lab gitter channel](https:// ## Contributors -This project exists thanks to all the people who contribute. - - +This project exists thanks to all the people who contribute. +[![Contributors Image](https://opencollective.com/Bassa/contributors.svg?width=890&button=false)](https://opencollective.com/Bassa#contributors) ## Backers From 64bc29faa96bf8e2186c8ab4300525d57a71ec5f Mon Sep 17 00:00:00 2001 From: Sahan Dilshan <32576163+sahandilshan@users.noreply.github.com> Date: Wed, 15 Apr 2020 22:21:06 +0530 Subject: [PATCH 10/19] Add register with strong password feature (#898) * Add register with a strong password feature * Add separate error messages * Update ui/src/app/signup/controllers/SignupCtrl.js Co-Authored-By: Dehami Koswatte Co-authored-by: Sahan Dilshan <32576163+Amstrom@users.noreply.github.com> Co-authored-by: Bhathiya Perera Co-authored-by: Dehami Koswatte --- ui/src/app/signup/controllers/SignupCtrl.js | 39 ++++++++++++++++++++- ui/src/app/views/signup.html | 8 ++--- 2 files changed, 42 insertions(+), 5 deletions(-) diff --git a/ui/src/app/signup/controllers/SignupCtrl.js b/ui/src/app/signup/controllers/SignupCtrl.js index 4d5f785c..6f31f615 100755 --- a/ui/src/app/signup/controllers/SignupCtrl.js +++ b/ui/src/app/signup/controllers/SignupCtrl.js @@ -13,6 +13,7 @@ }; $scope.incorrectCredentials = false; + $scope.errorMsg = ''; var validate = function(obj) { for (var key in obj) { @@ -46,6 +47,42 @@ ToastService.showToast('Please ensure entered details are correct'); } }; - }; + + + function passwordValid() { + $scope.loginForm.password.$valid = true; + $scope.loginForm.password.$dirty = false; + $scope.loginForm.password.$invalid = false; + } + function passwordInValid() { + $scope.loginForm.password.$valid = false; + $scope.loginForm.password.$invalid = true; + $scope.loginForm.password.$dirty = true; + } + + $scope.validatePwd = () => { + const lowerCaseLetters = /[a-z]/g; + const upperCaseLetters = /[A-Z]/g; + const numbers = /[0-9]/g; + if (typeof $scope.user.password !== 'undefined') { + if ($scope.user.password.length < 8) { + passwordInValid(); + $scope.errorMsg = 'Password must contains Minimum 8 characters'; + } else if (!$scope.user.password.match(upperCaseLetters)) { + passwordInValid(); + $scope.errorMsg = 'Password must contains a uppercase letter'; + } else if (!$scope.user.password.match(lowerCaseLetters)) { + passwordInValid(); + $scope.errorMsg = 'Password must contains a lowercase letter'; + } else if (!$scope.user.password.match(numbers)) { + passwordInValid(); + $scope.errorMsg = 'Password must contains a number'; + } else { + passwordValid(); + $scope.errorMsg = ''; + } + } + }; + } })(); diff --git a/ui/src/app/views/signup.html b/ui/src/app/views/signup.html index e747853a..6daaec5e 100755 --- a/ui/src/app/views/signup.html +++ b/ui/src/app/views/signup.html @@ -3,7 +3,7 @@

Sign up for Bassa

- +

Username should be more than 3 characters and less than 15

@@ -13,12 +13,12 @@

Sign up for Bassa

- -

Please Enter at least 8 characters

+ +

- +

Password did not match!

From 99f10a71c246c969d5c568e4f0d945ca5f9a1c3d Mon Sep 17 00:00:00 2001 From: Harshit Verma Date: Wed, 15 Apr 2020 22:24:36 +0530 Subject: [PATCH 11/19] fix(ui/views/main.html): changed menu to button (#882) added tooltip to log out icon --- ui/src/app/views/main.html | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/ui/src/app/views/main.html b/ui/src/app/views/main.html index 79d7fbe0..f146281f 100755 --- a/ui/src/app/views/main.html +++ b/ui/src/app/views/main.html @@ -26,14 +26,9 @@

Hi {{username}}

- - - menu - - - Logout - - + + Log out +
From 92701ffd3940a6a9a251e0cd852f7dd8a765a71a Mon Sep 17 00:00:00 2001 From: Mehant Kammakomati <411843@student.nitandhra.ac.in> Date: Thu, 7 May 2020 17:51:44 +0530 Subject: [PATCH 12/19] Make Bassa ready for Kubernetes deployments (#828) * Modify API endpoints for better use of ingress * Dockerfile for Bassa database for better cloud deployments * Change hostname relevent to kubernetes cluster * kubernetes starter script * Kubernetes manifests for complete Bassa projects * Update Killer.py * Update Starter.py * Get Bassa Url from config --- components/core/Killer.py | 4 +- components/core/Server.py | 1 - components/core/dl.conf | 2 +- components/database/dockerfile | 5 +++ scripts/kubernetes/deploy.sh | 17 ++++++++ .../manifests/api_and_aria2c/deployment.yml | 43 +++++++++++++++++++ .../api_and_aria2c/downloads-claim.yml | 11 +++++ .../manifests/api_and_aria2c/service.yml | 12 ++++++ .../manifests/database/database-claim.yml | 11 +++++ .../manifests/database/deployment.yml | 36 ++++++++++++++++ .../kubernetes/manifests/database/service.yml | 12 ++++++ .../kubernetes/manifests/web/00-namespace.yml | 4 ++ .../kubernetes/manifests/web/deployment.yml | 20 +++++++++ scripts/kubernetes/manifests/web/ingress.yml | 17 ++++++++ scripts/kubernetes/manifests/web/service.yml | 12 ++++++ ui/config.ts | 3 ++ ui/src/app/admin/services/AdminService.js | 4 +- .../app/admin/services/AdminService.spec.js | 4 +- ui/src/app/index.js | 3 +- 19 files changed, 212 insertions(+), 9 deletions(-) create mode 100644 components/database/dockerfile create mode 100644 scripts/kubernetes/deploy.sh create mode 100644 scripts/kubernetes/manifests/api_and_aria2c/deployment.yml create mode 100644 scripts/kubernetes/manifests/api_and_aria2c/downloads-claim.yml create mode 100644 scripts/kubernetes/manifests/api_and_aria2c/service.yml create mode 100644 scripts/kubernetes/manifests/database/database-claim.yml create mode 100644 scripts/kubernetes/manifests/database/deployment.yml create mode 100644 scripts/kubernetes/manifests/database/service.yml create mode 100644 scripts/kubernetes/manifests/web/00-namespace.yml create mode 100644 scripts/kubernetes/manifests/web/deployment.yml create mode 100644 scripts/kubernetes/manifests/web/ingress.yml create mode 100644 scripts/kubernetes/manifests/web/service.yml create mode 100644 ui/config.ts diff --git a/components/core/Killer.py b/components/core/Killer.py index 801d9415..a3674909 100644 --- a/components/core/Killer.py +++ b/components/core/Killer.py @@ -1,6 +1,6 @@ import urllib.request, urllib.error, urllib.parse -req = urllib.request.Request('http://localhost:5000/download/kill') +req = urllib.request.Request('http://localhost:5000/download/kill') req.add_header('key', '123456789') resp = urllib.request.urlopen(req) -content = resp.read() \ No newline at end of file +content = resp.read() diff --git a/components/core/Server.py b/components/core/Server.py index 73e6412b..57ca99e6 100644 --- a/components/core/Server.py +++ b/components/core/Server.py @@ -26,7 +26,6 @@ def serve_ui1(): return send_file(os.path.dirname(os.path.realpath(__file__)) + "/ui/index.html") - def register_blueprint(server): """Register Flask blueprints.""" server.register_blueprint(User_endpoints.userprint, url_prefix="/api") diff --git a/components/core/dl.conf b/components/core/dl.conf index c15a5685..a4737565 100644 --- a/components/core/dl.conf +++ b/components/core/dl.conf @@ -1 +1 @@ -{"max_downloads": 2, "down_folder": "../../downloads", "size_limit":100000, "max_age":1, "min_rating":3, "max_bandwidth": 300000, "aria_server": "ws://aria2c:6800/jsonrpc", "tmp_folder": "/tmp/bassa/"} +{"max_downloads": 2, "down_folder": "../../downloads", "size_limit":100000, "max_age":1, "min_rating":3, "max_bandwidth": 300000, "aria_server": "ws://localhost:6800/jsonrpc", "tmp_folder": "/tmp/bassa/"} diff --git a/components/database/dockerfile b/components/database/dockerfile new file mode 100644 index 00000000..4e5b9860 --- /dev/null +++ b/components/database/dockerfile @@ -0,0 +1,5 @@ +FROM mysql:5.7 +COPY ../../db_schema/Bassa.sql /tmp/schema/ +COPY ../../.docker/volumes/mysql/create-schema.sh /docker-entrypoint-initdb.d/ +EXPOSE 3306 +CMD ["mysqld"] diff --git a/scripts/kubernetes/deploy.sh b/scripts/kubernetes/deploy.sh new file mode 100644 index 00000000..e63a58e0 --- /dev/null +++ b/scripts/kubernetes/deploy.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +if [ "$1" = "delete" ]; then + echo "Clearing the cluster." + kubectl delete -f ./manifests/web/00-namespace.yml + echo "Bassa is removed from the cluster" +elif [ "$1" = "create" ]; then + echo "Deploying Bassa to kubernetes cluster" + # Deploy web pod + kubectl create -R -f ./manifests/web/ + # Deploy api and aria2c pod + kubectl create -R -f ./manifests/api_and_aria2c/ + # Deploy database pod + kubectl create -R -f ./manifests/database/ + echo "Bassa got deployed at" + kubectl get ingress -n bassa | awk '{ print $3 }' +fi \ No newline at end of file diff --git a/scripts/kubernetes/manifests/api_and_aria2c/deployment.yml b/scripts/kubernetes/manifests/api_and_aria2c/deployment.yml new file mode 100644 index 00000000..876cc9b3 --- /dev/null +++ b/scripts/kubernetes/manifests/api_and_aria2c/deployment.yml @@ -0,0 +1,43 @@ +kind: Deployment +apiVersion: apps/v1beta1 +metadata: + name: api + namespace: bassa +spec: + replicas: 1 + template: + metadata: + labels: + app: api + spec: + volumes: + - name: downloads + persistentVolumeClaim: + claimName: downloads-claim + containers: + - name: api + image: kmehant/api + imagePullPolicy: Always + ports: + - containerPort: 5000 + protocol: TCP + env: + - name: BASSA_DB_NAME + value: 'Bassa' + - name: BASSA_DB_USERNAME + value: 'bassa' + - name: BASSA_DB_PASSWORD + value: 'bassa@1234' + volumeMounts: + - mountPath: /downloads + name: downloads + - name: aria2c + image: kmehant/aria2c + imagePullPolicy: Always + ports: + - containerPort: 6800 + protocol: TCP + volumeMounts: + - mountPath: /data + name: downloads + restartPolicy: Always \ No newline at end of file diff --git a/scripts/kubernetes/manifests/api_and_aria2c/downloads-claim.yml b/scripts/kubernetes/manifests/api_and_aria2c/downloads-claim.yml new file mode 100644 index 00000000..e0e7e505 --- /dev/null +++ b/scripts/kubernetes/manifests/api_and_aria2c/downloads-claim.yml @@ -0,0 +1,11 @@ +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: downloads-claim + namespace: bassa +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 250Mi \ No newline at end of file diff --git a/scripts/kubernetes/manifests/api_and_aria2c/service.yml b/scripts/kubernetes/manifests/api_and_aria2c/service.yml new file mode 100644 index 00000000..7265a54a --- /dev/null +++ b/scripts/kubernetes/manifests/api_and_aria2c/service.yml @@ -0,0 +1,12 @@ +kind: Service +apiVersion: v1 +metadata: + name: api + namespace: bassa +spec: + ports: + - port: 5000 + protocol: TCP + targetPort: 5000 + selector: + app: api \ No newline at end of file diff --git a/scripts/kubernetes/manifests/database/database-claim.yml b/scripts/kubernetes/manifests/database/database-claim.yml new file mode 100644 index 00000000..49574e3c --- /dev/null +++ b/scripts/kubernetes/manifests/database/database-claim.yml @@ -0,0 +1,11 @@ +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: database-claim + namespace: bassa +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 250Mi \ No newline at end of file diff --git a/scripts/kubernetes/manifests/database/deployment.yml b/scripts/kubernetes/manifests/database/deployment.yml new file mode 100644 index 00000000..65d2f08b --- /dev/null +++ b/scripts/kubernetes/manifests/database/deployment.yml @@ -0,0 +1,36 @@ +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: db + namespace: bassa +spec: + replicas: 1 + template: + metadata: + labels: + name: db + spec: + containers: + - name: db + image: kmehant/db + imagePullPolicy: Always + env: + - name: MYSQL_ROOT_PASSWORD + value: bassa@1234 + - name: MYSQL_DATABASE + value: Bassa + - name: MYSQL_USER + value: bassa + - name: BASSA_DB_NAME + value: Bassa + - name: MYSQL_PASSWORD + value: bassa@1234 + ports: + - containerPort: 3306 + volumeMounts: + - mountPath: /var/lib/mysql + name: data-volume + volumes: + - name: data-volume + persistentVolumeClaim: + claimName: database-claim \ No newline at end of file diff --git a/scripts/kubernetes/manifests/database/service.yml b/scripts/kubernetes/manifests/database/service.yml new file mode 100644 index 00000000..84913b5f --- /dev/null +++ b/scripts/kubernetes/manifests/database/service.yml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: db + namespace: bassa + labels: + name: db +spec: + ports: + - port: 3306 + selector: + name: db \ No newline at end of file diff --git a/scripts/kubernetes/manifests/web/00-namespace.yml b/scripts/kubernetes/manifests/web/00-namespace.yml new file mode 100644 index 00000000..5e271360 --- /dev/null +++ b/scripts/kubernetes/manifests/web/00-namespace.yml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: bassa \ No newline at end of file diff --git a/scripts/kubernetes/manifests/web/deployment.yml b/scripts/kubernetes/manifests/web/deployment.yml new file mode 100644 index 00000000..a387fcb9 --- /dev/null +++ b/scripts/kubernetes/manifests/web/deployment.yml @@ -0,0 +1,20 @@ +kind: Deployment +apiVersion: apps/v1beta1 +metadata: + name: web + namespace: bassa +spec: + replicas: 1 + template: + metadata: + labels: + app: web + spec: + containers: + - name: web + image: kmehant/web + imagePullPolicy: Always + ports: + - containerPort: 80 + protocol: TCP + restartPolicy: Always \ No newline at end of file diff --git a/scripts/kubernetes/manifests/web/ingress.yml b/scripts/kubernetes/manifests/web/ingress.yml new file mode 100644 index 00000000..06f2f32d --- /dev/null +++ b/scripts/kubernetes/manifests/web/ingress.yml @@ -0,0 +1,17 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: web-ingress + namespace: bassa +spec: + rules: + - http: + paths: + - path: / + backend: + serviceName: web + servicePort: 80 + - path: /api/ + backend: + serviceName: api + servicePort: 5000 \ No newline at end of file diff --git a/scripts/kubernetes/manifests/web/service.yml b/scripts/kubernetes/manifests/web/service.yml new file mode 100644 index 00000000..20357ac4 --- /dev/null +++ b/scripts/kubernetes/manifests/web/service.yml @@ -0,0 +1,12 @@ +kind: Service +apiVersion: v1 +metadata: + name: web + namespace: bassa +spec: + ports: + - port: 80 + protocol: TCP + targetPort: 80 + selector: + app: web \ No newline at end of file diff --git a/ui/config.ts b/ui/config.ts new file mode 100644 index 00000000..4cc25912 --- /dev/null +++ b/ui/config.ts @@ -0,0 +1,3 @@ +export var config = { + "API_URL":"http://localhost:5000", + } \ No newline at end of file diff --git a/ui/src/app/admin/services/AdminService.js b/ui/src/app/admin/services/AdminService.js index 729dc32a..94323f21 100755 --- a/ui/src/app/admin/services/AdminService.js +++ b/ui/src/app/admin/services/AdminService.js @@ -8,7 +8,7 @@ var startDownloads = function() { return $http({ method: 'GET', - url: BassaUrl + '/download/start', + url: BassaUrl + '/api/download/start', headers: {'key': '123456789'} }); }; @@ -16,7 +16,7 @@ var killDownloads = function() { return $http({ method: 'GET', - url: BassaUrl + '/download/kill', + url: BassaUrl + '/api/download/kill', headers: {'key': '123456789'} }); }; diff --git a/ui/src/app/admin/services/AdminService.spec.js b/ui/src/app/admin/services/AdminService.spec.js index b71f25f6..89bdf156 100644 --- a/ui/src/app/admin/services/AdminService.spec.js +++ b/ui/src/app/admin/services/AdminService.spec.js @@ -18,7 +18,7 @@ describe("Service: AdminService", function() { it("should start downloads", function() { - localHttpBackend.expect("GET", localBassaUrl + "/download/start") + localHttpBackend.expect("GET", localBassaUrl + "/api/download/start") .respond(201, { "status": "12345" }); AdminService.startDownloads().then(function(response) { @@ -31,7 +31,7 @@ describe("Service: AdminService", function() { it("should kill downloads", function() { - localHttpBackend.expect("GET", localBassaUrl + "/download/kill") + localHttpBackend.expect("GET", localBassaUrl + "/api/download/kill") .respond(200, { status: "success"}); AdminService.killDownloads().then(function(response) { diff --git a/ui/src/app/index.js b/ui/src/app/index.js index 19f1bcd9..12529972 100755 --- a/ui/src/app/index.js +++ b/ui/src/app/index.js @@ -1,9 +1,10 @@ 'use strict'; +import { config } from "../../config"; angular.module('bassa', ['ngAnimate', 'ngCookies', 'ngTouch', 'ngSanitize', 'ui.router', 'ngMaterial', 'nvd3', 'app']) - .value('BassaUrl', 'http://localhost:5000') + .value('BassaUrl', config.API_URL) .config(function ($stateProvider, $httpProvider, $urlRouterProvider, $mdThemingProvider, $mdIconProvider, $qProvider) { From 2f622d46c723a06b101bce6791e8860f13bf3700 Mon Sep 17 00:00:00 2001 From: K mehant Date: Tue, 12 May 2020 22:46:37 +0530 Subject: [PATCH 13/19] Fix Server errors on download and other required fixes --- components/aria2c/Dockerfile | 4 +- components/core/DownloadDaemon.py | 9 +- components/core/DownloadManager.py | 99 +++++++++++++------ components/core/UserManager.py | 49 ++++++--- components/core/dl.conf | 2 +- components/core/routes/Download.py | 11 ++- .../core/routes/views/Download_endpoints.py | 4 +- ui/src/app/index.js | 4 +- 8 files changed, 127 insertions(+), 55 deletions(-) diff --git a/components/aria2c/Dockerfile b/components/aria2c/Dockerfile index 15f1c065..266b217f 100644 --- a/components/aria2c/Dockerfile +++ b/components/aria2c/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:edge +FROM alpine # Defining dockerfile maintainer MAINTAINER SCoRe Lab Community @@ -21,7 +21,7 @@ VOLUME [ "/data" ] WORKDIR /data # Starting the Aria2c RPC server -CMD ["aria2c", "--enable-rpc", "--rpc-listen-all"] +CMD ["aria2c", "--enable-rpc=true", "--rpc-listen-all", "--no-conf=true", "--disable-ipv6", "--console-log-level=debug"] # Labels for dockerfile information LABEL scorelab.bassa.name="Aria2c RPC server" \ diff --git a/components/core/DownloadDaemon.py b/components/core/DownloadDaemon.py index 01da9379..36ccf658 100644 --- a/components/core/DownloadDaemon.py +++ b/components/core/DownloadDaemon.py @@ -7,7 +7,6 @@ from EMail import send_mail from DiskMan import * from ConfReader import get_conf_reader -from MinioHandler import * import logging import websocket @@ -86,6 +85,9 @@ def worker(self): data = json.loads(message) print(threading.current_thread().name,data) print ("TOP LEVEL DATA", data) + if 'error' in data: + logging.error("Aria2c error: %s", data['error']) + return if 'id' in data and data['id'] == "act": toBeDownloaded = get_to_download() if toBeDownloaded: @@ -178,6 +180,7 @@ def add_uri(ws, download): def get_status(ws, id=None, gid=None): if id: gid = get_gid_from_id(id) + gid = str(gid) msg = JSONer("stat", 'aria2.tellStatus', [gid, ['gid', 'files']]) print ("Getting status") ws.send(msg) @@ -236,7 +239,7 @@ def on_message(ws, message): messageQueue.put(message) def on_error(ws, error): - print(error) + logging.error(error) def on_close(ws): @@ -252,6 +255,7 @@ def starter(socket): sc = socket remove_files(conf['max_age'], conf['min_rating']) folder_size=get_size(conf['down_folder']) + logging.info("SERVER: folder_size: %d", folder_size) websocket.enableTrace(False) ws = websocket.WebSocketApp(conf['aria_server'], on_message=on_message, @@ -268,4 +272,3 @@ def starter(socket): - diff --git a/components/core/DownloadManager.py b/components/core/DownloadManager.py index fbb20d23..97a5c2ed 100644 --- a/components/core/DownloadManager.py +++ b/components/core/DownloadManager.py @@ -3,13 +3,14 @@ import time import sys import sqlalchemy.pool as pool +import logging verbose = False if len(sys.argv) == 2 and sys.argv[1] == '-v': verbose = True -threadpool = pool.QueuePool(get_db_con, max_overflow=10, pool_size=20) +threadpool = pool.QueuePool(get_db_con, max_overflow=10, pool_size=0) def add_download(download): @@ -23,12 +24,12 @@ def add_download(download): sql = "INSERT into download(link, user_name, added_time, download_name) VALUES(%s, %s, %s, %s);" try: cursor.execute(sql, (download.link, download.userName, int(time.time()), download_name)) + cursor.close() db.commit() + db.close() except MySQLdb.Error as e: db.rollback() - # Shows error thrown up by database - print(e) - return e[1] + logging.error("Error %d: %s", e.args[0], e.args[1]) return "success" return "db connection error" @@ -42,14 +43,19 @@ def remove_download(id, userName): try: cursor.execute(sql1, [str(id)]) data = cursor.fetchone() + cursor.close() if data[0] != Status.DEFAULT and data[0] != Status.ERROR: db.commit() + db.close() return "Download started. Entry cannot be deleted." + cursor = db.cursor() cursor.execute(sql, [str(id), userName]) + cursor.close() db.commit() + db.close() except MySQLdb.Error as e: db.rollback() - return e[1] + logging.error("Error %d: %s", e.args[0], e.args[1]) return "success" return "db connection error" @@ -66,11 +72,13 @@ def rate_download(id, userName, rate): cursor.execute(sql1, (rate, id, userName)) if cursor.rowcount == 0: cursor.execute(sql, (userName, id, rate)) + cursor.close() db.commit() + db.close() update_rate(id) except MySQLdb.Error as e: db.rollback() - return e[1] + logging.error("Error %d: %s", e.args[0], e.args[1]) return "success" return "db connection error" @@ -85,10 +93,12 @@ def update_rate(id): cursor.execute(sql1, id) data = cursor.fetchone() cursor.execute(sql, (data[0], id)) + cursor.close() db.commit() + db.close() except MySQLdb.Error as e: db.rollback() - return e[1] + logging.error("Error %d: %s", e.args[0], e.args[1]) return "success" return "db connection error" @@ -105,7 +115,7 @@ def get_downloads_user(userName, limit): db.close() return results except MySQLdb.Error as e: - return e[1] + logging.error("Error %d: %s", e.args[0], e.args[1]) return "db connection error" @@ -121,7 +131,7 @@ def get_downloads(limit): db.close() return results except MySQLdb.Error as e: - return e[1] + logging.error("Error %d: %s", e.args[0], e.args[1]) return "db connection error" @@ -138,10 +148,12 @@ def update_status_gid(gid, status, completed=False): cursor.execute(sql, (status, int(time.time()), gid)) else: cursor.execute(sql, (status, gid)) + cursor.close() db.commit() + db.close() except MySQLdb.Error as e: db.rollback() - return e[1] + logging.error("Error %d: %s", e.args[0], e.args[1]) return "success" return "db connection error" @@ -153,10 +165,12 @@ def set_gid(id, gid): sql = "UPDATE download SET gid='%s' WHERE id=%s;" % (gid, id) try: cursor.execute(sql) + cursor.close() db.commit() + db.close() except MySQLdb.Error as e: db.rollback() - return e[1] + logging.error("Error %d: %s", e.args[0], e.args[1]) return "success" return "db connection error" @@ -168,10 +182,12 @@ def set_name(gid, name): sql = "UPDATE download SET download_name=%s WHERE gid=%s ;" try: cursor.execute(sql, (name, gid)) + cursor.close() db.commit() + db.close() except MySQLdb.Error as e: db.rollback() - return e[1] + logging.error("Error %d: %s", e.args[0], e.args[1]) return "success" return "db connection error" @@ -183,10 +199,12 @@ def set_size(gid, size): sql = "UPDATE download SET size=%s WHERE gid=%s ;" try: cursor.execute(sql, (size, gid)) + cursor.close() db.commit() + db.close() except MySQLdb.Error as e: db.rollback() - return e[1] + logging.error("Error %d: %s", e.args[0], e.args[1]) return "success" return "db connection error" @@ -202,11 +220,13 @@ def get_to_download(): if verbose: print("zero count") return None results = cursor.fetchall() + cursor.close() + db.close() if verbose: print("LIST", results) downloads = [Download(result['link'], result['user_name'], result['id']) for result in results] return downloads except MySQLdb.Error as e: - return e[1] + logging.error("Error %d: %s", e.args[0], e.args[1]) return "db connection error" @@ -217,10 +237,12 @@ def set_path(gid, path): sql = "UPDATE download SET path=%s WHERE gid=%s ;" try: cursor.execute(sql, (path, gid)) + cursor.close() db.commit() + db.close() except MySQLdb.Error as e: db.rollback() - return e[1] + logging.error("Error %d: %s", e.args[0], e.args[1]) return "success" return "db connection error" @@ -235,11 +257,12 @@ def get_download_path(id): if cursor.rowcount == 0: return None results = cursor.fetchone() + cursor.close() path = results['path'] db.close() return path except MySQLdb.Error as e: - return e[1] + logging.error("Error %d: %s", e.args[0], e.args[1]) return "db connection error" @@ -253,11 +276,12 @@ def get_download_email(gid): if cursor.rowcount == 0: return None results = cursor.fetchone() + cursor.close() path = results['email'] db.close() return path except MySQLdb.Error as e: - return e[1] + logging.error("Error %d: %s", e.args[0], e.args[1]) return "db connection error" @@ -270,12 +294,13 @@ def get_to_delete(time, rate): try: cursor.execute(sql, (time, rate)) results = cursor.fetchall() + cursor.close() db.close() if verbose: print("Time", time, "Rate", rate) if verbose: print("results", results) return results except MySQLdb.Error as e: - return e[1] + logging.error("Error %d: %s", e.args[0], e.args[1]) return "db connection error" @@ -286,10 +311,12 @@ def set_delete_status(path): sql = "UPDATE download SET status=2 WHERE path='%s' ;" % path try: cursor.execute(sql) + cursor.close() db.commit() + db.close() except MySQLdb.Error as e: db.rollback() - return e[1] + logging.error("Error %d: %s", e.args[0], e.args[1]) return "success" return "db connection error" @@ -304,12 +331,13 @@ def get_download_status(id): if cursor.rowcount == 0: return None results = cursor.fetchone() + cursor.close() print('results') status = results['status'] db.close() return status except MySQLdb.Error as e: - return e[1] + raise return "db connection error" @@ -323,11 +351,12 @@ def get_id_from_gid(gid): if cursor.rowcount == 0: return None results = cursor.fetchone() + cursor.close() status = results['id'] db.close() return status except MySQLdb.Error as e: - return e[1] + logging.error("Error %d: %s", e.args[0], e.args[1]) return "db connection error" @@ -341,11 +370,12 @@ def get_username_from_gid(gid): if cursor.rowcount == 0: return None results = cursor.fetchone() + cursor.close() status = results['user_name'] db.close() return status except MySQLdb.Error as e: - return e[1] + logging.error("Error %d: %s", e.args[0], e.args[1]) return "db connection error" @@ -359,11 +389,12 @@ def get_gid_from_id(id): if cursor.rowcount == 0: return None results = cursor.fetchone() + cursor.close() gid = results['gid'] db.close() return gid except MySQLdb.Error as e: - return e[1] + logging.error("Error %d: %s", e.args[0], e.args[1]) return "db connection error" @@ -377,11 +408,12 @@ def get_download_name_from_id(id): if cursor.rowcount == 0: return None results = cursor.fetchone() + cursor.close() download_name = results['download_name'] db.close() return download_name except MySQLdb.Error as e: - return e[1] + logging.error("Error %d: %s", e.args[0], e.args[1]) return "db connection error" @@ -396,12 +428,12 @@ def set_compression_progress(comp_id, status, **kwargs): 'completed_time'], comp_id) try: cursor.execute(sql, ) + cursor.close() db.commit() + db.close() except MySQLdb.Error as e: db.rollback() - # Shows error thrown up by database - print(e) - return e[1] + logging.error("Error %d: %s", e.args[0], e.args[1]) return "success" return "db connection error" @@ -416,11 +448,12 @@ def get_compression_progress(comp_id): if cursor.rowcount == 0: return None results = cursor.fetchone() + cursor.close() status = results['progress'] db.close() return status except MySQLdb.Error as e: - return e[1] + logging.error("Error %d: %s", e.args[0], e.args[1]) return "db connection error" @@ -433,12 +466,12 @@ def insert_compression_process(comp_id, start_time, completed_time, is_deleted): comp_id, start_time, completed_time, is_deleted) try: cursor.execute(sql, ) + cursor.close() db.commit() + db.close() except MySQLdb.Error as e: db.rollback() - # Shows error thrown up by database - print(e) - return e[1] + logging.error("Error %d: %s", e.args[0], e.args[1]) return 0 return "db connection error" @@ -451,6 +484,7 @@ def update_minio_indexes(file_name, file_path, gid, completedLength, username, d cursor.execute(sql, (download_id, gid, username, file_name, completedLength, file_path)) db.commit() cursor.close() + db.close() except MySQLdb.Error as e: db.rollback() # Shows error thrown up by database @@ -468,10 +502,11 @@ def get_file_name(id): return None results = cursor.fetchone() print(results) + cursor.close() file_name = results['mfile_name'] db.close() return file_name except MySQLdb.Error as e: print(e) - return e[1] + logging.error("Error %d: %s", e.args[0], e.args[1]) return "db connection error" diff --git a/components/core/UserManager.py b/components/core/UserManager.py index 74d5fe0e..465c376e 100644 --- a/components/core/UserManager.py +++ b/components/core/UserManager.py @@ -2,10 +2,11 @@ from DBCon import * from ConfReader import get_conf_reader import sqlalchemy.pool as pool +import logging conf = get_conf_reader("dl.conf") -threadpool = pool.QueuePool(get_db_con, max_overflow=10, pool_size=20) +threadpool = pool.QueuePool(get_db_con, max_overflow=10, pool_size=0) def user_login(username, password): db = threadpool.connect() @@ -14,6 +15,7 @@ def user_login(username, password): sql = "SELECT * FROM user WHERE user_name=%s AND password=MD5(%s) AND blocked=0;" cursor.execute(sql, (username, password)) data = cursor.fetchone() + cursor.close() db.close() if data == None: return False @@ -27,6 +29,7 @@ def check_approved(username, password): sql = "SELECT * FROM user WHERE user_name=%s AND password=MD5(%s) AND approved=1;" cursor.execute(sql, (username, password)) data = cursor.fetchone() + cursor.close() db.close() if data == None: return False @@ -40,6 +43,7 @@ def check_user_name(username): sql = "SELECT * FROM user WHERE user_name=%s;" cursor.execute(sql, (username)) data = cursor.fetchone() + cursor.close() db.close() if data == None: return False @@ -54,6 +58,7 @@ def get_user(username): sql = "SELECT * FROM user WHERE user_name='%s' and blocked=0;" % username cursor.execute(sql) data = cursor.fetchone() + cursor.close() db.close() if data == None: return None @@ -68,10 +73,12 @@ def add_user(user): sql = "INSERT into user VALUES(%s, MD5(%s), %s, %s, 0, 0);" try: cursor.execute(sql, (user.userName, user.password, user.auth, user.email)) + cursor.close() db.commit() + db.close() except MySQLdb.Error as e: db.rollback() - return e[1] + logging.error("Error %d: %s", e.args[0], e.args[1]) return "success" return "db connection error" @@ -82,7 +89,9 @@ def add_regular_user(user): sql = "INSERT into user VALUES(%s, MD5(%s), %s, %s, 0, 0);" try: cursor.execute(sql, (user.userName, user.password, user.auth, user.email)) + cursor.close() db.commit() + db.close() except MySQLdb.IntegrityError: db.rollback() return "username taken" @@ -96,10 +105,12 @@ def remove_user(username): sql = "DELETE from user WHERE user_name=%s;" try: cursor.execute(sql, (username)) + cursor.close() db.commit() + db.close() except MySQLdb.Error as e: db.rollback() - return e[1] + logging.error("Error %d: %s", e.args[0], e.args[1]) return "success" return "db connection error" @@ -111,10 +122,12 @@ def update_user(user, username): sql = "UPDATE user SET user_name=%s, auth=%s, email=%s WHERE user_name=%s;" try: cursor.execute(sql, (user.userName, user.auth, user.email, username)) + cursor.close() db.commit() + db.close() except MySQLdb.Error as e: db.rollback() - return e[1] + logging.error("Error %d: %s", e.args[0], e.args[1]) return "success" return "db connection error" @@ -126,10 +139,11 @@ def get_users(): try: cursor.execute(sql) results = cursor.fetchall() + cursor.close() db.close() return results except MySQLdb.Error as e: - return e[1] + logging.error("Error %d: %s", e.args[0], e.args[1]) return "db connection error" def get_blocked_users(): @@ -140,10 +154,11 @@ def get_blocked_users(): try: cursor.execute(sql) results = cursor.fetchall() + cursor.close() db.close() return results except MySQLdb.Error as e: - return e[1] + logging.error("Error %d: %s", e.args[0], e.args[1]) return "db connection error" def block_user(username): @@ -153,10 +168,12 @@ def block_user(username): sql = "UPDATE user SET blocked=%s WHERE user_name=%s;" try: cursor.execute(sql, (1, username)) + cursor.close() db.commit() + db.close() except MySQLdb.Error as e: db.rollback() - return e[1] + logging.error("Error %d: %s", e.args[0], e.args[1]) return "success" return "db connection error" @@ -167,10 +184,12 @@ def unblock_user(username): sql = "UPDATE user SET blocked=%s WHERE user_name=%s;" try: cursor.execute(sql, (0, username)) + cursor.close() db.commit() + db.close() except MySQLdb.Error as e: db.rollback() - return e[1] + logging.error("Error %d: %s", e.args[0], e.args[1]) return "success" return "db connection error" @@ -182,10 +201,11 @@ def get_signup_requests(): try: cursor.execute(sql) results = cursor.fetchall() + cursor.close() db.close() return results except MySQLdb.Error as e: - return e[1] + logging.error("Error %d: %s", e.args[0], e.args[1]) return "db connection error" def approve_user(username): @@ -195,10 +215,12 @@ def approve_user(username): sql = "UPDATE user SET approved=%s WHERE user_name=%s;" try: cursor.execute(sql, (1, username)) + cursor.close() db.commit() + db.close() except MySQLdb.Error as e: db.rollback() - return e[1] + logging.error("Error %d: %s", e.args[0], e.args[1]) return "success" return "db connection error" @@ -211,10 +233,11 @@ def get_heavy_users(): try: cursor.execute(sql) results = cursor.fetchall() + cursor.close() db.close() return results except MySQLdb.Error as e: - return e[1] + logging.error("Error %d: %s", e.args[0], e.args[1]) return "db connection error" def check_if_bandwidth_exceeded(username): @@ -226,6 +249,8 @@ def check_if_bandwidth_exceeded(username): try: cursor.execute(sql, (MONTH, username)) result = cursor.fetchone() + cursor.close() + db.close() if result['sum'] == None: return False elif result['sum'] > conf['max_bandwidth']: @@ -234,5 +259,5 @@ def check_if_bandwidth_exceeded(username): return False return result except MySQLdb.Error as e: - return e[1] + logging.error("Error %d: %s", e.args[0], e.args[1]) return "db connection error" diff --git a/components/core/dl.conf b/components/core/dl.conf index a4737565..a576b79c 100644 --- a/components/core/dl.conf +++ b/components/core/dl.conf @@ -1 +1 @@ -{"max_downloads": 2, "down_folder": "../../downloads", "size_limit":100000, "max_age":1, "min_rating":3, "max_bandwidth": 300000, "aria_server": "ws://localhost:6800/jsonrpc", "tmp_folder": "/tmp/bassa/"} +{"max_downloads": 2, "down_folder": "../../downloads", "size_limit":1000000000000, "max_age":1, "min_rating":3, "max_bandwidth": 1000000000000, "aria_server": "ws://aria2c:6800/jsonrpc", "tmp_folder": "/tmp/bassa/"} diff --git a/components/core/routes/Download.py b/components/core/routes/Download.py index 2debdace..2d2ce9c7 100644 --- a/components/core/routes/Download.py +++ b/components/core/routes/Download.py @@ -13,14 +13,23 @@ conf = get_conf_reader("dl.conf") +processes = [] def start(): try: token = request.headers['key'] if str(token) != SERVER_SECRET_KEY: return "{'error':'not authorized'}", 403 - global p + if len(processes) != 0: + p = processes[0] + processes.clear() + p.terminate() + p.join() + jsonreq = json.dumps({'jsonrpc': '2.0', 'id': 'qwer', 'method': 'aria2.pauseAll'}) + jsonreq = jsonreq.encode('ascii') + c = urllib.request.urlopen(conf['aria_server'], jsonreq) p = Process(target=starter, args=(socketio,)) + processes.append(p) p.start() return '{"status":"' + str(p.pid) + '"}' except Exception as e: diff --git a/components/core/routes/views/Download_endpoints.py b/components/core/routes/views/Download_endpoints.py index b5d30f2e..ab76bc39 100644 --- a/components/core/routes/views/Download_endpoints.py +++ b/components/core/routes/views/Download_endpoints.py @@ -3,11 +3,11 @@ downloadprint = Blueprint("download", __name__) -downloadprint.add_url_rule(rule='/download/start', +downloadprint.add_url_rule(rule='/api/download/start', endpoint='start', view_func=start) -downloadprint.add_url_rule(rule='/download/kill', +downloadprint.add_url_rule(rule='/api/download/kill', endpoint='kill', view_func=kill) diff --git a/ui/src/app/index.js b/ui/src/app/index.js index 12529972..158f98a3 100755 --- a/ui/src/app/index.js +++ b/ui/src/app/index.js @@ -1,10 +1,10 @@ 'use strict'; -import { config } from "../../config"; +// import { constants } from "./config"; angular.module('bassa', ['ngAnimate', 'ngCookies', 'ngTouch', 'ngSanitize', 'ui.router', 'ngMaterial', 'nvd3', 'app']) - .value('BassaUrl', config.API_URL) + .value('BassaUrl', "http://localhost:5000") .config(function ($stateProvider, $httpProvider, $urlRouterProvider, $mdThemingProvider, $mdIconProvider, $qProvider) { From 97ca456dc913e38877bc681684f80a87316829e2 Mon Sep 17 00:00:00 2001 From: K mehant Date: Tue, 12 May 2020 22:58:45 +0530 Subject: [PATCH 14/19] add import --- components/core/DownloadDaemon.py | 1 + 1 file changed, 1 insertion(+) diff --git a/components/core/DownloadDaemon.py b/components/core/DownloadDaemon.py index 36ccf658..9efa0339 100644 --- a/components/core/DownloadDaemon.py +++ b/components/core/DownloadDaemon.py @@ -8,6 +8,7 @@ from DiskMan import * from ConfReader import get_conf_reader import logging +from MinioHandler import * import websocket import sys From cb72797a81266715c98afe12e58df5c8e2c188cf Mon Sep 17 00:00:00 2001 From: K mehant Date: Wed, 13 May 2020 21:12:19 +0530 Subject: [PATCH 15/19] Fix Compression Feature --- components/aria2c/Dockerfile | 4 ++-- docker-compose.dev.yml | 2 +- docker-compose.yml | 2 +- ui/src/app/completed/controllers/TableCtrl.js | 2 +- ui/src/app/completed/services/tableService.js | 5 +++-- ui/src/app/views/table.html | 6 +++--- 6 files changed, 11 insertions(+), 10 deletions(-) diff --git a/components/aria2c/Dockerfile b/components/aria2c/Dockerfile index 266b217f..9c70521f 100644 --- a/components/aria2c/Dockerfile +++ b/components/aria2c/Dockerfile @@ -15,10 +15,10 @@ RUN apk update && \ EXPOSE 6800 # Volume for storing the downloaded data -VOLUME [ "/data" ] +VOLUME [ "/downloads" ] # Changing the current directory to data so that the files get downloaded there -WORKDIR /data +WORKDIR /downloads # Starting the Aria2c RPC server CMD ["aria2c", "--enable-rpc=true", "--rpc-listen-all", "--no-conf=true", "--disable-ipv6", "--console-log-level=debug"] diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 695ebc4f..15bda694 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -34,7 +34,7 @@ services: ports: - 6800:6800 volumes: - - ./downloads:/data + - ./downloads:/downloads restart: always db: diff --git a/docker-compose.yml b/docker-compose.yml index 4b7e6d6d..b9e2f088 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,7 +36,7 @@ services: ports: - 6800:6800 volumes: - - ./downloads:/data + - ./downloads:/downloads restart: always db: diff --git a/ui/src/app/completed/controllers/TableCtrl.js b/ui/src/app/completed/controllers/TableCtrl.js index 1b9b0ce8..65a274f5 100755 --- a/ui/src/app/completed/controllers/TableCtrl.js +++ b/ui/src/app/completed/controllers/TableCtrl.js @@ -44,7 +44,7 @@ $mdToast.hide($scope.zipToast); } if(token) { - window.location.href = BassaUrl + '/api/file?gid=' + downloadGID + "&token=" + token; + window.location.href = BassaUrl + '/api/file?gid=' + downloadGID + "&share=" + token; }else{ ToastService.showToast("Unable to process your download request") } diff --git a/ui/src/app/completed/services/tableService.js b/ui/src/app/completed/services/tableService.js index a5f66b07..3277471d 100755 --- a/ui/src/app/completed/services/tableService.js +++ b/ui/src/app/completed/services/tableService.js @@ -5,7 +5,7 @@ .service('TableService', ['$http', 'BassaUrl', TableService]); function TableService($http, BassaUrl){ - + var getCompletedDownloads = function() { return $http({ method: 'GET', @@ -16,7 +16,8 @@ return $http({ method: 'POST', url : BassaUrl + '/api/compress', - data : {'gid' : gids} + data : {'gid' : gids}, + headers: {'token': window.localStorage.getItem("Token")} }) }; var compressionProgress = function (progressId) { diff --git a/ui/src/app/views/table.html b/ui/src/app/views/table.html index 25e9e353..6a14baaa 100755 --- a/ui/src/app/views/table.html +++ b/ui/src/app/views/table.html @@ -5,7 +5,7 @@ check_circle_outline check_circle - +
@@ -23,7 +23,7 @@ - + @@ -37,7 +37,7 @@

There are no completed downloads yet!

    -
  • +
  • image caption From b4f700544bb5c365bb68be0e648ddf35ab3d48bd Mon Sep 17 00:00:00 2001 From: K mehant Date: Fri, 15 May 2020 18:50:25 +0530 Subject: [PATCH 16/19] Update Travis config and fix Codacy --- .travis.yml | 39 ++++++++++++------- components/aria2c/Dockerfile | 2 +- components/core/DownloadManager.py | 2 +- ui/src/app/completed/services/tableService.js | 34 ++++++++-------- 4 files changed, 44 insertions(+), 33 deletions(-) diff --git a/.travis.yml b/.travis.yml index 04249159..3fa99ce9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,27 +3,36 @@ language: minimal sudo: required -services: - - docker - before_install: - - | - if ! git diff --name-only $TRAVIS_COMMIT_RANGE | grep -qvE '(.md)|(.png)|(.)|^(LICENSE)|^(docs)|^(robots.txt)|^(package-list-yum)|^(package-list-pacman)|^(package-list-dnf)|^(package-list-brew)|^(package-list-aptget)|^(humans.txt)|^(docker-compose.yml)|^(docker-compose.dev.yml)|^(bassa)|^(.gitignore)|^(.gitattributes)|^(bassa-ui.yaml)|^(bassa-ingress.yaml)|^(bassa-api.yaml)|^(deploy_rds.sh)|^(rds-dev.json)|^(template.json)|^(setup-mysql.sh)|^(setup-database.sh)|^(create-schema.sh)|^(Bassa.sql)' - then - echo "Files changed not related to dockerfile, therefore travis build ends" - exit - fi + - sudo apt-get update + - sudo apt-get install -o Dpkg::Options::="--force-confold" --force-yes -y docker-engine + - docker-compose --version + - sudo rm /usr/local/bin/docker-compose + - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose + - chmod +x docker-compose + - sudo mv docker-compose /usr/local/bin + - docker-compose --version + - docker-compose -f docker-compose.dev.yml up + - docker ps script: - - docker build --build-arg VCS_REF=`git rev-parse --short HEAD` --build-arg BUILD_DATE=`date -u +”%Y-%m-%dT%H:%M:%SZ”` -t scoreucsc/bassa-ui ui - - docker build --build-arg VCS_REF=`git rev-parse --short HEAD` --build-arg BUILD_DATE=`date -u +”%Y-%m-%dT%H:%M:%SZ”` -t scoreucsc/bassa components/core + - "travis_wait 5 sleep 800 &" + - docker-compose down + - docker-compose down + - docker build --build-arg VCS_REF=`git rev-parse --short HEAD` --build-arg BUILD_DATE=`date -u +”%Y-%m-%dT%H:%M:%SZ”` -t scoreucsc/bassa-prod-ui ui + - docker build -f Dockerfile.dev --build-arg VCS_REF=`git rev-parse --short HEAD` --build-arg BUILD_DATE=`date -u +”%Y-%m-%dT%H:%M:%SZ”` -t scoreucsc/bassa-dev-ui ui + - docker build -f Dockerfile.prod --build-arg VCS_REF=`git rev-parse --short HEAD` --build-arg BUILD_DATE=`date -u +”%Y-%m-%dT%H:%M:%SZ”` -t scoreucsc/bassa-prod-server components/core + - docker build --build-arg VCS_REF=`git rev-parse --short HEAD` --build-arg BUILD_DATE=`date -u +”%Y-%m-%dT%H:%M:%SZ”` -t scoreucsc/bassa-dev-server components/core + - docker build --build-arg VCS_REF=`git rev-parse --short HEAD` --build-arg BUILD_DATE=`date -u +”%Y-%m-%dT%H:%M:%SZ”` -t scoreucsc/bassa-aria2c components/aria2c + after_success: - if [ "$TRAVIS_BRANCH" == "master" -a "$TRAVIS_PULL_REQUEST" == "false" ]; then docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD"; - docker push scoreucsc/bassa; - curl -X POST 'https://hooks.microbadger.com/images/scoreucsc/bassa/Wbi5zehDMooS3ri2bgC8LETGGuQ='; - docker push scoreucsc/bassa-ui; - curl -X POST 'https://hooks.microbadger.com/images/scoreucsc/bassa-ui/et6pa2aJq-tEvwFoW-PL8ixNxr0='; + docker push scoreucsc/bassa-prod-ui; + docker push scoreucsc/bassa-dev-ui; + docker push scoreucsc/bassa-prod-server; + docker push scoreucsc/bassa-dev-server + docker push scoreucsc/bassa-aria2c; fi diff --git a/components/aria2c/Dockerfile b/components/aria2c/Dockerfile index 9c70521f..f2da66fa 100644 --- a/components/aria2c/Dockerfile +++ b/components/aria2c/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine +FROM alpine:3 # Defining dockerfile maintainer MAINTAINER SCoRe Lab Community diff --git a/components/core/DownloadManager.py b/components/core/DownloadManager.py index 97a5c2ed..e3ef997a 100644 --- a/components/core/DownloadManager.py +++ b/components/core/DownloadManager.py @@ -3,7 +3,7 @@ import time import sys import sqlalchemy.pool as pool -import logging +import logging verbose = False diff --git a/ui/src/app/completed/services/tableService.js b/ui/src/app/completed/services/tableService.js index 3277471d..4b4d003c 100755 --- a/ui/src/app/completed/services/tableService.js +++ b/ui/src/app/completed/services/tableService.js @@ -1,12 +1,12 @@ -(function(){ +(function () { 'use strict'; angular.module('app') .service('TableService', ['$http', 'BassaUrl', TableService]); - function TableService($http, BassaUrl){ - - var getCompletedDownloads = function() { + function TableService($http, BassaUrl) { + + var getCompletedDownloads = function () { return $http({ method: 'GET', url: BassaUrl + '/api/downloads/1' @@ -14,31 +14,33 @@ }; var startCompression = function (gids) { return $http({ - method: 'POST', - url : BassaUrl + '/api/compress', - data : {'gid' : gids}, - headers: {'token': window.localStorage.getItem("Token")} + method: 'POST', + url: BassaUrl + '/api/compress', + data: { 'gid': gids }, + headers: { + 'token': window.localStorage.getItem("Token") + } }) }; var compressionProgress = function (progressId) { return $http({ - method: 'GET', - url : BassaUrl + '/api/compression-progress?gid=' + progressId + method: 'GET', + url: BassaUrl + '/api/compression-progress?gid=' + progressId }) }; - var downloadFromMinio = function(downloadId) { + var downloadFromMinio = function (downloadId) { return $http({ method: 'GET', - url : BassaUrl + '/api/file_from_minio/' + downloadId + url: BassaUrl + '/api/file_from_minio/' + downloadId }) }; return { - getCompletedDownloads : getCompletedDownloads, - startCompression : startCompression, - compressionProgress : compressionProgress, - downloadFromMinio : downloadFromMinio + getCompletedDownloads: getCompletedDownloads, + startCompression: startCompression, + compressionProgress: compressionProgress, + downloadFromMinio: downloadFromMinio }; } })(); From 9d2aa5d83b12aafda5eefca0e3d692fd9117c66d Mon Sep 17 00:00:00 2001 From: K mehant Date: Fri, 15 May 2020 18:54:18 +0530 Subject: [PATCH 17/19] Update Travis config and fix Codacy --- .travis.yml | 2 +- components/core/Starter.py | 2 +- .../core/file_sender/compression_handler.py | 6 ++- components/core/routes/Download.py | 2 +- downloads/data.txt | 2 - ui/src/app/signup/controllers/SignupCtrl.js | 41 +------------------ 6 files changed, 10 insertions(+), 45 deletions(-) delete mode 100644 downloads/data.txt diff --git a/.travis.yml b/.travis.yml index 3fa99ce9..0612d975 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ sudo: required before_install: - sudo apt-get update - - sudo apt-get install -o Dpkg::Options::="--force-confold" --force-yes -y docker-engine + - sudo apt-get install -o Dpkg::Options::="--force-confold" --force-yes -y docker.io - docker-compose --version - sudo rm /usr/local/bin/docker-compose - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose diff --git a/components/core/Starter.py b/components/core/Starter.py index 36c7252d..1440fbd3 100644 --- a/components/core/Starter.py +++ b/components/core/Starter.py @@ -1,6 +1,6 @@ import urllib.request, urllib.error, urllib.parse -req = urllib.request.Request('http://localhost:5000/download/start') +req = urllib.request.Request('http://localhost:5000/api/download/start') req.add_header('key', '123456789') resp = urllib.request.urlopen(req) content = resp.read() diff --git a/components/core/file_sender/compression_handler.py b/components/core/file_sender/compression_handler.py index 57638f1e..8ff1754a 100644 --- a/components/core/file_sender/compression_handler.py +++ b/components/core/file_sender/compression_handler.py @@ -14,10 +14,11 @@ def compress(file_list, file_name): timestamp = time.time() status = insert_compression_process(comp_id=file_name, start_time=timestamp, completed_time=timestamp, is_deleted=0) zip_path = get_zip_path(file_name) - zip_file = zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) + zip_file = zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_STORED) set_compression_progress(comp_id=file_name, status=status, completed_time=timestamp) for file in file_list: if os.path.exists(file): + logging.info("file: %s", file) zip_file.write(file, basename(file)) zip_file.close() logging.info('Compression of files is done, written to following file {0}'.format(zip_path)) @@ -55,7 +56,9 @@ def file_name_formatter(file_name): if __name__ == "__main__": logging.basicConfig(filename="./debug-server.log", level=logging.DEBUG) gid_list = sys.argv + logging.info("before pop %s", gid_list) gid_list.pop(0) # remove file name as an argument + logging.info("after pop %s", gid_list) gid_list.sort() file_path_list = [] ugid_of_compressed_file = '' @@ -64,4 +67,5 @@ def file_name_formatter(file_name): id = get_id_from_gid(elem) download_path = get_download_path(id) file_path_list.append(download_path) + logging.info(file_path_list) compress(file_path_list, ugid_of_compressed_file) diff --git a/components/core/routes/Download.py b/components/core/routes/Download.py index 2d2ce9c7..756cc555 100644 --- a/components/core/routes/Download.py +++ b/components/core/routes/Download.py @@ -27,7 +27,7 @@ def start(): p.join() jsonreq = json.dumps({'jsonrpc': '2.0', 'id': 'qwer', 'method': 'aria2.pauseAll'}) jsonreq = jsonreq.encode('ascii') - c = urllib.request.urlopen(conf['aria_server'], jsonreq) + urllib.request.urlopen(conf['aria_server'], jsonreq) p = Process(target=starter, args=(socketio,)) processes.append(p) p.start() diff --git a/downloads/data.txt b/downloads/data.txt deleted file mode 100644 index 033229c4..00000000 --- a/downloads/data.txt +++ /dev/null @@ -1,2 +0,0 @@ -This folder contains downloaded data from Aria2c -which is mapped to both Bassa API container and Aria2c RPC server \ No newline at end of file diff --git a/ui/src/app/signup/controllers/SignupCtrl.js b/ui/src/app/signup/controllers/SignupCtrl.js index 6f31f615..4bd067c2 100755 --- a/ui/src/app/signup/controllers/SignupCtrl.js +++ b/ui/src/app/signup/controllers/SignupCtrl.js @@ -13,7 +13,6 @@ }; $scope.incorrectCredentials = false; - $scope.errorMsg = ''; var validate = function(obj) { for (var key in obj) { @@ -47,42 +46,6 @@ ToastService.showToast('Please ensure entered details are correct'); } }; + }; - - function passwordValid() { - $scope.loginForm.password.$valid = true; - $scope.loginForm.password.$dirty = false; - $scope.loginForm.password.$invalid = false; - } - function passwordInValid() { - $scope.loginForm.password.$valid = false; - $scope.loginForm.password.$invalid = true; - $scope.loginForm.password.$dirty = true; - } - - $scope.validatePwd = () => { - const lowerCaseLetters = /[a-z]/g; - const upperCaseLetters = /[A-Z]/g; - const numbers = /[0-9]/g; - if (typeof $scope.user.password !== 'undefined') { - if ($scope.user.password.length < 8) { - passwordInValid(); - $scope.errorMsg = 'Password must contains Minimum 8 characters'; - } else if (!$scope.user.password.match(upperCaseLetters)) { - passwordInValid(); - $scope.errorMsg = 'Password must contains a uppercase letter'; - } else if (!$scope.user.password.match(lowerCaseLetters)) { - passwordInValid(); - $scope.errorMsg = 'Password must contains a lowercase letter'; - } else if (!$scope.user.password.match(numbers)) { - passwordInValid(); - $scope.errorMsg = 'Password must contains a number'; - } else { - passwordValid(); - $scope.errorMsg = ''; - } - } - }; - } - -})(); +})(); \ No newline at end of file From 4b112e1b837c2972b2fb748482e625c86ed2f5eb Mon Sep 17 00:00:00 2001 From: K mehant Date: Fri, 15 May 2020 18:59:18 +0530 Subject: [PATCH 18/19] Update Travis config and fix Codacy --- .travis.yml | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0612d975..4713501b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,22 +3,11 @@ language: minimal sudo: required -before_install: - - sudo apt-get update - - sudo apt-get install -o Dpkg::Options::="--force-confold" --force-yes -y docker.io - - docker-compose --version - - sudo rm /usr/local/bin/docker-compose - - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose - - chmod +x docker-compose - - sudo mv docker-compose /usr/local/bin - - docker-compose --version - - docker-compose -f docker-compose.dev.yml up - - docker ps +services: + - docker + script: - - "travis_wait 5 sleep 800 &" - - docker-compose down - - docker-compose down - docker build --build-arg VCS_REF=`git rev-parse --short HEAD` --build-arg BUILD_DATE=`date -u +”%Y-%m-%dT%H:%M:%SZ”` -t scoreucsc/bassa-prod-ui ui - docker build -f Dockerfile.dev --build-arg VCS_REF=`git rev-parse --short HEAD` --build-arg BUILD_DATE=`date -u +”%Y-%m-%dT%H:%M:%SZ”` -t scoreucsc/bassa-dev-ui ui - docker build -f Dockerfile.prod --build-arg VCS_REF=`git rev-parse --short HEAD` --build-arg BUILD_DATE=`date -u +”%Y-%m-%dT%H:%M:%SZ”` -t scoreucsc/bassa-prod-server components/core From 4a9439e47ed205ae51550d44ab21f5e14805ef5c Mon Sep 17 00:00:00 2001 From: K mehant Date: Fri, 15 May 2020 19:37:20 +0530 Subject: [PATCH 19/19] Update Travis config and fix Codacy --- .travis.yml | 6 +++--- ui/src/app/completed/services/tableService.js | 3 --- ui/src/app/signup/controllers/SignupCtrl.js | 4 ++-- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4713501b..4290ba94 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,14 +9,14 @@ services: script: - docker build --build-arg VCS_REF=`git rev-parse --short HEAD` --build-arg BUILD_DATE=`date -u +”%Y-%m-%dT%H:%M:%SZ”` -t scoreucsc/bassa-prod-ui ui - - docker build -f Dockerfile.dev --build-arg VCS_REF=`git rev-parse --short HEAD` --build-arg BUILD_DATE=`date -u +”%Y-%m-%dT%H:%M:%SZ”` -t scoreucsc/bassa-dev-ui ui - - docker build -f Dockerfile.prod --build-arg VCS_REF=`git rev-parse --short HEAD` --build-arg BUILD_DATE=`date -u +”%Y-%m-%dT%H:%M:%SZ”` -t scoreucsc/bassa-prod-server components/core + - docker build -f ui/Dockerfile.dev --build-arg VCS_REF=`git rev-parse --short HEAD` --build-arg BUILD_DATE=`date -u +”%Y-%m-%dT%H:%M:%SZ”` -t scoreucsc/bassa-dev-ui ui + - docker build -f components/core/Dockerfile.prod --build-arg VCS_REF=`git rev-parse --short HEAD` --build-arg BUILD_DATE=`date -u +”%Y-%m-%dT%H:%M:%SZ”` -t scoreucsc/bassa-prod-server components/core - docker build --build-arg VCS_REF=`git rev-parse --short HEAD` --build-arg BUILD_DATE=`date -u +”%Y-%m-%dT%H:%M:%SZ”` -t scoreucsc/bassa-dev-server components/core - docker build --build-arg VCS_REF=`git rev-parse --short HEAD` --build-arg BUILD_DATE=`date -u +”%Y-%m-%dT%H:%M:%SZ”` -t scoreucsc/bassa-aria2c components/aria2c after_success: - - if [ "$TRAVIS_BRANCH" == "master" -a "$TRAVIS_PULL_REQUEST" == "false" ]; then + - if [ "$TRAVIS_BRANCH" == "develop" -a "$TRAVIS_PULL_REQUEST" == "true" ]; then docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD"; docker push scoreucsc/bassa-prod-ui; docker push scoreucsc/bassa-dev-ui; diff --git a/ui/src/app/completed/services/tableService.js b/ui/src/app/completed/services/tableService.js index 4b4d003c..72e6e0ab 100755 --- a/ui/src/app/completed/services/tableService.js +++ b/ui/src/app/completed/services/tableService.js @@ -17,9 +17,6 @@ method: 'POST', url: BassaUrl + '/api/compress', data: { 'gid': gids }, - headers: { - 'token': window.localStorage.getItem("Token") - } }) }; var compressionProgress = function (progressId) { diff --git a/ui/src/app/signup/controllers/SignupCtrl.js b/ui/src/app/signup/controllers/SignupCtrl.js index 4bd067c2..ce466ad6 100755 --- a/ui/src/app/signup/controllers/SignupCtrl.js +++ b/ui/src/app/signup/controllers/SignupCtrl.js @@ -46,6 +46,6 @@ ToastService.showToast('Please ensure entered details are correct'); } }; - }; + } -})(); \ No newline at end of file +})();
#{{data.download_name}} {{data.size}}