Merge pull request #918 from kmehant/test_prod

Fix Server errors for download and compression feature and other required fixes

.travis.yml

@@ -6,24 +6,22 @@ sudo: required
 services:
   - docker
 
-before_install:
-  - |
-      if ! git diff --name-only $TRAVIS_COMMIT_RANGE | grep -qvE '(.md)|(.png)|(.)|^(LICENSE)|^(docs)|^(robots.txt)|^(package-list-yum)|^(package-list-pacman)|^(package-list-dnf)|^(package-list-brew)|^(package-list-aptget)|^(humans.txt)|^(docker-compose.yml)|^(docker-compose.dev.yml)|^(bassa)|^(.gitignore)|^(.gitattributes)|^(bassa-ui.yaml)|^(bassa-ingress.yaml)|^(bassa-api.yaml)|^(deploy_rds.sh)|^(rds-dev.json)|^(template.json)|^(setup-mysql.sh)|^(setup-database.sh)|^(create-schema.sh)|^(Bassa.sql)'
-      then
-        echo "Files changed not related to dockerfile, therefore travis build ends"
-        exit
-      fi
 
 script: 
-  - docker build --build-arg VCS_REF=`git rev-parse --short HEAD`   --build-arg BUILD_DATE=`date -u +”%Y-%m-%dT%H:%M:%SZ”` -t scoreucsc/bassa-ui  ui
-  - docker build --build-arg VCS_REF=`git rev-parse --short HEAD`   --build-arg BUILD_DATE=`date -u +”%Y-%m-%dT%H:%M:%SZ”` -t scoreucsc/bassa   components/core
+  - docker build --build-arg VCS_REF=`git rev-parse --short HEAD`   --build-arg BUILD_DATE=`date -u +”%Y-%m-%dT%H:%M:%SZ”` -t scoreucsc/bassa-prod-ui ui
+  - docker build -f ui/Dockerfile.dev --build-arg VCS_REF=`git rev-parse --short HEAD`   --build-arg BUILD_DATE=`date -u +”%Y-%m-%dT%H:%M:%SZ”` -t scoreucsc/bassa-dev-ui ui
+  - docker build -f components/core/Dockerfile.prod --build-arg VCS_REF=`git rev-parse --short HEAD`   --build-arg BUILD_DATE=`date -u +”%Y-%m-%dT%H:%M:%SZ”` -t scoreucsc/bassa-prod-server components/core
+  - docker build --build-arg VCS_REF=`git rev-parse --short HEAD`   --build-arg BUILD_DATE=`date -u +”%Y-%m-%dT%H:%M:%SZ”` -t scoreucsc/bassa-dev-server components/core
+  - docker build --build-arg VCS_REF=`git rev-parse --short HEAD`   --build-arg BUILD_DATE=`date -u +”%Y-%m-%dT%H:%M:%SZ”` -t scoreucsc/bassa-aria2c components/aria2c
+
 
 after_success:
-  - if [ "$TRAVIS_BRANCH" == "master" -a "$TRAVIS_PULL_REQUEST" == "false" ]; then
+  - if [ "$TRAVIS_BRANCH" == "develop" -a "$TRAVIS_PULL_REQUEST" == "true" ]; then
     docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD";
-    docker push scoreucsc/bassa;
-    curl -X POST 'https://hooks.microbadger.com/images/scoreucsc/bassa/Wbi5zehDMooS3ri2bgC8LETGGuQ=';
-    docker push scoreucsc/bassa-ui;
-    curl -X POST 'https://hooks.microbadger.com/images/scoreucsc/bassa-ui/et6pa2aJq-tEvwFoW-PL8ixNxr0=';
+    docker push scoreucsc/bassa-prod-ui;
+    docker push scoreucsc/bassa-dev-ui;
+    docker push scoreucsc/bassa-prod-server;
+    docker push scoreucsc/bassa-dev-server
+    docker push scoreucsc/bassa-aria2c;
     fi
 

components/aria2c/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:edge
+FROM alpine:3
 
 # Defining dockerfile maintainer
 MAINTAINER SCoRe Lab Community <[email protected]>
@@ -15,13 +15,13 @@ RUN apk update && \
 EXPOSE 6800
 
 # Volume for storing the downloaded data
-VOLUME [ "/data" ]   
+VOLUME [ "/downloads" ]   
 
 # Changing the current directory to data so that the files get downloaded there
-WORKDIR /data
+WORKDIR /downloads
 
 # Starting the Aria2c RPC server
-CMD ["aria2c", "--enable-rpc", "--rpc-listen-all"]
+CMD ["aria2c", "--enable-rpc=true", "--rpc-listen-all", "--no-conf=true", "--disable-ipv6", "--console-log-level=debug"]
 
 # Labels for dockerfile information
 LABEL scorelab.bassa.name="Aria2c RPC server" \

components/core/DownloadDaemon.py

@@ -7,8 +7,8 @@
 from EMail import send_mail
 from DiskMan import *
 from ConfReader import get_conf_reader
-from MinioHandler import *
 import logging
+from MinioHandler import *
 
 import websocket
 import sys
@@ -86,6 +86,9 @@ def worker(self):
             data = json.loads(message)
             print(threading.current_thread().name,data)
             print ("TOP LEVEL DATA", data)
+            if 'error' in data:
+                logging.error("Aria2c error: %s", data['error'])
+                return
             if 'id' in data and data['id'] == "act":
                 toBeDownloaded = get_to_download()
                 if toBeDownloaded:
@@ -178,6 +181,7 @@ def add_uri(ws, download):
 def get_status(ws, id=None, gid=None):
     if id:
         gid = get_gid_from_id(id)
+    gid = str(gid)
     msg = JSONer("stat", 'aria2.tellStatus', [gid, ['gid', 'files']])
     print ("Getting status")
     ws.send(msg)
@@ -236,7 +240,7 @@ def on_message(ws, message):
     messageQueue.put(message)
 
 def on_error(ws, error):
-    print(error)
+    logging.error(error)
 
 
 def on_close(ws):
@@ -252,6 +256,7 @@ def starter(socket):
     sc = socket
     remove_files(conf['max_age'], conf['min_rating'])
     folder_size=get_size(conf['down_folder'])
+    logging.info("SERVER: folder_size: %d", folder_size)
     websocket.enableTrace(False)
     ws = websocket.WebSocketApp(conf['aria_server'],
                                 on_message=on_message,
@@ -268,4 +273,3 @@ def starter(socket):
 
 
 
-