From 7b928e1d7b370b5d0e369f5f6a67f909ad7b1427 Mon Sep 17 00:00:00 2001 From: pamepeixinho Date: Fri, 10 Nov 2017 15:29:37 -0200 Subject: [PATCH 1/3] first take in split rest and websocket --- src/Controllers/EvaluatorController.py | 8 ++-- src/FileExamples/ExampleSort.py | 3 ++ src/WebSocket.py | 59 ++++++++++++++++++++++++++ src/server.py | 45 +------------------- 4 files changed, 68 insertions(+), 47 deletions(-) create mode 100644 src/WebSocket.py diff --git a/src/Controllers/EvaluatorController.py b/src/Controllers/EvaluatorController.py index 2f75999..6048f56 100644 --- a/src/Controllers/EvaluatorController.py +++ b/src/Controllers/EvaluatorController.py @@ -37,7 +37,7 @@ def __init__(self, complex_network=None): self.complex_network = complex_network self.nlp_module = NlpModule(weight=5, complex_network=complex_network) self.low_coupling_module = LowCouplingModule(weight=5) - self.understanding_module = UnderstandingModule(weight=1) + self.understanding_module = UnderstandingModule(weight=5) @staticmethod def get_recommendation_code(request_id, query, libs, comments, language): @@ -105,9 +105,9 @@ def evaluate_codes(self, code, idx, idy, input_bus): nlp_score = self.nlp_module.evaluate_code(input_bus_vo=input_bus, search_result_id=idx, code_id=idy) - print('low_coupling_score', low_coupling_score / self.low_coupling_module.weight) - print('understanding_score', understanding_score / self.understanding_module.weight) - print('nlp_score', nlp_score / self.nlp_module.weight) + # print('low_coupling_score', low_coupling_score / self.low_coupling_module.weight) + # print('understanding_score', understanding_score / self.understanding_module.weight) + # print('nlp_score', nlp_score / self.nlp_module.weight) sum_weight = (self.low_coupling_module.weight + self.understanding_module.weight + self.nlp_module.weight) print('----------') final_score = (low_coupling_score + understanding_score + nlp_score) / sum_weight diff --git a/src/FileExamples/ExampleSort.py b/src/FileExamples/ExampleSort.py index 9f1e58f..df77d84 100644 --- a/src/FileExamples/ExampleSort.py +++ b/src/FileExamples/ExampleSort.py @@ -32,6 +32,9 @@ def __save_file(self, filename=None): def __reading_file(self): # Return a file with list of lines + # Open the file for reading. + with open('my_file.txt', 'r') as infile: + data = infile.read() # Read the contents of the file into memory. diff --git a/src/WebSocket.py b/src/WebSocket.py new file mode 100644 index 0000000..b122312 --- /dev/null +++ b/src/WebSocket.py @@ -0,0 +1,59 @@ +from flask import Flask, request, json +from flask_socketio import SocketIO +import requests +from logging.config import fileConfig +import logging + +from Controllers.EvaluatorController import EvaluatorController + +# fileConfig('logging.conf') +# log = logging.getLogger(__name__) +from Modules.Concepts.ComplexNetwork import ComplexNetwork + +app = Flask(__name__, static_folder='') +app.config['SECRET_KEY'] = '@server-secret' +# socketio = SocketIO(app, allow_upgrades=True, engineio_logger=log, logger=log) + +socketio = SocketIO(app, allow_upgrades=True) +complex_network = ComplexNetwork() +evaluator_controller = EvaluatorController(complex_network=complex_network) + + +class Socket: + """ + Class used to emit answer to specific client + """ + + def __init__(self, sid): + self.sid = sid + self.connected = True + + # Emits data to a socket's unique room + def emit(self, event, data): + print('going to emit to', self.sid) + socketio.emit(event, data, room=self.sid, namespace='/code-recommendations') + + +@socketio.on('connect') +def connect(): + print('CONECTADO A UM CLIENTE ', request.sid) + + +@socketio.on('getCodes', namespace='/code-recommendations') +def get_recommendation_codes(data): + data = json.loads(data) + print(data) + evaluator_controller.get_recommendation_code(request_id=request.sid, + language=data['language'], + query=data['query'], + comments=data['comments'], + libs=data['libs']) + + +def run_server(): + socketio.run(app, host='0.0.0.0', port=10443, threaded=True) + + +if __name__ == "__main__": + socketio.start_background_task(run_server) + # socketio.run(app, host='0.0.0.0', port=10443, threaded=True) diff --git a/src/server.py b/src/server.py index 2c11bf9..77d313d 100644 --- a/src/server.py +++ b/src/server.py @@ -13,29 +13,13 @@ from Modules.Concepts.ComplexNetwork import ComplexNetwork app = Flask(__name__, static_folder='') -app.config['SECRET_KEY'] = '@server-secret' +# app.config['SECRET_KEY'] = '@server-secret' # socketio = SocketIO(app, allow_upgrades=True, engineio_logger=log, logger=log) -socketio = SocketIO(app, allow_upgrades=True) complex_network = ComplexNetwork() evaluator_controller = EvaluatorController(complex_network=complex_network) -class Socket: - """ - Class used to emit answer to specific client - """ - - def __init__(self, sid): - self.sid = sid - self.connected = True - - # Emits data to a socket's unique room - def emit(self, event, data): - print('going to emit to', self.sid) - socketio.emit(event, data, room=self.sid, namespace='/code-recommendations') - - @app.route('/') def index(): return 'Hello, World new version!' @@ -81,31 +65,6 @@ def get_complex_network_cluster(): return resp -# TODO: maybe use on connect -@socketio.on('connect') -def connect(): - print('connectttsssss') - - -@socketio.on('getCodes', namespace='/code-recommendations') -def get_recommendation_codes(data): - data = json.loads(data) - print(data) - evaluator_controller.get_recommendation_code(request_id=request.sid, - language=data['language'], - query=data['query'], - comments=data['comments'], - libs=data['libs']) - - -def emit_code_recommendations(request_id, data): - Socket(request_id).emit('recommendationCodes', data) - - -def run_server(): - socketio.run(app, host='0.0.0.0', port=10443, threaded=True) - - if __name__ == "__main__": - socketio.start_background_task(run_server) + app.run(host='0.0.0.0', port=10442) # socketio.run(app, host='0.0.0.0', port=10443, threaded=True) From 5182823d5041b22d3863ab11fee70e45e5eb61ad Mon Sep 17 00:00:00 2001 From: pamepeixinho Date: Fri, 10 Nov 2017 16:10:36 -0200 Subject: [PATCH 2/3] trying --- src/Controllers/EvaluatorController.py | 6 +++--- src/requests.py | 7 +++++++ src/server.py | 18 +++--------------- src/{WebSocket.py => webSocket.py} | 23 ++++++++++------------- 4 files changed, 23 insertions(+), 31 deletions(-) create mode 100644 src/requests.py rename src/{WebSocket.py => webSocket.py} (86%) diff --git a/src/Controllers/EvaluatorController.py b/src/Controllers/EvaluatorController.py index 6048f56..0804c5f 100644 --- a/src/Controllers/EvaluatorController.py +++ b/src/Controllers/EvaluatorController.py @@ -10,7 +10,7 @@ SCRIPT_DIR = os.path.dirname(os.path.realpath(os.path.join(os.getcwd(), os.path.expanduser(__file__)))) sys.path.append(os.path.normpath(os.path.join(SCRIPT_DIR, PACKAGE_PARENT))) -import server +import requests from Modules.NlpModule import NlpModule from Modules.LowCouplingModule import LowCouplingModule from Modules.UnderstandingModule import UnderstandingModule @@ -50,7 +50,7 @@ def get_recommendation_code(request_id, query, libs, comments, language): data = request_code.toRequestJSON() print(data) RequestDB().add(request_code) - server.get_source_codes(data=data) + requests.get_source_codes(data=data) @staticmethod def init_get_recommendation_code_with_mocked_data(): @@ -65,7 +65,7 @@ def init_get_recommendation_code_with_mocked_data(): data = request_code.toRequestJSON() print(data) RequestDB().add(request_code) - server.get_source_codes(data=data) + requests.get_source_codes(data=data) def evaluate_search_codes(self, request): max_score = 0 diff --git a/src/requests.py b/src/requests.py new file mode 100644 index 0000000..4ee3489 --- /dev/null +++ b/src/requests.py @@ -0,0 +1,7 @@ +import requests + +def get_source_codes(data): + url = 'http://0.0.0.0:1111/crawl' + headers = {'Content-Type': 'application/json'} + print('going to request new version') + requests.request(url=url, method='GET', data=data, headers=headers) \ No newline at end of file diff --git a/src/server.py b/src/server.py index 77d313d..a4bcf6a 100644 --- a/src/server.py +++ b/src/server.py @@ -1,10 +1,8 @@ import flask import time from flask import Flask, request, json -from flask_socketio import SocketIO -import requests -from logging.config import fileConfig -import logging +# from logging.config import fileConfig +# import logging from Controllers.EvaluatorController import EvaluatorController @@ -13,8 +11,6 @@ from Modules.Concepts.ComplexNetwork import ComplexNetwork app = Flask(__name__, static_folder='') -# app.config['SECRET_KEY'] = '@server-secret' -# socketio = SocketIO(app, allow_upgrades=True, engineio_logger=log, logger=log) complex_network = ComplexNetwork() evaluator_controller = EvaluatorController(complex_network=complex_network) @@ -34,13 +30,6 @@ def source_codes(): return json.dumps({'success': True}) -def get_source_codes(data): - url = 'http://0.0.0.0:1111/crawl' - headers = {'Content-Type': 'application/json'} - print('going to request new version') - requests.request(url=url, method='GET', data=data, headers=headers) - - @app.route('/train-network', methods=['POST']) def train_network(): print('Train Start') @@ -66,5 +55,4 @@ def get_complex_network_cluster(): if __name__ == "__main__": - app.run(host='0.0.0.0', port=10442) - # socketio.run(app, host='0.0.0.0', port=10443, threaded=True) + app.run(host='0.0.0.0', port=10443) diff --git a/src/WebSocket.py b/src/webSocket.py similarity index 86% rename from src/WebSocket.py rename to src/webSocket.py index b122312..f926a91 100644 --- a/src/WebSocket.py +++ b/src/webSocket.py @@ -4,21 +4,23 @@ from logging.config import fileConfig import logging -from Controllers.EvaluatorController import EvaluatorController - -# fileConfig('logging.conf') -# log = logging.getLogger(__name__) -from Modules.Concepts.ComplexNetwork import ComplexNetwork app = Flask(__name__, static_folder='') app.config['SECRET_KEY'] = '@server-secret' +socketio = SocketIO(app) + # socketio = SocketIO(app, allow_upgrades=True, engineio_logger=log, logger=log) -socketio = SocketIO(app, allow_upgrades=True) +# fileConfig('logging.conf') +# log = logging.getLogger(__name__) + + +from Controllers.EvaluatorController import EvaluatorController +from Modules.Concepts.ComplexNetwork import ComplexNetwork + complex_network = ComplexNetwork() evaluator_controller = EvaluatorController(complex_network=complex_network) - class Socket: """ Class used to emit answer to specific client @@ -50,10 +52,5 @@ def get_recommendation_codes(data): libs=data['libs']) -def run_server(): - socketio.run(app, host='0.0.0.0', port=10443, threaded=True) - - if __name__ == "__main__": - socketio.start_background_task(run_server) - # socketio.run(app, host='0.0.0.0', port=10443, threaded=True) + socketio.run(app, host='0.0.0.0', port=10442, threaded=True) From 288498aeed93f010992bca0af445fce70199c303 Mon Sep 17 00:00:00 2001 From: pamepeixinho Date: Fri, 10 Nov 2017 16:37:13 -0200 Subject: [PATCH 3/3] error in spliting code --- src/Controllers/EvaluatorController.py | 10 +++++----- src/Models/DTO/Client/CodeDTO.py | 2 +- src/__init__.py | 0 src/fake-crawler.py | 5 ++++- src/{requests.py => request.py} | 4 +++- src/server.py | 3 ++- src/testTrainServer.py | 4 ++-- src/webSocket.py | 7 +++++-- 8 files changed, 22 insertions(+), 13 deletions(-) create mode 100644 src/__init__.py rename src/{requests.py => request.py} (88%) diff --git a/src/Controllers/EvaluatorController.py b/src/Controllers/EvaluatorController.py index 0804c5f..e7c2396 100644 --- a/src/Controllers/EvaluatorController.py +++ b/src/Controllers/EvaluatorController.py @@ -2,6 +2,7 @@ import sys from uuid import uuid4 +import request # from flask import json # issue 12 https://github.com/quicksloth/source-code-recommendation-server/issues/11 @@ -10,7 +11,6 @@ SCRIPT_DIR = os.path.dirname(os.path.realpath(os.path.join(os.getcwd(), os.path.expanduser(__file__)))) sys.path.append(os.path.normpath(os.path.join(SCRIPT_DIR, PACKAGE_PARENT))) -import requests from Modules.NlpModule import NlpModule from Modules.LowCouplingModule import LowCouplingModule from Modules.UnderstandingModule import UnderstandingModule @@ -50,7 +50,7 @@ def get_recommendation_code(request_id, query, libs, comments, language): data = request_code.toRequestJSON() print(data) RequestDB().add(request_code) - requests.get_source_codes(data=data) + request.get_source_codes(data=data) @staticmethod def init_get_recommendation_code_with_mocked_data(): @@ -65,9 +65,9 @@ def init_get_recommendation_code_with_mocked_data(): data = request_code.toRequestJSON() print(data) RequestDB().add(request_code) - requests.get_source_codes(data=data) + request.get_source_codes(data=data) - def evaluate_search_codes(self, request): + def evaluate_search_codes(self, request, emitFn): max_score = 0 request_json = request.get_json() @@ -93,7 +93,7 @@ def evaluate_search_codes(self, request): code_results = code_results.toJSON() print(code_results) - server.emit_code_recommendations(request_id, code_results) + emitFn(request_id, code_results) return code_results def evaluate_codes(self, code, idx, idy, input_bus): diff --git a/src/Models/DTO/Client/CodeDTO.py b/src/Models/DTO/Client/CodeDTO.py index 57ae9fa..1146d22 100644 --- a/src/Models/DTO/Client/CodeDTO.py +++ b/src/Models/DTO/Client/CodeDTO.py @@ -1,5 +1,5 @@ import os -import requests +import request from flask import json diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/fake-crawler.py b/src/fake-crawler.py index 9aab0aa..6e4acc5 100644 --- a/src/fake-crawler.py +++ b/src/fake-crawler.py @@ -13,7 +13,9 @@ def hello_world(): # TODO: temporary until all set client and crawler @app.route('/crawl', methods=['GET']) def crawl(): + print('REQUEST AQUI') request_body = request.get_json() + print('request_body', request_body) url = 'http://0.0.0.0:10443/source-codes' data = { "requestID": request_body.get('requestID'), @@ -237,8 +239,9 @@ def crawl(): # "Days of the Week\nMonday\nTuesday\nWednesday\nThursday\nFriday\nSaturday\nSunday\n"]}]} headers = {'Content-Type': 'application/json'} + print("POST") requests.post(url=url, data=json.dumps(data), headers=headers) - + print("POST") return json.dumps({'success': True}) diff --git a/src/requests.py b/src/request.py similarity index 88% rename from src/requests.py rename to src/request.py index 4ee3489..401c1f2 100644 --- a/src/requests.py +++ b/src/request.py @@ -1,7 +1,9 @@ import requests + def get_source_codes(data): url = 'http://0.0.0.0:1111/crawl' headers = {'Content-Type': 'application/json'} print('going to request new version') - requests.request(url=url, method='GET', data=data, headers=headers) \ No newline at end of file + requests.request(url=url, method='GET', data=data, headers=headers) + print('REQUEST') diff --git a/src/server.py b/src/server.py index a4bcf6a..396edf3 100644 --- a/src/server.py +++ b/src/server.py @@ -9,6 +9,7 @@ # fileConfig('logging.conf') # log = logging.getLogger(__name__) from Modules.Concepts.ComplexNetwork import ComplexNetwork +import webSocket app = Flask(__name__, static_folder='') @@ -24,7 +25,7 @@ def index(): @app.route('/source-codes', methods=['POST']) def source_codes(): start = time.time() - evaluator_controller.evaluate_search_codes(request) + evaluator_controller.evaluate_search_codes(request, webSocket.emit_code_recommendations) end = time.time() print('Receive Source code and evaluate took', (end - start), 'seconds') return json.dumps({'success': True}) diff --git a/src/testTrainServer.py b/src/testTrainServer.py index b058384..a1c93eb 100644 --- a/src/testTrainServer.py +++ b/src/testTrainServer.py @@ -1,4 +1,4 @@ -import requests +import request from flask import json @@ -11,7 +11,7 @@ def train_database(): 'Python', 'sites stackoverflow lore lorem', 'lorem ipsumb amdksd'], } print('going to request') - requests.post(url=url, data=json.dumps(data), headers=headers) + request.post(url=url, data=json.dumps(data), headers=headers) train_database() diff --git a/src/webSocket.py b/src/webSocket.py index f926a91..0a0e1b8 100644 --- a/src/webSocket.py +++ b/src/webSocket.py @@ -1,10 +1,8 @@ from flask import Flask, request, json from flask_socketio import SocketIO -import requests from logging.config import fileConfig import logging - app = Flask(__name__, static_folder='') app.config['SECRET_KEY'] = '@server-secret' socketio = SocketIO(app) @@ -21,6 +19,7 @@ complex_network = ComplexNetwork() evaluator_controller = EvaluatorController(complex_network=complex_network) + class Socket: """ Class used to emit answer to specific client @@ -52,5 +51,9 @@ def get_recommendation_codes(data): libs=data['libs']) +def emit_code_recommendations(request_id, data): + Socket(request_id).emit('recommendationCodes', data) + + if __name__ == "__main__": socketio.run(app, host='0.0.0.0', port=10442, threaded=True)