From 28e8432ebe0d92122f4d13d9bdb3505cd1e07263 Mon Sep 17 00:00:00 2001 From: Andre Emmerichs Date: Sat, 26 Nov 2016 15:03:46 +0100 Subject: [PATCH 01/43] Adding Prototype of ncloc module Created new branch for project practical Adde first idea of ncloc module, need to add tests & more languages later --- modules/ncloc/__init__.py | 108 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 modules/ncloc/__init__.py diff --git a/modules/ncloc/__init__.py b/modules/ncloc/__init__.py new file mode 100644 index 00000000..57ab3dd6 --- /dev/null +++ b/modules/ncloc/__init__.py @@ -0,0 +1,108 @@ +import os +import json + +config = { + 'wantdiff': True, + 'wantsfiles': True, + 'threadsafe': True, + 'behavior': { + 'creates': [['resource', 'ncloc']], + 'uses': [['resource', 'lang']] + } +} + +# this is the actual logic of the module +def count_non_comment_lines(context, f): + #datastructure for languages and their corresponding comment symbols + languages = {'Java': {'Single': ['//'], 'BlockStart': ['/*'], 'BlockEnd': ['*/']}, + 'Python': {'Single': ['#'], 'BlockStart': ["'''"], 'BlockEnd': ["'''"]}, + 'Haskell': {'Single': ['--'], 'BlockStart': ["{-"], 'BlockEnd': ["-}"]}} + #get file + source = context.get_primary_resource(f) + #get language of file + lang = context.get_derived_resource(f,'lang') + + ncloc = 0 + block = False + lineNumber = 0 + blockLineBegin = -1 # only needed when block start and end symbol are equal + + #check if language is in datastructure + if languages.get(lang) != None : + #check if langauges have block/singe-line comments + singleExists = languages[lang].get("Single") != None + blockExists = languages[lang].get("BlockStart") != None + + if blockExists: + equalBlockSymbols = languages[lang].get("BlockStart") == languages[lang].get("BlockEnd") + #iterate over all lines + for line in source.split('\n'): + lineNumber = lineNumber +1 + + if blockExists: + #check if symbols are one of the given comment symbols for block comments + for blockstart in languages[lang].get("BlockStart"): + line = line.replace("\t","").replace(" ","").replace("\n","") + if line[0:len(blockstart)] == blockstart and block == False: + block = True + #set marker for blockLineBegin, if block symbols are equal, to prevent reading of + #block start as block end later + if equalBlockSymbols: + if len((line)) == len(blockstart): + blockLineBegin = lineNumber + + + + #if we are not in a Block comment, check if it's a single line comment... + singleCheck = False + if block == False: + if singleExists: + for single in languages[lang]["Single"]: + if len(line)>len(single): + if line[0:len(single)] == single: + singleCheck = True + #...if not increment sum of non comment lines of code + if singleCheck == False and block == False: + ncloc = ncloc + 1 + + + #check for block end symbols + if blockExists: + for blockend in languages[lang]["BlockEnd"]: + #invert line + line = line[::-1] + #check if the first found signs are now the inverted signs of the endblock + #of a comment (because we inverted the whole line before) + if line[0:len(blockend)] == (blockend)[::-1] and block == True: + #if block start and end sign are equal, take care that you do not read + #a block-start as a block-end + if equalBlockSymbols: + if not blockLineBegin == lineNumber: + block = False + else: + block = False + + return ncloc + +def update_file(context, f): + # reads the content of the file (primary resource) + try: + ncloc = count_non_comment_lines(context, f) + + context.write_derived_resource(f, ncloc, 'ncloc') + except UnicodeDecodeError: + context.write_derived_resource(f, 0, 'ncloc') + +def remove_file(context, f): + context.remove_derived_resource(f, 'ncloc') + +def run(context, change): + # dispatch the modified file + if change['type'] == 'NEW_FILE': + update_file(context, change['file']) + + elif change['type'] == 'FILE_CHANGED': + update_file(context, change['file']) + + else: + remove_file(context, change['file']) From 3f3aa9f517e82f060b705c133ecdadb049f93983 Mon Sep 17 00:00:00 2001 From: Andre Emmerichs Date: Fri, 2 Dec 2016 16:41:16 +0100 Subject: [PATCH 02/43] Added prototype for comment extractor Prototype for comment extactor, added "beta" version to support development of other modules. Improvements will come in the next few days, but JSON structure will keep equal over development --- modules/comments/__init__.py | 109 +++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 modules/comments/__init__.py diff --git a/modules/comments/__init__.py b/modules/comments/__init__.py new file mode 100644 index 00000000..666c697a --- /dev/null +++ b/modules/comments/__init__.py @@ -0,0 +1,109 @@ +import os +import json + +config = { + 'wantdiff': True, + 'wantsfiles': True, + 'threadsafe': True, + 'behavior': { + 'creates': [['resource', 'comments']], + 'uses': [['resource', 'lang']] + } +} + +# this is the actual logic of the module +def extract_comments(context, f): + #datastructure for languages and their corresponding comment symbols + languages = {'Java': {'Single': ['//'], 'BlockStart': ['/*'], 'BlockEnd': ['*/']}, + 'Python': {'Single': ['#'], 'BlockStart': ["'''"], 'BlockEnd': ["'''"]}, + 'Haskell': {'Single': ['--'], 'BlockStart': ["{-"], 'BlockEnd': ["-}"]}} + #init of comments array + comments = [] + + #get file & get language of file + source = context.get_primary_resource(f) + lang = context.get_derived_resource(f,'lang') + + + #block symbols if we are actually in a block-comment + block = False + #number of actual array entry + entry = 0 + + #check if language is in datastructure + if languages.get(lang) != None : + #check if languages have block/singe-line comments + singleExists = languages[lang].get("Single") != None + blockExists = languages[lang].get("BlockStart") != None + + #iterate over all lines + for line in source.split('\n'): + + if blockExists: + #check if we are already in a block + if block == True: + blockendFound = False + for blockend in languages[lang].get("BlockEnd"): + end = line.find(blockend) + #if we find a block-end symbol end the block + if end != -1: + comments[entry] = comments[entry]+'\n'+line[0:end+1] + entry = entry + 1 + block = False + blockendFound = True + break + if blockendFound == False: + comments[entry] = comments[entry]+'\n'+line + #if we are not in a block, try to find a start-symbol + else: + for blockstart in languages[lang].get("BlockStart"): + start = line.find(blockstart) + if start != -1 and block == False: + blockendFound = False + #try to find block-end symbol in same line + for blockend in languages[lang].get("BlockEnd"): + end = line[start+len(blockstart):len(line)].find(blockend) + if end != -1: + comments.append(line[start+len(blockstart):end+1]) + entry = entry + 1 + blockendFound = True + break + #if no block-end symbol has been found, set block to true + if blockendFound == False: + comments.append(line[start+len(blockstart):len(line)]) + block = True + + #check for single-line comment + for single in languages[lang].get("Single"): + if block == False and singleExists: + start = line.find(single) + if line.find(single) != -1: + comments.append(line[start+len(single):len(line)]) + entry = entry + 1 + break + + result = {'comments':comments} + return result + +def update_file(context, f): + # reads the content of the file (primary resource) + try: + comments = extract_comments(context, f) + + context.write_derived_resource(f, comments, 'comments') + except UnicodeDecodeError: + context.write_derived_resource(f, {'comments':[]}, 'comments') + +def remove_file(context, f): + context.remove_derived_resource(f, 'comments') + +def run(context, change): + # dispatch the modified file + if change['type'] == 'NEW_FILE': + update_file(context, change['file']) + + elif change['type'] == 'FILE_CHANGED': + update_file(context, change['file']) + + else: + remove_file(context, change['file']) From 65a89979fd6b1a6e2017098b03a5f78cd58f7d7a Mon Sep 17 00:00:00 2001 From: Andre Emmerichs Date: Wed, 7 Dec 2016 18:48:50 +0100 Subject: [PATCH 03/43] Improved Comments Module Now comments in the same line with code can be found, tests still missing --- modules/comments/__init__.py | 49 +++++++++++++++++++++++++++++------- 1 file changed, 40 insertions(+), 9 deletions(-) diff --git a/modules/comments/__init__.py b/modules/comments/__init__.py index 666c697a..52bf961a 100644 --- a/modules/comments/__init__.py +++ b/modules/comments/__init__.py @@ -39,39 +39,70 @@ def extract_comments(context, f): #iterate over all lines for line in source.split('\n'): - if blockExists: + #positionNumber marks later, where a block has ended and where the rest of the line continues + #lineRest is the rest of the line + positionNumber = 0 + lineRest = line + + #loop gets stopped, when a) we are already in a block and do not find a end symbol + # b) we are not in a block, but do not find a start symbol + while True and blockExists: #check if we are already in a block if block == True: blockendFound = False for blockend in languages[lang].get("BlockEnd"): - end = line.find(blockend) + end = lineRest.find(blockend) #if we find a block-end symbol end the block if end != -1: - comments[entry] = comments[entry]+'\n'+line[0:end+1] + comments[entry] = comments[entry]+'\n'+lineRest[0:end+1] entry = entry + 1 block = False blockendFound = True + #mark number of blockend, to determine where rest of line starts + positionNumber = end+len(blockend) break + #when we find a blockend, add the text to the comments if blockendFound == False: - comments[entry] = comments[entry]+'\n'+line - #if we are not in a block, try to find a start-symbol + comments[entry] = comments[entry]+'\n'+lineRest + #check if positionNumber is already over the length of the string, if not continue with + #the rest of the line... + if positionNumber+1 < len(lineRest) and blockendFound == True: + lineRest = lineRest[positionNumber+1:len(lineRest)] + #...otherwise stop while loop + else: + break + #if not in a block, try to find blockstart symbol else: for blockstart in languages[lang].get("BlockStart"): - start = line.find(blockstart) + start = lineRest.find(blockstart) + blockendFound = False if start != -1 and block == False: blockendFound = False #try to find block-end symbol in same line for blockend in languages[lang].get("BlockEnd"): - end = line[start+len(blockstart):len(line)].find(blockend) + #note that we change here the lineRest to a shorter path, where the blockstart + #symbol is at the beginning + lineRest = lineRest[start:len(lineRest)] + end = lineRest.find(blockend) + #if blockend symbol in same line, take only this part if end != -1: - comments.append(line[start+len(blockstart):end+1]) + comments.append(lineRest[len(blockstart):end]) entry = entry + 1 blockendFound = True + #mark number of blockend, to determine where rest of line starts + positionNumber = end+len(blockend) break #if no block-end symbol has been found, set block to true if blockendFound == False: - comments.append(line[start+len(blockstart):len(line)]) + comments.append(lineRest[start+len(blockstart):len(lineRest)]) block = True + #check if positionNumber is already over the length of the string and if we are in a block, + #if not continue with the rest of the line... + if positionNumber+1 < len(lineRest) and blockendFound == True: + lineRest = lineRest[positionNumber+1:len(lineRest)] + #...otherwise stop while loop + else: + break #check for single-line comment for single in languages[lang].get("Single"): From 7c29c8d10aff3898b1c0a5ec6948a8833568af84 Mon Sep 17 00:00:00 2001 From: Marcel Date: Thu, 8 Dec 2016 17:07:36 +0100 Subject: [PATCH 04/43] Added general information and developerRanks modules --- general_information/readme schema | 8 ++ modules/developerRanks/__init__.py | 69 ++++++++++++++++++ .../__init__.cpython-35.pyc.139678278124328 | Bin 0 -> 2011 bytes 3 files changed, 77 insertions(+) create mode 100644 general_information/readme schema create mode 100644 modules/developerRanks/__init__.py create mode 100644 modules/developerRanks/__pycache__/__init__.cpython-35.pyc.139678278124328 diff --git a/general_information/readme schema b/general_information/readme schema new file mode 100644 index 00000000..a7a497f4 --- /dev/null +++ b/general_information/readme schema @@ -0,0 +1,8 @@ +Readme schema: + +- Headline +- Needed imports +- Dependencies (links to other modules) +- Input / Output behaviour +- Example +- Natural language description diff --git a/modules/developerRanks/__init__.py b/modules/developerRanks/__init__.py new file mode 100644 index 00000000..cfcff690 --- /dev/null +++ b/modules/developerRanks/__init__.py @@ -0,0 +1,69 @@ +from collections import Counter + +import matplotlib.pylot as plt +import seaborn as sns + +config = { + 'wantdiff': False, + 'wantsfiles': False, + 'threadsafe': False, + 'behavior': { + 'uses': [['dump', 'wiki-links']], + 'creates': [['dump', 'developerRanks']] + } +} + +def run(env): + wiki_dump = env.read_dump('wiki-links') + + pages = wiki_dump['wiki']['pages'] + + data = {} + + contributions = filter(lambda p: 'Contribution' == p.get('p', ''), pages) + for c in contributions: + + languages = [c.get('Uses', [])] + languages = [p for language in languages for p in language] + + languages = list(filter(lambda u: u['p'] == 'Language', languages)) + languages = [language['n'].replace('_', ' ') for language in languages] + + contributors = [c.get('DevelopedBy', [])] + contributors = [p['n'] for contributor in contributors for p in contributor] + + + + + for language in languages: + if language in data.keys(): + data2 = data[language] + else: + data2 = {} + for contributor in contributors: + if contributor in data2.keys(): + data2[contributor] = data2[contributor] + 1 + else: + data2[contributor] = 1 + + data[language] = data2 + + env.write_dump('developerRanks', data) + + x = [] + y = [] + for (k,v) in data['Javascript']: + x.append(k) + y.append(v) + sns.barplot(x, y) + sns.plt.show() + + + + + + + + + + diff --git a/modules/developerRanks/__pycache__/__init__.cpython-35.pyc.139678278124328 b/modules/developerRanks/__pycache__/__init__.cpython-35.pyc.139678278124328 new file mode 100644 index 0000000000000000000000000000000000000000..2333b25e19d1361b4a2ee9b410279d8993d7673e GIT binary patch literal 2011 zcma)7T~8ZF6uq;%{=~*^5d=3$ixQ=ks|1I4gsKKrRaK#?QL73SS&L_EHeP$3*)=E# zo)Y!rF+Zb!Xoy4D@kZ2<4SdfqoNtdK&`;zU;Br7DVBx^KsXe3D1?dw|h8aNbqbc}@!l1+L; zbc`L=>3O&H4>C|!n98$gK24N1)eC=`$Y419#?<*S9tNR`O+7i&%9pV}RHk~W&iwNr z(#D&|nDgX(GP|r_1moaI7)-~psSFUDU}{~ea}`E2rGNA>F%X;%KV>wqCKoOK7u^-n z3rqC1pld-_;NVaPL_60GC3pkh5+lEI=(R)Fz;9Xeh)ajO3K5W?VuQSVC`nh>^3@)% z&Y~gGYBUn9<)K9)h?d5siT(u6W;XN(*+dxnooI;F>pbxrKxGI@mKo5t=&HoIg(T?S zKDwos$J&|F%pa+RMywTU^qe)?LI)dUGFb~?v$2D{yd%?i7EKc!oX(RVni?@PTvB(W zlCIF}Sos&}kMQs2yWUweQN4+;2P*9SpraRRoXoU8PJ;1#(%XBwcM<8a(!EI}=O}yc zcKLe!elQJ^et&oN%2fA5e{w4QgO3>>8eMMc`E++bj0S!fAK)RKFN*FUri5k*J;1}r z*8OBVt2i6L4uaWRhhCOJP8jbecnmz+SkQHhv-{LivV_u(0o>I0!yrxu(PVb8iT9Y3 z)#&oh3Zy)y0L2b0RDOt;3{)orb$HSsSP-^*>(a}%MYYJ?bv0)*)zF`g<|yC0uxTzH zw+vK~n=8yXtsq<{lov(r!1}hb?vdTjRKL%6=PE)IF<(H?OkZ!H+eWj6C=%RK{Bula z>feYRz8bWeh5kTw9m`eU;d~5ap}u9<;q zEP`&K%L|_RcQqZey^HogY#(6yL;Q8Tc9>mJ^6OW5){Ln|^A4u>IQ0rs_EBN(FBZw~ z7d~odK7z+P3ojw%Cy2HTb2qIG^5Y!o_ZsT{dqD8j%KF4l&}9ldWDtB;ooq1(AixEi z;Kp)jYZ=OA<&oTZyz@w+?2Q5h?vxB!TKiA2@vVSJ$17g=i~*zy={kpdrpDioK7T4q z8Q&EC)_8nm@tM&5 z_3WpZj|7R9v+iuV8)Cz`i?%8FspIG`F?R%HkUI?3fbY&&GiXyrzF{!35lh~)R^a1? Y%P+kl57^)_`=srb1^>FDCAPi)0JN-n;{X5v literal 0 HcmV?d00001 From 63df99e0be4979b51066aa6b84982089ce23e80a Mon Sep 17 00:00:00 2001 From: Andre Emmerichs Date: Mon, 19 Dec 2016 13:33:43 +0100 Subject: [PATCH 05/43] Improving CommentExtractor & adding tests Split up tests and program in sperate files, renamed folder name --- modules/comments/__init__.py | 140 ---------------------------- modules/extractComments/__init__.py | 12 +++ modules/extractComments/program.py | 134 ++++++++++++++++++++++++++ modules/extractComments/test.py | 131 ++++++++++++++++++++++++++ 4 files changed, 277 insertions(+), 140 deletions(-) delete mode 100644 modules/comments/__init__.py create mode 100644 modules/extractComments/__init__.py create mode 100644 modules/extractComments/program.py create mode 100644 modules/extractComments/test.py diff --git a/modules/comments/__init__.py b/modules/comments/__init__.py deleted file mode 100644 index 52bf961a..00000000 --- a/modules/comments/__init__.py +++ /dev/null @@ -1,140 +0,0 @@ -import os -import json - -config = { - 'wantdiff': True, - 'wantsfiles': True, - 'threadsafe': True, - 'behavior': { - 'creates': [['resource', 'comments']], - 'uses': [['resource', 'lang']] - } -} - -# this is the actual logic of the module -def extract_comments(context, f): - #datastructure for languages and their corresponding comment symbols - languages = {'Java': {'Single': ['//'], 'BlockStart': ['/*'], 'BlockEnd': ['*/']}, - 'Python': {'Single': ['#'], 'BlockStart': ["'''"], 'BlockEnd': ["'''"]}, - 'Haskell': {'Single': ['--'], 'BlockStart': ["{-"], 'BlockEnd': ["-}"]}} - #init of comments array - comments = [] - - #get file & get language of file - source = context.get_primary_resource(f) - lang = context.get_derived_resource(f,'lang') - - - #block symbols if we are actually in a block-comment - block = False - #number of actual array entry - entry = 0 - - #check if language is in datastructure - if languages.get(lang) != None : - #check if languages have block/singe-line comments - singleExists = languages[lang].get("Single") != None - blockExists = languages[lang].get("BlockStart") != None - - #iterate over all lines - for line in source.split('\n'): - - #positionNumber marks later, where a block has ended and where the rest of the line continues - #lineRest is the rest of the line - positionNumber = 0 - lineRest = line - - #loop gets stopped, when a) we are already in a block and do not find a end symbol - # b) we are not in a block, but do not find a start symbol - while True and blockExists: - #check if we are already in a block - if block == True: - blockendFound = False - for blockend in languages[lang].get("BlockEnd"): - end = lineRest.find(blockend) - #if we find a block-end symbol end the block - if end != -1: - comments[entry] = comments[entry]+'\n'+lineRest[0:end+1] - entry = entry + 1 - block = False - blockendFound = True - #mark number of blockend, to determine where rest of line starts - positionNumber = end+len(blockend) - break - #when we find a blockend, add the text to the comments - if blockendFound == False: - comments[entry] = comments[entry]+'\n'+lineRest - #check if positionNumber is already over the length of the string, if not continue with - #the rest of the line... - if positionNumber+1 < len(lineRest) and blockendFound == True: - lineRest = lineRest[positionNumber+1:len(lineRest)] - #...otherwise stop while loop - else: - break - #if not in a block, try to find blockstart symbol - else: - for blockstart in languages[lang].get("BlockStart"): - start = lineRest.find(blockstart) - blockendFound = False - if start != -1 and block == False: - blockendFound = False - #try to find block-end symbol in same line - for blockend in languages[lang].get("BlockEnd"): - #note that we change here the lineRest to a shorter path, where the blockstart - #symbol is at the beginning - lineRest = lineRest[start:len(lineRest)] - end = lineRest.find(blockend) - #if blockend symbol in same line, take only this part - if end != -1: - comments.append(lineRest[len(blockstart):end]) - entry = entry + 1 - blockendFound = True - #mark number of blockend, to determine where rest of line starts - positionNumber = end+len(blockend) - break - #if no block-end symbol has been found, set block to true - if blockendFound == False: - comments.append(lineRest[start+len(blockstart):len(lineRest)]) - block = True - #check if positionNumber is already over the length of the string and if we are in a block, - #if not continue with the rest of the line... - if positionNumber+1 < len(lineRest) and blockendFound == True: - lineRest = lineRest[positionNumber+1:len(lineRest)] - #...otherwise stop while loop - else: - break - - #check for single-line comment - for single in languages[lang].get("Single"): - if block == False and singleExists: - start = line.find(single) - if line.find(single) != -1: - comments.append(line[start+len(single):len(line)]) - entry = entry + 1 - break - - result = {'comments':comments} - return result - -def update_file(context, f): - # reads the content of the file (primary resource) - try: - comments = extract_comments(context, f) - - context.write_derived_resource(f, comments, 'comments') - except UnicodeDecodeError: - context.write_derived_resource(f, {'comments':[]}, 'comments') - -def remove_file(context, f): - context.remove_derived_resource(f, 'comments') - -def run(context, change): - # dispatch the modified file - if change['type'] == 'NEW_FILE': - update_file(context, change['file']) - - elif change['type'] == 'FILE_CHANGED': - update_file(context, change['file']) - - else: - remove_file(context, change['file']) diff --git a/modules/extractComments/__init__.py b/modules/extractComments/__init__.py new file mode 100644 index 00000000..00d7a3f6 --- /dev/null +++ b/modules/extractComments/__init__.py @@ -0,0 +1,12 @@ +config = { + 'wantdiff': True, + 'wantsfiles': True, + 'threadsafe': True, + 'behavior': { + 'creates': [['resource', 'comments']], + 'uses': [['resource', 'lang']] + } +} + +from .program import run +from .test import test diff --git a/modules/extractComments/program.py b/modules/extractComments/program.py new file mode 100644 index 00000000..eff8b9f9 --- /dev/null +++ b/modules/extractComments/program.py @@ -0,0 +1,134 @@ +import os +import json + +def getLanguages(): + #datastructure for languages and their corresponding comment symbols + languages = {'Java': {'Single': ['//'], 'Block': [{'BlockStart': '/*', 'BlockEnd': '*/'}]}, + 'Python': {'Single': ['#'], 'Block': [{'BlockStart': '"""', 'BlockEnd': '"""'},{'BlockStart': "'''", 'BlockEnd': "'''"},]}, + 'Haskell': {'Single': ['--'], 'Block': [{'BlockStart': '{-', 'BlockEnd': '-}'}]}} + return languages + +# this is the actual logic of the module +def extract_comments(context, f): + languages = getLanguages() + #init of comments array + comments = [] + + #get file & get language of file + source = context.get_primary_resource(f) + lang = context.get_derived_resource(f,'lang') + + + #block symbols if we are actually in a block-comment + block = False + #number of actual array entry + entry = 0 + globalBlockEnd = "" + + #check if language is in datastructure + if languages.get(lang) != None : + #check if languages have block/singe-line comments + singleExists = languages[lang].get("Single") != None + blockExists = languages[lang].get("Block") != None and languages[lang].get("Block") != [] + + #iterate over all lines + for line in source.split('\n'): + + #positionNumber marks later, where a block has ended and where the rest of the line continues + #lineRest is the rest of the line + positionNumber = 0 + lineRest = line + + #firstly check for single-line comment + for single in languages[lang].get("Single"): + if block == False and singleExists: + start = line.find(single) + if start != -1: + comments.append(line[start+len(single):len(line)]) + entry = entry+1 + lineRest = line[0:start] #here we set the line to the rest of the line without the single line comment, + #so that we can search for block comments in the rest + break + + #loop gets stopped, when a) we are already in a block and do not find a end symbol + # b) we are not in a block, but do not find a start symbol + if blockExists: + while True: + #check if we are already in a block + if block == True: + blockendFound = False + end = lineRest.find(globalBlockEnd) + #if we find a block-end symbol end the block + if end != -1: + comments[entry] = comments[entry]+'\n'+lineRest[0:end+1] + entry = entry + 1 + block = False + blockendFound = True + #mark number of blockend, to determine where rest of line starts + positionNumber = end+len(blockend) + #when we find a blockend, add the text to the comments + else: + comments[entry] = comments[entry]+'\n'+lineRest + #check if positionNumber is already over the length of the string, if not continue with + #the rest of the line... + if positionNumber+1 < len(lineRest) and blockendFound == True: + lineRest = lineRest[positionNumber+1:len(lineRest)] + #...otherwise stop while loop + else: + break + #if not in a block, try to find blockstart symbol + else: + positionNumber = -1 + for blocksymbol in languages[lang].get("Block"): + blockstart = blocksymbol.get("BlockStart") + start = lineRest.find(blockstart) + if start != -1 and block == False: + blockend = blocksymbol.get("BlockEnd") + #note that we change here the lineRest to a shorter path, where the blockstart + #symbol is at the beginning + lineRest = lineRest[start+len(blockstart):len(lineRest)] + end = lineRest.find(blockend) + #if blockend symbol in same line, take only this part + if end != -1: + comments.append(lineRest[0:end]) + entry = entry + 1 + #mark number of blockend, to determine where rest of line starts + positionNumber = end+len(blockend) + else: + #if no block-end symbol has been found, set block to true + comments.append(lineRest[start+len(blockstart):len(lineRest)]) + block = True + globalBlockEnd = blockend + #check if positionNumber is already over the length of the string and if we are in a block, + #if not continue with the rest of the line... + if positionNumber+1 < len(lineRest) and positionNumber != -1: + lineRest = lineRest[positionNumber+1:len(lineRest)] + #...otherwise stop while loop + else: + break + + result = {'comments':comments} + return result + +def update_file(context, f): + # reads the content of the file (primary resource) + try: + comments = extract_comments(context, f) + + context.write_derived_resource(f, comments, 'comments') + except UnicodeDecodeError: + context.write_derived_resource(f, {'comments':[]}, 'comments') + +def remove_file(context, f): + context.remove_derived_resource(f, 'comments') + +def run(context, change): + # dispatch the modified file + if change['type'] == 'NEW_FILE': + update_file(context, change['file']) + + elif change['type'] == 'FILE_CHANGED': + update_file(context, change['file']) + + else: + remove_file(context, change['file']) diff --git a/modules/extractComments/test.py b/modules/extractComments/test.py new file mode 100644 index 00000000..6039fad0 --- /dev/null +++ b/modules/extractComments/test.py @@ -0,0 +1,131 @@ +from .program import run, getLanguages +import unittest +from unittest.mock import Mock, patch + + +class ExtractCommentsTest(unittest.TestCase): + + #Test for Java Single Line Comments + def test_Java_singleCommentStandard(self): + change = { + 'type': 'NEW_FILE', + 'file': 'some-file.any' + } + sourceCode = "this is the regular \n code of a program \n which does something" + commentText = "this is a comment" + sourceCode = sourceCode + "\n "+"//"+commentText+"\n more code" + self.env = Mock() + self.env.get_derived_resource.return_value = "Java" + self.env.get_primary_resource.return_value = sourceCode + run(self.env, change) + self.env.write_derived_resource.assert_called_with('some-file.any', {'comments':[commentText]}, 'comments') + + #Test for Java Multiline Comments + def test_Java_multiCommentStandard(self): + change = { + 'type': 'NEW_FILE', + 'file': 'some-file.any' + } + sourceCode = "this is the regular \n code of a program \n which does something" + commentText = "this is a comment" + sourceCode = sourceCode + "\n "+"/*"+commentText+"*/"+"more code" + self.env = Mock() + self.env.get_derived_resource.return_value = "Java" + self.env.get_primary_resource.return_value = sourceCode + run(self.env, change) + self.env.write_derived_resource.assert_called_with('some-file.any', {'comments':[commentText]}, 'comments') + + + #Test ALL entries in the language datastructure of .program + def test_ALL_singleCommentStandard(self): + languages = getLanguages() + change = { + 'type': 'NEW_FILE', + 'file': 'some-file.any' + } + for key in languages.keys(): + sourceCode = "this is the regular \n code of a program \n which does something" + lang = key + singleComment = languages[key].get("Single") + if singleComment != None: + for comment in singleComment: + commentText = "this is a comment" + sourceCode = sourceCode + "\n "+comment+commentText+"\n more code" + else: + sourceCode = "" + commentText = "" + self.env = Mock() + self.env.get_derived_resource.return_value = key + self.env.get_primary_resource.return_value = sourceCode + run(self.env, change) + self.env.write_derived_resource.assert_called_with('some-file.any', {'comments':[commentText]}, 'comments') + + #Test ALL entries in the language datastructure of .program + def test_ALL_skipBlockInSingleLine(self): + languages = getLanguages() + change = { + 'type': 'NEW_FILE', + 'file': 'some-file.any' + } + for key in languages.keys(): + sourceCode = "this is the regular \n code of a program \n which does something" + lang = key + singleComment = languages[key].get("Single") + if singleComment != None: + if languages[key].get("Block") != None and languages[key].get("Block") != []: + blockStart = languages[key].get("Block")[0].get("BlockStart") + blockEnd = languages[key].get("Block")[0].get("BlockEnd") + for comment in singleComment: + if languages[key].get("Block") != None and languages[key].get("Block") != []: + blockStart = languages[key].get("Block")[0].get("BlockStart") + blockEnd = languages[key].get("Block")[0].get("BlockEnd") + commentText = "this is a comment"+blockStart+"inside"+blockEnd + else: + commentText = "this is a comment" + sourceCode = sourceCode + "\n "+comment+commentText+"\n more code" + else: + commentText = "" + sourceCode = "" + + self.env = Mock() + self.env.get_derived_resource.return_value = key + self.env.get_primary_resource.return_value = sourceCode + run(self.env, change) + self.env.write_derived_resource.assert_called_with('some-file.any', {'comments':[commentText]}, 'comments') + + #Test ALL entries in the language datastructure of .program + def test_ALL_MultiLineCommentAlone(self): + languages = getLanguages() + change = { + 'type': 'NEW_FILE', + 'file': 'some-file.any' + } + + for key in languages.keys(): + allComments = [] + sourceCode = "this is the regular \n code of a program \n which does something" + lang = key + if languages[key].get("Block") != None and languages[key].get("Block") != []: + for blocksymbols in languages[key].get("Block"): + blockStart = blocksymbols.get("BlockStart") + blockEnd = blocksymbols.get("BlockEnd") + commentText = "Block Comment 1 starting and ending in own line(s)" + sourceCode = sourceCode + "\n "+blockStart+commentText+blockEnd+"\n more code" + allComments.append(commentText) + commentText = "Block Comment 2 ending in line with code" + sourceCode = sourceCode + "\n "+blockStart+commentText+blockEnd+" more code" + allComments.append(commentText) + else: + allComments = [] + sourceCode = "" + + self.env = Mock() + self.env.get_derived_resource.return_value = key + self.env.get_primary_resource.return_value = sourceCode + run(self.env, change) + self.env.write_derived_resource.assert_called_with('some-file.any', {'comments':allComments}, 'comments') + + +def test(): + suite = unittest.TestLoader().loadTestsFromTestCase(ExtractCommentsTest) + unittest.TextTestRunner(verbosity=2).run(suite) From 93498834a45939fc6a3111fe21097f0afcde152c Mon Sep 17 00:00:00 2001 From: Andre Emmerichs Date: Thu, 22 Dec 2016 21:25:32 +0100 Subject: [PATCH 06/43] Added Visualisation approch, read README Added visualisation approach with matplotlib/numpy for module locPerContribution, visualisation is saved in data/views/locPerContribution --- README.md | 13 ++++++++++ bin/worker_lib/executor.py | 5 ++++ modules/locPerContribution/__init__.py | 6 +++-- modules/locPerContribution/visualisation.py | 27 +++++++++++++++++++++ 4 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 modules/locPerContribution/visualisation.py diff --git a/README.md b/README.md index 545e9ee9..65cf1765 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,16 @@ +# Actual information for project practical + +-Changed bin/workerlib/extractor to allow call of visualisation + +-Added boolean visualisation to locPerContribution, which is used (when True), to tell the extractor that +a chart should be created + +-We are actually using matplotlib and numpy as imports, which are not automatically installed +-> Need to add it to the makefile? + + + + # Installation It is advised to familiar yourself with the basic concepts of 101worker diff --git a/bin/worker_lib/executor.py b/bin/worker_lib/executor.py index ba7a800f..d1c724b0 100644 --- a/bin/worker_lib/executor.py +++ b/bin/worker_lib/executor.py @@ -38,6 +38,11 @@ def run(self, changes): } self._exec(change) + ####### added to create Image ################ + if self._module.config.get('visualisation') == True : + print("Creating Image") + self._module.createImage(self._env,env) + ############################################## class AllFullSweepExecutor(Executor): diff --git a/modules/locPerContribution/__init__.py b/modules/locPerContribution/__init__.py index e6d1909d..5dcffe9d 100644 --- a/modules/locPerContribution/__init__.py +++ b/modules/locPerContribution/__init__.py @@ -4,9 +4,11 @@ 'threadsafe': False, 'behavior': { 'creates': [['dump', 'locPerContribution']], - 'uses': [['resource', 'loc']] - } + 'uses': [['resource', 'loc']], + }, + 'visualisation': True } from .program import run from .test import test +from .visualisation import createImage diff --git a/modules/locPerContribution/visualisation.py b/modules/locPerContribution/visualisation.py new file mode 100644 index 00000000..d7fa0d91 --- /dev/null +++ b/modules/locPerContribution/visualisation.py @@ -0,0 +1,27 @@ +import numpy +import matplotlib.pyplot as plotting +import os + +def createImage(env,workerlibEnv): + data = env.read_dump('locPerContribution') + contributions = [] + loc = [] + for contribution in data.keys(): + contributions.append(contribution) + loc.append(data[contribution]) + plotting.bar(numpy.arange(len(contributions)),loc,0.35, color = 'b') + plotting.xlabel("Contribution") + plotting.ylabel("Lines of Code") + plotting.xticks(numpy.arange(len(contributions))+0.35/2., contributions) + plotting.title("LocPerContribution") + + ### check if path exists, else create it ### + if not os.path.exists(workerlibEnv['views101dir']): + os.mkdir(workerlibEnv['views101dir']) + path = workerlibEnv['views101dir']+'/locPerContribution' + if not os.path.exists(path): + os.mkdir(path) + ############################################ + + plotting.savefig(path+'/chart.png') + From 31d08d6621c17348a71803b007ed59f77bb0a7dd Mon Sep 17 00:00:00 2001 From: Skaduwa Date: Wed, 28 Dec 2016 16:19:22 +0100 Subject: [PATCH 07/43] added visualization module and concept --- modules/documentation.md | 39 ++++++++ .../101_contribution/locPerContribution.py | 10 ++ modules/visualization/101_contributor/loc.py | 4 + .../101_meta/locPerContribution.py | 13 +++ modules/visualization/README.md | 53 ++++++++++ modules/visualization/__init__.py | 12 +++ modules/visualization/program.py | 68 +++++++++++++ modules/visualization/sample/contribution.tsv | 4 + modules/visualization/sample/meta.csv | 2 + .../sample/sample_contribution.html | 68 +++++++++++++ modules/visualization/sample/sample_meta.html | 97 +++++++++++++++++++ modules/visualization/test.py | 38 ++++++++ 12 files changed, 408 insertions(+) create mode 100644 modules/documentation.md create mode 100644 modules/visualization/101_contribution/locPerContribution.py create mode 100644 modules/visualization/101_contributor/loc.py create mode 100644 modules/visualization/101_meta/locPerContribution.py create mode 100644 modules/visualization/README.md create mode 100644 modules/visualization/__init__.py create mode 100644 modules/visualization/program.py create mode 100644 modules/visualization/sample/contribution.tsv create mode 100644 modules/visualization/sample/meta.csv create mode 100644 modules/visualization/sample/sample_contribution.html create mode 100644 modules/visualization/sample/sample_meta.html create mode 100644 modules/visualization/test.py diff --git a/modules/documentation.md b/modules/documentation.md new file mode 100644 index 00000000..8f38a868 --- /dev/null +++ b/modules/documentation.md @@ -0,0 +1,39 @@ +# Headline - module Name + +## Characteristics +### config: +wantdiff: True | False +wantsfiles: True | False +threadsafe: True | False +### behavior: +creates: describe output +uses: describe input +### imports: +list of imports + +## Architecture +### __init__.py: +short description of the module +### program.py; +short description of the program +### test.py +short description of the tests +### other files +short description of other files + +## output +### value: +describe generated value +### structure: +describe output structure + +## Usage +explain usage and testing of the module + +## Metadata +### Author: name of creator +### Creation: +context: context of creation +date: date of creation +### Features: list of features +### Technology: used technology with sources diff --git a/modules/visualization/101_contribution/locPerContribution.py b/modules/visualization/101_contribution/locPerContribution.py new file mode 100644 index 00000000..137553b3 --- /dev/null +++ b/modules/visualization/101_contribution/locPerContribution.py @@ -0,0 +1,10 @@ +def contribution_locPerContribution(env, res): + + # locPerContribution - contribution - BarChart + data = env.read_dump('locPerContribution') + out = [] + header = ['contribution','loc'] + out.append(header) + for key, value in data.items(): + out.append([key, value]) + write_csv_contribution('locPerContribution', out) diff --git a/modules/visualization/101_contributor/loc.py b/modules/visualization/101_contributor/loc.py new file mode 100644 index 00000000..c38f9eef --- /dev/null +++ b/modules/visualization/101_contributor/loc.py @@ -0,0 +1,4 @@ +def contributor_locPerContribution(env, res): + + # loc - contributor - BarChart + # ToBedone diff --git a/modules/visualization/101_meta/locPerContribution.py b/modules/visualization/101_meta/locPerContribution.py new file mode 100644 index 00000000..ade433bb --- /dev/null +++ b/modules/visualization/101_meta/locPerContribution.py @@ -0,0 +1,13 @@ +def meta_locPerContribution(env, res): + + # locPerContribution - meta - Stacked BarChart + data = env.read_dump('locPerContribution') + out = [] + header = ['loc'] + sum_value = ['101_loc'] + for key, value in data.items(): + header.append(key) + sum_value.append(value) + out.append(header) + out.append(sum_value) + write_csv_meta('locPerContribution', out) diff --git a/modules/visualization/README.md b/modules/visualization/README.md new file mode 100644 index 00000000..60c2d0f9 --- /dev/null +++ b/modules/visualization/README.md @@ -0,0 +1,53 @@ +# visualization + +## Characteristics +### config: +wantdiff: False +wantsfiles: True +threadsafe: True +### behavior: +creates: csv files +uses: locPerContriution +### imports: +import os +import json +import csv + +## Architecture +### __init__.py: +This module uses resources, created by other modules, +extracts information from them and creates csv files holding those informations, +to quickly create graphs with the "insert name here". +### program.py; ++ meta +++ locPerContribution ++ contribution ++ contributor +### test.py +short description of the tests +### other files +short description of other files + +## output +### value: +describe generated value +### structure: +describe output structure + +## Usage +explain usage and testing of the module + +## Metadata +### Author: +Torsten Kaiser +### Creation: +context: pp 16/17 +date: WS 16/17 +### Features: +visualization +fact-extraction +### Technology: +JSON +CSV +d3js.org + diff --git a/modules/visualization/__init__.py b/modules/visualization/__init__.py new file mode 100644 index 00000000..4a053a92 --- /dev/null +++ b/modules/visualization/__init__.py @@ -0,0 +1,12 @@ +config = { + 'wantdiff': False, + 'wantsfiles': True, + 'threadsafe': False, + 'behavior': { + 'creates': [], + 'uses': [['resource', 'loc']] + } +} + +from .program import run +# from .test import test diff --git a/modules/visualization/program.py b/modules/visualization/program.py new file mode 100644 index 00000000..5aaf7ffa --- /dev/null +++ b/modules/visualization/program.py @@ -0,0 +1,68 @@ +import os +import json +import csv + +''' +output definitions +''' + +def check_path(path): + if not os.path.exists(path): + os.mkdir(path) + +def write_csv_meta(visualization_name, data): + path = '../101web/data/views/101_meta' + check_path(path) + d = os.path.join(path, visualization_name + '.csv') + with open(d, 'w') as f: + wr = csv.writer(f) + for item in data: + wr.writerow(item) + +def write_csv_contribution(visualization_name, data): + path = '../101web/data/views/101_contribution' + check_path(path) + d = os.path.join(path, visualization_name + '.csv') + with open(d, 'w') as f: + wr = csv.writer(f) + for item in data: + wr.writerow(item) + +def write_csv_contributor(visualization_name, data): + path = '../101web/data/views/101_contributor' + check_path(path) + d = os.path.join(path, visualization_name + '.csv') + with open(d, 'w') as f: + wr = csv.writer(f) + for item in data: + wr.writerow(item) + + +''' +program ++ graphs need to be put into seperate folders: meta, contribution, contributor ++ 1 file per graph +''' + +def run(env, res): + + # locPerContribution - meta - Stacked BarChart + data = env.read_dump('locPerContribution') + out = [] + header = ['loc'] + sum_value = ['101_loc'] + for key, value in data.items(): + header.append(key) + sum_value.append(value) + out.append(header) + out.append(sum_value) + write_csv_meta('locPerContribution', out) + + # locPerContribution - contribution - BarChart + data = env.read_dump('locPerContribution') + out = [] + header = ['contribution','loc'] + out.append(header) + for key, value in data.items(): + out.append([key, value]) + write_csv_contribution('locPerContribution', out) diff --git a/modules/visualization/sample/contribution.tsv b/modules/visualization/sample/contribution.tsv new file mode 100644 index 00000000..80031856 --- /dev/null +++ b/modules/visualization/sample/contribution.tsv @@ -0,0 +1,4 @@ +contribution loc +py3k 155 +javaTree 1523 +ruby 236 diff --git a/modules/visualization/sample/meta.csv b/modules/visualization/sample/meta.csv new file mode 100644 index 00000000..63341dde --- /dev/null +++ b/modules/visualization/sample/meta.csv @@ -0,0 +1,2 @@ +loc,py3k,javaTree,ruby +101_loc,155,1523,236 diff --git a/modules/visualization/sample/sample_contribution.html b/modules/visualization/sample/sample_contribution.html new file mode 100644 index 00000000..1b991fb9 --- /dev/null +++ b/modules/visualization/sample/sample_contribution.html @@ -0,0 +1,68 @@ + + + + + + + diff --git a/modules/visualization/sample/sample_meta.html b/modules/visualization/sample/sample_meta.html new file mode 100644 index 00000000..f550a10d --- /dev/null +++ b/modules/visualization/sample/sample_meta.html @@ -0,0 +1,97 @@ + + + + + + diff --git a/modules/visualization/test.py b/modules/visualization/test.py new file mode 100644 index 00000000..a2032560 --- /dev/null +++ b/modules/visualization/test.py @@ -0,0 +1,38 @@ +from .program import run + +import unittest +from unittest.mock import Mock, patch + +class LocPerContributionTest(unittest.TestCase): + + def setUp(self): + self.env = Mock() + self.env.read_dump.return_value = { 'python': 45 } + self.env.get_derived_resource.return_value = 10 + + def test_run(self): + res = { + 'file': 'contributions/python/some-file.py' + } + run(self.env, res) + + self.env.write_dump.assert_called_with('locPerContribution', { 'python': 55 }) + + def test_new_contribution(self): + res = { + 'file': 'contributions/ruby/some-file.rb' + } + run(self.env, res) + self.env.write_dump.assert_called_with('locPerContribution', { 'python': 45, 'ruby': 10 }) + + def test_run_no_contribution(self): + res = { + 'file': 'something/python/some-file.py' + } + run(self.env, res) + + self.env.write_dump.assert_called_with('locPerContribution', { 'python': 45 }) + +def test(): + suite = unittest.TestLoader().loadTestsFromTestCase(LocPerContributionTest) + unittest.TextTestRunner(verbosity=2).run(suite) From 4064d8f69357f60b744a8453f96bb53b6d2177d9 Mon Sep 17 00:00:00 2001 From: Andre Emmerichs Date: Sun, 15 Jan 2017 15:54:19 +0100 Subject: [PATCH 08/43] CommentExtractor -changed tests, readme --- README.md | 2 +- modules/extractComments/ReadMe.md | 48 ++++++++ modules/extractComments/test.py | 187 +++++++++++++----------------- 3 files changed, 132 insertions(+), 105 deletions(-) create mode 100644 modules/extractComments/ReadMe.md diff --git a/README.md b/README.md index 65cf1765..efe11e8e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Actual information for project practical --Changed bin/workerlib/extractor to allow call of visualisation +-Changed bin/workerlib/executor to allow call of visualisation -Added boolean visualisation to locPerContribution, which is used (when True), to tell the extractor that a chart should be created diff --git a/modules/extractComments/ReadMe.md b/modules/extractComments/ReadMe.md new file mode 100644 index 00000000..46e2f69a --- /dev/null +++ b/modules/extractComments/ReadMe.md @@ -0,0 +1,48 @@ +# commentExtractor + +##Characteristics +###config: +wantsdiff: True +wantsfiles: True +threadsafe: True + +###behavior: +creates: resource comments +uses: resource lang + +###imports: +os +json + +##Architecture +###init.py +Uses the source code of the file and its corresponding language provided by module matchlanguage +to extract the comments of the file and store them in a derived resource +###program.py +Does the work of the module, by iterating line by line and checking if a there is a comment symbol +and if it is acutally in a block. The comment symbols are actually saved in the file, too. +Note: In a multiline commment, all lines get connected with a \n +###test.py +Standard tests for all languages provided in the comment symbol datastructure in program.py. It tests +all single and multi line comments, and it has also a extended test, where it takes a look, if the +program ignores a blockstart in a single line comment + +##Output: +###value: +All comments of the language of the file +###structure: +{"comments":["comment1","comment2",...] + +##Usage: +Run & test module with standard commands (bin/run_module, bin/test) + +##Metadata: +###Author: +Andre Emmerichs +###Creation: +context: PP16/17 +date: WS 16/17 +###Features: + +###Technology: + diff --git a/modules/extractComments/test.py b/modules/extractComments/test.py index 6039fad0..781a606a 100644 --- a/modules/extractComments/test.py +++ b/modules/extractComments/test.py @@ -4,128 +4,107 @@ class ExtractCommentsTest(unittest.TestCase): + + def setUp(self): + self.change = { + 'type': 'NEW_FILE', + 'file': 'some-file.any' + } + self.commentText = "a great comment" #Test for Java Single Line Comments def test_Java_singleCommentStandard(self): - change = { - 'type': 'NEW_FILE', - 'file': 'some-file.any' - } - sourceCode = "this is the regular \n code of a program \n which does something" - commentText = "this is a comment" - sourceCode = sourceCode + "\n "+"//"+commentText+"\n more code" + sourceCode = "x=42 \n y=101 \n "+"//"+self.commentText+"\n"+"z=x+y" self.env = Mock() self.env.get_derived_resource.return_value = "Java" self.env.get_primary_resource.return_value = sourceCode - run(self.env, change) - self.env.write_derived_resource.assert_called_with('some-file.any', {'comments':[commentText]}, 'comments') + run(self.env, self.change) + self.env.write_derived_resource.assert_called_with('some-file.any', {'comments':[self.commentText]}, 'comments') #Test for Java Multiline Comments def test_Java_multiCommentStandard(self): - change = { - 'type': 'NEW_FILE', - 'file': 'some-file.any' - } - sourceCode = "this is the regular \n code of a program \n which does something" - commentText = "this is a comment" - sourceCode = sourceCode + "\n "+"/*"+commentText+"*/"+"more code" + sourceCode = "x=42 \n y=101 \n "+"/*"+self.commentText+"*/"+"z=x+y" self.env = Mock() self.env.get_derived_resource.return_value = "Java" self.env.get_primary_resource.return_value = sourceCode - run(self.env, change) - self.env.write_derived_resource.assert_called_with('some-file.any', {'comments':[commentText]}, 'comments') + run(self.env, self.change) + self.env.write_derived_resource.assert_called_with('some-file.any', {'comments':[self.commentText]}, 'comments') + #Test for Java SingleLine including multiLine Begin + def test_Java_singleMultiCommentExt(self): + sourceCode = "x=42 \n y=101 \n "+"//"+self.commentText+"/* abc\n"+"z=x+y \n"+"x=42 \n y=101 \n "+"/*"+self.commentText+"*/"+"z=x+y" + self.env = Mock() + self.env.get_derived_resource.return_value = "Java" + self.env.get_primary_resource.return_value = sourceCode + run(self.env, self.change) + self.env.write_derived_resource.assert_called_with('some-file.any', {'comments':[self.commentText+"/* abc",self.commentText]}, 'comments') - #Test ALL entries in the language datastructure of .program - def test_ALL_singleCommentStandard(self): - languages = getLanguages() - change = { - 'type': 'NEW_FILE', - 'file': 'some-file.any' - } - for key in languages.keys(): - sourceCode = "this is the regular \n code of a program \n which does something" - lang = key - singleComment = languages[key].get("Single") - if singleComment != None: - for comment in singleComment: - commentText = "this is a comment" - sourceCode = sourceCode + "\n "+comment+commentText+"\n more code" - else: - sourceCode = "" - commentText = "" - self.env = Mock() - self.env.get_derived_resource.return_value = key - self.env.get_primary_resource.return_value = sourceCode - run(self.env, change) - self.env.write_derived_resource.assert_called_with('some-file.any', {'comments':[commentText]}, 'comments') - - #Test ALL entries in the language datastructure of .program - def test_ALL_skipBlockInSingleLine(self): - languages = getLanguages() - change = { - 'type': 'NEW_FILE', - 'file': 'some-file.any' - } - for key in languages.keys(): - sourceCode = "this is the regular \n code of a program \n which does something" - lang = key - singleComment = languages[key].get("Single") - if singleComment != None: - if languages[key].get("Block") != None and languages[key].get("Block") != []: - blockStart = languages[key].get("Block")[0].get("BlockStart") - blockEnd = languages[key].get("Block")[0].get("BlockEnd") - for comment in singleComment: - if languages[key].get("Block") != None and languages[key].get("Block") != []: - blockStart = languages[key].get("Block")[0].get("BlockStart") - blockEnd = languages[key].get("Block")[0].get("BlockEnd") - commentText = "this is a comment"+blockStart+"inside"+blockEnd - else: - commentText = "this is a comment" - sourceCode = sourceCode + "\n "+comment+commentText+"\n more code" - else: - commentText = "" - sourceCode = "" + #Test for Haskell Single Line Comments + def test_Haskell_singleCommentStandard(self): + sourceCode = "x=42 \n y=101 \n "+"--"+self.commentText+"\n"+"z=x+y" + self.env = Mock() + self.env.get_derived_resource.return_value = "Haskell" + self.env.get_primary_resource.return_value = sourceCode + run(self.env, self.change) + self.env.write_derived_resource.assert_called_with('some-file.any', {'comments':[self.commentText]}, 'comments') - self.env = Mock() - self.env.get_derived_resource.return_value = key - self.env.get_primary_resource.return_value = sourceCode - run(self.env, change) - self.env.write_derived_resource.assert_called_with('some-file.any', {'comments':[commentText]}, 'comments') - - #Test ALL entries in the language datastructure of .program - def test_ALL_MultiLineCommentAlone(self): - languages = getLanguages() - change = { - 'type': 'NEW_FILE', - 'file': 'some-file.any' - } - - for key in languages.keys(): - allComments = [] - sourceCode = "this is the regular \n code of a program \n which does something" - lang = key - if languages[key].get("Block") != None and languages[key].get("Block") != []: - for blocksymbols in languages[key].get("Block"): - blockStart = blocksymbols.get("BlockStart") - blockEnd = blocksymbols.get("BlockEnd") - commentText = "Block Comment 1 starting and ending in own line(s)" - sourceCode = sourceCode + "\n "+blockStart+commentText+blockEnd+"\n more code" - allComments.append(commentText) - commentText = "Block Comment 2 ending in line with code" - sourceCode = sourceCode + "\n "+blockStart+commentText+blockEnd+" more code" - allComments.append(commentText) - else: - allComments = [] - sourceCode = "" + #Test for Haskell Multiline Comments + def test_Haskell_multiCommentStandard(self): + sourceCode = "x=42 \n y=101 \n "+"{-"+self.commentText+"-}"+"z=x+y" + self.env = Mock() + self.env.get_derived_resource.return_value = "Haskell" + self.env.get_primary_resource.return_value = sourceCode + run(self.env, self.change) + self.env.write_derived_resource.assert_called_with('some-file.any', {'comments':[self.commentText]}, 'comments') + + #Test for Haskell SingleLine including multiLine Begin + def test_Haskell_singleMultiCommentExt(self): + sourceCode = "x=42 \n y=101 \n "+"--"+self.commentText+"{- abc\n"+"z=x+y \n"+"x=42 \n y=101 \n "+"{-"+self.commentText+"-}"+"z=x+y" + self.env = Mock() + self.env.get_derived_resource.return_value = "Haskell" + self.env.get_primary_resource.return_value = sourceCode + run(self.env, self.change) + self.env.write_derived_resource.assert_called_with('some-file.any', {'comments':[self.commentText+"{- abc",self.commentText]}, 'comments') + + + #Test for Python Single Line Comments + def test_Python_singleCommentStandard(self): + sourceCode = "x=42 \n y=101 \n "+"#"+self.commentText+"\n"+"z=x+y" + self.env = Mock() + self.env.get_derived_resource.return_value = "Python" + self.env.get_primary_resource.return_value = sourceCode + run(self.env, self.change) + self.env.write_derived_resource.assert_called_with('some-file.any', {'comments':[self.commentText]}, 'comments') - self.env = Mock() - self.env.get_derived_resource.return_value = key - self.env.get_primary_resource.return_value = sourceCode - run(self.env, change) - self.env.write_derived_resource.assert_called_with('some-file.any', {'comments':allComments}, 'comments') + #Test for Python Multiline Comments1 + def test_Python_multiCommentStandard1(self): + sourceCode = "x=42 \n y=101 \n "+"'''"+self.commentText+"'''"+"z=x+y" + self.env = Mock() + self.env.get_derived_resource.return_value = "Python" + self.env.get_primary_resource.return_value = sourceCode + run(self.env, self.change) + self.env.write_derived_resource.assert_called_with('some-file.any', {'comments':[self.commentText]}, 'comments') + #Test for Python Multiline Comments2 + def test_Python_multiCommentStandard2(self): + sourceCode = "x=42 \n y=101 \n "+"\"\"\""+self.commentText+"\"\"\""+"z=x+y" + self.env = Mock() + self.env.get_derived_resource.return_value = "Python" + self.env.get_primary_resource.return_value = sourceCode + run(self.env, self.change) + self.env.write_derived_resource.assert_called_with('some-file.any', {'comments':[self.commentText]}, 'comments') + + #Test for Python SingleLine including multiLine Begin + def test_Python_singleMultiCommentExt(self): + sourceCode = "x=42 \n y=101 \n "+"#"+self.commentText+"''' abc\n"+"z=x+y \n"+"x=42 \n y=101 \n "+"'''"+self.commentText+"'''"+"z=x+y" + self.env = Mock() + self.env.get_derived_resource.return_value = "Python" + self.env.get_primary_resource.return_value = sourceCode + run(self.env, self.change) + self.env.write_derived_resource.assert_called_with('some-file.any', {'comments':[self.commentText+"''' abc",self.commentText]}, 'comments') def test(): suite = unittest.TestLoader().loadTestsFromTestCase(ExtractCommentsTest) unittest.TextTestRunner(verbosity=2).run(suite) + + From ef9d6cc62d6847974a492ba2c297ba744c0cff46 Mon Sep 17 00:00:00 2001 From: Skaduwa Date: Sun, 22 Jan 2017 15:52:40 +0100 Subject: [PATCH 09/43] visualization module: added module support added samples for all types --- bin/worker_lib/modules.py | 3 +- .../101_contribution/locPerContribution.py | 10 --- modules/visualization/101_contributor/loc.py | 4 -- .../visualization/contribution/__init__.py | 0 modules/visualization/contribution/test.py | 16 +++++ modules/visualization/contributor/__init__.py | 0 modules/visualization/contributor/test.py | 16 +++++ modules/visualization/meta/__init__.py | 0 .../locPerContribution.py => meta/test.py} | 9 ++- modules/visualization/module/__init__.py | 0 modules/visualization/module/test.py | 16 +++++ modules/visualization/program.py | 64 +++++++------------ modules/visualization/sample.py | 10 +++ 13 files changed, 89 insertions(+), 59 deletions(-) delete mode 100644 modules/visualization/101_contribution/locPerContribution.py delete mode 100644 modules/visualization/101_contributor/loc.py create mode 100644 modules/visualization/contribution/__init__.py create mode 100644 modules/visualization/contribution/test.py create mode 100644 modules/visualization/contributor/__init__.py create mode 100644 modules/visualization/contributor/test.py create mode 100644 modules/visualization/meta/__init__.py rename modules/visualization/{101_meta/locPerContribution.py => meta/test.py} (63%) create mode 100644 modules/visualization/module/__init__.py create mode 100644 modules/visualization/module/test.py create mode 100644 modules/visualization/sample.py diff --git a/bin/worker_lib/modules.py b/bin/worker_lib/modules.py index 665f0647..8b04e756 100644 --- a/bin/worker_lib/modules.py +++ b/bin/worker_lib/modules.py @@ -21,7 +21,8 @@ def import_module(module_name): 'moretagclouds', 'plainTextWikiDump', 'zip', - 'mongodump' + 'mongodump', + 'visualization' ] modules = [import_module(module) for module in modules] diff --git a/modules/visualization/101_contribution/locPerContribution.py b/modules/visualization/101_contribution/locPerContribution.py deleted file mode 100644 index 137553b3..00000000 --- a/modules/visualization/101_contribution/locPerContribution.py +++ /dev/null @@ -1,10 +0,0 @@ -def contribution_locPerContribution(env, res): - - # locPerContribution - contribution - BarChart - data = env.read_dump('locPerContribution') - out = [] - header = ['contribution','loc'] - out.append(header) - for key, value in data.items(): - out.append([key, value]) - write_csv_contribution('locPerContribution', out) diff --git a/modules/visualization/101_contributor/loc.py b/modules/visualization/101_contributor/loc.py deleted file mode 100644 index c38f9eef..00000000 --- a/modules/visualization/101_contributor/loc.py +++ /dev/null @@ -1,4 +0,0 @@ -def contributor_locPerContribution(env, res): - - # loc - contributor - BarChart - # ToBedone diff --git a/modules/visualization/contribution/__init__.py b/modules/visualization/contribution/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/modules/visualization/contribution/test.py b/modules/visualization/contribution/test.py new file mode 100644 index 00000000..08a71227 --- /dev/null +++ b/modules/visualization/contribution/test.py @@ -0,0 +1,16 @@ +from ..program import check_path +from ..program import write_csv + +def run(env, res): + + # test + data = env.read_dump('locPerContribution') + out = [] + header = ['loc'] + sum_value = ['101_loc'] + for key, value in data.items(): + header.append(key) + sum_value.append(value) + out.append(header) + out.append(sum_value) + write_csv('test', 'contribution', out) diff --git a/modules/visualization/contributor/__init__.py b/modules/visualization/contributor/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/modules/visualization/contributor/test.py b/modules/visualization/contributor/test.py new file mode 100644 index 00000000..5daf233c --- /dev/null +++ b/modules/visualization/contributor/test.py @@ -0,0 +1,16 @@ +from ..program import check_path +from ..program import write_csv + +def run(env, res): + + # test + data = env.read_dump('locPerContribution') + out = [] + header = ['loc'] + sum_value = ['101_loc'] + for key, value in data.items(): + header.append(key) + sum_value.append(value) + out.append(header) + out.append(sum_value) + write_csv('test', 'contributor', out) diff --git a/modules/visualization/meta/__init__.py b/modules/visualization/meta/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/modules/visualization/101_meta/locPerContribution.py b/modules/visualization/meta/test.py similarity index 63% rename from modules/visualization/101_meta/locPerContribution.py rename to modules/visualization/meta/test.py index ade433bb..ebf7b14d 100644 --- a/modules/visualization/101_meta/locPerContribution.py +++ b/modules/visualization/meta/test.py @@ -1,6 +1,9 @@ -def meta_locPerContribution(env, res): +from ..program import check_path +from ..program import write_csv + +def run(env, res): - # locPerContribution - meta - Stacked BarChart + # test data = env.read_dump('locPerContribution') out = [] header = ['loc'] @@ -10,4 +13,4 @@ def meta_locPerContribution(env, res): sum_value.append(value) out.append(header) out.append(sum_value) - write_csv_meta('locPerContribution', out) + write_csv('test', 'meta', out) diff --git a/modules/visualization/module/__init__.py b/modules/visualization/module/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/modules/visualization/module/test.py b/modules/visualization/module/test.py new file mode 100644 index 00000000..f6fec323 --- /dev/null +++ b/modules/visualization/module/test.py @@ -0,0 +1,16 @@ +from ..program import check_path +from ..program import write_csv + +def run(env, res): + + # test + data = env.read_dump('locPerContribution') + out = [] + header = ['loc'] + sum_value = ['101_loc'] + for key, value in data.items(): + header.append(key) + sum_value.append(value) + out.append(header) + out.append(sum_value) + write_csv('test', 'module', out) diff --git a/modules/visualization/program.py b/modules/visualization/program.py index 5aaf7ffa..221eb550 100644 --- a/modules/visualization/program.py +++ b/modules/visualization/program.py @@ -10,28 +10,15 @@ def check_path(path): if not os.path.exists(path): os.mkdir(path) -def write_csv_meta(visualization_name, data): - path = '../101web/data/views/101_meta' - check_path(path) - d = os.path.join(path, visualization_name + '.csv') - with open(d, 'w') as f: - wr = csv.writer(f) - for item in data: - wr.writerow(item) - -def write_csv_contribution(visualization_name, data): - path = '../101web/data/views/101_contribution' - check_path(path) - d = os.path.join(path, visualization_name + '.csv') - with open(d, 'w') as f: - wr = csv.writer(f) - for item in data: - wr.writerow(item) - -def write_csv_contributor(visualization_name, data): - path = '../101web/data/views/101_contributor' - check_path(path) - d = os.path.join(path, visualization_name + '.csv') +def write_csv(name, dest, data): + path = {'meta': '../101web/data/views/meta', + 'contribution': '../101web/data/views/contribution', + 'contributor': '../101web/data/views/contributor', + 'module': '../101web/data/views/module'} + + destination = path[dest] + check_path(destination) + d = os.path.join(destination, name + '.csv') with open(d, 'w') as f: wr = csv.writer(f) for item in data: @@ -46,23 +33,18 @@ def write_csv_contributor(visualization_name, data): def run(env, res): - # locPerContribution - meta - Stacked BarChart - data = env.read_dump('locPerContribution') - out = [] - header = ['loc'] - sum_value = ['101_loc'] - for key, value in data.items(): - header.append(key) - sum_value.append(value) - out.append(header) - out.append(sum_value) - write_csv_meta('locPerContribution', out) + # meta + from .meta.test import run as test_meta + test_meta(env, res) + + # contribution + from .contribution.test import run as test_contribution + test_contribution(env, res) + + # contributor + from .contributor.test import run as test_contributor + test_contributor(env, res) - # locPerContribution - contribution - BarChart - data = env.read_dump('locPerContribution') - out = [] - header = ['contribution','loc'] - out.append(header) - for key, value in data.items(): - out.append([key, value]) - write_csv_contribution('locPerContribution', out) + # modules + from .module.test import run as test_module + test_module(env, res) diff --git a/modules/visualization/sample.py b/modules/visualization/sample.py new file mode 100644 index 00000000..a9c64eed --- /dev/null +++ b/modules/visualization/sample.py @@ -0,0 +1,10 @@ +from ..program import check_path +from ..program import write_csv + +def run(env, res): + + ''' + insert code here + move file to according folder + import run in program.py + ''' From 2e68e46950b8852cac88761d8ea5256b5c3937af Mon Sep 17 00:00:00 2001 From: Andre Emmerichs Date: Wed, 25 Jan 2017 19:06:42 +0100 Subject: [PATCH 10/43] Extending visualisation protoype Please see comment in program.py - create_barchart --- .../module/locPerContribution.py | 15 ++++++++ modules/visualization/program.py | 35 +++++++++++-------- 2 files changed, 36 insertions(+), 14 deletions(-) create mode 100644 modules/visualization/module/locPerContribution.py diff --git a/modules/visualization/module/locPerContribution.py b/modules/visualization/module/locPerContribution.py new file mode 100644 index 00000000..947e1ac2 --- /dev/null +++ b/modules/visualization/module/locPerContribution.py @@ -0,0 +1,15 @@ +from ..program import check_path +from ..program import create_barchart + +def run(env, res): + + data = env.read_dump('locPerContribution') + out = [] + xName = ['Contribution'] + yName = ['Lines of Code'] + xValues = [] + yValues = [] + for key, value in data.items(): + xValues.append(key) + yValues.append(value) + create_barchart(xName, yName, xValues, yValues,'locPerContribution', env) diff --git a/modules/visualization/program.py b/modules/visualization/program.py index 221eb550..475c122d 100644 --- a/modules/visualization/program.py +++ b/modules/visualization/program.py @@ -24,6 +24,25 @@ def write_csv(name, dest, data): for item in data: wr.writerow(item) +def create_barchart(xName,yName,xValues,yValues,moduleName,env): + path = env.get_env("views101dir")+os.sep+'Module' + check_path(path) + path = path +os.sep+moduleName + check_path(path) + d = os.path.join(path, 'data.tsv') + data = [] + #labels = [xName,yName] actually not in use because of special read in format + labels = ['letter','frequency'] + data.append(labels) + for x,y in zip(xValues,yValues): + data.append([x,y]) + with open(d, 'w') as f: + wr = csv.writer(f, delimiter = "\t") + for item in data: + wr.writerow(item) + #TODO: shutil copy js data (like the one at http://bl.ocks.org/mbostock/3885304) to path + + ''' program @@ -32,19 +51,7 @@ def write_csv(name, dest, data): ''' def run(env, res): - - # meta - from .meta.test import run as test_meta - test_meta(env, res) - - # contribution - from .contribution.test import run as test_contribution - test_contribution(env, res) - - # contributor - from .contributor.test import run as test_contributor - test_contributor(env, res) # modules - from .module.test import run as test_module - test_module(env, res) + from .module.locPerContribution import run as runLocPerContribution + runLocPerContribution(env, res) From b2811bba03b1278000a3cccd8388ebf6c388ad83 Mon Sep 17 00:00:00 2001 From: Skaduwa Date: Fri, 27 Jan 2017 15:23:36 +0100 Subject: [PATCH 11/43] Updating structure after todays meeting --- modules/visualization/contribution/test.py | 16 --- modules/visualization/contributor/test.py | 16 --- modules/visualization/meta/test.py | 16 --- .../module/locPerContribution.py | 26 ++--- modules/visualization/module/test.py | 16 --- modules/visualization/program.py | 64 ++++++------ .../{sample_contribution.html => bar.html} | 2 +- modules/visualization/sample/contribution.tsv | 4 - modules/visualization/sample/meta.csv | 2 - modules/visualization/sample/pie.html | 63 ++++++++++++ modules/visualization/{ => sample}/sample.py | 3 +- modules/visualization/sample/sample_meta.html | 97 ------------------- 12 files changed, 115 insertions(+), 210 deletions(-) delete mode 100644 modules/visualization/contribution/test.py delete mode 100644 modules/visualization/contributor/test.py delete mode 100644 modules/visualization/meta/test.py delete mode 100644 modules/visualization/module/test.py rename modules/visualization/sample/{sample_contribution.html => bar.html} (97%) delete mode 100644 modules/visualization/sample/contribution.tsv delete mode 100644 modules/visualization/sample/meta.csv create mode 100644 modules/visualization/sample/pie.html rename modules/visualization/{ => sample}/sample.py (61%) delete mode 100644 modules/visualization/sample/sample_meta.html diff --git a/modules/visualization/contribution/test.py b/modules/visualization/contribution/test.py deleted file mode 100644 index 08a71227..00000000 --- a/modules/visualization/contribution/test.py +++ /dev/null @@ -1,16 +0,0 @@ -from ..program import check_path -from ..program import write_csv - -def run(env, res): - - # test - data = env.read_dump('locPerContribution') - out = [] - header = ['loc'] - sum_value = ['101_loc'] - for key, value in data.items(): - header.append(key) - sum_value.append(value) - out.append(header) - out.append(sum_value) - write_csv('test', 'contribution', out) diff --git a/modules/visualization/contributor/test.py b/modules/visualization/contributor/test.py deleted file mode 100644 index 5daf233c..00000000 --- a/modules/visualization/contributor/test.py +++ /dev/null @@ -1,16 +0,0 @@ -from ..program import check_path -from ..program import write_csv - -def run(env, res): - - # test - data = env.read_dump('locPerContribution') - out = [] - header = ['loc'] - sum_value = ['101_loc'] - for key, value in data.items(): - header.append(key) - sum_value.append(value) - out.append(header) - out.append(sum_value) - write_csv('test', 'contributor', out) diff --git a/modules/visualization/meta/test.py b/modules/visualization/meta/test.py deleted file mode 100644 index ebf7b14d..00000000 --- a/modules/visualization/meta/test.py +++ /dev/null @@ -1,16 +0,0 @@ -from ..program import check_path -from ..program import write_csv - -def run(env, res): - - # test - data = env.read_dump('locPerContribution') - out = [] - header = ['loc'] - sum_value = ['101_loc'] - for key, value in data.items(): - header.append(key) - sum_value.append(value) - out.append(header) - out.append(sum_value) - write_csv('test', 'meta', out) diff --git a/modules/visualization/module/locPerContribution.py b/modules/visualization/module/locPerContribution.py index 947e1ac2..b41f202a 100644 --- a/modules/visualization/module/locPerContribution.py +++ b/modules/visualization/module/locPerContribution.py @@ -1,15 +1,17 @@ -from ..program import check_path -from ..program import create_barchart +from ..program import assemble_barchart +from ..program import assemble_piechart def run(env, res): - data = env.read_dump('locPerContribution') - out = [] - xName = ['Contribution'] - yName = ['Lines of Code'] - xValues = [] - yValues = [] - for key, value in data.items(): - xValues.append(key) - yValues.append(value) - create_barchart(xName, yName, xValues, yValues,'locPerContribution', env) + data = env.read_dump('locPerContribution') + out = [] + xName = 'Contribution' + yName = 'Lines of Code' + xValues = [] + yValues = [] + for key, value in data.items(): + xValues.append(key) + yValues.append(value) + + assemble_barchart(xName, yName, xValues, yValues,'module', 'locPerContribution', env) + assemble_piechart(xName, yName, xValues, yValues,'module', 'locPerContribution', env) diff --git a/modules/visualization/module/test.py b/modules/visualization/module/test.py deleted file mode 100644 index f6fec323..00000000 --- a/modules/visualization/module/test.py +++ /dev/null @@ -1,16 +0,0 @@ -from ..program import check_path -from ..program import write_csv - -def run(env, res): - - # test - data = env.read_dump('locPerContribution') - out = [] - header = ['loc'] - sum_value = ['101_loc'] - for key, value in data.items(): - header.append(key) - sum_value.append(value) - out.append(header) - out.append(sum_value) - write_csv('test', 'module', out) diff --git a/modules/visualization/program.py b/modules/visualization/program.py index 475c122d..ea86d6cd 100644 --- a/modules/visualization/program.py +++ b/modules/visualization/program.py @@ -2,52 +2,60 @@ import json import csv + ''' -output definitions +write functions ''' def check_path(path): if not os.path.exists(path): - os.mkdir(path) - -def write_csv(name, dest, data): - path = {'meta': '../101web/data/views/meta', - 'contribution': '../101web/data/views/contribution', - 'contributor': '../101web/data/views/contributor', - 'module': '../101web/data/views/module'} + os.mkdir(path) - destination = path[dest] +def write_csv(name, source, dest, data, env): + destination = os.path.join(env.get_env('views101dir'), source) + check_path(destination) + destination = os.path.join(destination, dest) check_path(destination) d = os.path.join(destination, name + '.csv') with open(d, 'w') as f: wr = csv.writer(f) for item in data: wr.writerow(item) + +def write_tsv(name, source, dest, data, env): + destination = env.get_env('views101dir') + destination = os.path.join(destination, source, dest) + check_path(destination) + d = os.path.join(destination, name + '.tsv') + with open(d, 'w') as f: + wr = csv.writer(f, delimiter='\t') + for item in data: + wr.writerow(item) + -def create_barchart(xName,yName,xValues,yValues,moduleName,env): - path = env.get_env("views101dir")+os.sep+'Module' - check_path(path) - path = path +os.sep+moduleName - check_path(path) - d = os.path.join(path, 'data.tsv') - data = [] - #labels = [xName,yName] actually not in use because of special read in format - labels = ['letter','frequency'] - data.append(labels) - for x,y in zip(xValues,yValues): - data.append([x,y]) - with open(d, 'w') as f: - wr = csv.writer(f, delimiter = "\t") - for item in data: - wr.writerow(item) - #TODO: shutil copy js data (like the one at http://bl.ocks.org/mbostock/3885304) to path +''' +write functions +''' + +def assemble_barchart(xName,yName,xValues,yValues, source, dest, env): + out = [] + labels = ['letter','frequency'] + out.append(labels) + for x,y in zip(xValues,yValues): + out.append([x,y]) + write_tsv('bar', source, dest, out, env) +def assemble_piechart(xName,yName,xValues,yValues, source, dest, env): + out = [] + labels = [xName,yName] + out.append(labels) + for x,y in zip(xValues,yValues): + out.append([x,y]) + write_csv('pie', source, dest, out, env) ''' program -+ graphs need to be put into seperate folders: meta, contribution, contributor -+ 1 file per graph ''' def run(env, res): diff --git a/modules/visualization/sample/sample_contribution.html b/modules/visualization/sample/bar.html similarity index 97% rename from modules/visualization/sample/sample_contribution.html rename to modules/visualization/sample/bar.html index 1b991fb9..99b09516 100644 --- a/modules/visualization/sample/sample_contribution.html +++ b/modules/visualization/sample/bar.html @@ -30,7 +30,7 @@ var g = svg.append("g") .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); -d3.tsv("contribution.tsv", function(d) { +d3.tsv("bar.tsv", function(d) { d.frequency = +d.frequency; return d; }, function(error, data) { diff --git a/modules/visualization/sample/contribution.tsv b/modules/visualization/sample/contribution.tsv deleted file mode 100644 index 80031856..00000000 --- a/modules/visualization/sample/contribution.tsv +++ /dev/null @@ -1,4 +0,0 @@ -contribution loc -py3k 155 -javaTree 1523 -ruby 236 diff --git a/modules/visualization/sample/meta.csv b/modules/visualization/sample/meta.csv deleted file mode 100644 index 63341dde..00000000 --- a/modules/visualization/sample/meta.csv +++ /dev/null @@ -1,2 +0,0 @@ -loc,py3k,javaTree,ruby -101_loc,155,1523,236 diff --git a/modules/visualization/sample/pie.html b/modules/visualization/sample/pie.html new file mode 100644 index 00000000..d2384c50 --- /dev/null +++ b/modules/visualization/sample/pie.html @@ -0,0 +1,63 @@ + + + + + + diff --git a/modules/visualization/sample.py b/modules/visualization/sample/sample.py similarity index 61% rename from modules/visualization/sample.py rename to modules/visualization/sample/sample.py index a9c64eed..476c2dbc 100644 --- a/modules/visualization/sample.py +++ b/modules/visualization/sample/sample.py @@ -1,5 +1,4 @@ -from ..program import check_path -from ..program import write_csv +from ..program import assemble_barchart def run(env, res): diff --git a/modules/visualization/sample/sample_meta.html b/modules/visualization/sample/sample_meta.html deleted file mode 100644 index f550a10d..00000000 --- a/modules/visualization/sample/sample_meta.html +++ /dev/null @@ -1,97 +0,0 @@ - - - - - - From b5bbad980251578dbcc8d659b631ec073dfd65ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christopher=20Sch=C3=A4fer?= Date: Mon, 30 Jan 2017 11:56:34 +0100 Subject: [PATCH 12/43] Sentiment module added --- modules/commentSentimentPerLanguage/README.md | 12 ++ .../commentSentimentPerLanguage/__init__.py | 135 ++++++++++++++++++ modules/simpleSentiments/README.md | 11 ++ modules/simpleSentiments/__init__.py | 94 ++++++++++++ 4 files changed, 252 insertions(+) create mode 100644 modules/commentSentimentPerLanguage/README.md create mode 100644 modules/commentSentimentPerLanguage/__init__.py create mode 100644 modules/simpleSentiments/README.md create mode 100644 modules/simpleSentiments/__init__.py diff --git a/modules/commentSentimentPerLanguage/README.md b/modules/commentSentimentPerLanguage/README.md new file mode 100644 index 00000000..84ea975a --- /dev/null +++ b/modules/commentSentimentPerLanguage/README.md @@ -0,0 +1,12 @@ +# commentSentimentPerLanguage + +Min, max, median and mean sentiment of comments per language + +# Description + +This module calculates the minimum, maximum, median and mean Sentiment of comments per language. The results are saved as a sentimentsPerLanguage dump. +Additionally a sentimentsPerContribution dump is created containing the comment sentiment per file grouped by language and contribution. + +# Requirements + +This module requires simpleSentiment to be run first. \ No newline at end of file diff --git a/modules/commentSentimentPerLanguage/__init__.py b/modules/commentSentimentPerLanguage/__init__.py new file mode 100644 index 00000000..ed9b91ef --- /dev/null +++ b/modules/commentSentimentPerLanguage/__init__.py @@ -0,0 +1,135 @@ +import os +import statistics +import unittest +from unittest.mock import Mock, patch + +config = { + 'wantdiff': True, + 'wantsfiles': True, + 'threadsafe': False, + 'behavior': { + 'creates': [['dump', 'sentimentsPerContribution'], + ['dump', 'sentimentsPerLanguage']], + 'uses': [['dump', 'LangPerContribution'], + ['resource', 'sentiment']] + } +} + + +def run(env, res): + data = env.read_dump('sentimentsPerLanguage') + langData = env.read_dump('LangPerContribution') + helpData = env.read_dump('sentimentsPerContribution') + + if data is None: + data = {} + if helpData is None: + helpData = {} + f = res['file'] + if f.startswith('contributions' + os.sep): + folders = f.split(os.sep) + contribution = folders[1] + fileSentiment = env.get_derived_resource(f, 'sentiment') + lang = langData[contribution]['Main Language'] + fileName = folders[-1] + + if helpData.get(lang, None) is None: + helpData[lang] = {} + if helpData[lang].get(contribution, None) is None: + helpData[lang][contribution] = {} + if res['type'] == "NEW_FILE" or res['type'] == "FILE_CHANGED": + helpData[lang][contribution][fileName] = env.get_derived_resource(f, 'sentiment') + else: + del helpData[lang][contribution][fileName] + + if data.get(lang, None) is None: + data[lang] = {} + + for c_lang in helpData: + polarity = [] + subjectivity = [] + for c_name in helpData[c_lang]: + for c_file in helpData[c_lang][c_name]: + if helpData[c_lang][c_name][c_file] != 'N/A': + f_sentiment = helpData[c_lang][c_name][c_file] + polarity.append(f_sentiment[0]) + subjectivity.append(f_sentiment[1]) + if len(polarity) != 0: + p_min = min(polarity) + p_max = max(polarity) + #p_median = statistics.median(polarity) + #p_mean = statistics.mean(polarity) + + s_min = min(subjectivity) + s_max = max(subjectivity) + #s_median = statistics.median(subjectivity) + #s_mean = statistics.mean(subjectivity) + + data[c_lang]['Min'] = (p_min, s_min) + data[c_lang]['Max'] = (p_max, s_max) + #data[c_lang]['Median'] = (p_median, s_median) + #data[c_lang]['Mean'] = (p_mean, s_mean) + + + env.write_dump('sentimentsPerContribution', helpData) + env.write_dump('sentimentsPerLanguage', data) + +class CommentSentimentPerLanguageTest(unittest.TestCase): + + def setUp(self): + self.env = Mock() + + def side_effect(name): + if name == 'LangPerContribution': + return {'haskell':{'Main Language': 'Haskell'}, 'java':{'Main Language': 'Java'}} + elif name == 'sentimentsPerContribution': + return {'Java':{'stupidJava':{'Main.java':[-0.9, 0.5]}}} + elif name == 'sentimentsPerLanguage': + return {'Java': {'Min':(-0.9, 0.5), + 'Max':(-0.9, 0.5), + 'Median':(-0.9, 0.5), + 'Mean':(-0.9, 0.5),}} + + self.env.read_dump.side_effect = side_effect + self.env.get_derived_resource.return_value=[0.7, 0.5] + + def test_run(self): + res = { + 'file': 'contributions' + os.sep + 'java' + os.sep + 'EvenWorseJava.java', + 'type': 'NEW_FILE' + } + run(self.env, res) + self.env.write_dump.assert_called_with('sentimentsPerLanguage', {'Java': {'Min': (-0.9, 0.5), + 'Max': (0.7, 0.5), + 'Median': (statistics.median([-0.9, 0.7]), 0.5), + 'Mean': (statistics.median([-0.9, 0.7]), 0.5)}}) + + def test_new_Contribution(self): + res = { + 'file': 'contributions' + os.sep + 'haskell' + os.sep + 'haskell.hs', + 'type': 'NEW_FILE' + } + run(self.env, res) + self.env.write_dump.assert_called_with('sentimentsPerLanguage', {'Java': {'Min': (-0.9, 0.5), + 'Max': (-0.9, 0.5), + 'Median': (-0.9, 0.5), + 'Mean': (-0.9, 0.5)}, + 'Haskell': {'Min': (0.7, 0.5), + 'Max': (0.7, 0.5), + 'Median': (0.7, 0.5), + 'Mean': (0.7, 0.5)}}) + + def test_no_Contribution(self): + res = { + 'file': 'something' + os.sep + 'haskellIsGreat' + os.sep + 'really.hs', + 'type': 'NEW_FILE' + } + run(self.env, res) + self.env.write_dump.assert_called_with('sentimentsPerLanguage', {'Java': {'Min': (-0.9, 0.5), + 'Max': (-0.9, 0.5), + 'Median': (-0.9, 0.5), + 'Mean': (-0.9, 0.5)}}) + +def test(): + suite = unittest.TestLoader().loadTestsFromTestCase(CommentSentimentPerLanguageTest) + unittest.TextTestRunner(verbosity=2).run(suite) \ No newline at end of file diff --git a/modules/simpleSentiments/README.md b/modules/simpleSentiments/README.md new file mode 100644 index 00000000..75b0b60e --- /dev/null +++ b/modules/simpleSentiments/README.md @@ -0,0 +1,11 @@ +# simpleSentiment + +Sentiment of comments per File + +# Description + +This module uses TextBlob to analyse the sentiment of comments per File. A resource file named sentiment.json is saved containing the polarity and subjectivity of the comments. + +# Requirements + +This module requires a working installation of TextBlob and extractComments to be run first. \ No newline at end of file diff --git a/modules/simpleSentiments/__init__.py b/modules/simpleSentiments/__init__.py new file mode 100644 index 00000000..1eb54ac6 --- /dev/null +++ b/modules/simpleSentiments/__init__.py @@ -0,0 +1,94 @@ +import os +import json +import unittest +from unittest.mock import Mock +from textblob import TextBlob + + +config = { + 'wantdiff': True, + 'wantsfiles': True, + 'threadsafe': True, + 'behavior': { + 'creates': [['resource', 'sentiment']], + 'uses': [['resource', 'comments']] + } +} + + +def calc_sentiment(context, f): + comments = context.get_derived_resource(f, 'comments') + comments_blob = TextBlob(comments) + comment_sentiment = comments_blob.sentiment + if (comments == ''): + return 'N/A' + else: + return comment_sentiment + + +def update_file(context, f): + # reads the content of the file (primary resource) + try: + sentiment = calc_sentiment(context, f) + context.write_derived_resource(f, sentiment, 'sentiment') + except UnicodeDecodeError: + context.write_derived_resource(f, 0, 'sentiment') + + +def remove_file(context, f): + context.remove_derived_resource(f, 'sentiment') + + +def run(context, change): + # dispatch the modified file + if change['type'] == 'NEW_FILE': + update_file(context, change['file']) + + elif change['type'] == 'FILE_CHANGED': + update_file(context, change['file']) + + else: + remove_file(context, change['file']) + + +class simpleSentimentsTest(unittest.TestCase): + + def setUp(self): + self.env = Mock() + self.env.get_derived_resource.return_value = 'set x to 5 set y to 6 ' + + def test_run_new(self): + change = { + 'type': 'NEW_FILE', + 'file': 'some-file.java' + } + run(self.env, change) + + self.env.write_derived_resource.assert_called_with('some-file.java', + (0.0, 0.0), + 'sentiment') + + def test_run_changed(self): + change = { + 'type': 'FILE_CHANGED', + 'file': 'some-file.java' + } + run(self.env, change) + + self.env.write_derived_resource.assert_called_with('some-file.java', + (0.0, 0.0), + 'sentiment') + + def test_run_removed(self): + change = { + 'type': '', + 'file': 'some-file.java' + } + run(self.env, change) + + self.env.remove_derived_resource.assert_called_with('some-file.java', 'sentiment') + + +def test(): + suite = unittest.TestLoader().loadTestsFromTestCase(simpleSentimentsTest) + unittest.TextTestRunner(verbosity=2).run(suite) From 69b6f9531843ad28177ca78f0648b30813af8409 Mon Sep 17 00:00:00 2001 From: Andre Emmerichs Date: Wed, 15 Feb 2017 10:46:03 +0100 Subject: [PATCH 13/43] Reworking visualisation, allow multiple charts from same type TODO: Create template folder in modules/visualization with templates "chartname".html (see function program.py-copyTemplateToTarget) Create index.html file, which gets copied to mainPath(see program.y-assemble_pie_chart) and reads out links.json (-manageLinkFile) to create links General improvements of everything --- .../visualization/contribution/__init__.py | 0 modules/visualization/contributor/__init__.py | 0 modules/visualization/meta/__init__.py | 0 .../module/locPerContribution.py | 5 +- modules/visualization/program.py | 90 +++++++++---------- 5 files changed, 44 insertions(+), 51 deletions(-) delete mode 100644 modules/visualization/contribution/__init__.py delete mode 100644 modules/visualization/contributor/__init__.py delete mode 100644 modules/visualization/meta/__init__.py diff --git a/modules/visualization/contribution/__init__.py b/modules/visualization/contribution/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/modules/visualization/contributor/__init__.py b/modules/visualization/contributor/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/modules/visualization/meta/__init__.py b/modules/visualization/meta/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/modules/visualization/module/locPerContribution.py b/modules/visualization/module/locPerContribution.py index b41f202a..13af4b46 100644 --- a/modules/visualization/module/locPerContribution.py +++ b/modules/visualization/module/locPerContribution.py @@ -1,4 +1,3 @@ -from ..program import assemble_barchart from ..program import assemble_piechart def run(env, res): @@ -13,5 +12,5 @@ def run(env, res): xValues.append(key) yValues.append(value) - assemble_barchart(xName, yName, xValues, yValues,'module', 'locPerContribution', env) - assemble_piechart(xName, yName, xValues, yValues,'module', 'locPerContribution', env) + assemble_piechart("normal",xName, yName, xValues, yValues, 'locPerContribution', env) + assemble_piechart("inverted",yName, xName, xValues, yValues, 'locPerContribution', env) diff --git a/modules/visualization/program.py b/modules/visualization/program.py index ea86d6cd..be58c156 100644 --- a/modules/visualization/program.py +++ b/modules/visualization/program.py @@ -1,6 +1,7 @@ import os import json import csv +import shutil ''' @@ -8,58 +9,51 @@ ''' def check_path(path): - if not os.path.exists(path): - os.mkdir(path) - -def write_csv(name, source, dest, data, env): - destination = os.path.join(env.get_env('views101dir'), source) - check_path(destination) - destination = os.path.join(destination, dest) - check_path(destination) - d = os.path.join(destination, name + '.csv') - with open(d, 'w') as f: - wr = csv.writer(f) - for item in data: - wr.writerow(item) + if not os.path.exists(path): + os.makedirs(path) + +def write_csv(target,csvData): + d = os.path.join(target, 'data.csv') + with open(d, 'w') as f: + wr = csv.writer(f) + for item in csvData: + wr.writerow(item) + +def assemble_piechart(endFolderName,xName,yName,xValues,yValues, moduleName, env): + dataPath = os.path.join(env.get_env('views101dir'),moduleName,endFolderName) + mainPath = os.path.join(env.get_env('views101dir'),moduleName) + check_path(dataPath) + chartName = "pie" + csvData = [] + labels = ['age','population'] + csvData.append(labels) + for x,y in zip(xValues,yValues): + csvData.append([x,y]) + write_csv(dataPath,csvData) + copyTemplateToTarget(moduleName,endFolderName,chartName,env) + manageLinkFile(endFolderName,mainPath) + +def manageLinkFile(endFolderName,mainPath): + data = {"link":[endFolderName]} + if not os.path.isfile(mainPath+os.sep+'links.json'): + target = os.path.join(mainPath,'links.json') + else: + with open(mainPath+os.sep+'links.json', 'r') as f: + data=json.load(f) + if endFolderName not in data["link"]: + data["link"].append(endFolderName) + with open(mainPath+os.sep+'links.json', 'w') as f: + json.dump(data, f) + +def copyTemplateToTarget(moduleName,endFolderName,chartName,env): + shutil.copyfile(env.get_env('modules101dir')+os.sep+'visualization'+os.sep+'template'+os.sep+chartName+'.html',env.get_env('views101dir')+os.sep+moduleName+os.sep+endFolderName+os.sep+'chart.html') -def write_tsv(name, source, dest, data, env): - destination = env.get_env('views101dir') - destination = os.path.join(destination, source, dest) - check_path(destination) - d = os.path.join(destination, name + '.tsv') - with open(d, 'w') as f: - wr = csv.writer(f, delimiter='\t') - for item in data: - wr.writerow(item) - - -''' -write functions -''' - -def assemble_barchart(xName,yName,xValues,yValues, source, dest, env): - out = [] - labels = ['letter','frequency'] - out.append(labels) - for x,y in zip(xValues,yValues): - out.append([x,y]) - write_tsv('bar', source, dest, out, env) - -def assemble_piechart(xName,yName,xValues,yValues, source, dest, env): - out = [] - labels = [xName,yName] - out.append(labels) - for x,y in zip(xValues,yValues): - out.append([x,y]) - write_csv('pie', source, dest, out, env) - - ''' program ''' def run(env, res): - # modules - from .module.locPerContribution import run as runLocPerContribution - runLocPerContribution(env, res) + # modules + from .module.locPerContribution import run as runLocPerContribution + runLocPerContribution(env, res) From fe1069873f7a5f14983833f8d098ddc90ac89451 Mon Sep 17 00:00:00 2001 From: Andre Emmerichs Date: Sun, 12 Mar 2017 20:06:10 +0100 Subject: [PATCH 14/43] Improving visualistion withut module-Prototype TODO: Need a template folder in 101worker directory and more --- bin/worker_lib/env.py | 7 ++- bin/worker_lib/executor.py | 12 +++++- bin/worker_lib/visualiser.py | 48 +++++++++++++++++++++ modules/locPerContribution/visualisation.py | 33 +++++--------- 4 files changed, 75 insertions(+), 25 deletions(-) create mode 100644 bin/worker_lib/visualiser.py diff --git a/bin/worker_lib/env.py b/bin/worker_lib/env.py index 61635d62..af096a6e 100644 --- a/bin/worker_lib/env.py +++ b/bin/worker_lib/env.py @@ -1,6 +1,7 @@ import os import json import logging +from .visualiser import create_piechart as c_piechart def abs_path(path): return os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', path)) @@ -100,6 +101,9 @@ def write_dump(dump_name, data): json.dump(data, f, indent=4) logging.debug('Wrote dump %s at %s', dump_name, d) + def create_piechart(name,xName,yName,xValue,yValue): + c_piechart(name,xName,yName,xValue,yValue,get_env('views101dir')) + return AttrDict({ 'get_env': get_env, 'write_derived_resource': write_derived_resource, @@ -108,5 +112,6 @@ def write_dump(dump_name, data): 'read_dump': read_dump, 'write_dump': write_dump, 'get_derived_resource': get_derived_resource, - 'remove_dump': remove_dump + 'remove_dump': remove_dump, + 'create_piechart': create_piechart }) diff --git a/bin/worker_lib/executor.py b/bin/worker_lib/executor.py index d1c724b0..8870a6d6 100644 --- a/bin/worker_lib/executor.py +++ b/bin/worker_lib/executor.py @@ -23,6 +23,11 @@ def _exec(self, change): def run(self, changes): for change in changes: self._exec(change) + ####### added to create Image ################ + if self._module.config.get('visualisation') == True : + print("Creating Image") + self._module.createImage(self._env) + ############################################## class FileFullSweepExecutor(Executor): @@ -41,7 +46,7 @@ def run(self, changes): ####### added to create Image ################ if self._module.config.get('visualisation') == True : print("Creating Image") - self._module.createImage(self._env,env) + self._module.createImage(self._env) ############################################## class AllFullSweepExecutor(Executor): @@ -57,3 +62,8 @@ def _exec(self): def run(self, changes): self._exec() + ####### added to create Image ################ + if self._module.config.get('visualisation') == True : + print("Creating Image") + self._module.createImage(self._env) + ############################################## diff --git a/bin/worker_lib/visualiser.py b/bin/worker_lib/visualiser.py new file mode 100644 index 00000000..e14c2b60 --- /dev/null +++ b/bin/worker_lib/visualiser.py @@ -0,0 +1,48 @@ +import os +import json +import csv +import shutil + + +def check_path(path): + if not os.path.exists(path): + os.makedirs(path) + +def write_csv(target,csvData): + d = os.path.join(target, 'data.csv') + with open(d, 'w') as f: + wr = csv.writer(f) + for item in csvData: + wr.writerow(item) + +def create_piechart(name,xName,yName,xValues,yValues,path): + dataPath = os.path.join(path,name) + check_path(dataPath) + templateName = "pie" + csvData = [] + labels = ['age','population'] + csvData.append(labels) + for x,y in zip(xValues,yValues): + csvData.append([x,y]) + write_csv(dataPath,csvData) + copyTemplateToTarget(path,name,templateName) + manageLinkFile(name,path) + +def manageLinkFile(endFolderName,mainPath): + data = {"link":[endFolderName]} + if not os.path.isfile(mainPath+os.sep+'links.json'): + target = os.path.join(mainPath,'links.json') + else: + with open(mainPath+os.sep+'links.json', 'r') as f: + data=json.load(f) + if endFolderName not in data["link"]: + data["link"].append(endFolderName) + with open(mainPath+os.sep+'links.json', 'w') as f: + json.dump(data, f) + +def copyTemplateToTarget(path,name,templateName): + #hard-code template folder path?? + templatePath = os.path.join('template',templateName+'.html') + targetPath = os.path.join(path,name,'chart.html') + shutil.copyfile(templatePath,targetPath) + diff --git a/modules/locPerContribution/visualisation.py b/modules/locPerContribution/visualisation.py index d7fa0d91..baec8236 100644 --- a/modules/locPerContribution/visualisation.py +++ b/modules/locPerContribution/visualisation.py @@ -1,27 +1,14 @@ -import numpy -import matplotlib.pyplot as plotting -import os - -def createImage(env,workerlibEnv): +def createImage(env): data = env.read_dump('locPerContribution') - contributions = [] - loc = [] - for contribution in data.keys(): - contributions.append(contribution) - loc.append(data[contribution]) - plotting.bar(numpy.arange(len(contributions)),loc,0.35, color = 'b') - plotting.xlabel("Contribution") - plotting.ylabel("Lines of Code") - plotting.xticks(numpy.arange(len(contributions))+0.35/2., contributions) - plotting.title("LocPerContribution") + out = [] + xName = 'Contribution' + yName = 'Lines of Code' + xValue = [] + yValue = [] + for key, value in data.items(): + xValue.append(key) + yValue.append(value) - ### check if path exists, else create it ### - if not os.path.exists(workerlibEnv['views101dir']): - os.mkdir(workerlibEnv['views101dir']) - path = workerlibEnv['views101dir']+'/locPerContribution' - if not os.path.exists(path): - os.mkdir(path) - ############################################ + env.create_piechart('Piechart1',xName,yName,xValue,yValue) - plotting.savefig(path+'/chart.png') From db82a681a974f4cee3d8cd20191a9013cc75e2ef Mon Sep 17 00:00:00 2001 From: Torsten Date: Mon, 13 Mar 2017 09:12:47 +0100 Subject: [PATCH 15/43] Update pie.html added License information. --- modules/visualization/sample/pie.html | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/modules/visualization/sample/pie.html b/modules/visualization/sample/pie.html index d2384c50..ea864229 100644 --- a/modules/visualization/sample/pie.html +++ b/modules/visualization/sample/pie.html @@ -1,5 +1,6 @@ + + + + + + + + From ab61152c2893efec46770cc18262f47bba5d6e14 Mon Sep 17 00:00:00 2001 From: Torsten Date: Mon, 13 Mar 2017 09:14:43 +0100 Subject: [PATCH 16/43] Update bar.html added license information --- modules/visualization/sample/bar.html | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/modules/visualization/sample/bar.html b/modules/visualization/sample/bar.html index 99b09516..0520794a 100644 --- a/modules/visualization/sample/bar.html +++ b/modules/visualization/sample/bar.html @@ -1,5 +1,6 @@ + + + + + - + + + From d3fa38afd70d80d15583de04fcb5e10f1d9ed7a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christopher=20Sch=C3=A4fer?= Date: Fri, 17 Mar 2017 12:14:37 +0100 Subject: [PATCH 17/43] added simpleComments for testing, added documentation to simpleSentiments --- modules/simpleComments/README.md | 12 +++ modules/simpleComments/__init__.py | 152 +++++++++++++++++++++++++++ modules/simpleSentiments/README.md | 3 +- modules/simpleSentiments/__init__.py | 4 +- 4 files changed, 169 insertions(+), 2 deletions(-) create mode 100644 modules/simpleComments/README.md create mode 100644 modules/simpleComments/__init__.py diff --git a/modules/simpleComments/README.md b/modules/simpleComments/README.md new file mode 100644 index 00000000..300edbc2 --- /dev/null +++ b/modules/simpleComments/README.md @@ -0,0 +1,12 @@ +# simpleComments + +Comments per File + +# Description + +This module extracts the Comments of every contribution per file and saves them as a .comments.json resource file (Output: "comment1 comment2 com ment3"). It extracts both Inline and Block Comments. +This is a simpler version of the extractComments module. + +# Requirements + +This module requires matchLanguage to be run first. \ No newline at end of file diff --git a/modules/simpleComments/__init__.py b/modules/simpleComments/__init__.py new file mode 100644 index 00000000..48329ad5 --- /dev/null +++ b/modules/simpleComments/__init__.py @@ -0,0 +1,152 @@ +import os +import json +import unittest +from unittest.mock import Mock + + +config = { + 'wantdiff': True, + 'wantsfiles': True, + 'threadsafe': True, + 'behavior': { + 'creates': [['resource', 'comments']], + 'uses': [['resource', 'lang']] + } +} + +generic = ['//', '///'] +generic_block_start = ['/*', '/**'] +generic_block_end = ['*/'] +generic_ignore = ['*'] + +haskell = ['--'] +haskell_block_start = ['{-'] +haskell_block_end = ['-}'] + +perl_ruby = ['#'] +perl_ruby_block_start = ['=begin'] +perl_ruby_block_end = ['=end', '=cut'] + +python = ['#'] +python_block = ['\'\'\'', '\"\"\"'] + + +def collect_comments(context, f): + source = context.get_primary_resource(f) + lang = context.get_derived_resource(f, 'lang') + inline = [] + block_start = [] + block_end = [] + ignore = [] + if (lang == 'Python'): + inline = python + block_start = python_block + block_end = python_block + elif (lang == 'Perl' or lang == 'Ruby'): + inline = perl_ruby + block_start = perl_ruby_block_start + block_end = perl_ruby_block_end + elif (lang == 'Haskell'): + inline = haskell + block_start = haskell_block_start + block_end = haskell_block_end + else: + inline = generic + block_start = generic_block_start + block_end = generic_block_end + ignore = generic_ignore + + line_comment = False + block_comment = False + + comments = '' + for line in source.split('\n'): + for word in line.split(' '): + if (word in inline): + line_comment = True + elif (word in block_start): + if (lang == 'Python' and block_comment is True): + block_comment = False + else: + block_comment = True + elif (word in block_end): + block_comment = False + + if (line_comment or block_comment): + if not (word in inline or word in block_start or + word in block_end or word in ignore): + comments += word + ' ' + + line_comment = False + + return comments + + +def update_file(context, f): + # reads the content of the file (primary resource) + try: + comments = collect_comments(context, f) + context.write_derived_resource(f, comments, 'comments') + except UnicodeDecodeError: + context.write_derived_resource(f, '', 'comments') + + +def remove_file(context, f): + context.remove_derived_resource(f, 'comments') + + +def run(context, change): + # dispatch the modified file + if change['type'] == 'NEW_FILE': + update_file(context, change['file']) + + elif change['type'] == 'FILE_CHANGED': + update_file(context, change['file']) + + else: + remove_file(context, change['file']) + + +class extractCommentsTest(unittest.TestCase): + + def setUp(self): + self.env = Mock() + self.env.get_primary_resource.return_value = 'x = 5 // set x to 5\n/* set\ny to\n6 */\ny=6\nprint(x)\n' + self.env.get_derived_resource.return_value = 'Java' + + def test_run_new(self): + change = { + 'type': 'NEW_FILE', + 'file': 'some-file.java' + } + run(self.env, change) + + self.env.write_derived_resource.assert_called_with('some-file.java', + 'set x to 5 set y to 6 ', + 'comments') + + def test_run_changed(self): + change = { + 'type': 'FILE_CHANGED', + 'file': 'some-file.java' + } + run(self.env, change) + + self.env.write_derived_resource.assert_called_with('some-file.java', + 'set x to 5 set y to 6 ', + 'comments') + + def test_run_removed(self): + change = { + 'type': '', + 'file': 'some-file.java' + } + run(self.env, change) + + self.env.remove_derived_resource.assert_called_with('some-file.java', 'comments') + + +def test(): + suite = unittest.TestLoader().loadTestsFromTestCase(extractCommentsTest) + unittest.TextTestRunner(verbosity=2).run(suite) + diff --git a/modules/simpleSentiments/README.md b/modules/simpleSentiments/README.md index 75b0b60e..96428c88 100644 --- a/modules/simpleSentiments/README.md +++ b/modules/simpleSentiments/README.md @@ -4,7 +4,8 @@ Sentiment of comments per File # Description -This module uses TextBlob to analyse the sentiment of comments per File. A resource file named sentiment.json is saved containing the polarity and subjectivity of the comments. +This module uses TextBlob to analyse the sentiment of comments per File. Textblob is python library for language processing. A Textblob can be created by using any String. The sentiment property returns a tuple of the form (polarity, subjectivity), where polarity is a float within the range [-1.0, 1.0] and subjectivity within the range [0, 1.0] (0 means very objective and 1.0 very subjective). +A resource file named sentiment.json is saved containing the polarity and subjectivity of the comments (output-example: [0.3, 0.5] ). # Requirements diff --git a/modules/simpleSentiments/__init__.py b/modules/simpleSentiments/__init__.py index 1eb54ac6..e94eda89 100644 --- a/modules/simpleSentiments/__init__.py +++ b/modules/simpleSentiments/__init__.py @@ -15,11 +15,13 @@ } } - +# calculate Sentiments from comments ressource def calc_sentiment(context, f): comments = context.get_derived_resource(f, 'comments') + # creating a textblob object from comments.json comments_blob = TextBlob(comments) comment_sentiment = comments_blob.sentiment + # handling files without comments if (comments == ''): return 'N/A' else: From 30217fe12e0a576f497e52d26717b0ac67fe8a18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christopher=20Sch=C3=A4fer?= Date: Fri, 17 Mar 2017 13:08:39 +0100 Subject: [PATCH 18/43] small modifications commentSentimentPerLanguage --- modules/commentSentimentPerLanguage/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/commentSentimentPerLanguage/__init__.py b/modules/commentSentimentPerLanguage/__init__.py index ed9b91ef..011834ce 100644 --- a/modules/commentSentimentPerLanguage/__init__.py +++ b/modules/commentSentimentPerLanguage/__init__.py @@ -18,7 +18,7 @@ def run(env, res): data = env.read_dump('sentimentsPerLanguage') - langData = env.read_dump('LangPerContribution') + #langData = env.read_dump('LangPerContribution') helpData = env.read_dump('sentimentsPerContribution') if data is None: @@ -30,7 +30,9 @@ def run(env, res): folders = f.split(os.sep) contribution = folders[1] fileSentiment = env.get_derived_resource(f, 'sentiment') - lang = langData[contribution]['Main Language'] + #lang = langData[contribution]['Main Language'] + + lang = env.get_derived_resource(f, 'lang') fileName = folders[-1] if helpData.get(lang, None) is None: From b323c9234f392fd3d2139fa0f5c05819b9ed8db0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christopher=20Sch=C3=A4fer?= Date: Fri, 17 Mar 2017 14:02:37 +0100 Subject: [PATCH 19/43] small additions --- modules/simpleSentiments/README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/modules/simpleSentiments/README.md b/modules/simpleSentiments/README.md index 96428c88..70344584 100644 --- a/modules/simpleSentiments/README.md +++ b/modules/simpleSentiments/README.md @@ -9,4 +9,9 @@ A resource file named sentiment.json is saved containing the polarity and subjec # Requirements -This module requires a working installation of TextBlob and extractComments to be run first. \ No newline at end of file +This module requires a working installation of TextBlob and extractComments to be run first. + +# How to install TextBlob + +$ pip install -U textblob +$ python -m textblob.download_corpora \ No newline at end of file From 13bae7633b9e259da15901f33ac9e73403f7e4ff Mon Sep 17 00:00:00 2001 From: Andre Emmerichs Date: Fri, 17 Mar 2017 19:37:57 +0100 Subject: [PATCH 20/43] Improving ncloc Added test cases and ReadMe --- modules/ncloc/ReadMe.md | 48 ++++++++++++++++++ modules/ncloc/__init__.py | 101 +++++++++++++++++++++++++------------- 2 files changed, 116 insertions(+), 33 deletions(-) create mode 100644 modules/ncloc/ReadMe.md diff --git a/modules/ncloc/ReadMe.md b/modules/ncloc/ReadMe.md new file mode 100644 index 00000000..430606cb --- /dev/null +++ b/modules/ncloc/ReadMe.md @@ -0,0 +1,48 @@ +# ncloc + +##Characteristics +###config: +wantsdiff: True +wantsfiles: True +threadsafe: True + +###behavior: +creates: resource ncoc +uses: resource lang + +###imports: +None + +##Architecture +###init.py +Contains program and test functions. The program takes the language of the file, finds the corresponding +comment symbols in a language datastructure. + +Then it iterates over every single line and checks if it is in a single-line or in a block comment. +If we are not in a comment, it increments the ncloc value by 1. + +Limitations: + +The program only recognizes the start of a comment at the beginning of a line, and the end of a comment at the +end of a line. + + +##Output: +###value: +Contains the number of lines of code, which are not only comments +###structure: +{Integer} + +##Usage: +Run & test module with standard commands (bin/run_module, bin/test) + +##Metadata: +###Author: +Andre Emmerichs +###Creation: +context: PP16/17 +date: WS 16/17 +###Features: + +###Technology: + diff --git a/modules/ncloc/__init__.py b/modules/ncloc/__init__.py index 57ab3dd6..bee20891 100644 --- a/modules/ncloc/__init__.py +++ b/modules/ncloc/__init__.py @@ -1,6 +1,3 @@ -import os -import json - config = { 'wantdiff': True, 'wantsfiles': True, @@ -14,73 +11,62 @@ # this is the actual logic of the module def count_non_comment_lines(context, f): #datastructure for languages and their corresponding comment symbols - languages = {'Java': {'Single': ['//'], 'BlockStart': ['/*'], 'BlockEnd': ['*/']}, - 'Python': {'Single': ['#'], 'BlockStart': ["'''"], 'BlockEnd': ["'''"]}, - 'Haskell': {'Single': ['--'], 'BlockStart': ["{-"], 'BlockEnd': ["-}"]}} + languages = {'Java': {'Single': ['//'], 'Block': [{'BlockStart': '/*', 'BlockEnd': '*/'}]}, + 'Python': {'Single': ['#'], 'Block': [{'BlockStart': '"""', 'BlockEnd': '"""'},{'BlockStart': "'''", 'BlockEnd': "'''"},]}, + 'Haskell': {'Single': ['--'], 'Block': [{'BlockStart': '{-', 'BlockEnd': '-}'}]}} #get file source = context.get_primary_resource(f) #get language of file lang = context.get_derived_resource(f,'lang') ncloc = 0 - block = False - lineNumber = 0 - blockLineBegin = -1 # only needed when block start and end symbol are equal - + inBlock = False + #check if language is in datastructure if languages.get(lang) != None : #check if langauges have block/singe-line comments singleExists = languages[lang].get("Single") != None - blockExists = languages[lang].get("BlockStart") != None + blockExists = languages[lang].get("Block") != None - if blockExists: - equalBlockSymbols = languages[lang].get("BlockStart") == languages[lang].get("BlockEnd") #iterate over all lines for line in source.split('\n'): - lineNumber = lineNumber +1 if blockExists: #check if symbols are one of the given comment symbols for block comments - for blockstart in languages[lang].get("BlockStart"): + for block in languages[lang].get("Block"): + blockStart = block["BlockStart"] line = line.replace("\t","").replace(" ","").replace("\n","") - if line[0:len(blockstart)] == blockstart and block == False: - block = True - #set marker for blockLineBegin, if block symbols are equal, to prevent reading of - #block start as block end later - if equalBlockSymbols: - if len((line)) == len(blockstart): - blockLineBegin = lineNumber + if line[0:len(blockStart)] == blockStart and inBlock == False: + inBlock = True + blockEnd = block["BlockEnd"] + line = line[len(blockStart):len(line)] + break #if we are not in a Block comment, check if it's a single line comment... singleCheck = False - if block == False: + if inBlock == False: if singleExists: for single in languages[lang]["Single"]: if len(line)>len(single): if line[0:len(single)] == single: singleCheck = True #...if not increment sum of non comment lines of code - if singleCheck == False and block == False: + if singleCheck == False and inBlock == False: ncloc = ncloc + 1 #check for block end symbols if blockExists: - for blockend in languages[lang]["BlockEnd"]: + if inBlock == True: #invert line line = line[::-1] + line = line.replace("\t","").replace(" ","").replace("\n","") #check if the first found signs are now the inverted signs of the endblock #of a comment (because we inverted the whole line before) - if line[0:len(blockend)] == (blockend)[::-1] and block == True: - #if block start and end sign are equal, take care that you do not read - #a block-start as a block-end - if equalBlockSymbols: - if not blockLineBegin == lineNumber: - block = False - else: - block = False + if line[0:len(blockEnd)] == (blockEnd)[::-1] and inBlock == True: + inBlock = False return ncloc @@ -106,3 +92,52 @@ def run(context, change): else: remove_file(context, change['file']) + + + +import unittest +from unittest.mock import Mock +import io + +class NclocTest(unittest.TestCase): + + def test_run_java(self): + change = { + 'type': 'NEW_FILE', + 'file': 'some-file.java' + } + self.env = Mock() + self.env.get_primary_resource.return_value = '//x = 5\n/*y=6*/\nprint(x)\nz=7\nw=8' + self.env.get_derived_resource.return_value = 'Java' + run(self.env, change) + + self.env.write_derived_resource.assert_called_with('some-file.java', 3, 'ncloc') + + def test_run_haskell(self): + change = { + 'type': 'FILE_CHANGED', + 'file': 'some-file.hs' + } + self.env = Mock() + self.env.get_primary_resource.return_value = "\t--x = 5\n\t{-y=6\nprint(x)-}\nz=7\nw=8" + self.env.get_derived_resource.return_value = 'Haskell' + run(self.env, change) + + self.env.write_derived_resource.assert_called_with('some-file.hs', 2, 'ncloc') + + def test_run_python(self): + change = { + 'type': 'FILE_CHANGED', + 'file': 'some-file.py' + } + self.env = Mock() + self.env.get_primary_resource.return_value = '\t#x = 5\n\t"""y=6\nprint(x)""" \nz=7\nw=8' + self.env.get_derived_resource.return_value = 'Python' + run(self.env, change) + + self.env.write_derived_resource.assert_called_with('some-file.py', 2, 'ncloc') + + +def test(): + suite = unittest.TestLoader().loadTestsFromTestCase(NclocTest) + unittest.TextTestRunner(verbosity=2).run(suite) From 8a2f458d9870f9035a86a151005d738383ac0223 Mon Sep 17 00:00:00 2001 From: Marcel Date: Wed, 10 May 2017 15:31:49 +0200 Subject: [PATCH 21/43] Update --- bin/worker_lib/env.py | 4 +- bin/worker_lib/visualiser.py | 73 +++++++++++++++---- general_information/readme schema | 8 -- modules/locPerContribution/ReadMe.md | 39 ++++++++++ modules/locPerContribution/visualisation.py | 2 +- modules/matchLanguage/ReadMe.md | 39 ++++++++++ modules/simpleLOC/ReadMe.md | 39 ++++++++++ .../visualization/module/developerRanks.py | 16 ++++ modules/visualization/sample/sample.py | 9 --- .../sample => templates}/bar.html | 0 .../sample => templates}/pie.html | 4 +- 11 files changed, 196 insertions(+), 37 deletions(-) delete mode 100644 general_information/readme schema create mode 100644 modules/locPerContribution/ReadMe.md create mode 100644 modules/matchLanguage/ReadMe.md create mode 100644 modules/simpleLOC/ReadMe.md create mode 100644 modules/visualization/module/developerRanks.py delete mode 100644 modules/visualization/sample/sample.py rename {modules/visualization/sample => templates}/bar.html (100%) rename {modules/visualization/sample => templates}/pie.html (92%) diff --git a/bin/worker_lib/env.py b/bin/worker_lib/env.py index af096a6e..316c10bd 100644 --- a/bin/worker_lib/env.py +++ b/bin/worker_lib/env.py @@ -101,8 +101,8 @@ def write_dump(dump_name, data): json.dump(data, f, indent=4) logging.debug('Wrote dump %s at %s', dump_name, d) - def create_piechart(name,xName,yName,xValue,yValue): - c_piechart(name,xName,yName,xValue,yValue,get_env('views101dir')) + def create_piechart(name,moduleName,xName,yName,xValue,yValue): + c_piechart(name,moduleName,xName,yName,xValue,yValue,get_env('views101dir')) return AttrDict({ 'get_env': get_env, diff --git a/bin/worker_lib/visualiser.py b/bin/worker_lib/visualiser.py index e14c2b60..27fe2c40 100644 --- a/bin/worker_lib/visualiser.py +++ b/bin/worker_lib/visualiser.py @@ -8,15 +8,16 @@ def check_path(path): if not os.path.exists(path): os.makedirs(path) -def write_csv(target,csvData): - d = os.path.join(target, 'data.csv') +def write_csv(target,csvData,name): + d = os.path.join(target, name + '.csv') with open(d, 'w') as f: wr = csv.writer(f) for item in csvData: wr.writerow(item) + return name + '.csv' -def create_piechart(name,xName,yName,xValues,yValues,path): - dataPath = os.path.join(path,name) +def create_piechart(name,moduleName,xName,yName,xValues,yValues,path): + dataPath = os.path.join(path,moduleName) check_path(dataPath) templateName = "pie" csvData = [] @@ -24,25 +25,67 @@ def create_piechart(name,xName,yName,xValues,yValues,path): csvData.append(labels) for x,y in zip(xValues,yValues): csvData.append([x,y]) - write_csv(dataPath,csvData) - copyTemplateToTarget(path,name,templateName) - manageLinkFile(name,path) + csvname = write_csv(dataPath,csvData, name) + copyTemplateToTarget(path,name,moduleName, csvname,templateName) + folderIterator(path) -def manageLinkFile(endFolderName,mainPath): - data = {"link":[endFolderName]} + +def folderIterator(path): + data = {} + for dirs in os.listdir(path): + htmlfiles = [] + for files in os.listdir(path + os.sep + dirs): + if files.endswith('.html'): + htmlfiles.append(files) + data[dirs] = htmlfiles + updateIndex(path, data) + +def updateIndex(path, data): + if not os.path.isfile(path + os.sep + 'index.html'): + target = open(path + os.sep + 'index.html', 'w') + + + + +def manageLinkFile(moduleName, chartName,mainPath): + name = moduleName + os.sep + chartName + data = {"link":[name]} if not os.path.isfile(mainPath+os.sep+'links.json'): target = os.path.join(mainPath,'links.json') else: with open(mainPath+os.sep+'links.json', 'r') as f: data=json.load(f) - if endFolderName not in data["link"]: - data["link"].append(endFolderName) + if name not in data["link"]: + data["link"].append(name) with open(mainPath+os.sep+'links.json', 'w') as f: json.dump(data, f) -def copyTemplateToTarget(path,name,templateName): +def copyTemplateToTarget(path,name,moduleName, dataname, templateName): #hard-code template folder path?? - templatePath = os.path.join('template',templateName+'.html') - targetPath = os.path.join(path,name,'chart.html') - shutil.copyfile(templatePath,targetPath) + templatePath = os.path.join('templates',templateName+'.html') + targetPath = os.path.join(path,moduleName, name + '.html') + insertTemplate(templatePath, targetPath, dataname) + +def insertTemplate(template, targetpath, dataname): + inputfile = open(template) + outputfile = open(targetpath, 'w') + + for line in inputfile: + if line.find('enterDataHere'): + line = line.replace('enterDataHere', dataname) + outputfile.writelines(line) + + + + + + + + + + + + + + diff --git a/general_information/readme schema b/general_information/readme schema deleted file mode 100644 index a7a497f4..00000000 --- a/general_information/readme schema +++ /dev/null @@ -1,8 +0,0 @@ -Readme schema: - -- Headline -- Needed imports -- Dependencies (links to other modules) -- Input / Output behaviour -- Example -- Natural language description diff --git a/modules/locPerContribution/ReadMe.md b/modules/locPerContribution/ReadMe.md new file mode 100644 index 00000000..6d9eb213 --- /dev/null +++ b/modules/locPerContribution/ReadMe.md @@ -0,0 +1,39 @@ +# simpleLOC + +##Characteristics +###config: +wantsdiff: True +wantsfiles: True +threadsafe: True + +###behavior: +creates: resource loc + +###imports: +os +json + +##Architecture +###init.py +Uses the source code of the files to count the number of lines of code and stores it into a derived resource. +The file also contains tests for all possibilities of change types. + +##Output: +###value: +Number of lines of code in the files. +###structure: +{Number of lines} + +##Usage: +Run & test module with standard commands (bin/run_module, bin/test) + +##Metadata: +###Author: +Marcel Michels +###Creation: +context: PP16/17 +date: SS 17 +###Features: + +###Technology: + diff --git a/modules/locPerContribution/visualisation.py b/modules/locPerContribution/visualisation.py index baec8236..d7237cd4 100644 --- a/modules/locPerContribution/visualisation.py +++ b/modules/locPerContribution/visualisation.py @@ -9,6 +9,6 @@ def createImage(env): xValue.append(key) yValue.append(value) - env.create_piechart('Piechart1',xName,yName,xValue,yValue) + env.create_piechart('Piechart1','locPerContribution',xName,yName,xValue,yValue) diff --git a/modules/matchLanguage/ReadMe.md b/modules/matchLanguage/ReadMe.md new file mode 100644 index 00000000..c3f6dff0 --- /dev/null +++ b/modules/matchLanguage/ReadMe.md @@ -0,0 +1,39 @@ +# matchLanguage + +##Characteristics +###config: +wantsdiff: True +wantsfiles: True +threadsafe: True + +###behavior: +creates: resource lang + +###imports: +os +json + +##Architecture +###init.py +Extracts the file suffix and writes the corresponding language into a derived resource. +Also contains tests to cover all possibilities of change types. + +##Output: +###value: +All comments of the language of the file +###structure: +{language} + +##Usage: +Run & test module with standard commands (bin/run_module, bin/test) + +##Metadata: +###Author: +Marcel Michels +###Creation: +context: PP16/17 +date: SS 17 +###Features: + +###Technology: + diff --git a/modules/simpleLOC/ReadMe.md b/modules/simpleLOC/ReadMe.md new file mode 100644 index 00000000..6d9eb213 --- /dev/null +++ b/modules/simpleLOC/ReadMe.md @@ -0,0 +1,39 @@ +# simpleLOC + +##Characteristics +###config: +wantsdiff: True +wantsfiles: True +threadsafe: True + +###behavior: +creates: resource loc + +###imports: +os +json + +##Architecture +###init.py +Uses the source code of the files to count the number of lines of code and stores it into a derived resource. +The file also contains tests for all possibilities of change types. + +##Output: +###value: +Number of lines of code in the files. +###structure: +{Number of lines} + +##Usage: +Run & test module with standard commands (bin/run_module, bin/test) + +##Metadata: +###Author: +Marcel Michels +###Creation: +context: PP16/17 +date: SS 17 +###Features: + +###Technology: + diff --git a/modules/visualization/module/developerRanks.py b/modules/visualization/module/developerRanks.py new file mode 100644 index 00000000..13af4b46 --- /dev/null +++ b/modules/visualization/module/developerRanks.py @@ -0,0 +1,16 @@ +from ..program import assemble_piechart + +def run(env, res): + + data = env.read_dump('locPerContribution') + out = [] + xName = 'Contribution' + yName = 'Lines of Code' + xValues = [] + yValues = [] + for key, value in data.items(): + xValues.append(key) + yValues.append(value) + + assemble_piechart("normal",xName, yName, xValues, yValues, 'locPerContribution', env) + assemble_piechart("inverted",yName, xName, xValues, yValues, 'locPerContribution', env) diff --git a/modules/visualization/sample/sample.py b/modules/visualization/sample/sample.py deleted file mode 100644 index 476c2dbc..00000000 --- a/modules/visualization/sample/sample.py +++ /dev/null @@ -1,9 +0,0 @@ -from ..program import assemble_barchart - -def run(env, res): - - ''' - insert code here - move file to according folder - import run in program.py - ''' diff --git a/modules/visualization/sample/bar.html b/templates/bar.html similarity index 100% rename from modules/visualization/sample/bar.html rename to templates/bar.html diff --git a/modules/visualization/sample/pie.html b/templates/pie.html similarity index 92% rename from modules/visualization/sample/pie.html rename to templates/pie.html index ea864229..ed92fb2d 100644 --- a/modules/visualization/sample/pie.html +++ b/templates/pie.html @@ -16,7 +16,7 @@ - + + + + + + + + + +
+ + + + + diff --git a/templates/googleCharts/pieChart.html b/templates/googleCharts/pieChart.html new file mode 100644 index 00000000..5fc3d4fb --- /dev/null +++ b/templates/googleCharts/pieChart.html @@ -0,0 +1,37 @@ + + + + + + + + + + + + + +
+ + + + + From f28d9b9e9e25722e593d0e03569310596222d192 Mon Sep 17 00:00:00 2001 From: Marcel Date: Wed, 17 May 2017 13:58:38 +0200 Subject: [PATCH 25/43] UPdating creation of index.html --- bin/worker_lib/env.py | 5 +- bin/worker_lib/executor.py | 4 ++ bin/worker_lib/visualiser.py | 54 +++++++++++++++++++-- modules/locPerContribution/visualisation.py | 3 +- templates/index.html | 6 +++ 5 files changed, 64 insertions(+), 8 deletions(-) create mode 100644 templates/index.html diff --git a/bin/worker_lib/env.py b/bin/worker_lib/env.py index 316c10bd..a83bb359 100644 --- a/bin/worker_lib/env.py +++ b/bin/worker_lib/env.py @@ -101,9 +101,12 @@ def write_dump(dump_name, data): json.dump(data, f, indent=4) logging.debug('Wrote dump %s at %s', dump_name, d) - def create_piechart(name,moduleName,xName,yName,xValue,yValue): + def create_piechart(name,xName,yName,xValue,yValue): + moduleName = str(module).replace("'","").split(" ")[1] c_piechart(name,moduleName,xName,yName,xValue,yValue,get_env('views101dir')) + + return AttrDict({ 'get_env': get_env, 'write_derived_resource': write_derived_resource, diff --git a/bin/worker_lib/executor.py b/bin/worker_lib/executor.py index 8870a6d6..16d5b1c8 100644 --- a/bin/worker_lib/executor.py +++ b/bin/worker_lib/executor.py @@ -4,6 +4,7 @@ import os import sys import traceback +import shutil class Executor(object): @@ -46,6 +47,9 @@ def run(self, changes): ####### added to create Image ################ if self._module.config.get('visualisation') == True : print("Creating Image") + folderName = str(self._module).replace("'","").split(" ")[1] + path = self._env.get_env('views101dir') + shutil.rmtree(path + os.sep + folderName) self._module.createImage(self._env) ############################################## diff --git a/bin/worker_lib/visualiser.py b/bin/worker_lib/visualiser.py index 27fe2c40..d608239d 100644 --- a/bin/worker_lib/visualiser.py +++ b/bin/worker_lib/visualiser.py @@ -34,6 +34,8 @@ def folderIterator(path): data = {} for dirs in os.listdir(path): htmlfiles = [] + if dirs.endswith('index.html'): + continue for files in os.listdir(path + os.sep + dirs): if files.endswith('.html'): htmlfiles.append(files) @@ -41,11 +43,53 @@ def folderIterator(path): updateIndex(path, data) def updateIndex(path, data): - if not os.path.isfile(path + os.sep + 'index.html'): - target = open(path + os.sep + 'index.html', 'w') - - - + + targetpath = path + os.sep + 'index.html' + templatepath = os.path.join('templates','index.html') + shutil.copyfile(templatepath, targetpath) + for key in data.keys(): + skiplines = False + inputfile = open(targetpath, 'r') + tempfile = open(path + os.sep + 'temp', 'w') + keyfound = False + + for line in inputfile: + if line.find('') != -1: + skiplines = True + keyfound = True + tempfile.writelines(line) + if line.find('') != -1: + skiplines = False + line = "" + if not skiplines: + tempfile.writelines(line) + + if not keyfound: + os.remove(path + os.sep + 'temp') + inputfile = open(path + os.sep + 'index.html', 'r') + tempfile = open(path + os.sep + 'temp', 'w') + for line in inputfile: + if line.find('') != -1: + line = '' + "\n" + '' + "\n" + tempfile.writelines(line) + tempfile = open(path + os.sep + 'temp', 'r') + inputfile = open(path + os.sep + 'index.html', 'w') + for line in tempfile: + if line.find('') != -1: + line = createHtmlTag(key, data[key]) + inputfile.writelines(line) + os.remove(path + os.sep + 'temp') + +def createHtmlTag(key, array): + htmlNote = '' + "\n" + header = "

"+key+"


\n" + fullCode = htmlNote + header + for item in array: + linkperitem = ""+item+"
\n" + fullCode = fullCode + linkperitem + endNode = "\n \n" + fullcode = fullCode + endNode + return fullcode def manageLinkFile(moduleName, chartName,mainPath): name = moduleName + os.sep + chartName diff --git a/modules/locPerContribution/visualisation.py b/modules/locPerContribution/visualisation.py index d7237cd4..9bc5de1d 100644 --- a/modules/locPerContribution/visualisation.py +++ b/modules/locPerContribution/visualisation.py @@ -9,6 +9,5 @@ def createImage(env): xValue.append(key) yValue.append(value) - env.create_piechart('Piechart1','locPerContribution',xName,yName,xValue,yValue) - + env.create_piechart('Piechart1',xName,yName,xValue,yValue) diff --git a/templates/index.html b/templates/index.html new file mode 100644 index 00000000..e3a463f2 --- /dev/null +++ b/templates/index.html @@ -0,0 +1,6 @@ + + + + + + From dd9c1f77fc2aeb203e359828f618387369091932 Mon Sep 17 00:00:00 2001 From: Skaduwa Date: Wed, 17 May 2017 14:50:37 +0200 Subject: [PATCH 26/43] google implementation for locPerContribution --- bin/worker_lib/env.py | 12 ++++++++- bin/worker_lib/visualiser.py | 30 +++++++++++++++++++++ modules/locPerContribution/visualisation.py | 10 ++++++- templates/googleCharts/barChart.html | 4 +-- templates/googleCharts/pieChart.html | 6 ++--- 5 files changed, 55 insertions(+), 7 deletions(-) diff --git a/bin/worker_lib/env.py b/bin/worker_lib/env.py index 316c10bd..6b6a6649 100644 --- a/bin/worker_lib/env.py +++ b/bin/worker_lib/env.py @@ -2,6 +2,8 @@ import json import logging from .visualiser import create_piechart as c_piechart +from .visualiser import create_googleChart_pie as create_googleChart_pie_be +from .visualiser import create_googleChart_bar as create_googleChart_bar_be def abs_path(path): return os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', path)) @@ -104,6 +106,12 @@ def write_dump(dump_name, data): def create_piechart(name,moduleName,xName,yName,xValue,yValue): c_piechart(name,moduleName,xName,yName,xValue,yValue,get_env('views101dir')) + def create_googleChart_pie(ModuleName, fileName, options, data): + create_googleChart_pie_be(ModuleName, fileName, options, data, get_env('views101dir')) + + def create_googleChart_bar(ModuleName, fileName, options, data): + create_googleChart_bar_be(ModuleName, fileName, options, data, get_env('views101dir')) + return AttrDict({ 'get_env': get_env, 'write_derived_resource': write_derived_resource, @@ -113,5 +121,7 @@ def create_piechart(name,moduleName,xName,yName,xValue,yValue): 'write_dump': write_dump, 'get_derived_resource': get_derived_resource, 'remove_dump': remove_dump, - 'create_piechart': create_piechart + 'create_piechart': create_piechart, + 'create_googleChart_pie': create_googleChart_pie, + 'create_googleChart_bar': create_googleChart_pie }) diff --git a/bin/worker_lib/visualiser.py b/bin/worker_lib/visualiser.py index 27fe2c40..792c9759 100644 --- a/bin/worker_lib/visualiser.py +++ b/bin/worker_lib/visualiser.py @@ -76,6 +76,36 @@ def insertTemplate(template, targetpath, dataname): outputfile.writelines(line) +################# +# Google Charts # +################# + +# barChart +def create_googleChart_bar(moduleName, fileName, options, data, path): + sourcePath = os.path.join('templates', 'googleCharts', 'barChart.html') + targetPath = os.path.join(path, moduleName, fileName + '.html') + inputFile = open(sourcePath) + check_path(targetPath) + outputFile = open(targetPath, 'w') + for line in inputFile: + if line.find('INSERT_DATA'): + line = line.replace('INSERT_DATA', str(data)) + if line.find('INSERT_OPTIONS'): + line = line.replace('INSERT_OPTIONS', options) +# pieChart +def create_googleChart_pie(moduleName, fileName, options, data, path): + sourcePath = os.path.join('templates', 'googleCharts', 'pieChart.html') + targetPath = os.path.join(path, moduleName, fileName + '.html') + inputFile = open(sourcePath) + check_path(targetPath) + outputFile = open(targetPath, 'w') + for line in inputFile: + if line.find('INSERT_DATA'): + line = line.replace('INSERT_DATA', str(data)) + if line.find('INSERT_OPTIONS'): + line = line.replace('INSERT_OPTIONS', options) + outputFile.writelines(line) + diff --git a/modules/locPerContribution/visualisation.py b/modules/locPerContribution/visualisation.py index d7237cd4..1c6138aa 100644 --- a/modules/locPerContribution/visualisation.py +++ b/modules/locPerContribution/visualisation.py @@ -5,10 +5,18 @@ def createImage(env): yName = 'Lines of Code' xValue = [] yValue = [] + + data_in = [] + options_in = '{title: \'locPerContribution\'}' + + data_in.append(['contribution', 'loc']) + for key, value in data.items(): xValue.append(key) yValue.append(value) + data_in.append([key, value]) - env.create_piechart('Piechart1','locPerContribution',xName,yName,xValue,yValue) + #env.create_piechart('Piechart1','locPerContribution',xName,yName,xValue,yValue) + env.create_googleChart_pie('locPerContribution', 'googlePieFTW', options_in, data_in) diff --git a/templates/googleCharts/barChart.html b/templates/googleCharts/barChart.html index d9e9d60b..6763a835 100644 --- a/templates/googleCharts/barChart.html +++ b/templates/googleCharts/barChart.html @@ -10,9 +10,9 @@ function drawChart() { - var data_in = INSTERT_DATA; + var data_in = INSERT_DATA; - var options_in = INSERT_DATA; + var options_in = INSERT_OPTIONS; var data = google.visualization.arrayToDataTable(data_in); diff --git a/templates/googleCharts/pieChart.html b/templates/googleCharts/pieChart.html index 5fc3d4fb..f7000743 100644 --- a/templates/googleCharts/pieChart.html +++ b/templates/googleCharts/pieChart.html @@ -10,13 +10,13 @@ function drawChart() { - var data_in = INSTERT_DATA; + var data_in = INSERT_DATA; - var options_in = INSERT_DATA; + var options_in = INSERT_OPTIONS; var data = google.visualization.arrayToDataTable(data_in); - var options = {options_in}; + var options = options_in; var chart = new google.visualization.PieChart(document.getElementById('googleChart')); From e45b1591d2de45352fc0e54f48ca776f8a639433 Mon Sep 17 00:00:00 2001 From: Marcel Date: Wed, 17 May 2017 14:56:32 +0200 Subject: [PATCH 27/43] update devRanks and more --- bin/worker_lib/executor.py | 6 +++ bin/worker_lib/visualiser.py | 8 ++-- modules/developerRanks/__init__.py | 49 ++------------------- modules/developerRanks/program.py | 36 +++++++++++++++ modules/developerRanks/visualisation.py | 12 +++++ modules/locPerContribution/visualisation.py | 3 +- templates/index.html | 47 +++++++++++++++++++- 7 files changed, 109 insertions(+), 52 deletions(-) create mode 100644 modules/developerRanks/program.py create mode 100644 modules/developerRanks/visualisation.py diff --git a/bin/worker_lib/executor.py b/bin/worker_lib/executor.py index 16d5b1c8..c4a23cdd 100644 --- a/bin/worker_lib/executor.py +++ b/bin/worker_lib/executor.py @@ -27,6 +27,9 @@ def run(self, changes): ####### added to create Image ################ if self._module.config.get('visualisation') == True : print("Creating Image") + folderName = str(self._module).replace("'","").split(" ")[1] + path = self._env.get_env('views101dir') + shutil.rmtree(path + os.sep + folderName) self._module.createImage(self._env) ############################################## @@ -69,5 +72,8 @@ def run(self, changes): ####### added to create Image ################ if self._module.config.get('visualisation') == True : print("Creating Image") + folderName = str(self._module).replace("'","").split(" ")[1] + path = self._env.get_env('views101dir') + shutil.rmtree(path + os.sep + folderName) self._module.createImage(self._env) ############################################## diff --git a/bin/worker_lib/visualiser.py b/bin/worker_lib/visualiser.py index d608239d..d773d4b2 100644 --- a/bin/worker_lib/visualiser.py +++ b/bin/worker_lib/visualiser.py @@ -70,7 +70,7 @@ def updateIndex(path, data): tempfile = open(path + os.sep + 'temp', 'w') for line in inputfile: if line.find('') != -1: - line = '' + "\n" + '' + "\n" + line = '' + "\n" + '' + "\n\n" tempfile.writelines(line) tempfile = open(path + os.sep + 'temp', 'r') inputfile = open(path + os.sep + 'index.html', 'w') @@ -82,12 +82,12 @@ def updateIndex(path, data): def createHtmlTag(key, array): htmlNote = '' + "\n" - header = "

"+key+"


\n" + header = "
  • "+key+"
    • \n
      \n" fullCode = htmlNote + header for item in array: - linkperitem = ""+item+"
      \n" + linkperitem = "
    • "+item+"
      • \n" fullCode = fullCode + linkperitem - endNode = "\n \n" + endNode = "\n
    \n
\n"+" \n" fullcode = fullCode + endNode return fullcode diff --git a/modules/developerRanks/__init__.py b/modules/developerRanks/__init__.py index cfcff690..4c1a1a08 100644 --- a/modules/developerRanks/__init__.py +++ b/modules/developerRanks/__init__.py @@ -1,7 +1,5 @@ from collections import Counter -import matplotlib.pylot as plt -import seaborn as sns config = { 'wantdiff': False, @@ -10,53 +8,14 @@ 'behavior': { 'uses': [['dump', 'wiki-links']], 'creates': [['dump', 'developerRanks']] - } + }, + 'visualisation': True } -def run(env): - wiki_dump = env.read_dump('wiki-links') +from .program import run +from .visualisation import createImage - pages = wiki_dump['wiki']['pages'] - - data = {} - contributions = filter(lambda p: 'Contribution' == p.get('p', ''), pages) - for c in contributions: - - languages = [c.get('Uses', [])] - languages = [p for language in languages for p in language] - - languages = list(filter(lambda u: u['p'] == 'Language', languages)) - languages = [language['n'].replace('_', ' ') for language in languages] - - contributors = [c.get('DevelopedBy', [])] - contributors = [p['n'] for contributor in contributors for p in contributor] - - - - - for language in languages: - if language in data.keys(): - data2 = data[language] - else: - data2 = {} - for contributor in contributors: - if contributor in data2.keys(): - data2[contributor] = data2[contributor] + 1 - else: - data2[contributor] = 1 - - data[language] = data2 - - env.write_dump('developerRanks', data) - - x = [] - y = [] - for (k,v) in data['Javascript']: - x.append(k) - y.append(v) - sns.barplot(x, y) - sns.plt.show() diff --git a/modules/developerRanks/program.py b/modules/developerRanks/program.py new file mode 100644 index 00000000..d19d58af --- /dev/null +++ b/modules/developerRanks/program.py @@ -0,0 +1,36 @@ +def run(env): + wiki_dump = env.read_dump('wiki-links') + + pages = wiki_dump['wiki']['pages'] + + data = {} + + contributions = filter(lambda p: 'Contribution' == p.get('p', ''), pages) + for c in contributions: + + languages = [c.get('Uses', [])] + languages = [p for language in languages for p in language] + + languages = list(filter(lambda u: u['p'] == 'Language', languages)) + languages = [language['n'].replace('_', ' ') for language in languages] + + contributors = [c.get('DevelopedBy', [])] + contributors = [p['n'] for contributor in contributors for p in contributor] + + + + + for language in languages: + if language in data.keys(): + data2 = data[language] + else: + data2 = {} + for contributor in contributors: + if contributor in data2.keys(): + data2[contributor] = data2[contributor] + 1 + else: + data2[contributor] = 1 + + data[language] = data2 + + env.write_dump('developerRanks', data) diff --git a/modules/developerRanks/visualisation.py b/modules/developerRanks/visualisation.py new file mode 100644 index 00000000..15ad8d16 --- /dev/null +++ b/modules/developerRanks/visualisation.py @@ -0,0 +1,12 @@ +def createImage(env): + data = env.read_dump('developerRanks') + xName = 'Language' + yName = 'Contributor' + for keys in data.keys(): + xValue = [] + yValue = [] + for key, value in data[keys].items(): + xValue.append(key) + yValue.append(value) + env.create_piechart(keys,xName,yName,xValue,yValue) + diff --git a/modules/locPerContribution/visualisation.py b/modules/locPerContribution/visualisation.py index 9bc5de1d..6fd1ac28 100644 --- a/modules/locPerContribution/visualisation.py +++ b/modules/locPerContribution/visualisation.py @@ -10,4 +10,5 @@ def createImage(env): yValue.append(value) env.create_piechart('Piechart1',xName,yName,xValue,yValue) - + env.create_piechart('Piechart6',xName,yName,xValue,yValue) + env.create_piechart('Piechart999',xName,yName,xValue,yValue) diff --git a/templates/index.html b/templates/index.html index e3a463f2..e4db9ad8 100644 --- a/templates/index.html +++ b/templates/index.html @@ -1,6 +1,49 @@ + - + + +101worker visualizer + + + - + + + +
+ +
+ +
+
+ + + + From 91ee87eb4017572dc5b52309e8d7d24fd02326a6 Mon Sep 17 00:00:00 2001 From: Marcel Date: Thu, 18 May 2017 08:46:24 +0200 Subject: [PATCH 28/43] update for index page --- bin/worker_lib/visualiser.py | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/bin/worker_lib/visualiser.py b/bin/worker_lib/visualiser.py index 263f698d..68061797 100644 --- a/bin/worker_lib/visualiser.py +++ b/bin/worker_lib/visualiser.py @@ -82,28 +82,15 @@ def updateIndex(path, data): def createHtmlTag(key, array): htmlNote = '' + "\n" - header = "
  • "+key+"
    • \n
      \n" + header = "
      \n"+key+"\n
        \n" fullCode = htmlNote + header for item in array: linkperitem = "
      • "+item+"
        • \n" fullCode = fullCode + linkperitem - endNode = "\n
      \n
    \n"+" \n" + endNode = "\n
\n\n"+" \n" fullcode = fullCode + endNode return fullcode -def manageLinkFile(moduleName, chartName,mainPath): - name = moduleName + os.sep + chartName - data = {"link":[name]} - if not os.path.isfile(mainPath+os.sep+'links.json'): - target = os.path.join(mainPath,'links.json') - else: - with open(mainPath+os.sep+'links.json', 'r') as f: - data=json.load(f) - if name not in data["link"]: - data["link"].append(name) - with open(mainPath+os.sep+'links.json', 'w') as f: - json.dump(data, f) - def copyTemplateToTarget(path,name,moduleName, dataname, templateName): #hard-code template folder path?? templatePath = os.path.join('templates',templateName+'.html') From ed57166fe7ccb8563723fa107cbd5cbeef46fab1 Mon Sep 17 00:00:00 2001 From: Skaduwa Date: Wed, 31 May 2017 11:59:04 +0200 Subject: [PATCH 29/43] resync --- modules/locPerContribution/visualisation.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/modules/locPerContribution/visualisation.py b/modules/locPerContribution/visualisation.py index 4e4d7b17..d5bd0f22 100644 --- a/modules/locPerContribution/visualisation.py +++ b/modules/locPerContribution/visualisation.py @@ -16,11 +16,10 @@ def createImage(env): yValue.append(value) data_in.append([key, value]) -<<<<<<< HEAD + #env.create_piechart('Piechart1','locPerContribution',xName,yName,xValue,yValue) env.create_googleChart_pie('locPerContribution', 'googlePieFTW', options_in, data_in) -======= env.create_piechart('Piechart1',xName,yName,xValue,yValue) ->>>>>>> f28d9b9e9e25722e593d0e03569310596222d192 + From 7fc4f28bad8c76cca63b6e0ea0fbe4fe316f154d Mon Sep 17 00:00:00 2001 From: Skaduwa Date: Wed, 31 May 2017 12:10:49 +0200 Subject: [PATCH 30/43] update visualizer --- bin/worker_lib/visualiser.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bin/worker_lib/visualiser.py b/bin/worker_lib/visualiser.py index 68061797..33746306 100644 --- a/bin/worker_lib/visualiser.py +++ b/bin/worker_lib/visualiser.py @@ -114,9 +114,10 @@ def insertTemplate(template, targetpath, dataname): # barChart def create_googleChart_bar(moduleName, fileName, options, data, path): sourcePath = os.path.join('templates', 'googleCharts', 'barChart.html') + targetFolder = os.path.join(path, moduleName) targetPath = os.path.join(path, moduleName, fileName + '.html') inputFile = open(sourcePath) - check_path(targetPath) + check_path(targetFolder) outputFile = open(targetPath, 'w') for line in inputFile: if line.find('INSERT_DATA'): @@ -126,9 +127,10 @@ def create_googleChart_bar(moduleName, fileName, options, data, path): # pieChart def create_googleChart_pie(moduleName, fileName, options, data, path): sourcePath = os.path.join('templates', 'googleCharts', 'pieChart.html') + targetFolder = os.path.join(path, moduleName) targetPath = os.path.join(path, moduleName, fileName + '.html') inputFile = open(sourcePath) - check_path(targetPath) + check_path(targetFolder) outputFile = open(targetPath, 'w') for line in inputFile: if line.find('INSERT_DATA'): From cf6ccde1772d22b358de33d715102d8f3267f185 Mon Sep 17 00:00:00 2001 From: Marcel Date: Wed, 31 May 2017 12:57:04 +0200 Subject: [PATCH 31/43] new templates --- templates/ReadMe.md | 0 templates/googleCharts/bubbleChart.html | 37 +++++++++++++++++++++++++ templates/googleCharts/lineChart.html | 37 +++++++++++++++++++++++++ templates/index.html | 3 ++ 4 files changed, 77 insertions(+) create mode 100644 templates/ReadMe.md create mode 100644 templates/googleCharts/bubbleChart.html create mode 100644 templates/googleCharts/lineChart.html diff --git a/templates/ReadMe.md b/templates/ReadMe.md new file mode 100644 index 00000000..e69de29b diff --git a/templates/googleCharts/bubbleChart.html b/templates/googleCharts/bubbleChart.html new file mode 100644 index 00000000..0b62c3d5 --- /dev/null +++ b/templates/googleCharts/bubbleChart.html @@ -0,0 +1,37 @@ + + + + + + + + + + + + + +
+ + + + + diff --git a/templates/googleCharts/lineChart.html b/templates/googleCharts/lineChart.html new file mode 100644 index 00000000..1df2bf84 --- /dev/null +++ b/templates/googleCharts/lineChart.html @@ -0,0 +1,37 @@ + + + + + + + + + + + + + +
+ + + + + diff --git a/templates/index.html b/templates/index.html index e4db9ad8..854780c3 100644 --- a/templates/index.html +++ b/templates/index.html @@ -3,6 +3,9 @@ 101worker visualizer + + +