PP 1617 - visualisation

bin/worker_lib/env.py

@@ -2,6 +2,9 @@
 import json
 import logging
 
+from .visualiser import create_googleChart as create_googleChart_viz
+
+
 def abs_path(path):
     return os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', path))
 
@@ -100,6 +103,26 @@ def write_dump(dump_name, data):
             json.dump(data, f, indent=4)
             logging.debug('Wrote dump %s at %s', dump_name, d)
 
+
+########
+# d3js #
+########
+    def create_piechart(name,xName,yName,xValue,yValue):
+        moduleName = str(module).replace("'","").split(" ")[1]
+        c_piechart(name,moduleName,xName,yName,xValue,yValue,get_env('views101dir'))
+#
+########
+
+
+#################
+# Google Charts #
+#################
+#
+    def create_googleChart(chartType, fileName, data, options = None):
+        create_googleChart_viz(chartType, str(module).replace("'","").split(" ")[1], fileName, get_env('views101dir'), data, options)
+#
+#################
+
     return AttrDict({
         'get_env': get_env,
         'write_derived_resource': write_derived_resource,
@@ -108,5 +131,7 @@ def write_dump(dump_name, data):
         'read_dump': read_dump,
         'write_dump': write_dump,
         'get_derived_resource': get_derived_resource,
-        'remove_dump': remove_dump
+        'remove_dump': remove_dump,
+        'create_piechart': create_piechart,
+	'create_googleChart': create_googleChart
     })

bin/worker_lib/executor.py

@@ -4,6 +4,7 @@
 import os
 import sys
 import traceback
+import shutil
 
 class Executor(object):
 
@@ -23,6 +24,15 @@ def _exec(self, change):
     def run(self, changes):
         for change in changes:
             self._exec(change)
+        ####### added to create Image ################
+        if self._module.config.get('visualization') == True :        
+            print("Creating Image")
+            folderName = str(self._module).replace("'","").split(" ")[1]
+            path = self._env.get_env('views101dir')
+            if(os.path.isdir(path + os.sep + folderName)):
+                shutil.rmtree(path + os.sep + folderName)
+            self._module.visualize(self._env)
+        ##############################################
 
 class FileFullSweepExecutor(Executor):
 
@@ -38,6 +48,15 @@ def run(self, changes):
                 }
 
                 self._exec(change)
+        ####### added to create Image ################
+        if self._module.config.get('visualization') == True :        
+            print("Creating Image")
+            folderName = str(self._module).replace("'","").split(" ")[1]
+            path = self._env.get_env('views101dir')
+            if(os.path.isdir(path + os.sep + folderName)):
+                shutil.rmtree(path + os.sep + folderName)
+            self._module.visualize(self._env)
+        ##############################################
 
 class AllFullSweepExecutor(Executor):
 
@@ -52,3 +71,12 @@ def _exec(self):
 
     def run(self, changes):
         self._exec()
+        ####### added to create Image ################
+        if self._module.config.get('visualization') == True :        
+            print("Creating Image")
+            folderName = str(self._module).replace("'","").split(" ")[1]
+            path = self._env.get_env('views101dir')
+            if(os.path.isdir(path + os.sep + folderName)):
+                shutil.rmtree(path + os.sep + folderName)
+            self._module.visualize(self._env)
+        ##############################################

bin/worker_lib/modules.py

@@ -21,7 +21,7 @@ def import_module(module_name):
     'moretagclouds',
     'plainTextWikiDump',
     'zip',
-    'mongodump'
+	'mongodump'
 ]
 
 modules = [import_module(module) for module in modules]

bin/worker_lib/visualiser.py

@@ -0,0 +1,167 @@
+import os
+import json
+import csv
+import shutil
+
+
+###################
+# Service methods #
+###################
+#
+# check if folder exists or create
+def check_path(path):
+    if not os.path.exists(path):
+        os.makedirs(path)
+#
+###################
+
+
+
+########
+# d3JS #
+########
+#
+# Write CSV file
+def write_csv(target,csvData,name):
+    d = os.path.join(target, name + '.csv')
+    with open(d, 'w') as f:
+        wr = csv.writer(f)
+        for item in csvData:
+            wr.writerow(item)
+    return name + '.csv'
+#
+# create pieChart
+def create_piechart(name,moduleName,xName,yName,xValues,yValues,path):
+    dataPath = os.path.join(path,moduleName)
+    check_path(dataPath)
+    templateName = "pie"
+    csvData = []
+    labels = ['age','population']
+    csvData.append(labels)
+    for x,y in zip(xValues,yValues): 
+        csvData.append([x,y])
+    csvname = write_csv(dataPath,csvData, name)
+    copyTemplateToTarget(path,name,moduleName, csvname,templateName)
+    folderIterator(path)
+#
+# copy template to file
+def copyTemplateToTarget(path,name,moduleName, dataname, templateName):
+    #hard-code template folder path??
+    templatePath = os.path.join('templates',templateName+'.html')
+    targetPath = os.path.join(path,moduleName, name + '.html')
+    insertTemplate(templatePath, targetPath, dataname)
+#
+# insetr template
+def insertTemplate(template, targetpath, dataname):
+    inputfile = open(template)
+    outputfile = open(targetpath, 'w')
+    for line in inputfile:
+        if line.find('enterDataHere'):
+            line = line.replace('enterDataHere', dataname)
+        outputfile.writelines(line)
+#
+########
+
+
+#################
+# Google Charts #
+#################
+#
+# GOOGLE-Chart
+def create_googleChart(chartType, moduleName, fileName, path, data, options):
+
+    sourcePath = os.path.join('templates', 'googleCharts', chartType + '.html')
+
+    if not os.path.isfile(sourcePath):
+        raise FileNotFoundError('Could not find template: ' + '\'' + sourcePath + '\'\nPlease check the charttype.')
+
+    targetFolder = os.path.join(path, moduleName)
+    targetPath = os.path.join(path, moduleName, fileName.replace(' ', '_') + '.html')
+    inputFile = open(sourcePath, 'r')
+    check_path(targetFolder)
+    outputFile = open(targetPath, 'w')
+    for line in inputFile:
+        if line.find('INSERT_DATA'):
+            line = line.replace('INSERT_DATA', str(data))
+        if line.find('INSERT_OPTIONS'):
+            if options != None:
+                line = line.replace('INSERT_OPTIONS', options)
+            else:
+                line = line.replace('INSERT_OPTIONS', '')
+        outputFile.writelines(line)
+    folderIterator(path)
+#
+#################
+
+
+##################
+# Index Creation #
+##################
+#
+# folderIterator
+def folderIterator(path):
+    data = {}
+    for dirs in os.listdir(path):
+        htmlfiles = []
+        if dirs.endswith('index.html'):
+            continue
+        for files in os.listdir(path + os.sep + dirs):
+            if files.endswith('.html'):
+                htmlfiles.append(files)
+        data[dirs] = htmlfiles
+    updateIndex(path, data)
+#
+# Index update
+def updateIndex(path, data):
+
+    targetpath = path + os.sep + 'index.html'
+    templatepath = os.path.join('templates','index.html')
+    shutil.copyfile(templatepath, targetpath)
+    for key in data.keys():
+        skiplines = False
+        inputfile = open(targetpath, 'r')
+        tempfile = open(path + os.sep + 'temp', 'w')
+        keyfound = False
+
+        for line in inputfile:
+            if line.find('<!--'+ key + '-->') != -1:
+                skiplines = True
+                keyfound = True
+                tempfile.writelines(line)
+            if line.find('<!--'+key+'-END-->') != -1:
+                skiplines = False
+                line = ""
+            if not skiplines:
+                tempfile.writelines(line)
+
+        if not keyfound:
+            os.remove(path + os.sep + 'temp')
+            inputfile = open(path + os.sep + 'index.html', 'r')
+            tempfile = open(path + os.sep + 'temp', 'w')
+            for line in inputfile:
+                if line.find('<!-- xxx -->') != -1:
+                    line = '<!--'+ key + '-->' + "\n" + '<!--'+key+'-END-->' + "\n<!-- xxx -->\n"
+                tempfile.writelines(line)
+        tempfile = open(path + os.sep + 'temp', 'r')
+        inputfile = open(path + os.sep + 'index.html', 'w')
+        for line in tempfile:
+            if line.find('<!--'+ key + '-->') != -1:
+                line = createHtmlTag(key, data[key])
+            inputfile.writelines(line)
+        os.remove(path + os.sep + 'temp')
+#
+# HTML tag creation
+def createHtmlTag(key, array):
+    htmlNote = '<!--'+ key + '-->' + "\n"
+    header = "<details>\n<summary>"+key+"</summary>\n<ul>\n"
+    fullCode = htmlNote + header
+    for item in array:
+        linkperitem = "<li><a href="+key+os.sep+item+" target=\'content_frame\'>"+item+"</a></li>\n"
+        fullCode = fullCode + linkperitem
+    endNode = "\n</ul>\n</details>\n"+"<!--"+key+"-END--> \n"
+    fullcode = fullCode + endNode
+    return fullcode
+#
+##################
+
+

config/test_folders.txt

@@ -1,3 +1,4 @@
 contributions/py3k
 contributions/ruby
 contributions/javaTree
+contributions/cppMpl

modules/commentSentimentPerLanguage/README.md

@@ -0,0 +1,12 @@
+# commentSentimentPerLanguage
+
+Min, max, median and mean sentiment of comments per language
+
+# Description
+
+This module calculates the minimum, maximum, median and mean Sentiment of comments per language. The results are saved as a sentimentsPerLanguage dump. 
+Additionally a sentimentsPerContribution dump is created containing the comment sentiment per file grouped by language and contribution.
+
+# Requirements
+
+This module requires simpleSentiment to be run first.