diff --git a/.gitignore b/.gitignore index d646835b..3253a2d2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *.pyc __pycache__/ +Data diff --git a/common/generic_utils.py b/common/generic_utils.py index 27db31e2..2809a246 100644 --- a/common/generic_utils.py +++ b/common/generic_utils.py @@ -1,4 +1,5 @@ from __future__ import absolute_import +from __future__ import print_function import numpy as np import time import sys diff --git a/common/solr_keras.py b/common/solr_keras.py index 4d8bb8fe..319f772f 100644 --- a/common/solr_keras.py +++ b/common/solr_keras.py @@ -23,13 +23,20 @@ def compute_trainable_params(model): class CandleRemoteMonitor(Callback): """Capture Run level output and store/send for monitoring + + Arguments: + params: the gParams global parameters dictionary + monitor: an array of strings specifying the metrics to be monitored + For every string s, both s and 'val_'+s will be monitored """ def __init__(self, - params=None): + params=None, + monitor=None): super(CandleRemoteMonitor, self).__init__() self.global_params = params + self.monitor = monitor or [] self.has_solr_config = False if 'solr_root' in params and params['solr_root'] != '': self.has_solr_config = True @@ -77,13 +84,17 @@ def on_epoch_end(self, epoch, logs=None): epoch_in_sec = epoch_duration.total_seconds() epoch_line = "epoch: {}/{}, duration: {}s, loss: {}, val_loss: {}".format( (epoch + 1), epoch_total, epoch_in_sec, loss, val_loss) - - send = {'run_id': self.run_id, - 'status': {'set': 'Running'}, - 'training_loss': {'set': loss}, - 'validation_loss': {'set': val_loss}, - 'run_progress': {'add': [epoch_line]} - } + send = {} + for s in self.monitor: + send[s] = {'set': logs.get(s)} + send['val_'+s] = {'set': logs.get('val_' + s)} + + send.update({'run_id': self.run_id, + 'status': {'set': 'Running'}, + 'training_loss': {'set': loss}, + 'validation_loss': {'set': val_loss}, + 'run_progress': {'add': [epoch_line]} + }) # print("on_epoch_end", send) self.log_messages.append(send) if self.has_solr_config: