Skip to content

Commit

Permalink
Merge pull request #28 from JC1DA/overhaul_loc
Browse files Browse the repository at this point in the history
Show avg_latency, token_consumed and token_reduction on the UI
  • Loading branch information
JC1DA authored Dec 14, 2024
2 parents ceada9c + 468e4ae commit 69f723c
Showing 1 changed file with 7 additions and 3 deletions.
10 changes: 7 additions & 3 deletions guidance/models/_model.py
Original file line number Diff line number Diff line change
Expand Up @@ -253,9 +253,6 @@ def _msg_recv(engine_weakref: weakref.ReferenceType, message: GuidanceMessage) -
last_model = list(engine.model_dict.values())[-1]
last_trace_id = last_model._id

# send stats to the renderer
# self.post_exec_metrics.emit_messages(last_model)

failed = False
processed_gen_tokens: list[GenTokenExtra] = [] # suppress IDE warnings by definition
try:
Expand Down Expand Up @@ -283,6 +280,13 @@ def _msg_recv(engine_weakref: weakref.ReferenceType, message: GuidanceMessage) -
if engine.periodic_metrics_generator is not None:
engine.periodic_metrics_generator.pause()

# sending extra metrics
try:
# send stats to the renderer
engine.post_exec_metrics.emit_messages(last_model)
except:
pass


class Engine:
"""The engine owns the inference computation and is used/created by the Model class.
Expand Down

0 comments on commit 69f723c

Please sign in to comment.