kv_cache_limit满了之后停掉最长的询问

ztxz16 · Jul 26, 2024 · 5fa3466 · 5fa3466
1 parent 9ad5f65
commit 5fa3466
Showing 1 changed file with 18 additions and 0 deletions.
diff --git a/src/models/basellm.cpp b/src/models/basellm.cpp
@@ -750,6 +750,24 @@ printf("len = %d, spend = %f s. tokens / s = %f\n", (int)total, spend, (float)to
                                     }
                                 }
                             }
+                        } else {
+                            int maxLen = -1, select = -1;
+                            for (auto &it: model->responseContextDict.dicts) {
+                                if (it.second->isEnding) {
+                                    continue;
+                                }
+                                if (it.second->pastKeyValues[0].first.expansionDims.size() > 0) {
+                                    int curLen = it.second->pastKeyValues[0].first.expansionDims[1];
+                                    if (curLen > maxLen) {
+                                        maxLen = curLen;
+                                        select = it.first;
+                                    }
+                                }
+                            }
+                            if (select != -1) {
+                                model->responseContextDict.dicts[select]->isEnding = true;
+                                continue;
+                            }
                         }
 
                         for (int i = 0; i < attentionMasks.size(); i++) {