refactor: reduce the impacts of common files

srctag/tagger.py

@@ -46,6 +46,9 @@ def top_n_tags(self, file_name: str, n: int) -> typing.List[str]:
     def top_n_files(self, tag_name: str, n: int) -> typing.List[str]:
         return self.files_by_tag(tag_name).nlargest(n).index.tolist()
 
+    def normalize_score(self, origin: float) -> float:
+        return origin / len(self.files())
+
 
 class TaggerConfig(BaseSettings):
     tags: typing.Set[str] = set()
@@ -107,6 +110,13 @@ def tag(self, storage: Storage) -> TagResult:
         # END tag_results
 
         scores_df = pd.DataFrame.from_dict(ret, orient="index")
+
+        # reduce the impacts of common files
+        row_variances = scores_df.var(axis=1)
+        max_variance = row_variances.max()
+        weights = 1.0 - row_variances / max_variance
+        scores_df = scores_df.multiply(weights, axis=0)
+
         # convert score matrix into rank (use reversed rank as score). because:
         # 1. score/distance is meaningless to users
         # 2. can not be evaluated both rows and cols

tests/test_api.py

@@ -72,10 +72,14 @@ def test_query(setup_tagger):
 
     tags_series = tag_result.tags_by_file("examples/write.py")
     assert len(tags_series) == len(all_tags)
+    tags_series = tags_series[:5]
+    assert len(tags_series) == 5
     for k, v in tags_series.items():
-        logger.info(f"tag: {k}, score: {v}")
+        normalize_score = tag_result.normalize_score(v)
+        logger.info(f"tag: {k}, score: {normalize_score}")
 
     files_series = tag_result.files_by_tag("example")
     assert len(files_series) > 10
     for k, v in files_series.items():
-        logger.info(f"file: {k}, score: {v}")
+        normalize_score = tag_result.normalize_score(v)
+        logger.info(f"file: {k}, score: {normalize_score}")