diff --git a/src/google/adk/dependencies/rouge_scorer.py b/src/google/adk/dependencies/rouge_scorer.py index 622a190ab73..e7e6cfcd8c0 100644 --- a/src/google/adk/dependencies/rouge_scorer.py +++ b/src/google/adk/dependencies/rouge_scorer.py @@ -15,3 +15,4 @@ from __future__ import annotations from rouge_score import rouge_scorer as rouge_scorer +from rouge_score import tokenizers as tokenizers diff --git a/src/google/adk/evaluation/final_response_match_v1.py b/src/google/adk/evaluation/final_response_match_v1.py index f7c07cec737..5e4d71747fa 100644 --- a/src/google/adk/evaluation/final_response_match_v1.py +++ b/src/google/adk/evaluation/final_response_match_v1.py @@ -20,6 +20,7 @@ from typing_extensions import override from ..dependencies.rouge_scorer import rouge_scorer +from ..dependencies.rouge_scorer import tokenizers from .eval_case import ConversationScenario from .eval_case import Invocation from .eval_metrics import EvalMetric @@ -30,6 +31,19 @@ from .evaluator import PerInvocationResult +class _UnicodeFriendlyTokenizer(tokenizers.Tokenizer): + """A tokenizer that splits on whitespace and preserves Unicode characters. + + The default rouge_score tokenizer strips all non-ASCII characters, causing + non-English text (e.g., Thai, Chinese, Arabic) to tokenize to empty strings + and receive a score of 0 regardless of content. This tokenizer uses simple + whitespace splitting and lowercasing, which correctly handles any language. + """ + + def tokenize(self, text: str) -> list[str]: + return text.lower().split() + + class RougeEvaluator(Evaluator): """Evaluates if agent's final response matches a golden/expected final response using Rouge_1 metric. @@ -114,7 +128,9 @@ def _calculate_rouge_1_scores(candidate: str, reference: str): Returns: A dictionary containing the ROUGE-1 precision, recall, and f-measure. """ - scorer = rouge_scorer.RougeScorer(["rouge1"], use_stemmer=True) + scorer = rouge_scorer.RougeScorer( + ["rouge1"], tokenizer=_UnicodeFriendlyTokenizer() + ) # The score method returns a dictionary where keys are the ROUGE types # and values are Score objects (tuples) with precision, recall, and fmeasure. diff --git a/tests/unittests/evaluation/test_final_response_match_v1.py b/tests/unittests/evaluation/test_final_response_match_v1.py index 111ca8415c3..9a380c5b7f5 100644 --- a/tests/unittests/evaluation/test_final_response_match_v1.py +++ b/tests/unittests/evaluation/test_final_response_match_v1.py @@ -87,6 +87,24 @@ def test_calculate_rouge_1_scores(): assert rouge_1_score.fmeasure == pytest.approx(8 / 11) +def test_calculate_rouge_1_scores_non_english_identical(): + """Non-English text that is identical should score 1.0, not 0.""" + candidate = "สวัสดี" + reference = "สวัสดี" + rouge_1_score = _calculate_rouge_1_scores(candidate, reference) + assert rouge_1_score.precision == pytest.approx(1.0) + assert rouge_1_score.recall == pytest.approx(1.0) + assert rouge_1_score.fmeasure == pytest.approx(1.0) + + +def test_calculate_rouge_1_scores_non_english_partial_overlap(): + """Non-English text with partial overlap should return a partial score.""" + candidate = "สวัสดี โลก" + reference = "สวัสดี ทุกคน" + rouge_1_score = _calculate_rouge_1_scores(candidate, reference) + assert rouge_1_score.fmeasure == pytest.approx(0.5) + + @pytest.mark.parametrize( "candidates, references, expected_score, expected_status", [