From 3c12871bd882a977bd31beef71ce3df2e80194fa Mon Sep 17 00:00:00 2001 From: JosselinSomervilleRoberts Date: Tue, 26 Sep 2023 06:43:54 -0700 Subject: [PATCH] Added special case to support Antrhopic Critique --- src/helm/proxy/clients/model_critique_client.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/helm/proxy/clients/model_critique_client.py b/src/helm/proxy/clients/model_critique_client.py index aa6a0bfbd3..5cc41ce428 100644 --- a/src/helm/proxy/clients/model_critique_client.py +++ b/src/helm/proxy/clients/model_critique_client.py @@ -14,6 +14,8 @@ from helm.proxy.clients.client import Client from helm.proxy.clients.critique_client import CritiqueClient +import anthropic + class CritiqueParseError(Exception): pass @@ -61,6 +63,12 @@ def _task_to_requests(self, task: CritiqueTaskTemplate, fields: Dict[str, str]) max_tokens = len(question.options) * 2 else: max_tokens = 1 + + # Special case for Anthropic to handle prefix and suffix. + # TODO(josselin): Fix this once refactor of HELM allows for automatic model prefix and suffix. + if self._model_name.startswith("anthropic"): + prompt = anthropic.HUMAN_PROMPT + prompt + anthropic.AI_PROMPT + request = Request( model=self._model_name, prompt=prompt,