Add Palmyra-X-004 model (#2990)

src/helm/clients/palmyra_client.py

@@ -154,15 +154,14 @@ def __init__(
         tokenizer_name: str,
         cache_config: CacheConfig,
         api_key: str,
-        base_url: str,
     ):
         super().__init__(
             tokenizer=tokenizer,
             tokenizer_name=tokenizer_name,
             cache_config=cache_config,
             api_key=api_key,
             org_id=None,
-            base_url=base_url,
+            base_url="https://api.writer.com/v1/chat",
         )
 
     def _is_chat_model_engine(self, model_engine: str) -> bool:

src/helm/config/model_deployments.yaml

@@ -2277,6 +2277,15 @@ model_deployments:
     client_spec:
       class_name: "helm.clients.vision_language.palmyra_vision_client.PalmyraVisionClient"
 
+  - name: writer/palmyra-x-004
+    model_name: writer/palmyra-x-004
+    # Actual tokenizer is Llama 2, but it cannot be used in HELM due to this issue:
+    # https://github.com/stanford-crfm/helm/issues/2467
+    # Work around by using Llama 3 tokenizer for now.
+    tokenizer_name: meta/llama-3-8b
+    max_sequence_length: 8192
+    client_spec:
+      class_name: "helm.clients.palmyra_client.PalmyraChatClient"
 
   # Qwen
 

src/helm/config/model_metadata.yaml

@@ -2945,6 +2945,13 @@ models:
     # Does not support echo
     tags: [VISION_LANGUAGE_MODEL_TAG, LIMITED_FUNCTIONALITY_VLM_TAG]
 
+  - name: writer/palmyra-x-004
+    display_name: Palmyra-X-004
+    description: Palmyra-X-004 language model with a large context window of up to 128,000 tokens that excels in processing and understanding complex tasks.
+    creator_organization_name: Writer
+    access: limited
+    release_date: 2024-09-12
+    tags: [TEXT_MODEL_TAG, LIMITED_FUNCTIONALITY_TEXT_MODEL_TAG, INSTRUCTION_FOLLOWING_MODEL_TAG]
 
   # Yandex
   - name: yandex/yalm