Add --disable-cache flag (#2143)

scripts/compute_request_limits.py

@@ -4,6 +4,8 @@
 # python compute_request_limits.py --model_deployment_name="writer/palmyra-base" --tokenizer_name="Writer/palmyra-base"
 
 from typing import Any, Optional, Dict
+from helm.common.cache_backend_config import SqliteCacheBackendConfig
+from helm.common.general import ensure_directory_exists
 from helm.proxy.clients.auto_client import AutoClient
 from helm.benchmark.model_deployment_registry import ModelDeployment, get_model_deployment
 from helm.proxy.tokenizers.auto_tokenizer import AutoTokenizer
@@ -345,8 +347,11 @@ def main():
     cache_path = os.path.join(current_path, args.cache_path)
     print(f"cache_path: {cache_path}")
 
-    client = AutoClient(credentials=credentials, cache_path=cache_path)
-    auto_tokenizer = AutoTokenizer(credentials=credentials, cache_path=cache_path)
+    ensure_directory_exists(cache_path)
+    client = AutoClient(
+        credentials=credentials, file_storage_path=cache_path, cache_backend_config=SqliteCacheBackendConfig(cache_path)
+    )
+    auto_tokenizer = AutoTokenizer(credentials=credentials, cache_backend_config=SqliteCacheBackendConfig(cache_path))
     print("client successfully created")
 
     print("Making short request...")

scripts/efficiency/generate_instances.py

@@ -8,6 +8,7 @@
 
 import os
 from typing import Dict, List, Tuple
+from helm.common.cache_backend_config import SqliteCacheBackendConfig
 
 from helm.common.general import ensure_directory_exists, ensure_file_downloaded, write, get_credentials
 from helm.common.tokenization_request import (
@@ -52,7 +53,7 @@ def get_tokenizer(base_path: str = "prod_env") -> AutoTokenizer:
     ensure_directory_exists(cache_path)
 
     # TODO: Pass mongo_uri to AutoClient
-    tokenizer = AutoTokenizer(credentials, cache_path)
+    tokenizer = AutoTokenizer(credentials, SqliteCacheBackendConfig(cache_path))
 
     return tokenizer
 

src/helm/benchmark/adaptation/adapters/multimodal/test_in_context_learning_multimodal_adapter.py

@@ -1,6 +1,7 @@
 import shutil
 import tempfile
 import unittest
+from helm.common.cache_backend_config import BlackHoleCacheBackendConfig
 
 from helm.common.media_object import MediaObject, MultimediaObject
 from helm.benchmark.scenarios.scenario import Instance, Reference, Input, Output, TEST_SPLIT, TRAIN_SPLIT, CORRECT_TAG
@@ -14,7 +15,7 @@
 class TestInContextLearningMultimodalAdapter(unittest.TestCase):
     def setup_method(self, _):
         self._path: str = tempfile.mkdtemp()
-        self._tokenizer_service = get_tokenizer_service(self._path)
+        self._tokenizer_service = get_tokenizer_service(self._path, BlackHoleCacheBackendConfig())
 
     def teardown_method(self, _):
         shutil.rmtree(self._path)

src/helm/benchmark/adaptation/adapters/test_adapter.py

@@ -2,6 +2,7 @@
 import tempfile
 
 from helm.common.authentication import Authentication
+from helm.common.cache_backend_config import BlackHoleCacheBackendConfig
 from helm.proxy.services.server_service import ServerService
 from helm.benchmark.window_services.tokenizer_service import TokenizerService
 
@@ -13,7 +14,7 @@ class TestAdapter:
 
     def setup_method(self):
         self.path: str = tempfile.mkdtemp()
-        service = ServerService(base_path=self.path, root_mode=True)
+        service = ServerService(base_path=self.path, root_mode=True, cache_backend_config=BlackHoleCacheBackendConfig())
         self.tokenizer_service = TokenizerService(service, Authentication("test"))
 
     def teardown_method(self, _):

src/helm/benchmark/executor.py

@@ -1,5 +1,11 @@
 from typing import Optional
 from dataclasses import dataclass, replace
+from helm.common.cache_backend_config import (
+    CacheBackendConfig,
+    BlackHoleCacheBackendConfig,
+    MongoCacheBackendConfig,
+    SqliteCacheBackendConfig,
+)
 
 from helm.common.general import parallel_map
 from helm.common.hierarchical_logger import htrack, hlog
@@ -18,28 +24,36 @@ class ExecutorError(Exception):
 
 @dataclass(frozen=True)
 class ExecutionSpec:
-    # If non-empty, URL of the proxy server we send requests to (e.g., http://localhost:1959).
+
     url: Optional[str]
+    """If non-empty, URL of the proxy server we send requests to (e.g., http://localhost:1959)."""
 
-    # Pass into the service
     auth: Authentication
+    """Authentication that will be passed into the local service, if using the local service."""
 
-    # Path where API credentials and cache is stored.
-    # This path is the same as `--base-path` when launching the proxy server (see server.py).
-    # Required when url is not set.
     local_path: Optional[str]
+    """Path where API credentials and cache is stored.
+
+    This path is the same as `--base-path` when launching the proxy server (see server.py).
+    Required when url is not set."""
 
-    # How many threads to have at once
     parallelism: int
+    """How many threads to have at once"""
 
-    # Whether to skip execution
     dry_run: bool = False
+    """Whether to skip execution"""
+
+    sqlite_cache_backend_config: Optional[SqliteCacheBackendConfig] = None
+    """If set, SQLite will be used for the cache.
+
+    This specifies the directory in which the SQLite cache will store files.
+    At most one of sqlite_cache_backend_config and mongo_cache_backend_config can be set."""
 
-    # URL to the MongoDB database.
-    # If non-empty, the MongoDB database will be used for caching instead of SQLite.
-    # Example format: mongodb://[username:password@]host1[:port1]/[dbname]
-    # For full format, see: https://www.mongodb.com/docs/manual/reference/connection-string/
-    mongo_uri: str = ""
+    mongo_cache_backend_config: Optional[MongoCacheBackendConfig] = None
+    """If set, MongoDB will be used for the cache.
+
+    This specifies the MongoDB database to be used by the MongoDB cache.
+    At most one of sqlite_cache_backend_config and mongo_cache_backend_config can be set."""
 
 
 class Executor:
@@ -51,14 +65,26 @@ class Executor:
     def __init__(self, execution_spec: ExecutionSpec):
         self.execution_spec = execution_spec
 
+        cache_backend_config: CacheBackendConfig
+        if execution_spec.sqlite_cache_backend_config and execution_spec.mongo_cache_backend_config:
+            raise ExecutorError("At most one of sqlite_cache_backend_config and mongo_cache_backend_config can be set.")
+        elif execution_spec.sqlite_cache_backend_config:
+            cache_backend_config = execution_spec.sqlite_cache_backend_config
+        elif execution_spec.mongo_cache_backend_config:
+            cache_backend_config = execution_spec.mongo_cache_backend_config
+        else:
+            cache_backend_config = BlackHoleCacheBackendConfig()
+
         self.service: Service
         if execution_spec.url:
             hlog(f"Running using remote API proxy server: {execution_spec.url}")
             self.service = RemoteService(execution_spec.url)
         elif execution_spec.local_path:
             hlog(f"Running in local mode with base path: {execution_spec.local_path}")
             self.service = ServerService(
-                base_path=execution_spec.local_path, root_mode=True, mongo_uri=execution_spec.mongo_uri
+                base_path=execution_spec.local_path,
+                root_mode=True,
+                cache_backend_config=cache_backend_config,
             )
         else:
             raise ValueError("Either the proxy server URL or the local path must be set")

src/helm/benchmark/run.py

@@ -1,14 +1,17 @@
 import argparse
 from dataclasses import replace
+import os
 from typing import List, Optional
 
 
 from helm.benchmark.presentation.run_entry import RunEntry, read_run_entries
+from helm.common.cache_backend_config import MongoCacheBackendConfig, SqliteCacheBackendConfig
 from helm.common.general import ensure_directory_exists
 from helm.common.hierarchical_logger import hlog, htrack, htrack_block
 from helm.common.authentication import Authentication
 from helm.common.object_spec import parse_object_spec, get_class_by_name
 from helm.proxy.services.remote_service import create_authentication, add_service_args
+from helm.proxy.services.service import CACHE_DIR
 
 from helm.benchmark.config_registry import (
     register_configs_from_directory,
@@ -83,16 +86,29 @@ def run_benchmarking(
     skip_completed_runs: bool,
     exit_on_error: bool,
     runner_class_name: Optional[str],
-    mongo_uri: str = "",
+    mongo_uri: Optional[str] = None,
+    disable_cache: Optional[bool] = None,
 ) -> List[RunSpec]:
     """Runs RunSpecs given a list of RunSpec descriptions."""
+    sqlite_cache_backend_config: Optional[SqliteCacheBackendConfig] = None
+    mongo_cache_backend_config: Optional[MongoCacheBackendConfig] = None
+
+    if not disable_cache:
+        if mongo_uri:
+            mongo_cache_backend_config = MongoCacheBackendConfig(mongo_uri)
+        else:
+            sqlite_cache_path = os.path.join(local_path, CACHE_DIR)
+            ensure_directory_exists(sqlite_cache_path)
+            sqlite_cache_backend_config = SqliteCacheBackendConfig(sqlite_cache_path)
+
     execution_spec = ExecutionSpec(
         auth=auth,
         url=url,
         local_path=local_path,
         parallelism=num_threads,
         dry_run=dry_run,
-        mongo_uri=mongo_uri,
+        sqlite_cache_backend_config=sqlite_cache_backend_config,
+        mongo_cache_backend_config=mongo_cache_backend_config,
     )
     with htrack_block("run_specs"):
         for run_spec in run_specs:
@@ -177,6 +193,11 @@ def add_run_args(parser: argparse.ArgumentParser):
         help="If non-empty, the URL of the MongoDB database that will be used for caching instead of SQLite",
         default="",
     )
+    parser.add_argument(
+        "--disable-cache",
+        action="store_true",
+        help="If true, the request-response cache for model clients and tokenizers will be disabled.",
+    )
 
 
 def validate_args(args):
@@ -310,6 +331,7 @@ def main():
         exit_on_error=args.exit_on_error,
         runner_class_name=args.runner_class_name,
         mongo_uri=args.mongo_uri,
+        disable_cache=args.disable_cache,
     )
 
     if args.local:

src/helm/benchmark/test_model_deployment_definition.py

@@ -14,6 +14,7 @@
 from helm.benchmark.model_metadata_registry import get_model_metadata, ModelMetadata
 from helm.benchmark.tokenizer_config_registry import TokenizerConfig, get_tokenizer_config
 from helm.benchmark.window_services.test_utils import get_tokenizer_service
+from helm.common.cache_backend_config import BlackHoleCacheBackendConfig
 from helm.proxy.clients.client import Client
 from helm.proxy.tokenizers.tokenizer import Tokenizer
 from helm.benchmark.window_services.window_service import WindowService
@@ -30,9 +31,9 @@ class TestModelProperties:
     @pytest.mark.parametrize("deployment_name", [deployment.name for deployment in ALL_MODEL_DEPLOYMENTS])
     def test_models_has_window_service(self, deployment_name: str):
         with TemporaryDirectory() as tmpdir:
-            auto_client = AutoClient({}, tmpdir, "")
-            auto_tokenizer = AutoTokenizer({}, tmpdir, "")
-            tokenizer_service = get_tokenizer_service(tmpdir)
+            auto_client = AutoClient({}, tmpdir, BlackHoleCacheBackendConfig())
+            auto_tokenizer = AutoTokenizer({}, BlackHoleCacheBackendConfig())
+            tokenizer_service = get_tokenizer_service(tmpdir, BlackHoleCacheBackendConfig())
 
             # Loading the TokenizerConfig and ModelMetadat ensures that they are valid.
             deployment: ModelDeployment = get_model_deployment(deployment_name)

src/helm/benchmark/window_services/image_generation/test_clip_window_service.py

@@ -4,12 +4,13 @@
 from helm.benchmark.window_services.tokenizer_service import TokenizerService
 from helm.benchmark.window_services.test_utils import get_tokenizer_service
 from helm.benchmark.window_services.window_service_factory import WindowServiceFactory
+from helm.common.cache_backend_config import BlackHoleCacheBackendConfig
 
 
 class TestCLIPWindowService:
     def setup_method(self):
         self.path: str = tempfile.mkdtemp()
-        service: TokenizerService = get_tokenizer_service(self.path)
+        service: TokenizerService = get_tokenizer_service(self.path, BlackHoleCacheBackendConfig())
         self.window_service = WindowServiceFactory.get_window_service("huggingface/dreamlike-photoreal-v2-0", service)
 
     def teardown_method(self, method):

src/helm/benchmark/window_services/image_generation/test_openai_dalle_window_service.py

@@ -5,12 +5,13 @@
 from helm.proxy.clients.image_generation.dalle2_client import DALLE2Client
 from helm.benchmark.window_services.test_utils import get_tokenizer_service, TEST_PROMPT
 from helm.benchmark.window_services.window_service_factory import WindowServiceFactory
+from helm.common.cache_backend_config import BlackHoleCacheBackendConfig
 
 
 class TestOpenAIDALLEWindowService:
     def setup_method(self):
         self.path: str = tempfile.mkdtemp()
-        service: TokenizerService = get_tokenizer_service(self.path)
+        service: TokenizerService = get_tokenizer_service(self.path, BlackHoleCacheBackendConfig())
         self.window_service = WindowServiceFactory.get_window_service("openai/dall-e-2", service)
 
     def teardown_method(self, method):

src/helm/benchmark/window_services/test_anthropic_window_service.py

@@ -2,6 +2,7 @@
 import tempfile
 from typing import List
 
+from helm.common.cache_backend_config import BlackHoleCacheBackendConfig
 from .tokenizer_service import TokenizerService
 from .window_service_factory import WindowServiceFactory
 from .test_utils import get_tokenizer_service, TEST_PROMPT
@@ -120,7 +121,7 @@ class TestAnthropicWindowService:
 
     def setup_method(self):
         self.path: str = tempfile.mkdtemp()
-        service: TokenizerService = get_tokenizer_service(self.path)
+        service: TokenizerService = get_tokenizer_service(self.path, BlackHoleCacheBackendConfig())
         self.window_service = WindowServiceFactory.get_window_service("anthropic/claude-v1.3", service)
 
     def teardown_method(self, method):

src/helm/benchmark/window_services/test_bloom_window_service.py

@@ -2,6 +2,7 @@
 import tempfile
 from typing import List
 
+from helm.common.cache_backend_config import BlackHoleCacheBackendConfig
 from .tokenizer_service import TokenizerService
 from .window_service_factory import WindowServiceFactory
 from .test_utils import get_tokenizer_service, TEST_PROMPT
@@ -64,7 +65,7 @@ class TestBloomWindowService:
 
     def setup_method(self):
         self.path: str = tempfile.mkdtemp()
-        service: TokenizerService = get_tokenizer_service(self.path)
+        service: TokenizerService = get_tokenizer_service(self.path, BlackHoleCacheBackendConfig())
         self.window_service = WindowServiceFactory.get_window_service("together/bloom", service)
 
     def teardown_method(self, method):

src/helm/benchmark/window_services/test_cohere_window_service.py

@@ -6,6 +6,7 @@
 
 from sqlitedict import SqliteDict
 
+from helm.common.cache_backend_config import SqliteCacheBackendConfig
 from helm.common.general import ensure_directory_exists
 from .test_cohere_window_service_utils import REQUESTS_TO_RESPONSES, TEST_PROMPT, TOKENIZED_PROMPT
 from .tokenizer_service import TokenizerService
@@ -30,7 +31,7 @@ def setup_class(cls):
         with open(os.path.join(cls.path, "credentials.conf"), "w") as f:
             f.write("cohereApiKey: secret")
 
-        service: TokenizerService = get_tokenizer_service(cls.path)
+        service: TokenizerService = get_tokenizer_service(cls.path, SqliteCacheBackendConfig(cache_path))
         cls.window_service = WindowServiceFactory.get_window_service("cohere/xlarge-20220609", service)
         cls.prompt: str = TEST_PROMPT
         cls.tokenized_prompt: List[str] = TOKENIZED_PROMPT

src/helm/benchmark/window_services/test_flan_t5_window_service.py

@@ -1,5 +1,6 @@
 import tempfile
 
+from helm.common.cache_backend_config import BlackHoleCacheBackendConfig
 from helm.benchmark.window_services.test_t511b_window_service import TestT511bWindowService
 from helm.benchmark.window_services.window_service_factory import TokenizerService, WindowServiceFactory
 from helm.benchmark.window_services.test_utils import get_tokenizer_service
@@ -8,5 +9,5 @@
 class TestFlanT5WindowService(TestT511bWindowService):
     def setup_method(self):
         self.path: str = tempfile.mkdtemp()
-        service: TokenizerService = get_tokenizer_service(self.path)
+        service: TokenizerService = get_tokenizer_service(self.path, BlackHoleCacheBackendConfig())
         self.window_service = WindowServiceFactory.get_window_service("together/flan-t5-xxl", service)

src/helm/benchmark/window_services/test_gpt2_window_service.py

@@ -2,15 +2,15 @@
 import tempfile
 
 from helm.benchmark.window_services.tokenizer_service import TokenizerService
-
+from helm.common.cache_backend_config import BlackHoleCacheBackendConfig
 from .test_utils import get_tokenizer_service, TEST_PROMPT, GPT2_TEST_TOKENS, GPT2_TEST_TOKEN_IDS
 from .window_service_factory import WindowServiceFactory
 
 
 class TestGPT2WindowService:
     def setup_method(self):
         self.path: str = tempfile.mkdtemp()
-        service: TokenizerService = get_tokenizer_service(self.path)
+        service: TokenizerService = get_tokenizer_service(self.path, BlackHoleCacheBackendConfig())
         self.window_service = WindowServiceFactory.get_window_service("huggingface/gpt2", service)
 
     def teardown_method(self, method):

src/helm/benchmark/window_services/test_gpt4_window_service.py

@@ -1,6 +1,7 @@
 import shutil
 import tempfile
 
+from helm.common.cache_backend_config import BlackHoleCacheBackendConfig
 from .test_utils import get_tokenizer_service, TEST_PROMPT, GPT4_TEST_TOKEN_IDS, GPT4_TEST_TOKENS
 from .tokenizer_service import TokenizerService
 from .window_service_factory import WindowServiceFactory
@@ -9,7 +10,7 @@
 class TestOpenAIWindowService:
     def setup_method(self):
         self.path: str = tempfile.mkdtemp()
-        service: TokenizerService = get_tokenizer_service(self.path)
+        service: TokenizerService = get_tokenizer_service(self.path, BlackHoleCacheBackendConfig())
         self.window_service = WindowServiceFactory.get_window_service("openai/gpt-3.5-turbo-0301", service)
 
     def teardown_method(self, method):