diff --git a/CHANGELOG.md b/CHANGELOG.md index 60ba1ff..f5cbede 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [0.1.15] - 2024-03-07 ### Added -- Amazon Bedrock LLM plugin now suports anthropic.claude-3-sonnet model, and deprecates anthropic.claude-v1 - PR #26. +- Amazon Bedrock LLM plugin now suports anthropic.claude-3-sonnet model, and deprecates anthropic.claude-v1 - PR #26 & PR #27. ## [0.1.14] - 2023-12-22 ### Added diff --git a/lambdas/bedrock-embeddings-and-llm/src/lambdahook.py b/lambdas/bedrock-embeddings-and-llm/src/lambdahook.py index be384b2..3534a5d 100644 --- a/lambdas/bedrock-embeddings-and-llm/src/lambdahook.py +++ b/lambdas/bedrock-embeddings-and-llm/src/lambdahook.py @@ -20,11 +20,20 @@ def get_request_body(modelId, parameters, prompt): provider = modelId.split(".")[0] request_body = None if provider == "anthropic": - request_body = { - "prompt": prompt, - "max_tokens_to_sample": DEFAULT_MAX_TOKENS - } - request_body.update(parameters) + # claude-3 models use new messages format + if modelId.startswith("anthropic.claude-3"): + request_body = { + "anthropic_version": "bedrock-2023-05-31", + "messages": [{"role": "user", "content": [{'type':'text','text': prompt}]}], + "max_tokens": DEFAULT_MAX_TOKENS + } + request_body.update(parameters) + else: + request_body = { + "prompt": prompt, + "max_tokens_to_sample": DEFAULT_MAX_TOKENS + } + request_body.update(parameters) elif provider == "ai21": request_body = { "prompt": prompt, @@ -59,20 +68,21 @@ def get_request_body(modelId, parameters, prompt): def get_generate_text(modelId, response): provider = modelId.split(".")[0] generated_text = None + response_body = json.loads(response.get("body").read()) + print("Response body: ", json.dumps(response_body)) if provider == "anthropic": - response_body = json.loads(response.get("body").read().decode()) - generated_text = response_body.get("completion") + # claude-3 models use new messages format + if modelId.startswith("anthropic.claude-3"): + generated_text = response_body.get("content")[0].get("text") + else: + generated_text = response_body.get("completion") elif provider == "ai21": - response_body = json.loads(response.get("body").read()) generated_text = response_body.get("completions")[0].get("data").get("text") elif provider == "amazon": - response_body = json.loads(response.get("body").read()) generated_text = response_body.get("results")[0].get("outputText") elif provider == "cohere": - response_body = json.loads(response.get('body').read()) generated_text = response_body.get("generations")[0].get("text") elif provider == "meta": - response_body = json.loads(response.get('body').read()) generated_text = response_body.get("generation") else: raise Exception("Unsupported provider: ", provider) @@ -89,13 +99,15 @@ def replace_template_placeholders(prompt, event): def format_prompt(modelId, prompt): provider = modelId.split(".")[0] if provider == "anthropic": - print("Model provider is Anthropic. Checking prompt format.") - if not prompt.startswith("\n\nHuman:") or not prompt.startswith("\n\nSystem:"): - prompt = "\n\nHuman: " + prompt - print("Prepended '\\n\\nHuman:'") - if not prompt.endswith("\n\nAssistant:"): - prompt = prompt + "\n\nAssistant:" - print("Appended '\\n\\nHuman:'") + # Claude models prior to v3 required 'Human/Assistant' formatting + if not modelId.startswith("anthropic.claude-3"): + print("Model provider is Anthropic v2. Checking prompt format.") + if not prompt.startswith("\n\nHuman:") or not prompt.startswith("\n\nSystem:"): + prompt = "\n\nHuman: " + prompt + print("Prepended '\\n\\nHuman:'") + if not prompt.endswith("\n\nAssistant:"): + prompt = prompt + "\n\nAssistant:" + print("Appended '\\n\\nHuman:'") print(f"Prompt: {json.dumps(prompt)}") return prompt diff --git a/lambdas/bedrock-embeddings-and-llm/src/settings.py b/lambdas/bedrock-embeddings-and-llm/src/settings.py index 63da152..2a075f8 100644 --- a/lambdas/bedrock-embeddings-and-llm/src/settings.py +++ b/lambdas/bedrock-embeddings-and-llm/src/settings.py @@ -52,6 +52,9 @@ def getModelSettings(modelId): "top_p": 1, "system": "You are a helpful AI assistant." } + else: + params.update({"max_tokens_to_sample": 256}) + params_qa.update({"max_tokens_to_sample": 256}) lambdahook_args = {"Prefix":"LLM Answer:", "Model_params": params} settings = { 'LLM_GENERATE_QUERY_MODEL_PARAMS': json.dumps(params), diff --git a/lambdas/bedrock-embeddings-and-llm/template.yml b/lambdas/bedrock-embeddings-and-llm/template.yml index 81a8d33..a435a09 100644 --- a/lambdas/bedrock-embeddings-and-llm/template.yml +++ b/lambdas/bedrock-embeddings-and-llm/template.yml @@ -303,7 +303,7 @@ Resources: ServiceToken: !GetAtt OutputSettingsFunction.Arn EmbeddingsModelId: !Ref EmbeddingsModelId LLMModelId: !Ref LLMModelId - LastUpdate: '03/07/2024 11:45' + LastUpdate: '03/07/2024 12:20' TestBedrockModelFunction: Type: AWS::Lambda::Function