Markdown JSON Mode

examples/vision/run.py

@@ -0,0 +1,63 @@
+import instructor
+from openai import OpenAI
+from typing import Iterable
+from pydantic import BaseModel
+import base64
+
+client = instructor.patch(OpenAI(), mode=instructor.function_calls.Mode.MD_JSON)
+
+class Circle(BaseModel):
+    x: int
+    y: int
+    color: str
+
+def encode_image(image_path):
+    with open(image_path, "rb") as image_file:
+        return base64.b64encode(image_file.read()).decode('utf-8')
+
+def draw_circle(image_size, num_circles, path):
+
+    from PIL import Image, ImageDraw
+    import random
+
+    image = Image.new("RGB", image_size, "white")
+
+    draw = ImageDraw.Draw(image)
+    for _ in range(num_circles):
+        # Randomize the circle properties
+        radius = 100#random.randint(10, min(image_size)//5)  # Radius between 10 and 1/5th of the smallest dimension
+        x = random.randint(radius, image_size[0] - radius)
+        y = random.randint(radius, image_size[1] - radius)
+        color = ['red', 'black', 'blue', 'green'][random.randint(0, 3)]
+
+        circle_position = (x - radius, y - radius, x + radius, y + radius)
+        print(f"Generating circle at {x, y} with color {color}")
+        draw.ellipse(circle_position, fill=color, outline="black")
+
+    image.save(path)
+
+img_path = 'circle.jpg'
+draw_circle((1024,1024), 1, img_path)
+base64_image = encode_image(img_path)
+
+response = client.chat.completions.create(
+    model="gpt-4-vision-preview",
+    max_tokens=1800,
+    response_model=Circle,
+    messages=[
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": 'find the circle'},
+                {
+                    "type": "image_url",
+                    "image_url": {
+                        "url": f"data:image/jpeg;base64,{base64_image}"
+                    },
+                },
+            ],
+        }
+    ],
+)
+
+print(f"Found circle with center at x: {response.x}, y: {response.y} and color: {response.color}")

instructor/function_calls.py

@@ -1,4 +1,5 @@
 import json
+import re 
 from docstring_parser import parse
 from functools import wraps
 from typing import Any, Callable
@@ -13,6 +14,7 @@ class Mode(enum.Enum):
     FUNCTIONS: str = "function_call"
     TOOLS: str = "tool_call"
     JSON: str = "json_mode"
+    MD_JSON: str = "markdown_json_mode"
 
 
 class openai_function:
@@ -237,6 +239,14 @@ def from_response(
                 context=validation_context,
                 strict=strict,
             )
+        elif mode == Mode.MD_JSON:
+            pattern = r'```(?:json)?(.*?)```'
+            json_content = re.findall(pattern, message.content, re.DOTALL)[0].strip()
+            return cls.model_validate_json(
+                json_content.strip(),
+                context=validation_context,
+                strict=strict,
+            )
         else:
             raise ValueError(f"Invalid patch mode: {mode}")
 

instructor/patch.py

@@ -79,13 +79,21 @@ def handle_response_model(
                 "type": "function",
                 "function": {"name": response_model.openai_schema["name"]},
             }
-        elif mode == Mode.JSON:
-            new_kwargs["response_format"] = {"type": "json_object"}
-
-            # check that the first message is a system message
-            # if it is not, add a system message to the beginning
-            message = f"Make sure that your response to any message matches the json_schema below, do not deviate at all: \n{response_model.model_json_schema()['properties']}"
-
+        elif mode == Mode.JSON or mode == Mode.MD_JSON:
+            if mode == Mode.JSON:
+                new_kwargs["response_format"] = {"type": "json_object"}
+                # check that the first message is a system message
+                # if it is not, add a system message to the beginning
+                message = f"""Make sure that your response to any message matches the json_schema below,
+                        do not deviate at all: \n{response_model.model_json_schema()['properties']}
+                        """
+            else:
+                message = f"""
+                    As a genius expert, your task is to understand the content and provide 
+                    the parsed objects in json that match the following json_schema (do not deviate at all and its okay if you cant be exact). Use a
+                    ```json<JSON>``` block to provide your response:\n
+                    {response_model.model_json_schema()['properties']}
+                    """
             if new_kwargs["messages"][0]["role"] != "system":
                 new_kwargs["messages"].insert(
                     0,