forked from rlancemartin/auto-evaluator
-
Notifications
You must be signed in to change notification settings - Fork 0
/
text_utils.py
75 lines (58 loc) · 2.89 KB
/
text_utils.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
import re
from langchain.prompts import PromptTemplate
def clean_pdf_text(text: str) -> str:
"""Cleans text extracted from a PDF file."""
# TODO: Remove References/Bibliography section.
return remove_citations(text)
def remove_citations(text: str) -> str:
"""Removes in-text citations from a string."""
# (Author, Year)
text = re.sub(r'\([A-Za-z0-9,.\s]+\s\d{4}\)', '', text)
# [1], [2], [3-5], [3, 33, 49, 51]
text = re.sub(r'\[[0-9,-]+(,\s[0-9,-]+)*\]', '', text)
return text
template = """You are a teacher grading a quiz.
You are given a question, the student's answer, and the true answer, and are asked to score the student answer as either CORRECT or INCORRECT.
Example Format:
QUESTION: question here
STUDENT ANSWER: student's answer here
TRUE ANSWER: true answer here
GRADE: CORRECT or INCORRECT here
Grade the student answers based ONLY on their factual accuracy. Ignore differences in punctuation and phrasing between the student answer and true answer. It is OK if the student answer contains more information than the true answer, as long as it does not contain any conflicting statements. Begin!
QUESTION: {query}
STUDENT ANSWER: {result}
TRUE ANSWER: {answer}
GRADE:
And explain why the STUDENT ANSWER is correct or incorrect.
"""
GRADE_ANSWER_PROMPT = PromptTemplate(input_variables=["query", "result", "answer"], template=template)
template = """You are a teacher grading a quiz.
You are given a question, the student's answer, and the true answer, and are asked to score the student answer as either CORRECT or INCORRECT.
Example Format:
QUESTION: question here
STUDENT ANSWER: student's answer here
TRUE ANSWER: true answer here
GRADE: CORRECT or INCORRECT here
Grade the student answers based ONLY on their factual accuracy. Ignore differences in punctuation and phrasing between the student answer and true answer. It is OK if the student answer contains more information than the true answer, as long as it does not contain any conflicting statements. Begin!
QUESTION: {query}
STUDENT ANSWER: {result}
TRUE ANSWER: {answer}
GRADE:"""
GRADE_ANSWER_PROMPT_FAST = PromptTemplate(input_variables=["query", "result", "answer"], template=template)
template = """
Given the question: \n
{query}
Decide if the following retreived context is relevant: \n
{result}
Answer in the following format: \n
"Context is relevant: True or False." \n
And explain why it supports or does not support the correct answer: {answer}"""
GRADE_DOCS_PROMPT = PromptTemplate(input_variables=["query", "result", "answer"], template=template)
template = """
Given the question: \n
{query}
Decide if the following retreived context is relevant to the {answer}: \n
{result}
Answer in the following format: \n
"Context is relevant: True or False." \n """
GRADE_DOCS_PROMPT_FAST = PromptTemplate(input_variables=["query", "result", "answer"], template=template)