-
Notifications
You must be signed in to change notification settings - Fork 0
/
CLIFS.py
48 lines (37 loc) · 1.5 KB
/
CLIFS.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
from PIL import Image
import torchvision
import numpy as np
import torch
import clip
import cv2
# Load the CLIP model and normalize images.
device = "cuda" if torch.cuda.is_available() else "cpu"
model, preprocess = clip.load('ViT-B/32', device=device)
model.eval()
def CLIFS(video_path, text_prompt):
# Load the video and extract frames.
cap = cv2.VideoCapture(video_path)
frames = []
while cap.isOpened():
ret, frame = cap.read()
if not ret:
break
frames.append(Image.fromarray(frame)) # convert numpy array to PIL Image
cap.release()
# Process each frame and caption with CLIP.
with torch.no_grad():
#text_prompt = ["A truck with the text 'odwalla'"]
text_prompt = clip.tokenize(text_prompt).to(device)
images = [preprocess(frame).unsqueeze(0).to(device) for frame in frames]
features = model.encode_image(torch.cat(images))
# Compute the similarity between each caption and frame.
similarities = features @ model.encode_text(text_prompt).T
best_match_idx = torch.argmax(similarities)
# Convert PIL image to NumPy array
image_array = np.asarray(frames[best_match_idx])
# Convert the color format of the image
bgr_image = cv2.cvtColor(image_array, cv2.COLOR_RGB2BGR)
# Save the output frame
cv2.imwrite('zwl.png', bgr_image)
#Generate an Output
CLIFS('Sample.mp4', ["A tree with the text 'ZWL'"])