a fork of shap-e for gc
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 

270 lines
9.6 KiB

from typing import Iterable, List, Optional, Union
import numpy as np
import torch
import torch.nn as nn
from PIL import Image
from shap_e.models.download import default_cache_dir
ImageType = Union[np.ndarray, torch.Tensor, Image.Image]
class ImageCLIP(nn.Module):
"""
A wrapper around a pre-trained CLIP model that automatically handles
batches of texts, images, and embeddings.
"""
def __init__(
self,
device: torch.device,
dtype: Optional[torch.dtype] = torch.float32,
ensure_used_params: bool = True,
clip_name: str = "ViT-L/14",
cache_dir: Optional[str] = None,
):
super().__init__()
assert clip_name in ["ViT-L/14", "ViT-B/32"]
self.device = device
self.ensure_used_params = ensure_used_params
# Lazy import because of torchvision.
import clip
self.clip_model, self.preprocess = clip.load(
clip_name, device=device, download_root=cache_dir or default_cache_dir()
)
self.clip_name = clip_name
if dtype is not None:
self.clip_model.to(dtype)
self._tokenize = clip.tokenize
@property
def feature_dim(self) -> int:
if self.clip_name == "ViT-L/14":
return 768
else:
return 512
@property
def grid_size(self) -> int:
if self.clip_name == "ViT-L/14":
return 16
else:
return 7
@property
def grid_feature_dim(self) -> int:
if self.clip_name == "ViT-L/14":
return 1024
else:
return 768
def forward(
self,
batch_size: int,
images: Optional[Iterable[Optional[ImageType]]] = None,
texts: Optional[Iterable[Optional[str]]] = None,
embeddings: Optional[Iterable[Optional[torch.Tensor]]] = None,
) -> torch.Tensor:
"""
Generate a batch of embeddings from a mixture of images, texts,
precomputed embeddings, and possibly empty values.
For each batch element, at most one of images, texts, and embeddings
should have a non-None value. Embeddings from multiple modalities
cannot be mixed for a single batch element. If no modality is provided,
a zero embedding will be used for the batch element.
"""
image_seq = [None] * batch_size if images is None else list(images)
text_seq = [None] * batch_size if texts is None else list(texts)
embedding_seq = [None] * batch_size if embeddings is None else list(embeddings)
assert len(image_seq) == batch_size, "number of images should match batch size"
assert len(text_seq) == batch_size, "number of texts should match batch size"
assert len(embedding_seq) == batch_size, "number of embeddings should match batch size"
if self.ensure_used_params:
return self._static_multimodal_embed(
images=image_seq, texts=text_seq, embeddings=embedding_seq
)
result = torch.zeros((batch_size, self.feature_dim), device=self.device)
index_images = []
index_texts = []
for i, (image, text, emb) in enumerate(zip(image_seq, text_seq, embedding_seq)):
assert (
sum([int(image is not None), int(text is not None), int(emb is not None)]) < 2
), "only one modality may be non-None per batch element"
if image is not None:
index_images.append((i, image))
elif text is not None:
index_texts.append((i, text))
elif emb is not None:
result[i] = emb.to(result)
if len(index_images):
embs = self.embed_images((img for _, img in index_images))
for (i, _), emb in zip(index_images, embs):
result[i] = emb.to(result)
if len(index_texts):
embs = self.embed_text((text for _, text in index_texts))
for (i, _), emb in zip(index_texts, embs):
result[i] = emb.to(result)
return result
def _static_multimodal_embed(
self,
images: List[Optional[ImageType]] = None,
texts: List[Optional[str]] = None,
embeddings: List[Optional[torch.Tensor]] = None,
) -> torch.Tensor:
"""
Like forward(), but always runs all encoders to ensure that
the forward graph looks the same on every rank.
"""
image_emb = self.embed_images(images)
text_emb = self.embed_text(t if t else "" for t in texts)
joined_embs = torch.stack(
[
emb.to(device=self.device, dtype=torch.float32)
if emb is not None
else torch.zeros(self.feature_dim, device=self.device)
for emb in embeddings
],
dim=0,
)
image_flag = torch.tensor([x is not None for x in images], device=self.device)[
:, None
].expand_as(image_emb)
text_flag = torch.tensor([x is not None for x in texts], device=self.device)[
:, None
].expand_as(image_emb)
emb_flag = torch.tensor([x is not None for x in embeddings], device=self.device)[
:, None
].expand_as(image_emb)
return (
image_flag.float() * image_emb
+ text_flag.float() * text_emb
+ emb_flag.float() * joined_embs
+ self.clip_model.logit_scale * 0 # avoid unused parameters
)
def embed_images(self, xs: Iterable[Optional[ImageType]]) -> torch.Tensor:
"""
:param xs: N images, stored as numpy arrays, tensors, or PIL images.
:return: an [N x D] tensor of features.
"""
clip_inputs = self.images_to_tensor(xs)
results = self.clip_model.encode_image(clip_inputs).float()
return results / torch.linalg.norm(results, dim=-1, keepdim=True)
def embed_text(self, prompts: Iterable[str]) -> torch.Tensor:
"""
Embed text prompts as an [N x D] tensor.
"""
enc = self.clip_model.encode_text(
self._tokenize(list(prompts), truncate=True).to(self.device)
).float()
return enc / torch.linalg.norm(enc, dim=-1, keepdim=True)
def embed_images_grid(self, xs: Iterable[Optional[ImageType]]) -> torch.Tensor:
"""
Embed images into latent grids.
:param xs: an iterable of images to embed.
:return: a tensor of shape [N x C x L], where L = self.grid_size**2.
"""
if self.ensure_used_params:
extra_value = 0.0
for p in self.parameters():
extra_value = extra_value + p.mean() * 0.0
else:
extra_value = 0.0
x = self.images_to_tensor(xs).to(self.clip_model.dtype)
# https://github.com/openai/CLIP/blob/4d120f3ec35b30bd0f992f5d8af2d793aad98d2a/clip/model.py#L225
vt = self.clip_model.visual
x = vt.conv1(x) # shape = [*, width, grid, grid]
x = x.reshape(x.shape[0], x.shape[1], -1) # shape = [*, width, grid ** 2]
x = x.permute(0, 2, 1) # shape = [*, grid ** 2, width]
x = torch.cat(
[
vt.class_embedding.to(x.dtype)
+ torch.zeros(x.shape[0], 1, x.shape[-1], dtype=x.dtype, device=x.device),
x,
],
dim=1,
) # shape = [*, grid ** 2 + 1, width]
x = x + vt.positional_embedding.to(x.dtype)
x = vt.ln_pre(x)
x = x.permute(1, 0, 2) # NLD -> LND
x = vt.transformer(x)
x = x.permute(1, 2, 0) # LND -> NDL
return x[..., 1:].contiguous().float() + extra_value
def images_to_tensor(self, xs: Iterable[Optional[ImageType]]) -> torch.Tensor:
return torch.stack([self.preprocess(_image_to_pil(x)) for x in xs], dim=0).to(self.device)
class FrozenImageCLIP:
def __init__(self, device: torch.device, **kwargs):
self.model = ImageCLIP(device, dtype=None, ensure_used_params=False, **kwargs)
for parameter in self.model.parameters():
parameter.requires_grad_(False)
@property
def feature_dim(self) -> int:
return self.model.feature_dim
@property
def grid_size(self) -> int:
return self.model.grid_size
@property
def grid_feature_dim(self) -> int:
return self.model.grid_feature_dim
def __call__(
self,
batch_size: int,
images: Optional[Iterable[Optional[ImageType]]] = None,
texts: Optional[Iterable[Optional[str]]] = None,
embeddings: Optional[Iterable[Optional[torch.Tensor]]] = None,
) -> torch.Tensor:
# We don't do a no_grad() here so that gradients could still
# flow to the input embeddings argument.
# This behavior is currently not used, but it could be.
return self.model(batch_size=batch_size, images=images, texts=texts, embeddings=embeddings)
def embed_images(self, xs: Iterable[Optional[ImageType]]) -> torch.Tensor:
with torch.no_grad():
return self.model.embed_images(xs)
def embed_text(self, prompts: Iterable[str]) -> torch.Tensor:
with torch.no_grad():
return self.model.embed_text(prompts)
def embed_images_grid(self, xs: Iterable[Optional[ImageType]]) -> torch.Tensor:
with torch.no_grad():
return self.model.embed_images_grid(xs)
def _image_to_pil(obj: Optional[ImageType]) -> Image.Image:
if obj is None:
return Image.fromarray(np.zeros([64, 64, 3], dtype=np.uint8))
if isinstance(obj, np.ndarray):
return Image.fromarray(obj.astype(np.uint8))
elif isinstance(obj, torch.Tensor):
return Image.fromarray(obj.detach().cpu().numpy().astype(np.uint8))
else:
return obj