ningshuliang / picture Goto Github PK

from ldm.modules.encoders.modules import ClipImageProjector
from torchvision import transforms

version="./../pretrain_models/clip-vit-large-patch14"
clip_model2 = ClipImageProjector(version=version).to(device)
tform = transforms.ToTensor()

# text = tokenizer(text_description,truncation=True, max_length=77, return_length=True,
#             return_overflowing_tokens=False, padding="max_length", return_tensors="pt")

# text_features = clip_model(text["input_ids"].cuda(non_blocking=True))
# text_features = text_features.last_hidden_state # torch.Size([1, 77, 768])

garment_condition_path = os.path.join("./Sample_data/Cloth_White_Background", file_name[0])
garment_condition = tform(Image.open(garment_condition_path).convert("RGB"))

garment_condition = garment_condition * 2. - 1.
garment_condition = clip_model2.preprocess(garment_condition.unsqueeze(0)) # got similar results with / without preprocessing
text_features = clip_model2(garment_condition.cuda(non_blocking=True))

c = [concat_feature,text_features]
sampler.sample(S=opt.ddim_steps,
               conditioning=c,
               ...)

labels = { # from https://github.com/shadow2496/VITON-HD
    0:  ['background',  [0]], 
    1:  ['paste',       [2, 4, 7, 8, 9, 10, 11]], 
    2:  ['upper',       [3]],
    3:  ['hair',        [1]],
    4:  ['left_arm',    [5]],
    5:  ['right_arm',   [6]],
    6:  ['noise',       [12]]
}
labels_to_colours = [(0,0,0), ...]