Demo for inference a single image with prompt?

[perp]

could you provide a demo for inference a single image with prompt?

[ForestWang]

+1

[MinusOne2]

+1

[benakhovan32561]

+1

[benakhovan32561]

Anyone created by any chance?

[Xamexer]

I'm working on my bachelor thesis right now for LLM Wildlive trapcam segmentation. Please, we need a demo or a script with image and prompt input to test it out! Just an easy copy-paste python snippet would already work, no interface needed!
Specially for HyperSeg-3B basic segmentation would a small snippet be amazing!

<3 <3 <3

[wj-on-un]

The code works, but feel free to modify it if needed.

import torch
import torch.distributed as distributed
import os
from enum import Enum
import json
from tqdm import tqdm
import shortuuid
import numpy as np
import cv2
from torch.utils.data import Dataset, DataLoader
from transformers import SiglipImageProcessor
import re
from hyperseg.utils.constants import IGNORE_INDEX, IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, \
    DEFAULT_IM_END_TOKEN, DEFAULT_SEG_TOKEN, SEG_TOKEN_INDEX, DEFAULT_CLS_TOKEN, CLS_TOKEN_INDEX, DEFAULT_REGION_TOKEN, \
    REGION_TOKEN_INDEX, REFER_TOKEN_INDEX, DEFAULT_TEMPORAL_TOKEN, TEMPORAL_TOKEN_INDEX, \
    YTVIS_CATEGORIES_2019, OVIS_CATEGORIES
from hyperseg.utils.builder import load_pretrained_model
from hyperseg.utils import conversation as conversation_lib
from hyperseg.eval.eval_dataset.eval_datasets import DataCollatorForCOCODatasetV2, RefCOCO_dataset_test
from detectron2.data import MetadataCatalog, DatasetCatalog
from pycocotools import mask
from typing import Dict, Optional, Sequence, List
from dataclasses import dataclass, field
import torch.distributed as dist
import transformers
import pickle
from pathlib import Path
from dataclasses import dataclass, field
from typing import Optional
import argparse
from hyperseg.model.mipha import conversation as conversation_lib
from PIL import Image

def preprocess_llama2(sources, tokenizer, class_name_id_pred=None):
    conv = conversation_lib.default_conversation.copy()
    roles = {"human": conv.roles[0], "gpt": conv.roles[1]}

    # Apply prompt templates
    conversations = []
    for i, source in enumerate(sources):
        if roles[source[0]["from"]] != conv.roles[0]:
            # Skip the first one if it is not from human
            source = source[1:]

        conv.messages = []
        for j, sentence in enumerate(source):
            role = roles[sentence["from"]]
            assert role == conv.roles[j % 2], f"{i}"
            conv.append_message(role, sentence["value"])
        conversations.append(conv.get_prompt())

    # Tokenize conversations
    input_ids = torch.stack(
        [tokenizer_special_tokens(prompt, tokenizer, return_tensors='pt', class_name_id_pred=class_name_id_pred) for prompt in conversations], dim=0)

    targets = input_ids.clone()
    # Mask targets
    sep = conv.sep + conv.roles[1] + ": "
    idx = 0
    for conversation, target in zip(conversations, targets):
        total_len = int(target.ne(tokenizer.pad_token_id).sum())

        rounds = conversation.split(conv.sep2)
        if conv.version == 'v0':
            cur_len = 0
            end_token_cnt = 0
            idx = 0
            for i, rou in enumerate(rounds):
                if rou == "":
                    continue
                parts = rou.split(sep)
                if len(parts) != 2:
                    break
                parts[0] += sep
                if idx > 0:
                    round_len = len(tokenizer_special_tokens(rou, tokenizer, class_name_id_pred=class_name_id_pred)) + 1
                else:
                    round_len = len(tokenizer_special_tokens(rou, tokenizer, class_name_id_pred=class_name_id_pred)) + 1
                if idx > 0:
                    instruction_len = len(tokenizer_special_tokens(parts[0], tokenizer, class_name_id_pred=class_name_id_pred))
                else:
                    instruction_len = len(tokenizer_special_tokens(parts[0], tokenizer, class_name_id_pred=class_name_id_pred)) - 2
                target[cur_len: cur_len + instruction_len] = IGNORE_INDEX
                end_token_cnt += 1
                cur_len += round_len
                idx += 1
            target[cur_len:] = IGNORE_INDEX
            cur_len -= end_token_cnt
        else:
            cur_len = 1
            target[:cur_len] = IGNORE_INDEX
            for i, rou in enumerate(rounds):
                if rou == "":
                    continue
                parts = rou.split(sep)
                if len(parts) != 2:
                    break
                parts[0] += sep
                round_len = len(tokenizer_special_tokens(rou, tokenizer, class_name_id_pred=class_name_id_pred))
                instruction_len = len(tokenizer_special_tokens(parts[0], tokenizer, class_name_id_pred=class_name_id_pred)) - 2
                target[cur_len: cur_len + instruction_len] = IGNORE_INDEX
                cur_len += round_len
                idx += 1
            target[cur_len:] = IGNORE_INDEX

        if cur_len < tokenizer.model_max_length:
            if cur_len != total_len:
                target[:] = IGNORE_INDEX
                print(
                    f"WARNING: tokenization mismatch: {cur_len} vs. {total_len}."
                    f" (ignored)"
                )
    return dict(
        input_ids=input_ids,
        labels=targets,
    )

def tokenizer_special_tokens(prompt, tokenizer, image_token_index=IMAGE_TOKEN_INDEX,
                             seg_token_index=SEG_TOKEN_INDEX, cls_token_index=CLS_TOKEN_INDEX,
                             region_token_index=REGION_TOKEN_INDEX,refer_token_index=REFER_TOKEN_INDEX, return_tensors=None, class_name_id_pred=None):
    input_ids = []
    special_token_map = {'<image>': image_token_index, '<seg>': seg_token_index, '<cls>': cls_token_index, '<region>':region_token_index, '<refer>':refer_token_index, DEFAULT_TEMPORAL_TOKEN:TEMPORAL_TOKEN_INDEX}
    prompt_chunks = re.split('(<image>|<seg>|<cls>|<region>|<refer>|<gt>|<temporal>)', prompt)

    for chunk in prompt_chunks:
        if chunk == '<gt>':
            input_ids.extend(class_name_id_pred)
        elif chunk in special_token_map:
            input_ids.append(special_token_map[chunk])
        elif chunk != '':
            input_ids.extend(tokenizer.encode(chunk, add_special_tokens=False))
    if return_tensors is not None:
        if return_tensors == 'pt':
            return torch.tensor(input_ids, dtype=torch.long).squeeze()
        raise ValueError(f'Unsupported tensor type: {return_tensors}')
    else:
        return input_ids
    
def preprocess_referring_instruction(instruction, REFER_token='[SEG]', sum_ref_pred=False):
        tokenized = tokenizer.encode(instruction, add_special_tokens=False)
        REFER_token_id = [tokenizer.encode(REFER_token, add_special_tokens=False)[0]]
        tokenized = tokenized + REFER_token_id
        if sum_ref_pred:
            pred_ref_id = tokenized
        else:
            pred_ref_id = REFER_token_id

        token_refer_id = torch.tensor(tokenized)

        return token_refer_id, pred_ref_id

@dataclass
class DataArguments:
    vision_tower: str = "./pretrained_model/siglip-so400m-patch14-384"
    vision_tower_mask: str = "./pretrained_model/mask2former/model_final_54b88a.pkl"
    lazy_preprocess: bool = False
    is_multimodal: bool = False
    image_folder: Optional[str] = "./dataset/coco/train2014"
    model_path: Optional[str] = "./model/HyperSeg-3B"
    mask_config: Optional[str] = "./hyperseg/model/mask_decoder/mask_config/maskformer2_swin_base_384_bs16_50ep.yaml"
    image_aspect_ratio: str = "square"
    image_grid_pinpoints: Optional[str] = None
    model_map_name: str = "HyperSeg"
    version: str = "llava_phi"
    segmentation: bool = True
    eval_batch_size: int = 1
    dataloader_num_workers: int = 8
    seg_task: Optional[str] = "referring"
    sum_ref_pred_answer: bool = False
    enable_mgvp_seg_query: bool = True
    visualize: bool = True

def get_data_args():
    return DataArguments()

data_args = get_data_args()
model_path = os.path.expanduser(data_args.model_path)
tokenizer, model, image_processor, context_len = load_pretrained_model(model_path, model_args=data_args, mask_config=data_args.mask_config, device='cuda')
device = torch.device("cuda") 
model.to(dtype=torch.float32, device=device)
print("Loaded")

data_args.image_processor = image_processor
data_args.is_multimodal = True
clip_image_processor = SiglipImageProcessor.from_pretrained(data_args.vision_tower)

####### Input Sentence #######
prefix_inst = 'This is an image <image>\n<temporal>\n, Please doing Referring Segmentation according to the following instruction:'
instruction = "knife"
##############################

sum_ref_pred = data_args.sum_ref_pred_answer
token_refer_id, pred_ref_id = preprocess_referring_instruction(instruction, sum_ref_pred=sum_ref_pred)

sources = [[{'from': 'human', 'value': prefix_inst + '\n<refer>'}, {'from': 'gpt', 'value': '\nSure, the segmentation result is <gt><seg>'}]]

text_dict = preprocess_llama2(sources, tokenizer, pred_ref_id)
input_ids = text_dict['input_ids'][0]
labels = text_dict['labels'][0]


refer_embedding_indices = torch.zeros_like(input_ids)
refer_embedding_indices[input_ids == REFER_TOKEN_INDEX] = 1

model_input_ids = text_dict['input_ids']
model_labels = text_dict['labels']
model_dataset_type = 'referring_coco'

model_token_refer_id = token_refer_id
model_refer_embedding_indices = refer_embedding_indices

######### Input image ########
image_path = "./knife_test.jpg"
##############################

image_clip = cv2.imread(image_path)
np_h, np_w, np_c = image_clip.shape
image_clip = cv2.cvtColor(image_clip, cv2.COLOR_BGR2RGB)
model_image_clip = clip_image_processor.preprocess(image_clip, return_tensors="pt")["pixel_values"].to(device)

padding_mask = np.ones((np_h, np_w), dtype=np.uint8)

seg_info = [{"height": np_h, "width": np_w, "segments": [{"id": 0, "class": "knife",},], "image_path": str, "padding_mask": padding_mask}]

with torch.no_grad():
    outputs = model.eval_seg(
        input_ids=model_input_ids.to(device),
        images=model_image_clip.float(),
        images_clip=model_image_clip.float(),
        seg_info=seg_info,
        token_refer_id = model_token_refer_id.unsqueeze(0).to(device),
        refer_embedding_indices=model_refer_embedding_indices.unsqueeze(0).to(device),
    )
    
scores = outputs[0]['instances'].scores
max_index = torch.argmax(scores).item()

Image.fromarray(outputs[0]['instances'].pred_masks[max_index].detach().cpu().numpy().astype(np.uint8)*255).save("pred.png")

[temiwale88]

Thanks @wj-on-un for sharing! Hope to try it.

[deaodaoo]

Thank you, I have successfully used your code. @wj-on-un