# Visualbert config.json

**URL:** <https://discuss.huggingface.co/t/visualbert-config-json/47148>\
**Category:** Beginners\
**Created:** [July 18, 2023, 5:15am UTC](https://discuss.huggingface.co/t/visualbert-config-json/47148 "2023-07-18T05:15:32Z")\
**Posts on this page:** 1\
**Page:** 1

<div class="post-metadata">

**Author:** ![wayne123456](https://avatars.discourse-cdn.com/v4/letter/w/35a633/32.png) [@wayne123456](https://discuss.huggingface.co/u/wayne123456)\
**Post date:** [July 18, 2023, 5:15am UTC](https://discuss.huggingface.co/t/visualbert-config-json/47148/1 "2023-07-18T05:15:32Z")

</div>

I am doing visualBERT VQA evaluation but the target tensor is always 0. Could share the pytorch dataset for local files or update the config. json? Thank you  
Here is my code:

# ===========================================================================================

# Read questions

import json

# Checking path and Opening JSON file,

f\_read\_questions = open(‘./vqa2/v2\_OpenEnded\_mscoco\_val2014\_questions.json’)

# Return JSON object as dictionary

data\_questions = json.load(f\_read\_questions)  
print(data\_questions.keys())

questions = data\_questions[‘questions’]  
print(“Number of questions:”, len(questions))

from os import listdir  
from os.path import isfile, join

# root at which all images are stored

root = ‘./multimodal\_data/vqa2/val2014’  
file\_names = [f for f in listdir(root) if isfile(join(root, f))]

import re  
from typing import Optional

filename\_re = re.compile(r".\*(\d{12}).((jpg)|(png))")

# source: [allennlp-models/allennlp\_models/vision/dataset\_readers/vqav2.py at a36aed540e605c4293c25f73d6674071ca9edfc3 · allenai/allennlp-models · GitHub](https://github.com/allenai/allennlp-models/blob/a36aed540e605c4293c25f73d6674071ca9edfc3/allennlp_models/vision/dataset_readers/vqav2.py#L141)

def id\_from\_filename(filename: str) → Optional[int]:  
match = filename\_re.fullmatch(filename)  
if match is None:  
return None  
return int(match.group(1))

filename\_to\_id = {root + “/” + file: id\_from\_filename(file) for file in file\_names}  
id\_to\_filename = {v: k for k, v in filename\_to\_id.items()}

from os import listdir  
from os.path import isfile, join

# root at which all images are stored

root = ‘./multimodal\_data/vqa2/val2014’  
file\_names = [f for f in listdir(root) if isfile(join(root, f))]

# Read annotations

f\_read\_annotations = open(“./vqa2/v2\_mscoco\_val2014\_annotations.json”)

# Return JSON object as dictionary

data\_annotations = json.load(f\_read\_annotations)  
print(data\_annotations.keys())

# show answwers

annotations = data\_annotations[‘annotations’]  
print(“Number of annotations:”, len(annotations))

from transformers import VisualBertConfig, VisualBertModel  
config = VisualBertConfig.from\_pretrained(“./pretrained/visualBERT/visualbert-vqa”)

from tqdm.notebook import tqdm

def get\_score(count: int) → float:  
return min(1.0, count / 3)

for annotation in tqdm(annotations):  
answers = annotation[‘answers’]  
answer\_count = {}  
for answer in answers:  
answer\_ = answer[“answer”]  
answer\_count[answer\_] = answer\_count.get(answer\_, 0) + 1  
labels =   
scores =   
for answer in answer\_count:  
if answer not in list(config.label2id.keys()):  
continue  
labels.append(config.label2id[answer])  
score = get\_score(answer\_count[answer])  
scores.append(score)  
annotation[‘labels’] = labels  
annotation[‘scores’] = scores

class VQADataset(torch.utils.data.Dataset):  
“”“VQA (v2) dataset.”“”

```auto
def __init__ (self, questions, annotations, image_preprocess):
    self.questions = questions
    self.annotations = annotations
    self.image_preprocess = image_preprocess

def __len__ (self):
    return len(self.annotations)

def __getitem__ (self, idx):
    # get image + text
    annotation = self.annotations[idx]
    questions = self.questions[idx]
    image = id_to_filename[annotation['image_id']]
    text = questions['question']

    inputs = tokenizer(
                text,
                padding="max_length",
                max_length=40,
                truncation=True,
                return_token_type_ids=True,
                return_attention_mask=True,
                add_special_tokens=True,
                return_tensors="pt",
            )

     #faster-rcnn
    images, sizes, scales_yx = self.image_preprocess(image)
    output_dict = frcnn(
        images,
        sizes,
        scales_yx=scales_yx,
        padding="max_detections",
        max_detections=frcnn_cfg.max_detections,
        return_tensors="pt",
    )

    features = output_dict.get("roi_features")

    inputs.update(
        {"input_ids": inputs.input_ids,
         "attention_mask": inputs.attention_mask,
         "token_type_ids": inputs.token_type_ids,
         "visual_embeds": features,
         "visual_attention_mask": torch.ones(features.shape[:-1], dtype=torch.float),
         "visual_token_type_ids": torch.ones(features .shape[:-1], dtype=torch.long),
         # "output_attentions": False
         }
    )

  # remove batch dimension
    for k, v in inputs.items():
        inputs[k] = v.squeeze()

   # add labels
    labels = annotation['labels']
    scores = torch.tensor(annotation['scores'])

    targets = torch.zeros(len(config.id2label), dtype=torch.float)
    for label, score in zip(labels, scores):
        targets[label] = score
    inputs["labels"] = targets # get the index of the highest score as target

    # based on: https://github.com/dandelin/ViLT/blob/762fd3975c180db6fc88f577cf39549983fa373a/vilt/modules/objectives.py#L301
    return inputs

```

if **name** == ‘ **main** ’:  
from visual\_bert.processing\_image import Preprocess  
from visual\_bert.visualizing\_image import SingleImageViz  
from visual\_bert.modeling\_frcnn import GeneralizedRCNN  
from visual\_bert.utils import Config

```auto
frcnn_cfg = Config.from_pretrained("unc-nlp/frcnn-vg-finetuned")
frcnn = GeneralizedRCNN.from_pretrained("unc-nlp/frcnn-vg-finetuned", config=frcnn_cfg)
image_preprocess = Preprocess(frcnn_cfg)

from transformers import VisualBertForQuestionAnswering, BertTokenizerFast
tokenizer = BertTokenizerFast.from_pretrained("./pretrained/bert-base-uncased")
model = VisualBertForQuestionAnswering.from_pretrained("./pretrained/visualBERT/visualbert-vqa")

dataset = VQADataset(questions=questions[:100],
                     annotations=annotations[:100],
                     image_preprocess=image_preprocess)

test_dataloader = DataLoader(dataset, batch_size=1, shuffle=False)

correct = 0.0
total = 0

loss_function = nn.CrossEntropyLoss()
# optimizer = optim.Adam(model.classifier.parameters(), lr=0.001)

for batch in tqdm(test_dataloader):
    # with torch.no_grad():
   
        batch = {k: v.to(device) for k, v in batch.items()}
        # here is the probelm
        print(batch["labels"])

        # forward pass
        outputs = model(**batch)
        logits= outputs.logits
        _, pre = torch.max(logits.data, 1)
        _, target = torch.max(batch["labels"].data, 1)
        print("prediction:", pre)
        print("target:", target)

        correct += (pre == target).sum().item()
        total = total + 1
        print(total)
        print("==============================================================")

final_acc = correct / float(len(test_dataloader.dataset))
print('Accuracy of test: %f %%' % (100 * float(final_acc)))

```
