Issues with fine-tuning on the ScienceQA dataset. about llama-adapter HOT 5 OPEN

We only change the dataloader for different tasks. The network architecture, loss and optimizer can be reused without any modification. Please be careful about the loss masking, namely, only apply loss upon the answer part.

from llama-adapter.

We follow the original template of ScienceQA.

https://scienceqa.github.io/

input1 = create_one_example(self.prompt_format, question, context, choice, answer, lecture, solution, test_example=True)
input2 = create_one_example(self.prompt_format, question, context, choice, answer, lecture, solution, test_example=False)
input1 = torch.tensor(self.tokenizer1.encode(input1, bos=True, eos=False), dtype=torch.int64)
input2 = torch.tensor(self.tokenizer1.encode(input2, bos=True, eos=True), dtype=torch.int64)

from llama-adapter.

282 def getitem(self, index):
283
284 if self.index[index] in self.visual_features_map.keys():
285 visual = self.visual_features[int(self.visual_features_map[self.index[index]])]
286 visual = torch.tensor(visual).float()
287 else:
288 visual = torch.zeros(100, 256).float()
289 ann = self.ann[self.index[index]]
290
291 question = get_question_text(ann)
292 context = get_context_text(ann, self.use_caption)
293 choice = get_choice_text(ann, self.options)
294 answer = get_answer(ann, self.options)
295 lecture = get_lecture_text(ann)
296 solution = get_solution_text(ann)
297 input1 = create_one_example(self.prompt_format, question, context, choice, answer, lecture, solution, test_example=True)
298 input2 = create_one_example(self.prompt_format, question, context, choice, answer, lecture, solution, test_example=False)
299 input1 = torch.tensor(self.tokenizer1.encode(input1, bos=True, eos=False), dtype=torch.int64)
300 input2 = torch.tensor(self.tokenizer1.encode(input2, bos=True, eos=True), dtype=torch.int64)
301 padding = self.max_words - input2.shape[0]
302 if padding > 0:
303 input2 = torch.cat((input2, torch.zeros(padding, dtype=torch.int64) - 1))
304 elif padding < 0:
305 input2 = input2[:self.max_words]
306 labels = copy.deepcopy(input2)
307 labels[:len(input1)] = -1
308 input2_mask = input2.ge(0)
309 label_mask = labels.ge(0)
310 input2[~input2_mask] = 0
311 labels[~label_mask] = 0
312 input2_mask = input2_mask.float()
313 label_mask = label_mask.float()
314 return input2, labels, input2_mask, visual

from llama-adapter.

282 def getitem(self, index): 283 284 if self.index[index] in self.visual_features_map.keys(): 285 visual = self.visual_features[int(self.visual_features_map[self.index[index]])] 286 visual = torch.tensor(visual).float() 287 else: 288 visual = torch.zeros(100, 256).float() 289 ann = self.ann[self.index[index]] 290 291 question = get_question_text(ann) 292 context = get_context_text(ann, self.use_caption) 293 choice = get_choice_text(ann, self.options) 294 answer = get_answer(ann, self.options) 295 lecture = get_lecture_text(ann) 296 solution = get_solution_text(ann) 297 input1 = create_one_example(self.prompt_format, question, context, choice, answer, lecture, solution, test_example=True) 298 input2 = create_one_example(self.prompt_format, question, context, choice, answer, lecture, solution, test_example=False) 299 input1 = torch.tensor(self.tokenizer1.encode(input1, bos=True, eos=False), dtype=torch.int64) 300 input2 = torch.tensor(self.tokenizer1.encode(input2, bos=True, eos=True), dtype=torch.int64) 301 padding = self.max_words - input2.shape[0] 302 if padding > 0: 303 input2 = torch.cat((input2, torch.zeros(padding, dtype=torch.int64) - 1)) 304 elif padding < 0: 305 input2 = input2[:self.max_words] 306 labels = copy.deepcopy(input2) 307 labels[:len(input1)] = -1 308 input2_mask = input2.ge(0) 309 label_mask = labels.ge(0) 310 input2[~input2_mask] = 0 311 labels[~label_mask] = 0 312 input2_mask = input2_mask.float() 313 label_mask = label_mask.float() 314 return input2, labels, input2_mask, visual

Thank you for your reply! Can the loss function on the ScienceQA dataset also be applied directly to the current V1 version?

from llama-adapter.

Hi @Gary3410 were you successful in replicating the results? I am getting accuracy in the 60s range.

from llama-adapter.