import os
import json
import random

folder_path = ''
# file_names = os.listdir(folder_path)
file_names = ['refexp_train.jsonl', 'widget_caption_maxlen_train.jsonl',  'taperception_train.jsonl']
step_i = 0
train_step = []
for file_name in file_names:    
    json_data = []
    with open(folder_path + file_name, 'r') as file:
        for line in file:
            json_data.append(json.loads(line))

    for sample in json_data:
        sample_json = {}
        conversations = []
        img_path = '' + '/' + sample['img_name']
        conv_user = {"role": "user", "content": ""}
        conv_user["content"] += sample['prompt']
        conv_ai = {"role": "assistant", "content": sample['text']}
        conversations.append(conv_user)
        conversations.append(conv_ai)
        sample_json['conversations'] = conversations
        sample_json['image'] = img_path
        sample_json['id'] = step_i

        train_step.append(sample_json)
        step_i += 1
random.shuffle(train_step)
print("Num of total step: " + str(len(train_step)))
json.dump(train_step, open('public_glm4_grouding_train.json', "w"), ensure_ascii=False)