Preprocessing Issues about mgtab HOT 8 OPEN

df_train = read_info_data("./train_new.json")
df_train_feature = exact_and_process_feature(df_train, with_label, 'labeled_df_train.json')
df_test = read_info_data("./test_new.json")
df_test_feature = exact_and_process_feature(df_test, with_label, 'labeled_df_test.json')

if task == 1:
    df_train_feature.pop("isBot")
    df_test_feature.pop("isBot")
else:
    df_train_feature.pop("category")
    df_test_feature.pop("category")

numerical_cols = [
    "followers_count",
    "friends_count",
    "listed_count",
    "created_at",
    "favourites_count",
    "statuses_count",
    'screen_name_length',
    'name_length',
    'description_length',
    'followers_friends_ratios',
]

df = pd.concat([df_train_feature, df_test_feature], ignore_index=True)
df_name = df['screen_name']
df[numerical_cols] = MinMaxScaler().fit_transform(df[numerical_cols])

def exact_and_process_feature(df, with_label, file_name):
    if not os.path.exists('./process/'+file_name):
        to_drop = list(df.keys())
        not_to_drop = [
            # 'has_extended_profile',
            "profile_use_background_image",
            "default_profile",
            "default_profile_image",
            "verified",
            "geo_enabled",
            'profile_background_image_url',
            'url',
            'profile_background_color',
            'profile_sidebar_fill_color',
            'profile_sidebar_border_color',
            "followers_count",
            "friends_count",
            "listed_count",
            "created_at",
            "favourites_count",
            "statuses_count",
            "screen_name",
            'name',
            'description',
            'id',
            'friends_list',
            'followers_list',
            'mention_list',
            'url_list',
            'hashtag_list'
        ]

        if with_label:
            not_to_drop.append('category')
            not_to_drop.append('isBot')

        for key in not_to_drop:
            to_drop.remove(key)

        df.drop(columns=to_drop, axis=1, inplace=True)
        df = change_df_dtypes(df)
        df.to_json('./process/'+file_name)
        print('saving {}'.format(file_name))
    else:
        df = pd.read_json('./process/'+file_name)
        print('loading existing {}'.format(file_name))
    return df


   def change_df_dtypes(df):
    df = df.fillna(0)
    df["followers_count"] = np.log2((df["followers_count"].astype("int64") + 1))
    df["friends_count"] = np.log2((df["friends_count"].astype("int64") + 1))
    df["listed_count"] = np.log2((df["listed_count"].astype("int64") + 1))
    df["created_at"] = pd.to_numeric(pd.to_datetime(df["created_at"])) / 365 / 24 / 60 / 60 / 1000000000
    df["favourites_count"] = np.log2((df["favourites_count"].astype("int64") + 1))
    df["statuses_count"] = np.log2((df["statuses_count"].astype("int64") + 1))
    df['screen_name_length'] = ""
    for i, each in enumerate(df['screen_name']):
        df['screen_name_length'][i] = len(each)

    df['name_length'] = ""
    for i, each in enumerate(df['name']):
        df['name_length'][i] = len(each)
    del df['name']

    df['description_length'] = ""
    for i, each in enumerate(df['description']):
        df['description_length'][i] = len(each)
    del df['description']

    df['followers_friends_ratios'] = ""
    for i, each in enumerate(df['followers_count']):
        df['followers_friends_ratios'][i] = df['friends_count'][i] / (each + 1)

    # bool feature
    df["default_profile"] = df["default_profile"].astype("int8")
    df["default_profile_image"] = df["default_profile_image"].astype("int8")
    df["geo_enabled"] = df["geo_enabled"].astype("int8")
    df["profile_use_background_image"] = df["profile_use_background_image"].astype("int8")
    df["verified"] = df["verified"].astype("int8")

    df['is_default_profile_background_color'] = ''
    for i, each in enumerate(df['profile_background_color']):
        if each is not None:
            if each == 'F5F8FA':
                df['is_default_profile_background_color'][i] = 1
            elif each == '':
                df['is_default_profile_background_color'][i] = 1
            else:
                df['is_default_profile_background_color'][i] = 0
        else:
            df['is_default_profile_background_color'][i] = 1
    del df['profile_background_color']

    df['is_default_profile_sidebar_fill_color'] = ''
    for i, each in enumerate(df['profile_sidebar_fill_color']):
        if each is not None:
            if each == 'DDEEF6':
                df['is_default_profile_sidebar_fill_color'][i] = 1
            elif each == '':
                df['is_default_profile_sidebar_fill_color'][i] = 1
            else:
                df['is_default_profile_sidebar_fill_color'][i] = 0
        else:
            df['is_default_profile_sidebar_fill_color'][i] = 1
    del df['profile_sidebar_fill_color']

    df['is_default_profile_sidebar_border_color'] = ''
    for i, each in enumerate(df['profile_sidebar_border_color']):
        if each is not None:
            if each == 'C0DEED':
                df['is_default_profile_sidebar_border_color'][i] = 1
            elif each == '':
                df['is_default_profile_sidebar_border_color'][i] = 1
            else:
                df['is_default_profile_sidebar_border_color'][i] = 0
        else:
            df['is_default_profile_sidebar_border_color'][i] = 1
    del df['profile_sidebar_border_color']

    df['has_url'] = ''
    for i, each in enumerate(df['url']):
        if each is not None:
            if each != 0:
                df['has_url'][i] = 1
            else:
                df['has_url'][i] = 0
        else:
            df['has_url'][i] = 0
    del df['url']

    df['has_profile_background_image_url'] = ''
    for i, each in enumerate(df['profile_background_image_url']):
        if each is not None:
            if each != 0:
                df['has_profile_background_image_url'][i] = 1
            else:
                df['has_profile_background_image_url'][i] = 0
        else:
            df['has_profile_background_image_url'][i] = 0
    del df['profile_background_image_url']

    return df

def read_info_data(json_path):
    with open(json_path, "r") as f:
        data = json.loads(f.read())
    df = pd.json_normalize(data=data)

    return df

X = df[features].to_numpy()
y = df[target_label].to_numpy()
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=test_size, random_state=random_state)
    
scaler = MinMaxScaler()
    
X_train = scaler.fit_transform(X_train)
X_test = scaler.transform(X_test)

model.fit(x_train, x_test)

x_1 = scaler.transform(x_1)
model.predict(x_1)
...