Scorecard about optbinning HOT 21 CLOSED

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import pandas as pd
from optbinning import OptimalBinning


class OptScoreCard:
    """Create ScoreCards using optbinning.

    Args:
            df (pandas.DataFrame): Dataframe containing the features.
            dict_parameters (dict): Dicionary with name of column as key and tuple
                with value to replace and datatype as value.
            target_col (str): Name of the target binary column. 

    Attributes:
        df (pandas.DataFrame): Stores input dataframe with the variables.
        target_array (numpy.array): Array containing the target variable.
        dict_parameters (dict): Dictionary containing each feature column of 
            interest from the df as key, and a respective tuple with the fillna 
            placeholder and dtype as values.
        df_bins (pandas.DataFrame): Dataframe with bin categories replacing 
            original values
        df_dummies (pandas.DataFrame): Dataframe with one-hot encoding applied
            from df_bins. 
        dict_optbins (dict): Store OptimalBinning objects from each variable
            processed.
        dict_optbins_filtered (dict): Filtered variables from dict_optbins.
        reference_bin_categories (dict): Variable name as key and bin 
            category for each selected variable.

    """

    def __init__(self, df, dict_parameters, target_col):
        self.df = df
        self.target_array = self.df[target_col].to_numpy()
        self.dict_parameters = dict_parameters
        self.df_bins = pd.DataFrame()
        self.df_dummies = pd.DataFrame()
        self.dict_optbins = {}
        self.dict_optbins_filtered = {}
        self.reference_bin_categories = {}

    def fillna(self, filter_cols=False):
        """
        Replace null values based on pre-defined parameters.

        Args:
            filter_cols (bool): Select columns dedine in dict_parameters in df.
        """
        dict_median = {
            k: v[0]
            for k, v in self.dict_parameters.items()
            if v[0] == "median"
        }
        dict_mean = {
            k: v[0] for k, v in self.dict_parameters.items() if v[0] == "mean"
        }
        dict_values = {
            k: v[0]
            for k, v in self.dict_parameters.items()
            if v[0] not in ["median", "mean"]
        }
        dict_datatype = {
            k: v[1]
            for k, v in self.dict_parameters.items()
            if v[1] is not None
        }
        dict_median = (
            self.df[list(dict_median.keys())].median().round(2).to_dict()
        )
        dict_mean = self.df[list(dict_mean.keys())].mean().round(2).to_dict()

        # Fillna
        dict_fillna = {**dict_median, **dict_mean, **dict_values}
        self.df.fillna(dict_fillna, inplace=True)

        # Convert datatype
        self.df = self.df.astype(dict_datatype)
        if filter_cols:
            self.df[list(dict_fillna.columns)]
        else:
            self.df

    def get_optimal_bins(self, solver="cp"):
        """Perform optimal binning in each column of the dataframe.
        
        Args:
            solver (str, default='cp'): Type of algorithm to solve the binning
                task.

        """
        for k, v in self.dict_parameters.items():
            if v[1] in ("category", "str"):
                x = self.df[k].to_numpy()
                optb = OptimalBinning(
                    name=k, dtype="categorical", solver=solver
                )
                optb.fit(x, self.target_array)
                self.dict_optbins[k] = [
                    optb,
                    optb.binning_table.build().at["Totals", "IV"],
                    {
                        k: str(v)
                        for k, v in dict(
                            zip(
                                optb._binning_table.build().WoE,
                                optb._binning_table.build().Bin,
                            )
                        ).items()
                        if str(v) not in ("", "Special", "Missing")
                    },
                ]

            elif v[1] in ("float", "int"):
                x = self.df[k].to_numpy()
                optb = OptimalBinning(name=k, solver=solver)
                optb.fit(x, self.target_array)
                self.dict_optbins[k] = [
                    optb,
                    optb.binning_table.build().at["Totals", "IV"],
                    {
                        k: str(v)
                        for k, v in dict(
                            zip(
                                optb._binning_table.build().WoE,
                                optb._binning_table.build().Bin,
                            )
                        ).items()
                        if str(v) not in ("", "Special", "Missing")
                    },
                ]
            else:
                self.dict_optbins[k] = [optb, 0, 'dtype_error']

    def show_iv_table(self):
        """Show a dataframe with variables and IV in ascending order."""
        return pd.DataFrame.from_dict(
            self.dict_optbins,
            orient="index",
            columns=["OptimalBinning_obj", "IV", "WoE-index"],
        )[["IV"]].sort_values(by="IV", ascending=False)

    def filter_features_by_IV(self, by='best', threshold=50):
        """Select the features based on the IV score.

        Need to implement other ways of filtering (eg: Percentage)
        """
        self.dict_optbins_filtered = {
            k: v
            for k, v in self.dict_optbins.items()
            if k in self.show_iv_table().IV[:threshold].index
        }
        self.df = self.df[self.dict_optbins_filtered.keys()]

    def transform_into_bin_categories(self):
        """
        Transform values into binning categories based on OptBinning.
        """

        for k, v in self.dict_optbins_filtered.items():
            self.df_bins[k] = pd.Series(
                self.dict_optbins_filtered.get(k)[0].transform(
                    self.df[k], metric="bins"
                )
            )

    def create_dummies_from_bins(self):
        """
        Create a One-Hot encoded dataframe from df_bins.
        """
        for k, v in self.dict_optbins_filtered.items():
            self.df_dummies = pd.concat(
                [
                    self.df_dummies,
                    pd.get_dummies(
                        self.df_bins[k],
                        prefix=k,
                        prefix_sep=":",
                        columns=list(
                            self.dict_optbins_filtered[k][2].values()
                        ),
                    ),
                ],
                axis=1,
            )

    def get_reference_bin_categories(self):
        """
        Define reference category for binned variables.

        The reference categories are selected by the lowest WoE of the bins
        of a variable.
        """
        self.reference_bin_categories = {
            k: f'{k}:'
            + str(
                self.dict_optbins_filtered[k][2].get(
                    min(self.dict_optbins_filtered[k][2].keys())
                )
            )
            for k in self.dict_optbins_filtered.keys()
        }

dict_parameters = {
    'var1': ["ind", "category"],
    'var2': ["median", "int"],
    'var3': ["mean", "float"],
    'var4': [0, "bool"]
}

    min_iv : float or None, optional (default=None)
        The minimum information value. Applicable if target type is binary.

    max_iv : float or None, optional (default=None)
        The maximum information value. Applicable if target type is binary.

    min_js : float or None, optional (default=None)
        The minimum Jensen-Shannon divergence value. Applicable if target type
        is binary or multiclass.

    max_js : float or None, optional (default=None)
        The maximum Jensen-Shannon divergence value. Applicable if target type
        is binary or multiclass.

    quality_score_cutoff : float or None, optional (default=None)
        The quality score cutoff value. Applicable if target type is binary or
        multiclass.

{"iv": {"min_value": 0.02, "max_value": 0.5, "strategy": "best", "top": 20}}

{"iv": {"min_value": 0.02, "max_value": 0.5, "strategy": "best", "top": 20},
"quality_score": {"min_value": 0.001}}

    scaling_method = "pdo_odds"
    scaling_method_data = {"pdo": 20, "odds": 50, "scorecard_points": 600}

scaling_method = "min_max"
scaling_method_data = {"min": 300, "max": 850}

import matplotlib.pyplot as plt
import pandas as pd
from sklearn.datasets import load_breast_cancer
from sklearn.linear_model import LogisticRegression
from optbinning import BinningProcess
from optbinning.scorecard.scorecard import Scorecard


data = load_breast_cancer()
variable_names = data.feature_names
target = "target"
df = pd.DataFrame(data.data, columns=variable_names)
df[target] = data.target

# Estimator
lr = LogisticRegression()

# Binning process
selection_criteria = {"iv": {"min": 0.025, "max": 1, "strategy": "highest", "top": 20}}
binning_process = BinningProcess(variable_names=variable_names, selection_criteria=selection_criteria)

# Scorecard
scaling_method = "min_max"
scaling_method_data = {"min": 300, "max": 850}
scorecard = Scorecard(target=target, binning_process=binning_process, estimator=lr,
                      scaling_method=scaling_method, scaling_method_data=scaling_method_data,
                      intercept_based=False, reverse_scorecard=False)
scorecard.fit(df)

# Check min_max points
sc = scorecard.table(style="detailed")
sc.groupby("Variable").agg({'Points' : [np.min, np.max]}).sum()

# Compute score
score = scorecard.score(df)

# Compute predicted probabilities
pred_proba = scorecard.predict_proba(df)

# Plots
score_good = score[df[target] == 0]
score_bad = score[df[target] == 1]

plt.hist(score_good, alpha=0.5, label="good")
plt.hist(score_bad, alpha=0.5, label="bad")
plt.legend()
plt.show()

plt.scatter(score, pred_proba[:, 1])

import matplotlib.pyplot as plt
import pandas as pd
from sklearn.datasets import load_boston
from sklearn.linear_model import LinearRegression, HuberRegressor, Ridge
from optbinning import BinningProcess
from optbinning.scorecard.scorecard import Scorecard

data = load_boston()
variable_names = data.feature_names
target = "target"
df = pd.DataFrame(data.data, columns=variable_names)
df[target] = data.target

# Estimator
lr = Ridge()

# Binnng process
binning_process = BinningProcess(variable_names=variable_names)

# Scorecard
scaling_method = "min_max"
scaling_method_data = {"min": 0, "max": 100}

scorecard = Scorecard(target=target, binning_process=binning_process, estimator=lr,
                      scaling_method=scaling_method, scaling_method_data=scaling_method_data,
                      intercept_based=False, reverse_scorecard=True)

scorecard.fit(df)

# Check min_max points
sc = scorecard.table(style="detailed")
sc.groupby("Variable").agg({'Points' : [np.min, np.max]}).sum()

# Compute score
score = scorecard.score(df)

# Compute predicted target
pred = scorecard.predict(df)

# Plots
plt.hist(score)
plt.plot()

plt.scatter(score, pred)