ExplainWrapper.h

Go to the documentation of this file.

 
#ifndef __EXPLAIN_WRAPPER_H__
#define __EXPLAIN_WRAPPER_H__
 
#include <vector>
#include <string>
#include <MedAlgo/MedAlgo/MedAlgo.h>
#include <MedProcessTools/MedProcessTools/PostProcessor.h>
#include <MedStat/MedStat/GibbsSampler.h>
#include <MedAlgo/MedAlgo/tree_shap.h>
#include <MedAlgo/MedAlgo/SamplesGenerator.h>
 
using namespace std;
 
class ExplainFilters : public SerializableObject {
public:
    int sort_mode; 
    int max_count; 
    float sum_ratio; 
 
    ExplainFilters();
 
    int init(map<string, string> &map);
 
    void filter(map<string, float> &explain_list) const;
 
    ADD_CLASS_NAME(ExplainFilters)
        ADD_SERIALIZATION_FUNCS(sort_mode, max_count, sum_ratio)
 
};
 
class ExplainProcessings : public SerializableObject {
private:
    bool postprocessing_cov = false; 
public:
    bool group_by_sum = false; 
    bool learn_cov_matrix = false; 
    int normalize_vals = 0; 
    int zero_missing = 0; 
    bool keep_b0 = false; 
    bool iterative = false; 
    int iteration_cnt = 0; 
    bool use_max_cov = false; 
 
    bool use_mutual_information; 
    BinSettings mutual_inf_bin_setting; 
 
    MedMat<float> abs_cov_features; 
 
    string grouping; 
    vector<vector<int>> group2Inds;
    vector<string> groupNames;
    map<string, vector<int>> groupName2Inds;
 
    ExplainProcessings();
 
    int init(map<string, string> &map);
 
    void learn(const MedFeatures &train_mat);
 
    void process(map<string, float> &explain_list) const;
    // same as process but zero-ing all contributions of missing values features and groups with all the participants inside missing
    void process(map<string, float> &explain_list, unsigned char *missing_value_mask) const;
 
    float get_group_normalized_contrib(const vector<int> &group_inds, vector<float> &contribs, float total_normalization_factor) const;
 
    void post_deserialization();
 
    static void read_feature_grouping(const string &file_name, const vector<string>& features, vector<vector<int>>& group2index,
        vector<string>& group_names, bool verbose = true);
 
    ADD_CLASS_NAME(ExplainProcessings)
        ADD_SERIALIZATION_FUNCS(group_by_sum, abs_cov_features, normalize_vals, zero_missing, groupNames, group2Inds, keep_b0,
            iterative, iteration_cnt, postprocessing_cov, use_mutual_information, mutual_inf_bin_setting, use_max_cov)
};
 
class GlobalExplainerParams : public SerializableObject {
public:
    string attr_name = ""; 
    bool store_as_json = false; 
    bool denorm_features = true; 
 
    //No init - will be initialized directly in ModelExplainer::init
 
    ADD_CLASS_NAME(GlobalExplainerParams)
        ADD_SERIALIZATION_FUNCS(attr_name, store_as_json, denorm_features)
};
 
class ModelExplainer : public PostProcessor {
private:
    virtual void _init(map<string, string> &mapper) = 0;
    unordered_map<string, const FeatureNormalizer *> feats_to_norm;
public:
    MedPredictor * original_predictor = NULL; 
    ExplainFilters filters; 
    ExplainProcessings processing; 
    GlobalExplainerParams global_explain_params;
    
    virtual int init(map<string, string> &mapper);
 
    virtual int update(map<string, string>& mapper);
    virtual void _learn(const MedFeatures &train_mat) = 0;
 
    virtual void Learn(const MedFeatures &train_mat);
    void Apply(MedFeatures &matrix) { explain(matrix); } 
 
    void get_input_fields(vector<Effected_Field> &fields) const;
    void get_output_fields(vector<Effected_Field> &fields) const;
 
    void init_post_processor(MedModel& model);
 
    virtual void explain(const MedFeatures &matrix, vector<map<string, float>> &sample_explain_reasons) const = 0;
 
    virtual void explain(MedFeatures &matrix) const; //stores _explain results in MedFeatures
 
    static void print_explain(MedSample &smp, int sort_mode = 0);
 
    void dprint(const string &pref) const;
 
    virtual ~ModelExplainer() {};
};
 
enum TreeExplainerMode {
    ORIGINAL_IMPL = 0,
    CONVERTED_TREES_IMPL = 1,
    PROXY_IMPL = 2
};
 
class TreeExplainer : public ModelExplainer {
private:
    MedPredictor * proxy_predictor = NULL; //uses this if model has no tree implementation
    //Tree structure of generic ensamble trees
private:
    bool convert_qrf_trees();
    bool convert_lightgbm_trees();
    bool convert_xgb_trees();
    void _init(map<string, string> &mapper);
public:
    bool try_convert_trees();
    TreeEnsemble generic_tree_model;
    string proxy_model_type = ""; 
    string proxy_model_init = ""; 
    bool interaction_shap = false; 
    int approximate = false; 
    float missing_value = MED_MAT_MISSING_VALUE; 
    bool verbose = false;
 
    TreeExplainer() { processor_type = FTR_POSTPROCESS_TREE_SHAP; }
 
    void init_post_processor(MedModel& model);
 
    TreeExplainerMode get_mode() const;
 
    void _learn(const MedFeatures &train_mat);
 
    void explain(const MedFeatures &matrix, vector<map<string, float>> &sample_explain_reasons) const;
 
    void post_deserialization();
 
    ~TreeExplainer();
 
    ADD_CLASS_NAME(TreeExplainer)
        ADD_SERIALIZATION_FUNCS(proxy_predictor, interaction_shap, filters, processing, global_explain_params, verbose)
};
 
class MissingShapExplainer : public ModelExplainer {
private:
    MedPredictor * retrain_predictor = NULL; //the retrain model
 
    void _init(map<string, string> &mapper);
 
    float avg_bias_score;
public:
    int add_new_data; 
    bool no_relearn; 
 
    int max_test; 
    float missing_value; 
    bool sample_masks_with_repeats; 
    float select_from_all; 
    bool uniform_rand; 
    bool use_shuffle; 
    string predictor_args; 
    string predictor_type;
    bool verbose_learn; 
    string verbose_apply; 
    float max_weight; 
    int subsample_train; 
    int limit_mask_size; 
 
    // parameters for minimal_set usage if use_minimal_set is true will do different thing
    bool use_minimal_set; 
    float sort_params_a; 
    float sort_params_b; 
    float sort_params_k1; 
    float sort_params_k2; 
    int max_set_size; 
    float override_score_bias; 
    float split_to_test; 
 
 
    MissingShapExplainer();
 
    void _learn(const MedFeatures &train_mat);
 
    void explain(const MedFeatures &matrix, vector<map<string, float>> &sample_explain_reasons) const;
 
    ~MissingShapExplainer();
 
    ADD_CLASS_NAME(MissingShapExplainer)
        ADD_SERIALIZATION_FUNCS(retrain_predictor, max_test, missing_value, sample_masks_with_repeats,
            select_from_all, uniform_rand, use_shuffle, no_relearn, avg_bias_score, filters, processing, global_explain_params,
            predictor_type, predictor_args, max_weight, use_minimal_set, sort_params_a, sort_params_b,
            sort_params_k1, sort_params_k2, max_set_size, override_score_bias, verbose_apply, subsample_train,
            limit_mask_size, split_to_test)
};
 
class ShapleyExplainer : public ModelExplainer {
private:
    unique_ptr<SamplesGenerator<float>> _sampler = NULL;
    void *sampler_sampling_args = NULL;
 
    GibbsSampler<float> _gibbs;
    GibbsSamplingParams _gibbs_sample_params;
 
    float avg_bias_score;
 
    void init_sampler(bool with_sampler = true);
 
    void _init(map<string, string> &mapper);
public:
    GeneratorType gen_type = GeneratorType::GIBBS; 
    string generator_args = ""; 
    string sampling_args = ""; 
    int n_masks = 100; 
    bool use_random_sampling = true; 
    float missing_value = MED_MAT_MISSING_VALUE; 
 
    ShapleyExplainer() { processor_type = FTR_POSTPROCESS_SHAPLEY; avg_bias_score = 0; }
 
    void _learn(const MedFeatures &train_mat);
 
    void explain(const MedFeatures &matrix, vector<map<string, float>> &sample_explain_reasons) const;
 
    void post_deserialization();
 
    void load_GIBBS(MedPredictor *original_pred, const GibbsSampler<float> &gibbs, const GibbsSamplingParams &sampling_args);
    void load_GAN(MedPredictor *original_pred, const string &gan_path);
    void load_MISSING(MedPredictor *original_pred);
    void load_sampler(MedPredictor *original_pred, unique_ptr<SamplesGenerator<float>> &&generator);
 
    void dprint(const string &pref) const;
 
    ADD_CLASS_NAME(ShapleyExplainer)
        ADD_SERIALIZATION_FUNCS(_sampler, gen_type, generator_args, n_masks, missing_value, sampling_args,
            use_random_sampling, avg_bias_score, filters, processing, global_explain_params)
};
 
class LimeExplainer : public ModelExplainer {
private:
    unique_ptr<SamplesGenerator<float>> _sampler = NULL;
    void *sampler_sampling_args = NULL;
 
    //just for gibbs memory hold when init & learn
    GibbsSampler<float> _gibbs;
    GibbsSamplingParams _gibbs_sample_params;
 
    void init_sampler(bool with_sampler = true);
    void _init(map<string, string> &mapper);
    medial::shapley::LimeWeightMethod get_weight_method(string method_s);
public:
    GeneratorType gen_type = GeneratorType::GIBBS; 
    string generator_args = ""; 
    string sampling_args = ""; 
    float missing_value = MED_MAT_MISSING_VALUE; 
    float p_mask = 0; 
    medial::shapley::LimeWeightMethod weighting = medial::shapley::LimeWeightSum;
    int n_masks = 1250; 
 
    LimeExplainer() { processor_type = FTR_POSTPROCESS_LIME_SHAP; }
 
    void _learn(const MedFeatures &train_mat);
 
    void explain(const MedFeatures &matrix, vector<map<string, float>> &sample_explain_reasons) const;
 
    void load_GIBBS(MedPredictor *original_pred, const GibbsSampler<float> &gibbs, const GibbsSamplingParams &sampling_args);
    void load_GAN(MedPredictor *original_pred, const string &gan_path);
    void load_MISSING(MedPredictor *original_pred);
    void load_sampler(MedPredictor *original_pred, unique_ptr<SamplesGenerator<float>> &&generator);
 
    void post_deserialization();
 
    void dprint(const string &pref) const;
 
    ADD_CLASS_NAME(LimeExplainer)
        ADD_SERIALIZATION_FUNCS(_sampler, gen_type, generator_args, missing_value, sampling_args, p_mask, n_masks, weighting,
            filters, processing, global_explain_params)
};
 
class KNN_Explainer : public ModelExplainer {
private:
    MedFeatures trainingMap;
    vector<float> average, std;
 
    // do the calculation for a single sample after normalization
    void computeExplanation(vector<float> thisRow, map<string, float> &sample_explain_reasons, vector <vector<int>> knnGroups, vector<string> knnGroupNames)const;
 
    void _init(map<string, string> &mapper);
public:
 
    int numClusters = -1; 
    float fraction = (float)0.02; 
    float chosenThreshold = MED_MAT_MISSING_VALUE; 
    float thresholdQ = MED_MAT_MISSING_VALUE;
 
    KNN_Explainer() { processor_type = FTR_POSTPROCESS_KNN_EXPLAIN; }
 
    void _learn(const MedFeatures &train_mat);
 
    void explain(const MedFeatures &matrix, vector<map<string, float>> &sample_explain_reasons) const;
 
    ADD_CLASS_NAME(KNN_Explainer)
        ADD_SERIALIZATION_FUNCS(numClusters, trainingMap, average, std, fraction, chosenThreshold, filters, processing, global_explain_params)
};
 
class LinearExplainer : public ModelExplainer {
private:
    void _init(map<string, string> &mapper);
 
    float avg_bias_score;
public:
    LinearExplainer() { processor_type = FTR_POSTPROCESS_LINEAR; avg_bias_score = 0; }
 
    void _learn(const MedFeatures &train_mat);
 
    void explain(const MedFeatures &matrix, vector<map<string, float>> &sample_explain_reasons) const;
 
    ADD_CLASS_NAME(LinearExplainer)
        ADD_SERIALIZATION_FUNCS(avg_bias_score, filters, processing, global_explain_params)
};
 
class IterativeSetExplainer : public ModelExplainer {
private:
    unique_ptr<SamplesGenerator<float>> _sampler = NULL;
    void *sampler_sampling_args = NULL;
 
    GibbsSampler<float> _gibbs;
    GibbsSamplingParams _gibbs_sample_params;
 
    float avg_bias_score;
 
    void init_sampler(bool with_sampler = true);
 
    void _init(map<string, string> &mapper);
public:
    GeneratorType gen_type = GeneratorType::GIBBS; 
    string generator_args = ""; 
    string sampling_args = ""; 
    int n_masks = 100; 
    bool use_random_sampling = true; 
    float missing_value = MED_MAT_MISSING_VALUE; 
 
    float sort_params_a; 
    float sort_params_b; 
    float sort_params_k1; 
    float sort_params_k2; 
    int max_set_size; 
 
    IterativeSetExplainer() { processor_type = FTR_POSTPROCESS_ITERATIVE_SET; avg_bias_score = 0; }
 
    void _learn(const MedFeatures &train_mat);
 
    void explain(const MedFeatures &matrix, vector<map<string, float>> &sample_explain_reasons) const;
 
    void post_deserialization();
 
    void load_GIBBS(MedPredictor *original_pred, const GibbsSampler<float> &gibbs, const GibbsSamplingParams &sampling_args);
    void load_GAN(MedPredictor *original_pred, const string &gan_path);
    void load_MISSING(MedPredictor *original_pred);
    void load_sampler(MedPredictor *original_pred, unique_ptr<SamplesGenerator<float>> &&generator);
 
    void dprint(const string &pref) const;
 
    ADD_CLASS_NAME(IterativeSetExplainer)
        ADD_SERIALIZATION_FUNCS(_sampler, gen_type, generator_args, n_masks, missing_value, sampling_args,
            use_random_sampling, avg_bias_score, filters, processing, global_explain_params, max_set_size,
            sort_params_a, sort_params_b, sort_params_k1, sort_params_k2)
};
 
MEDSERIALIZE_SUPPORT(ExplainFilters)
MEDSERIALIZE_SUPPORT(ExplainProcessings)
MEDSERIALIZE_SUPPORT(GlobalExplainerParams)
MEDSERIALIZE_SUPPORT(TreeExplainer)
MEDSERIALIZE_SUPPORT(MissingShapExplainer)
MEDSERIALIZE_SUPPORT(ShapleyExplainer)
MEDSERIALIZE_SUPPORT(LimeExplainer)
MEDSERIALIZE_SUPPORT(LinearExplainer)
MEDSERIALIZE_SUPPORT(KNN_Explainer)
MEDSERIALIZE_SUPPORT(IterativeSetExplainer)
 
 
 
#endif