bootstrap.h

Go to the documentation of this file.

#ifndef __BOOTSTRAP_ANALYSIS_H__
#define __BOOTSTRAP_ANALYSIS_H__
#include <vector>
#include <string>
#include <map>
#include <random>
#include <MedTime/MedTime/MedTime.h>
#include <SerializableObject/SerializableObject/SerializableObject.h>
#include <Logger/Logger/Logger.h>
#include <MedStat/MedStat/MedStat.h>
 
using namespace std;
 
static MedTime med_time;
 
class Lazy_Iterator
{
public:
    Lazy_Iterator(const vector<int> *p_pids, const vector<float> *p_preds,
                  const vector<float> *p_y, const vector<float> *p_w, float p_sample_ratio, int p_sample_per_pid, int max_loops, int seed, const vector<int> *p_preds_order = NULL);
 
    void init(const vector<int> *p_pids, const vector<float> *p_preds,
              const vector<float> *p_y, const vector<float> *p_w, float p_sample_ratio, int p_sample_per_pid, int max_loops, int seed);
 
    inline bool fetch_next(int thread, float &ret_y, float &ret_pred, float &weight);
    inline bool fetch_next(int thread, float &ret_y, const float *&ret_pred, float &weight, const int *&preds_order);
    bool fetch_next_external(int thread, float &ret_y, float &ret_pred, float &weight);
    bool fetch_next_external(int thread, float &ret_y, float &ret_pred, float &weight, const int *&preds_order);
 
    void restart_iterator(int thread);
    void set_static(const vector<float> *p_y, const vector<float> *p_preds, const vector<float> *p_w, const vector<int> *p_preds_order, int thread_num);
 
    void set_thread_sample_all(int thread);
 
    ~Lazy_Iterator();
 
    // sampling params:
    float sample_ratio;          
    int sample_per_pid;          
    bool sample_all_no_sampling; 
    size_t num_categories;       
private:
    // internal structure - one time init
    static random_device rd;
    vector<mt19937> rd_gen;
    uniform_int_distribution<> rand_pids;
    vector<int> ind_to_pid;
    vector<vector<int>> pid_index_to_indexes; // for each pid_index retrieve the indexes in the original vectors
    vector<uniform_int_distribution<>> internal_random;
    int cohort_size;
    int min_pid_start;
    // init each time again
    // save for each Thread!
    vector<int> current_pos;
    vector<int> inner_pos;     // only used when sample_per_pid==0
    vector<int> sel_pid_index; // only used when sample_per_pid==0
    vector<int> vec_size;
    vector<const float *> vec_y;
    vector<const float *> vec_preds;
    vector<const float *> vec_weights;
    vector<const int *> vec_preds_order;
    // original vectors
    const float *preds;
    const float *y;
    const float *weights;
    const vector<int> *pids;
    const int *preds_order;
 
    // threading:
    int maxThreadCount;
};
 
class Mem_Iterator
{
public:
    // no calc
    Mem_Iterator() {}
 
    Mem_Iterator(const vector<int> &pids, const vector<int> &cohort_indexes, float p_sample_ratio, int p_sample_per_pid, int seed);
 
    inline void fetch_selection(vector<int> &indexes) const;
 
    void fetch_selection_external(vector<int> &indexes) const;
 
    inline void fetch_selection(mt19937 &rd_gen, vector<int> &indexes) const;
 
    void fetch_selection_external(mt19937 &rd_gen, vector<int> &indexes) const;
 
    // sampling params:
    float sample_ratio; 
    int sample_per_pid; 
private:
    // internal structure - one time init
    mt19937 _rd_gen;
 
    int cohort_size;
    int tot_rec_cnt;
    vector<vector<int>> pid_to_inds;
    vector<int> ind_to_pid;
    vector<int> cohort_idx;
};
 
class Measurement_Params : public SerializableObject
{
public:
    bool show_warns; 
    Measurement_Params();
    virtual ~Measurement_Params() {};
};
 
#pragma region Measurements Functions
map<string, float> calc_npos_nneg(Lazy_Iterator *iterator, int thread_num, Measurement_Params *function_params);
map<string, float> calc_only_auc(Lazy_Iterator *iterator, int thread_num, Measurement_Params *function_params);
map<string, float> calc_roc_measures_with_inc(Lazy_Iterator *iterator, int thread_num, Measurement_Params *function_params); // with PPV and PR
map<string, float> calc_kandel_tau(Lazy_Iterator *iterator, int thread_num, Measurement_Params *function_params);
map<string, float> calc_multi_class(Lazy_Iterator *iterator, int thread_num, Measurement_Params *function_params);
 
map<string, float> calc_harrell_c_statistic(Lazy_Iterator *iterator, int thread_num, Measurement_Params *function_params);
 
map<string, float> calc_regression(Lazy_Iterator *iterator, int thread_num, Measurement_Params *function_params);
 
// For example we can put here statistical measures for regression problem or more measurements for classification..
#pragma endregion
 
class Incident_Stats : public SerializableObject
{
public:
    // age bin config:
    int age_bin_years; 
    float min_age;     
    float max_age;     
    vector<float> sorted_outcome_labels;
    // male:
    vector<vector<double>> male_labels_count_per_age; 
    // female:
    vector<vector<double>> female_labels_count_per_age; 
 
    void read_from_text_file(const string &text_file);
    void write_to_text_file(const string &text_file);
 
    ADD_SERIALIZATION_FUNCS(age_bin_years, min_age, max_age, sorted_outcome_labels,
                            male_labels_count_per_age, female_labels_count_per_age)
};
 
class ROC_Params : public Measurement_Params
{
public:
    vector<float> working_point_FPR;        
    vector<int> working_point_TOPN;         
    vector<float> working_point_SENS;       
    vector<float> working_point_PR;         
    vector<float> working_point_auc;        
    vector<float> working_point_Score;      
    bool use_score_working_points;          
    float max_diff_working_point;           
    int score_bins;                         
    int score_min_samples;                  
    float score_resolution;                 
    bool fix_label_to_binary;               
    int min_score_quants_to_force_score_wp; 
    Incident_Stats inc_stats;               
 
    ROC_Params()
    {
        max_diff_working_point = (float)0.05;
        use_score_working_points = false;
        working_point_FPR = {(float)0.1, 1, 5, 10, 20, 30, 40, 50, 55, 60, 65, 70, 75, 80, 85, 90, 95};
        score_bins = 0;
        score_resolution = 0;
        incidence_fix = 0;
        score_min_samples = 0;
        fix_label_to_binary = true;
        show_warns = true;
        min_score_quants_to_force_score_wp = 10;
    }
    ROC_Params(const string &init_string);
    int init(map<string, string> &map);
 
    double incidence_fix; 
    ADD_SERIALIZATION_FUNCS(working_point_FPR, working_point_SENS, working_point_PR, working_point_TOPN, working_point_Score, working_point_auc, use_score_working_points,
                            max_diff_working_point, score_bins, score_resolution, score_min_samples, fix_label_to_binary, show_warns, inc_stats, min_score_quants_to_force_score_wp)
};
 
class Multiclass_Params : public Measurement_Params
{
public:
    vector<int> top_n;                 
    int n_categ;                       
    vector<float> dist_weights;        
    vector<vector<float>> dist_matrix; 
    string dist_name = "JACCARD";
    string dist_file;
    bool do_class_auc = false;
 
    Multiclass_Params()
    {
        top_n = {1, 5};
        n_categ = 1;
    }
    Multiclass_Params(const string &init_string);
 
    int init(map<string, string> &map);
    void read_dist_matrix_from_file(const string &fileName);
 
    ADD_CLASS_NAME(Multiclass_Params)
    ADD_SERIALIZATION_FUNCS(top_n, n_categ, dist_weights, dist_file, dist_matrix, dist_name, do_class_auc)
};
 
class Regression_Params : public Measurement_Params
{
public:
    bool do_logloss = false;
    double epsilon = 1e-5;
    vector<float> coverage_quantile_percentages;
 
    int init(map<string, string> &mapper);
 
    ADD_CLASS_NAME(Regression_Params)
    ADD_SERIALIZATION_FUNCS(do_logloss, coverage_quantile_percentages, epsilon)
};
 
#pragma region Cohort Functions
bool filter_range_param(const map<string, vector<float>> &record_info, int index, void *cohort_params); // on single param
bool filter_range_params(const map<string, vector<float>> &record_info, int index, void *cohort_params); // on vector of params
#pragma endregion
 
class Filter_Param : public SerializableObject
{ // for example Age and range for filter
public:
    string param_name; 
    float min_range;   
    float max_range;   
 
    Filter_Param(const string &init_string);
 
    int init_from_string(string init_string);
 
    int init(map<string, string> &map);
 
    Filter_Param() { param_name = ""; }
 
    ADD_SERIALIZATION_FUNCS(param_name, min_range, max_range)
};
 
struct ROC_And_Filter_Params : public Measurement_Params
{
    ROC_Params *roc_params;
    vector<Filter_Param> *filter;
};
 
// Infra
typedef map<string, float> (*MeasurementFunctions)(Lazy_Iterator *iterator, int thread_num, Measurement_Params *function_params);
typedef bool (*FilterCohortFunc)(const map<string, vector<float>> &record_info, int index, void *cohort_params);
typedef void (*ProcessMeasurementParamFunc)(const map<string, vector<float>> &additional_info, const vector<float> &y, const vector<int> &pids, Measurement_Params *function_params,
                                            const vector<int> &filtered_indexes, const vector<float> &y_full, const vector<int> &pids_full);
typedef void (*PreprocessScoresFunc)(vector<float> &preds, Measurement_Params *function_params);
 
#pragma region Process Measurement Param Functions
void fix_cohort_sample_incidence(const map<string, vector<float>> &additional_info,
                                 const vector<float> &y, const vector<int> &pids, Measurement_Params *function_params,
                                 const vector<int> &filtered_indexes, const vector<float> &y_full, const vector<int> &pids_full);
 
void fix_cohort_sample_incidence_old(const map<string, vector<float>> &additional_info,
                                     const vector<float> &y, const vector<int> &pids, Measurement_Params *function_params,
                                     const vector<int> &filtered_indexes, const vector<float> &y_full, const vector<int> &pids_full);
#pragma endregion
 
#pragma region Process Scores Functions
void preprocess_bin_scores(vector<float> &preds, Measurement_Params *function_params);
#pragma endregion
 
inline string format_working_point(const string &init_str, float wp, bool perc = true)
{
    char res[100];
    if (perc)
        wp *= 100;
    snprintf(res, sizeof(res), "%s_%06.3f", init_str.c_str(), wp);
    return string(res);
}
 
inline string format_working_point_topn(const string &init_str, int wp, bool perc = true)
{
    char res[100];
    snprintf(res, sizeof(res), "%s_%d", init_str.c_str(), wp);
    return string(res);
}
 
map<string, float> booststrap_analyze_cohort(const vector<float> &preds, const vector<int> &preds_order, const vector<float> &y,
                                             const vector<int> &pids, float sample_ratio, int sample_per_pid, int loopCnt,
                                             const vector<MeasurementFunctions> &meas_functions, const vector<Measurement_Params *> &function_params,
                                             ProcessMeasurementParamFunc process_measurments_params,
                                             const map<string, vector<float>> &additional_info, const vector<float> &y_full,
                                             const vector<int> &pids_full, const vector<float> *weights, const vector<int> &filter_indexes, FilterCohortFunc cohort_def,
                                             void *cohort_params, int &warn_cnt, const string &cohort_name, int seed = 0);
 
 
map<string, map<string, float>> booststrap_analyze(
    const vector<float> &preds,
    const vector<int> &preds_order,
    const vector<float> &y,
    const vector<float> *weights,
    const vector<int> &pids,
    const map<string, vector<float>> &additional_info,
    const map<string, FilterCohortFunc> &filter_cohort,
    const vector<MeasurementFunctions> &meas_functions = {calc_roc_measures_with_inc},
    const map<string, void *> *cohort_params = NULL,
    const vector<Measurement_Params *> *function_params = NULL,
    ProcessMeasurementParamFunc process_measurments_params = NULL,
    PreprocessScoresFunc preprocess_scores = NULL,
    Measurement_Params *preprocess_scores_params = NULL,
    float sample_ratio = (float)1.0,
    int sample_per_pid = 1,
    int loopCnt = 500,
    int seed = 0,
    bool binary_outcome = true);
 
void prepare_for_bootstrap(const vector<int> &pids,
                           const map<string, vector<float>> &additional_info, FilterCohortFunc &filter_cohort, void *cohort_params, float sample_ratio, int sample_per_pid, int seed, vector<int> &indexes);
 
void write_bootstrap_results(const string &file_name, const map<string, map<string, float>> &all_cohorts_measurments, const string &run_id = "");
void read_bootstrap_results(const string &file_name, map<string, map<string, float>> &all_cohorts_measurments);
 
void write_pivot_bootstrap_results(const string &file_name, const map<string, map<string, float>> &all_cohorts_measurments, const string &run_id = "");
void read_pivot_bootstrap_results(const string &file_name, map<string, map<string, float>> &all_cohorts_measurments);
 
MEDSERIALIZE_SUPPORT(Incident_Stats)
MEDSERIALIZE_SUPPORT(ROC_Params)
MEDSERIALIZE_SUPPORT(Filter_Param)
MEDSERIALIZE_SUPPORT(Regression_Params)
MEDSERIALIZE_SUPPORT(Multiclass_Params)
 
#endif // !__BOOTSTRAP_ANALYSIS_H__