FeatureProcess.h

Go to the documentation of this file.

#ifndef _FTR_PROCESS_H_
#define _FTR_PROCESS_H_
 
#include <SerializableObject/SerializableObject/SerializableObject.h>
#include <MedProcessTools/MedProcessTools/MedFeatures.h>
#include <MedProcessTools/MedProcessTools/MedProcessUtils.h>
#include <MedProcessTools/MedProcessTools/MedValueCleaner.h>
#include <MedProcessTools/MedProcessTools/MedProcessUtils.h>
#include <MedUtils/MedUtils/MedUtils.h>
#include <MedAlgo/MedAlgo/BinSplitOptimizer.h>
#include <unordered_set>
 
#define DEFAULT_FEAT_CLNR_NTHREADS 24
 
// Forward definition of MedBootstrap
class MedBootstrapResult;
 
typedef enum {
    FTR_PROCESS_MULTI, 
    FTR_PROCESS_BASIC_OUTLIER_CLEANER, 
    FTR_PROCESS_NORMALIZER, 
    FTR_PROCESS_IMPUTER, 
    FTR_PROCESS_DO_CALC, 
    FTR_PROCESS_UNIVARIATE_SELECTOR, 
    FTR_PROCESSOR_MRMR_SELECTOR, 
    FTR_PROCESSOR_LASSO_SELECTOR, 
    FTR_PROCESSOR_TAGS_SELECTOR, 
    FTR_PROCESSOR_IMPORTANCE_SELECTOR, 
    FTR_PROCESSOR_ITERATIVE_SELECTOR, 
    FTR_PROCESS_REMOVE_DGNRT_FTRS, 
    FTR_PROCESS_ITERATIVE_IMPUTER, 
    FTR_PROCESS_ENCODER_PCA, 
    FTR_PROCESS_ONE_HOT, 
    FTR_PROCESS_GET_PROB, 
    FTR_PROCESS_PREDICTOR_IMPUTER, 
    FTR_PROCESS_MULTIPLIER, 
    FTR_PROCESS_RESAMPLE_WITH_MISSING, 
    FTR_PROCESS_DUPLICATE, 
    FTR_PROCESS_MISSING_INDICATOR, 
    FTR_PROCESS_BINNING, 
    FTR_PROCESS_LAST
} FeatureProcessorTypes;
 
class FeatureProcessor : public SerializableObject {
public:
 
    string feature_name = "unset_feature_name";
    string resolved_feature_name;
 
    virtual string select_learn_matrix(const vector<string> &matrix_tags) const { return ""; };
 
    // Type
    FeatureProcessorTypes processor_type = FTR_PROCESS_LAST;
 
    // Threading
    int learn_nthreads, clean_nthreads;
 
    // Constructor/Destructor
    FeatureProcessor() { init_defaults(); };
    virtual ~FeatureProcessor() { clear(); };
    virtual void clear() {};
    void init_defaults() { learn_nthreads = DEFAULT_FEAT_CLNR_NTHREADS;  clean_nthreads = DEFAULT_FEAT_CLNR_NTHREADS; processor_type = FTR_PROCESS_LAST; };
 
    // Copy
    virtual void copy(FeatureProcessor *processor) { *this = *processor; }
 
    // Virtual Set Feature Name
    virtual void set_feature_name(const string& feature_name) { this->feature_name = feature_name; }
    virtual string get_feature_name() { return this->feature_name; }
    virtual void get_feature_names(vector<string> & feature_names) { feature_names.clear(); feature_names.push_back(feature_name); };
 
    // Learn cleaning model
    virtual int Learn(MedFeatures& features, unordered_set<int>& ids) { return 0; }
 
    int learn(MedFeatures& features);
    int learn(MedFeatures& features, unordered_set<int>& ids) { return Learn(features, ids); }
 
    // Apply feature processing
    virtual int _apply(MedFeatures& features, unordered_set<int>& ids, bool learning) {
        // For most processors - application is the same for model - learning/applying
        return _apply(features, ids);
    }
 
    virtual int _apply(MedFeatures& features, unordered_set<int>& ids);
    virtual int _conditional_apply(MedFeatures& features, unordered_set<int>& ids, unordered_set<string>& req_features, bool learning);
 
    int apply(MedFeatures& features, bool learning);
    int apply(MedFeatures& features, unordered_set<string>& req_features, bool learning);
    int apply(MedFeatures& features, unordered_set<int>& ids, bool learning);
    int apply(MedFeatures& features, unordered_set<int>& ids, unordered_set<string>& req_features, bool learning);
 
    int apply(MedFeatures& features) { return apply(features, true); };
    int apply(MedFeatures& features, unordered_set<string>& req_features) { return apply(features, req_features, true); };
    int apply(MedFeatures& features, unordered_set<int>& ids) { return apply(features, ids, true); };
    int apply(MedFeatures& features, unordered_set<int>& ids, unordered_set<string>& req_features) { return apply(features, ids, req_features, true); };
 
    // Init
    static FeatureProcessor *make_processor(string processor_name);
    static FeatureProcessor *make_processor(FeatureProcessorTypes type);
    static FeatureProcessor *make_processor(string processor_name, string params);
    static FeatureProcessor *make_processor(FeatureProcessorTypes type, string params);
 
    virtual int init(void *processor_params) { return 0; };
    virtual int init(map<string, string>& mapper) { return 0; };
 
    virtual int filter(unordered_set<string>& features) { return (features.find(feature_name) == features.end()) ? 0 : 1; };
 
    string resolve_feature_name(MedFeatures& features, string substr);
 
    virtual bool are_features_affected(unordered_set<string>& out_req_features) { return (out_req_features.empty() || out_req_features.find(resolved_feature_name) != out_req_features.end()); }
 
    virtual void update_req_features_vec(unordered_set<string>& out_req_features, unordered_set<string>& in_req_features) { in_req_features = out_req_features; };
 
    virtual bool is_selector() { return false; }
 
    // Serialization (including type)
    ADD_CLASS_NAME(FeatureProcessor)
        ADD_SERIALIZATION_FUNCS(feature_name, resolved_feature_name, processor_type)
        void *new_polymorphic(string derived_class_name);
 
    size_t get_processor_size();
    size_t processor_serialize(unsigned char *blob);
 
    // debug prints
    virtual void dprint(const string &pref, int rp_flag);
 
};
 
// Utilities
FeatureProcessorTypes feature_processor_name_to_type(const string& cleaner_name);
 
class MultiFeatureProcessor : public FeatureProcessor {
public:
 
    // For generating processors only at learning, we need type + init_string
    FeatureProcessorTypes members_type;
    string init_string;
    int duplicate;
    string tag;
    bool use_parallel_learn;
    bool use_parallel_apply;
 
    // Processors (if empty, will be generated upon learning for all featuers)
    vector<FeatureProcessor *> processors;
 
    string select_learn_matrix(const vector<string> &matrix_tags) const;
 
    // Constructor/Destructor
    MultiFeatureProcessor() { init_defaults(); };
    ~MultiFeatureProcessor() { clear(); };
 
    void init_defaults() { processor_type = FTR_PROCESS_MULTI; duplicate = 0; members_type = FTR_PROCESS_LAST; init_string = ""; tag = ""; use_parallel_learn = true; use_parallel_apply = true; };
 
    void clear();
 
    int init(map<string, string>& mapper);
 
    // Copy
    virtual void copy(FeatureProcessor *processor);
 
    // Learn cleaning model
    int Learn(MedFeatures& features, unordered_set<int>& ids);
 
    // Apply cleaning model
    int _apply(MedFeatures& features, unordered_set<int>& ids, bool learning);
    int _conditional_apply(MedFeatures& features, unordered_set<int>& ids, unordered_set<string>& req_features, bool learning);
 
    virtual void get_feature_names(vector<string>& feature_names);
 
    // Add processors
    void add_processors_set(FeatureProcessorTypes type, vector<string>& features);
    void add_processors_set(FeatureProcessorTypes type, vector<string>& features, string init_string);
 
    // Filter according to a subset of features
    int filter(unordered_set<string>& features);
 
    // debug print
    void dprint(const string &pref, int fp_flag);
 
    bool are_features_affected(unordered_set<string>& out_req_features);
 
    void update_req_features_vec(unordered_set<string>& out_req_features, unordered_set<string>& in_req_features);
 
    // Serialization
    ADD_CLASS_NAME(MultiFeatureProcessor)
        ADD_SERIALIZATION_FUNCS(processor_type, members_type, init_string, duplicate, tag, processors, use_parallel_apply)
};
 
#define DEF_FTR_TRIMMING_SD_NUM 7
#define DEF_FTR_REMOVING_SD_NUM 14
class FeatureBasicOutlierCleaner : public FeatureProcessor, public MedValueCleaner {
public:
 
    // Constructor
    FeatureBasicOutlierCleaner() : FeatureProcessor() { init_defaults(); }
    FeatureBasicOutlierCleaner(string& feature_name) : FeatureProcessor() { set_feature_name(feature_name); init_defaults(); }
    FeatureBasicOutlierCleaner(string& feature_name, string init_string) : FeatureProcessor() { set_feature_name(feature_name);  init_defaults();  init_from_string(init_string); }
    FeatureBasicOutlierCleaner(string& feature_name, ValueCleanerParams *_params) : FeatureProcessor() { init_defaults(); set_feature_name(feature_name);  MedValueCleaner::init(_params); }
 
    void init_defaults() {
        processor_type = FTR_PROCESS_BASIC_OUTLIER_CLEANER;
        params.missing_value = MED_MAT_MISSING_VALUE;
        params.trimming_sd_num = DEF_FTR_TRIMMING_SD_NUM;
        params.removing_sd_num = DEF_FTR_REMOVING_SD_NUM;
        params.nbrs_sd_num = 0;
        params.take_log = 0;
        params.doTrim = params.doRemove = true;
        params.type = VAL_CLNR_ITERATIVE;
 
    };
 
    // Init
    int init(void *processor_params) { return MedValueCleaner::init(processor_params); };
    int init(map<string, string>& mapper);
 
    // Copy
    virtual void copy(FeatureProcessor *processor) { *this = *(dynamic_cast<FeatureBasicOutlierCleaner *>(processor)); }
 
    // Learn cleaning model
    int Learn(MedFeatures& features, unordered_set<int>& ids);
    int iterativeLearn(MedFeatures& features, unordered_set<int>& ids);
    int quantileLearn(MedFeatures& features, unordered_set<int>& ids);
 
    // Apply cleaning model
    int _apply(MedFeatures& features, unordered_set<int>& ids);
 
    // Serialization
    ADD_CLASS_NAME(FeatureBasicOutlierCleaner)
        ADD_SERIALIZATION_FUNCS(processor_type, feature_name, resolved_feature_name, params.doTrim, params.doRemove, trimMax, trimMin, removeMax, removeMin)
 
};
 
class FeatureNormalizer : public FeatureProcessor {
public:
 
    float missing_value;
 
    bool normalizeSd;
 
    bool fillMissing;
 
    float mean, sd;
 
    int resolution = 0;
 
    float resolution_bin;
 
    int max_samples = 0;
 
    bool resolution_only = false;
 
    int verbosity = 0;
 
    bool use_linear_transform = false;
    float max_val_prctile = 1;
    float max_val_for_triming = 2;
    float prctile_th = (float)0.001;
    float min_x, max_x; 
 
    // Constructor
    FeatureNormalizer() : FeatureProcessor() { init_defaults(); }
    FeatureNormalizer(const  string& feature_name) : FeatureProcessor() { init_defaults(); set_feature_name(feature_name); }
    FeatureNormalizer(const  string& feature_name, string init_string) : FeatureProcessor() { init_defaults(); init_from_string(init_string);  set_feature_name(feature_name); }
 
    string select_learn_matrix(const vector<string> &matrix_tags) const;
 
    // Learn cleaning model
    int Learn(MedFeatures& features, unordered_set<int>& ids);
 
    // Apply cleaning model
    int _apply(MedFeatures& features, unordered_set<int>& ids);
 
    void reverse_apply(float &feature_value) const;
 
    int init(map<string, string>& mapper);
    void init_defaults() { missing_value = MED_MAT_MISSING_VALUE; normalizeSd = true; fillMissing = false; processor_type = FTR_PROCESS_NORMALIZER; resolution_bin = 0; };
 
    // Copy
    virtual void copy(FeatureProcessor *processor) { *this = *(dynamic_cast<FeatureNormalizer *>(processor)); }
 
    // Serialization
    ADD_CLASS_NAME(FeatureNormalizer)
        ADD_SERIALIZATION_FUNCS(processor_type, feature_name, resolved_feature_name, mean, sd, resolution, normalizeSd, fillMissing, resolution_only, verbosity, resolution_bin,
            use_linear_transform, max_val_prctile, max_val_for_triming, prctile_th, min_x, max_x)
 
};
 
//.......................................................................................
//.......................................................................................
// Feature Imputer
//.......................................................................................
//.......................................................................................
 
typedef enum {
    IMPUTE_MMNT_MEAN,
    IMPUTE_MMNT_MEDIAN,
    IMPUTE_MMNT_COMMON,
    IMPUTE_MMNT_SAMPLE,
    IMPUTE_MMNT_LAST
} imputeMomentTypes;
 
class featureStrata :SerializableObject {
public:
    string name;
    float resolution, min, max;
    int nValues;
 
    featureStrata() {};
    featureStrata(string& _name, float _resolution, float _min, float _max) { name = _name; resolution = _resolution; min = _min; max = _max; }
 
    void SetNValues() { nValues = ((int)(max / resolution) - (int)(min / resolution) + 1); }
    // returns the correct strata for a value. 
    // E.g. if "strata": "Age,40,80,5" 42 will return 0, the first bin
    int getIndex(float value, float missing_val) const {
        if (value == missing_val)
            return nValues / 2;
        else {
            if (value >= max)
                return nValues - 1;
            else if (value <= min)
                return 0;
            else
                return ((int)(value / resolution) - (int)(min / resolution));
        }
    }
 
    // Serialization
    ADD_CLASS_NAME(featureStrata)
        ADD_SERIALIZATION_FUNCS(name, resolution, min, max, nValues)
 
};
class featureSetStrata :SerializableObject {
public:
    vector<featureStrata> stratas;
    vector<int> factors;
 
    size_t nStratas() const { return stratas.size(); }
 
    void getFactors() {
 
        if (stratas.size() == 0)
            return;
 
        factors.resize(stratas.size());
 
        for (auto& strata : stratas)
            strata.SetNValues();
 
        factors[0] = 1;
        for (int i = 1; i < stratas.size(); i++)
            factors[i] = factors[i - 1] * stratas[i - 1].nValues;
    }
 
    int nValues() const {
        if (stratas.size() == 0)
            return 1;
        else
            return factors.back() * stratas.back().nValues;
    }
 
    int getIndex(float missing_val,
        const vector<const vector<float> *> &strataValues, int row) const {
        int index = 0;
        for (int i = 0; i < nStratas(); i++)
            index += factors[i] * stratas[i].getIndex(strataValues[i]->at(row), missing_val);
        return index;
    }
 
    // Serialization
    ADD_CLASS_NAME(featureSetStrata)
        ADD_SERIALIZATION_FUNCS(stratas, factors)
};
 
class FeatureImputer : public FeatureProcessor {
private:
    float round_to_closest(float val) const;
public:
 
    // Missing Value
    float missing_value = MED_MAT_MISSING_VALUE;
    bool verbose = true; 
    bool verbose_learn = false; 
 
    // Strata for setting moment
    featureSetStrata imputerStrata;
 
    // minimum samples required for learning
    int min_samples = 50;
 
    // if true, doesn't impute missing values that are left due to small stratas
    int leave_missing_for_small_stratas = 0;
 
    // if false (default), do NOT impute ANY case in which the strata contains a missing value
    int impute_strata_with_missing = 0;
 
    // Moment (learning/applying)
    vector<imputeMomentTypes> moment_type_vec;
    vector<float> default_moment_vec;
    vector<vector<float>> moments_vec;
 
    // For backword compatability ...
    imputeMomentTypes moment_type = IMPUTE_MMNT_MEAN;
    float default_moment;
    vector<float> moments;
 
    // for sampling-imputation
    vector < pair<float, float> > default_histogram;
    vector < vector<pair<float, float> > > histograms;
 
    vector<int> strata_sizes;
 
    int max_samples = 100000;
 
    bool round_to_existing_value = true;
    vector<float> existing_values;
 
    // Constructor
    FeatureImputer() : FeatureProcessor() { init_defaults(); }
    FeatureImputer(const  string& feature_name) : FeatureProcessor() { init_defaults(); set_feature_name(feature_name); }
    FeatureImputer(const  string& feature_name, string init_string) : FeatureProcessor() { init_defaults(); init_from_string(init_string);  set_feature_name(feature_name); }
 
    // Add stratifier
    void addStrata(string& init_string);
    void addStrata(featureStrata& strata) { imputerStrata.stratas.push_back(strata); }
    void addStrata(string& name, float resolution, float min, float max) { imputerStrata.stratas.push_back(featureStrata(name, resolution, min, max)); }
 
    int init(map<string, string>& mapper);
    void init_defaults() { missing_value = MED_MAT_MISSING_VALUE; moment_type = IMPUTE_MMNT_MEAN;  processor_type = FTR_PROCESS_IMPUTER; verbose = true; verbose_learn = false; };
    imputeMomentTypes getMomentType(string& entry);
 
    void update_req_features_vec(unordered_set<string>& out_req_features, unordered_set<string>& in_req_features);
 
    // Copy
    virtual void copy(FeatureProcessor *processor) { *this = *(dynamic_cast<FeatureImputer *>(processor)); }
 
    // Learn cleaning model
    int Learn(MedFeatures& features, unordered_set<int>& ids);
 
    // Apply cleaning model
    int _apply(MedFeatures& features, unordered_set<int>& ids, bool learning);
 
    // check startas name
    void check_stratas_name(MedFeatures& features, map <string, string> &strata_name_conversion);
 
    // Serialization
    ADD_CLASS_NAME(FeatureImputer)
        ADD_SERIALIZATION_FUNCS(processor_type, feature_name, resolved_feature_name, missing_value, imputerStrata, moment_type, moments, histograms, strata_sizes, default_moment, default_histogram,
            moment_type_vec, moments_vec, default_moment_vec, leave_missing_for_small_stratas, impute_strata_with_missing, round_to_existing_value, existing_values)
 
        void dprint(const string &pref, int fp_flag);
    void print();
 
};
 
 
class FeatureSelector : public FeatureProcessor {
public:
 
    float missing_value = (float)MED_MAT_MISSING_VALUE;
 
    unordered_set<string> required;
 
    vector<string> selected;
 
    int numToSelect = 0;
 
    int numToSelectDelta = 0;
 
    // Constructor
    FeatureSelector() : FeatureProcessor() { missing_value = MED_MAT_MISSING_VALUE; numToSelect = 0; }
 
    virtual int Learn(MedFeatures& features, unordered_set<int>& ids);
 
    virtual int _apply(MedFeatures& features, unordered_set<int>& ids);
    virtual int _conditional_apply(MedFeatures& features, unordered_set<int>& ids, unordered_set<string>& out_req_features);
 
    bool is_selector() { return true; }
    bool are_features_affected(unordered_set<string>& out_req_features);
 
    void update_req_features_vec(unordered_set<string>& out_req_features, unordered_set<string>& in_req_features);
 
    ADD_CLASS_NAME(FeatureSelector)
        ADD_SERIALIZATION_FUNCS(processor_type, missing_value, required, selected, numToSelect)
 
private:
    virtual int _learn(MedFeatures& features, unordered_set<int>& ids) { return 0; }
};
 
class LassoSelector : public FeatureSelector {
public:
    LassoSelector() : FeatureSelector() { init_defaults(); }
 
    int init(map<string, string>& mapper);
 
    float initMaxLambda = (float)0.005;
 
    float lambdaRatio = (float)0.1;
    vector<string> lax_lasso_features;
 
    float rate = (float)0.01;
 
    float momentum = (float)0.95;
 
    float stop_at_err = (float)1e-4;
 
    int nthreads = 12;
 
    void init_defaults() { missing_value = MED_MAT_MISSING_VALUE; processor_type = FTR_PROCESSOR_LASSO_SELECTOR; };
 
    // Copy
    virtual void copy(FeatureProcessor *processor) { *this = *(dynamic_cast<LassoSelector *>(processor)); }
 
    // Serialization
    ADD_CLASS_NAME(LassoSelector)
        ADD_SERIALIZATION_FUNCS(processor_type, initMaxLambda, nthreads, missing_value, required, selected, numToSelect)
 
private:
    // Find set of selected features
    int _learn(MedFeatures& features, unordered_set<int>& ids);
};
 
class DgnrtFeatureRemvoer : public FeatureSelector {
public:
 
    // Percantage covered by single value to define as degenerate
    float percentage = 1.0F;
 
    // Constructor
    DgnrtFeatureRemvoer() : FeatureSelector() { init_defaults(); }
 
    int init(map<string, string>& mapper);
    void init_defaults() { missing_value = MED_MAT_MISSING_VALUE; processor_type = FTR_PROCESS_REMOVE_DGNRT_FTRS; numToSelect = 0; };
 
    // Copy
    virtual void copy(FeatureProcessor *processor) { *this = *(dynamic_cast<DgnrtFeatureRemvoer *>(processor)); }
 
    // Serialization
    ADD_CLASS_NAME(DgnrtFeatureRemvoer)
        ADD_SERIALIZATION_FUNCS(processor_type, percentage, missing_value, selected)
 
private:
    // Find set of selected features
    int _learn(MedFeatures& features, unordered_set<int>& ids);
};
 
typedef enum {
    UNIV_SLCT_PRSN = 0,
    UNIV_SLCT_MI = 1,
    UNIV_SLCT_DCORR = 2,
    UNIV_SLCT_LAST
} UnivariateSelectionMethod;
 
class univariateSelectionParams : SerializableObject {
public:
    UnivariateSelectionMethod method;
    float minStat;
 
    int nBins = 10;
    MedBinningType binMethod = BIN_EQUIDIST;
 
    int takeSquare = 0;
 
    float pDistance;
 
    int max_samples = 10000;
 
    UnivariateSelectionMethod get_method(string name) {
 
        boost::algorithm::to_lower(name);
        if (name == "pearson")
            return UNIV_SLCT_PRSN;
        else if (name == "mi" || name == "mutual_information" || name == "mutualinformation")
            return UNIV_SLCT_MI;
        else if (name == "dcorr" || name == "dist_corr" || name == "distcorr")
            return UNIV_SLCT_DCORR;
        else
            return UNIV_SLCT_LAST;
    }
 
    MedBinningType get_binning_method(string name) {
 
        boost::algorithm::to_lower(name);
        if (name == "equi_dist")
            return BIN_EQUIDIST;
        else if (name == "equi_size")
            return BIN_EQUISIZE;
        else
            return BIN_LAST;
    }
 
    ADD_CLASS_NAME(univariateSelectionParams)
        ADD_SERIALIZATION_FUNCS(method, minStat, nBins, binMethod, takeSquare, pDistance, max_samples)
};
 
class UnivariateFeatureSelector : public FeatureSelector {
public:
 
    univariateSelectionParams params;
 
    // Constructor
    UnivariateFeatureSelector() : FeatureSelector() { init_defaults(); }
 
    int init(map<string, string>& mapper);
    void init_defaults() { missing_value = MED_MAT_MISSING_VALUE; processor_type = FTR_PROCESS_UNIVARIATE_SELECTOR;  params.method = UNIV_SLCT_PRSN;  params.minStat = 0.05F; };
 
    // Copy
    virtual void copy(FeatureProcessor *processor) { *this = *(dynamic_cast<UnivariateFeatureSelector *>(processor)); }
 
    // Serialization
    ADD_CLASS_NAME(UnivariateFeatureSelector)
        ADD_SERIALIZATION_FUNCS(processor_type, params, missing_value, required, selected, numToSelect)
 
private:
    // Scores 
    int getAbsPearsonCorrs(MedFeatures& features, unordered_set<int>& ids, vector<float>& stats);
    int getMIs(MedFeatures& features, unordered_set<int>& ids, vector<float>& stats);
    int getDistCorrs(MedFeatures& features, unordered_set<int>& ids, vector<float>& stats);
    // Find set of selected features
    int _learn(MedFeatures& features, unordered_set<int>& ids);
};
 
typedef enum {
    MRMR_MAX = 0,
    MRMR_MEAN = 1,
    MRMR_LAST
} MRMRPenaltyMethod;
 
class MRMRFeatureSelector : public FeatureSelector {
public:
    univariateSelectionParams params;
    float penalty;
    MRMRPenaltyMethod penaltyMethod;
 
    // Constructor
    MRMRFeatureSelector() : FeatureSelector() { init_defaults(); }
 
    int init(map<string, string>& mapper);
    void init_defaults();
    MRMRPenaltyMethod get_penalty_method(string _method);
 
    // Copy
    virtual void copy(FeatureProcessor *processor) { *this = *(dynamic_cast<MRMRFeatureSelector *>(processor)); }
 
    // Serialization
    ADD_CLASS_NAME(MRMRFeatureSelector)
        ADD_SERIALIZATION_FUNCS(processor_type, params, penalty, penaltyMethod, missing_value, required, selected, numToSelect)
 
private:
    // Find set of selected features
    int _learn(MedFeatures& features, unordered_set<int>& ids);
private:
    // Scores 
    int fillStatsMatrix(MedFeatures& features, unordered_set<int>& ids, MedMat<float>& stats, int index);
    int fillAbsPearsonCorrsMatrix(MedFeatures& features, unordered_set<int>& ids, MedMat<float>& stats, int index);
    int fillMIsMatrix(MedFeatures& features, unordered_set<int>& ids, MedMat<float>& stats, int index);
    int fillDistCorrsMatrix(MedFeatures& features, unordered_set<int>& ids, MedMat<float>& stats, int index);
};
 
//.......................................................................................
//.......................................................................................
// Additional implimentations in other h files
//.......................................................................................
//.......................................................................................
#include "IterativeImputer.h"
 
//.......................................................................................
//.......................................................................................
// Utilities
//.......................................................................................
//.......................................................................................
 
#define DEF_MAX_SAMPLE 1000
void get_all_values(MedFeatures& features, string& signalName, unordered_set<int>& ids, vector<float>& values, int max_sample = DEF_MAX_SAMPLE);
void get_all_outcomes(MedFeatures& features, unordered_set<int>& ids, vector<float>& values, int max_sample = DEF_MAX_SAMPLE);
void smearBins(vector<int>& bins, int nBins, int reqNbins);
 
/************************************************************************************/
class TagFeatureSelector : public FeatureSelector {
public:
    int verbose = 0;
    vector<string> selected_tags; 
    vector<string> removed_tags; 
    // Constructor
    TagFeatureSelector() : FeatureSelector() { init_defaults(); }
 
    int init(map<string, string>& mapper);
    void init_defaults() { missing_value = MED_MAT_MISSING_VALUE; processor_type = FTR_PROCESSOR_TAGS_SELECTOR; };
 
    // Copy
    virtual void copy(FeatureProcessor *processor) { *this = *(dynamic_cast<TagFeatureSelector *>(processor)); }
 
    //print function
    void dprint(const string &pref, int fp_flag);
 
    // Serialization
    ADD_CLASS_NAME(TagFeatureSelector)
        ADD_SERIALIZATION_FUNCS(processor_type, selected_tags, removed_tags, selected)
private:
    // Find set of selected features
    int _learn(MedFeatures& features, unordered_set<int>& ids);
};
 
class ImportanceFeatureSelector : public FeatureSelector {
public:
    string predictor; 
    string predictor_params; 
    string importance_params; 
    float minStat;
    bool verbose; 
    // Constructor
    ImportanceFeatureSelector() : FeatureSelector() { init_defaults(); }
 
    int init(map<string, string>& mapper);
    virtual void init_defaults() { missing_value = MED_MAT_MISSING_VALUE; processor_type = FTR_PROCESSOR_IMPORTANCE_SELECTOR; };
 
    // Copy
    virtual void copy(FeatureProcessor *processor) { *this = *(dynamic_cast<ImportanceFeatureSelector *>(processor)); }
 
 
    // Serialization
    ADD_CLASS_NAME(ImportanceFeatureSelector)
        ADD_SERIALIZATION_FUNCS(processor_type, predictor, predictor_params, importance_params, minStat, selected)
 
private:
    // Find set of selected features
    int _learn(MedFeatures& features, unordered_set<int>& ids);
};
 
class IterativeFeatureSelector : public FeatureSelector {
public:
    string predictor; 
    string predictor_params; 
    string predictor_params_file; 
    int nfolds = 5; 
    bool do_internal_cv = true; 
    vector<int> folds; 
    string mode = "top2bottom"; 
    string rates = "50:1,100:2,500:5,5000:10"; 
    string cohort_params; 
    string bootstrap_params = "sample_per_pid:1"; 
    string msr_params = "AUC"; 
    bool work_on_sets = false; 
    bool group_to_sigs = false; 
    unordered_set<string> ungroupd_names = { "Drug","RC","ICD9" }; 
    unordered_set<string> ignored; 
    bool verbose; 
    string progress_file_path = ""; 
    string grouping_mode = "BY_SIGNAL_CATEG"; 
 
    vector<int> rates_vec;
    vector<string> predictor_params_vec;
    string measurement_name;
    vector<string> report;
 
    // Constructor
    IterativeFeatureSelector() : FeatureSelector() { init_defaults(); }
 
    int init(map<string, string>& mapper);
    virtual void init_defaults() { missing_value = MED_MAT_MISSING_VALUE; processor_type = FTR_PROCESSOR_ITERATIVE_SELECTOR; };
 
    // Bootstrapper initialization
    void init_bootstrap_cohort(MedBootstrapResult& bootstrapper, string& init);
    void init_bootstrap_params(MedBootstrapResult& bootstrapper, string& init);
 
    // Copy
    virtual void copy(FeatureProcessor *processor) { *this = *(dynamic_cast<IterativeFeatureSelector *>(processor)); }
 
    // Print report to file
    void print_report(string& fileName);
 
    // Retracing a pre-given order of families
    void retrace(MedFeatures& features, unordered_set<int>& ids, vector<string>& families_order, int start, int end);
    void retrace(MedFeatures& features, vector<string>& families_order, int start, int end) { unordered_set<int> empty; retrace(features, empty, families_order, start, end); }
 
    // Serialization
    ADD_CLASS_NAME(IterativeFeatureSelector)
        ADD_SERIALIZATION_FUNCS(processor_type, predictor, predictor_params, predictor_params_vec, nfolds, folds, mode, rates_vec, cohort_params, bootstrap_params, msr_params, work_on_sets,
            required, ignored, numToSelect, selected, report, do_internal_cv, grouping_mode)
 
private:
    // Resolved names of required signals
    unordered_set<string> resolved_required, resolved_ignored;
 
    // Find set of selected features
    int _learn(MedFeatures& features, unordered_set<int>& ids);
 
    // Parse rates string
    void get_rates_vec();
 
    // Read parameters file
    void read_params_vec();
 
    // Get Families of signals
    void get_features_families(MedFeatures& features, map<string, vector<string> >& featureFamilies);
 
    // Utilities
    void prepare_for_iterations(MedBootstrapResult& bootstrapper, MedFeatures& features, vector<int>& folds, vector<vector<int>>& trainRows, vector<vector<int>>& testRows, vector<vector<float>>&trainLabels,
        vector<vector<MedSample>>&testSamples, MedFeatures& bootstrapFeatures);
    void pre_learn(MedFeatures& features, MedBootstrapResult& bootstrapper, map<string, vector<string> >& featureFamilies, vector<int>& orig_folds);
 
    // Actual selection
    void doTop2BottomSelection(MedFeatures& features, map<string, vector<string>>& featureFamilies, MedBootstrapResult& bootstrapper);
    void doBottom2TopSelection(MedFeatures& features, map<string, vector<string>>& featureFamilies, MedBootstrapResult& bootstrapper);
 
    // Actual retracing a pre-given order of families
    void retraceTop2BottomSelection(MedFeatures& features, map<string, vector<string>>& featureFamilies, MedBootstrapResult& bootstrapper, vector<string>& order, int start, int end);
    void retraceBottom2TopSelection(MedFeatures& features, map<string, vector<string>>& featureFamilies, MedBootstrapResult& bootstrapper, vector<string>& order, int start, int end);
};
 
 
class FeatureEncoder : public FeatureProcessor {
public:
 
    vector<string> names;
 
    // Constructor
    FeatureEncoder() : FeatureProcessor() { init_defaults(); }
 
    bool are_features_affected(unordered_set<string>& out_req_features);
 
    ADD_CLASS_NAME(FeatureEncoder)
        ADD_SERIALIZATION_FUNCS(processor_type, names)
 
        void update_req_features_vec(unordered_set<string>& out_req_features, unordered_set<string>& in_req_features);
 
};
 
class FeaturePCAParams : SerializableObject {
public:
    int pca_top; 
    float pca_cutoff;
    int subsample_count;
 
    ADD_CLASS_NAME(FeaturePCAParams)
        ADD_SERIALIZATION_FUNCS(pca_top, pca_cutoff)
};
 
class FeaturePCA :public FeatureEncoder {
public:
    FeaturePCAParams params;
 
    // Constructor
    FeaturePCA() : FeatureEncoder() { init_defaults(); }
 
    int init(map<string, string>& mapper);
    void init_defaults() { processor_type = FTR_PROCESS_ENCODER_PCA;  params.pca_cutoff = 0; params.pca_top = 100; };
 
    virtual void copy(FeatureProcessor *processor) { *this = *(dynamic_cast<FeaturePCA *>(processor)); }
 
    ADD_CLASS_NAME(FeaturePCA)
        ADD_SERIALIZATION_FUNCS(processor_type, names, params, selected_indexes, W)
 
private:
    MedMat<float> W;
    vector<int> selected_indexes;
 
    int _learn(MedFeatures& features, unordered_set<int>& ids);
 
    // Apply encoding
    int _apply(MedFeatures& features, unordered_set<int>& ids);
};
 
class OneHotFeatProcessor :public FeatureProcessor {
public:
 
    string index_feature_prefix = ""; 
    string other_feature_name = ""; 
    string removed_feature_name = ""; 
    bool rem_origin = true; 
    bool add_other = false; 
    bool allow_other = false; 
    bool remove_last = false; 
    int max_values = 32; 
    vector<string> regex_list; 
    vector<string> regex_list_names; 
    string other_suffix = "other";
 
    //map<float, string> value2feature;
    map<float, vector<string>> value2feature;
 
    // Constructor
    OneHotFeatProcessor() { init_defaults(); }
 
    int init(map<string, string>& mapper);
 
    void init_defaults() { processor_type = FTR_PROCESS_ONE_HOT; }
    virtual void copy(FeatureProcessor *processor) { *this = *(dynamic_cast<OneHotFeatProcessor *>(processor)); }
 
    bool are_features_affected(unordered_set<string>& out_req_features);
 
    void update_req_features_vec(unordered_set<string>& out_req_features, unordered_set<string>& in_req_features);
 
    ADD_CLASS_NAME(OneHotFeatProcessor);
    ADD_SERIALIZATION_FUNCS(processor_type, feature_name, resolved_feature_name, index_feature_prefix, other_feature_name, removed_feature_name, rem_origin, add_other,
        remove_last, allow_other, value2feature, regex_list, regex_list_names, max_values)
private:
    int Learn(MedFeatures& features, unordered_set<int>& ids);
    int _apply(MedFeatures& features, unordered_set<int>& ids);
    string get_feature_name(float value, const string &out_prefix, unordered_map<float, string> &value2Name, float missing_value);
};
 
class GetProbFeatProcessor : public FeatureProcessor {
public:
 
    float missing_value = MED_MAT_MISSING_VALUE; 
    int overall_count = 5; 
    map<float, int> target_labels; 
    map<float, string> feature_names; 
    bool remove_origin = true; 
    bool all_labels = false; 
    int min_obs = 100; 
 
    vector<map<float, float>> probs; 
    vector<float> overall_prob; 
 
    // Constructor
    GetProbFeatProcessor() : FeatureProcessor() { init_defaults(); processor_type = FTR_PROCESS_GET_PROB; }
    GetProbFeatProcessor(const  string& feature_name) : FeatureProcessor() { init_defaults(); processor_type = FTR_PROCESS_GET_PROB; set_feature_name(feature_name); }
    GetProbFeatProcessor(const  string& feature_name, string init_string) : FeatureProcessor() { init_defaults(); processor_type = FTR_PROCESS_GET_PROB;; init_from_string(init_string);  set_feature_name(feature_name); }
 
    // Learn probabilities
    int Learn(MedFeatures& features, unordered_set<int>& ids);
 
    // Apply transformation
    int _apply(MedFeatures& features, unordered_set<int>& ids);
 
    int init(map<string, string>& mapper);
 
    // Copy
    virtual void copy(FeatureProcessor *processor) { *this = *(dynamic_cast<GetProbFeatProcessor *>(processor)); }
 
    // Serialization
    ADD_CLASS_NAME(GetProbFeatProcessor)
        ADD_SERIALIZATION_FUNCS(processor_type, feature_name, resolved_feature_name, missing_value, overall_count, probs,
            overall_prob, target_labels, remove_origin, feature_names, min_obs);
 
};
 
class MultiplierProcessor : public FeatureProcessor {
public:
    vector<string> selected_tags; 
    string multiplier_name; 
    bool divide; 
    bool verbose;
 
    MultiplierProcessor() : FeatureProcessor() { init_defaults(); }
 
    int init(map<string, string>& mapper);
    void init_defaults() { multiplier_name = ""; divide = false; verbose = true; processor_type = FTR_PROCESS_MULTIPLIER; };
 
    // Copy
    virtual void copy(FeatureProcessor *processor) { *this = *(dynamic_cast<MultiplierProcessor *>(processor)); }
 
    //print function
    void dprint(const string &pref, int fp_flag);
 
    // Serialization
    ADD_CLASS_NAME(MultiplierProcessor)
        ADD_SERIALIZATION_FUNCS(processor_type, selected_tags, multiplier_name, divide, verbose)
private:
    // Apply multiplier
    int _apply(MedFeatures& features, unordered_set<int>& ids);
};
class MissingIndicatorProcessor : public FeatureProcessor {
public:
    float missing_value; 
    string name;  
    float replace_value; 
    string new_feature_name; 
 
    MissingIndicatorProcessor() { init_defaults(); }
 
    void init_defaults() { processor_type = FTR_PROCESS_MISSING_INDICATOR;  missing_value = MED_MAT_MISSING_VALUE; name = "is_missing"; }
    int _apply(MedFeatures& features, unordered_set<int>& ids);
    int init(map<string, string>& mapper);
    void update_req_features_vec(unordered_set<string>& out_req_features, unordered_set<string>& in_req_features);
 
    // Serialization
    ADD_CLASS_NAME(MissingIndicatorProcessor)
        ADD_SERIALIZATION_FUNCS(processor_type, name, replace_value, new_feature_name, feature_name, resolved_feature_name)
};
 
 
class Binning_Wrapper : public SerializableObject {
public:
    vector<double> bin_cutoffs; 
    vector<double> bin_repr_vals; 
 
    string use_bin_settings; 
 
    int init(map<string, string>& mapper);
 
    void load_bin_settings(const vector<float> &nums, vector<float> &y);
 
    int get_idx(float v) const;
 
    int num_of_bins() const;
 
    float normalize(float v) const;
 
    ADD_CLASS_NAME(Binning_Wrapper)
        ADD_SERIALIZATION_FUNCS(bin_cutoffs, bin_repr_vals, use_bin_settings)
};
 
 
class BinningFeatProcessor : public FeatureProcessor {
private:
    string get_bin_name(float num) const;
public:
 
    float missing_value = MED_MAT_MISSING_VALUE; 
    float missing_target_val = MED_MAT_MISSING_VALUE; 
    bool remove_origin = true;
    bool one_hot = true;
    bool keep_original_val = false;
    string bin_format = "%2.1f";
    Binning_Wrapper bin_sett;
    // Constructor
    BinningFeatProcessor() : FeatureProcessor() { init_defaults(); processor_type = FTR_PROCESS_BINNING; }
    BinningFeatProcessor(const  string& feature_name) : FeatureProcessor() { init_defaults(); processor_type = FTR_PROCESS_BINNING; set_feature_name(feature_name); }
    BinningFeatProcessor(const  string& feature_name, string init_string) : FeatureProcessor() { init_defaults(); processor_type = FTR_PROCESS_BINNING; init_from_string(init_string);  set_feature_name(feature_name); }
 
    // Learn probabilities
    int Learn(MedFeatures& features, unordered_set<int>& ids);
 
    // Apply transformation
    int _apply(MedFeatures& features, unordered_set<int>& ids);
 
    int init(map<string, string>& mapper);
 
    // Copy
    virtual void copy(FeatureProcessor *processor) { *this = *(dynamic_cast<BinningFeatProcessor *>(processor)); }
 
    // Serialization
    ADD_CLASS_NAME(BinningFeatProcessor)
        ADD_SERIALIZATION_FUNCS(processor_type, feature_name, resolved_feature_name, missing_value, missing_target_val,
            remove_origin, bin_sett, bin_format, one_hot, keep_original_val);
 
};
 
 
//=======================================
// Joining the MedSerialze wagon
//=======================================
MEDSERIALIZE_SUPPORT(FeatureProcessor)
MEDSERIALIZE_SUPPORT(MultiFeatureProcessor)
MEDSERIALIZE_SUPPORT(FeatureBasicOutlierCleaner)
MEDSERIALIZE_SUPPORT(FeatureNormalizer)
MEDSERIALIZE_SUPPORT(featureStrata)
MEDSERIALIZE_SUPPORT(featureSetStrata)
MEDSERIALIZE_SUPPORT(FeatureImputer)
MEDSERIALIZE_SUPPORT(FeatureSelector)
MEDSERIALIZE_SUPPORT(LassoSelector)
MEDSERIALIZE_SUPPORT(DgnrtFeatureRemvoer)
MEDSERIALIZE_SUPPORT(UnivariateFeatureSelector)
MEDSERIALIZE_SUPPORT(MRMRFeatureSelector)
MEDSERIALIZE_SUPPORT(FeatureEncoder)
MEDSERIALIZE_SUPPORT(FeaturePCAParams)
MEDSERIALIZE_SUPPORT(FeaturePCA)
MEDSERIALIZE_SUPPORT(TagFeatureSelector)
MEDSERIALIZE_SUPPORT(ImportanceFeatureSelector)
MEDSERIALIZE_SUPPORT(IterativeFeatureSelector)
MEDSERIALIZE_SUPPORT(OneHotFeatProcessor)
MEDSERIALIZE_SUPPORT(univariateSelectionParams)
MEDSERIALIZE_SUPPORT(MultiplierProcessor)
MEDSERIALIZE_SUPPORT(MissingIndicatorProcessor)
MEDSERIALIZE_SUPPORT(Binning_Wrapper)
MEDSERIALIZE_SUPPORT(BinningFeatProcessor)
#endif