FeatureGenerator.h

Go to the documentation of this file.

#ifndef _FTR_GENERATOR_H_
#define _FTR_GENERATOR_H_
 
#include <InfraMed/InfraMed/InfraMed.h>
#include <Logger/Logger/Logger.h>
#include <MedProcessTools/MedProcessTools/RepProcess.h>
#include <MedProcessTools/MedProcessTools/MedFeatures.h>
#include <SerializableObject/SerializableObject/SerializableObject.h>
#include <MedProcessTools/MedProcessTools/MedModelExceptions.h>
#include <MedTime/MedTime/MedTime.h>
#include <MedAlgo/MedAlgo/MedAlgo.h>
#include <MedAlgo/MedAlgo/MedLM.h>
#include <cfloat>
#include <boost/regex.hpp>
 
#define DEFAULT_FEAT_GNRTR_NTHREADS 8
 
// For ModelFeatureGenerator
class MedModel;
 
// TBD : Add wrapper for management of features list (read/write to file, etc.)
 
typedef enum {
    FTR_GEN_NOT_SET,
    FTR_GEN_BASIC, 
    FTR_GEN_AGE, 
    FTR_GEN_SINGLETON, 
    FTR_GEN_GENDER, 
    FTR_GEN_BINNED_LM, 
    FTR_GEN_SMOKING, 
    FTR_GEN_KP_SMOKING, 
    FTR_GEN_UNIFIED_SMOKING, 
    FTR_GEN_RANGE, 
    FTR_GEN_DRG_INTAKE, 
    FTR_GEN_ALCOHOL, 
    FTR_GEN_MODEL, 
    FTR_GEN_TIME, 
    FTR_GEN_ATTR, 
    FTR_GEN_CATEGORY_DEPEND, 
    FTR_GEN_EMBEDDING, 
    FTR_GEN_EXTRACT_TBL, 
    FTR_GEN_ELIXHAUSER, 
    FTR_GEN_DIABETES_FINDER, 
    FTR_GEN_LAST
} FeatureGeneratorTypes;
 
class FeatureGenerator : public SerializableObject {
public:
 
    FeatureGeneratorTypes generator_type = FTR_GEN_LAST;
 
    vector<string> names;
 
    // Threading
    int learn_nthreads = 16, pred_nthreads = 16;
 
    float missing_val = (float)MED_MAT_MISSING_VALUE;
 
    vector<string> tags;
 
    int iGenerateWeights = 0;
 
    // Naming
    virtual void set_names() { names.clear(); }
 
    // Helper - pointers to data vectors in MedFeatures (to save time in generation)
    vector <float *> p_data;
 
    // Prepare for feature generation
    virtual void prepare(MedFeatures &features, MedPidRepository& rep, MedSamples& samples);
    virtual void get_p_data(MedFeatures& features, vector<float *> &_p_data);
    void get_p_data(MedFeatures& features) { get_p_data(features, p_data); }
 
    // Constructor/Destructor
    FeatureGenerator() { learn_nthreads = DEFAULT_FEAT_GNRTR_NTHREADS; pred_nthreads = DEFAULT_FEAT_GNRTR_NTHREADS;  missing_val = MED_MAT_MISSING_VALUE; serial_id = ++MedFeatures::global_serial_id_cnt; };
    virtual ~FeatureGenerator() { clear(); };
    virtual void clear() { };
 
    // Required Signals
    vector<string> req_signals;
    vector<int> req_signal_ids;
 
    void get_required_signal_names(unordered_set<string>& signalNames);
    virtual void set_required_signal_ids(MedDictionarySections& dict);
    void get_required_signal_ids(unordered_set<int>& signalIds);
 
    // generated features
    virtual void get_generated_features(unordered_set<string>& names_list) { for (auto &s : names) names_list.insert(s); }
 
    // Signal Ids
    virtual void set_signal_ids(MedSignals& sigs) { return; }
 
    // Init required tables
    virtual void init_tables(MedDictionarySections& dict) { return; }
 
    virtual void fit_for_repository(MedPidRepository &rep) { return; }
 
    // Learn a generator
    virtual int _learn(MedPidRepository& rep, const MedSamples& samples, vector<RepProcessor *> processors) { return 0; }
    int learn(MedPidRepository& rep, const MedSamples& samples, vector<RepProcessor *> processors) { set_names(); return _learn(rep, samples, processors); }
    int learn(MedPidRepository& rep, const MedSamples& samples) { set_names(); return _learn(rep, samples, vector<RepProcessor *>()); }
 
    // generate feature data from repository
    // We assume the corresponding MedSamples have been inserted to MedFeatures : either at the end or at position index
    int _generate(PidDynamicRec& in_rep, MedFeatures& features, int index, int num) {return _generate(in_rep, features, index, num, p_data); }
 
    // the following is the MAIN generation routine to implement.
    // note that it is given a p_data of its own. This is in order to allow different records to write results to different places.
    // the default run will use it with the generator p_data.
    virtual int _generate(PidDynamicRec& in_rep, MedFeatures& features, int index, int num, vector<float *> &_p_data) { return 0; }
 
    int generate(PidDynamicRec& in_rep, MedFeatures& features, int index, int num) { return _generate(in_rep, features, index, num); }
    int generate(PidDynamicRec& in_rep, MedFeatures& features);
    int generate(MedPidRepository& rep, int id, MedFeatures& features);
    int generate(MedPidRepository& rep, int id, MedFeatures& features, int index, int num);
 
    // generate feature data from other features
    virtual int _generate(MedFeatures& features) { return 0; }
    int generate(MedFeatures& features) { return _generate(features); }
 
    // Init
    // create a generator
    static FeatureGenerator *make_generator(string name);
    static FeatureGenerator *make_generator(string name, string params);
    static FeatureGenerator *make_generator(FeatureGeneratorTypes type);
    static FeatureGenerator *make_generator(FeatureGeneratorTypes type, string params);
 
    static FeatureGenerator *create_generator(string &params); // must include fg_type
 
    virtual int init(void *generator_params) { return 0; };
    virtual int init(map<string, string>& mapper);
    virtual void init_defaults() {};
 
    // Copy
    virtual void copy(FeatureGenerator *generator) { *this = *generator; }
 
    // Number of features generated
    virtual int nfeatures() { return (int)names.size(); }
 
    virtual void get_required_signal_categories(unordered_map<string, vector<string>> &signal_categories_in_use) const {};
 
    // Filter generated features according to a set. return number of valid features
    virtual int filter_features(unordered_set<string>& validFeatures);
 
    virtual void make_summary() {};
 
    // Serialization
    ADD_CLASS_NAME(FeatureGenerator)
    ADD_SERIALIZATION_FUNCS(generator_type, names, learn_nthreads, pred_nthreads, missing_val, tags, iGenerateWeights)
    void *new_polymorphic(string derived_class_name);
 
    size_t get_generator_size();
    size_t generator_serialize(unsigned char *blob);
 
    virtual void print() { fprintf(stderr, "Print Not Implemented for feature\n"); }
 
 
    // debug print for a feature generator. fg_flag can 
    virtual void dprint(const string &pref, int fg_flag);
 
 
    int serial_id;      // serial id of feature
};
 
FeatureGeneratorTypes ftr_generator_name_to_type(const string& generator_name);
 
//..............................................................................................
// FeatureSingleChannel -
// This class is a mediator between FeatureGenerator and classes that generate
// Features on a single variable (not including age and gender) and in it in a single channel.
//..............................................................................................
 
//.......................................................................................
//.......................................................................................
// Single signal features that do not require learning(e.g. last hemoglobin)
//.......................................................................................
//.......................................................................................
 
typedef enum {
    FTR_LAST_VALUE = 0, 
    FTR_FIRST_VALUE = 1, 
    FTR_LAST2_VALUE = 2, 
    FTR_AVG_VALUE = 3, 
    FTR_MAX_VALUE = 4, 
    FTR_MIN_VALUE = 5, 
    FTR_STD_VALUE = 6, 
    FTR_LAST_DELTA_VALUE = 7, 
    FTR_LAST_DAYS = 8, 
    FTR_LAST2_DAYS = 9,
    FTR_SLOPE_VALUE = 10, 
    FTR_WIN_DELTA_VALUE = 11, 
    FTR_CATEGORY_SET = 12, 
    FTR_CATEGORY_SET_COUNT = 13,
    FTR_CATEGORY_SET_SUM = 14, 
    FTR_NSAMPLES = 15, 
    FTR_EXISTS = 16, 
    FTR_CATEGORY_SET_FIRST = 17, 
    FTR_MAX_DIFF = 18, 
    FTR_FIRST_DAYS = 19, 
    FTR_RANGE_WIDTH = 20, 
    FTR_CATEGORY_SET_FIRST_TIME = 21, 
    FTR_SUM_VALUE=22, 
    FTR_LAST_NTH_VALUE = 23, 
    FTR_CATEGORY_SET_LAST_NTH = 24, 
    FTR_TIME_SINCE_LAST_CHANGE = 25, 
    FTR_LAST
} BasicFeatureTypes;
 
typedef enum {
    TIME_RANGE_CURRENT = 0, 
    TIME_RANGE_BEFORE = 1, 
    TIME_RANGE_LAST
} TimeRangeTypes;
 
class BasicFeatGenerator : public FeatureGenerator {
private:
    // actual generators
    float uget_last(UniversalSigVec &usv, int time_point, int _win_from, int _win_to, int outcomeTime); // Added the win as needed to be called on different ones in uget_win_delta
    float uget_last_nth(UniversalSigVec &usv, int time_point, int _win_from, int _win_to, int outcomeTime);
    float uget_first(UniversalSigVec &usv, int time_point, int _win_from, int _win_to, int outcomeTime);
    float uget_last2(UniversalSigVec &usv, int time_point, int _win_from, int _win_to, int outcomeTime);
    float uget_avg(UniversalSigVec &usv, int time_point, int _win_from, int _win_to, int outcomeTime);
    float uget_max(UniversalSigVec &usv, int time_point, int _win_from, int _win_to, int outcomeTime);
    float uget_min(UniversalSigVec &usv, int time_point, int _win_from, int _win_to, int outcomeTime);
    float uget_sum(UniversalSigVec &usv, int time_point, int _win_from, int _win_to, int outcomeTime);
    float uget_std(UniversalSigVec &usv, int time_point, int _win_from, int _win_to, int outcomeTime);
    float uget_last_delta(UniversalSigVec &usv, int time_point, int _win_from, int _win_to, int outcomeTime);
    float uget_last_time(UniversalSigVec &usv, int time_point, int _win_from, int _win_to, int outcomeTime);
    float uget_last2_time(UniversalSigVec &usv, int time_point, int _win_from, int _win_to, int outcomeTime);
    float uget_slope(UniversalSigVec &usv, int time_point, int _win_from, int _win_to, int outcomeTime);
    float uget_win_delta(UniversalSigVec &usv, int time_point, int _win_from, int _win_to, int _d_win_from, int _d_win_to, int outcomeTime);
    float uget_category_set(PidDynamicRec &rec, UniversalSigVec &usv, int time_point, int _win_from, int _win_to, int outcomeTime);
    float uget_category_set_last_nth(PidDynamicRec &rec, UniversalSigVec &usv, int time_point, int _win_from, int _win_to, int outcomeTime);
    float uget_category_set_count(PidDynamicRec &rec, UniversalSigVec &usv, int time_point, int _win_from, int _win_to, int outcomeTime);
    float uget_category_set_sum(PidDynamicRec &rec, UniversalSigVec &usv, int time_point, int _win_from, int _win_to, int outcomeTime);
    float uget_nsamples(UniversalSigVec &usv, int time, int _win_from, int _win_to, int outcomeTime);
    float uget_exists(UniversalSigVec &usv, int time, int _win_from, int _win_to, int outcomeTime);
    float uget_range_width(UniversalSigVec &usv, int time_point, int _win_from, int _win_to, int outcomeTime);
    float uget_max_diff(UniversalSigVec &usv, int time_point, int _win_from, int _win_to, int outcomeTime);
    float uget_first_time(UniversalSigVec &usv, int time_point, int _win_from, int _win_to, int outcomeTime);
    float uget_category_set_first(PidDynamicRec &rec, UniversalSigVec &usv, int time_point, int _win_from, int _win_to, int outcomeTime);
    float uget_category_set_first_time(PidDynamicRec &rec, UniversalSigVec &usv, int time_point, int _win_from, int _win_to, int outcomeTime);
    float uget_time_since_last_change(UniversalSigVec &usv, int time_point, int _win_from, int _win_to, int outcomeTime);
 
    // Applying non FTR_CATEGORY_SET_* types to categorical data
    unordered_set<int> categ_require_dict = { FTR_LAST_VALUE, FTR_FIRST_VALUE, FTR_LAST2_VALUE, FTR_LAST_NTH_VALUE }; // Types that requriew dictionary if applied on categorical data
    unordered_set<int> categ_forbidden = { FTR_AVG_VALUE, FTR_MAX_VALUE, FTR_MIN_VALUE,  FTR_STD_VALUE, FTR_LAST_DELTA_VALUE,
        FTR_SLOPE_VALUE,FTR_WIN_DELTA_VALUE, FTR_MAX_DIFF,FTR_RANGE_WIDTH, FTR_SUM_VALUE }; // Types that are not allowed for categorical data
    map<string, int> categ_value2id;
    bool needs_categ_dict = true;
 
public:
    // Feature Descrption
    string signalName;
    int signalId;
 
    // Signal to determine allowed time-range (e.g. current stay/admission for inpatients)
    string timeRangeSignalName = "";
    int timeRangeSignalId;
    TimeRangeTypes timeRangeType = TIME_RANGE_CURRENT;
    int time_unit_range_sig = MedTime::Undefined;       
 
    // parameters (should be serialized)
    BasicFeatureTypes type = FTR_LAST;
    int win_from = 0;
    int win_to = 360000;
    int d_win_from = 360; 
    int d_win_to = 360000;  
    int time_unit_win = MedTime::Undefined;         
    int time_channel = 0;                       
    int val_channel = 0;                        
    int sum_channel = 1;                        
    vector<string> sets;                        
    int time_unit_sig = MedTime::Undefined;     
    string in_set_name = "";                    
    bool bound_outcomeTime = false; 
    float min_value = -FLT_MAX, max_value = FLT_MAX; 
    int N_th = 0; 
    int zero_missing = 0; 
    float zero_missing_val = 0; 
    int full_name = 0; 
    string rename_signal = "";
 
    // helpers
    vector<char> lut;                           
    vector<float> categ_map;                    
    bool apply_categ_map;
 
    // Naming 
    void set_names();
 
    void get_required_signal_categories(unordered_map<string, vector<string>> &signal_categories_in_use) const;
 
    // Constructor/Destructor
    BasicFeatGenerator() : FeatureGenerator() { init_defaults(); };
    //~BasicFeatGenerator() {};
    void set(string& _signalName, BasicFeatureTypes _type) {
        set(_signalName, _type, 0, 360000);
        req_signals.assign(1, signalName);
        if (timeRangeSignalName != "")
            req_signals.push_back(timeRangeSignalName);
    }
 
    void set(string& _signalName, BasicFeatureTypes _type, int _time_win_from, int _time_win_to) {
        signalName = _signalName; type = _type; win_from = _time_win_from; win_to = _time_win_to;
        set_names();
        req_signals.assign(1, signalName);
        if (timeRangeSignalName != "")
            req_signals.push_back(timeRangeSignalName);
    }
 
    BasicFeatureTypes name_to_type(const string &name);
 
    TimeRangeTypes time_range_name_to_type(const string& name);
    string time_range_type_to_name(TimeRangeTypes type);
 
    int init(map<string, string>& mapper);
    void init_defaults();
 
    // Copy
    virtual void copy(FeatureGenerator *generator) { *this = *(dynamic_cast<BasicFeatGenerator *>(generator)); }
 
    int _learn(MedPidRepository& rep, const MedSamples& samples, vector<RepProcessor *> processors) {
        time_unit_sig = rep.sigs.Sid2Info[rep.sigs.sid(signalName)].time_unit;
        if (timeRangeSignalName != "")
            time_unit_range_sig = rep.sigs.Sid2Info[rep.sigs.sid(timeRangeSignalName)].time_unit;
        return 0;
    }
 
    int _generate(PidDynamicRec& rec, MedFeatures& features, int index, int num, vector<float *> &_p_data);
    float get_value(PidDynamicRec& rec, int index, int time, int outcomeTime);
 
    void set_signal_ids(MedSignals& sigs);
 
    void init_tables(MedDictionarySections& dict);
 
    void prepare(MedFeatures &features, MedPidRepository& rep, MedSamples& samples);
 
    // Serialization
    ADD_CLASS_NAME(BasicFeatGenerator)
    ADD_SERIALIZATION_FUNCS(generator_type, type, tags, serial_id, win_from, win_to, d_win_from, d_win_to, time_unit_win, time_channel, val_channel, sum_channel, min_value, max_value, signalName, sets,
            names, req_signals, in_set_name, bound_outcomeTime, timeRangeSignalName, timeRangeType, time_unit_sig, N_th, zero_missing, missing_val, categ_value2id, zero_missing_val, full_name, rename_signal)
 
};
 
class AgeGenerator : public FeatureGenerator {
public:
 
    string signalName;
    int signalId;
 
    ~AgeGenerator() { clear(); }
    void clear() { }
 
    // Constructor/Destructor
    AgeGenerator() {
        generator_type = FTR_GEN_AGE; names.push_back("Age"); signalId = -1; signalName = "BDATE"; req_signals.assign(1, signalName);
    }
    //~AgeGenerator() {};
 
    // Name
    void set_names() { if (names.empty()) names.push_back("FTR_" + int_to_string_digits(serial_id, 6) + ".Age"); tags.push_back("Age"); }
 
    // Copy
    virtual void copy(FeatureGenerator *generator) { *this = *(dynamic_cast<AgeGenerator *>(generator)); }
 
    // generate a new feature
    int _generate(PidDynamicRec& rec, MedFeatures& features, int index, int num, vector<float *> &_p_data);
 
    // Signal Ids
    void set_signal_ids(MedSignals& sigs) { signalId = sigs.sid(signalName); }
 
    // Serialization
    ADD_CLASS_NAME(AgeGenerator)
        ADD_SERIALIZATION_FUNCS(generator_type, names, tags, iGenerateWeights, signalName, req_signals)
 
        virtual int init(map<string, string>& mapper);
};
 
class SingletonGenerator : public FeatureGenerator {
private:
    vector<char> lut;           
    unordered_map<string, float> name2Value; 
    vector<float> id2Value; 
 
    void get_id2Value(MedDictionarySections& dict);
public:
 
    string signalName;
    int signalId;
 
    vector<string> sets = {};       
    string in_set_name = "";
 
    // Constructor/Destructor
    SingletonGenerator() : FeatureGenerator() { generator_type = FTR_GEN_SINGLETON; names.push_back(signalName); signalId = -1; req_signals.assign(1, signalName); }
    SingletonGenerator(int _signalId) : FeatureGenerator() { generator_type = FTR_GEN_SINGLETON; names.push_back(signalName); signalId = _signalId; req_signals.assign(1, signalName); }
 
    // Name
    void set_names();
 
    // Init LUT for categorial variable
    void init_tables(MedDictionarySections& dict);
    int init(map<string, string>& mapper);
 
    // Copy
    virtual void copy(FeatureGenerator *generator) { *this = *(dynamic_cast<SingletonGenerator *>(generator)); }
 
    // learn generator (learning name2Value)
    int _learn(MedPidRepository& rep, const MedSamples& samples, vector<RepProcessor *> processors);
 
    // generate a new feature
    int _generate(PidDynamicRec& rec, MedFeatures& features, int index, int num, vector<float *> &_p_data);
 
    // Preparation - just fill the value2Name attribute
    void prepare(MedFeatures &features, MedPidRepository& rep, MedSamples& samples);
 
    // Signal Ids
    void set_signal_ids(MedSignals& sigs) { signalId = sigs.sid(signalName); }
    void set_required_signal_ids(MedDictionarySections& dict) { req_signal_ids.assign(1, dict.id(signalName)); }
 
    void get_required_signal_categories(unordered_map<string, vector<string>> &signal_categories_in_use) const;
 
    // Serialization
    ADD_CLASS_NAME(SingletonGenerator)
        ADD_SERIALIZATION_FUNCS(generator_type, req_signals, signalName, names, tags, iGenerateWeights, sets, lut, name2Value)
};
 
 
class GenderGenerator : public FeatureGenerator {
private:
    vector<string> category_values;
public:
 
    int genderId;
 
    // Constructor/Destructor
    GenderGenerator() : FeatureGenerator() { generator_type = FTR_GEN_GENDER; names.push_back("Gender"); genderId = -1; req_signals.assign(1, "GENDER"); }
    GenderGenerator(int _genderId) : FeatureGenerator() { generator_type = FTR_GEN_GENDER; names.push_back("Gender"); genderId = _genderId; req_signals.assign(1, "GENDER"); }
 
    //~GenderGenerator() {};
 
    // Name
    void set_names() { if (names.empty()) names.push_back("Gender"); tags.push_back("Gender"); }
 
    // Copy
    virtual void copy(FeatureGenerator *generator) { *this = *(dynamic_cast<GenderGenerator *>(generator)); }
 
    int init(map<string, string>& mapper);
 
    // generate a new feature
    int _generate(PidDynamicRec& rec, MedFeatures& features, int index, int num, vector<float *> &_p_data);
 
    // Signal Ids
    void set_signal_ids(MedSignals& sigs) { genderId = sigs.sid("GENDER"); }
    void set_required_signal_ids(MedDictionarySections& dict) { 
        req_signal_ids.assign(1, dict.id("GENDER")); 
    }
 
    void init_tables(MedDictionarySections& dict) {
        category_values.clear();
        if (dict.SectionName2Id.find("GENDER") != dict.SectionName2Id.end()) {
            int section_id = dict.section_id("GENDER");
            for (const auto &it : dict.dicts[section_id].Id2Name)
                category_values.push_back(it.second);
        }
    }
 
    void get_required_signal_categories(unordered_map<string, vector<string>> &signal_categories_in_use) const;
 
    // Serialization
    ADD_CLASS_NAME(GenderGenerator)
        ADD_SERIALIZATION_FUNCS(generator_type, names, tags, iGenerateWeights)
};
 
struct BinnedLmEstimatesParams : public SerializableObject {
    vector<int> bin_bounds;
    int min_period;
    int max_period;
    float rfactor;
 
    vector<int> estimation_points;
    ADD_CLASS_NAME(BinnedLmEstimatesParams)
        ADD_SERIALIZATION_FUNCS(bin_bounds, min_period, max_period, rfactor, estimation_points)
 
};
 
typedef enum {
    BINNED_LM_TAKE_ALL = 0,
    BINNED_LM_STOP_AT_FIRST = 1,
    BINNED_LM_STOP_AT_LAST = 2,
    BINNED_LM_LAST
} BinnedLMSamplingStrategy;
 
class BinnedLmEstimates : public FeatureGenerator {
public:
    // Feature Descrption
    string signalName;
    int signalId, bdateId, genderId;
 
    BinnedLmEstimatesParams params;
    BinnedLMSamplingStrategy sampling_strategy = BINNED_LM_TAKE_ALL;
    vector<MedLM> models;
    vector<float> xmeans, xsdvs, ymeans, ysdvs;
    vector<vector<float>> means = { {}, {} };
 
    int time_unit_periods = MedTime::Undefined;     
    int time_unit_sig = MedTime::Undefined;         
    int time_channel = 0;                       
    int val_channel = 0;                        
 
    void set_names();
 
    // Constructor/Destructor
    BinnedLmEstimates() : FeatureGenerator() { signalName = ""; init_defaults(); };
    BinnedLmEstimates(string _signalName) : FeatureGenerator() { signalName = _signalName; init_defaults(); req_signals.push_back(signalName); names.clear();  set_names(); };
    BinnedLmEstimates(string _signalName, string init_string) : FeatureGenerator() { signalName = _signalName; init_defaults(); req_signals.push_back(signalName); init_from_string(init_string); };
 
    //~BinnedLmEstimates() {};
 
    void set(string& _signalName);
    void set(string& _signalName, BinnedLmEstimatesParams* _params);
 
    void init_defaults();
    int init(map<string, string>& mapper);
 
    // Copy
    virtual void copy(FeatureGenerator *generator) { *this = *(dynamic_cast<BinnedLmEstimates *>(generator)); }
 
    int _learn(MedPidRepository& rep, const MedSamples& samples, vector<RepProcessor *> processors);
 
    int _generate(PidDynamicRec& rec, MedFeatures& features, int index, int num, vector<float *> &_p_data);
 
    int filter_features(unordered_set<string>& validFeatures);
 
    // get pointers to data
    void get_p_data(MedFeatures& features, vector<float *> &_p_data);
 
    // Signal Ids
    void set_signal_ids(MedSignals& sigs);
 
    // Sampling strategy
    void set_sampling_strategy(string& strategy);
 
    // Age Related functions
    void prepare_for_age(PidDynamicRec& rec, UniversalSigVec& ageUsv, int &age, int &byear);
    void prepare_for_age(MedPidRepository& rep, int id, UniversalSigVec& ageUsv, int &age, int &byear);
    inline void get_age(int time, int time_unit_from, int& age, int byear);
 
    void dprint(const string &pref, int fg_flag);
 
    // Serialization
    ADD_CLASS_NAME(BinnedLmEstimates)
        ADD_SERIALIZATION_FUNCS(generator_type, signalName, names, tags, req_signals, time_unit_periods, iGenerateWeights, params, xmeans, xsdvs, ymeans, means, models, time_unit_sig, sampling_strategy)
 
        // print 
        void print();
};
 
 
typedef enum {
    FTR_RANGE_CURRENT = 0, 
    FTR_RANGE_LATEST = 1, 
    FTR_RANGE_MAX = 2, 
    FTR_RANGE_MIN = 3, 
    FTR_RANGE_EVER = 4,
    FTR_RANGE_TIME_DIFF = 5,
    FTR_RANGE_RECURRENCE_COUNT = 6,
    FTR_RANGE_TIME_COVERED = 7,
    FTR_RANGE_LAST_NTH_TIME_LENGTH = 8,
    FTR_RANGE_TIME_DIFF_START = 9,
    FTR_RANGE_TIME_INSIDE = 10, 
    FTR_RANGE_LAST
} RangeFeatureTypes;
 
class RangeFeatGenerator : public FeatureGenerator {
private:
    // actual generators
    float uget_range_current(UniversalSigVec &usv, int updated_win_from, int updated_win_to, int time);
    float uget_range_latest(UniversalSigVec &usv, int updated_win_from, int updated_win_to, int time);
    float uget_range_min(UniversalSigVec &usv, int updated_win_from, int updated_win_to, int time);
    float uget_range_max(UniversalSigVec &usv, int updated_win_from, int updated_win_to, int time);
    float uget_range_ever(UniversalSigVec &usv, int updated_win_from, int updated_win_to, int time);
    float uget_range_time_diff(UniversalSigVec &usv, int updated_win_from, int updated_win_to, int time);
    float uget_range_recurrence_count(UniversalSigVec &usv, int updated_win_from, int updated_win_to, int time);
    float uget_range_time_covered(UniversalSigVec &usv, int updated_win_from, int updated_win_to, int time);
    float uget_range_last_nth_time_len(UniversalSigVec &usv, int updated_win_from, int updated_win_to, int time);
    float uget_range_time_diff_start(UniversalSigVec &usv, int updated_win_from, int updated_win_to, int time);
    float uget_range_time_inside(UniversalSigVec &usv, int updated_win_from, int updated_win_to, int time);
 
public:
 
    string signalName; 
    int signalId;
    vector<string> sets;                        
    RangeFeatureTypes type; 
    int win_from = 0; 
    int win_to = 360000;            
    int time_unit_win = MedTime::Undefined;         
    int time_unit_sig = MedTime::Undefined;     
    int val_channel = 0;                        
    int check_first = 1;                        
    float div_factor = 1.0f;                    
 
    vector<char> lut;                           
    int recurrence_delta = 30 * 24 * 60;        
    int min_range_time = -1;                    
    int N_th = 0;                               
    int zero_missing = 0;   
    int strict_times = 0;  
    int conditional_channel = -1; 
    bool regex_on_sets= false;        
 
    int first_evidence_time_channel = 1;     
 
    // Signal to determine allowed time-range (e.g. current stay/admission for inpatients)
    string timeRangeSignalName = "";
    int timeRangeSignalId;
    TimeRangeTypes timeRangeType = TIME_RANGE_CURRENT;
    int time_unit_range_sig = MedTime::Undefined;       
 
 
 
    // Constructor/Destructor
    RangeFeatGenerator() : FeatureGenerator() { init_defaults(); };
    //~RangeFeatGenerator() {};
    void set(string& _signalName, RangeFeatureTypes _type) { set(_signalName, _type, 0, 360000); req_signals.assign(1, signalName); }
    void set(string& _signalName, RangeFeatureTypes _type, int _time_win_from, int _time_win_to) {
        signalName = _signalName; type = _type; win_from = _time_win_from; win_to = _time_win_to;
        set_names(); req_signals.assign(1, signalName);
    }
 
    // Naming 
    void set_names();
 
    void get_required_signal_categories(unordered_map<string, vector<string>> &signal_categories_in_use) const;
 
    int init(map<string, string>& mapper);
    void init_defaults();
    RangeFeatureTypes name_to_type(const string &name); 
    void init_tables(MedDictionarySections& dict);
    // Copy
    virtual void copy(FeatureGenerator *generator) { *this = *(dynamic_cast<RangeFeatGenerator *>(generator)); }
 
    // Learn a generator
    int _learn(MedPidRepository& rep, const MedSamples& samples, vector<RepProcessor *> processors) { time_unit_sig = rep.sigs.Sid2Info[rep.sigs.sid(signalName)].time_unit; return 0; }
 
    // generate a new feature
    int _generate(PidDynamicRec& rec, MedFeatures& features, int index, int num, vector<float *> &_p_data);
    float get_value(PidDynamicRec& rec, int index, int date);
 
    // Signal Ids
    void set_signal_ids(MedSignals& sigs) { signalId = sigs.sid(signalName); }
 
 
    // Serialization
    ADD_CLASS_NAME(RangeFeatGenerator)
    ADD_SERIALIZATION_FUNCS(generator_type, signalName, type, win_from, win_to, val_channel, names, tags, req_signals, sets, check_first, timeRangeSignalName, timeRangeType, recurrence_delta, min_range_time,
            time_unit_sig, time_unit_win, div_factor, N_th, zero_missing, strict_times,conditional_channel, regex_on_sets, first_evidence_time_channel)
};
 
class ModelFeatGenerator : public FeatureGenerator {
public:
 
    string modelFile = ""; 
    MedModel *model = NULL; 
    string modelName = ""; 
    string model_json = ""; 
    string model_train_samples = ""; 
    bool ensure_patient_ids = true; 
    int n_preds = 1;  
    int impute_existing_feature = 0; 
    int use_overriden_predictions = 0; 
    int time_unit_win = global_default_windows_time_unit; 
    int time_unit_sig = global_default_windows_time_unit; 
    vector<int> times;
 
    void set_names();
 
    int init(map<string, string>& mapper);
    int init_from_model();
 
    void override_predictions(MedSamples& inSamples, MedSamples& modelSamples);
 
    void prepare(MedFeatures & features, MedPidRepository& rep, MedSamples& samples);
 
    int _learn(MedPidRepository& rep, const MedSamples& samples, vector<RepProcessor *> processors);
    int _generate(PidDynamicRec& rec, MedFeatures& features, int index, int num, vector<float *> &_p_data);
 
    void modifySampleTime(MedSamples& samples, int time);
 
    // (De)Serialize
    ADD_CLASS_NAME(ModelFeatGenerator)
        ADD_SERIALIZATION_HEADERS()
 
        //dctor:
        ~ModelFeatGenerator();
private:
    vector<vector<vector<float>>> preds;
};
 
 
typedef enum {
    FTR_TIME_YEAR = 0, 
    FTR_TIME_MONTH = 1, 
    FTR_TIME_DAY_IN_MONTH = 2, 
    FTR_TIME_DAY_IN_WEEK = 3, 
    FTR_TIME_HOUR = 4, 
    FTR_TIME_MINUTE = 5, 
    FTR_TIME_DATE = 6, 
    FTR_TIME_LAST,
} TimeFeatTypes;
 
 
class TimeFeatGenerator : public FeatureGenerator {
public:
 
    // Time Unit
    TimeFeatTypes time_unit = FTR_TIME_LAST;
 
    // Binning of time units
    vector<int> time_bins;
    vector<string> time_bin_names;
 
    // Constructor/Destructor
    TimeFeatGenerator() { generator_type = FTR_GEN_TIME; }
    ~TimeFeatGenerator() {}
 
    // Naming 
    void set_names();
 
    int init(map<string, string>& mapper);
    int get_time_unit(string name);
    int get_time_bins(string& binsInfo);
    int get_nBins();
    void set_default_bins();
    string time_unit_to_string(TimeFeatTypes time_unit);
 
    // Copy
    virtual void copy(FeatureGenerator *generator) { *this = *(dynamic_cast<TimeFeatGenerator *>(generator)); }
 
    // generate a new feature
    int _generate(PidDynamicRec& rec, MedFeatures& features, int index, int num, vector<float *> &_p_data);
 
    // Serialization
    ADD_CLASS_NAME(TimeFeatGenerator)
        ADD_SERIALIZATION_FUNCS(generator_type, names, time_unit, time_bins, time_bin_names, tags)
};
 
class AttrFeatGenerator : public FeatureGenerator {
public:
 
    // Attribute to use
    string attribute;
 
    // Feature name (if empty - use attribute)
    string ftr_name;
 
    // Constructor/Destructor
    AttrFeatGenerator() { generator_type = FTR_GEN_ATTR; }
    ~AttrFeatGenerator() {}
 
    // Naming 
    void set_names();
 
    int init(map<string, string>& mapper);
 
    // Copy
    virtual void copy(FeatureGenerator *generator) { *this = *(dynamic_cast<AttrFeatGenerator *>(generator)); }
 
    // generate a new feature
    int _generate(PidDynamicRec& rec, MedFeatures& features, int index, int num, vector<float *> &_p_data);
 
    // Serialization
    ADD_CLASS_NAME(AttrFeatGenerator);
    ADD_SERIALIZATION_FUNCS(generator_type, ftr_name, attribute, names);
};
 
enum class category_stat_test {
    chi_square = 1,
    mcnemar = 2
};
 
class CategoryDependencyGenerator : public FeatureGenerator {
private:
    int bdate_sid;
    int gender_sid;
    map<int, vector<string>> categoryId_to_name; //for regex filter
    map<int, vector<int>> _member2Sets; //for hierarchy
    map<int, vector<int>> _set2Members; //for hierarchy
    unordered_map<int, vector<int>> _member2Sets_flat_cache; //for hierarchy cache in get_parents
 
    vector<string> top_codes;
    vector<vector<char>> luts;
    vector<vector<char>> filter_luts;
    vector<int> filter_vals_idx; // stores filter indexes
    int input_sig_num_val_ch; // store num val channels for validation
 
    void get_parents(int codeGroup, vector<int> &parents, const boost::regex &reg_pat, const boost::regex & remove_reg_pat);
 
    void get_stats(const unordered_map<int, vector<vector<vector<int>>>> &categoryVal_to_stats,
        vector<int> &all_signal_values, vector<int> &signal_indexes, vector<double> &valCnts, vector<double> &posCnts,
        vector<double> &lift, vector<double> &scores, vector<double> &p_values, vector<double> &pos_ratio, vector<int> &dof, const vector<vector<double>> &prior_per_bin) const;
public:
    string signalName; 
    int signalId;
    int time_channel;                       
    int val_channel;                        
    int win_from;
    int win_to;
    int time_unit_win;          
    int min_age; 
    int max_age; 
    int age_bin; 
    string regex_filter; 
    string remove_regex_filter; 
    int min_code_cnt; 
    float fdr; 
    int take_top; 
    float lift_below; 
    float lift_above; 
    float filter_child_pval_diff; 
    float filter_child_lift_ratio; 
    float filter_child_count_ratio; 
    float filter_child_removed_ratio; 
    category_stat_test stat_metric; 
    float chi_square_at_least; 
    int minimal_chi_cnt; 
    int sort_by_chi = 0; 
    int max_depth; 
    int max_parents; 
    bool use_fixed_lift; 
    bool filter_hierarchy; 
    bool verbose; 
    bool verbose_full; 
    string verbose_full_file; 
    string feature_prefix; 
    bool generate_with_counts; 
    vector<vector<string>> filter_set_by_val_channel; 
    vector<string> filter_set_by_val_channel_names; 
 
    float male_regression_cntrl_lower; 
    float male_regression_cntrl_upper; 
    float male_regression_case_lower; 
    float male_regression_case_upper; 
    float female_regression_cntrl_lower; 
    float female_regression_cntrl_upper; 
    float female_regression_case_lower; 
    float female_regression_case_upper; 
 
    void set_signal_ids(MedSignals& sigs);
 
    void init_tables(MedDictionarySections& dict);
 
    // Constructor/Destructor
    CategoryDependencyGenerator() : FeatureGenerator() { init_defaults(); };
    void init_defaults();
 
    virtual void copy(FeatureGenerator *generator) { *this = *(dynamic_cast<CategoryDependencyGenerator *>(generator)); }
 
    int init(map<string, string>& mapper);
 
    int update(map<string, string>& mapper);
 
    void set_names();
    int filter_features(unordered_set<string>& validFeatures);
 
    int _learn(MedPidRepository& rep, const MedSamples& samples, vector<RepProcessor *> processors);
    int _generate(PidDynamicRec& rec, MedFeatures& features, int index, int num, vector<float *> &_p_data);
 
    int nfeatures();
 
    void get_required_signal_categories(unordered_map<string, vector<string>> &signal_categories_in_use) const;
 
    ADD_CLASS_NAME(CategoryDependencyGenerator)
    ADD_SERIALIZATION_FUNCS(generator_type, req_signals, top_codes, names, signalName, time_channel, val_channel, win_from, win_to, time_unit_win, 
        feature_prefix, generate_with_counts, tags, male_regression_cntrl_lower, male_regression_cntrl_upper, 
        male_regression_case_lower, male_regression_case_upper, female_regression_cntrl_lower, female_regression_cntrl_upper,
        female_regression_case_lower, female_regression_case_upper, filter_set_by_val_channel, filter_set_by_val_channel_names)
};
 
//=======================================
// Helpers
//=======================================
 
void get_window_in_sig_time(int _win_from, int _win_to, int _time_unit_win, int _time_unit_sig, int _win_time, int &_min_time, int &_max_time,
    bool boundOutcomeTime = false, int outcome_time = -1);
 
TimeRangeTypes time_range_name_to_type(const string& name);
string time_range_type_to_name(TimeRangeTypes type);
 
// update time window according to time-range signal
void get_updated_time_window(UniversalSigVec& time_range_usv, TimeRangeTypes type, int time_unit_range_sig, int time_unit_win, int time_unit_sig, int time,
    int win_from, int& updated_win_from, int win_to, int& updated_win_to, bool delta_win, int d_win_from, int& updated_d_win_from, int d_win_to, int& updated_d_win_to);
void get_updated_time_window(TimeRangeTypes type, int range_from, int range_to, int time, int _win_from, int _win_to, int& updated_win_from, int& updated_win_to);
 
 
//=======================================
// Joining the MedSerialze wagon
//=======================================
MEDSERIALIZE_SUPPORT(FeatureGenerator)
MEDSERIALIZE_SUPPORT(BasicFeatGenerator)
MEDSERIALIZE_SUPPORT(AgeGenerator)
MEDSERIALIZE_SUPPORT(GenderGenerator)
MEDSERIALIZE_SUPPORT(SingletonGenerator)
MEDSERIALIZE_SUPPORT(BinnedLmEstimatesParams)
MEDSERIALIZE_SUPPORT(BinnedLmEstimates)
MEDSERIALIZE_SUPPORT(RangeFeatGenerator)
MEDSERIALIZE_SUPPORT(ModelFeatGenerator)
 
#endif