MedFeatures.h

// MedFeatures - holding data as a map from name to vector
 
#ifndef __MED_FEATURES_H__
#define __MED_FEATURES_H__
 
#include <InfraMed/InfraMed/InfraMed.h>
#include <SerializableObject/SerializableObject/SerializableObject.h>
#include <MedProcessTools/MedProcessTools/MedProcessUtils.h>
#include <MedProcessTools/MedProcessTools/MedSamples.h>
#include <MedMat/MedMat/MedMat.h>
#include <random>
 
//.......................................................................................
//  A structure holding feature attributes
//.......................................................................................
class FeatureAttr : public SerializableObject {
public:
    bool normalized = false; 
    bool imputed = false; 
    float denorm_mean = 0.0, denorm_sdv = 1.0; 
 
    unordered_map<float, string> value2Name; 
 
    // Serialization
    ADD_CLASS_NAME(FeatureAttr)
    ADD_SERIALIZATION_FUNCS(normalized, imputed , denorm_mean, denorm_sdv, value2Name)
};
 
//.......................................................................................
//.......................................................................................
class MedFeatures final : public SerializableObject {
public:
 
    // Data
    map<string, vector<float> > data; 
    vector<float> weights; 
    vector<MedSample> samples; 
 
    map<int, pair<int, int>> pid_pos_len;
 
    // Attributes
    map<string, FeatureAttr> attributes; 
    map<string, unordered_set<string> > tags; 
 
    // masks for cleaning , imputing
    const static unsigned char cleaned_mask = (unsigned char)0x01;
    const static unsigned char imputed_mask = (unsigned char)0x02;
    map<string, vector<unsigned char>> masks;
    float medf_missing_value = (float)MED_MAT_MISSING_VALUE;
 
    // time Unit
    int time_unit; 
 
    static int global_serial_id_cnt;
 
    // Functions
 
    MedFeatures(int _time_unit) { time_unit = _time_unit; }
    MedFeatures() { time_unit = global_default_time_unit; global_serial_id_cnt = 0; }
 
    // Initialization
    void clear() { data.clear(); samples.clear(); pid_pos_len.clear(); attributes.clear(); weights.clear(); tags.clear(); masks.clear(); }
    void set_time_unit(int _time_unit) { time_unit = _time_unit; }
 
    void get_feature_names(vector<string>& names) const;
    void get_as_matrix(MedMat<float>& mat) const;
    void get_as_matrix(MedMat<float>& mat, vector<string>& names) const;
    void get_as_matrix(MedMat<float>& mat, const vector<string>& names, vector<int> &idx) const;
 
    void set_as_matrix(const MedMat<float>& mat);
 
    void append_samples(MedIdSamples& in_samples);
    void insert_samples(MedIdSamples& in_samples, int index);
    void init_all_samples(vector<MedIdSamples> &in_samples) { samples.clear(); for (auto& id : in_samples) append_samples(id); init_pid_pos_len(); }
    void init_pid_pos_len();
    int get_pid_pos(int pid) const { if (pid_pos_len.find(pid) == pid_pos_len.end()) return -1; return (pid_pos_len.at(pid).first); }
    int get_pid_len(int pid) const { if (pid_pos_len.find(pid) == pid_pos_len.end()) return 0; return (pid_pos_len.at(pid).second); }
    unsigned int get_crc();
    void print_csv() const;
    void get_samples(MedSamples& outSamples) const;
    int get_max_serial_id_cnt() const;
 
    int write_as_csv_mat(const string &csv_fname, bool write_attributes = false) const;
    int add_to_csv_mat(const string &csv_fname, bool write_attributes, int start_idx) const;
    void write_csv_data(ofstream& out_f, bool write_attributes, vector<string>& col_names, int start_idx) const;
 
    int read_from_csv_mat(const string &csv_fname, bool read_time_raw = true);
 
    int filter(unordered_set<string>& selectedFeatures);
 
    int prep_selected_list(vector<string>& search_str, unordered_set<string> &selected);
 
    // masks functions
    int init_masks();
    int get_masks_as_mat(MedMat<unsigned char> &masks_mat);
    int mark_imputed_in_masks(float _missing_val);
    int mark_imputed_in_masks() { return mark_imputed_in_masks(medf_missing_value); }
 
    void round_data(float r);
    void noise_data(float r);
 
    void samples_sort();
 
    string resolve_name(string& substr) const;
 
    // Serialization
    ADD_CLASS_NAME(MedFeatures)
    ADD_SERIALIZATION_FUNCS(data, weights, samples, attributes, tags, time_unit)
 
};
 
MEDSERIALIZE_SUPPORT(MedFeatures)
MEDSERIALIZE_SUPPORT(FeatureAttr)
 
 
namespace medial {
    namespace process {
        template<class T> void commit_selection(vector<T> &vec, const vector<int> &idx);
        void filter_row_indexes(MedFeatures &dataMat, vector<int> &selected_indexes, bool op_flag = false);
        void filter_row_indexes_safe(MedFeatures &dataMat, const vector<int> &selected_indexes, bool op_flag = false);
        void down_sample(MedFeatures &dataMat, double take_ratio, bool with_repeats = false,
            vector<int> *selected_indexes = NULL);
        double reweight_by_general(MedFeatures &data_records, const vector<string> &groups,
            vector<float> &weigths, bool print_verbose);
        void match_by_general(MedFeatures &data_records, const vector<string> &groups,
            vector<int> &filtered_row_ids, float price_ratio, int min_grp_size, bool print_verbose);
        void match_by_general(MedFeatures &data_records, const vector<string> &groups,
            vector<int> &filtered_row_ids, float price_ratio, float max_ratio, int min_grp_size, bool print_verbose);
 
        void split_matrix(const MedFeatures& matrix, vector<int>& folds, int iFold,
            MedFeatures& trainMatrix, MedFeatures& testMatrix, const vector<string> *selected_features = NULL);
        void split_matrix(const MedFeatures& matrix, unordered_map<int, int>& folds, int iFold,
            MedFeatures& trainMatrix, MedFeatures& testMatrix, const vector<string> *selected_features = NULL);
        void convert_prctile(vector<float> &features_prctiles);
        void match_to_prior(const vector<float> &outcome,
            const vector<float> &group_values, float target_prior, vector<int> &sel_idx);
 
        double match_to_prior(MedSamples &samples, float target_prior, vector<int> &sel_idx);
 
        double match_to_prior(MedFeatures &features, float target_prior, vector<int> &sel_idx);
 
        void match_to_prior(MedFeatures &features,
            const vector<string> &group_values, float target_prior, vector<int> &sel_idx, bool print_verbose = true);
 
        int nSplits(vector<MedSample>& samples);
 
        float match_multi_class(MedFeatures& data, const vector<string> &groups, vector<int> &filtered_row_ids, vector<float>& price_ratios, int nRand = 10000, int verbose = false);
        float match_multi_class(vector<MedSample>& data, const vector<string> &groups, vector<int> &filtered_row_ids, vector<float>& price_ratios, int nRand = 10000, int verbose = false);
 
        void match_multi_class_to_dist(MedFeatures& data, const vector<string> &groups, vector<int> &filtered_row_ids, vector<float> probs);
        void match_multi_class_to_dist(vector<MedSample>& data, const vector<string> &groups, vector<int> &filtered_row_ids, vector<float> probs);
 
    }
 
}
 
#endif