MedSamples.h

// Classes for holding samples
 
#ifndef __MED_SAMPLES_H__
#define __MED_SAMPLES_H__
 
 
#include <SerializableObject/SerializableObject/SerializableObject.h>
#include <MedTime/MedTime/MedTime.h>
#include <unordered_set>
#include <nlohmann/json.hpp>
 
using json = nlohmann::json;
 
class MedFeatures;
 
//.......................................................................................
//.......................................................................................
 
class MedSample : public SerializableObject {
public:
    int id = -1;            
    int split = -1;         
    int time = 0;           
    float outcome = 0;      
    int outcomeTime = 0;    
    vector<float> prediction;   
    map<string, float> attributes;  
    map<string, string> str_attributes; 
    json jrec;  
 
    MedSample() { prediction.clear(); }
    MedSample(int _id, int _time, int _outcome, int _outcomeTime) { id = _id; time = _time; outcome = _outcome; outcomeTime = _outcomeTime; prediction.clear(); }
    ~MedSample() { prediction.clear(); }
 
    void print(const string prefix);
 
    void print() { print(""); }
 
    int parse_from_string(string &s, int time_unit);
    int parse_from_string(string &s, const map <string, int> & pos, const vector<int>& pred_pos, const map<string, int>& attr_pos,
        const map<string, int>& str_attr_pos, int time_unit, int raw_format, const string &delimeter = "\t");
    int parse_from_string(const vector<string> &fields, const map<string, int> & pos, const vector<int>& pred_pos, const map<string, int>& attr_pos,
        const map<string, int>& str_attr_pos, int time_unit, int raw_format, const string &delimeter);
 
    void write_to_string(string &s, int time_unit, bool write_attrib = true, const string &delimeter = "\t", int pred_precision = -1) const;
 
    int get_all_attributes(vector<string>& attributes, vector<string>& str_attributes) const;
 
    // Serialization
    ADD_CLASS_NAME(MedSample)
        ADD_SERIALIZATION_FUNCS(id, split, time, outcome, outcomeTime, prediction, attributes, str_attributes)
};
 
inline bool comp_sample_pred(const MedSample &pr1, const MedSample &pr2) {
    return pr1.prediction[0] < pr2.prediction[0];
}
 
inline bool comp_sample_id_time(const MedSample &pr1, const MedSample &pr2) {
    if (pr1.id == pr2.id)
        return pr1.time < pr2.time;
    else
        return pr1.id < pr2.id;
}
 
//.......................................................................................
//.......................................................................................
class MedIdSamples : public SerializableObject {
public:
    int id = -1;        
    int split = -1;
    vector<MedSample> samples;      
 
    MedIdSamples(int _id) { id = _id; split = -1; samples.clear(); }
    MedIdSamples() { id = -1; split = -1; samples.clear(); }
 
    void set_split(int _split) { split = _split; for (auto& s : samples) s.split = _split; }
 
    bool same_as(MedIdSamples &other, int mode);
 
    void get_times(vector<int> &times) const { times.clear(); for (auto &s : samples) times.push_back(s.time); }
 
    // Serialization
    ADD_CLASS_NAME(MedIdSamples)
        ADD_SERIALIZATION_FUNCS(id, split, samples)
 
};
 
inline bool comp_patient_id_time(const MedIdSamples &pr1, const MedIdSamples &pr2) {
    return pr1.id < pr2.id;
}
 
//.......................................................................................
//.......................................................................................
 
class MedSamples final : public SerializableObject {
public:
    int time_unit = MedTime::Date;  
    vector<MedIdSamples> idSamples; 
    int raw_format = 0; // read times as is, no conversions
 
    MedSamples() { time_unit = global_default_time_unit; }
    ~MedSamples() {}
 
    void clear() { time_unit = global_default_time_unit; idSamples.clear(); }
 
    int insert_preds(MedFeatures& featuresData);
 
    int copy_attributes(const vector<MedSample>& samples);
 
    void get_ids(vector<int>& ids) const;
 
    void append(MedSamples& newSamples) { idSamples.insert(idSamples.end(), newSamples.idSamples.begin(), newSamples.idSamples.end()); }
 
    int read_from_bin_file(const string& file_name) { return SerializableObject::read_from_file(file_name); }
    int write_to_bin_file(const string& file_name) { return SerializableObject::write_to_file(file_name); }
 
    int read_from_file(const string& file_name, bool sort_rows = true);
 
    int write_to_file(const string &fname, int pred_precision=-1, bool print_attributes =true);
    void write_to_file(ofstream& of, int pred_precision, bool print_attributes, bool print_header);
 
    void get_preds(vector<float>& preds) const;
    void get_preds_channel(vector<float>& preds, int channel);
    void get_y(vector<float>& y) const;
    void get_categs(vector<float> &categs) const;
    void get_attr_values(const string& attr_name, vector<float>& values) const;
    void export_to_sample_vec(vector<MedSample> &vec_samples) const;
    void import_from_sample_vec(const vector<MedSample> &vec_samples, bool allow_split_inconsistency = false);
 
    void sort_by_id_date();
    void normalize();
 
    bool same_as(MedSamples &other, int mode);
 
    int nSamples() const;
 
    int nSplits();
 
    int get_predictions_size(int& nPreds);
 
    int get_all_attributes(vector<string>& attributes, vector<string>& str_attributes) const;
 
    void dilute(float prob);
 
    void binary_dilute(float p0, float p1);
 
    void subtract(MedSamples &_dont_include);
 
    void split_train_test(MedSamples &train, MedSamples &test, float p_test);
 
    void split_by_split(MedSamples &in_split, MedSamples &off_split, int split);
 
    void add_splits_from_file(string f_splits);
 
    void init_all_jrecs();
 
 
    void flatten(vector<MedSample> &flat) const;
 
    void insertRec(int pid, int time, float outcome, int outcomeTime);
    void insertRec(int pid, int time, float outcome, int outcomeTime, float pred);
    void insertRec(int pid, int time) { insertRec(pid, time, -1, 0); }
 
    //Serialization, version 1: Added version, model_features, features_count to serialization
    //               version 2: Added attributes
    ADD_CLASS_NAME(MedSamples)
        ADD_SERIALIZATION_FUNCS(time_unit, idSamples)
};
 
namespace medial {
    namespace print {
        void print_samples_stats(const vector<MedSample> &samples, const string &log_file = "");
        void print_samples_stats(const MedSamples &samples, const string &log_file = "");
        void print_by(const vector<MedSample> &data_records, const vector<string> &groups,
            bool unique_ids = false, const string &log_file = "");
        void print_by(const MedSamples &data_records, const vector<string> &groups,
            bool unique_ids = false, const string &log_file = "");
        void print_by_year(const vector<MedSample> &data_records, int year_bin_size, bool unique_ids = false,
            bool take_prediction_time = true, const string &log_file = "");
        void print_by_year(const MedSamples &data_records, int year_bin_size, bool unique_ids = false,
            bool take_prediction_time = true, const string &log_file = "");
    }
    namespace process {
        void down_sample(MedSamples &samples, double take_ratio, bool with_repeats = false);
        void down_sample_by_pid(MedSamples &samples, double take_ratio, bool with_repeats = false);
 
        void down_sample(MedSamples &samples, int no_more_than, bool with_repeats = false);
        void down_sample_by_pid(MedSamples &samples, int no_more_than, bool with_repeats = false);
    }
 
    namespace stats {
        double kaplan_meir_on_samples(const MedSamples &incidence_samples, int time_period, const vector<pair<int, int>> *filtered_idx = NULL);
        double kaplan_meir_on_samples(const vector<MedSample> &incidence_samples, int time_unit, int time_period, const vector<int> *filtered_idx = NULL);
    }
}
 
//=======================================
// Joining the MedSerialze wagon
//=======================================
MEDSERIALIZE_SUPPORT(MedSample)
MEDSERIALIZE_SUPPORT(MedIdSamples)
MEDSERIALIZE_SUPPORT(MedSamples)
 
#endif