RepProcess.h

Go to the documentation of this file.

#ifndef _REP_PROCESS_H_
#define _REP_PROCESS_H_
 
#include "InfraMed/InfraMed/InfraMed.h"
#include "InfraMed/InfraMed/MedPidRepository.h"
#include "MedProcessTools/MedProcessTools/MedSamples.h"
#include "MedProcessTools/MedProcessTools/MedProcessUtils.h"
#include <SerializableObject/SerializableObject/SerializableObject.h>
#include "MedProcessTools/MedProcessTools/MedValueCleaner.h"
#include <MedMat/MedMat/MedMat.h>
#include <omp.h>
#include <cmath>
 
#define DEFAULT_REP_CLNR_NTHREADS 8
 
//.......................................................................................
//.......................................................................................
typedef enum {
    REP_PROCESS_MULTI, 
    REP_PROCESS_BASIC_OUTLIER_CLEANER,
    REP_PROCESS_NBRS_OUTLIER_CLEANER,
    REP_PROCESS_CONFIGURED_OUTLIER_CLEANER,
    REP_PROCESS_RULEBASED_OUTLIER_CLEANER,
    REP_PROCESS_CALC_SIGNALS,
    REP_PROCESS_COMPLETE, 
    REP_PROCESS_CHECK_REQ, 
    REP_PROCESS_SIM_VAL, 
    REP_PROCESS_SIGNAL_RATE, 
    REP_PROCESS_COMBINE, 
    REP_PROCESS_SPLIT, 
    REP_PROCESS_AGGREGATION_PERIOD, 
    REP_PROCESS_BASIC_RANGE_CLEANER,
    REP_PROCESS_AGGREGATE, 
    REP_PROCESS_HISTORY_LIMIT, 
    REP_PROCESS_CREATE_REGISTRY, 
    REP_PROCESS_CREATE_BIT_SIGNAL, 
    REP_PROCESS_CATEGORY_DESCENDERS, 
    REP_PROCESS_REODER_CHANNELS, 
    REP_PROCESS_FILTER_BY_CHANNELS, 
    REP_PROCESS_NUMERIC_NOISER, 
    REP_PROCESS_FILTER_BY_DIAG, 
    REP_PROCESS_LAST
} RepProcessorTypes;
 
class RepProcessor : public SerializableObject {
public:
 
    RepProcessorTypes processor_type = REP_PROCESS_LAST; 
 
    unordered_set<string> req_signals; 
    unordered_set<int> req_signal_ids; 
 
    unordered_set<string> aff_signals; 
    unordered_set<int> aff_signal_ids; 
    bool unconditional = false; 
 
    vector<string> attributes; 
 
    virtual ~RepProcessor() { clear(); }
    virtual void clear() {  };
 
    vector<pair<string, int>> virtual_signals; 
    vector<pair<string, string>> virtual_signals_generic;
 
    // create a new rep_processor
    static RepProcessor *make_processor(string name);
    static RepProcessor *make_processor(string type, string params);
    static RepProcessor *make_processor(RepProcessorTypes type);
    static RepProcessor *make_processor(RepProcessorTypes type, string params);
 
    static RepProcessor *create_processor(string &params);
 
    virtual int init(void *params) { return 0; };
    virtual int init(map<string, string>& mapper) { return 0; };
    virtual void init_defaults() {};
 
    virtual void set_signal(const string& _signalName) { return; };
 
    virtual void set_signal_ids(MedSignals& sigs) { return; }
 
    // Required Signals functions : get all signals that are required by the processor
    virtual void get_required_signal_names(unordered_set<string>& signalNames);
    // Required Signals functions : get all signals that are required by the processor
    virtual void get_required_signal_names(unordered_set<string>& signalNames, unordered_set<string> preReqSignals);
 
    virtual void set_required_signal_ids(MedDictionarySections& dict);
 
    virtual void add_virtual_signals(map<string, int> &_virtual_signals, map<string, string> &_virtual_signals_generic) const {
        for (auto &v : virtual_signals) _virtual_signals[v.first] = v.second;
        for (auto &v : virtual_signals_generic) _virtual_signals_generic[v.first] = v.second;
    };
 
    // Required Signals functions : get all signals that are required by the processor
    virtual void get_required_signal_ids(unordered_set<int>& signalIds);
    // Required Signals functions : get all signals that are required by the processor
    virtual void get_required_signal_ids(unordered_set<int>& signalIds, unordered_set<int> preReqSignals);
 
    // Affected Signals functions;
    virtual void set_affected_signal_ids(MedDictionarySections& dict);
    inline bool is_signal_affected(int signalId) { return (aff_signal_ids.find(signalId) != aff_signal_ids.end()); }
    inline bool is_signal_affected(string& signalName) { return (aff_signals.find(signalName) != aff_signals.end()); }
 
    virtual void fit_for_repository(MedPidRepository& rep) {};
 
    virtual void register_virtual_section_name_id(MedDictionarySections& dict) { };
 
    // check filtering
    virtual bool filter(unordered_set<string>& reqSignals);
 
    virtual void init_tables(MedDictionarySections& dict, MedSignals& sigs) { return; }
 
    virtual void init_attributes() { return; }
 
    // Learning
    // Should be implemented for inheriting classes that require learning </summary>
    virtual int _learn(MedPidRepository& rep, MedSamples& samples, vector<RepProcessor *>& prev_processors) { return 0; };
    // May be implemented for inheriting classes that require learning </summary>
    virtual int _conditional_learn(MedPidRepository& rep, MedSamples& samples, vector<RepProcessor *>& prev_processors, unordered_set<int>& neededSignalIds);
 
    // Learning envelopes - Here because of issues with overloading and inheritance
    int learn(MedPidRepository& rep, MedSamples& samples, vector<RepProcessor *>& prev_processors) { return _learn(rep, samples, prev_processors); };
    int learn(MedPidRepository& rep);
    int learn(MedPidRepository& rep, MedSamples& samples) { vector<RepProcessor *> temp;  return _learn(rep, samples, temp); }
    virtual int conditional_learn(MedPidRepository& rep, MedSamples& samples, vector<RepProcessor *>& prev_processors, unordered_set<int>& neededSignalIds) {
        return _conditional_learn(rep, samples, prev_processors, neededSignalIds);
    }
    int conditional_learn(MedPidRepository& rep, MedSamples& samples, unordered_set<int>& neededSignalIds) { vector<RepProcessor *> temp;  return _conditional_learn(rep, samples, temp, neededSignalIds); }
 
    // Applying
    virtual int _apply(PidDynamicRec& rec, vector<int>& time_points, vector<vector<float>>& attributes_vals) { return -1; };
    virtual int _conditional_apply(PidDynamicRec& rec, vector<int>& time_points, unordered_set<int>& neededSignalIds, vector<vector<float>>& attributes_vals);
 
    // next is needed for efficient applying of a single records
    virtual int _apply_simple(PidDynamicRec& rec, vector<int>& time_points) {
        vector<vector<float>> attributes_vals;
        return _apply(rec, time_points, attributes_vals);
    }
 
    // Applying envelopes - Here because of issues with overloading and inheritance
    int apply(PidDynamicRec& rec, vector<int>& time_points, vector<vector<float>>& attributes_vals) { return _apply(rec, time_points, attributes_vals); }
    int conditional_apply(PidDynamicRec& rec, vector<int>& time_points, unordered_set<int>& neededSignalIds, vector<vector<float>>& attributes_vals) {
        return _conditional_apply(rec, time_points, neededSignalIds, attributes_vals);
    }
    int apply(PidDynamicRec& rec, MedIdSamples& samples);
    int conditional_apply(PidDynamicRec& rec, MedIdSamples& samples, unordered_set<int>& neededSignalIds);
    int conditional_apply_without_attributes(PidDynamicRec& rec, const MedIdSamples& samples, unordered_set<int>& neededSignalIds);
 
 
    // debug prints
    virtual void dprint(const string &pref, int rp_flag);
 
    virtual void make_summary() {};
 
    virtual void get_required_signal_categories(unordered_map<string, vector<string>> &signal_categories_in_use) const {};
 
    // Serialization (including type)
    ADD_CLASS_NAME(RepProcessor)
        ADD_SERIALIZATION_FUNCS(processor_type, req_signals, aff_signals, unconditional)
        void *new_polymorphic(string derived_class_name);
 
    size_t get_processor_size();
    size_t processor_serialize(unsigned char *blob);
 
    virtual void print() { fprintf(stderr, "No implementation for print()\n"); }
};
 
// Utilities
RepProcessorTypes rep_processor_name_to_type(const string& procesor_name);
 
//.......................................................................................
//.......................................................................................
class RepMultiProcessor : public RepProcessor {
public:
    vector<RepProcessor *> processors; 
 
    vector<vector<int>> attributes_map; 
 
    RepMultiProcessor() { processor_type = REP_PROCESS_MULTI; };
    ~RepMultiProcessor() { clear(); };
 
    void clear();
 
 
    void add_processors_set(RepProcessorTypes type, vector<string>& signals);
    void add_processors_set(RepProcessorTypes type, vector<string>& signals, string init_string);
    void set_required_signal_ids(MedDictionarySections& dict);
    void add_virtual_signals(map<string, int> &_virtual_signals, map<string, string> &_virtual_signals_generic) const;
 
    void get_required_signal_names(unordered_set<string>& signalNames);
    virtual void get_required_signal_names(unordered_set<string>& signalNames, unordered_set<string> preReqSignals);
 
    void get_required_signal_ids(unordered_set<int>& signalIds);
    virtual void get_required_signal_ids(unordered_set<int>& signalIds, unordered_set<int> preReqSignals);
 
    void set_affected_signal_ids(MedDictionarySections& dict);
 
 
    void register_virtual_section_name_id(MedDictionarySections& dict);
    // check filtering
    bool filter(unordered_set<string>& reqSignals);
 
    void set_signal_ids(MedSignals& sigs);
 
    void init_tables(MedDictionarySections& dict, MedSignals& sigs) { for (RepProcessor * proc : processors) { proc->init_tables(dict, sigs); } }
 
    void init_attributes();
 
    int _learn(MedPidRepository& rep, MedSamples& samples, vector<RepProcessor *>& prev_processors);
    int _conditional_learn(MedPidRepository& rep, MedSamples& samples, vector<RepProcessor *>& prev_processors, unordered_set<int>& neededSignalIds);
 
    int _apply(PidDynamicRec& rec, vector<int>& time_points, vector<vector<float>>& attributes_vals);
    int _apply_simple(PidDynamicRec& rec, vector<int>& time_points);
    int _conditional_apply(PidDynamicRec& rec, vector<int>& time_points, unordered_set<int>& neededSignals, vector<vector<float>>& attributes_mat);
 
    void fit_for_repository(MedPidRepository& rep);
 
    void dprint(const string &pref, int rp_flag);
 
    void make_summary();
 
    void get_required_signal_categories(unordered_map<string, vector<string>> &signal_categories_in_use) const;
 
    ADD_CLASS_NAME(RepMultiProcessor)
        ADD_SERIALIZATION_FUNCS(processor_type, processors)
 
        
        void print() { for (auto& processor : processors) processor->print(); }
};
 
#define DEF_REP_TRIMMING_SD_NUM 7
#define DEF_REP_REMOVING_SD_NUM 14
 
class remove_stats {
public:
    int total_removed = 0, total_pids_touched = 0;
    int total_records = 0, total_pids = 0;
 
    void restart();
 
    void print_summary(const string &cleaner_info, const string &signal_name,
        int minimal_pid_cnt, float print_summary_critical_cleaned, bool prnt_flg) const;
};
 
//.......................................................................................
//.......................................................................................
class RepBasicOutlierCleaner : public RepProcessor, public MedValueCleaner {
private:
    ofstream log_file;
    remove_stats _stats;
    bool is_categ = false;
public:
 
    string signalName;  
    int signalId;   
    int time_channel = 0; 
    int val_channel = 0; 
 
    string nRem_attr = ""; 
    string nTrim_attr = ""; 
    string nRem_attr_suffix = ""; 
    string nTrim_attr_suffix = ""; 
    string verbose_file; 
 
    bool print_summary = false; 
    float print_summary_critical_cleaned = (float)0.05; 
 
    RepBasicOutlierCleaner() { init_defaults(); }
    RepBasicOutlierCleaner(const string& _signalName) { init_defaults(); signalId = -1; signalName = _signalName; init_lists(); }
    RepBasicOutlierCleaner(const string& _signalName, string init_string) { init_defaults(); signalId = -1; signalName = _signalName; init_from_string(init_string); }
    RepBasicOutlierCleaner(const string& _signalName, ValueCleanerParams *_params) { init_defaults(); signalId = -1; signalName = _signalName; init_lists(); MedValueCleaner::init(_params); }
 
    void init_defaults() {
        processor_type = REP_PROCESS_BASIC_OUTLIER_CLEANER;
        params.trimming_sd_num = DEF_REP_TRIMMING_SD_NUM; params.removing_sd_num = DEF_REP_REMOVING_SD_NUM; params.nbrs_sd_num = 0;
        params.take_log = 0;
        params.doTrim = params.doRemove = true;
        signalId = -1;
        params.type = VAL_CLNR_ITERATIVE;
        params.missing_value = MED_MAT_MISSING_VALUE;
        verbose_file = "";
    };
 
    void set_signal(const string& _signalName) { signalId = -1; signalName = _signalName; init_lists(); }
 
    virtual void set_signal_ids(MedSignals& sigs);
 
    int init(void *processor_params) { return MedValueCleaner::init(processor_params); };
    virtual int init(map<string, string>& mapper);
    void init_lists();
 
    void init_attributes();
 
    int _learn(MedPidRepository& rep, MedSamples& samples, vector<RepProcessor *>& prev_processor);
    int iterativeLearn(MedPidRepository& rep, MedSamples& samples, vector<RepProcessor *>& prev_processor);
    int quantileLearn(MedPidRepository& rep, MedSamples& samples, vector<RepProcessor *>& prev_processor);
 
    int _apply(PidDynamicRec& rec, vector<int>& time_points, vector<vector<float>>& attributes_mat);
 
    virtual ~RepBasicOutlierCleaner() { if (!verbose_file.empty() && log_file.is_open()) log_file.close(); };
 
    void print() { dprint("", 1); }
    void dprint(const string &pref, int rp_flag);
 
    void make_summary();
 
    ADD_CLASS_NAME(RepBasicOutlierCleaner)
        ADD_SERIALIZATION_FUNCS(processor_type, signalName, time_channel, val_channel, req_signals, aff_signals, params.take_log, params.missing_value, params.doTrim, params.doRemove,
            trimMax, trimMin, removeMax, removeMin, nRem_attr, nTrim_attr, nRem_attr_suffix, nTrim_attr_suffix, verbose_file,
            print_summary, print_summary_critical_cleaned)
};
 
//.......................................................................................
class confRecord : public SerializableObject {
public:
    double logicalLow, logicalHigh, confirmedLow, confirmedHigh;
    double trimLow, trimHigh; //<trimming if not above remove borders
    string distLow, distHigh; //"none" "norm" or "log" 
    int val_channel = 0;
    ADD_CLASS_NAME(confRecord)
        ADD_SERIALIZATION_FUNCS(logicalLow, logicalHigh, confirmedLow, confirmedHigh,
            distLow, distHigh, trimLow, trimHigh, val_channel)
};
MEDSERIALIZE_SUPPORT(confRecord)
 
//.......................................................................................
//.......................................................................................
class RepConfiguredOutlierCleaner : public  RepBasicOutlierCleaner {
private:
    string confFileName; 
    confRecord outlierParam; 
public:
    string cleanMethod; 
 
    RepConfiguredOutlierCleaner() { init_defaults(); }
 
    void init_defaults() {
        processor_type = REP_PROCESS_CONFIGURED_OUTLIER_CLEANER;
        params.trimming_sd_num = DEF_REP_TRIMMING_SD_NUM; params.removing_sd_num = DEF_REP_REMOVING_SD_NUM; params.nbrs_sd_num = 0;
        params.take_log = 0;
        params.doTrim = params.doRemove = true;
        signalId = -1;
        params.type = VAL_CLNR_ITERATIVE;
        params.missing_value = MED_MAT_MISSING_VALUE;
        verbose_file = "";
    };
 
    int _learn(MedPidRepository& rep, MedSamples& samples, vector<RepProcessor *>& prev_processor);
 
    int init(map<string, string>& mapper);
 
    // Apply cleaning model -inheritted
    void set_signal_ids(MedSignals& sigs);
 
    ADD_CLASS_NAME(RepConfiguredOutlierCleaner)
        ADD_SERIALIZATION_FUNCS(processor_type, signalName, time_channel, val_channel, req_signals, aff_signals, params.take_log, params.missing_value, params.doTrim, params.doRemove,
            trimMax, trimMin, removeMax, removeMin, confFileName, cleanMethod, outlierParam, nRem_attr, nTrim_attr, nRem_attr_suffix, nTrim_attr_suffix)
 
        void print();
};
 
void learnDistributionBorders(float& borderHi, float& borderLo, vector<float> filteredValues);
// a function that takes sorted vector of filtered values and estimates the +- 7 sd borders based on the center of distribution
// predefined calibration constants are used for estimation of the borders. 
 
 
class RepRuleBasedOutlierCleaner : public RepProcessor {
    // get multiple signals and clean them according to consistency  with other signals from same date
public:
 
    vector <int> signalIds;
    vector<int> consideredRules;
 
    string nRem_attr = ""; 
    string nRem_attr_suffix = ""; 
 
    float tolerance = 0.1F;
    int time_window = 0; 
    string verbose_file; 
    float calc_res = 0; 
 
    bool print_summary = false; 
    float print_summary_critical_cleaned = (float)0.05; 
 
    map <int, vector<string>>rules2Signals = {
    {1,{"BMI","Weight","Height"}},
    {2,{"MCH", "Hemoglobin","RBC"}},
    {3,{"MCV","Hematocrit","RBC"} },
    {4,{"MCHC-M","MCH","MCV"}},
    {5,{"Eosinophils#","Monocytes#","Basophils#","Lymphocytes#","Neutrophils#","WBC" }},
    {6,{ "MPV","Platelets_Hematocrit","Platelets" }},
    {7,{"UrineAlbumin","UrineTotalProtein" }},
    {8,{"UrineAlbumin_over_Creatinine","UrineAlbumin","UrineCreatinine" }},
    {9,{"LDL","HDL","Cholesterol"}},
    {10,{"NonHDLCholesterol","HDL","Cholesterol"}},
    {11,{"HDL_over_nonHDL","HDL","NonHDLCholesterol"}},
    {12,{"HDL_over_Cholesterol","HDL","Cholesterol"}},
    {13,{"HDL_over_LDL","HDL","LDL"}},
    {14,{"HDL_over_LDL","LDL_over_HDL"}},
    {15,{"Cholesterol_over_HDL","Cholesterol","HDL"}},
    {17,{"Cholesterol_over_HDL","HDL_over_Cholesterol"}}, //rule 16 canceled
    {18,{"LDL_over_HDL","LDL","HDL"}},
    {19,{"Albumin","Protein_Total"}},
    {20,{"FreeT4","T4"}},
    {21,{"NRBC","RBC"}},
    {22,{"CHADS2","CHADS2_VASC"}},
    {23, {"BP"}}
    };
 
    map<int, string> rules2RemoveSignal; 
 
    vector <int> rulesToApply;
    unordered_map<string, pair<int, int>> signal_channels; 
    unordered_map<int, pair<int, int>> signal_id_channels; 
 
    set <int> reqSignalIds, affSignalIds;
    unordered_map<int, vector<int>> rules_sids;
    unordered_map<int, vector<bool>> affected_by_rules;
 
    // Constructors 
    RepRuleBasedOutlierCleaner() : RepProcessor() { init_defaults(); }
 
    void init_defaults();
 
    void parse_rules_signals(const string &path);
    void parse_sig_channels(const string &path);
 
 
    // Init
    int init(void *processor_params) { return 0; };
 
    int init(map<string, string>& mapper);
 
    void init_attributes();
 
    void set_signal_ids(MedSignals& sigs);
 
    void init_tables(MedDictionarySections& dict, MedSignals& sigs);
 
    // Learning
    int _learn(MedPidRepository& rep, MedSamples& samples, vector<RepProcessor *>& prev_processors) { init_tables(rep.dict, rep.sigs); return 0; };
 
    int _apply(PidDynamicRec& rec, vector<int>& time_points, vector<vector<float>>& attributes_mat);
 
    ~RepRuleBasedOutlierCleaner() { if (!verbose_file.empty() && log_file.is_open()) log_file.close(); };
 
    // Check if some required signals are missing and remove corresponding rules
    void fit_for_repository(MedPidRepository& rep);
 
    // set affected and required signals lists according to rules
    void init_lists();
 
    void make_summary();
 
    void dprint(const string &pref, int rp_flag);
 
    ADD_CLASS_NAME(RepRuleBasedOutlierCleaner)
        ADD_SERIALIZATION_FUNCS(processor_type, time_window, calc_res, rules2Signals, rulesToApply, rules2RemoveSignal, signal_channels, consideredRules, tolerance, req_signals, aff_signals, nRem_attr,
            nRem_attr_suffix, verbose_file, print_summary, print_summary_critical_cleaned)
 
private:
    bool  applyRule(int rule, const  vector<UniversalSigVec> &ruleUsvs,
        const vector<int> &val_channels, const vector<int> &sPointer);
    unordered_map<int, string> affected_ids_to_name;
    ofstream log_file;
    unordered_map<string, remove_stats> _rmv_stats;
 
    void select_rules_to_apply();
 
};
 
#define DEF_REP_NBRS_NBRS_SD_NUM 5
#define DEF_REP_NBRS_TRIM_SD_NUM 7
#define DEF_REP_NBRS_REMOVING_SD_NUM 14
//.......................................................................................
//.......................................................................................
class RepNbrsOutlierCleaner : public RepProcessor, public MedValueCleaner {
public:
 
    string signalName; 
    int signalId; 
    int time_channel = 0; 
    int val_channel = 0; 
 
    int nbr_time_width = 7; 
    int nbr_time_unit = MedTime::Days; 
 
    string nRem_attr = ""; 
    string nTrim_attr = ""; 
    string nRem_attr_suffix = ""; 
    string nTrim_attr_suffix = ""; 
 
    RepNbrsOutlierCleaner() { init_defaults(); }
    RepNbrsOutlierCleaner(const string& _signalName) { init_defaults(); signalId = -1; signalName = _signalName; init_lists(); }
    RepNbrsOutlierCleaner(const string& _signalName, string init_string) { init_defaults(); signalId = -1; signalName = _signalName; init_from_string(init_string); }
    RepNbrsOutlierCleaner(const string& _signalName, ValueCleanerParams *_params) { init_defaults(); signalId = -1; signalName = _signalName; init_lists(); MedValueCleaner::init(_params); }
 
    void init_defaults() {
        processor_type = REP_PROCESS_NBRS_OUTLIER_CLEANER;
        params.trimming_sd_num = DEF_REP_NBRS_TRIM_SD_NUM; params.removing_sd_num = DEF_REP_NBRS_REMOVING_SD_NUM; params.nbrs_sd_num = DEF_REP_NBRS_NBRS_SD_NUM;
        params.take_log = 0;
        params.doTrim = params.doRemove = true;
        signalId = -1;
        params.type = VAL_CLNR_ITERATIVE;
        params.missing_value = MED_MAT_MISSING_VALUE;
    };
 
    void set_signal(const string& _signalName) { signalId = -1; signalName = _signalName; init_lists(); }
 
    void set_signal_ids(MedSignals& sigs) { signalId = sigs.sid(signalName); }
 
    int init(void *processor_params) { return MedValueCleaner::init(processor_params); };
    int init(map<string, string>& mapper);
    void init_lists();
    void init_attributes();
 
    int _learn(MedPidRepository& rep, MedSamples& samples, vector<RepProcessor *>& prev_processor);
    int iterativeLearn(MedPidRepository& rep, MedSamples& samples, vector<RepProcessor *>& prev_processor);
    int quantileLearn(MedPidRepository& rep, MedSamples& samples, vector<RepProcessor *>& prev_processor);
 
    int _apply(PidDynamicRec& rec, vector<int>& time_points, vector<vector<float>>& attributes_mat);
 
    // Serialization
    ADD_CLASS_NAME(RepNbrsOutlierCleaner)
        ADD_SERIALIZATION_FUNCS(processor_type, signalName, time_channel, val_channel, req_signals, aff_signals, params.take_log, params.missing_value, params.doTrim, params.doRemove,
            trimMax, trimMin, removeMax, removeMin, nbr_time_unit, nbr_time_width, nbrsMax, nbrsMin, nRem_attr, nTrim_attr, nRem_attr_suffix, nTrim_attr_suffix)
 
        void print();
};
 
//.......................................................................................
//.......................................................................................
 
// Modes of handling multiple values at the same time
typedef enum {
    SIM_VAL_FIRST_VAL,
    SIM_VAL_LAST_VAL,
    SIM_VAL_MEAN,
    SIM_VAL_REM,
    SIM_VAL_REM_DIFF,
    SIM_VAL_MIN,
    SIM_VAL_MAX,
    MULT_VAL_LAST
} SimValHandleTypes;
 
class RepSimValHandler : public RepProcessor {
public:
    string signalName;  
    int signalId;   
    vector<int> time_channels; 
    SimValHandleTypes handler_type; 
 
    int nValChannels; 
 
    string nHandle_attr = ""; 
    string nHandle_attr_suffix = ""; 
    bool debug = false; 
 
    RepSimValHandler() { init_defaults(); processor_type = REP_PROCESS_SIM_VAL; }
    RepSimValHandler(const string& _signalName) { init_defaults(); processor_type = REP_PROCESS_SIM_VAL;; signalId = -1; signalName = _signalName; init_lists(); }
    RepSimValHandler(const string& _signalName, string init_string) { init_defaults(); processor_type = REP_PROCESS_SIM_VAL; signalId = -1; signalName = _signalName; init_from_string(init_string); }
 
    void set_signal(const string& _signalName) { signalId = -1; signalName = _signalName; init_lists(); }
 
    void set_signal_ids(MedSignals& sigs) { signalId = sigs.sid(signalName); }
 
    virtual int init(map<string, string>& mapper);
 
    void init_lists();
 
    void init_tables(MedDictionarySections& dict, MedSignals& sigs);
 
    void init_attributes();
 
    int _apply(PidDynamicRec& rec, vector<int>& time_points, vector<vector<float>>& attributes_mat);
 
    static SimValHandleTypes get_sim_val_handle_type(string& name);
 
    ADD_CLASS_NAME(RepSimValHandler)
        ADD_SERIALIZATION_FUNCS(processor_type, signalName, time_channels, req_signals, aff_signals, nHandle_attr, nHandle_attr_suffix, handler_type)
private:
    void handle_block(int start, int end, UniversalSigVec& usv, vector<int>& remove, int& nRemove, vector<pair<int, vector<float>>>& change, int& nChange, int& nTimes);
    int verbose_cnt = 0;
};
 
//.......................................................................................
//.......................................................................................
 
typedef enum {
    REP_CMPLT_RED_LINE_PANEL, 
    REP_CMPLT_WHITE_LINE_PANEL, 
    REP_CMPLT_PLATELETS_PANEL, 
    REP_CMPLT_LIPIDS_PANEL, 
    REP_CMPLT_EGFR_PANEL, 
    REP_CMPLT_BMI_PANEL, 
    REP_CMPLT_GCS, 
    REP_CMPLT_LAST 
} PanelCompleterTypes;
 
typedef enum {
    RED_PNL_MCV,
    RED_PNL_HCT,
    RED_PNL_RBC,
    RED_PNL_MCH,
    RED_PNL_MCHC,
    RED_PNL_HGB,
    RED_PNL_LAST
} RedPanelSignals;
 
typedef enum {
    WHITE_PNL_WBC,
    WHITE_PNL_EOS_N, WHITE_PNL_EOS_P,
    WHITE_PNL_NEU_N, WHITE_PNL_NEU_P,
    WHITE_PNL_LYM_N, WHITE_PNL_LYM_P,
    WHITE_PNL_MON_N, WHITE_PNL_MON_P,
    WHITE_PNL_BAS_N, WHITE_PNL_BAS_P,
    WHITE_PNL_LAST
} WhitePanelSignals;
 
typedef enum {
    PLT_PNL_PLTS,
    PLT_PNL_PLT_HCT,
    PLT_PNL_MPV,
    PLT_PNL_LAST
} PltsPanelSignals;
 
typedef enum {
    LIPIDS_PNL_CHOL,
    LIPIDS_PNL_LDL,
    LIPIDS_PNL_HDL,
    LIPIDS_PNL_HDL_OVER_CHOL,
    LIPIDS_PNL_CHOL_OVER_HDL,
    LIPIDS_PNL_HDL_OVER_LDL,
    LIPIDS_PNL_LDL_OVER_HDL,
    LIPIDS_PNL_NON_HDL_CHOL,
    LIPIDS_PNL_HDL_OVER_NON_HDL,
    LIPIDS_PNL_TRGS,
    LIPIDS_PNL_VLDL,
    LIPIDS_PNL_LAST
} LipidsPanelSignals;
 
typedef enum {
    EGFR_PNL_CRT,
    EGFR_PNL_CKD_EPI,
    EGFR_PNL_MDRD,
    EGFR_PNL_LAST
} eGFRPanelSignals;
 
typedef enum {
    BMI_PNL_BMI,
    BMI_PNL_WGT,
    BMI_PNL_HGT,
    BMI_PNL_HGT_SQR,
    BMI_PNL_LAST
} BMIPanelSignals;
 
typedef enum {
    GCS_PNL,
    GCS_PNL_EYE,
    GCS_PNL_MOTOR,
    GCS_PNL_VERBAL,
    GCS_PNL_LAST
} GCSPanelSignals;
 
class RepPanelCompleter : public RepProcessor {
public:
    // Signals for completions
    vector<vector<string> > panel_signal_names;
    vector<vector<int> > panel_signal_ids;
 
    // Extra signal ids
    int bdateId, genderId;
    string genderSignalName;
 
    // Missing value indication
    float missing_val = MED_MAT_MISSING_VALUE;
 
    // Handling multiple values
    SimValHandleTypes sim_val_handler = SIM_VAL_LAST_VAL;
 
    // Signals meta-data : original and final resolution and factors
    string metadata_file;
    vector<vector<float> > original_sig_res, final_sig_res, sig_conversion_factors;
 
    RepPanelCompleter() { processor_type = REP_PROCESS_COMPLETE; init_defaults(); }
 
    int init(map<string, string>& mapper);
 
    void init_defaults();
 
    // Change signal names from defualt
    int update_signal_names(string panel, string& names);
 
    // Change panels to handle
    int update_panels(string& panels);
 
    // initialize signal ids
    void set_signal_ids(MedSignals& sigs);
 
    // Check if some required signals are missing and make them virtual or remove relevant panel completer
    void fit_for_repository(MedPidRepository& rep);
 
    // dictionary based initializations
    void init_tables(MedDictionarySections &dict, MedSignals& sigs);
 
    void init_lists();
 
    // Read conversion and resolution info
    void read_metadata();
 
    // Learning
    int _learn(MedPidRepository& rep, MedSamples& samples, vector<RepProcessor *>& prev_processors) { init_tables(rep.dict, rep.sigs); read_metadata(); return 0; };
 
    // Applying
    int _apply(PidDynamicRec& rec, vector<int>& time_points, vector<vector<float>>& attributes_mat);
 
    // calculators implementations
    int apply_red_line_completer(PidDynamicRec& rec, vector<int>& time_points);
    int apply_white_line_completer(PidDynamicRec& rec, vector<int>& time_points);
    int apply_platelets_completer(PidDynamicRec& rec, vector<int>& time_points);
    int apply_lipids_completer(PidDynamicRec& rec, vector<int>& time_points);
    int apply_eGFR_completer(PidDynamicRec& rec, vector<int>& time_points);
    int apply_BMI_completer(PidDynamicRec& rec, vector<int>& time_points);
    int apply_GCS_completer(PidDynamicRec& rec, vector<int>& time_points);
 
    // Utilities
    // Age/Gender
    int perpare_for_age_and_gender(PidDynamicRec& rec, int& age, int& bYear, int& gender);
 
    // Rounding - take care of resolution both in original and final units 
    inline float completer_round(float value, float orig_res, float final_res, float factor) { return  set_resolution(set_resolution(value / factor, orig_res) * factor, final_res); }
    inline float set_resolution(float value, float res) { return res * std::round(value / res); }
 
    // Generating panels from usvs
    void get_panels(vector<UniversalSigVec>& usvs, vector<int>& panel_times, vector<vector<float>>& panels, int time_limit, int panel_size);
 
    // Applying formulas
    int triplet_complete(vector<float>& panel, float factor, int x_idx, int y_idx, int z_idx, vector<float>& orig_res, vector<float>& final_res, vector<float>& conv, vector<int>& changed);
    int sum_complete(vector<float>& panel, int sum, vector<int>& summands, vector<float>& orig_res, vector<float>& final_res, vector<float>& conv, vector<int>& changed);
    int reciprocal_complete(vector<float>& panel, float factor, int x_idx, int y_idx, vector<float>& orig_res, vector<float>& final_res, vector<float>& conv, vector<int>& changed);
    int egfr_complete(vector<float>& panel, float age, int gender, vector<float>& orig_res, vector<float>& final_res, vector<float>& conv, vector<int>& changed);
 
    // Updating signals in dynamic-rec
    int update_signals(PidDynamicRec& rec, int iver, vector<vector<float>>& panels, vector<int>& panel_times, vector<int>& sigs_ids, vector<int>& changed);
 
    // serialization. meta-data file is kept for information but not used in apply
    void print();
    ADD_CLASS_NAME(RepPanelCompleter)
        ADD_SERIALIZATION_FUNCS(processor_type, panel_signal_names, missing_val, sim_val_handler, original_sig_res, final_sig_res, sig_conversion_factors, metadata_file, req_signals, aff_signals, virtual_signals)
 
private:
 
    map<string, PanelCompleterTypes> panel2type = {
        { "red_line",REP_CMPLT_RED_LINE_PANEL },
        { "white_line",REP_CMPLT_WHITE_LINE_PANEL },
        { "platelets",REP_CMPLT_PLATELETS_PANEL },
        { "lipids",REP_CMPLT_LIPIDS_PANEL },
        { "egfr",REP_CMPLT_EGFR_PANEL },
        { "bmi",REP_CMPLT_BMI_PANEL },
        { "gcs", REP_CMPLT_GCS }
    };
 
    // definitions and defaults for each panel-completer
    map<string, vector<string> > panel2signals = {
        { "red_line", {"MCV","Hematocrit","RBC","MCH","MCHC-M","Hemoglobin"}},
        { "white_line", {"WBC", "Eosinophils#", "Eosinophils%", "Neutrophils#", "Neutrophils%", "Lymphocytes#", "Lymphocytes%", "Monocytes#", "Monocytes%", "Basophils#", "Basophils%"}},
        { "platelets", {"Platelets", "Platelets_Hematocrit","MPV"}},
        { "lipids",{"Cholesterol", "LDL", "HDL", "HDL_over_Cholesterol", "Cholesterol_over_HDL", "HDL_over_LDL", "LDL_over_HDL", "NonHDLCholesterol", "HDL_over_nonHDL", "Triglycerides", "VLDL"}},
        { "egfr", {"Creatinine","eGFR_CKD_EPI","eGFR_MDRD"}},
        { "bmi", {"BMI","Weight","Height"}},
        {"gcs", {"GCS", "GCS_Eye", "GCS_Motor", "GCS_Verbal"}}
    };
 
    vector<int> white_panel_nums = { WHITE_PNL_EOS_N,WHITE_PNL_NEU_N,WHITE_PNL_LYM_N,WHITE_PNL_MON_N,WHITE_PNL_BAS_N };
    vector<int> white_panel_precs = { WHITE_PNL_EOS_P,WHITE_PNL_NEU_P,WHITE_PNL_LYM_P,WHITE_PNL_MON_P,WHITE_PNL_BAS_P };
    vector<int> chol_types1 = { LIPIDS_PNL_NON_HDL_CHOL,LIPIDS_PNL_HDL };
    vector<int> chol_types2 = { LIPIDS_PNL_LDL, LIPIDS_PNL_HDL, LIPIDS_PNL_VLDL };
    vector<int> gcs_panel_parts = { GCS_PNL_EYE,GCS_PNL_MOTOR,GCS_PNL_VERBAL };
 
    //Convertion map for GCS:
    unordered_map<int, int> eye_vals, verbal_vals, motor_vals;
    void convert_gcs_signals(vector<float> &panel);
};
 
//.......................................................................................
 
 
//.......................................................................................
//.......................................................................................
class SimpleCalculator : public SerializableObject {
public:
    vector<string> output_signal_names;
    float missing_value = (float)MED_MAT_MISSING_VALUE; 
    string calculator_name = ""; 
    int work_channel = 0; 
    bool need_time = false; 
    bool keep_only_in_range = false; 
 
    virtual int init(map<string, string>& mapper) { return 0; };
    virtual void validate_arguments(const vector<string> &input_signals, const vector<string> &output_signals) const {};
    virtual bool do_calc(const vector<float> &vals, float &res) const { HMTHROW_AND_ERR("Error %s::do_calc not implemented\n", my_class_name().c_str()); };
    virtual void list_output_signals(const vector<string> &input_signals, vector<pair<string, string>> &_virtual_signals, const string &output_type) { HMTHROW_AND_ERR("Error %s::do_calc not implemented\n", my_class_name().c_str()); };
    virtual void init_tables(MedDictionarySections& dict, MedSignals& sigs, const vector<string> &input_signals) {};
 
    virtual void get_required_signal_categories(unordered_map<string, vector<string>> &signal_categories_in_use) const {};
 
    virtual void fit_for_repository(MedPidRepository &rep, vector<pair<string, string>> &_virtual_signals) {};
 
    static SimpleCalculator *make_calculator(const string &calc_type);
 
    virtual ~SimpleCalculator() {};
 
    void *new_polymorphic(string derived_class_name);
 
    ADD_CLASS_NAME(SimpleCalculator)
        ADD_SERIALIZATION_FUNCS(calculator_name, missing_value, work_channel, need_time, keep_only_in_range)
 
 
};
 
class EmptyCalculator : public SimpleCalculator {
public:
    EmptyCalculator() { calculator_name = "empty"; keep_only_in_range = false; };
 
    void validate_arguments(const vector<string> &input_signals, const vector<string> &output_signals) const;
    void list_output_signals(const vector<string> &input_signals, vector<pair<string, string>> &_virtual_signals, const string &output_type);
    bool do_calc(const vector<float> &vals, float &res) const;
 
    ADD_CLASS_NAME(EmptyCalculator)
        ADD_SERIALIZATION_FUNCS(calculator_name, missing_value, work_channel, need_time, keep_only_in_range)
};
 
class RatioCalculator : public SimpleCalculator {
public:
    float factor = 1; 
    float power_base = 1; 
    float power_mone = 1; 
 
    RatioCalculator() { calculator_name = "ratio"; keep_only_in_range = false; };
    int init(map<string, string>& mapper);
 
    void validate_arguments(const vector<string> &input_signals, const vector<string> &output_signals) const;
    void list_output_signals(const vector<string> &input_signals, vector<pair<string, string>> &_virtual_signals, const string &output_type);
    bool do_calc(const vector<float> &vals, float &res) const;
 
    ADD_CLASS_NAME(RatioCalculator)
        ADD_SERIALIZATION_FUNCS(calculator_name, missing_value, work_channel, need_time, keep_only_in_range, factor, power_base, power_mone)
};
 
class KfreCalculator : public SimpleCalculator {
public:
 
    // Valid values are 2,3 or 6
    //int model_id = 2;
    int n_variables = 4;
 
    int kfre_version = 2;
    int prediction_years = 5;
    std::string region = "original";        // original,north_american,non_noth_american,pooled
    int region_id;
    bool discard_range_check = false;       // when true, allow computation for input parameters out of valid range
 
    std::map<std::string, int> region2id = {
        { "original",           0 },
        { "north_american",     1 },
        { "non_north_american", 2 },
        { "pooled",             3 },
    };
 
    KfreCalculator() { calculator_name = "KFRE"; need_time = true; keep_only_in_range = true; };
    int init(map<string, string>& mapper);
    void validate_arguments(const vector<string> &input_signals, const vector<string> &output_signals) const;
    void list_output_signals(const vector<string> &input_signals, vector<pair<string, string>> &_virtual_signals, const string &output_type);
    bool do_calc(const vector<float> &vals, float &res) const;
 
    ADD_CLASS_NAME(KfreCalculator)
        ADD_SERIALIZATION_FUNCS(calculator_name, missing_value, work_channel, need_time, n_variables, n_variables, prediction_years, kfre_version, region, discard_range_check)
};
 
class eGFRCalculator : public SimpleCalculator {
public:
    float ethnicity = 0; 
    bool mdrd = false; 
 
    eGFRCalculator() { calculator_name = "eGFR_CKD_EPI"; need_time = true; keep_only_in_range = true; };
    int init(map<string, string>& mapper);
    void validate_arguments(const vector<string> &input_signals, const vector<string> &output_signals) const;
    void list_output_signals(const vector<string> &input_signals, vector<pair<string, string>> &_virtual_signals, const string &output_type);
    bool do_calc(const vector<float> &vals, float &res) const;
 
    ADD_CLASS_NAME(eGFRCalculator)
        ADD_SERIALIZATION_FUNCS(calculator_name, missing_value, work_channel, need_time, keep_only_in_range, ethnicity, mdrd)
};
 
class logCalculator : public SimpleCalculator {
public:
    logCalculator() { calculator_name = "log"; keep_only_in_range = false; };
 
    int init(map<string, string>& mapper);
    void validate_arguments(const vector<string> &input_signals, const vector<string> &output_signals) const;
    void list_output_signals(const vector<string> &input_signals, vector<pair<string, string>> &_virtual_signals, const string &output_type);
 
    bool do_calc(const vector<float> &vals, float &res) const;
 
    ADD_CLASS_NAME(logCalculator)
        ADD_SERIALIZATION_FUNCS(calculator_name, missing_value, work_channel, need_time, keep_only_in_range)
};
 
class SumCalculator : public SimpleCalculator {
public:
    vector<float> factors; 
    float b0 = 0; 
 
    SumCalculator() { calculator_name = "sum"; };
    int init(map<string, string>& mapper);
    void validate_arguments(const vector<string> &input_signals, const vector<string> &output_signals) const;
    void list_output_signals(const vector<string> &input_signals, vector<pair<string, string>> &_virtual_signals, const string &output_type);
    bool do_calc(const vector<float> &vals, float &res) const;
 
    ADD_CLASS_NAME(SumCalculator)
        ADD_SERIALIZATION_FUNCS(calculator_name, missing_value, work_channel, need_time, keep_only_in_range, factors, b0)
};
 
class RangeCalculator : public SimpleCalculator {
public:
    float min_range; 
    float max_range; 
    float in_range_val = 1; 
    float out_range_val = 0; 
 
    RangeCalculator() { calculator_name = "range"; min_range = MED_MAT_MISSING_VALUE; max_range = MED_MAT_MISSING_VALUE; };
    int init(map<string, string>& mapper);
    void validate_arguments(const vector<string> &input_signals, const vector<string> &output_signals) const;
    void list_output_signals(const vector<string> &input_signals, vector<pair<string, string>> &_virtual_signals, const string &output_type);
    bool do_calc(const vector<float> &vals, float &res) const;
 
    ADD_CLASS_NAME(RangeCalculator)
        ADD_SERIALIZATION_FUNCS(calculator_name, missing_value, work_channel, need_time, keep_only_in_range, min_range, max_range, in_range_val, out_range_val)
};
 
class MultiplyCalculator : public SimpleCalculator {
public:
    vector<float> powers; 
    float b0 = 1; 
 
    MultiplyCalculator() { calculator_name = "multiply"; };
    int init(map<string, string>& mapper);
 
    void validate_arguments(const vector<string> &input_signals, const vector<string> &output_signals) const;
    void list_output_signals(const vector<string> &input_signals, vector<pair<string, string>> &_virtual_signals, const string &output_type);
 
    bool do_calc(const vector<float> &vals, float &res) const;
 
    ADD_CLASS_NAME(MultiplyCalculator)
        ADD_SERIALIZATION_FUNCS(calculator_name, missing_value, work_channel, need_time, keep_only_in_range, powers, b0)
};
 
class SetCalculator : public SimpleCalculator {
public:
    vector<string> sets;
    float in_range_val = 1; 
    float out_range_val = 0; 
    bool regex_on_sets = false;
    SetCalculator() { calculator_name = "set"; };
    int init(map<string, string>& mapper);
 
    void validate_arguments(const vector<string> &input_signals, const vector<string> &output_signals) const;
    void list_output_signals(const vector<string> &input_signals, vector<pair<string, string>> &_virtual_signals, const string &output_type);
    void init_tables(MedDictionarySections& dict, MedSignals& sigs, const vector<string> &input_signals);
 
    bool do_calc(const vector<float> &vals, float &res) const;
 
    void get_required_signal_categories(unordered_map<string, vector<string>> &signal_categories_in_use) const;
 
    ADD_CLASS_NAME(SetCalculator)
        ADD_SERIALIZATION_FUNCS(calculator_name, missing_value, work_channel, need_time, keep_only_in_range, input_signal, sets, in_range_val, out_range_val)
private:
    vector<char> Flags;
    string input_signal = "";
};
 
class ExistsCalculator : public SimpleCalculator {
public:
    float in_range_val = 1; 
    float out_range_val = 0; 
 
    ExistsCalculator() { calculator_name = "exists"; keep_only_in_range = true; need_time = true; };
    int init(map<string, string>& mapper);
 
    void validate_arguments(const vector<string> &input_signals, const vector<string> &output_signals) const;
    void list_output_signals(const vector<string> &input_signals, vector<pair<string, string>> &_virtual_signals, const string &output_type);
 
    bool do_calc(const vector<float> &vals, float &res) const;
 
    ADD_CLASS_NAME(ExistsCalculator)
        ADD_SERIALIZATION_FUNCS(calculator_name, missing_value, work_channel, need_time, keep_only_in_range, in_range_val, out_range_val)
};
 
class ConstantValueCalculator : public SimpleCalculator {
private:
    float numeric_val;
public:
    string value;
    bool is_numeric;
    vector<string> additional_dict_vals;
    ConstantValueCalculator() {
        calculator_name = "constant_value"; is_numeric = false; value = ""; numeric_val = 1;
    };
 
    int init(map<string, string>& mapper);
 
    void validate_arguments(const vector<string> &input_signals, const vector<string> &output_signals) const;
    void list_output_signals(const vector<string> &input_signals, vector<pair<string, string>> &_virtual_signals, const string &output_type);
 
    bool do_calc(const vector<float> &vals, float &res) const;
 
    void fit_for_repository(MedPidRepository &rep, vector<pair<string, string>> &_virtual_signals);
 
    ADD_CLASS_NAME(ConstantValueCalculator)
        ADD_SERIALIZATION_FUNCS(calculator_name, missing_value, work_channel, need_time, keep_only_in_range, value, is_numeric, numeric_val, output_signal_names, additional_dict_vals)
};
 
class RepCalcSimpleSignals : public RepProcessor {
 
public:
    vector<string> V_names; 
 
    string calculator; 
    int work_channel = 0; 
    int time_channel = 0; 
    string output_signal_type; 
 
    float missing_value = (float)MED_MAT_MISSING_VALUE;
 
    //vector<float> coeff; ///< it is possible to transfer a vector of params to the calculator, to enable parametric calculators.
 
    vector<string> signals; 
    int signals_time_unit = MedTime::Undefined; 
 
    int max_time_search_range = 0; 
    string calculator_init_params = ""; 
 
    RepCalcSimpleSignals() {
        processor_type = REP_PROCESS_CALC_SIGNALS;
        output_signal_type = "T(i),V(f)";
    }
    ~RepCalcSimpleSignals();
 
    int init(map<string, string>& mapper);
 
    // making sure V_ids and sigs_ids are initialized
    void init_tables(MedDictionarySections& dict, MedSignals& sigs);
 
    void fit_for_repository(MedPidRepository &rep);
 
    // Learning
    int _learn(MedPidRepository& rep, MedSamples& samples, vector<RepProcessor *>& prev_processors) { init_tables(rep.dict, rep.sigs); return 0; };
 
    // Applying
    int _apply(PidDynamicRec& rec, vector<int>& time_points, vector<vector<float>>& attributes_mat);
 
    void print();
 
    void get_required_signal_categories(unordered_map<string, vector<string>> &signal_categories_in_use) const;
 
    void register_virtual_section_name_id(MedDictionarySections& dict);
 
    // serialization
    ADD_CLASS_NAME(RepCalcSimpleSignals)
        ADD_SERIALIZATION_FUNCS(processor_type, calculator, calculator_init_params, max_time_search_range, signals_time_unit,
            signals, V_names, req_signals, aff_signals, virtual_signals, virtual_signals_generic, work_channel, time_channel, calculator_logic, output_signal_type)
        void post_deserialization() {
        SimpleCalculator *p = SimpleCalculator::make_calculator(calculator);
        pass_time_last = p->need_time;
        delete p;
        //if (calculator_logic == NULL) {
        //  printf("write calculator_logic again for %s\n%s\n",
        //      calculator.c_str(), calculator_init_params.c_str());
        //  calculator_logic = SimpleCalculator::make_calculator(calculator);
        //  if (!calculator_init_params.empty())
        //      calculator_logic->init_from_string(calculator_init_params);
        //  calculator_logic->missing_value = missing_value;
        //  vector<pair<string, string>> default_virtual_signals;
        //  calculator_logic->list_output_signals(signals, default_virtual_signals); //init calculator
        //}
 
    }
 
private:
    // definitions and defaults for each calculator - all must be filled in for a new calculator
    bool pass_time_last = false; 
    const map<string, vector<string>> calc2req_sigs = {
        //--------- level 1 - calculated from raw signals (level0)
        //the general hospital processor's signals must be overridden from outside
        { "calc_eGFR", {"Creatinine", "GENDER", "BDATE"}}
    };
 
    vector<int> V_ids; 
    vector<int> sigs_ids; 
    vector<bool> static_input_signals;
 
    int apply_calc_in_time(PidDynamicRec& rec, vector<int>& time_points);
 
    SimpleCalculator *calculator_logic = NULL;
    int out_n_val_ch, out_n_time_ch;
};
 
class RepCombineSignals : public RepProcessor {
public:
    string output_name; 
    vector<string> signals; 
    vector<float> factors; 
    int factor_channel; 
    string signal_type; 
 
    RepCombineSignals() {
        processor_type = REP_PROCESS_COMBINE;
        output_name = "";
        factor_channel = 1;
        signal_type = "T(i,i),V(f,f)";
    }
 
    void register_virtual_section_name_id(MedDictionarySections& dict);
 
    int init(map<string, string>& mapper);
 
    void init_tables(MedDictionarySections& dict, MedSignals& sigs);
    void set_required_signal_ids(MedDictionarySections& dict) {};
    void set_affected_signal_ids(MedDictionarySections& dict) {};
 
    void fit_for_repository(MedPidRepository& rep);
    // Applying
    int _apply(PidDynamicRec& rec, vector<int>& time_points, vector<vector<float>>& attributes_mat);
 
    void print();
    ADD_CLASS_NAME(RepCombineSignals)
        ADD_SERIALIZATION_FUNCS(processor_type, output_name, signals, factors,
            unconditional, req_signals, aff_signals, virtual_signals,
            virtual_signals_generic, signal_type, factor_channel)
private:
    int v_out_sid = -1;
    vector<int> sigs_ids;
    int v_out_n_time_ch, v_out_n_val_ch;
};
 
class RepSplitSignal : public RepProcessor {
public:
    string input_name; 
    vector<string> names; 
    vector<float> factors; 
    vector<string> sets; 
    int val_channel; 
    string output_signal_type; 
 
    RepSplitSignal() { processor_type = REP_PROCESS_SPLIT; input_name = ""; val_channel = 0; output_signal_type = "T(i),T(i),V(f),V(f)"; }
 
    void register_virtual_section_name_id(MedDictionarySections& dict);
 
    void init_tables(MedDictionarySections& dict, MedSignals& sigs);
    void set_required_signal_ids(MedDictionarySections& dict) {};
    void set_affected_signal_ids(MedDictionarySections& dict) {};
 
    int init(map<string, string>& mapper);
 
    // Applying
    int _apply(PidDynamicRec& rec, vector<int>& time_points, vector<vector<float>>& attributes_mat);
 
    void get_required_signal_categories(unordered_map<string, vector<string>> &signal_categories_in_use) const;
 
    void print();
    ADD_CLASS_NAME(RepSplitSignal)
        ADD_SERIALIZATION_FUNCS(processor_type, input_name, names, factors, sets, unconditional,
            req_signals, aff_signals, virtual_signals, virtual_signals_generic, val_channel, output_signal_type)
private:
    int in_sid = -1;
    vector<int> V_ids;
    vector<char> Flags;
    int v_out_n_time_ch, v_out_n_val_ch;
};
 
 
class RepAggregationPeriod : public RepProcessor {
public:
    string input_name; 
    string output_name; 
    vector<string> sets; 
    int period; 
 
    int time_unit_win = MedTime::Undefined;     
    int time_unit_sig = MedTime::Undefined;     
 
private:
    int in_sid;
    vector<int> V_ids;
    vector<char> lut;
 
public:
    RepAggregationPeriod() :
        input_name(""), output_name(""), period(0), time_unit_sig(global_default_windows_time_unit), time_unit_win(global_default_windows_time_unit), in_sid(-1) {
        processor_type = REP_PROCESS_AGGREGATION_PERIOD;
    }
 
    void init_tables(MedDictionarySections& dict, MedSignals& sigs);
 
    int init(map<string, string>& mapper);
 
    // Applying
    int _apply(PidDynamicRec& rec, vector<int>& time_points, vector<vector<float>>& attributes_mat);
 
    void print();
 
    void get_required_signal_categories(unordered_map<string, vector<string>> &signal_categories_in_use) const;
 
    ADD_CLASS_NAME(RepAggregationPeriod)
        ADD_SERIALIZATION_FUNCS(processor_type, input_name, output_name, sets, period, req_signals, aff_signals, virtual_signals, virtual_signals_generic, time_unit_win, time_unit_sig, in_sid, V_ids, lut)
 
};
 
typedef enum {
    all = 0,
    first = 1,
    last = 2
} range_op_type;
class RepBasicRangeCleaner : public RepProcessor {
private:
    vector<char> lut;
public:
 
    string signal_name;     
    string ranges_name; 
    string output_name; 
    int signal_id;  
    int ranges_id; 
    int output_id; 
    int time_channel; 
    int range_time_channel; 
    int output_type; 
    int get_values_in_range = 1; 
    range_op_type range_operator = range_op_type::all; 
    int range_val_channel = -1; 
    vector<string> sets; 
    bool regex_on_sets = 0;  
    int last_n = 0;  
    bool do_on_last_n = false; 
 
    RepBasicRangeCleaner() :
        signal_name(""), ranges_name(""), output_name(""), signal_id(-1), ranges_id(-1),
        output_id(-1), time_channel(0), output_type(3) {
        processor_type = REP_PROCESS_BASIC_RANGE_CLEANER;
        range_time_channel = 1;
    }
 
    void init_tables(MedDictionarySections& dict, MedSignals& sigs);
 
    void register_virtual_section_name_id(MedDictionarySections& dict);
 
    bool get_last_n_value(int time, const UniversalSigVec& range_sig, float& last_value);
 
    virtual int init(map<string, string>& mapper);
 
    int _apply(PidDynamicRec& rec, vector<int>& time_points, vector<vector<float>>& attributes_mat);
 
    void fit_for_repository(MedPidRepository& rep);
 
    ADD_CLASS_NAME(RepBasicRangeCleaner)
        ADD_SERIALIZATION_FUNCS(processor_type, signal_name, ranges_name, output_name, time_channel,
            req_signals, aff_signals, signal_id, ranges_id, output_id, virtual_signals, virtual_signals_generic,
            output_type, get_values_in_range, range_operator, range_val_channel, sets, range_time_channel, last_n, do_on_last_n, regex_on_sets)
 
        void print();
};
 
class RepSignalRate : public RepProcessor {
public:
    string output_name; 
    string input_name; 
    int work_channel; 
    float factor; 
    string output_signal_type; 
 
    RepSignalRate() {
        processor_type = REP_PROCESS_SIGNAL_RATE; output_name = { "calc_drug_rate" };
        work_channel = 0;  factor = 1;
        output_signal_type = "T(i,i),V(f)";
    }
 
    int init(map<string, string>& mapper);
    void init_tables(MedDictionarySections& dict, MedSignals& sigs);
    void register_virtual_section_name_id(MedDictionarySections& dict);
    void set_required_signal_ids(MedDictionarySections& dict) {};
    void set_affected_signal_ids(MedDictionarySections& dict) {};
 
    // Applying
    int _apply(PidDynamicRec& rec, vector<int>& time_points, vector<vector<float>>& attributes_mat);
 
    void print();
    ADD_CLASS_NAME(RepSignalRate)
        ADD_SERIALIZATION_FUNCS(processor_type, input_name, output_name,
            work_channel, factor, unconditional, req_signals, aff_signals,
            virtual_signals, virtual_signals_generic, output_signal_type)
private:
    int v_out_sid = -1;
    int in_sid = -1;
    int v_out_n_time_ch, v_out_n_val_ch;
};
 
class RepAggregateSignal : public RepProcessor {
public:
    string signalName; 
    string output_name; 
    int work_channel; 
    int start_time_channel;; 
    int end_time_channel; 
    float factor; 
    int time_window; 
    int time_unit; 
    float drop_missing_rate; 
    bool buffer_first; 
    string output_signal_type; 
 
 
    RepAggregateSignal() {
        processor_type = REP_PROCESS_AGGREGATE;
        work_channel = 0;
        start_time_channel = 0;
        end_time_channel = 0;
        factor = 1;
        time_unit = global_default_windows_time_unit;
        drop_missing_rate = 1;
 
        output_name = "calc_aggregate";
        time_window = 0;
        buffer_first = true;
        output_signal_type = "T(i),V(f)";
    }
 
    int init(map<string, string>& mapper);
    void init_tables(MedDictionarySections& dict, MedSignals& sigs);
    void register_virtual_section_name_id(MedDictionarySections& dict);
    void set_required_signal_ids(MedDictionarySections& dict) {};
    void set_affected_signal_ids(MedDictionarySections& dict) {};
 
    // Applying
    int _apply(PidDynamicRec& rec, vector<int>& time_points, vector<vector<float>>& attributes_mat);
 
    void print();
    ADD_CLASS_NAME(RepAggregateSignal)
        ADD_SERIALIZATION_FUNCS(processor_type, signalName, output_name, work_channel, factor, time_window, time_unit,
            start_time_channel, end_time_channel, drop_missing_rate, buffer_first, unconditional, req_signals, aff_signals, virtual_signals, virtual_signals_generic, output_signal_type)
private:
    int v_out_sid = -1;
    int in_sid = -1;
    int v_out_n_time_ch, v_out_n_val_ch;
};
 
//---------------------------------------------------------------------------------------------------------------
// helper structures for RepCreateBitSignal:
//---------------------------------------------------------------------------------------------------------------
struct category_time_interval {
    int first_appearance = 0;
    int last_appearance = 0;
    int n_appearances = 0;
    int last_time = 0; // last_appearance + duration
 
    category_time_interval() {};
    category_time_interval(int _first, int _last, int _n, int _last_time) { first_appearance = _first; last_appearance = _last; n_appearances = _n; last_time = _last_time; }
 
    void set(int _first, int _last, int _n, int _last_time) { first_appearance = _first; last_appearance = _last; n_appearances = _n; last_time = _last_time; }
};
 
struct category_event_state {
    int time = 0;
    int appear_time = 0;
    int categ = 0;
    int type = 0; // 0 : stop, 1: start, 2: last appearance
 
    category_event_state() {};
    category_event_state(int _time, int _appear_time, int _categ, int _type) { time = _time; appear_time = _appear_time; categ = _categ; type = _type; }
    void set(int _time, int _appear_time, int _categ, int _type) { time = _time; appear_time = _appear_time; categ = _categ; type = _type; }
 
 
    bool operator <(const category_event_state &c) const {
        if (time < c.time) return true;
        if (time > c.time) return false;
        if (type < c.type) return true;
        //if (type > c.type) return false;
        //if (end_time < c.end_time) return true;
        return false;
    }
};
 
struct combination_state {
    int start;
    int state;
    vector<int> last;
 
    combination_state() {};
    combination_state(int time, int _state, int N) { start = time; state = _state; last.assign(N, -1); }
};
 
 
//---------------------------------------------------------------------------------------------------------------
// RepCreateBitSignal
// Given N<32 categories defined as groups of sets on a categorial signal, creates a united signal
// holding in bit j the occurance of each category. Each occurance of a category at time t implies 
// a time span of [t, t+max(min_duration, duration in signal (if available))] in which we define it as 
// occuring.
// The resulted signal includes each time point in which any of the categories changed, and the new "state" 
// at each such time point. "0" state starts at the first sig time point (unless it starts with the category).
// This is useful for example when working on drugs for a specific conditions (such as diabetes drugs)
// It then creates all states of treatment for the patient in a single easy to use signal.
// A virtual dictionary will be added as well.
// 
//---------------------------------------------------------------------------------------------------------------
class RepCreateBitSignal : public RepProcessor {
public:
    string in_sig = ""; 
    string out_virtual = ""; 
    int t_chan = 0; 
    int c_chan = 0; 
    int duration_chan = 1; 
    int min_duration = 60; 
    vector<int> min_durations = {}; 
    int max_duration = 180; 
    float duration_add = 0;
    float duration_mult = 0.1f; 
    int dont_look_back = 7; 
    int min_clip_time = 7; 
    int last_clip_period = 30; 
    vector<int> min_jitters = { 0,0,0 };    
    int max_min_jitters;
    vector<string> categories_names; 
    vector<vector<string>> categories_sets; 
    int time_unit_sig = MedTime::Date; 
    int time_unit_duration = MedTime::Days; 
    bool change_at_prescription_mode = false; 
    MedDictionarySections *_dict; // for debug
    // next will NOT be serialized, and is here for debug reasons
    string print_dict = "";
    int time_channels = 1;
    vector<char> all_cat_lut;
    vector<vector<char>> categories_luts;
 
    RepCreateBitSignal() { processor_type = REP_PROCESS_CREATE_BIT_SIGNAL; };
 
    int init(map<string, string>& mapper);
    void get_min_jitters(string& jitters_s);
    void init_tables(MedDictionarySections& dict, MedSignals& sigs);
    void register_virtual_section_name_id(MedDictionarySections& dict);
    void set_required_signal_ids(MedDictionarySections& dict) {};
    void set_affected_signal_ids(MedDictionarySections& dict) {};
 
    // Applying
    int _apply(PidDynamicRec& rec, vector<int>& time_points, vector<vector<float>>& attributes_mat);
 
    void get_required_signal_categories(unordered_map<string, vector<string>> &signal_categories_in_use) const;
 
    //void print();
    ADD_CLASS_NAME(RepCreateBitSignal)
        ADD_SERIALIZATION_FUNCS(processor_type, in_sig, out_virtual, aff_signals, req_signals,
            t_chan, c_chan, duration_chan, min_duration, min_durations, max_duration, duration_add, duration_mult, dont_look_back, min_clip_time, last_clip_period, categories_names, categories_sets, time_unit_sig, time_unit_duration, change_at_prescription_mode,
            virtual_signals_generic, time_channels, min_jitters)
 
private:
    int v_out_sid = -1;
    int in_sid = -1;
    vector<string> registry_values;
};
 
//---------------------------------------------------------------------------------------------------------------
 
 
//.......................................................................................
//.......................................................................................
 
class RepCheckReq : public RepProcessor {
 
public:
    vector<string> signalNames; 
    vector<int> signalIds; 
 
    vector<int> time_channels; 
    int win_from = 0, win_to = 0; 
    int window_time_unit = MedTime::Days; 
 
    string attrName = "MissingReq"; 
 
    // Helper
    vector<int> sig_time_units;
 
    RepCheckReq() { processor_type = REP_PROCESS_CHECK_REQ; unconditional = true; }
 
    int init(map<string, string>& mapper);
 
    void set_signal_ids(MedSignals& sigs);
 
    void init_lists();
 
    void init_tables(MedDictionarySections& dict, MedSignals& sigs);
 
    void init_attributes() { attributes = { attrName }; }
 
    // Learning - none done
 
    // Applying
    int _apply(PidDynamicRec& rec, vector<int>& time_points, vector<vector<float>>& attributes_mat);
 
    // serialization.
    ADD_CLASS_NAME(RepCheckReq)
        ADD_SERIALIZATION_FUNCS(processor_type, signalNames, time_channels, win_from, win_to, window_time_unit, attrName, req_signals)
};
 
//----------------------------------------------------------------------------------------
// RepHistoryLimit : given a signal : chomps history to be at a given window relative
//                   to prediction points
//----------------------------------------------------------------------------------------
class RepHistoryLimit : public RepProcessor {
public:
    string signalName;  
    int signalId;   
    int time_channel = 0; 
    int truncate_time_channel = -1; 
    int win_time_unit = global_default_windows_time_unit;
    int rep_time_unit = global_default_time_unit; // we assume this is also the samples time unit
    int win_from = 0;
    int win_to = 0;
    int delete_sig = 0; 
    int take_last_events = -1; 
 
    RepHistoryLimit() { init_defaults(); }
 
    // default init
    void init_defaults() { processor_type = REP_PROCESS_HISTORY_LIMIT; }
 
    // preparations
    void set_signal_ids(MedSignals& sigs) { signalId = sigs.sid(signalName); }
 
    // learn - nothing to do
    int _learn(MedPidRepository& rep, MedSamples& samples, vector<RepProcessor *>& prev_processor) { return 0; }
 
    // apply
    int _apply(PidDynamicRec& rec, vector<int>& time_points, vector<vector<float>>& attributes_mat);
 
    int init(map<string, string>& mapper);
 
    // helper
    int get_sub_usv_data(UniversalSigVec &usv, int from_time, int to_time, vector<char> &data, int &len);
 
    void init_lists();
 
    ADD_CLASS_NAME(RepHistoryLimit)
        ADD_SERIALIZATION_FUNCS(processor_type, signalName, time_channel, win_time_unit, rep_time_unit,
            win_from, win_to, delete_sig, take_last_events, req_signals, aff_signals, truncate_time_channel)
 
};
 
//----------------------------------------------------------------------------------------
// RepNumericNoiser : given a numeric signal : adds gaussian noise to each value, with std as user-determined
// fraction of signal std, uniform distribution to time, and probability of dropping value
//----------------------------------------------------------------------------------------
class RepNumericNoiser : public RepProcessor {
private:
    int signalId =-1;   
    double stdev;
    bool on_learning = false;
    bool apply_in_test;
 
    random_device rd;
    //mt19937 gens;
    vector<mt19937> gens;
 
    //mt19937 generator;
 
public:
    string signalName;  
    
    int val_channel = 0; 
    int time_channel = 0; 
    int time_noise = 0; 
    float value_noise = 0; 
    int truncation = 2; 
    float drop_probability = (float)0.0; 
 
 
    RepNumericNoiser() { init_defaults(); }
 
 
    // default init
    void init_defaults() { 
        processor_type = REP_PROCESS_NUMERIC_NOISER; 
        int N_TH = omp_get_max_threads();
        gens.resize(3*N_TH);
        for (size_t i = 0; i < 3*N_TH; ++i)
            gens[i] = mt19937(rd());
    }
 
    // preparations
    void set_signal_ids(MedSignals& sigs) { signalId = sigs.sid(signalName); }
 
    // learn - nothing to do
    int _learn(MedPidRepository& rep, MedSamples& samples, vector<RepProcessor *>& prev_processor);
 
    // apply
    int _apply(PidDynamicRec& rec, vector<int>& time_points, vector<vector<float>>& attributes_mat);
 
    int init(map<string, string>& mapper);
 
    void init_lists();
 
    ADD_CLASS_NAME(RepNumericNoiser)
        ADD_SERIALIZATION_FUNCS(processor_type, signalName, val_channel, time_channel,
            time_noise, value_noise, drop_probability, req_signals, aff_signals, stdev, truncation, apply_in_test)
 
};
 
class RepReoderChannels : public RepProcessor {
private:
    int sid = -1;
public:
    vector<int> new_order;
    string signal_name = "";
 
    RepReoderChannels() {
        processor_type = REP_PROCESS_REODER_CHANNELS;
    }
 
    void set_signal_ids(MedSignals& sigs);
    void set_signal(const string& _signalName) { sid = -1; signal_name = _signalName; }
 
    int init(map<string, string>& mapper);
 
    // Applying
    int _apply(PidDynamicRec& rec, vector<int>& time_points, vector<vector<float>>& attributes_mat);
 
    void print();
 
    ADD_CLASS_NAME(RepReoderChannels)
        ADD_SERIALIZATION_FUNCS(processor_type, req_signals, aff_signals, virtual_signals, virtual_signals_generic, signal_name, new_order)
};
 
//.......................................................................................
//.......................................................................................
// Utility Functions
//.......................................................................................
//.......................................................................................
 
int get_values(MedRepository& rep, MedSamples& samples, int signalId, int time_channel, int val_channel, float range_min, float range_max, vector<float>& values,
    vector<RepProcessor *>& prev_cleaners);
int get_values(MedRepository& rep, MedSamples& samples, int signalId, int time_channel, int val_channel, float range_min, float range_max, vector<float>& values);
 
//=======================================
// Joining the MedSerialze wagon
//=======================================
MEDSERIALIZE_SUPPORT(RepProcessor)
MEDSERIALIZE_SUPPORT(RepMultiProcessor)
MEDSERIALIZE_SUPPORT(RepBasicOutlierCleaner)
MEDSERIALIZE_SUPPORT(RepRuleBasedOutlierCleaner)
MEDSERIALIZE_SUPPORT(RepConfiguredOutlierCleaner)
MEDSERIALIZE_SUPPORT(RepNbrsOutlierCleaner)
MEDSERIALIZE_SUPPORT(RepCalcSimpleSignals)
MEDSERIALIZE_SUPPORT(RepSimValHandler)
MEDSERIALIZE_SUPPORT(RepPanelCompleter)
MEDSERIALIZE_SUPPORT(RepCombineSignals)
MEDSERIALIZE_SUPPORT(RepSplitSignal)
MEDSERIALIZE_SUPPORT(RepAggregationPeriod)
MEDSERIALIZE_SUPPORT(RepBasicRangeCleaner)
MEDSERIALIZE_SUPPORT(RepSignalRate)
MEDSERIALIZE_SUPPORT(RepAggregateSignal)
MEDSERIALIZE_SUPPORT(RepCheckReq)
MEDSERIALIZE_SUPPORT(RepHistoryLimit)
MEDSERIALIZE_SUPPORT(RepCreateBitSignal)
MEDSERIALIZE_SUPPORT(RepReoderChannels)
MEDSERIALIZE_SUPPORT(RepNumericNoiser)
MEDSERIALIZE_SUPPORT(SimpleCalculator)
MEDSERIALIZE_SUPPORT(SetCalculator)
MEDSERIALIZE_SUPPORT(MultiplyCalculator)
MEDSERIALIZE_SUPPORT(logCalculator)
MEDSERIALIZE_SUPPORT(RatioCalculator)
MEDSERIALIZE_SUPPORT(eGFRCalculator)
MEDSERIALIZE_SUPPORT(SumCalculator)
MEDSERIALIZE_SUPPORT(RangeCalculator)
MEDSERIALIZE_SUPPORT(ExistsCalculator)
MEDSERIALIZE_SUPPORT(EmptyCalculator)
MEDSERIALIZE_SUPPORT(ConstantValueCalculator)
 
#endif