MR_LIBS/RepProcess_8h_source.html

#ifndef _REP_PROCESS_H_

#define _REP_PROCESS_H_


#include "InfraMed/InfraMed/InfraMed.h"

#include "InfraMed/InfraMed/MedPidRepository.h"

#include "MedProcessTools/MedProcessTools/MedSamples.h"

#include "MedProcessTools/MedProcessTools/MedProcessUtils.h"

#include <SerializableObject/SerializableObject/SerializableObject.h>

#include "MedProcessTools/MedProcessTools/MedValueCleaner.h"

#include <MedMat/MedMat/MedMat.h>

#include <omp.h>


#define DEFAULT_REP_CLNR_NTHREADS 8


//.......................................................................................

//.......................................................................................


typedef enum {

    REP_PROCESS_MULTI,

    REP_PROCESS_BASIC_OUTLIER_CLEANER,

    REP_PROCESS_NBRS_OUTLIER_CLEANER,

    REP_PROCESS_CONFIGURED_OUTLIER_CLEANER,

    REP_PROCESS_RULEBASED_OUTLIER_CLEANER,

    REP_PROCESS_CALC_SIGNALS,

    REP_PROCESS_COMPLETE,

    REP_PROCESS_CHECK_REQ,

    REP_PROCESS_SIM_VAL,

    REP_PROCESS_SIGNAL_RATE,

    REP_PROCESS_COMBINE,

    REP_PROCESS_SPLIT,

    REP_PROCESS_AGGREGATION_PERIOD,

    REP_PROCESS_BASIC_RANGE_CLEANER,

    REP_PROCESS_AGGREGATE,

    REP_PROCESS_HISTORY_LIMIT,

    REP_PROCESS_CREATE_REGISTRY,

    REP_PROCESS_CREATE_BIT_SIGNAL,

    REP_PROCESS_CATEGORY_DESCENDERS,

    REP_PROCESS_REODER_CHANNELS,

    REP_PROCESS_FILTER_BY_CHANNELS,

    REP_PROCESS_NUMERIC_NOISER,

    REP_PROCESS_FILTER_BY_DIAG,

    REP_PROCESS_LAST

} RepProcessorTypes;


class RepProcessor : public SerializableObject {

public:


    RepProcessorTypes processor_type = REP_PROCESS_LAST;


    unordered_set<string> req_signals;

    unordered_set<int> req_signal_ids;


    unordered_set<string> aff_signals;

    unordered_set<int> aff_signal_ids;

    bool unconditional = false;


    vector<string> attributes;


    virtual ~RepProcessor() { clear(); }

    virtual void clear() {  };


    vector<pair<string, int>> virtual_signals;

    vector<pair<string, string>> virtual_signals_generic;


    // create a new rep_processor

    static RepProcessor *make_processor(string name);

    static RepProcessor *make_processor(string type, string params);

    static RepProcessor *make_processor(RepProcessorTypes type);

    static RepProcessor *make_processor(RepProcessorTypes type, string params);


    static RepProcessor *create_processor(string &params);


    virtual int init(void *params) { return 0; };

    virtual int init(map<string, string>& mapper) { return 0; };

    virtual void init_defaults() {};


    virtual void set_signal(const string& _signalName) { return; };


    virtual void set_signal_ids(MedSignals& sigs) { return; }


    // Required Signals functions : get all signals that are required by the processor

    virtual void get_required_signal_names(unordered_set<string>& signalNames);

    // Required Signals functions : get all signals that are required by the processor

    virtual void get_required_signal_names(unordered_set<string>& signalNames, unordered_set<string> preReqSignals);


    virtual void set_required_signal_ids(MedDictionarySections& dict);


    virtual void add_virtual_signals(map<string, int> &_virtual_signals, map<string, string> &_virtual_signals_generic) const {

        for (auto &v : virtual_signals) _virtual_signals[v.first] = v.second;

        for (auto &v : virtual_signals_generic) _virtual_signals_generic[v.first] = v.second;

    };


    // Required Signals functions : get all signals that are required by the processor

    virtual void get_required_signal_ids(unordered_set<int>& signalIds);

    // Required Signals functions : get all signals that are required by the processor

    virtual void get_required_signal_ids(unordered_set<int>& signalIds, unordered_set<int> preReqSignals);


    // Affected Signals functions;

    virtual void set_affected_signal_ids(MedDictionarySections& dict);

    inline bool is_signal_affected(int signalId) { return (aff_signal_ids.find(signalId) != aff_signal_ids.end()); }

    inline bool is_signal_affected(string& signalName) { return (aff_signals.find(signalName) != aff_signals.end()); }


    virtual void fit_for_repository(MedPidRepository& rep) {};


    virtual void register_virtual_section_name_id(MedDictionarySections& dict) { };


    // check filtering

    virtual bool filter(unordered_set<string>& reqSignals);


    virtual void init_tables(MedDictionarySections& dict, MedSignals& sigs) { return; }


    virtual void init_attributes() { return; }


    // Learning

    // Should be implemented for inheriting classes that require learning </summary>

    virtual int _learn(MedPidRepository& rep, MedSamples& samples, vector<RepProcessor *>& prev_processors) { return 0; };

    // May be implemented for inheriting classes that require learning </summary>

    virtual int _conditional_learn(MedPidRepository& rep, MedSamples& samples, vector<RepProcessor *>& prev_processors, unordered_set<int>& neededSignalIds);


    // Learning envelopes - Here because of issues with overloading and inheritance

    int learn(MedPidRepository& rep, MedSamples& samples, vector<RepProcessor *>& prev_processors) { return _learn(rep, samples, prev_processors); };

    int learn(MedPidRepository& rep);

    int learn(MedPidRepository& rep, MedSamples& samples) { vector<RepProcessor *> temp;  return _learn(rep, samples, temp); }


    virtual int conditional_learn(MedPidRepository& rep, MedSamples& samples, vector<RepProcessor *>& prev_processors, unordered_set<int>& neededSignalIds) {

        return _conditional_learn(rep, samples, prev_processors, neededSignalIds);

    }


    int conditional_learn(MedPidRepository& rep, MedSamples& samples, unordered_set<int>& neededSignalIds) { vector<RepProcessor *> temp;  return _conditional_learn(rep, samples, temp, neededSignalIds); }


    // Applying

    virtual int _apply(PidDynamicRec& rec, vector<int>& time_points, vector<vector<float>>& attributes_vals) { return -1; };

    virtual int _conditional_apply(PidDynamicRec& rec, vector<int>& time_points, unordered_set<int>& neededSignalIds, vector<vector<float>>& attributes_vals);


    // next is needed for efficient applying of a single records

    virtual int _apply_simple(PidDynamicRec& rec, vector<int>& time_points) {

        vector<vector<float>> attributes_vals;

        return _apply(rec, time_points, attributes_vals);

    }


    // Applying envelopes - Here because of issues with overloading and inheritance

    int apply(PidDynamicRec& rec, vector<int>& time_points, vector<vector<float>>& attributes_vals) { return _apply(rec, time_points, attributes_vals); }


    int conditional_apply(PidDynamicRec& rec, vector<int>& time_points, unordered_set<int>& neededSignalIds, vector<vector<float>>& attributes_vals) {

        return _conditional_apply(rec, time_points, neededSignalIds, attributes_vals);

    }


    int apply(PidDynamicRec& rec, MedIdSamples& samples);

    int conditional_apply(PidDynamicRec& rec, MedIdSamples& samples, unordered_set<int>& neededSignalIds);

    int conditional_apply_without_attributes(PidDynamicRec& rec, const MedIdSamples& samples, unordered_set<int>& neededSignalIds);


    // debug prints

    virtual void dprint(const string &pref, int rp_flag);


    virtual void make_summary() {};


    virtual void get_required_signal_categories(unordered_map<string, vector<string>> &signal_categories_in_use) const {};


    // Serialization (including type)

    ADD_CLASS_NAME(RepProcessor)

        ADD_SERIALIZATION_FUNCS(processor_type, req_signals, aff_signals, unconditional)

        void *new_polymorphic(string derived_class_name);


    size_t get_processor_size();

    size_t processor_serialize(unsigned char *blob);


    virtual void print() { fprintf(stderr, "No implementation for print()\n"); }

};


// Utilities

RepProcessorTypes rep_processor_name_to_type(const string& procesor_name);


//.......................................................................................

//.......................................................................................


class RepMultiProcessor : public RepProcessor {

public:

    vector<RepProcessor *> processors;


    vector<vector<int>> attributes_map;


    RepMultiProcessor() { processor_type = REP_PROCESS_MULTI; };

    ~RepMultiProcessor() { clear(); };


    void clear();


    void add_processors_set(RepProcessorTypes type, vector<string>& signals);

    void add_processors_set(RepProcessorTypes type, vector<string>& signals, string init_string);

    void set_required_signal_ids(MedDictionarySections& dict);

    void add_virtual_signals(map<string, int> &_virtual_signals, map<string, string> &_virtual_signals_generic) const;


    void get_required_signal_names(unordered_set<string>& signalNames);

    virtual void get_required_signal_names(unordered_set<string>& signalNames, unordered_set<string> preReqSignals);


    void get_required_signal_ids(unordered_set<int>& signalIds);

    virtual void get_required_signal_ids(unordered_set<int>& signalIds, unordered_set<int> preReqSignals);


    void set_affected_signal_ids(MedDictionarySections& dict);


    void register_virtual_section_name_id(MedDictionarySections& dict);

    // check filtering

    bool filter(unordered_set<string>& reqSignals);


    void set_signal_ids(MedSignals& sigs);


    void init_tables(MedDictionarySections& dict, MedSignals& sigs) { for (RepProcessor * proc : processors) { proc->init_tables(dict, sigs); } }


    void init_attributes();


    int _learn(MedPidRepository& rep, MedSamples& samples, vector<RepProcessor *>& prev_processors);

    int _conditional_learn(MedPidRepository& rep, MedSamples& samples, vector<RepProcessor *>& prev_processors, unordered_set<int>& neededSignalIds);


    int _apply(PidDynamicRec& rec, vector<int>& time_points, vector<vector<float>>& attributes_vals);

    int _apply_simple(PidDynamicRec& rec, vector<int>& time_points);

    int _conditional_apply(PidDynamicRec& rec, vector<int>& time_points, unordered_set<int>& neededSignals, vector<vector<float>>& attributes_mat);


    void fit_for_repository(MedPidRepository& rep);


    void dprint(const string &pref, int rp_flag);


    void make_summary();


    void get_required_signal_categories(unordered_map<string, vector<string>> &signal_categories_in_use) const;


    ADD_CLASS_NAME(RepMultiProcessor)

        ADD_SERIALIZATION_FUNCS(processor_type, processors)


        void print() { for (auto& processor : processors) processor->print(); }

};


#define DEF_REP_TRIMMING_SD_NUM 7

#define DEF_REP_REMOVING_SD_NUM 14


class remove_stats {

public:

    int total_removed = 0, total_pids_touched = 0;

    int total_records = 0, total_pids = 0;


    void restart();


    void print_summary(const string &cleaner_info, const string &signal_name,

        int minimal_pid_cnt, float print_summary_critical_cleaned, bool prnt_flg) const;

};


//.......................................................................................

//.......................................................................................


class RepBasicOutlierCleaner : public RepProcessor, public MedValueCleaner {

private:

    ofstream log_file;

    remove_stats _stats;

    bool is_categ = false;

public:


    string signalName;

    int signalId;

    int time_channel = 0;

    int val_channel = 0;


    string nRem_attr = "";

    string nTrim_attr = "";

    string nRem_attr_suffix = "";

    string nTrim_attr_suffix = "";

    string verbose_file;


    bool print_summary = false;

    float print_summary_critical_cleaned = (float)0.05;


    RepBasicOutlierCleaner() { init_defaults(); }

    RepBasicOutlierCleaner(const string& _signalName) { init_defaults(); signalId = -1; signalName = _signalName; init_lists(); }

    RepBasicOutlierCleaner(const string& _signalName, string init_string) { init_defaults(); signalId = -1; signalName = _signalName; init_from_string(init_string); }

    RepBasicOutlierCleaner(const string& _signalName, ValueCleanerParams *_params) { init_defaults(); signalId = -1; signalName = _signalName; init_lists(); MedValueCleaner::init(_params); }


    void init_defaults() {

        processor_type = REP_PROCESS_BASIC_OUTLIER_CLEANER;

        params.trimming_sd_num = DEF_REP_TRIMMING_SD_NUM; params.removing_sd_num = DEF_REP_REMOVING_SD_NUM; params.nbrs_sd_num = 0;

        params.take_log = 0;

        params.doTrim = params.doRemove = true;

        signalId = -1;

        params.type = VAL_CLNR_ITERATIVE;

        params.missing_value = MED_MAT_MISSING_VALUE;

        verbose_file = "";

    };


    void set_signal(const string& _signalName) { signalId = -1; signalName = _signalName; init_lists(); }


    virtual void set_signal_ids(MedSignals& sigs);


    int init(void *processor_params) { return MedValueCleaner::init(processor_params); };

    virtual int init(map<string, string>& mapper);

    void init_lists();


    void init_attributes();


    int _learn(MedPidRepository& rep, MedSamples& samples, vector<RepProcessor *>& prev_processor);

    int iterativeLearn(MedPidRepository& rep, MedSamples& samples, vector<RepProcessor *>& prev_processor);

    int quantileLearn(MedPidRepository& rep, MedSamples& samples, vector<RepProcessor *>& prev_processor);


    int _apply(PidDynamicRec& rec, vector<int>& time_points, vector<vector<float>>& attributes_mat);


    virtual ~RepBasicOutlierCleaner() { if (!verbose_file.empty() && log_file.is_open()) log_file.close(); };


    void print() { dprint("", 1); }

    void dprint(const string &pref, int rp_flag);


    void make_summary();


    ADD_CLASS_NAME(RepBasicOutlierCleaner)

        ADD_SERIALIZATION_FUNCS(processor_type, signalName, time_channel, val_channel, req_signals, aff_signals, params.take_log, params.missing_value, params.doTrim, params.doRemove,

            trimMax, trimMin, removeMax, removeMin, nRem_attr, nTrim_attr, nRem_attr_suffix, nTrim_attr_suffix, verbose_file,

            print_summary, print_summary_critical_cleaned)

};


//.......................................................................................


class confRecord : public SerializableObject {

public:

    double logicalLow, logicalHigh, confirmedLow, confirmedHigh;

    double trimLow, trimHigh; //<trimming if not above remove borders

    string distLow, distHigh; //"none" "norm" or "log"

    int val_channel = 0;

    ADD_CLASS_NAME(confRecord)

        ADD_SERIALIZATION_FUNCS(logicalLow, logicalHigh, confirmedLow, confirmedHigh,

            distLow, distHigh, trimLow, trimHigh, val_channel)

};


MEDSERIALIZE_SUPPORT(confRecord)


//.......................................................................................

//.......................................................................................


class RepConfiguredOutlierCleaner : public  RepBasicOutlierCleaner {

private:

    string confFileName;

    confRecord outlierParam;

public:

    string cleanMethod;


    RepConfiguredOutlierCleaner() { init_defaults(); }


    void init_defaults() {

        processor_type = REP_PROCESS_CONFIGURED_OUTLIER_CLEANER;

        params.trimming_sd_num = DEF_REP_TRIMMING_SD_NUM; params.removing_sd_num = DEF_REP_REMOVING_SD_NUM; params.nbrs_sd_num = 0;

        params.take_log = 0;

        params.doTrim = params.doRemove = true;

        signalId = -1;

        params.type = VAL_CLNR_ITERATIVE;

        params.missing_value = MED_MAT_MISSING_VALUE;

        verbose_file = "";

    };


    int _learn(MedPidRepository& rep, MedSamples& samples, vector<RepProcessor *>& prev_processor);


    int init(map<string, string>& mapper);


    // Apply cleaning model -inheritted

    void set_signal_ids(MedSignals& sigs);


    ADD_CLASS_NAME(RepConfiguredOutlierCleaner)

        ADD_SERIALIZATION_FUNCS(processor_type, signalName, time_channel, val_channel, req_signals, aff_signals, params.take_log, params.missing_value, params.doTrim, params.doRemove,

            trimMax, trimMin, removeMax, removeMin, confFileName, cleanMethod, outlierParam, nRem_attr, nTrim_attr, nRem_attr_suffix, nTrim_attr_suffix)


        void print();

};


void learnDistributionBorders(float& borderHi, float& borderLo, vector<float> filteredValues);

// a function that takes sorted vector of filtered values and estimates the +- 7 sd borders based on the center of distribution

// predefined calibration constants are used for estimation of the borders.


class RepRuleBasedOutlierCleaner : public RepProcessor {

    // get multiple signals and clean them according to consistency  with other signals from same date

public:


    vector <int> signalIds;

    vector<int> consideredRules;


    string nRem_attr = "";

    string nRem_attr_suffix = "";


    float tolerance = 0.1F;

    int time_window = 0;

    string verbose_file;

    float calc_res = 0;


    bool print_summary = false;

    float print_summary_critical_cleaned = (float)0.05;


    map <int, vector<string>>rules2Signals = {

    {1,{"BMI","Weight","Height"}},

    {2,{"MCH", "Hemoglobin","RBC"}},

    {3,{"MCV","Hematocrit","RBC"} },

    {4,{"MCHC-M","MCH","MCV"}},

    {5,{"Eosinophils#","Monocytes#","Basophils#","Lymphocytes#","Neutrophils#","WBC" }},

    {6,{ "MPV","Platelets_Hematocrit","Platelets" }},

    {7,{"UrineAlbumin","UrineTotalProtein" }},

    {8,{"UrineAlbumin_over_Creatinine","UrineAlbumin","UrineCreatinine" }},

    {9,{"LDL","HDL","Cholesterol"}},

    {10,{"NonHDLCholesterol","HDL","Cholesterol"}},

    {11,{"HDL_over_nonHDL","HDL","NonHDLCholesterol"}},

    {12,{"HDL_over_Cholesterol","HDL","Cholesterol"}},

    {13,{"HDL_over_LDL","HDL","LDL"}},

    {14,{"HDL_over_LDL","LDL_over_HDL"}},

    {15,{"Cholesterol_over_HDL","Cholesterol","HDL"}},

    {17,{"Cholesterol_over_HDL","HDL_over_Cholesterol"}}, //rule 16 canceled

    {18,{"LDL_over_HDL","LDL","HDL"}},

    {19,{"Albumin","Protein_Total"}},

    {20,{"FreeT4","T4"}},

    {21,{"NRBC","RBC"}},

    {22,{"CHADS2","CHADS2_VASC"}},

    {23, {"BP"}}

    };


    map<int, string> rules2RemoveSignal;


    vector <int> rulesToApply;

    unordered_map<string, pair<int, int>> signal_channels;

    unordered_map<int, pair<int, int>> signal_id_channels;


    set <int> reqSignalIds, affSignalIds;

    unordered_map<int, vector<int>> rules_sids;

    unordered_map<int, vector<bool>> affected_by_rules;


    // Constructors

    RepRuleBasedOutlierCleaner() : RepProcessor() { init_defaults(); }


    void init_defaults();


    void parse_rules_signals(const string &path);

    void parse_sig_channels(const string &path);


    // Init

    int init(void *processor_params) { return 0; };


    int init(map<string, string>& mapper);


    void init_attributes();


    void set_signal_ids(MedSignals& sigs);


    void init_tables(MedDictionarySections& dict, MedSignals& sigs);


    // Learning

    int _learn(MedPidRepository& rep, MedSamples& samples, vector<RepProcessor *>& prev_processors) { init_tables(rep.dict, rep.sigs); return 0; };


    int _apply(PidDynamicRec& rec, vector<int>& time_points, vector<vector<float>>& attributes_mat);


    ~RepRuleBasedOutlierCleaner() { if (!verbose_file.empty() && log_file.is_open()) log_file.close(); };


    // Check if some required signals are missing and remove corresponding rules

    void fit_for_repository(MedPidRepository& rep);


    // set affected and required signals lists according to rules

    void init_lists();


    void make_summary();


    void dprint(const string &pref, int rp_flag);


    ADD_CLASS_NAME(RepRuleBasedOutlierCleaner)

        ADD_SERIALIZATION_FUNCS(processor_type, time_window, calc_res, rules2Signals, rulesToApply, rules2RemoveSignal, signal_channels, consideredRules, tolerance, req_signals, aff_signals, nRem_attr,

            nRem_attr_suffix, verbose_file, print_summary, print_summary_critical_cleaned)


private:

    bool  applyRule(int rule, const  vector<UniversalSigVec> &ruleUsvs,

        const vector<int> &val_channels, const vector<int> &sPointer);

    unordered_map<int, string> affected_ids_to_name;

    ofstream log_file;

    unordered_map<string, remove_stats> _rmv_stats;


    void select_rules_to_apply();


};


#define DEF_REP_NBRS_NBRS_SD_NUM 5

#define DEF_REP_NBRS_TRIM_SD_NUM 7

#define DEF_REP_NBRS_REMOVING_SD_NUM 14

//.......................................................................................

//.......................................................................................


class RepNbrsOutlierCleaner : public RepProcessor, public MedValueCleaner {

public:


    string signalName;

    int signalId;

    int time_channel = 0;

    int val_channel = 0;


    int nbr_time_width = 7;

    int nbr_time_unit = MedTime::Days;


    string nRem_attr = "";

    string nTrim_attr = "";

    string nRem_attr_suffix = "";

    string nTrim_attr_suffix = "";


    RepNbrsOutlierCleaner() { init_defaults(); }

    RepNbrsOutlierCleaner(const string& _signalName) { init_defaults(); signalId = -1; signalName = _signalName; init_lists(); }

    RepNbrsOutlierCleaner(const string& _signalName, string init_string) { init_defaults(); signalId = -1; signalName = _signalName; init_from_string(init_string); }

    RepNbrsOutlierCleaner(const string& _signalName, ValueCleanerParams *_params) { init_defaults(); signalId = -1; signalName = _signalName; init_lists(); MedValueCleaner::init(_params); }


    void init_defaults() {

        processor_type = REP_PROCESS_NBRS_OUTLIER_CLEANER;

        params.trimming_sd_num = DEF_REP_NBRS_TRIM_SD_NUM; params.removing_sd_num = DEF_REP_NBRS_REMOVING_SD_NUM; params.nbrs_sd_num = DEF_REP_NBRS_NBRS_SD_NUM;

        params.take_log = 0;

        params.doTrim = params.doRemove = true;

        signalId = -1;

        params.type = VAL_CLNR_ITERATIVE;

        params.missing_value = MED_MAT_MISSING_VALUE;

    };


    void set_signal(const string& _signalName) { signalId = -1; signalName = _signalName; init_lists(); }


    void set_signal_ids(MedSignals& sigs) { signalId = sigs.sid(signalName); }


    int init(void *processor_params) { return MedValueCleaner::init(processor_params); };

    int init(map<string, string>& mapper);

    void init_lists();

    void init_attributes();


    int _learn(MedPidRepository& rep, MedSamples& samples, vector<RepProcessor *>& prev_processor);

    int iterativeLearn(MedPidRepository& rep, MedSamples& samples, vector<RepProcessor *>& prev_processor);

    int quantileLearn(MedPidRepository& rep, MedSamples& samples, vector<RepProcessor *>& prev_processor);


    int _apply(PidDynamicRec& rec, vector<int>& time_points, vector<vector<float>>& attributes_mat);


    // Serialization

    ADD_CLASS_NAME(RepNbrsOutlierCleaner)

        ADD_SERIALIZATION_FUNCS(processor_type, signalName, time_channel, val_channel, req_signals, aff_signals, params.take_log, params.missing_value, params.doTrim, params.doRemove,

            trimMax, trimMin, removeMax, removeMin, nbr_time_unit, nbr_time_width, nbrsMax, nbrsMin, nRem_attr, nTrim_attr, nRem_attr_suffix, nTrim_attr_suffix)


        void print();

};


//.......................................................................................

//.......................................................................................


// Modes of handling multiple values at the same time


typedef enum {

    SIM_VAL_FIRST_VAL,

    SIM_VAL_LAST_VAL,

    SIM_VAL_MEAN,

    SIM_VAL_REM,

    SIM_VAL_REM_DIFF,

    SIM_VAL_MIN,

    SIM_VAL_MAX,

    MULT_VAL_LAST

} SimValHandleTypes;


class RepSimValHandler : public RepProcessor {

public:

    string signalName;

    int signalId;

    vector<int> time_channels;

    SimValHandleTypes handler_type;


    int nValChannels;


    string nHandle_attr = "";

    string nHandle_attr_suffix = "";

    bool debug = false;


    RepSimValHandler() { init_defaults(); processor_type = REP_PROCESS_SIM_VAL; }

    RepSimValHandler(const string& _signalName) { init_defaults(); processor_type = REP_PROCESS_SIM_VAL;; signalId = -1; signalName = _signalName; init_lists(); }

    RepSimValHandler(const string& _signalName, string init_string) { init_defaults(); processor_type = REP_PROCESS_SIM_VAL; signalId = -1; signalName = _signalName; init_from_string(init_string); }


    void set_signal(const string& _signalName) { signalId = -1; signalName = _signalName; init_lists(); }


    void set_signal_ids(MedSignals& sigs) { signalId = sigs.sid(signalName); }


    virtual int init(map<string, string>& mapper);


    void init_lists();


    void init_tables(MedDictionarySections& dict, MedSignals& sigs);


    void init_attributes();


    int _apply(PidDynamicRec& rec, vector<int>& time_points, vector<vector<float>>& attributes_mat);


    static SimValHandleTypes get_sim_val_handle_type(string& name);


    ADD_CLASS_NAME(RepSimValHandler)

        ADD_SERIALIZATION_FUNCS(processor_type, signalName, time_channels, req_signals, aff_signals, nHandle_attr, nHandle_attr_suffix, handler_type)

private:

    void handle_block(int start, int end, UniversalSigVec& usv, vector<int>& remove, int& nRemove, vector<pair<int, vector<float>>>& change, int& nChange, int& nTimes);

    int verbose_cnt = 0;

};


//.......................................................................................

//.......................................................................................


typedef enum {

    REP_CMPLT_RED_LINE_PANEL,

    REP_CMPLT_WHITE_LINE_PANEL,

    REP_CMPLT_PLATELETS_PANEL,

    REP_CMPLT_LIPIDS_PANEL,

    REP_CMPLT_EGFR_PANEL,

    REP_CMPLT_BMI_PANEL,

    REP_CMPLT_GCS,

    REP_CMPLT_LAST

} PanelCompleterTypes;


typedef enum {

    RED_PNL_MCV,

    RED_PNL_HCT,

    RED_PNL_RBC,

    RED_PNL_MCH,

    RED_PNL_MCHC,

    RED_PNL_HGB,

    RED_PNL_LAST

} RedPanelSignals;


typedef enum {

    WHITE_PNL_WBC,

    WHITE_PNL_EOS_N, WHITE_PNL_EOS_P,

    WHITE_PNL_NEU_N, WHITE_PNL_NEU_P,

    WHITE_PNL_LYM_N, WHITE_PNL_LYM_P,

    WHITE_PNL_MON_N, WHITE_PNL_MON_P,

    WHITE_PNL_BAS_N, WHITE_PNL_BAS_P,

    WHITE_PNL_LAST

} WhitePanelSignals;


typedef enum {

    PLT_PNL_PLTS,

    PLT_PNL_PLT_HCT,

    PLT_PNL_MPV,

    PLT_PNL_LAST

} PltsPanelSignals;


typedef enum {

    LIPIDS_PNL_CHOL,

    LIPIDS_PNL_LDL,

    LIPIDS_PNL_HDL,

    LIPIDS_PNL_HDL_OVER_CHOL,

    LIPIDS_PNL_CHOL_OVER_HDL,

    LIPIDS_PNL_HDL_OVER_LDL,

    LIPIDS_PNL_LDL_OVER_HDL,

    LIPIDS_PNL_NON_HDL_CHOL,

    LIPIDS_PNL_HDL_OVER_NON_HDL,

    LIPIDS_PNL_TRGS,

    LIPIDS_PNL_VLDL,

    LIPIDS_PNL_LAST

} LipidsPanelSignals;


typedef enum {

    EGFR_PNL_CRT,

    EGFR_PNL_CKD_EPI,

    EGFR_PNL_MDRD,

    EGFR_PNL_LAST

} eGFRPanelSignals;


typedef enum {

    BMI_PNL_BMI,

    BMI_PNL_WGT,

    BMI_PNL_HGT,

    BMI_PNL_HGT_SQR,

    BMI_PNL_LAST

} BMIPanelSignals;


typedef enum {

    GCS_PNL,

    GCS_PNL_EYE,

    GCS_PNL_MOTOR,

    GCS_PNL_VERBAL,

    GCS_PNL_LAST

} GCSPanelSignals;


class RepPanelCompleter : public RepProcessor {

public:

    // Signals for completions

    vector<vector<string> > panel_signal_names;

    vector<vector<int> > panel_signal_ids;


    // Extra signal ids

    int bdateId, genderId;

    string genderSignalName;


    // Missing value indication

    float missing_val = MED_MAT_MISSING_VALUE;


    // Handling multiple values

    SimValHandleTypes sim_val_handler = SIM_VAL_LAST_VAL;


    // Signals meta-data : original and final resolution and factors

    string metadata_file;

    vector<vector<float> > original_sig_res, final_sig_res, sig_conversion_factors;


    RepPanelCompleter() { processor_type = REP_PROCESS_COMPLETE; init_defaults(); }


    int init(map<string, string>& mapper);


    void init_defaults();


    // Change signal names from defualt

    int update_signal_names(string panel, string& names);


    // Change panels to handle

    int update_panels(string& panels);


    // initialize signal ids

    void set_signal_ids(MedSignals& sigs);


    // Check if some required signals are missing and make them virtual or remove relevant panel completer

    void fit_for_repository(MedPidRepository& rep);


    // dictionary based initializations

    void init_tables(MedDictionarySections &dict, MedSignals& sigs);


    void init_lists();


    // Read conversion and resolution info

    void read_metadata();


    // Learning

    int _learn(MedPidRepository& rep, MedSamples& samples, vector<RepProcessor *>& prev_processors) { init_tables(rep.dict, rep.sigs); read_metadata(); return 0; };


    // Applying

    int _apply(PidDynamicRec& rec, vector<int>& time_points, vector<vector<float>>& attributes_mat);


    // calculators implementations

    int apply_red_line_completer(PidDynamicRec& rec, vector<int>& time_points);

    int apply_white_line_completer(PidDynamicRec& rec, vector<int>& time_points);

    int apply_platelets_completer(PidDynamicRec& rec, vector<int>& time_points);

    int apply_lipids_completer(PidDynamicRec& rec, vector<int>& time_points);

    int apply_eGFR_completer(PidDynamicRec& rec, vector<int>& time_points);

    int apply_BMI_completer(PidDynamicRec& rec, vector<int>& time_points);

    int apply_GCS_completer(PidDynamicRec& rec, vector<int>& time_points);


    // Utilities

    // Age/Gender

    int perpare_for_age_and_gender(PidDynamicRec& rec, int& age, int& bYear, int& gender);


    // Rounding - take care of resolution both in original and final units

    inline float completer_round(float value, float orig_res, float final_res, float factor) { return  set_resolution(set_resolution(value / factor, orig_res) * factor, final_res); }

    inline float set_resolution(float value, float res) { return res * (int)(value / res + 0.5); }


    // Generating panels from usvs

    void get_panels(vector<UniversalSigVec>& usvs, vector<int>& panel_times, vector<vector<float>>& panels, int time_limit, int panel_size);


    // Applying formulas

    int triplet_complete(vector<float>& panel, float factor, int x_idx, int y_idx, int z_idx, vector<float>& orig_res, vector<float>& final_res, vector<float>& conv, vector<int>& changed);

    int sum_complete(vector<float>& panel, int sum, vector<int>& summands, vector<float>& orig_res, vector<float>& final_res, vector<float>& conv, vector<int>& changed);

    int reciprocal_complete(vector<float>& panel, float factor, int x_idx, int y_idx, vector<float>& orig_res, vector<float>& final_res, vector<float>& conv, vector<int>& changed);

    int egfr_complete(vector<float>& panel, float age, int gender, vector<float>& orig_res, vector<float>& final_res, vector<float>& conv, vector<int>& changed);


    // Updating signals in dynamic-rec

    int update_signals(PidDynamicRec& rec, int iver, vector<vector<float>>& panels, vector<int>& panel_times, vector<int>& sigs_ids, vector<int>& changed);


    // serialization. meta-data file is kept for information but not used in apply

    void print();

    ADD_CLASS_NAME(RepPanelCompleter)

        ADD_SERIALIZATION_FUNCS(processor_type, panel_signal_names, missing_val, sim_val_handler, original_sig_res, final_sig_res, sig_conversion_factors, metadata_file, req_signals, aff_signals, virtual_signals)


private:


    map<string, PanelCompleterTypes> panel2type = {

        { "red_line",REP_CMPLT_RED_LINE_PANEL },

        { "white_line",REP_CMPLT_WHITE_LINE_PANEL },

        { "platelets",REP_CMPLT_PLATELETS_PANEL },

        { "lipids",REP_CMPLT_LIPIDS_PANEL },

        { "egfr",REP_CMPLT_EGFR_PANEL },

        { "bmi",REP_CMPLT_BMI_PANEL },

        { "gcs", REP_CMPLT_GCS }

    };


    // definitions and defaults for each panel-completer

    map<string, vector<string> > panel2signals = {

        { "red_line", {"MCV","Hematocrit","RBC","MCH","MCHC-M","Hemoglobin"}},

        { "white_line", {"WBC", "Eosinophils#", "Eosinophils%", "Neutrophils#", "Neutrophils%", "Lymphocytes#", "Lymphocytes%", "Monocytes#", "Monocytes%", "Basophils#", "Basophils%"}},

        { "platelets", {"Platelets", "Platelets_Hematocrit","MPV"}},

        { "lipids",{"Cholesterol", "LDL", "HDL", "HDL_over_Cholesterol", "Cholesterol_over_HDL", "HDL_over_LDL", "LDL_over_HDL", "NonHDLCholesterol", "HDL_over_nonHDL", "Triglycerides", "VLDL"}},

        { "egfr", {"Creatinine","eGFR_CKD_EPI","eGFR_MDRD"}},

        { "bmi", {"BMI","Weight","Height"}},

        {"gcs", {"GCS", "GCS_Eye", "GCS_Motor", "GCS_Verbal"}}

    };


    vector<int> white_panel_nums = { WHITE_PNL_EOS_N,WHITE_PNL_NEU_N,WHITE_PNL_LYM_N,WHITE_PNL_MON_N,WHITE_PNL_BAS_N };

    vector<int> white_panel_precs = { WHITE_PNL_EOS_P,WHITE_PNL_NEU_P,WHITE_PNL_LYM_P,WHITE_PNL_MON_P,WHITE_PNL_BAS_P };

    vector<int> chol_types1 = { LIPIDS_PNL_NON_HDL_CHOL,LIPIDS_PNL_HDL };

    vector<int> chol_types2 = { LIPIDS_PNL_LDL, LIPIDS_PNL_HDL, LIPIDS_PNL_VLDL };

    vector<int> gcs_panel_parts = { GCS_PNL_EYE,GCS_PNL_MOTOR,GCS_PNL_VERBAL };


    //Convertion map for GCS:

    unordered_map<int, int> eye_vals, verbal_vals, motor_vals;

    void convert_gcs_signals(vector<float> &panel);

};


//.......................................................................................


//.......................................................................................

//.......................................................................................


class SimpleCalculator : public SerializableObject {

public:

    vector<string> output_signal_names;

    float missing_value = (float)MED_MAT_MISSING_VALUE;

    string calculator_name = "";

    int work_channel = 0;

    bool need_time = false;

    bool keep_only_in_range = false;


    virtual int init(map<string, string>& mapper) { return 0; };

    virtual void validate_arguments(const vector<string> &input_signals, const vector<string> &output_signals) const {};

    virtual bool do_calc(const vector<float> &vals, float &res) const { HMTHROW_AND_ERR("Error %s::do_calc not implemented\n", my_class_name().c_str()); };

    virtual void list_output_signals(const vector<string> &input_signals, vector<pair<string, string>> &_virtual_signals, const string &output_type) { HMTHROW_AND_ERR("Error %s::do_calc not implemented\n", my_class_name().c_str()); };

    virtual void init_tables(MedDictionarySections& dict, MedSignals& sigs, const vector<string> &input_signals) {};


    virtual void get_required_signal_categories(unordered_map<string, vector<string>> &signal_categories_in_use) const {};


    virtual void fit_for_repository(MedPidRepository &rep, vector<pair<string, string>> &_virtual_signals) {};


    static SimpleCalculator *make_calculator(const string &calc_type);


    virtual ~SimpleCalculator() {};


    void *new_polymorphic(string derived_class_name);


    ADD_CLASS_NAME(SimpleCalculator)

        ADD_SERIALIZATION_FUNCS(calculator_name, missing_value, work_channel, need_time, keep_only_in_range)


};


class EmptyCalculator : public SimpleCalculator {

public:

    EmptyCalculator() { calculator_name = "empty"; keep_only_in_range = false; };


    void validate_arguments(const vector<string> &input_signals, const vector<string> &output_signals) const;

    void list_output_signals(const vector<string> &input_signals, vector<pair<string, string>> &_virtual_signals, const string &output_type);

    bool do_calc(const vector<float> &vals, float &res) const;


    ADD_CLASS_NAME(EmptyCalculator)

        ADD_SERIALIZATION_FUNCS(calculator_name, missing_value, work_channel, need_time, keep_only_in_range)

};


class RatioCalculator : public SimpleCalculator {

public:

    float factor = 1;

    float power_base = 1;

    float power_mone = 1;


    RatioCalculator() { calculator_name = "ratio"; keep_only_in_range = false; };

    int init(map<string, string>& mapper);


    void validate_arguments(const vector<string> &input_signals, const vector<string> &output_signals) const;

    void list_output_signals(const vector<string> &input_signals, vector<pair<string, string>> &_virtual_signals, const string &output_type);

    bool do_calc(const vector<float> &vals, float &res) const;


    ADD_CLASS_NAME(RatioCalculator)

        ADD_SERIALIZATION_FUNCS(calculator_name, missing_value, work_channel, need_time, keep_only_in_range, factor, power_base, power_mone)

};


class KfreCalculator : public SimpleCalculator {

public:


    // Valid values are 2,3 or 6

    //int model_id = 2;

    int n_variables = 4;


    int kfre_version = 2;

    int prediction_years = 5;

    std::string region = "original";        // original,north_american,non_noth_american,pooled

    int region_id;

    bool discard_range_check = false;       // when true, allow computation for input parameters out of valid range


    std::map<std::string, int> region2id = {

        { "original",           0 },

        { "north_american",     1 },

        { "non_north_american", 2 },

        { "pooled",             3 },

    };


    KfreCalculator() { calculator_name = "KFRE"; need_time = true; keep_only_in_range = true; };

    int init(map<string, string>& mapper);

    void validate_arguments(const vector<string> &input_signals, const vector<string> &output_signals) const;

    void list_output_signals(const vector<string> &input_signals, vector<pair<string, string>> &_virtual_signals, const string &output_type);

    bool do_calc(const vector<float> &vals, float &res) const;


    ADD_CLASS_NAME(KfreCalculator)

        ADD_SERIALIZATION_FUNCS(calculator_name, missing_value, work_channel, need_time, n_variables, n_variables, prediction_years, kfre_version, region, discard_range_check)

};


class eGFRCalculator : public SimpleCalculator {

public:

    float ethnicity = 0;

    bool mdrd = false;


    eGFRCalculator() { calculator_name = "eGFR_CKD_EPI"; need_time = true; keep_only_in_range = true; };

    int init(map<string, string>& mapper);

    void validate_arguments(const vector<string> &input_signals, const vector<string> &output_signals) const;

    void list_output_signals(const vector<string> &input_signals, vector<pair<string, string>> &_virtual_signals, const string &output_type);

    bool do_calc(const vector<float> &vals, float &res) const;


    ADD_CLASS_NAME(eGFRCalculator)

        ADD_SERIALIZATION_FUNCS(calculator_name, missing_value, work_channel, need_time, keep_only_in_range, ethnicity, mdrd)

};


class logCalculator : public SimpleCalculator {

public:

    logCalculator() { calculator_name = "log"; keep_only_in_range = false; };


    int init(map<string, string>& mapper);

    void validate_arguments(const vector<string> &input_signals, const vector<string> &output_signals) const;

    void list_output_signals(const vector<string> &input_signals, vector<pair<string, string>> &_virtual_signals, const string &output_type);


    bool do_calc(const vector<float> &vals, float &res) const;


    ADD_CLASS_NAME(logCalculator)

        ADD_SERIALIZATION_FUNCS(calculator_name, missing_value, work_channel, need_time, keep_only_in_range)

};


class SumCalculator : public SimpleCalculator {

public:

    vector<float> factors;

    float b0 = 0;


    SumCalculator() { calculator_name = "sum"; };

    int init(map<string, string>& mapper);

    void validate_arguments(const vector<string> &input_signals, const vector<string> &output_signals) const;

    void list_output_signals(const vector<string> &input_signals, vector<pair<string, string>> &_virtual_signals, const string &output_type);

    bool do_calc(const vector<float> &vals, float &res) const;


    ADD_CLASS_NAME(SumCalculator)

        ADD_SERIALIZATION_FUNCS(calculator_name, missing_value, work_channel, need_time, keep_only_in_range, factors, b0)

};


class RangeCalculator : public SimpleCalculator {

public:

    float min_range;

    float max_range;

    float in_range_val = 1;

    float out_range_val = 0;


    RangeCalculator() { calculator_name = "range"; min_range = MED_MAT_MISSING_VALUE; max_range = MED_MAT_MISSING_VALUE; };

    int init(map<string, string>& mapper);

    void validate_arguments(const vector<string> &input_signals, const vector<string> &output_signals) const;

    void list_output_signals(const vector<string> &input_signals, vector<pair<string, string>> &_virtual_signals, const string &output_type);

    bool do_calc(const vector<float> &vals, float &res) const;


    ADD_CLASS_NAME(RangeCalculator)

        ADD_SERIALIZATION_FUNCS(calculator_name, missing_value, work_channel, need_time, keep_only_in_range, min_range, max_range, in_range_val, out_range_val)

};


class MultiplyCalculator : public SimpleCalculator {

public:

    vector<float> powers;

    float b0 = 1;


    MultiplyCalculator() { calculator_name = "multiply"; };

    int init(map<string, string>& mapper);


    void validate_arguments(const vector<string> &input_signals, const vector<string> &output_signals) const;

    void list_output_signals(const vector<string> &input_signals, vector<pair<string, string>> &_virtual_signals, const string &output_type);


    bool do_calc(const vector<float> &vals, float &res) const;


    ADD_CLASS_NAME(MultiplyCalculator)

        ADD_SERIALIZATION_FUNCS(calculator_name, missing_value, work_channel, need_time, keep_only_in_range, powers, b0)

};


class SetCalculator : public SimpleCalculator {

public:

    vector<string> sets;

    float in_range_val = 1;

    float out_range_val = 0;

    bool regex_on_sets = false;

    SetCalculator() { calculator_name = "set"; };

    int init(map<string, string>& mapper);


    void validate_arguments(const vector<string> &input_signals, const vector<string> &output_signals) const;

    void list_output_signals(const vector<string> &input_signals, vector<pair<string, string>> &_virtual_signals, const string &output_type);

    void init_tables(MedDictionarySections& dict, MedSignals& sigs, const vector<string> &input_signals);


    bool do_calc(const vector<float> &vals, float &res) const;


    void get_required_signal_categories(unordered_map<string, vector<string>> &signal_categories_in_use) const;


    ADD_CLASS_NAME(SetCalculator)

        ADD_SERIALIZATION_FUNCS(calculator_name, missing_value, work_channel, need_time, keep_only_in_range, input_signal, sets, in_range_val, out_range_val)

private:

    vector<char> Flags;

    string input_signal = "";

};


class ExistsCalculator : public SimpleCalculator {

public:

    float in_range_val = 1;

    float out_range_val = 0;


    ExistsCalculator() { calculator_name = "exists"; keep_only_in_range = true; need_time = true; };

    int init(map<string, string>& mapper);


    void validate_arguments(const vector<string> &input_signals, const vector<string> &output_signals) const;

    void list_output_signals(const vector<string> &input_signals, vector<pair<string, string>> &_virtual_signals, const string &output_type);


    bool do_calc(const vector<float> &vals, float &res) const;


    ADD_CLASS_NAME(ExistsCalculator)

        ADD_SERIALIZATION_FUNCS(calculator_name, missing_value, work_channel, need_time, keep_only_in_range, in_range_val, out_range_val)

};


class ConstantValueCalculator : public SimpleCalculator {

private:

    float numeric_val;

public:

    string value;

    bool is_numeric;

    vector<string> additional_dict_vals;

    ConstantValueCalculator() {

        calculator_name = "constant_value"; is_numeric = false; value = ""; numeric_val = 1;

    };


    int init(map<string, string>& mapper);


    void validate_arguments(const vector<string> &input_signals, const vector<string> &output_signals) const;

    void list_output_signals(const vector<string> &input_signals, vector<pair<string, string>> &_virtual_signals, const string &output_type);


    bool do_calc(const vector<float> &vals, float &res) const;


    void fit_for_repository(MedPidRepository &rep, vector<pair<string, string>> &_virtual_signals);


    ADD_CLASS_NAME(ConstantValueCalculator)

        ADD_SERIALIZATION_FUNCS(calculator_name, missing_value, work_channel, need_time, keep_only_in_range, value, is_numeric, numeric_val, output_signal_names, additional_dict_vals)

};


class RepCalcSimpleSignals : public RepProcessor {


public:

    vector<string> V_names;


    string calculator;

    int work_channel = 0;

    int time_channel = 0;

    string output_signal_type;


    float missing_value = (float)MED_MAT_MISSING_VALUE;


    //vector<float> coeff; ///< it is possible to transfer a vector of params to the calculator, to enable parametric calculators.


    vector<string> signals;

    int signals_time_unit = MedTime::Undefined;


    int max_time_search_range = 0;

    string calculator_init_params = "";


    RepCalcSimpleSignals() {

        processor_type = REP_PROCESS_CALC_SIGNALS;

        output_signal_type = "T(i),V(f)";

    }

    ~RepCalcSimpleSignals();


    int init(map<string, string>& mapper);


    // making sure V_ids and sigs_ids are initialized

    void init_tables(MedDictionarySections& dict, MedSignals& sigs);


    void fit_for_repository(MedPidRepository &rep);


    // Learning

    int _learn(MedPidRepository& rep, MedSamples& samples, vector<RepProcessor *>& prev_processors) { init_tables(rep.dict, rep.sigs); return 0; };


    // Applying

    int _apply(PidDynamicRec& rec, vector<int>& time_points, vector<vector<float>>& attributes_mat);


    void print();


    void get_required_signal_categories(unordered_map<string, vector<string>> &signal_categories_in_use) const;


    void register_virtual_section_name_id(MedDictionarySections& dict);


    // serialization

    ADD_CLASS_NAME(RepCalcSimpleSignals)

        ADD_SERIALIZATION_FUNCS(processor_type, calculator, calculator_init_params, max_time_search_range, signals_time_unit,

            signals, V_names, req_signals, aff_signals, virtual_signals, virtual_signals_generic, work_channel, time_channel, calculator_logic, output_signal_type)

        void post_deserialization() {

        SimpleCalculator *p = SimpleCalculator::make_calculator(calculator);

        pass_time_last = p->need_time;

        delete p;

        //if (calculator_logic == NULL) {

        //  printf("write calculator_logic again for %s\n%s\n",

        //      calculator.c_str(), calculator_init_params.c_str());

        //  calculator_logic = SimpleCalculator::make_calculator(calculator);

        //  if (!calculator_init_params.empty())

        //      calculator_logic->init_from_string(calculator_init_params);

        //  calculator_logic->missing_value = missing_value;

        //  vector<pair<string, string>> default_virtual_signals;

        //  calculator_logic->list_output_signals(signals, default_virtual_signals); //init calculator

        //}


    }


private:

    // definitions and defaults for each calculator - all must be filled in for a new calculator

    bool pass_time_last = false;

    const map<string, vector<string>> calc2req_sigs = {

        //--------- level 1 - calculated from raw signals (level0)

        //the general hospital processor's signals must be overridden from outside

        { "calc_eGFR", {"Creatinine", "GENDER", "BDATE"}}

    };


    vector<int> V_ids;

    vector<int> sigs_ids;

    vector<bool> static_input_signals;


    int apply_calc_in_time(PidDynamicRec& rec, vector<int>& time_points);


    SimpleCalculator *calculator_logic = NULL;

    int out_n_val_ch, out_n_time_ch;

};


class RepCombineSignals : public RepProcessor {

public:

    string output_name;

    vector<string> signals;

    vector<float> factors;

    int factor_channel;

    string signal_type;


    RepCombineSignals() {

        processor_type = REP_PROCESS_COMBINE;

        output_name = "";

        factor_channel = 1;

        signal_type = "T(i,i),V(f,f)";

    }


    void register_virtual_section_name_id(MedDictionarySections& dict);


    int init(map<string, string>& mapper);


    void init_tables(MedDictionarySections& dict, MedSignals& sigs);

    void set_required_signal_ids(MedDictionarySections& dict) {};

    void set_affected_signal_ids(MedDictionarySections& dict) {};


    void fit_for_repository(MedPidRepository& rep);

    // Applying

    int _apply(PidDynamicRec& rec, vector<int>& time_points, vector<vector<float>>& attributes_mat);


    void print();

    ADD_CLASS_NAME(RepCombineSignals)

        ADD_SERIALIZATION_FUNCS(processor_type, output_name, signals, factors,

            unconditional, req_signals, aff_signals, virtual_signals,

            virtual_signals_generic, signal_type, factor_channel)

private:

    int v_out_sid = -1;

    vector<int> sigs_ids;

    int v_out_n_time_ch, v_out_n_val_ch;

};


class RepSplitSignal : public RepProcessor {

public:

    string input_name;

    vector<string> names;

    vector<float> factors;

    vector<string> sets;

    int val_channel;

    string output_signal_type;


    RepSplitSignal() { processor_type = REP_PROCESS_SPLIT; input_name = ""; val_channel = 0; output_signal_type = "T(i),T(i),V(f),V(f)"; }


    void register_virtual_section_name_id(MedDictionarySections& dict);


    void init_tables(MedDictionarySections& dict, MedSignals& sigs);

    void set_required_signal_ids(MedDictionarySections& dict) {};

    void set_affected_signal_ids(MedDictionarySections& dict) {};


    int init(map<string, string>& mapper);


    // Applying

    int _apply(PidDynamicRec& rec, vector<int>& time_points, vector<vector<float>>& attributes_mat);


    void get_required_signal_categories(unordered_map<string, vector<string>> &signal_categories_in_use) const;


    void print();

    ADD_CLASS_NAME(RepSplitSignal)

        ADD_SERIALIZATION_FUNCS(processor_type, input_name, names, factors, sets, unconditional,

            req_signals, aff_signals, virtual_signals, virtual_signals_generic, val_channel, output_signal_type)

private:

    int in_sid = -1;

    vector<int> V_ids;

    vector<char> Flags;

    int v_out_n_time_ch, v_out_n_val_ch;

};


class RepAggregationPeriod : public RepProcessor {

public:

    string input_name;

    string output_name;

    vector<string> sets;

    int period;


    int time_unit_win = MedTime::Undefined;

    int time_unit_sig = MedTime::Undefined;


private:

    int in_sid;

    vector<int> V_ids;

    vector<char> lut;


public:

    RepAggregationPeriod() :

        input_name(""), output_name(""), period(0), time_unit_sig(global_default_windows_time_unit), time_unit_win(global_default_windows_time_unit), in_sid(-1) {

        processor_type = REP_PROCESS_AGGREGATION_PERIOD;

    }


    void init_tables(MedDictionarySections& dict, MedSignals& sigs);


    int init(map<string, string>& mapper);


    // Applying

    int _apply(PidDynamicRec& rec, vector<int>& time_points, vector<vector<float>>& attributes_mat);


    void print();


    void get_required_signal_categories(unordered_map<string, vector<string>> &signal_categories_in_use) const;


    ADD_CLASS_NAME(RepAggregationPeriod)

        ADD_SERIALIZATION_FUNCS(processor_type, input_name, output_name, sets, period, req_signals, aff_signals, virtual_signals, virtual_signals_generic, time_unit_win, time_unit_sig, in_sid, V_ids, lut)


};


typedef enum {

    all = 0,

    first = 1,

    last = 2

} range_op_type;


class RepBasicRangeCleaner : public RepProcessor {

private:

    vector<char> lut;

public:


    string signal_name;

    string ranges_name;

    string output_name;

    int signal_id;

    int ranges_id;

    int output_id;

    int time_channel;

    int range_time_channel;

    int output_type;

    int get_values_in_range = 1;

    range_op_type range_operator = range_op_type::all;

    int range_val_channel = -1;

    vector<string> sets;

    bool regex_on_sets = 0;

    int last_n = 0;

    bool do_on_last_n = false;


    RepBasicRangeCleaner() :

        signal_name(""), ranges_name(""), output_name(""), signal_id(-1), ranges_id(-1),

        output_id(-1), time_channel(0), output_type(3) {

        processor_type = REP_PROCESS_BASIC_RANGE_CLEANER;

        range_time_channel = 1;

    }


    void init_tables(MedDictionarySections& dict, MedSignals& sigs);


    void register_virtual_section_name_id(MedDictionarySections& dict);


    bool get_last_n_value(int time, const UniversalSigVec& range_sig, float& last_value);


    virtual int init(map<string, string>& mapper);


    int _apply(PidDynamicRec& rec, vector<int>& time_points, vector<vector<float>>& attributes_mat);


    void fit_for_repository(MedPidRepository& rep);


    ADD_CLASS_NAME(RepBasicRangeCleaner)

        ADD_SERIALIZATION_FUNCS(processor_type, signal_name, ranges_name, output_name, time_channel,

            req_signals, aff_signals, signal_id, ranges_id, output_id, virtual_signals, virtual_signals_generic,

            output_type, get_values_in_range, range_operator, range_val_channel, sets, range_time_channel, last_n, do_on_last_n, regex_on_sets)


        void print();

};


class RepSignalRate : public RepProcessor {

public:

    string output_name;

    string input_name;

    int work_channel;

    float factor;

    string output_signal_type;


    RepSignalRate() {

        processor_type = REP_PROCESS_SIGNAL_RATE; output_name = { "calc_drug_rate" };

        work_channel = 0;  factor = 1;

        output_signal_type = "T(i,i),V(f)";

    }


    int init(map<string, string>& mapper);

    void init_tables(MedDictionarySections& dict, MedSignals& sigs);

    void register_virtual_section_name_id(MedDictionarySections& dict);

    void set_required_signal_ids(MedDictionarySections& dict) {};

    void set_affected_signal_ids(MedDictionarySections& dict) {};


    // Applying

    int _apply(PidDynamicRec& rec, vector<int>& time_points, vector<vector<float>>& attributes_mat);


    void print();

    ADD_CLASS_NAME(RepSignalRate)

        ADD_SERIALIZATION_FUNCS(processor_type, input_name, output_name,

            work_channel, factor, unconditional, req_signals, aff_signals,

            virtual_signals, virtual_signals_generic, output_signal_type)

private:

    int v_out_sid = -1;

    int in_sid = -1;

    int v_out_n_time_ch, v_out_n_val_ch;

};


class RepAggregateSignal : public RepProcessor {

public:

    string signalName;

    string output_name;

    int work_channel;

    int start_time_channel;;

    int end_time_channel;

    float factor;

    int time_window;

    int time_unit;

    float drop_missing_rate;

    bool buffer_first;

    string output_signal_type;


    RepAggregateSignal() {

        processor_type = REP_PROCESS_AGGREGATE;

        work_channel = 0;

        start_time_channel = 0;

        end_time_channel = 0;

        factor = 1;

        time_unit = global_default_windows_time_unit;

        drop_missing_rate = 1;


        output_name = "calc_aggregate";

        time_window = 0;

        buffer_first = true;

        output_signal_type = "T(i),V(f)";

    }


    int init(map<string, string>& mapper);

    void init_tables(MedDictionarySections& dict, MedSignals& sigs);

    void register_virtual_section_name_id(MedDictionarySections& dict);

    void set_required_signal_ids(MedDictionarySections& dict) {};

    void set_affected_signal_ids(MedDictionarySections& dict) {};


    // Applying

    int _apply(PidDynamicRec& rec, vector<int>& time_points, vector<vector<float>>& attributes_mat);


    void print();

    ADD_CLASS_NAME(RepAggregateSignal)

        ADD_SERIALIZATION_FUNCS(processor_type, signalName, output_name, work_channel, factor, time_window, time_unit,

            start_time_channel, end_time_channel, drop_missing_rate, buffer_first, unconditional, req_signals, aff_signals, virtual_signals, virtual_signals_generic, output_signal_type)

private:

    int v_out_sid = -1;

    int in_sid = -1;

    int v_out_n_time_ch, v_out_n_val_ch;

};


//---------------------------------------------------------------------------------------------------------------

// helper structures for RepCreateBitSignal:

//---------------------------------------------------------------------------------------------------------------


struct category_time_interval {

    int first_appearance = 0;

    int last_appearance = 0;

    int n_appearances = 0;

    int last_time = 0; // last_appearance + duration


    category_time_interval() {};

    category_time_interval(int _first, int _last, int _n, int _last_time) { first_appearance = _first; last_appearance = _last; n_appearances = _n; last_time = _last_time; }


    void set(int _first, int _last, int _n, int _last_time) { first_appearance = _first; last_appearance = _last; n_appearances = _n; last_time = _last_time; }

};


struct category_event_state {

    int time = 0;

    int appear_time = 0;

    int categ = 0;

    int type = 0; // 0 : stop, 1: start, 2: last appearance


    category_event_state() {};

    category_event_state(int _time, int _appear_time, int _categ, int _type) { time = _time; appear_time = _appear_time; categ = _categ; type = _type; }

    void set(int _time, int _appear_time, int _categ, int _type) { time = _time; appear_time = _appear_time; categ = _categ; type = _type; }


    bool operator <(const category_event_state &c) const {

        if (time < c.time) return true;

        if (time > c.time) return false;

        if (type < c.type) return true;

        //if (type > c.type) return false;

        //if (end_time < c.end_time) return true;

        return false;

    }

};


struct combination_state {

    int start;

    int state;

    vector<int> last;


    combination_state() {};

    combination_state(int time, int _state, int N) { start = time; state = _state; last.assign(N, -1); }

};


//---------------------------------------------------------------------------------------------------------------

// RepCreateBitSignal

// Given N<32 categories defined as groups of sets on a categorial signal, creates a united signal

// holding in bit j the occurance of each category. Each occurance of a category at time t implies

// a time span of [t, t+max(min_duration, duration in signal (if available))] in which we define it as

// occuring.

// The resulted signal includes each time point in which any of the categories changed, and the new "state"

// at each such time point. "0" state starts at the first sig time point (unless it starts with the category).

// This is useful for example when working on drugs for a specific conditions (such as diabetes drugs)

// It then creates all states of treatment for the patient in a single easy to use signal.

// A virtual dictionary will be added as well.

//

//---------------------------------------------------------------------------------------------------------------


class RepCreateBitSignal : public RepProcessor {

public:

    string in_sig = "";

    string out_virtual = "";

    int t_chan = 0;

    int c_chan = 0;

    int duration_chan = 1;

    int min_duration = 60;

    vector<int> min_durations = {};

    int max_duration = 180;

    float duration_add = 0;

    float duration_mult = 0.1f;

    int dont_look_back = 7;

    int min_clip_time = 7;

    int last_clip_period = 30;

    vector<int> min_jitters = { 0,0,0 };

    int max_min_jitters;

    vector<string> categories_names;

    vector<vector<string>> categories_sets;

    int time_unit_sig = MedTime::Date;

    int time_unit_duration = MedTime::Days;

    bool change_at_prescription_mode = false;

    MedDictionarySections *_dict; // for debug

    // next will NOT be serialized, and is here for debug reasons

    string print_dict = "";

    int time_channels = 1;

    vector<char> all_cat_lut;

    vector<vector<char>> categories_luts;


    RepCreateBitSignal() { processor_type = REP_PROCESS_CREATE_BIT_SIGNAL; };


    int init(map<string, string>& mapper);

    void get_min_jitters(string& jitters_s);

    void init_tables(MedDictionarySections& dict, MedSignals& sigs);

    void register_virtual_section_name_id(MedDictionarySections& dict);

    void set_required_signal_ids(MedDictionarySections& dict) {};

    void set_affected_signal_ids(MedDictionarySections& dict) {};


    // Applying

    int _apply(PidDynamicRec& rec, vector<int>& time_points, vector<vector<float>>& attributes_mat);


    void get_required_signal_categories(unordered_map<string, vector<string>> &signal_categories_in_use) const;


    //void print();

    ADD_CLASS_NAME(RepCreateBitSignal)

        ADD_SERIALIZATION_FUNCS(processor_type, in_sig, out_virtual, aff_signals, req_signals,

            t_chan, c_chan, duration_chan, min_duration, min_durations, max_duration, duration_add, duration_mult, dont_look_back, min_clip_time, last_clip_period, categories_names, categories_sets, time_unit_sig, time_unit_duration, change_at_prescription_mode,

            virtual_signals_generic, time_channels, min_jitters)


private:

    int v_out_sid = -1;

    int in_sid = -1;

    vector<string> registry_values;

};


//---------------------------------------------------------------------------------------------------------------


//.......................................................................................

//.......................................................................................


class RepCheckReq : public RepProcessor {


public:

    vector<string> signalNames;

    vector<int> signalIds;


    vector<int> time_channels;

    int win_from = 0, win_to = 0;

    int window_time_unit = MedTime::Days;


    string attrName = "MissingReq";


    // Helper

    vector<int> sig_time_units;


    RepCheckReq() { processor_type = REP_PROCESS_CHECK_REQ; unconditional = true; }


    int init(map<string, string>& mapper);


    void set_signal_ids(MedSignals& sigs);


    void init_lists();


    void init_tables(MedDictionarySections& dict, MedSignals& sigs);


    void init_attributes() { attributes = { attrName }; }


    // Learning - none done


    // Applying

    int _apply(PidDynamicRec& rec, vector<int>& time_points, vector<vector<float>>& attributes_mat);


    // serialization.

    ADD_CLASS_NAME(RepCheckReq)

        ADD_SERIALIZATION_FUNCS(processor_type, signalNames, time_channels, win_from, win_to, window_time_unit, attrName, req_signals)

};


//----------------------------------------------------------------------------------------

// RepHistoryLimit : given a signal : chomps history to be at a given window relative

//                   to prediction points

//----------------------------------------------------------------------------------------


class RepHistoryLimit : public RepProcessor {

public:

    string signalName;

    int signalId;

    int time_channel = 0;

    int truncate_time_channel = -1;

    int win_time_unit = global_default_windows_time_unit;

    int rep_time_unit = global_default_time_unit; // we assume this is also the samples time unit

    int win_from = 0;

    int win_to = 0;

    int delete_sig = 0;

    int take_last_events = -1;


    RepHistoryLimit() { init_defaults(); }


    // default init

    void init_defaults() { processor_type = REP_PROCESS_HISTORY_LIMIT; }


    // preparations

    void set_signal_ids(MedSignals& sigs) { signalId = sigs.sid(signalName); }


    // learn - nothing to do

    int _learn(MedPidRepository& rep, MedSamples& samples, vector<RepProcessor *>& prev_processor) { return 0; }


    // apply

    int _apply(PidDynamicRec& rec, vector<int>& time_points, vector<vector<float>>& attributes_mat);


    int init(map<string, string>& mapper);


    // helper

    int get_sub_usv_data(UniversalSigVec &usv, int from_time, int to_time, vector<char> &data, int &len);


    void init_lists();


    ADD_CLASS_NAME(RepHistoryLimit)

        ADD_SERIALIZATION_FUNCS(processor_type, signalName, time_channel, win_time_unit, rep_time_unit,

            win_from, win_to, delete_sig, take_last_events, req_signals, aff_signals, truncate_time_channel)


};


//----------------------------------------------------------------------------------------

// RepNumericNoiser : given a numeric signal : adds gaussian noise to each value, with std as user-determined

// fraction of signal std, uniform distribution to time, and probability of dropping value

//----------------------------------------------------------------------------------------


class RepNumericNoiser : public RepProcessor {

private:

    int signalId =-1;

    double stdev;

    bool on_learning = false;

    bool apply_in_test;


    random_device rd;

    //mt19937 gens;

    vector<mt19937> gens;


    //mt19937 generator;


public:

    string signalName;


    int val_channel = 0;

    int time_channel = 0;

    int time_noise = 0;

    float value_noise = 0;

    int truncation = 2;

    float drop_probability = (float)0.0;


    RepNumericNoiser() { init_defaults(); }


    // default init


    void init_defaults() {

        processor_type = REP_PROCESS_NUMERIC_NOISER;

        int N_TH = omp_get_max_threads();

        gens.resize(3*N_TH);

        for (size_t i = 0; i < 3*N_TH; ++i)

            gens[i] = mt19937(rd());

    }


    // preparations

    void set_signal_ids(MedSignals& sigs) { signalId = sigs.sid(signalName); }


    // learn - nothing to do

    int _learn(MedPidRepository& rep, MedSamples& samples, vector<RepProcessor *>& prev_processor);


    // apply

    int _apply(PidDynamicRec& rec, vector<int>& time_points, vector<vector<float>>& attributes_mat);


    int init(map<string, string>& mapper);


    void init_lists();


    ADD_CLASS_NAME(RepNumericNoiser)

        ADD_SERIALIZATION_FUNCS(processor_type, signalName, val_channel, time_channel,

            time_noise, value_noise, drop_probability, req_signals, aff_signals, stdev, truncation, apply_in_test)


};


class RepReoderChannels : public RepProcessor {

private:

    int sid = -1;

public:

    vector<int> new_order;

    string signal_name = "";


    RepReoderChannels() {

        processor_type = REP_PROCESS_REODER_CHANNELS;

    }


    void set_signal_ids(MedSignals& sigs);

    void set_signal(const string& _signalName) { sid = -1; signal_name = _signalName; }


    int init(map<string, string>& mapper);


    // Applying

    int _apply(PidDynamicRec& rec, vector<int>& time_points, vector<vector<float>>& attributes_mat);


    void print();


    ADD_CLASS_NAME(RepReoderChannels)

        ADD_SERIALIZATION_FUNCS(processor_type, req_signals, aff_signals, virtual_signals, virtual_signals_generic, signal_name, new_order)

};


//.......................................................................................

//.......................................................................................

// Utility Functions

//.......................................................................................

//.......................................................................................


int get_values(MedRepository& rep, MedSamples& samples, int signalId, int time_channel, int val_channel, float range_min, float range_max, vector<float>& values,

    vector<RepProcessor *>& prev_cleaners);

int get_values(MedRepository& rep, MedSamples& samples, int signalId, int time_channel, int val_channel, float range_min, float range_max, vector<float>& values);


//=======================================

// Joining the MedSerialze wagon

//=======================================

MEDSERIALIZE_SUPPORT(RepProcessor)

MEDSERIALIZE_SUPPORT(RepMultiProcessor)

MEDSERIALIZE_SUPPORT(RepBasicOutlierCleaner)

MEDSERIALIZE_SUPPORT(RepRuleBasedOutlierCleaner)

MEDSERIALIZE_SUPPORT(RepConfiguredOutlierCleaner)

MEDSERIALIZE_SUPPORT(RepNbrsOutlierCleaner)

MEDSERIALIZE_SUPPORT(RepCalcSimpleSignals)

MEDSERIALIZE_SUPPORT(RepSimValHandler)

MEDSERIALIZE_SUPPORT(RepPanelCompleter)

MEDSERIALIZE_SUPPORT(RepCombineSignals)

MEDSERIALIZE_SUPPORT(RepSplitSignal)

MEDSERIALIZE_SUPPORT(RepAggregationPeriod)

MEDSERIALIZE_SUPPORT(RepBasicRangeCleaner)

MEDSERIALIZE_SUPPORT(RepSignalRate)

MEDSERIALIZE_SUPPORT(RepAggregateSignal)

MEDSERIALIZE_SUPPORT(RepCheckReq)

MEDSERIALIZE_SUPPORT(RepHistoryLimit)

MEDSERIALIZE_SUPPORT(RepCreateBitSignal)

MEDSERIALIZE_SUPPORT(RepReoderChannels)

MEDSERIALIZE_SUPPORT(RepNumericNoiser)

MEDSERIALIZE_SUPPORT(SimpleCalculator)

MEDSERIALIZE_SUPPORT(SetCalculator)

MEDSERIALIZE_SUPPORT(MultiplyCalculator)

MEDSERIALIZE_SUPPORT(logCalculator)

MEDSERIALIZE_SUPPORT(RatioCalculator)

MEDSERIALIZE_SUPPORT(eGFRCalculator)

MEDSERIALIZE_SUPPORT(SumCalculator)

MEDSERIALIZE_SUPPORT(RangeCalculator)

MEDSERIALIZE_SUPPORT(ExistsCalculator)

MEDSERIALIZE_SUPPORT(EmptyCalculator)

MEDSERIALIZE_SUPPORT(ConstantValueCalculator)


#endif

MedValueCleaner.h
A parent class for single-value cleaners.

VAL_CLNR_ITERATIVE
@ VAL_CLNR_ITERATIVE
"iterative"
Definition MedValueCleaner.h:12

RepProcessorTypes
RepProcessorTypes
Define types of repository processors.
Definition RepProcess.h:20

REP_PROCESS_BASIC_RANGE_CLEANER
@ REP_PROCESS_BASIC_RANGE_CLEANER
"basic_range_cleaner" or "range_cln" to activate RepBasicRangeCleaner
Definition RepProcess.h:34

REP_PROCESS_COMPLETE
@ REP_PROCESS_COMPLETE
"complete" to activate RepPanelCompleter
Definition RepProcess.h:27

REP_PROCESS_CREATE_BIT_SIGNAL
@ REP_PROCESS_CREATE_BIT_SIGNAL
"bit_signal" creates a state of categories (typically drugs) encoded in bits. creates RepCreateBitSig...
Definition RepProcess.h:38

REP_PROCESS_REODER_CHANNELS
@ REP_PROCESS_REODER_CHANNELS
"reoder_channels" reorder signal channels. Creates RepReoderChannels
Definition RepProcess.h:40

REP_PROCESS_CATEGORY_DESCENDERS
@ REP_PROCESS_CATEGORY_DESCENDERS
"category_descenders" creates all descenders values for each category value. Creates RepCategoryDesce...
Definition RepProcess.h:39

REP_PROCESS_HISTORY_LIMIT
@ REP_PROCESS_HISTORY_LIMIT
"history_limit" chomps the history for a signal to be at a certain given time window relative to the ...
Definition RepProcess.h:36

REP_PROCESS_SIM_VAL
@ REP_PROCESS_SIM_VAL
"sim_val" or "sim_val_handler" handle multiple simultanous values to activate RepSimValHandler
Definition RepProcess.h:29

REP_PROCESS_BASIC_OUTLIER_CLEANER
@ REP_PROCESS_BASIC_OUTLIER_CLEANER
"basic_outlier_cleaner" or "basic_cln" to activate RepBasicOutlierCleaner
Definition RepProcess.h:22

REP_PROCESS_NUMERIC_NOISER
@ REP_PROCESS_NUMERIC_NOISER
"numeric_noiser" adds gaussian noise to value and uniform noise to time for numeric signal....
Definition RepProcess.h:42

REP_PROCESS_SIGNAL_RATE
@ REP_PROCESS_SIGNAL_RATE
"signal_rate" combine complition for Drug rate based on Drug amount to activate RepSignalRate
Definition RepProcess.h:30

REP_PROCESS_AGGREGATION_PERIOD
@ REP_PROCESS_AGGREGATION_PERIOD
"aggregation_period" - creates RepAggregationPeriod
Definition RepProcess.h:33

REP_PROCESS_CONFIGURED_OUTLIER_CLEANER
@ REP_PROCESS_CONFIGURED_OUTLIER_CLEANER
"configured_outlier_cleaner" or "conf_cln" to activate RepConfiguredOutlierCleaner
Definition RepProcess.h:24

REP_PROCESS_CREATE_REGISTRY
@ REP_PROCESS_CREATE_REGISTRY
"create_registry" creates a registry signal (TimeRange to values). creates RepCreateRegistry
Definition RepProcess.h:37

REP_PROCESS_SPLIT
@ REP_PROCESS_SPLIT
"split" split signal to two signals based on set of values - usefull for example to give diffrent rul...
Definition RepProcess.h:32

REP_PROCESS_AGGREGATE
@ REP_PROCESS_AGGREGATE
"aggregate" - aggregate signal in sliding time window to calc some aggregation function....
Definition RepProcess.h:35

REP_PROCESS_FILTER_BY_CHANNELS
@ REP_PROCESS_FILTER_BY_CHANNELS
"filter_channels" reorder signal channels. Creates RepFilterByChannel
Definition RepProcess.h:41

REP_PROCESS_CALC_SIGNALS
@ REP_PROCESS_CALC_SIGNALS
"calc_signals" or "calculator" to activate RepCalcSimpleSignals
Definition RepProcess.h:26

REP_PROCESS_COMBINE
@ REP_PROCESS_COMBINE
"combine" flatten signals to 1 signal by dates. if conflict chooses based on order given....
Definition RepProcess.h:31

REP_PROCESS_NBRS_OUTLIER_CLEANER
@ REP_PROCESS_NBRS_OUTLIER_CLEANER
"nbrs_outlier_cleaner" or "nbrs_cln" to activate RepNbrsOutlierCleaner
Definition RepProcess.h:23

REP_PROCESS_RULEBASED_OUTLIER_CLEANER
@ REP_PROCESS_RULEBASED_OUTLIER_CLEANER
"rulebased_outlier_cleaner" or "rule_cln" to activate RepRuleBasedOutlierCleaner
Definition RepProcess.h:25

REP_PROCESS_FILTER_BY_DIAG
@ REP_PROCESS_FILTER_BY_DIAG
"filter_by_diag". Creates RepClearSignalByDiag
Definition RepProcess.h:43

REP_PROCESS_CHECK_REQ
@ REP_PROCESS_CHECK_REQ
"req" or "requirements" check compliance with minimal requirement to activate RepCheckReq
Definition RepProcess.h:28

REP_PROCESS_MULTI
@ REP_PROCESS_MULTI
"multi_processor" or "multi" to activate RepMultiProcessor
Definition RepProcess.h:21

rep_processor_name_to_type
RepProcessorTypes rep_processor_name_to_type(const string &procesor_name)
get RepProcessorTypes from name
Definition RepProcess.cpp:25

PanelCompleterTypes
PanelCompleterTypes
RepPanelCompleter fills-in calculatable signal values.
Definition RepProcess.h:802

REP_CMPLT_RED_LINE_PANEL
@ REP_CMPLT_RED_LINE_PANEL
complete values of the red blood line
Definition RepProcess.h:803

REP_CMPLT_LAST
@ REP_CMPLT_LAST
Last not in use.
Definition RepProcess.h:810

REP_CMPLT_BMI_PANEL
@ REP_CMPLT_BMI_PANEL
complete BMI/HIGHT/WEIGHT values
Definition RepProcess.h:808

REP_CMPLT_WHITE_LINE_PANEL
@ REP_CMPLT_WHITE_LINE_PANEL
complete values of the white blood line
Definition RepProcess.h:804

REP_CMPLT_GCS
@ REP_CMPLT_GCS
complete Glasgow Coma Score
Definition RepProcess.h:809

REP_CMPLT_LIPIDS_PANEL
@ REP_CMPLT_LIPIDS_PANEL
complete lipd values
Definition RepProcess.h:806

REP_CMPLT_EGFR_PANEL
@ REP_CMPLT_EGFR_PANEL
complete eGFR values
Definition RepProcess.h:807

REP_CMPLT_PLATELETS_PANEL
@ REP_CMPLT_PLATELETS_PANEL
complete values of platelets measurements
Definition RepProcess.h:805

SimValHandleTypes
SimValHandleTypes
SimValHandler handles multiple values at the same time.
Definition RepProcess.h:721

range_op_type
range_op_type
A filter on signal events using other range signal with time periods.
Definition RepProcess.h:1555

SerializableObject.h
An Abstract class that can be serialized and written/read from file.

ADD_SERIALIZATION_FUNCS
#define ADD_SERIALIZATION_FUNCS(...)
Definition SerializableObject.h:122

MEDSERIALIZE_SUPPORT
#define MEDSERIALIZE_SUPPORT(Type)
Definition SerializableObject.h:108

ConstantValueCalculator
Definition RepProcess.h:1296

EmptyCalculator
Dummy top create empty virtual signal.
Definition RepProcess.h:1076

ExistsCalculator
A is in as exists operation which return binary output res := in_range_val if signal exists otherwise...
Definition RepProcess.h:1278

KfreCalculator
KFRE calculator based on Age, Gender and eGFR (in that order)
Definition RepProcess.h:1113

MedDictionarySections
Definition MedDictionary.h:87

MedIdSamples
MedIdSamples represent a collection of samples of a given id   Additional (optinal) entries: split
Definition MedSamples.h:90

MedPidRepository
Definition MedPidRepository.h:87

MedRepository
Definition InfraMed.h:303

MedSamples
MedSamples represent a collection of samples per different id   The data is conatined in a vector of ...
Definition MedSamples.h:129

MedSignals
Definition MedSignals.h:719

MedTime::Undefined
static const int Undefined
undefined time unit
Definition MedTime.h:24

MedTime::Days
static const int Days
days since 1900/01/01
Definition MedTime.h:28

MedTime::Date
static const int Date
dates are in full regular format YYYYMMDD
Definition MedTime.h:25

MedValueCleaner
Definition MedValueCleaner.h:61

MedValueCleaner::removeMax
float removeMax
Thresholds for removing.
Definition MedValueCleaner.h:71

MedValueCleaner::trimMax
float trimMax
Thresholds for trimming.
Definition MedValueCleaner.h:68

MedValueCleaner::nbrsMax
float nbrsMax
Thresholds for neighbors.
Definition MedValueCleaner.h:74

MedValueCleaner::params
ValueCleanerParams params
Learning parameters.
Definition MedValueCleaner.h:65

MultiplyCalculator
A multiply operation with power on both arguments.
Definition RepProcess.h:1227

MultiplyCalculator::powers
vector< float > powers
power for args
Definition RepProcess.h:1229

PidDynamicRec
Definition MedPidRepository.h:127

RangeCalculator
A simple Range check that return in_range_val if within range and returns out_range_val if outside ra...
Definition RepProcess.h:1205

RangeCalculator::max_range
float max_range
max range check
Definition RepProcess.h:1208

RangeCalculator::min_range
float min_range
min range check
Definition RepProcess.h:1207

RatioCalculator
Calcs ratio between 2 inputs signals.
Definition RepProcess.h:1092

RepAggregateSignal
An Aggregator for signal in sliding time window.
Definition RepProcess.h:1659

RepAggregateSignal::signalName
string signalName
the name of inout signal
Definition RepProcess.h:1661

RepAggregateSignal::set_required_signal_ids
void set_required_signal_ids(MedDictionarySections &dict)
Fill req_signal_ids : parent function just fills from req_signals.
Definition RepProcess.h:1693

RepAggregateSignal::time_window
int time_window
back time window to search for
Definition RepProcess.h:1667

RepAggregateSignal::work_channel
int work_channel
channel for value
Definition RepProcess.h:1663

RepAggregateSignal::output_signal_type
string output_signal_type
same as input signal - will remove later after change init process of RepProcessor to fetch input sig...
Definition RepProcess.h:1671

RepAggregateSignal::set_affected_signal_ids
void set_affected_signal_ids(MedDictionarySections &dict)
Fill aff_signal_ids : parent function just fills from aff_signals.
Definition RepProcess.h:1694

RepAggregateSignal::end_time_channel
int end_time_channel
time channel for start time
Definition RepProcess.h:1665

RepAggregateSignal::buffer_first
bool buffer_first
If true will wait for first buffer to complete.
Definition RepProcess.h:1670

RepAggregateSignal::drop_missing_rate
float drop_missing_rate
If missing time points beyond this threshold will drop measurement.
Definition RepProcess.h:1669

RepAggregateSignal::factor
float factor
final factor to multiply
Definition RepProcess.h:1666

RepAggregateSignal::time_unit
int time_unit
time unit
Definition RepProcess.h:1668

RepAggregateSignal::output_name
string output_name
the name of virtual signal
Definition RepProcess.h:1662

RepAggregationPeriod
creates a signal of time ranges in which the input signal (usually medication) was active ranges are ...
Definition RepProcess.h:1512

RepAggregationPeriod::input_name
string input_name
name of input signal used by the processor
Definition RepProcess.h:1514

RepAggregationPeriod::period
int period
period to consider active after signal in win time units
Definition RepProcess.h:1517

RepAggregationPeriod::sets
vector< string > sets
the sets to check if signal value is in set
Definition RepProcess.h:1516

RepAggregationPeriod::output_name
string output_name
name of signal created by the processor
Definition RepProcess.h:1515

RepBasicOutlierCleaner
A simple cleaner considering each value of a certain signal separatley.
Definition RepProcess.h:341

RepBasicOutlierCleaner::nRem_attr_suffix
string nRem_attr_suffix
Attribute suffix (name is sample is signalName_suffix) for number of removed. not recorded if empty.
Definition RepProcess.h:355

RepBasicOutlierCleaner::nTrim_attr
string nTrim_attr
Attribute name (in sample) for number of trimmed. not recorded if empty.
Definition RepProcess.h:354

RepBasicOutlierCleaner::iterativeLearn
int iterativeLearn(MedPidRepository &rep, MedSamples &samples, vector< RepProcessor * > &prev_processor)
Learning : learn cleaning boundaries using MedValueCleaner's iterative approximation of moments.
Definition RepProcess.cpp:955

RepBasicOutlierCleaner::verbose_file
string verbose_file
cleaning output_file for debuging
Definition RepProcess.h:357

RepBasicOutlierCleaner::init_lists
void init_lists()
Fill req- and aff-signals vectors.
Definition RepProcess.cpp:845

RepBasicOutlierCleaner::init_attributes
void init_attributes()
Init attributes information : Should be implemented for inheriting classes that have attributes.
Definition RepProcess.cpp:918

RepBasicOutlierCleaner::_apply
int _apply(PidDynamicRec &rec, vector< int > &time_points, vector< vector< float > > &attributes_mat)
Apply cleaning model.
Definition RepProcess.cpp:1004

RepBasicOutlierCleaner::signalId
int signalId
id of signal to clean
Definition RepProcess.h:349

RepBasicOutlierCleaner::signalName
string signalName
name of signal to clean
Definition RepProcess.h:348

RepBasicOutlierCleaner::print_summary
bool print_summary
If true will always print clean summary.
Definition RepProcess.h:359

RepBasicOutlierCleaner::quantileLearn
int quantileLearn(MedPidRepository &rep, MedSamples &samples, vector< RepProcessor * > &prev_processor)
Learning : learn cleaning boundaries using MedValueCleaner's quantile approximation of moments.
Definition RepProcess.cpp:978

RepBasicOutlierCleaner::val_channel
int val_channel
value cahnnel to consider in cleaning
Definition RepProcess.h:351

RepBasicOutlierCleaner::dprint
void dprint(const string &pref, int rp_flag)
used for debug prints, each inheriting class can overload this one to get a more precise debug print.
Definition RepProcess.cpp:1184

RepBasicOutlierCleaner::print_summary_critical_cleaned
float print_summary_critical_cleaned
beyond this value will print summary
Definition RepProcess.h:360

RepBasicOutlierCleaner::nTrim_attr_suffix
string nTrim_attr_suffix
Attribute suffix (name is sample is signalName_suffix) for number of trimmed. not recorded if empty.
Definition RepProcess.h:356

RepBasicOutlierCleaner::init
int init(void *processor_params)
Fill required- and affected-signals sets.
Definition RepProcess.h:390

RepBasicOutlierCleaner::init_defaults
void init_defaults()
Initialize to default values.
Definition RepProcess.h:372

RepBasicOutlierCleaner::RepBasicOutlierCleaner
RepBasicOutlierCleaner(const string &_signalName, ValueCleanerParams *_params)
default constructor + setting signal name + initialize from parameters
Definition RepProcess.h:369

RepBasicOutlierCleaner::set_signal_ids
virtual void set_signal_ids(MedSignals &sigs)
Set signal id.
Definition RepProcess.cpp:832

RepBasicOutlierCleaner::RepBasicOutlierCleaner
RepBasicOutlierCleaner(const string &_signalName)
default constructor + setting signal name
Definition RepProcess.h:365

RepBasicOutlierCleaner::time_channel
int time_channel
time channel to consider in cleaning
Definition RepProcess.h:350

RepBasicOutlierCleaner::nRem_attr
string nRem_attr
Attribute name (in sample) for number of removed. not recorded if empty.
Definition RepProcess.h:353

RepBasicOutlierCleaner::RepBasicOutlierCleaner
RepBasicOutlierCleaner()
default constructor
Definition RepProcess.h:363

RepBasicOutlierCleaner::RepBasicOutlierCleaner
RepBasicOutlierCleaner(const string &_signalName, string init_string)
default constructor + setting signal name + initialize from string
Definition RepProcess.h:367

RepBasicOutlierCleaner::_learn
int _learn(MedPidRepository &rep, MedSamples &samples, vector< RepProcessor * > &prev_processor)
learn cleaning boundaries
Definition RepProcess.cpp:939

RepBasicOutlierCleaner::print
void print()
Debug printing.
Definition RepProcess.h:413

RepBasicOutlierCleaner::set_signal
void set_signal(const string &_signalName)
Set signal name and fill affected and required signals sets.
Definition RepProcess.h:384

RepBasicRangeCleaner
Definition RepProcess.h:1560

RepBasicRangeCleaner::signal_name
string signal_name
name of signal to clean
Definition RepProcess.h:1565

RepBasicRangeCleaner::output_name
string output_name
name of output virtual signal - defaults to signal_name + "_" + ranges_name
Definition RepProcess.h:1567

RepBasicRangeCleaner::signal_id
int signal_id
id of signal to clean
Definition RepProcess.h:1568

RepBasicRangeCleaner::ranges_id
int ranges_id
id of signal the defines ranges
Definition RepProcess.h:1569

RepBasicRangeCleaner::time_channel
int time_channel
time channel to consider in cleaning in the original signal
Definition RepProcess.h:1571

RepBasicRangeCleaner::ranges_name
string ranges_name
name of signal that defines ranges
Definition RepProcess.h:1566

RepBasicRangeCleaner::sets
vector< string > sets
sets use to filter ranges_name signal on range_val_channel. If empty or range_val_channel,...
Definition RepProcess.h:1577

RepBasicRangeCleaner::output_id
int output_id
id of output signal
Definition RepProcess.h:1570

RepBasicRangeCleaner::RepBasicRangeCleaner
RepBasicRangeCleaner()
default constructor
Definition RepProcess.h:1583

RepBasicRangeCleaner::range_time_channel
int range_time_channel
range time channel to consider in cleaning. this time channel will be used to check we're not using t...
Definition RepProcess.h:1572

RepBasicRangeCleaner::output_type
int output_type
output signal type - should be identical to input signal type default to range + val type....
Definition RepProcess.h:1573

RepCalcSimpleSignals
A simple class for calculating virtual signals.
Definition RepProcess.h:1325

RepCalcSimpleSignals::signals
vector< string > signals
it is possible to transfer a vector of required signals, to override default ones.
Definition RepProcess.h:1339

RepCalcSimpleSignals::output_signal_type
string output_signal_type
a parameter to be pass to list signal to afect output type
Definition RepProcess.h:1333

RepCalcSimpleSignals::calculator
string calculator
calculator asked for by user
Definition RepProcess.h:1330

RepCalcSimpleSignals::_learn
int _learn(MedPidRepository &rep, MedSamples &samples, vector< RepProcessor * > &prev_processors)
In this class there's never learning - we return 0 immediately.
Definition RepProcess.h:1361

RepCalcSimpleSignals::V_names
vector< string > V_names
names of signals created by the calculator (a calculator can create more than a single signal at a ti...
Definition RepProcess.h:1328

RepCheckReq
RepCheckReq does not actually process the repository but rather check each sample for compliance with...
Definition RepProcess.h:1843

RepCheckReq::init_attributes
void init_attributes()
Init attributes information : Should be implemented for inheriting classes that have attributes.
Definition RepProcess.h:1874

RepCheckReq::signalNames
vector< string > signalNames
Required signals.
Definition RepProcess.h:1846

RepCheckReq::signalIds
vector< int > signalIds
Required signals ids.
Definition RepProcess.h:1847

RepCheckReq::RepCheckReq
RepCheckReq()
constructor
Definition RepProcess.h:1859

RepCheckReq::time_channels
vector< int > time_channels
signals' time channels
Definition RepProcess.h:1849

RepCombineSignals
RepCreateReigsty creates registries (e.g.
Definition RepProcess.h:1423

RepCombineSignals::set_affected_signal_ids
void set_affected_signal_ids(MedDictionarySections &dict)
Fill aff_signal_ids : parent function just fills from aff_signals.
Definition RepProcess.h:1445

RepCombineSignals::signal_type
string signal_type
the signal type definition to create
Definition RepProcess.h:1429

RepCombineSignals::signals
vector< string > signals
names of input signals used by the processor
Definition RepProcess.h:1426

RepCombineSignals::output_name
string output_name
names of signal created by the processor
Definition RepProcess.h:1425

RepCombineSignals::factor_channel
int factor_channel
the factor_channel_number
Definition RepProcess.h:1428

RepCombineSignals::factors
vector< float > factors
factor for each signal
Definition RepProcess.h:1427

RepCombineSignals::set_required_signal_ids
void set_required_signal_ids(MedDictionarySections &dict)
Fill req_signal_ids : parent function just fills from req_signals.
Definition RepProcess.h:1444

RepConfiguredOutlierCleaner
RepConfiguredOutlierCleaner is a simple cleaner considering each value of a certain signal separatley...
Definition RepProcess.h:446

RepConfiguredOutlierCleaner::init_defaults
void init_defaults()
Initialize to default values.
Definition RepProcess.h:456

RepConfiguredOutlierCleaner::cleanMethod
string cleanMethod
cleaning method : "logical" "confirmed" or "learned"
Definition RepProcess.h:451

RepCreateBitSignal
Definition RepProcess.h:1771

RepCreateBitSignal::set_affected_signal_ids
void set_affected_signal_ids(MedDictionarySections &dict)
Fill aff_signal_ids : parent function just fills from aff_signals.
Definition RepProcess.h:1808

RepCreateBitSignal::categories_sets
vector< vector< string > > categories_sets
the sets defining each category.
Definition RepProcess.h:1789

RepCreateBitSignal::set_required_signal_ids
void set_required_signal_ids(MedDictionarySections &dict)
Fill req_signal_ids : parent function just fills from req_signals.
Definition RepProcess.h:1807

RepCreateBitSignal::categories_names
vector< string > categories_names
the names of the categories to create, categories_names[j] will sit at bit j (1 << j)
Definition RepProcess.h:1788

RepHistoryLimit
Definition RepProcess.h:1891

RepHistoryLimit::RepHistoryLimit
RepHistoryLimit()
It given >0 and delete_sig==0 and win_from==0,win_to==0 will use only last events number in the signa...
Definition RepProcess.h:1904

RepHistoryLimit::signalName
string signalName
name of signal to clean
Definition RepProcess.h:1893

RepHistoryLimit::signalId
int signalId
id of signal to clean
Definition RepProcess.h:1894

RepHistoryLimit::_learn
int _learn(MedPidRepository &rep, MedSamples &samples, vector< RepProcessor * > &prev_processor)
learn processing model on a subset of samples. Apply set of preceeding processors on DynamicPidRec be...
Definition RepProcess.h:1913

RepHistoryLimit::set_signal_ids
void set_signal_ids(MedSignals &sigs)
set signal-ids : Should be implemented for inheriting classes that have signalId
Definition RepProcess.h:1910

RepHistoryLimit::init_defaults
void init_defaults()
initialize to default values : Should be implemented for inheriting classes that have parameters
Definition RepProcess.h:1907

RepMultiProcessor
RepMultiProcessor is a repository processor which contains a vector of simpler processors that can be...
Definition RepProcess.h:241

RepMultiProcessor::register_virtual_section_name_id
void register_virtual_section_name_id(MedDictionarySections &dict)
Register section id to section name of new virtual signals.
Definition RepProcess.cpp:826

RepMultiProcessor::add_processors_set
void add_processors_set(RepProcessorTypes type, vector< string > &signals)
Add processors to set.
Definition RepProcess.cpp:731

RepMultiProcessor::init_attributes
void init_attributes()
Init attributes information : Should be implemented for inheriting classes that have attributes.
Definition RepProcess.cpp:757

RepMultiProcessor::set_affected_signal_ids
void set_affected_signal_ids(MedDictionarySections &dict)
Affected Signals : Fill the member set aff_signal_ids.
Definition RepProcess.cpp:473

RepMultiProcessor::init_tables
void init_tables(MedDictionarySections &dict, MedSignals &sigs)
Init required tables : Should be implemented for inheriting classes that have such tables.
Definition RepProcess.h:287

RepMultiProcessor::print
void print()
serialization
Definition RepProcess.h:317

RepMultiProcessor::filter
bool filter(unordered_set< string > &reqSignals)
Check if processor (and 'sub'-processors within) should be applied according to set of required signa...
Definition RepProcess.cpp:489

RepMultiProcessor::_learn
int _learn(MedPidRepository &rep, MedSamples &samples, vector< RepProcessor * > &prev_processors)
learn processors
Definition RepProcess.cpp:584

RepMultiProcessor::_apply
int _apply(PidDynamicRec &rec, vector< int > &time_points, vector< vector< float > > &attributes_vals)
Apply processors.
Definition RepProcess.cpp:622

RepMultiProcessor::get_required_signal_ids
void get_required_signal_ids(unordered_set< int > &signalIds)
Required Signals ids : Fill the unordered set signalNames.
Definition RepProcess.cpp:559

RepMultiProcessor::set_required_signal_ids
void set_required_signal_ids(MedDictionarySections &dict)
Required Signals ids : Fill the member vector - req_signal_ids.
Definition RepProcess.cpp:457

RepMultiProcessor::fit_for_repository
void fit_for_repository(MedPidRepository &rep)
make changes to RepProcessor according to available signals in Repository
Definition RepProcess.cpp:524

RepMultiProcessor::add_virtual_signals
void add_virtual_signals(map< string, int > &_virtual_signals, map< string, string > &_virtual_signals_generic) const
Reporting back virtual signals if there are any.
Definition RepProcess.cpp:567

RepMultiProcessor::attributes_map
vector< vector< int > > attributes_map
A map from the index of an attribute in the list of attributes of each processor to the index in the ...
Definition RepProcess.h:245

RepMultiProcessor::_conditional_apply
int _conditional_apply(PidDynamicRec &rec, vector< int > &time_points, unordered_set< int > &neededSignals, vector< vector< float > > &attributes_mat)
Apply processors that affect any of the needed signals.
Definition RepProcess.cpp:680

RepMultiProcessor::processors
vector< RepProcessor * > processors
Set of processors.
Definition RepProcess.h:243

RepMultiProcessor::get_required_signal_categories
void get_required_signal_categories(unordered_map< string, vector< string > > &signal_categories_in_use) const
returns for each used signal it's used categories
Definition RepProcess.cpp:801

RepMultiProcessor::dprint
void dprint(const string &pref, int rp_flag)
debug prints
Definition RepProcess.cpp:781

RepMultiProcessor::_conditional_learn
int _conditional_learn(MedPidRepository &rep, MedSamples &samples, vector< RepProcessor * > &prev_processors, unordered_set< int > &neededSignalIds)
learn processing model on a subset of samples only if required. Apply set of preceeding processors on...
Definition RepProcess.cpp:603

RepMultiProcessor::set_signal_ids
void set_signal_ids(MedSignals &sigs)
Set signal-ids for all linked signals.
Definition RepProcess.cpp:533

RepMultiProcessor::get_required_signal_names
void get_required_signal_names(unordered_set< string > &signalNames)
Required Signals names : Fill the unordered set signalNames.
Definition RepProcess.cpp:542

RepMultiProcessor::RepMultiProcessor
RepMultiProcessor()
Constructor.
Definition RepProcess.h:248

RepNbrsOutlierCleaner
RepNbrsOutlierCleaner is cleaner that looks at the neighbourhood of a certain signal value.
Definition RepProcess.h:646

RepNbrsOutlierCleaner::_learn
int _learn(MedPidRepository &rep, MedSamples &samples, vector< RepProcessor * > &prev_processor)
learn cleaning boundaries
Definition RepProcess.cpp:2156

RepNbrsOutlierCleaner::iterativeLearn
int iterativeLearn(MedPidRepository &rep, MedSamples &samples, vector< RepProcessor * > &prev_processor)
Learning : learn cleaning boundaries using MedValueCleaner's iterative approximation of moments.
Definition RepProcess.cpp:2171

RepNbrsOutlierCleaner::_apply
int _apply(PidDynamicRec &rec, vector< int > &time_points, vector< vector< float > > &attributes_mat)
Apply cleaning model.
Definition RepProcess.cpp:2209

RepNbrsOutlierCleaner::quantileLearn
int quantileLearn(MedPidRepository &rep, MedSamples &samples, vector< RepProcessor * > &prev_processor)
Learning : learn cleaning boundaries using MedValueCleaner's quantile approximation of moments.
Definition RepProcess.cpp:2190

RepNbrsOutlierCleaner::nbr_time_unit
int nbr_time_unit
time unit for defining neighboring values
Definition RepProcess.h:655

RepNbrsOutlierCleaner::init_defaults
void init_defaults()
Initialize to default values.
Definition RepProcess.h:672

RepNbrsOutlierCleaner::RepNbrsOutlierCleaner
RepNbrsOutlierCleaner(const string &_signalName, string init_string)
default constructor + setting signal name + initialize from string
Definition RepProcess.h:667

RepNbrsOutlierCleaner::val_channel
int val_channel
value cahnnel to consider in cleaning
Definition RepProcess.h:652

RepNbrsOutlierCleaner::set_signal
void set_signal(const string &_signalName)
Set signal name and fill affected and required signals sets.
Definition RepProcess.h:683

RepNbrsOutlierCleaner::time_channel
int time_channel
time channel to consider in cleaning
Definition RepProcess.h:651

RepNbrsOutlierCleaner::RepNbrsOutlierCleaner
RepNbrsOutlierCleaner(const string &_signalName)
default constructor + setting signal name
Definition RepProcess.h:665

RepNbrsOutlierCleaner::RepNbrsOutlierCleaner
RepNbrsOutlierCleaner(const string &_signalName, ValueCleanerParams *_params)
default constructor + setting signal name + initialize from parameters
Definition RepProcess.h:669

RepNbrsOutlierCleaner::init_attributes
void init_attributes()
Init attributes information : Should be implemented for inheriting classes that have attributes.
Definition RepProcess.cpp:2135

RepNbrsOutlierCleaner::nTrim_attr
string nTrim_attr
Attribute name (in sample) for number of trimmed. not recorded if empty.
Definition RepProcess.h:658

RepNbrsOutlierCleaner::nbr_time_width
int nbr_time_width
size of neighborhood for defining neighboring values
Definition RepProcess.h:654

RepNbrsOutlierCleaner::nRem_attr_suffix
string nRem_attr_suffix
Attribute suffix (name is sample is signalName_suffix) for number of removed. not recorded if empty.
Definition RepProcess.h:659

RepNbrsOutlierCleaner::signalName
string signalName
name of signal to clean
Definition RepProcess.h:649

RepNbrsOutlierCleaner::signalId
int signalId
id of signal to clean
Definition RepProcess.h:650

RepNbrsOutlierCleaner::set_signal_ids
void set_signal_ids(MedSignals &sigs)
Set signal id.
Definition RepProcess.h:686

RepNbrsOutlierCleaner::init
int init(void *processor_params)
Fill required- and affected-signals sets.
Definition RepProcess.h:689

RepNbrsOutlierCleaner::nTrim_attr_suffix
string nTrim_attr_suffix
Attribute suffix (name is sample is signalName_suffix) for number of trimmed. not recorded if empty.
Definition RepProcess.h:660

RepNbrsOutlierCleaner::RepNbrsOutlierCleaner
RepNbrsOutlierCleaner()
default constructor
Definition RepProcess.h:663

RepNbrsOutlierCleaner::nRem_attr
string nRem_attr
Attribute name (in sample) for number of removed. not recorded if empty.
Definition RepProcess.h:657

RepNumericNoiser
Definition RepProcess.h:1937

RepNumericNoiser::set_signal_ids
void set_signal_ids(MedSignals &sigs)
set signal-ids : Should be implemented for inheriting classes that have signalId
Definition RepProcess.h:1974

RepNumericNoiser::init_defaults
void init_defaults()
initialize to default values : Should be implemented for inheriting classes that have parameters
Definition RepProcess.h:1965

RepNumericNoiser::signalName
string signalName
name of signal to clean
Definition RepProcess.h:1951

RepPanelCompleter
A Repository panel completer for a complete panel of signals that are dependent and given on same tim...
Definition RepProcess.h:882

RepPanelCompleter::_learn
int _learn(MedPidRepository &rep, MedSamples &samples, vector< RepProcessor * > &prev_processors)
In this class there's never learning - we init tables and return 0 immediately.
Definition RepProcess.h:933

RepProcessor
Definition RepProcess.h:53

RepProcessor::make_processor
static RepProcessor * make_processor(string name)
create a new repository processor from name
Definition RepProcess.cpp:120

RepProcessor::get_processor_size
size_t get_processor_size()
get size of processor + processor_type
Definition RepProcess.cpp:421

RepProcessor::attributes
vector< string > attributes
attributes generated by the processor (optional)
Definition RepProcess.h:65

RepProcessor::conditional_apply
int conditional_apply(PidDynamicRec &rec, vector< int > &time_points, unordered_set< int > &neededSignalIds, vector< vector< float > > &attributes_vals)
apply processing on a single PidDynamicRec at a set of time-points only if required : if any of the s...
Definition RepProcess.h:193

RepProcessor::get_required_signal_names
virtual void get_required_signal_names(unordered_set< string > &signalNames)
Append required signal names to set : parent function just uses req_signals.
Definition RepProcess.cpp:339

RepProcessor::aff_signals
unordered_set< string > aff_signals
names of signals affected by processing
Definition RepProcess.h:61

RepProcessor::unconditional
bool unconditional
indicated that processor should ALWAYS be applied
Definition RepProcess.h:63

RepProcessor::set_affected_signal_ids
virtual void set_affected_signal_ids(MedDictionarySections &dict)
Fill aff_signal_ids : parent function just fills from aff_signals.
Definition RepProcess.cpp:391

RepProcessor::req_signals
unordered_set< string > req_signals
names of signals required for processsing
Definition RepProcess.h:58

RepProcessor::dprint
virtual void dprint(const string &pref, int rp_flag)
used for debug prints, each inheriting class can overload this one to get a more precise debug print.
Definition RepProcess.cpp:399

RepProcessor::_apply
virtual int _apply(PidDynamicRec &rec, vector< int > &time_points, vector< vector< float > > &attributes_vals)
apply processing on a single PidDynamicRec at a set of time-points : Should be implemented for all in...
Definition RepProcess.h:179

RepProcessor::init_defaults
virtual void init_defaults()
initialize to default values : Should be implemented for inheriting classes that have parameters
Definition RepProcess.h:98

RepProcessor::fit_for_repository
virtual void fit_for_repository(MedPidRepository &rep)
make changes to RepProcessor according to available signals in Repository
Definition RepProcess.h:138

RepProcessor::new_polymorphic
void * new_polymorphic(string derived_class_name)
for polymorphic classes that want to be able to serialize/deserialize a pointer * to the derived clas...
Definition RepProcess.cpp:80

RepProcessor::set_signal
virtual void set_signal(const string &_signalName)
set signal-name : Should be implemented for inheriting classes that have signalName
Definition RepProcess.h:101

RepProcessor::conditional_learn
int conditional_learn(MedPidRepository &rep, MedSamples &samples, unordered_set< int > &neededSignalIds)
learn processing model on a subset of ids only if required without preceesing processors
Definition RepProcess.h:174

RepProcessor::register_virtual_section_name_id
virtual void register_virtual_section_name_id(MedDictionarySections &dict)
Register section id to section name of new virtual signals.
Definition RepProcess.h:141

RepProcessor::processor_type
RepProcessorTypes processor_type
type of repository processor
Definition RepProcess.h:56

RepProcessor::set_required_signal_ids
virtual void set_required_signal_ids(MedDictionarySections &dict)
Fill req_signal_ids : parent function just fills from req_signals.
Definition RepProcess.cpp:330

RepProcessor::set_signal_ids
virtual void set_signal_ids(MedSignals &sigs)
set signal-ids : Should be implemented for inheriting classes that have signalId
Definition RepProcess.h:104

RepProcessor::_conditional_apply
virtual int _conditional_apply(PidDynamicRec &rec, vector< int > &time_points, unordered_set< int > &neededSignalIds, vector< vector< float > > &attributes_vals)
apply processing on a single PidDynamicRec at a set of time-points only if required : May be implemen...
Definition RepProcess.cpp:313

RepProcessor::apply
int apply(PidDynamicRec &rec, vector< int > &time_points, vector< vector< float > > &attributes_vals)
apply processing on a single PidDynamicRec at a set of time-points
Definition RepProcess.h:191

RepProcessor::init
virtual int init(void *params)
initialize from a params object : Should be implemented for inheriting classes that have parameters
Definition RepProcess.h:94

RepProcessor::get_required_signal_categories
virtual void get_required_signal_categories(unordered_map< string, vector< string > > &signal_categories_in_use) const
returns for each used signal it's used categories
Definition RepProcess.h:216

RepProcessor::init_attributes
virtual void init_attributes()
Init attributes information : Should be implemented for inheriting classes that have attributes.
Definition RepProcess.h:152

RepProcessor::create_processor
static RepProcessor * create_processor(string &params)
create a new repository processor from parameters string which contains rp_type
Definition RepProcess.cpp:111

RepProcessor::req_signal_ids
unordered_set< int > req_signal_ids
ids of signals required for processing
Definition RepProcess.h:59

RepProcessor::print
virtual void print()
optional printing of processor
Definition RepProcess.h:229

RepProcessor::aff_signal_ids
unordered_set< int > aff_signal_ids
ids of signals affected by processing
Definition RepProcess.h:62

RepProcessor::conditional_apply_without_attributes
int conditional_apply_without_attributes(PidDynamicRec &rec, const MedIdSamples &samples, unordered_set< int > &neededSignalIds)
apply processing on a single PidDynamicRec at a set of time-points given by samples only if required,...
Definition RepProcess.cpp:300

RepProcessor::is_signal_affected
bool is_signal_affected(int signalId)
Check if a signal is affected by processor </summray> true if affected, false if not
Definition RepProcess.h:134

RepProcessor::learn
int learn(MedPidRepository &rep, MedSamples &samples, vector< RepProcessor * > &prev_processors)
learn processing model on a subset of ids. Apply set of preceeding processors on DynamicPidRec before...
Definition RepProcess.h:164

RepProcessor::_learn
virtual int _learn(MedPidRepository &rep, MedSamples &samples, vector< RepProcessor * > &prev_processors)
learn processing model on a subset of samples. Apply set of preceeding processors on DynamicPidRec be...
Definition RepProcess.h:157

RepProcessor::filter
virtual bool filter(unordered_set< string > &reqSignals)
Check if processor (and 'sub'-processors within) should be applied according to set of required signa...
Definition RepProcess.cpp:227

RepProcessor::get_required_signal_ids
virtual void get_required_signal_ids(unordered_set< int > &signalIds)
Append required signal names to set : parent function just uses req_signals.
Definition RepProcess.cpp:367

RepProcessor::conditional_learn
virtual int conditional_learn(MedPidRepository &rep, MedSamples &samples, vector< RepProcessor * > &prev_processors, unordered_set< int > &neededSignalIds)
learn processing model on a subset of samples only if required. Apply set of preceeding processors on...
Definition RepProcess.h:170

RepProcessor::init
virtual int init(map< string, string > &mapper)
initialize from a map : Should be implemented for inheriting classes that have parameters
Definition RepProcess.h:96

RepProcessor::add_virtual_signals
virtual void add_virtual_signals(map< string, int > &_virtual_signals, map< string, string > &_virtual_signals_generic) const
rep processors CREATING virtual signals need to implement this: adding their signals to the pile
Definition RepProcess.h:117

RepProcessor::init_tables
virtual void init_tables(MedDictionarySections &dict, MedSignals &sigs)
Init required tables : Should be implemented for inheriting classes that have such tables.
Definition RepProcess.h:149

RepProcessor::processor_serialize
size_t processor_serialize(unsigned char *blob)
seialize processor + processor_type
Definition RepProcess.cpp:427

RepProcessor::learn
int learn(MedPidRepository &rep, MedSamples &samples)
learn on subset of samples without preceesing processors
Definition RepProcess.h:168

RepProcessor::virtual_signals
vector< pair< string, int > > virtual_signals
virtual signals are created only in rep processors but can be used by any rep processor that comes af...
Definition RepProcess.h:77

RepProcessor::_conditional_learn
virtual int _conditional_learn(MedPidRepository &rep, MedSamples &samples, vector< RepProcessor * > &prev_processors, unordered_set< int > &neededSignalIds)
learn processing model on a subset of samples only if required. Apply set of preceeding processors on...
Definition RepProcess.cpp:215

RepReoderChannels
Definition RepProcess.h:1994

RepReoderChannels::set_signal
void set_signal(const string &_signalName)
set signal-name : Should be implemented for inheriting classes that have signalName
Definition RepProcess.h:2006

RepRuleBasedOutlierCleaner
A cleaner that is based on rules that describe relations of signal values to each other.
Definition RepProcess.h:519

RepRuleBasedOutlierCleaner::signal_id_channels
unordered_map< int, pair< int, int > > signal_id_channels
signal channels (if exists). first is time, second is for val
Definition RepProcess.h:569

RepRuleBasedOutlierCleaner::verbose_file
string verbose_file
cleaning output_file for debuging
Definition RepProcess.h:533

RepRuleBasedOutlierCleaner::init
int init(void *processor_params)
initialize from a params object : Should be implemented for inheriting classes that have parameters
Definition RepProcess.h:586

RepRuleBasedOutlierCleaner::signal_channels
unordered_map< string, pair< int, int > > signal_channels
signal channels (if exists). first is time, second is for val
Definition RepProcess.h:568

RepRuleBasedOutlierCleaner::consideredRules
vector< int > consideredRules
only rules in this list will be considered in this cleaner (read list from jason) rule number 0 means...
Definition RepProcess.h:525

RepRuleBasedOutlierCleaner::rules2RemoveSignal
map< int, string > rules2RemoveSignal
which signal to remove if contradiction found. If not exists default to remove all
Definition RepProcess.h:565

RepRuleBasedOutlierCleaner::signalIds
vector< int > signalIds
Signals to clean.
Definition RepProcess.h:524

RepRuleBasedOutlierCleaner::_learn
int _learn(MedPidRepository &rep, MedSamples &samples, vector< RepProcessor * > &prev_processors)
In this class there's never learning - we init tables and return 0 immediately.
Definition RepProcess.h:603

RepRuleBasedOutlierCleaner::affected_ids_to_name
ADD_SERIALIZATION_FUNCS(processor_type, time_window, calc_res, rules2Signals, rulesToApply, rules2RemoveSignal, signal_channels, consideredRules, tolerance, req_signals, aff_signals, nRem_attr, nRem_attr_suffix, verbose_file, print_summary, print_summary_critical_cleaned) private unordered_map< int, string > affected_ids_to_name
Serialization.
Definition RepProcess.h:622

RepRuleBasedOutlierCleaner::reqSignalIds
set< int > reqSignalIds
Helpers.
Definition RepProcess.h:572

RepSignalRate
Normalize Signal Values by time - divide by time to calculate rate.
Definition RepProcess.h:1620

RepSignalRate::output_name
string output_name
names of signals created by the completer
Definition RepProcess.h:1622

RepSignalRate::set_required_signal_ids
void set_required_signal_ids(MedDictionarySections &dict)
Fill req_signal_ids : parent function just fills from req_signals.
Definition RepProcess.h:1638

RepSignalRate::output_signal_type
string output_signal_type
the output signal type
Definition RepProcess.h:1626

RepSignalRate::work_channel
int work_channel
which channel to change and divide by time
Definition RepProcess.h:1624

RepSignalRate::factor
float factor
additional constant factor
Definition RepProcess.h:1625

RepSignalRate::input_name
string input_name
names of input signals used by the completer
Definition RepProcess.h:1623

RepSignalRate::set_affected_signal_ids
void set_affected_signal_ids(MedDictionarySections &dict)
Fill aff_signal_ids : parent function just fills from aff_signals.
Definition RepProcess.h:1639

RepSimValHandler
Definition RepProcess.h:732

RepSimValHandler::RepSimValHandler
RepSimValHandler(const string &_signalName, string init_string)
default constructor + setting signal name + initialize from string
Definition RepProcess.h:750

RepSimValHandler::RepSimValHandler
RepSimValHandler()
default constructor
Definition RepProcess.h:746

RepSimValHandler::nValChannels
int nValChannels
number of value-channels, important for rem-diff and mean modes
Definition RepProcess.h:739

RepSimValHandler::signalName
string signalName
name of signal to handle
Definition RepProcess.h:734

RepSimValHandler::handler_type
SimValHandleTypes handler_type
type of handling multiple-values
Definition RepProcess.h:737

RepSimValHandler::set_signal
void set_signal(const string &_signalName)
Set signal name and fill affected and required signals sets.
Definition RepProcess.h:753

RepSimValHandler::signalId
int signalId
id of signal to handle
Definition RepProcess.h:735

RepSimValHandler::time_channels
vector< int > time_channels
time channels to consider. All if empty
Definition RepProcess.h:736

RepSimValHandler::RepSimValHandler
RepSimValHandler(const string &_signalName)
default constructor + setting signal name
Definition RepProcess.h:748

RepSimValHandler::set_signal_ids
void set_signal_ids(MedSignals &sigs)
Set signal id.
Definition RepProcess.h:756

RepSplitSignal
split signal based on set of values.
Definition RepProcess.h:1468

RepSplitSignal::output_signal_type
string output_signal_type
same as input signal - will remove later after change init process of RepProcessor to fetch input sig...
Definition RepProcess.h:1475

RepSplitSignal::set_affected_signal_ids
void set_affected_signal_ids(MedDictionarySections &dict)
Fill aff_signal_ids : parent function just fills from aff_signals.
Definition RepProcess.h:1484

RepSplitSignal::factors
vector< float > factors
factor for each output signal
Definition RepProcess.h:1472

RepSplitSignal::names
vector< string > names
names of signal created by the processor
Definition RepProcess.h:1471

RepSplitSignal::set_required_signal_ids
void set_required_signal_ids(MedDictionarySections &dict)
Fill req_signal_ids : parent function just fills from req_signals.
Definition RepProcess.h:1483

RepSplitSignal::sets
vector< string > sets
the sets to check if signal value is in set
Definition RepProcess.h:1473

RepSplitSignal::val_channel
int val_channel
the val channel to look for the sets
Definition RepProcess.h:1474

RepSplitSignal::input_name
string input_name
names of input signal used by the processor
Definition RepProcess.h:1470

SerializableObject
Definition SerializableObject.h:32

SerializableObject::init_from_string
int init_from_string(string init_string)
Init from string.
Definition SerializableObject.cpp:121

SetCalculator
A is in set operation which return binary output res := in_range_val if is in set otherwise out_range...
Definition RepProcess.h:1249

SimpleCalculator
RepCalcSimpleSignals is a rep processor containing several calculators to calculate new signals.
Definition RepProcess.h:1036

SimpleCalculator::list_output_signals
virtual void list_output_signals(const vector< string > &input_signals, vector< pair< string, string > > &_virtual_signals, const string &output_type)
list output signals with default naming
Definition RepProcess.h:1052

SimpleCalculator::do_calc
virtual bool do_calc(const vector< float > &vals, float &res) const
the calc option
Definition RepProcess.h:1050

SimpleCalculator::make_calculator
static SimpleCalculator * make_calculator(const string &calc_type)
Definition RepProcess.cpp:2948

SimpleCalculator::init
virtual int init(map< string, string > &mapper)
init function of calculator
Definition RepProcess.h:1046

SimpleCalculator::validate_arguments
virtual void validate_arguments(const vector< string > &input_signals, const vector< string > &output_signals) const
validates correctness of inputs
Definition RepProcess.h:1048

SimpleCalculator::init_tables
virtual void init_tables(MedDictionarySections &dict, MedSignals &sigs, const vector< string > &input_signals)
init operator based on repo if needed
Definition RepProcess.h:1054

SumCalculator
does linear sum with factors on vector of input signals, has b0 as constant res := b0 + sum_sigma(i=1...
Definition RepProcess.h:1185

SumCalculator::factors
vector< float > factors
the factors given by ","
Definition RepProcess.h:1187

ValueCleanerParams
Definition MedValueCleaner.h:17

confRecord
Parameters for configured outliers cleaner.
Definition RepProcess.h:428

eGFRCalculator
eGFR calculator based on Creatinine, Gender and Age.
Definition RepProcess.h:1147

logCalculator
does log operation
Definition RepProcess.h:1166

remove_stats
Definition RepProcess.h:323

remove_stats::restart
void restart()
restarts stats for new apply
Definition RepProcess.cpp:1141

remove_stats::print_summary
void print_summary(const string &cleaner_info, const string &signal_name, int minimal_pid_cnt, float print_summary_critical_cleaned, bool prnt_flg) const
prints stats to screen for cleaner
Definition RepProcess.cpp:1149

omp.h
header to handle OpenMP compatibility issues

category_event_state
Definition RepProcess.h:1727

category_time_interval
Definition RepProcess.h:1715

combination_state
Definition RepProcess.h:1748