SampleFilter.h

#ifndef _SAMPLE_FILTER_H_
#define _SAMPLE_FILTER_H_
 
#include "InfraMed/InfraMed/InfraMed.h"
#include "MedProcessTools/MedProcessTools/MedSamples.h"
#include <SerializableObject/SerializableObject/SerializableObject.h>
#include "InfraMed/InfraMed/MedPidRepository.h"
#include "MedProcessTools/MedProcessTools/MedValueCleaner.h"
#include <MedMat/MedMat/MedMat.h>
 
#define DEFAULT_SMPL_FLTR_NTHREADS 8
 
//.......................................................................................
//.......................................................................................
 
//.......................................................................................
//.......................................................................................
typedef enum {
    SMPL_FILTER_TRN,
    SMPL_FILTER_TST,
    SMPL_FILTER_OUTLIERS,
    SMPL_FILTER_MATCH,
    SMPL_FILTER_REQ_SIGNAL,
    SMPL_FILTER_BASIC,
    SMPL_FILTER_LAST
} SampleFilterTypes;
 
class SampleFilter : public SerializableObject {
public:
 
    // Type
    SampleFilterTypes filter_type; 
 
    // create a new sample filter
    static SampleFilter *make_filter(string name); 
    static SampleFilter *make_filter(SampleFilterTypes type); 
    static SampleFilter *make_filter(string name, string params); 
    static SampleFilter *make_filter(SampleFilterTypes type, string params); 
 
    // Initialization
    virtual int init(void *params) { return 0; }; 
    virtual int init(map<string, string>& mapper) { return 0; }; 
    virtual void init_defaults() {}; 
 
    // Learning : Actually learn 
    virtual int _learn(MedRepository& rep, MedSamples& samples) { return _learn(samples); } 
    virtual int _learn(MedSamples& samples) { return 0; }
 
    // Learning : Envelopes (Here because of probelsm with overload + inheritance)
    virtual int learn(MedRepository& rep, MedSamples& samples) { return _learn(rep,samples); }
    virtual int learn(MedSamples& samples) { return _learn(samples); }
 
    // Filtering
    virtual int _filter(MedRepository& rep, MedSamples& inSamples, MedSamples& outSamples) {return _filter(inSamples,outSamples) ;}
    virtual int _filter(MedSamples& inSamples, MedSamples& outSamples) = 0;
 
    // Filtering : Envelopes (Here because of probelsm with overload + inheritance)
    virtual int filter(MedRepository& rep, MedSamples& inSamples, MedSamples& outSamples) { return _filter(rep, inSamples, outSamples); }
    int filter(MedRepository& rep, MedSamples& samples);
    virtual int filter(MedSamples& inSamples, MedSamples& outSamples) { return _filter(inSamples, outSamples); }
    int filter(MedSamples& samples);
 
    virtual void get_required_signals(vector<string>& req_sigs) {return; }
 
    // Serialization (including type)
    ADD_CLASS_NAME(SampleFilter)
    ADD_SERIALIZATION_FUNCS(filter_type)
    virtual void *new_polymorphic(string derived_class_name);
 
    size_t get_filter_size();
    size_t filter_serialize(unsigned char *blob);
};
 
// Utilities
SampleFilterTypes sample_filter_name_to_type(const string& filter_name);
 
//.......................................................................................
//.......................................................................................
class BasicTrainFilter : public SampleFilter {
public:
 
    BasicTrainFilter() { filter_type = SMPL_FILTER_TRN; };
 
    int _filter(MedSamples& inSamples, MedSamples& outSamples);
    ADD_CLASS_NAME(BasicTrainFilter)
    ADD_SERIALIZATION_FUNCS(filter_type)
};
 
//.......................................................................................
//.......................................................................................
class BasicTestFilter : public SampleFilter {
public:
 
    BasicTestFilter() { filter_type = SMPL_FILTER_TST; };
    ~BasicTestFilter() {};
 
    int _filter(MedSamples& inSamples, MedSamples& outSamples);
 
    ADD_CLASS_NAME(BasicTestFilter)
    ADD_SERIALIZATION_FUNCS(filter_type)
};
 
#define SMPL_FLTR_TRIMMING_SD_NUM 7
#define SMPL_FLTR_REMOVING_SD_NUM 7
//.......................................................................................
//.......................................................................................
class OutlierSampleFilter : public SampleFilter, public MedValueCleaner {
public:
 
    OutlierSampleFilter() : SampleFilter() { init_defaults(); };
 
    int _learn(MedSamples& samples);
    int iterativeLearn(MedSamples& samples);
    int quantileLearn(MedSamples& samples);
    void get_values(MedSamples& samples, vector<float>& values);
 
    int _filter(MedSamples& inSamples, MedSamples& outSamples);
 
    int init(map<string, string>& mapper) { return MedValueCleaner::init(mapper); }
    void init_defaults() {
        filter_type = SMPL_FILTER_OUTLIERS;
        params.trimming_sd_num = SMPL_FLTR_TRIMMING_SD_NUM; params.removing_sd_num = SMPL_FLTR_REMOVING_SD_NUM;
        params.take_log = 0;
        params.doTrim = false;
        params.doRemove = true;
        params.type = VAL_CLNR_ITERATIVE;
        params.missing_value = MED_MAT_MISSING_VALUE;
    };
 
    // Serialization
    ADD_CLASS_NAME(OutlierSampleFilter)
    ADD_SERIALIZATION_FUNCS(filter_type, params.take_log, removeMax, removeMin)
};
 
//.......................................................................................
//.......................................................................................
typedef enum {
    SMPL_MATCH_SIGNAL, 
    SMPL_MATCH_AGE,    
    SMPL_MATCH_TIME,   
    SMPL_MATCH_FTR,   
    SMPL_MATCH_LAST
} SampleMatchingType;
 
//.......................................................................................
//.......................................................................................
class matchingParams : public SerializableObject {
public:
 
    SampleMatchingType match_type; 
 
    // Matching details
    string signalName; 
    string featureName,resolvedFeatureName; 
    int timeWindow, windowTimeUnit ; 
    int matchingTimeUnit; 
    float resolution ; 
 
    int signalId; 
    bool isTimeDependent; 
    int signalTimeUnit; 
 
    // Serialization
    // Serialization
    ADD_CLASS_NAME(matchingParams)
    ADD_SERIALIZATION_FUNCS(match_type, signalName, featureName, timeWindow, matchingTimeUnit, resolution)
};
 
//.......................................................................................
//.......................................................................................
class MatchingSampleFilter : public SampleFilter {
public:
 
    vector<matchingParams> matchingStrata; 
 
    float eventToControlPriceRatio = 100.0; 
    int min_group_size = 5; 
    float maxControlToEventRatio = -1.0; 
    int verbose = 0; 
    float match_to_prior = -1; 
 
    // helpers
    int samplesTimeUnit; 
    int bdateId; 
 
    MatchingSampleFilter() { init_defaults(); };
 
    // Initialization
    int init(map<string, string>& mapper);
    void init_defaults() { filter_type = SMPL_FILTER_MATCH; };
 
    int addMatchingStrata(string& init_string);
 
    // Utilities
    bool isRepRequired();
    bool isAgeRequired();
    int getSampleSignature(MedSample& sample, MedFeatures& features, int i, MedRepository& rep, string& signature);
    int addToSampleSignature(MedSample& sample, matchingParams& stratum, MedFeatures& features, int i, MedRepository& rep, string& signature);
    int initHelpers(MedSamples& inSamples, MedFeatures& features, MedRepository& rep);
    void get_required_signals(vector<string>& req_sigs);
 
    int _filter(MedRepository& rep, MedSamples& inSamples, MedSamples& outSamples);
    int _filter(MedSamples& inSamples, MedSamples& outSamples);
 
    int _filter(MedFeatures& features, MedRepository& rep, MedSamples& inSamples, MedSamples& outSamples);
    int _filter(MedFeatures& features, MedSamples& inSamples, MedSamples& outSamples);
 
    // Serialization
    ADD_CLASS_NAME(MatchingSampleFilter)
    ADD_SERIALIZATION_FUNCS(filter_type, matchingStrata, eventToControlPriceRatio, maxControlToEventRatio, min_group_size, match_to_prior)
};
 
//.......................................................................................
//.......................................................................................
class RequiredSignalFilter : public SampleFilter {
public:
 
    string signalName; 
    int timeWindow, windowTimeUnit; 
 
    RequiredSignalFilter() { init_defaults(); };
 
    int init(map<string, string>& mapper);
    void init_defaults();
 
    // Filter
    int _filter(MedRepository& rep, MedSamples& inSamples, MedSamples& outSamples);
    int _filter(MedSamples& inSamples, MedSamples& outSamples);
 
    // Serialization
    ADD_CLASS_NAME(RequiredSignalFilter)
    ADD_SERIALIZATION_FUNCS(filter_type, signalName, timeWindow, windowTimeUnit)
 
};
 
//.......................................................................................
//.......................................................................................
struct BasicFilteringParams : public SerializableObject {
    string sig_name; 
    int win_from = 0; 
    int win_to = (int)(1<<30); 
    float min_val = -1e10; 
    float max_val = 1e10;
    int min_Nvals = 1; ;
    int time_channel = 0; 
    int val_channel = 0; 
 
    int init_from_string(const string &init_str);
    int test_filter(MedSample &sample, MedRepository &rep, int win_time_unit);
 
    ADD_CLASS_NAME(BasicFilteringParams)
    ADD_SERIALIZATION_FUNCS(sig_name, time_channel, val_channel, win_from, win_to, min_val, max_val, min_Nvals)
 
private:
    int sig_id = -1; 
    int use_byear= 0; 
};              
 
//...........................................................................................................
//...........................................................................................................
class BasicSampleFilter : public SampleFilter {
public:
 
    // filtering parameters
    int min_sample_time = 0; 
    int max_sample_time = (int)(1<<30); 
    vector<BasicFilteringParams> bfilters; 
    int min_bfilter = -1; 
    int winsTimeUnit = MedTime::Days; 
 
    // next is initialized with init string
    vector<string> req_sigs; 
 
    int init(map<string, string>& mapper);
    void get_required_signals(vector<string>& req_sigs);
 
    // Filter
    int _filter(MedRepository& rep, MedSamples& inSamples, MedSamples& outSamples);
    int _filter(MedSamples& inSamples, MedSamples& outSamples);
 
    ADD_CLASS_NAME(BasicSampleFilter)
    ADD_SERIALIZATION_FUNCS(filter_type, min_sample_time, max_sample_time, bfilters, min_bfilter, winsTimeUnit)
};
 
//...........................................................................................................
//...........................................................................................................
class SanitySimpleFilter : public SerializableObject {
public:
    string sig_name; 
    int win_from = 0; 
    int win_to = (int)(1 << 30); 
    float min_val = -1e10; 
    float max_val = 1e10;
    int min_Nvals = -1; ;
    int max_Nvals = -1; ;
    int time_channel = 0; 
    int val_channel = 0; 
    int max_outliers = -1; 
    int win_time_unit = MedTime::Days; 
    int samples_time_unit = MedTime::Date; 
    unordered_set<float> allowed_values; 
    int values_in_dictionary = 0; 
    int min_left = -1; 
 
 
 
    int init_from_string(const string &init_str);
    int test_filter(MedSample &sample, MedRepository &rep) {
        int nvals, noutliers; return test_filter(sample, rep, nvals, noutliers);
    }
    int test_filter(MedSample &sample, MedRepository &rep, int &nvals, int &noutliers);
 
    // test_filter return codes
    const static int Passed = 0;
    const static int Failed = 1; // General fail due to reasons different than the following
    const static int Signal_Not_Valid = 2;
    const static int Failed_Min_Nvals = 3;
    const static int Failed_Max_Nvals = 4;
    const static int Failed_Outliers = 5;
    const static int Failed_Age = 6;
    const static int Failed_Age_No_Byear = 7;
    const static int Failed_Allowed_Values = 8;
    const static int Failed_Dictionary_Test = 9;
    const static int Failed_Not_Enough_Non_Outliers_Left = 10;
 
    ADD_CLASS_NAME(SanitySimpleFilter)
    ADD_SERIALIZATION_FUNCS(sig_name, time_channel, val_channel, win_from, win_to, min_val, max_val, min_Nvals, max_Nvals, allowed_values, values_in_dictionary, max_outliers, win_time_unit)
 
private:
    int sig_id = -1; 
    int section_id = -1; 
    int bdate_id = -1;
    bool used_byear = false; 
};
 
//=======================================
// Joining the MedSerialze wagon
//=======================================
MEDSERIALIZE_SUPPORT(SampleFilter)
MEDSERIALIZE_SUPPORT(SanitySimpleFilter)
MEDSERIALIZE_SUPPORT(BasicFilteringParams)
MEDSERIALIZE_SUPPORT(BasicSampleFilter)
MEDSERIALIZE_SUPPORT(MatchingSampleFilter)
MEDSERIALIZE_SUPPORT(BasicTrainFilter)
MEDSERIALIZE_SUPPORT(BasicTestFilter)
MEDSERIALIZE_SUPPORT(OutlierSampleFilter)
MEDSERIALIZE_SUPPORT(matchingParams)
MEDSERIALIZE_SUPPORT(RequiredSignalFilter)
 
#endif