dataset_loader.h

#ifndef LIGHTGBM_DATASET_LOADER_H_
#define LIGHTGBM_DATASET_LOADER_H_
 
#include <LightGBM/dataset.h>
 
namespace LightGBM {
 
class DatasetLoader {
public:
  LIGHTGBM_EXPORT DatasetLoader(const Config& io_config, const PredictFunction& predict_fun, int num_class, const char* filename);
 
  LIGHTGBM_EXPORT ~DatasetLoader();
 
  LIGHTGBM_EXPORT Dataset* LoadFromFile(const char* filename, const char* initscore_file, int rank, int num_machines);
 
  LIGHTGBM_EXPORT Dataset* LoadFromFile(const char* filename, const char* initscore_file) {
    return LoadFromFile(filename, initscore_file, 0, 1);
  }
 
  LIGHTGBM_EXPORT Dataset* LoadFromFileAlignWithOtherDataset(const char* filename, const char* initscore_file, const Dataset* train_data);
 
  LIGHTGBM_EXPORT Dataset* CostructFromSampleData(double** sample_values,
    int** sample_indices, int num_col, const int* num_per_col,
    size_t total_sample_size, data_size_t num_data);
 
  DatasetLoader& operator=(const DatasetLoader&) = delete;
  DatasetLoader(const DatasetLoader&) = delete;
 
private:
  Dataset* LoadFromBinFile(const char* data_filename, const char* bin_filename, int rank, int num_machines, int* num_global_data, std::vector<data_size_t>* used_data_indices);
 
  void SetHeader(const char* filename);
 
  void CheckDataset(const Dataset* dataset);
 
  std::vector<std::string> LoadTextDataToMemory(const char* filename, const Metadata& metadata, int rank, int num_machines, int* num_global_data, std::vector<data_size_t>* used_data_indices);
 
  std::vector<std::string> SampleTextDataFromMemory(const std::vector<std::string>& data);
 
  std::vector<std::string> SampleTextDataFromFile(const char* filename, const Metadata& metadata, int rank, int num_machines, int* num_global_data, std::vector<data_size_t>* used_data_indices);
 
  void ConstructBinMappersFromTextData(int rank, int num_machines, const std::vector<std::string>& sample_data, const Parser* parser, Dataset* dataset);
 
  void ExtractFeaturesFromMemory(std::vector<std::string>& text_data, const Parser* parser, Dataset* dataset);
 
  void ExtractFeaturesFromFile(const char* filename, const Parser* parser, const std::vector<data_size_t>& used_data_indices, Dataset* dataset);
 
  std::string CheckCanLoadFromBin(const char* filename);
 
  const Config& config_;
  Random random_;
  const PredictFunction& predict_fun_;
  int num_class_;
  int label_idx_;
  int weight_idx_;
  int group_idx_;
  std::unordered_set<int> ignore_features_;
  std::vector<std::string> feature_names_;
  std::unordered_set<int> categorical_features_;
};
 
}
 
#endif  // LIGHTGBM_DATASET_LOADER_H_