dataset.h

#ifndef LIGHTGBM_DATASET_H_
#define LIGHTGBM_DATASET_H_
 
#include <LightGBM/utils/random.h>
#include <LightGBM/utils/text_reader.h>
#include <LightGBM/utils/openmp_wrapper.h>
 
#include <LightGBM/meta.h>
#include <LightGBM/config.h>
#include <LightGBM/feature_group.h>
 
#include <vector>
#include <utility>
#include <functional>
#include <string>
#include <unordered_set>
#include <mutex>
 
namespace LightGBM {
 
class DatasetLoader;
class Metadata {
public:
  Metadata();
  void Init(const char* data_filename, const char* initscore_file);
  void Init(const Metadata& metadata, const data_size_t* used_indices, data_size_t num_used_indices);
  void LoadFromMemory(const void* memory);
  ~Metadata();
 
  void Init(data_size_t num_data, int weight_idx, int query_idx);
 
  void PartitionLabel(const std::vector<data_size_t>& used_indices);
 
  void CheckOrPartition(data_size_t num_all_data,
                        const std::vector<data_size_t>& used_data_indices);
 
  void SetLabel(const label_t* label, data_size_t len);
 
  void SetWeights(const label_t* weights, data_size_t len);
 
  void SetQuery(const data_size_t* query, data_size_t len);
 
  void SetInitScore(const double* init_score, data_size_t len);
 
 
  void SaveBinaryToFile(const VirtualFileWriter* writer) const;
 
  size_t SizesInByte() const;
 
  inline const label_t* label() const { return label_.data(); }
 
  inline void SetLabelAt(data_size_t idx, label_t value) {
    label_[idx] = value;
  }
 
  inline void SetWeightAt(data_size_t idx, label_t value) {
    weights_[idx] = value;
  }
 
  inline void SetQueryAt(data_size_t idx, data_size_t value) {
    queries_[idx] = static_cast<data_size_t>(value);
  }
 
  inline const label_t* weights() const {
    if (!weights_.empty()) {
      return weights_.data();
    } else {
      return nullptr;
    }
  }
 
  inline const data_size_t* query_boundaries() const {
    if (!query_boundaries_.empty()) {
      return query_boundaries_.data();
    } else {
      return nullptr;
    }
  }
 
  inline data_size_t num_queries() const { return num_queries_; }
 
  inline const label_t* query_weights() const {
    if (!query_weights_.empty()) {
      return query_weights_.data();
    } else {
      return nullptr;
    }
  }
 
  inline const double* init_score() const {
    if (!init_score_.empty()) {
      return init_score_.data();
    } else {
      return nullptr;
    }
  }
 
  inline int64_t num_init_score() const { return num_init_score_; }
 
  Metadata& operator=(const Metadata&) = delete;
  Metadata(const Metadata&) = delete;
 
private:
  void LoadInitialScore(const char* initscore_file);
  void LoadWeights();
  void LoadQueryBoundaries();
  void LoadQueryWeights();
  std::string data_filename_;
  data_size_t num_data_;
  data_size_t num_weights_;
  std::vector<label_t> label_;
  std::vector<label_t> weights_;
  std::vector<data_size_t> query_boundaries_;
  std::vector<label_t> query_weights_;
  data_size_t num_queries_;
  int64_t num_init_score_;
  std::vector<double> init_score_;
  std::vector<data_size_t> queries_;
  std::mutex mutex_;
  bool weight_load_from_file_;
  bool query_load_from_file_;
  bool init_score_load_from_file_;
};
 
 
class Parser {
public:
  virtual ~Parser() {}
 
  virtual void ParseOneLine(const char* str,
                            std::vector<std::pair<int, double>>* out_features, double* out_label) const = 0;
 
  virtual int TotalColumns() const = 0;
 
  static Parser* CreateParser(const char* filename, bool header, int num_features, int label_idx);
};
 
class Dataset {
public:
  friend DatasetLoader;
 
  LIGHTGBM_EXPORT Dataset();
 
  LIGHTGBM_EXPORT Dataset(data_size_t num_data);
 
  void Construct(
    std::vector<std::unique_ptr<BinMapper>>& bin_mappers,
    int** sample_non_zero_indices,
    const int* num_per_col,
    size_t total_sample_cnt,
    const Config& io_config);
 
  LIGHTGBM_EXPORT ~Dataset();
 
  LIGHTGBM_EXPORT bool CheckAlign(const Dataset& other) const {
    if (num_features_ != other.num_features_) {
      return false;
    }
    if (num_total_features_ != other.num_total_features_) {
      return false;
    }
    if (label_idx_ != other.label_idx_) {
      return false;
    }
    for (int i = 0; i < num_features_; ++i) {
      if (!FeatureBinMapper(i)->CheckAlign(*(other.FeatureBinMapper(i)))) {
        return false;
      }
    }
    return true;
  }
 
  inline void PushOneRow(int tid, data_size_t row_idx, const std::vector<double>& feature_values) {
    if (is_finish_load_) { return; }
    for (size_t i = 0; i < feature_values.size() && i < static_cast<size_t>(num_total_features_); ++i) {
      int feature_idx = used_feature_map_[i];
      if (feature_idx >= 0) {
        const int group = feature2group_[feature_idx];
        const int sub_feature = feature2subfeature_[feature_idx];
        feature_groups_[group]->PushData(tid, sub_feature, row_idx, feature_values[i]);
      }
    }
  }
 
  inline void PushOneRow(int tid, data_size_t row_idx, const std::vector<std::pair<int, double>>& feature_values) {
    if (is_finish_load_) { return; }
    for (auto& inner_data : feature_values) {
      if (inner_data.first >= num_total_features_) { continue; }
      int feature_idx = used_feature_map_[inner_data.first];
      if (feature_idx >= 0) {
        const int group = feature2group_[feature_idx];
        const int sub_feature = feature2subfeature_[feature_idx];
        feature_groups_[group]->PushData(tid, sub_feature, row_idx, inner_data.second);
      }
    }
  }
 
  inline void PushOneData(int tid, data_size_t row_idx, int group, int sub_feature, double value) {
    feature_groups_[group]->PushData(tid, sub_feature, row_idx, value);
  }
 
  inline int RealFeatureIndex(int fidx) const {
    return real_feature_idx_[fidx];
  }
 
  inline int InnerFeatureIndex(int col_idx) const {
    return used_feature_map_[col_idx];
  }
  inline int Feature2Group(int feature_idx) const {
    return feature2group_[feature_idx];
  }
  inline int Feture2SubFeature(int feature_idx) const {
    return feature2subfeature_[feature_idx];
  }
  inline uint64_t GroupBinBoundary(int group_idx) const {
    return group_bin_boundaries_[group_idx];
  }
  inline uint64_t NumTotalBin() const {
    return group_bin_boundaries_.back();
  }
  inline std::vector<int> ValidFeatureIndices() const {
    std::vector<int> ret;
    for (int i = 0; i < num_total_features_; ++i) {
      if (used_feature_map_[i] >= 0) {
        ret.push_back(i);
      }
    }
    return ret;
  }
  void ReSize(data_size_t num_data);
 
  void CopySubset(const Dataset* fullset, const data_size_t* used_indices, data_size_t num_used_indices, bool need_meta_data);
 
  LIGHTGBM_EXPORT void FinishLoad();
 
  LIGHTGBM_EXPORT bool SetFloatField(const char* field_name, const float* field_data, data_size_t num_element);
 
  LIGHTGBM_EXPORT bool SetDoubleField(const char* field_name, const double* field_data, data_size_t num_element);
 
  LIGHTGBM_EXPORT bool SetIntField(const char* field_name, const int* field_data, data_size_t num_element);
 
  LIGHTGBM_EXPORT bool GetFloatField(const char* field_name, data_size_t* out_len, const float** out_ptr);
 
  LIGHTGBM_EXPORT bool GetDoubleField(const char* field_name, data_size_t* out_len, const double** out_ptr);
 
  LIGHTGBM_EXPORT bool GetIntField(const char* field_name, data_size_t* out_len, const int** out_ptr);
 
  LIGHTGBM_EXPORT void SaveBinaryFile(const char* bin_filename);
 
  LIGHTGBM_EXPORT void CopyFeatureMapperFrom(const Dataset* dataset);
 
  LIGHTGBM_EXPORT void CreateValid(const Dataset* dataset);
 
  void ConstructHistograms(const std::vector<int8_t>& is_feature_used,
                           const data_size_t* data_indices, data_size_t num_data,
                           int leaf_idx,
                           std::vector<std::unique_ptr<OrderedBin>>& ordered_bins,
                           const score_t* gradients, const score_t* hessians,
                           score_t* ordered_gradients, score_t* ordered_hessians,
                           bool is_constant_hessian,
                           HistogramBinEntry* histogram_data) const;
 
  void FixHistogram(int feature_idx, double sum_gradient, double sum_hessian, data_size_t num_data,
                    HistogramBinEntry* data) const;
 
  inline data_size_t Split(int feature,
                           const uint32_t* threshold, int num_threshold,  bool default_left,
                           data_size_t* data_indices, data_size_t num_data,
                           data_size_t* lte_indices, data_size_t* gt_indices) const {
    const int group = feature2group_[feature];
    const int sub_feature = feature2subfeature_[feature];
    return feature_groups_[group]->Split(sub_feature, threshold, num_threshold, default_left, data_indices, num_data, lte_indices, gt_indices);
  }
 
  inline int SubFeatureBinOffset(int i) const {
    const int sub_feature = feature2subfeature_[i];
    if (sub_feature == 0) {
      return 1;
    } else {
      return 0;
    }
  }
 
  inline int FeatureNumBin(int i) const {
    const int group = feature2group_[i];
    const int sub_feature = feature2subfeature_[i];
    return feature_groups_[group]->bin_mappers_[sub_feature]->num_bin();
  }
 
  inline int8_t FeatureMonotone(int i) const {
    if (monotone_types_.empty()) {
      return 0;
    } else {
      return monotone_types_[i];
    }
  }
 
  inline double FeaturePenalte(int i) const {
    if (feature_penalty_.empty()) {
      return 1;
    } else {
      return feature_penalty_[i];
    }
  }
 
  bool HasMonotone() const {
    if (monotone_types_.empty()) {
      return false;
    } else {
      for (size_t i = 0; i < monotone_types_.size(); ++i) {
        if (monotone_types_[i] != 0) {
          return true;
        }
      }
      return false;
    }
  }
 
  inline int FeatureGroupNumBin(int group) const {
    return feature_groups_[group]->num_total_bin_;
  }
 
  inline const BinMapper* FeatureBinMapper(int i) const {
    const int group = feature2group_[i];
    const int sub_feature = feature2subfeature_[i];
    return feature_groups_[group]->bin_mappers_[sub_feature].get();
  }
 
  inline const Bin* FeatureBin(int i) const {
    const int group = feature2group_[i];
    return feature_groups_[group]->bin_data_.get();
  }
 
  inline const Bin* FeatureGroupBin(int group) const {
    return feature_groups_[group]->bin_data_.get();
  }
 
  inline bool FeatureGroupIsSparse(int group) const {
    return feature_groups_[group]->is_sparse_;
  }
 
  inline BinIterator* FeatureIterator(int i) const {
    const int group = feature2group_[i];
    const int sub_feature = feature2subfeature_[i];
    return feature_groups_[group]->SubFeatureIterator(sub_feature);
  }
 
  inline BinIterator* FeatureGroupIterator(int group) const {
    return feature_groups_[group]->FeatureGroupIterator();
  }
 
  inline double RealThreshold(int i, uint32_t threshold) const {
    const int group = feature2group_[i];
    const int sub_feature = feature2subfeature_[i];
    return feature_groups_[group]->bin_mappers_[sub_feature]->BinToValue(threshold);
  }
 
  // given a real threshold, find the closest threshold bin
  inline uint32_t BinThreshold(int i, double threshold_double) const {
    const int group = feature2group_[i];
    const int sub_feature = feature2subfeature_[i];
    return feature_groups_[group]->bin_mappers_[sub_feature]->ValueToBin(threshold_double);
  }
 
  inline void CreateOrderedBins(std::vector<std::unique_ptr<OrderedBin>>* ordered_bins) const {
    ordered_bins->resize(num_groups_);
    OMP_INIT_EX();
    #pragma omp parallel for schedule(guided)
    for (int i = 0; i < num_groups_; ++i) {
      OMP_LOOP_EX_BEGIN();
      ordered_bins->at(i).reset(feature_groups_[i]->bin_data_->CreateOrderedBin());
      OMP_LOOP_EX_END();
    }
    OMP_THROW_EX();
  }
 
  inline const Metadata& metadata() const { return metadata_; }
 
  inline int num_features() const { return num_features_; }
 
  inline int num_feature_groups() const { return num_groups_;}
 
  inline int num_total_features() const { return num_total_features_; }
 
  inline int label_idx() const { return label_idx_; }
 
  inline const std::vector<std::string>& feature_names() const { return feature_names_; }
 
  inline void set_feature_names(const std::vector<std::string>& feature_names) {
    if (feature_names.size() != static_cast<size_t>(num_total_features_)) {
      Log::Fatal("Size of feature_names error, should equal with total number of features");
    }
    feature_names_ = std::vector<std::string>(feature_names);
    // replace ' ' in feature_names with '_'
    bool spaceInFeatureName = false;
    for (auto& feature_name : feature_names_) {
      if (feature_name.find(' ') != std::string::npos) {
        spaceInFeatureName = true;
        std::replace(feature_name.begin(), feature_name.end(), ' ', '_');
      }
    }
    if (spaceInFeatureName) {
      Log::Warning("Find whitespaces in feature_names, replace with underlines");
    }
  }
 
  inline std::vector<std::string> feature_infos() const {
    std::vector<std::string> bufs;
    for (int i = 0; i < num_total_features_; i++) {
      int fidx = used_feature_map_[i];
      if (fidx == -1) {
        bufs.push_back("none");
      } else {
        const auto bin_mapper = FeatureBinMapper(fidx);
        bufs.push_back(bin_mapper->bin_info());
      }
    }
    return bufs;
  }
 
  void ResetConfig(const char* parameters);
 
  inline data_size_t num_data() const { return num_data_; }
 
  Dataset& operator=(const Dataset&) = delete;
  Dataset(const Dataset&) = delete;
 
private:
  std::string data_filename_;
  std::vector<std::unique_ptr<FeatureGroup>> feature_groups_;
  std::vector<int> used_feature_map_;
  int num_features_;
  int num_total_features_;
  data_size_t num_data_;
  Metadata metadata_;
  int label_idx_ = 0;
  double sparse_threshold_;
  std::vector<std::string> feature_names_;
  static const char* binary_file_token;
  int num_groups_;
  std::vector<int> real_feature_idx_;
  std::vector<int> feature2group_;
  std::vector<int> feature2subfeature_;
  std::vector<uint64_t> group_bin_boundaries_;
  std::vector<int> group_feature_start_;
  std::vector<int> group_feature_cnt_;
  std::vector<int8_t> monotone_types_;
  std::vector<double> feature_penalty_;
  bool is_finish_load_;
  int max_bin_;
  int bin_construct_sample_cnt_;
  int min_data_in_bin_;
  bool use_missing_;
  bool zero_as_missing_;
};
 
}  // namespace LightGBM
 
#endif   // LightGBM_DATA_H_