recordio_split.h

Go to the documentation of this file.

 
#ifndef DMLC_IO_RECORDIO_SPLIT_H_
#define DMLC_IO_RECORDIO_SPLIT_H_
 
#include <dmlc/io.h>
#include <dmlc/recordio.h>
#include <vector>
#include <cstdio>
#include <string>
#include <cstring>
#include "./input_split_base.h"
 
namespace dmlc {
namespace io {
class RecordIOSplitter : public InputSplitBase {
 public:
  RecordIOSplitter(FileSystem *fs,
                   const char *uri,
                   unsigned rank,
                   unsigned nsplit,
                   const bool recurse_directories) {
    this->Init(fs, uri, 4, recurse_directories);
    this->ResetPartition(rank, nsplit);
  }
 
  bool IsTextParser(void) {
    return false;
  }
  virtual bool ExtractNextRecord(Blob *out_rec, Chunk *chunk);
 
 protected:
  virtual size_t SeekRecordBegin(Stream *fi);
  virtual const char*
  FindLastRecordBegin(const char *begin, const char *end);
};
}  // namespace io
}  // namespace dmlc
#endif  // DMLC_IO_RECORDIO_SPLIT_H_