allreduce_base.h

Go to the documentation of this file.

 
#ifndef RABIT_ALLREDUCE_BASE_H_
#define RABIT_ALLREDUCE_BASE_H_
 
#include <functional>
#include <future>
#include <vector>
#include <string>
#include <algorithm>
#include "rabit/internal/utils.h"
#include "rabit/internal/engine.h"
#include "rabit/internal/socket.h"
 
#ifdef RABIT_CXXTESTDEFS_H
#define private   public
#define protected public
#endif  // RABIT_CXXTESTDEFS_H
 
 
namespace MPI {  // NOLINT
// MPI data type to be compatible with existing MPI interface
class Datatype {
 public:
  size_t type_size;
  explicit Datatype(size_t type_size) : type_size(type_size) {}
};
}
namespace rabit {
namespace engine {
 
class AllreduceBase : public IEngine {
 public:
  // magic number to verify server
  static const int kMagic = 0xff99;
  // constant one byte out of band message to indicate error happening
  AllreduceBase();
  virtual ~AllreduceBase() = default;
  // initialize the manager
  virtual bool Init(int argc, char* argv[]);
  // shutdown the engine
  virtual bool Shutdown();
  virtual void SetParam(const char *name, const char *val);
  void TrackerPrint(const std::string &msg) override;
 
  int GetRingPrevRank() const override {
    return ring_prev->rank;
  }
  int GetRank() const override {
    return rank;
  }
  int GetWorldSize() const override {
    if (world_size == -1) return 1;
    return world_size;
  }
  bool IsDistributed() const override {
    return tracker_uri != "NULL";
  }
  std::string GetHost() const override {
    return host_uri;
  }
 
  void Allgather(void *sendrecvbuf_, size_t total_size, size_t slice_begin,
                 size_t slice_end, size_t size_prev_slice) override {
    if (world_size == 1 || world_size == -1) {
      return;
    }
    utils::Assert(TryAllgatherRing(sendrecvbuf_, total_size, slice_begin,
                                   slice_end, size_prev_slice) == kSuccess,
                  "AllgatherRing failed");
  }
  void Allreduce(void *sendrecvbuf_, size_t type_nbytes, size_t count,
                 ReduceFunction reducer, PreprocFunction prepare_fun = nullptr,
                 void *prepare_arg = nullptr) override {
    if (prepare_fun != nullptr) prepare_fun(prepare_arg);
    if (world_size == 1 || world_size == -1) return;
    utils::Assert(TryAllreduce(sendrecvbuf_, type_nbytes, count, reducer) ==
                      kSuccess,
                  "Allreduce failed");
  }
  void Broadcast(void *sendrecvbuf_, size_t total_size, int root) override {
    if (world_size == 1 || world_size == -1) return;
    utils::Assert(TryBroadcast(sendrecvbuf_, total_size, root) == kSuccess,
                  "Broadcast failed");
  }
  int LoadCheckPoint() override { return 0; }
 
  // deprecated, increase internal version number
  void CheckPoint() override { version_number += 1; }
  int VersionNumber() const override {
    return version_number;
  }
  inline void ReportStatus() const {
    if (hadoop_mode != 0) {
      LOG(CONSOLE) << "reporter:status:Rabit Phase[" << version_number << "] Operation " << seq_counter << "\n";
    }
  }
 
 protected:
  enum ReturnTypeEnum {
    kSuccess,
    kConnReset,
    kRecvZeroLen,
    kSockError,
    kGetExcept
  };
  struct ReturnType {
    ReturnTypeEnum value;
    // constructor
    ReturnType() = default;
    ReturnType(ReturnTypeEnum value) : value(value) {}  // NOLINT(*)
    inline bool operator==(const ReturnTypeEnum &v) const {
      return value == v;
    }
    inline bool operator!=(const ReturnTypeEnum &v) const {
      return value != v;
    }
  };
  static ReturnType Errno2Return() {
    int errsv = xgboost::system::LastError();
    if (errsv == EAGAIN || errsv == EWOULDBLOCK || errsv == 0) return kSuccess;
#ifdef _WIN32
    if (errsv == WSAEWOULDBLOCK) return kSuccess;
    if (errsv == WSAECONNRESET) return kConnReset;
#endif  // _WIN32
    if (errsv == ECONNRESET) return kConnReset;
    return kSockError;
  }
  // link record to a neighbor
  struct LinkRecord {
   public:
    // socket to get data from/to link
    xgboost::collective::TCPSocket sock;
    // rank of the node in this link
    int rank;
    // size of data readed from link
    size_t size_read;
    // size of data sent to the link
    size_t size_write;
    // pointer to buffer head
    char *buffer_head {nullptr};
    // buffer size, in bytes
    size_t buffer_size {0};
    // constructor
    LinkRecord() = default;
    // initialize buffer
    void InitBuffer(size_t type_nbytes, size_t count,
                    size_t reduce_buffer_size) {
      size_t n = (type_nbytes * count + 7)/ 8;
      auto to = Min(reduce_buffer_size, n);
      buffer_.resize(to);
      // make sure align to type_nbytes
      buffer_size =
          buffer_.size() * sizeof(uint64_t) / type_nbytes * type_nbytes;
      utils::Assert(type_nbytes <= buffer_size,
                    "too large type_nbytes=%lu, buffer_size=%lu",
                    type_nbytes, buffer_size);
      // set buffer head
      buffer_head = reinterpret_cast<char*>(BeginPtr(buffer_));
    }
    // reset the recv and sent size
    inline void ResetSize() {
      size_write = size_read = 0;
    }
    inline ReturnType ReadToRingBuffer(size_t protect_start, size_t max_size_read) {
      utils::Assert(buffer_head != nullptr, "ReadToRingBuffer: buffer not allocated");
      utils::Assert(size_read <= max_size_read, "ReadToRingBuffer: max_size_read check");
      size_t ngap = size_read - protect_start;
      utils::Assert(ngap <= buffer_size, "Allreduce: boundary check");
      size_t offset = size_read % buffer_size;
      size_t nmax = max_size_read - size_read;
      nmax = Min(nmax, buffer_size - ngap);
      nmax = Min(nmax, buffer_size - offset);
      if (nmax == 0) return kSuccess;
      ssize_t len = sock.Recv(buffer_head + offset, nmax);
      // length equals 0, remote disconnected
      if (len == 0) {
        sock.Close(); return kRecvZeroLen;
      }
      if (len == -1) return Errno2Return();
      size_read += static_cast<size_t>(len);
      return kSuccess;
    }
    inline ReturnType ReadToArray(void *recvbuf_, size_t max_size) {
      if (max_size == size_read) return kSuccess;
      char *p = static_cast<char*>(recvbuf_);
      ssize_t len = sock.Recv(p + size_read, max_size - size_read);
      // length equals 0, remote disconnected
      if (len == 0) {
        sock.Close(); return kRecvZeroLen;
      }
      if (len == -1) return Errno2Return();
      size_read += static_cast<size_t>(len);
      return kSuccess;
    }
    inline ReturnType WriteFromArray(const void *sendbuf_, size_t max_size) {
      const char *p = static_cast<const char*>(sendbuf_);
      ssize_t len = sock.Send(p + size_write, max_size - size_write);
      if (len == -1) return Errno2Return();
      size_write += static_cast<size_t>(len);
      return kSuccess;
    }
 
   private:
    // recv buffer to get data from child
    // aligned with 64 bits, will be able to perform 64 bits operations freely
    std::vector<uint64_t> buffer_;
  };
  struct RefLinkVector {
    std::vector<LinkRecord*> plinks;
    inline LinkRecord &operator[](size_t i) {
      return *plinks[i];
    }
    inline size_t Size() const {
      return plinks.size();
    }
  };
  xgboost::collective::TCPSocket ConnectTracker() const;
  bool ReConnectLinks(const char *cmd = "start");
  ReturnType TryAllreduce(void *sendrecvbuf_,
                          size_t type_nbytes,
                          size_t count,
                          ReduceFunction reducer);
  ReturnType TryBroadcast(void *sendrecvbuf_, size_t size, int root);
  ReturnType TryAllreduceTree(void *sendrecvbuf_,
                              size_t type_nbytes,
                              size_t count,
                              ReduceFunction reducer);
  ReturnType TryAllgatherRing(void *sendrecvbuf_, size_t total_size,
                              size_t slice_begin, size_t slice_end,
                              size_t size_prev_slice);
  ReturnType TryReduceScatterRing(void *sendrecvbuf_,
                                  size_t type_nbytes,
                                  size_t count,
                                  ReduceFunction reducer);
  ReturnType TryAllreduceRing(void *sendrecvbuf_,
                              size_t type_nbytes,
                              size_t count,
                              ReduceFunction reducer);
  inline ReturnType ReportError(LinkRecord *link, ReturnType err) {
    err_link = link; return err;
  }
  //---- data structure related to model ----
  // call sequence counter, records how many calls we made so far
  // from last call to CheckPoint, LoadCheckPoint
  int seq_counter{0}; // NOLINT
  // version number of model
  int version_number {0};  // NOLINT
  // whether the job is running in Hadoop
  bool hadoop_mode;  // NOLINT
  //---- local data related to link ----
  // index of parent link, can be -1, meaning this is root of the tree
  int parent_index;  // NOLINT
  // rank of parent node, can be -1
  int parent_rank;  // NOLINT
  // sockets of all links this connects to
  std::vector<LinkRecord> all_links;  // NOLINT
  // used to record the link where things goes wrong
  LinkRecord *err_link;  // NOLINT
  // all the links in the reduction tree connection
  RefLinkVector tree_links;  // NOLINT
  // pointer to links in the ring
  LinkRecord *ring_prev, *ring_next;  // NOLINT
  //----- meta information-----
  // list of enviroment variables that are of possible interest
  std::vector<std::string> env_vars;  // NOLINT
  // unique identifier of the possible job this process is doing
  // used to assign ranks, optional, default to NULL
  std::string task_id;  // NOLINT
  // uri of current host, to be set by Init
  std::string host_uri;  // NOLINT
  // uri of tracker
  std::string tracker_uri;  // NOLINT
  // role in dmlc jobs
  std::string dmlc_role;  // NOLINT
  // port of tracker address
  int tracker_port;  // NOLINT
  // reduce buffer size
  size_t reduce_buffer_size;  // NOLINT
  // reduction method
  int reduce_method;  // NOLINT
  // minimum count of cells to use ring based method
  size_t reduce_ring_mincount;  // NOLINT
  // minimum block size per tree reduce
  size_t tree_reduce_minsize;  // NOLINT
  // current rank
  int rank;  // NOLINT
  // world size
  int world_size;  // NOLINT
  // connect retry time
  int connect_retry;  // NOLINT
  // by default, if rabit worker not recover in half an hour exit
  std::chrono::seconds timeout_sec{std::chrono::seconds{1800}}; // NOLINT
  // flag to enable rabit_timeout
  bool rabit_timeout = false;  // NOLINT
  // Enable TCP node delay
  bool rabit_enable_tcp_no_delay = false;  // NOLINT
};
}  // namespace engine
}  // namespace rabit
#endif  // RABIT_ALLREDUCE_BASE_H_