data partition within group

Makefile.am

@@ -23,9 +23,7 @@ SINGA_SRCS := src/utils/cluster.cc \
 			  src/utils/blob.cc \
 			  src/trainer/server.cc \
 			  src/trainer/worker.cc \
-			  src/trainer/pm_server.cc \
 			  src/trainer/trainer.cc \
-			  src/trainer/pm_worker.cc \
 			  src/neuralnet/base_layer.cc \
 			  src/neuralnet/neuralnet.cc \
 			  src/neuralnet/layer.cc \
@@ -43,9 +41,7 @@ SINGA_HDRS := include/utils/cluster.h \
 			  include/utils/updater.h \
 			  include/trainer/server.h \
 			  include/trainer/worker.h \
-			  include/trainer/pm_worker.h \
 			  include/trainer/trainer.h \
-			  include/trainer/pm_server.h \
 			  include/neuralnet/base_layer.h \
 			  include/neuralnet/layer.h \
 			  include/neuralnet/neuralnet.h \

Makefile.in

@@ -75,9 +75,7 @@ am__objects_3 = src/utils/libsinga_la-cluster.lo \
 	src/utils/libsinga_la-blob.lo \
 	src/trainer/libsinga_la-server.lo \
 	src/trainer/libsinga_la-worker.lo \
-	src/trainer/libsinga_la-pm_server.lo \
 	src/trainer/libsinga_la-trainer.lo \
-	src/trainer/libsinga_la-pm_worker.lo \
 	src/neuralnet/libsinga_la-base_layer.lo \
 	src/neuralnet/libsinga_la-neuralnet.lo \
 	src/neuralnet/libsinga_la-layer.lo \
@@ -242,9 +240,7 @@ SINGA_SRCS := src/utils/cluster.cc \
 			  src/utils/blob.cc \
 			  src/trainer/server.cc \
 			  src/trainer/worker.cc \
-			  src/trainer/pm_server.cc \
 			  src/trainer/trainer.cc \
-			  src/trainer/pm_worker.cc \
 			  src/neuralnet/base_layer.cc \
 			  src/neuralnet/neuralnet.cc \
 			  src/neuralnet/layer.cc \
@@ -262,9 +258,7 @@ SINGA_HDRS := include/utils/cluster.h \
 			  include/utils/updater.h \
 			  include/trainer/server.h \
 			  include/trainer/worker.h \
-			  include/trainer/pm_worker.h \
 			  include/trainer/trainer.h \
-			  include/trainer/pm_server.h \
 			  include/neuralnet/base_layer.h \
 			  include/neuralnet/layer.h \
 			  include/neuralnet/neuralnet.h \
@@ -445,12 +439,8 @@ src/trainer/libsinga_la-server.lo: src/trainer/$(am__dirstamp) \
 	src/trainer/$(DEPDIR)/$(am__dirstamp)
 src/trainer/libsinga_la-worker.lo: src/trainer/$(am__dirstamp) \
 	src/trainer/$(DEPDIR)/$(am__dirstamp)
-src/trainer/libsinga_la-pm_server.lo: src/trainer/$(am__dirstamp) \
-	src/trainer/$(DEPDIR)/$(am__dirstamp)
 src/trainer/libsinga_la-trainer.lo: src/trainer/$(am__dirstamp) \
 	src/trainer/$(DEPDIR)/$(am__dirstamp)
-src/trainer/libsinga_la-pm_worker.lo: src/trainer/$(am__dirstamp) \
-	src/trainer/$(DEPDIR)/$(am__dirstamp)
 src/neuralnet/$(am__dirstamp):
 	@$(mkdir_p) src/neuralnet
 	@: > src/neuralnet/$(am__dirstamp)
@@ -535,10 +525,6 @@ mostlyclean-compile:
 	-rm -f src/proto/libsinga_la-model.pb.$(OBJEXT)
 	-rm -f src/proto/libsinga_la-model.pb.lo
 	-rm -f src/singa-main.$(OBJEXT)
-	-rm -f src/trainer/libsinga_la-pm_server.$(OBJEXT)
-	-rm -f src/trainer/libsinga_la-pm_server.lo
-	-rm -f src/trainer/libsinga_la-pm_worker.$(OBJEXT)
-	-rm -f src/trainer/libsinga_la-pm_worker.lo
 	-rm -f src/trainer/libsinga_la-server.$(OBJEXT)
 	-rm -f src/trainer/libsinga_la-server.lo
 	-rm -f src/trainer/libsinga_la-trainer.$(OBJEXT)
@@ -571,8 +557,6 @@ distclean-compile:
 @AMDEP_TRUE@@am__include@ @am__quote@src/neuralnet/$(DEPDIR)/libsinga_la-neuralnet.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@src/proto/$(DEPDIR)/libsinga_la-cluster.pb.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@src/proto/$(DEPDIR)/libsinga_la-model.pb.Plo@am__quote@
-@AMDEP_TRUE@@am__include@ @am__quote@src/trainer/$(DEPDIR)/libsinga_la-pm_server.Plo@am__quote@
-@AMDEP_TRUE@@am__include@ @am__quote@src/trainer/$(DEPDIR)/libsinga_la-pm_worker.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@src/trainer/$(DEPDIR)/libsinga_la-server.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@src/trainer/$(DEPDIR)/libsinga_la-trainer.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@src/trainer/$(DEPDIR)/libsinga_la-worker.Plo@am__quote@
@@ -685,27 +669,13 @@ src/trainer/libsinga_la-worker.lo: src/trainer/worker.cc
 @AMDEP_TRUE@@am__fastdepCXX_FALSE@	DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
 @am__fastdepCXX_FALSE@	$(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libsinga_la_CXXFLAGS) $(CXXFLAGS) -c -o src/trainer/libsinga_la-worker.lo `test -f 'src/trainer/worker.cc' || echo '$(srcdir)/'`src/trainer/worker.cc
 
-src/trainer/libsinga_la-pm_server.lo: src/trainer/pm_server.cc
-@am__fastdepCXX_TRUE@	if $(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libsinga_la_CXXFLAGS) $(CXXFLAGS) -MT src/trainer/libsinga_la-pm_server.lo -MD -MP -MF "src/trainer/$(DEPDIR)/libsinga_la-pm_server.Tpo" -c -o src/trainer/libsinga_la-pm_server.lo `test -f 'src/trainer/pm_server.cc' || echo '$(srcdir)/'`src/trainer/pm_server.cc; \
-@am__fastdepCXX_TRUE@	then mv -f "src/trainer/$(DEPDIR)/libsinga_la-pm_server.Tpo" "src/trainer/$(DEPDIR)/libsinga_la-pm_server.Plo"; else rm -f "src/trainer/$(DEPDIR)/libsinga_la-pm_server.Tpo"; exit 1; fi
-@AMDEP_TRUE@@am__fastdepCXX_FALSE@	source='src/trainer/pm_server.cc' object='src/trainer/libsinga_la-pm_server.lo' libtool=yes @AMDEPBACKSLASH@
-@AMDEP_TRUE@@am__fastdepCXX_FALSE@	DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
-@am__fastdepCXX_FALSE@	$(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libsinga_la_CXXFLAGS) $(CXXFLAGS) -c -o src/trainer/libsinga_la-pm_server.lo `test -f 'src/trainer/pm_server.cc' || echo '$(srcdir)/'`src/trainer/pm_server.cc
-
 src/trainer/libsinga_la-trainer.lo: src/trainer/trainer.cc
 @am__fastdepCXX_TRUE@	if $(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libsinga_la_CXXFLAGS) $(CXXFLAGS) -MT src/trainer/libsinga_la-trainer.lo -MD -MP -MF "src/trainer/$(DEPDIR)/libsinga_la-trainer.Tpo" -c -o src/trainer/libsinga_la-trainer.lo `test -f 'src/trainer/trainer.cc' || echo '$(srcdir)/'`src/trainer/trainer.cc; \
 @am__fastdepCXX_TRUE@	then mv -f "src/trainer/$(DEPDIR)/libsinga_la-trainer.Tpo" "src/trainer/$(DEPDIR)/libsinga_la-trainer.Plo"; else rm -f "src/trainer/$(DEPDIR)/libsinga_la-trainer.Tpo"; exit 1; fi
 @AMDEP_TRUE@@am__fastdepCXX_FALSE@	source='src/trainer/trainer.cc' object='src/trainer/libsinga_la-trainer.lo' libtool=yes @AMDEPBACKSLASH@
 @AMDEP_TRUE@@am__fastdepCXX_FALSE@	DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
 @am__fastdepCXX_FALSE@	$(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libsinga_la_CXXFLAGS) $(CXXFLAGS) -c -o src/trainer/libsinga_la-trainer.lo `test -f 'src/trainer/trainer.cc' || echo '$(srcdir)/'`src/trainer/trainer.cc
 
-src/trainer/libsinga_la-pm_worker.lo: src/trainer/pm_worker.cc
-@am__fastdepCXX_TRUE@	if $(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libsinga_la_CXXFLAGS) $(CXXFLAGS) -MT src/trainer/libsinga_la-pm_worker.lo -MD -MP -MF "src/trainer/$(DEPDIR)/libsinga_la-pm_worker.Tpo" -c -o src/trainer/libsinga_la-pm_worker.lo `test -f 'src/trainer/pm_worker.cc' || echo '$(srcdir)/'`src/trainer/pm_worker.cc; \
-@am__fastdepCXX_TRUE@	then mv -f "src/trainer/$(DEPDIR)/libsinga_la-pm_worker.Tpo" "src/trainer/$(DEPDIR)/libsinga_la-pm_worker.Plo"; else rm -f "src/trainer/$(DEPDIR)/libsinga_la-pm_worker.Tpo"; exit 1; fi
-@AMDEP_TRUE@@am__fastdepCXX_FALSE@	source='src/trainer/pm_worker.cc' object='src/trainer/libsinga_la-pm_worker.lo' libtool=yes @AMDEPBACKSLASH@
-@AMDEP_TRUE@@am__fastdepCXX_FALSE@	DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
-@am__fastdepCXX_FALSE@	$(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libsinga_la_CXXFLAGS) $(CXXFLAGS) -c -o src/trainer/libsinga_la-pm_worker.lo `test -f 'src/trainer/pm_worker.cc' || echo '$(srcdir)/'`src/trainer/pm_worker.cc
-
 src/neuralnet/libsinga_la-base_layer.lo: src/neuralnet/base_layer.cc
 @am__fastdepCXX_TRUE@	if $(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libsinga_la_CXXFLAGS) $(CXXFLAGS) -MT src/neuralnet/libsinga_la-base_layer.lo -MD -MP -MF "src/neuralnet/$(DEPDIR)/libsinga_la-base_layer.Tpo" -c -o src/neuralnet/libsinga_la-base_layer.lo `test -f 'src/neuralnet/base_layer.cc' || echo '$(srcdir)/'`src/neuralnet/base_layer.cc; \
 @am__fastdepCXX_TRUE@	then mv -f "src/neuralnet/$(DEPDIR)/libsinga_la-base_layer.Tpo" "src/neuralnet/$(DEPDIR)/libsinga_la-base_layer.Plo"; else rm -f "src/neuralnet/$(DEPDIR)/libsinga_la-base_layer.Tpo"; exit 1; fi

examples/cifar10/Makefile

@@ -5,7 +5,7 @@ libs :=singa glog protobuf
 download: cifar-10-binary-bin
 
 cifar-10-binary-bin:
-	wget http://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz
+	#wget http://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz
 	tar xf cifar-10-binary.tar.gz
 
 create:

examples/cifar10/cluster.conf

@@ -1,5 +1,6 @@
 nworker_groups: 1
 nserver_groups: 1
 nservers_per_group: 1
-nworkers_per_group: 1
+nworkers_per_group: 2
+nworkers_per_procs: 2
 workspace: "examples/cifar10/"

examples/cifar10/model.conf

@@ -2,7 +2,7 @@ name: "cifar10-convnet"
 train_steps: 70000
 test_steps:100
 test_frequency:1000
-display_frequency:50
+display_frequency:30
 updater{
   momentum:0.9
   weight_decay:0.004
@@ -15,6 +15,7 @@ updater{
   step_lr:0.00001
 }
 neuralnet {
+partition_type: kDataPartition
 layer{
   name: "data"
   type: "kShardData"

include/communication/msg.h

@@ -13,27 +13,28 @@ class BaseMsg{
     */
   virtual ~BaseMsg(){};
   /**
-    * @param group_id worker/server group id
+    * @param first worker/server group id
     * @param id worker/server id within the group
     * @param flag 0 for server, 1 for worker, 2 for stub
     */
-  virtual void set_src(int group_id, int id, int flag)=0;
-  virtual void set_dst(int group_id, int id, int flag)=0;
+  virtual void set_src(int first, int second, int flag)=0;
+  virtual void set_dst(int first, int second, int flag)=0;
   virtual void set_src(int procs_id, int flag)=0;
   virtual void set_dst(int procs_id, int flag)=0;
-  virtual int src_group_id() const=0;
-  virtual int dst_group_id() const=0;
-  virtual int src_id() const=0;
-  virtual int dst_id() const=0;
+  virtual int src_first() const=0;
+  virtual int dst_first() const=0;
+  virtual int src_second() const=0;
+  virtual int dst_second() const=0;
   virtual int src_flag() const=0;
   virtual int dst_flag() const=0;
   virtual void set_type(int type)=0;
   virtual int type() const=0;
-  virtual void set_target(int target)=0;
-  virtual int target() const=0;
+  virtual void set_target(int first, int second)=0;
+  virtual int target_first() const=0;
+  virtual int target_second() const=0;
 
   /**
-   * Copy src and dst address, including group_id, id, flag
+   * Copy src and dst address, including first, id, flag
    */
   virtual BaseMsg* CopyAddr()=0;
   virtual void SetAddr(BaseMsg* msg)=0;
@@ -64,11 +65,11 @@ class Msg : public BaseMsg{
     if(msg_!=NULL)
       zmsg_destroy(&msg_);
   }
-  virtual void set_src(int group_id, int id, int flag){
-    src_=(group_id<<kOff1)|(id<<kOff2)|flag;
+  virtual void set_src(int first, int second, int flag){
+    src_=(first<<kOff1)|(second<<kOff2)|flag;
   }
-  virtual void set_dst(int group_id, int id, int flag){
-    dst_=(group_id<<kOff1)|(id<<kOff2)|flag;
+  virtual void set_dst(int first, int second, int flag){
+    dst_=(first<<kOff1)|(second<<kOff2)|flag;
   }
   virtual void set_src(int procs_id, int flag){
     set_src(procs_id, 0, flag);
@@ -82,20 +83,20 @@ class Msg : public BaseMsg{
   int dst() const {
     return dst_;
   }
-  virtual int src_group_id() const {
+  virtual int src_first() const {
     int ret=src_>>kOff1;
     return ret;
   }
 
-  virtual int dst_group_id() const{
+  virtual int dst_first() const{
     int ret=dst_>>kOff1;
     return ret;
   }
-  virtual int src_id() const{
+  virtual int src_second() const{
     int ret=(src_&kMask1)>>kOff2;
     return ret;
   }
-  virtual int dst_id() const{
+  virtual int dst_second() const{
     int ret=(dst_&kMask1)>>kOff2;
     return ret;
   }
@@ -113,22 +114,24 @@ class Msg : public BaseMsg{
   }
 
   virtual void set_type(int type){
-    target_=(type<<kOff3)|(target_&kMask3);
-  }
-  virtual void set_target(int target){
-    target_=(target_>>kOff3)<<kOff3;
-    target_=target_|target;
+    type_=type;
   }
   virtual int type() const{
-    int ret=target_>>kOff3;
-    return ret;
+    return type_;
   }
-  virtual int target() const{
-    int ret=target_&kMask3;
-    return ret;
+
+  virtual void set_target(int first, int second){
+    target_first_=first;
+    target_second_=second;
+  }
+  virtual int target_first() const{
+    return target_first_;
+  }
+  virtual int target_second() const{
+    return target_second_;
   }
 
-  virtual BaseMsg* CopyAddr(){
+ virtual BaseMsg* CopyAddr(){
     Msg* msg=new Msg();
     msg->src_=src_;
     msg->dst_=dst_;
@@ -158,25 +161,27 @@ class Msg : public BaseMsg{
 
   void ParseFromZmsg(zmsg_t* msg){
     char* tmp=zmsg_popstr(msg);
-    sscanf(tmp, "%d %d %d", &src_, &dst_, &target_);
+    sscanf(tmp, "%d %d %d %d %d",
+        &src_, &dst_, &type_, &target_first_, &target_second_);
     //LOG(ERROR)<<"recv "<<src_<<" "<<dst_<<" "<<target_;
     frame_=zmsg_next(msg);
     msg_=msg;
   }
 
   zmsg_t* DumpToZmsg(){
-    zmsg_pushstrf(msg_, "%d %d %d",src_, dst_,target_);
+    zmsg_pushstrf(msg_, "%d %d %d %d %d",
+        src_, dst_, type_, target_first_, target_second_);
     //LOG(ERROR)<<"send "<<src_<<" "<<dst_<<" "<<target_;
-    zmsg_t* tmp=msg_;
+    zmsg_t *tmp=msg_;
     msg_=NULL;
     return tmp;
   }
 
  protected:
-  static const unsigned int kOff1=16, kOff2=4, kOff3=24;
-  static const unsigned int kMask1=(1<<kOff1)-1, kMask2=(1<<kOff2)-1,
-               kMask3=(1<<kOff3)-1;
-  unsigned int src_, dst_, target_;
+  static const unsigned int kOff1=16, kOff2=4;
+  static const unsigned int kMask1=(1<<kOff1)-1, kMask2=(1<<kOff2)-1;
+  int src_, dst_;
+  int type_, target_first_, target_second_;
   zmsg_t* msg_;
   zframe_t *frame_;
 };