Permalink
Browse files

init commit

  • Loading branch information...
1 parent 1d5f0c0 commit eaed92a0d4806009e5646db6c438e8a0ea01753e @mli mli committed Apr 26, 2015
Showing with 11,771 additions and 33 deletions.
  1. +17 −28 .gitignore
  2. +2 −3 LICENSE
  3. +60 −0 Makefile
  4. +20 −2 README.md
  5. +2,303 −0 doc/Doxyfile
  6. +3 −0 doc/README.md
  7. +5 −0 doc/gendoc.sh
  8. +147 −0 guide/README.md
  9. BIN guide/deps.png
  10. +25 −0 guide/example_a.cc
  11. +28 −0 guide/example_b.cc
  12. +26 −0 guide/example_c.cc
  13. +31 −0 guide/example_d.cc
  14. +39 −0 guide/local.sh
  15. +5 −0 guide/ps_guide.mk
  16. +27 −0 make/README.md
  17. +6 −0 make/install_third.sh
  18. 0 src/README.mk
  19. +52 −0 src/base/assign_op.h
  20. +105 −0 src/base/auc.h
  21. +41 −0 src/base/barrier.h
  22. +42 −0 src/base/base.h
  23. +99 −0 src/base/bitmap.h
  24. +213 −0 src/base/blob.h
  25. +71 −0 src/base/block_bloom_filter.h
  26. +48 −0 src/base/bloom_filter.h
  27. +72 −0 src/base/common.h
  28. +50 −0 src/base/countmin.h
  29. +338 −0 src/base/crc32c.cc
  30. +42 −0 src/base/crc32c.h
  31. +65 −0 src/base/evaluation.h
  32. +423 −0 src/base/file.cc
  33. +126 −0 src/base/file.h
  34. +8 −0 src/base/hdfs.h
  35. +91 −0 src/base/integral_types.h
  36. +135 −0 src/base/local_machine.h
  37. +223 −0 src/base/localizer.h
  38. +15 −0 src/base/macros.h
  39. +114 −0 src/base/parallel_ordered_match.h
  40. +43 −0 src/base/parallel_sort.h
  41. +54 −0 src/base/producer_consumer.h
  42. +133 −0 src/base/range.h
  43. +41 −0 src/base/recordio.cc
  44. +79 −0 src/base/recordio.h
  45. +144 −0 src/base/resource_usage.h
  46. +310 −0 src/base/shared_array.h
  47. +274 −0 src/base/shared_array_inl.h
  48. +35 −0 src/base/sketch.h
  49. +30 −0 src/base/split.h
  50. +47 −0 src/base/strtonum.h
  51. +59 −0 src/base/threadpool.cc
  52. +48 −0 src/base/threadpool.h
  53. +73 −0 src/base/threadsafe_limited_queue.h
  54. +70 −0 src/base/threadsafe_queue.h
  55. +40 −0 src/filter/add_noise.h
  56. +40 −0 src/filter/compressing.h
  57. +33 −0 src/filter/filter.cc
  58. +27 −0 src/filter/filter.h
  59. +105 −0 src/filter/fixing_float.h
  60. +87 −0 src/filter/frequency_filter.h
  61. +78 −0 src/filter/key_caching.h
  62. +26 −0 src/filter/sparse_filter.h
  63. +93 −0 src/kv/kv_cache.h
  64. +64 −0 src/kv/kv_store.h
  65. +1 −0 src/kv/kv_store_dense.h
  66. +68 −0 src/kv/kv_store_sparse.h
  67. +101 −0 src/kv/kv_store_sparse_dynamic.h
  68. +9 −0 src/parameter/README.org
  69. +212 −0 src/parameter/kv_layer.h
  70. +107 −0 src/parameter/kv_map.h
  71. +264 −0 src/parameter/kv_vector.h
  72. +51 −0 src/parameter/parameter.cc
  73. +79 −0 src/parameter/parameter.h
  74. +13 −0 src/proto/assign_op.proto
  75. +49 −0 src/proto/data.proto
  76. +38 −0 src/proto/filter.proto
  77. +31 −0 src/proto/heartbeat.proto
  78. +21 −0 src/proto/node.proto
  79. +48 −0 src/proto/param.proto
  80. +7 −0 src/proto/range.proto
  81. +92 −0 src/proto/task.proto
  82. +79 −0 src/ps-inl.h
  83. +351 −0 src/ps.h
  84. +18 −0 src/ps_main.cc
  85. +98 −0 src/server_handle.h
  86. +34 −0 src/system/assigner.cc
  87. +57 −0 src/system/assigner.h
  88. +281 −0 src/system/customer.h
  89. +130 −0 src/system/dashboard.cc
  90. +24 −0 src/system/dashboard.h
  91. +120 −0 src/system/env.cc
  92. +20 −0 src/system/env.h
  93. +377 −0 src/system/executor.cc
  94. +112 −0 src/system/executor.h
  95. +163 −0 src/system/heartbeat_info.cc
  96. +89 −0 src/system/heartbeat_info.h
  97. +314 −0 src/system/manager.cc
  98. +97 −0 src/system/manager.h
  99. +49 −0 src/system/message.cc
  100. +161 −0 src/system/message.h
  101. +99 −0 src/system/monitor.h
  102. +67 −0 src/system/network_usage.h
  103. +130 −0 src/system/postoffice.cc
  104. +54 −0 src/system/postoffice.h
  105. +58 −0 src/system/remote_node.cc
  106. +70 −0 src/system/remote_node.h
  107. +309 −0 src/system/van.cc
  108. +69 −0 src/system/van.h
View
@@ -1,28 +1,17 @@
-# Compiled Object files
-*.slo
-*.lo
-*.o
-*.obj
-
-# Precompiled Headers
-*.gch
-*.pch
-
-# Compiled Dynamic libraries
-*.so
-*.dylib
-*.dll
-
-# Fortran module files
-*.mod
-
-# Compiled Static libraries
-*.lai
-*.la
-*.a
-*.lib
-
-# Executables
-*.exe
-*.out
-*.app
+example/**/data
+example/**/model
+build
+third_party
+**/log/**
+**/cache/**
+**/output/**
+*.pb.cc
+*.pb.h
+.*
+*core*
+config.mk
+**/bak/**
+doc/html
+doc/latex
+test/
+test-bak
View
@@ -1,4 +1,4 @@
- Apache License
+Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
@@ -186,7 +186,7 @@
same "printed page" as the copyright notice for easier
identification within third-party archives.
- Copyright {yyyy} {name of copyright owner}
+ Copyright 2015 Carnegie Mellon University
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@@ -199,4 +199,3 @@
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-
View
@@ -0,0 +1,60 @@
+ifneq ("$(wildcard ./config.mk)","")
+include ./config.mk
+else
+include make/config.mk
+endif
+
+ifeq ($(STATIC_THIRD_LIB), 1)
+THIRD_LIB=$(addprefix $(THIRD_PATH)/lib/, libgflags.a libzmq.a libprotobuf.a libglog.a libz.a libsnappy.a)
+ ifeq ($(USE_S3),1)
+ THIRD_LIB+=$(addprefix $(THIRD_PATH)/lib/, libxml2.a)
+ endif
+else
+THIRD_LIB=-L$(THIRD_PATH)/lib -lgflags -lzmq -lprotobuf -lglog -lz -lsnappy
+ ifeq ($(USE_S3),1)
+ THIRD_LIB+=-lxml2
+ endif
+endif
+
+WARN = -Wall -Wno-unused-function -finline-functions -Wno-sign-compare #-Wconversion
+INCPATH = -I./src -I./include -I$(THIRD_PATH)/include
+CFLAGS = -std=c++0x $(WARN) $(OPT) $(INCPATH) $(EXTRA_CFLAGS)
+ifeq ($(USE_S3), 1)
+CFLAGS += -DUSE_S3=1
+endif
+LDFLAGS = $(EXTRA_LDFLAGS) $(THIRD_LIB) -lpthread # -lrt
+
+PS_LIB = build/libps.a
+PS_MAIN = build/libps_main.a
+# TEST_MAIN = build/test_main.o
+
+clean:
+ rm -rf build
+ find src -name "*.pb.[ch]*" -delete
+
+ps: $(PS_LIB) $(PS_MAIN) #$(TEST_MAIN)
+
+# PS system
+ps_srcs = $(wildcard src/*.cc src/*/*.cc)
+ps_protos = $(wildcard src/proto/*.proto)
+ps_objs = $(patsubst src/%.proto, build/%.pb.o, $(ps_protos)) \
+ $(patsubst src/%.cc, build/%.o, $(ps_srcs))
+
+build/libps.a: $(patsubst %.proto, %.pb.h, $(ps_protos)) $(ps_objs)
+ ar crv $@ $(filter %.o, $?)
+
+build/libps_main.a: build/ps_main.o
+ ar crv $@ $?
+
+build/%.o: src/%.cc
+ @mkdir -p $(@D)
+ $(CC) $(INCPATH) -std=c++0x -MM -MT build/$*.o $< >build/$*.d
+ $(CC) $(CFLAGS) -c $< -o $@
+
+%.pb.cc %.pb.h : %.proto
+ ${THIRD_PATH}/bin/protoc --cpp_out=./src --proto_path=./src $<
+
+-include build/*/*.d
+-include build/*/*/*.d
+-include test/ps_test.mk
+-include guide/ps_guide.mk
View
@@ -1,2 +1,20 @@
-# ps-lite
-A light parameter server interface
+<img src="http://parameterserver.org/images/parameterserver.png" alt="Parameter Server" style="width: 500px;">
+
+<!-- [xx](http://parameterserver.org/images/parameterserver.png) -->
+
+The parameter server is a distributed system scaling to industry size machine
+learning problems. It provides asynchronous and zero-copy key-value pair
+communications between worker machines and server machines. It also supports
+flexiable data consistency model, data filters, and flexiable server machine
+programming.
+
+- [Tutorial](guide/)
+- [API Document](doc/)
+- [Wiki](https://github.com/dmlc/parameter_server/wiki/)
+- How to [build](make/)
+- Examples
+ - [Linear method](example/linear)
+ - Deep neural network, see [CXXNET](https://github.com/dmlc/cxxnet) and [Minverva](https://github.com/minerva-developers/minerva)
+- Research papers: System
+ [OSDI'14](http://www.cs.cmu.edu/~muli/file/parameter_server_osdi14.pdf),
+ Algorithm [NIPS'14](http://www.cs.cmu.edu/~muli/file/parameter_server_nips14.pdf)
Oops, something went wrong.

0 comments on commit eaed92a

Please sign in to comment.