Skip to content

Commit

Permalink
Merge pull request #228 from stingergraph/dev
Browse files Browse the repository at this point in the history 
Release 17.04
  • Loading branch information
@davidediger
davidediger committed Apr 26, 2017
2 parents 2dad748 + 92348d4 commit 149d5b5
Show file tree
Hide file tree
Showing 225 changed files with 8,695 additions and 15,198 deletions.
137 changes: 98 additions & 39 deletions CMakeLists.txt
View file
Expand Up @@ -4,8 +4,8 @@ project (STINGER)
include (ExternalProject)
include (CTest)

set(STINGER_VERSION_MAJOR 2015)
set(STINGER_VERSION_MINOR 10)
set(STINGER_VERSION_MAJOR 2017)
set(STINGER_VERSION_MINOR 02)

set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib)
set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib)
Expand Down Expand Up @@ -47,7 +47,8 @@ ENDIF()
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=gnu9x")

# Doxygen
if(BUILD_DOCUMENTATION)
option(BUILD_DOCUMENTATION "Use Doxygen to create the HTML based API documentation" OFF)
if (BUILD_DOCUMENTATION)
find_package(Doxygen)
if (DOXYGEN_FOUND)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/doxygen/Doxyfile.in
Expand All @@ -60,19 +61,52 @@ if(BUILD_DOCUMENTATION)
endif()
endif()

if(APPLE)
set(SHARED_LIB_EXT .dylib)
else()
set(SHARED_LIB_EXT .so)
endif()

#protobuf
ExternalProject_Add(protobuf
PREFIX ${CMAKE_BINARY_DIR}/external
URL file://${CMAKE_SOURCE_DIR}/external/protobuf-2.5.0.tar.gz
URL_MD5 b751f772bdeb2812a2a8e7202bf1dae8
UPDATE_COMMAND ""
PATCH_COMMAND ""
CONFIGURE_COMMAND CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} ${CMAKE_BINARY_DIR}/external/src/protobuf/configure
--prefix=${CMAKE_BINARY_DIR}
--enable-shared
BUILD_COMMAND make
INSTALL_COMMAND make install
)
find_package(Protobuf)
# Look for installed version
if (${PROTOBUF_FOUND})
add_custom_target("protobuf")

add_library(_protobuf_library STATIC IMPORTED)
add_dependencies(_protobuf_library protobuf)
set_target_properties(_protobuf_library PROPERTIES IMPORTED_LOCATION ${PROTOBUF_LIBRARY})

add_library(_protobuf_lite_library STATIC IMPORTED)
add_dependencies(_protobuf_lite_library protobuf)
set_target_properties(_protobuf_lite_library PROPERTIES IMPORTED_LOCATION ${PROTOBUF_LITE_LIBRARY})

include_directories("${PROTOBUF_INCLUDE_DIR}")
# Download and build it ourselves
else ()
ExternalProject_Add(protobuf
PREFIX ${CMAKE_BINARY_DIR}/external
URL file://${CMAKE_SOURCE_DIR}/external/protobuf-cpp-3.1.0.tar.gz
UPDATE_COMMAND ""
PATCH_COMMAND ""
CONFIGURE_COMMAND CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} ${CMAKE_BINARY_DIR}/external/src/protobuf/configure
--prefix=${CMAKE_BINARY_DIR}
--enable-shared
BUILD_COMMAND $(MAKE)
INSTALL_COMMAND $(MAKE) install
)

set(PROTOBUF_PROTOC_EXECUTABLE "./protoc")

add_library(_protobuf_library STATIC IMPORTED)
add_dependencies(_protobuf_library protobuf)
set_target_properties(_protobuf_library PROPERTIES IMPORTED_LOCATION ${CMAKE_BINARY_DIR}/lib/libprotobuf${SHARED_LIB_EXT})

add_library(_protobuf_lite_library STATIC IMPORTED)
add_dependencies(_protobuf_lite_library protobuf)
set_target_properties(_protobuf_lite_library PROPERTIES IMPORTED_LOCATION ${CMAKE_BINARY_DIR}/lib/libprotobuf-lite${SHARED_LIB_EXT})

endif()

#libconfig
ExternalProject_Add(libconfig
Expand All @@ -85,16 +119,10 @@ ExternalProject_Add(libconfig
--prefix=${CMAKE_BINARY_DIR}
--enable-shared
--disable-examples
BUILD_COMMAND cp ${CMAKE_BINARY_DIR}/external/src/libconfig/lib/libconfig.h ${CMAKE_BINARY_DIR}/external/src/libconfig-build/lib/ && make
INSTALL_COMMAND make install
BUILD_COMMAND cp ${CMAKE_BINARY_DIR}/external/src/libconfig/lib/libconfig.h ${CMAKE_BINARY_DIR}/external/src/libconfig-build/lib/ && $(MAKE)
INSTALL_COMMAND $(MAKE) install
)

if(APPLE)
set(SHARED_LIB_EXT .dylib)
else()
set(SHARED_LIB_EXT .so)
endif()

add_library(_libconfig_library STATIC IMPORTED)
add_dependencies(_libconfig_library libconfig)
set_target_properties(_libconfig_library PROPERTIES IMPORTED_LOCATION ${CMAKE_BINARY_DIR}/lib/libconfig${SHARED_LIB_EXT})
Expand All @@ -103,18 +131,6 @@ add_library(_libconfig++_library STATIC IMPORTED)
add_dependencies(_libconfig++_library libconfig)
set_target_properties(_libconfig++_library PROPERTIES IMPORTED_LOCATION ${CMAKE_BINARY_DIR}/lib/libconfig++${SHARED_LIB_EXT})

add_library(_protobuf_library STATIC IMPORTED)
add_dependencies(_protobuf_library protobuf)
set_target_properties(_protobuf_library PROPERTIES IMPORTED_LOCATION ${CMAKE_BINARY_DIR}/lib/libprotobuf${SHARED_LIB_EXT})

add_library(_protobuf_lite_library STATIC IMPORTED)
add_dependencies(_protobuf_lite_library protobuf)
set_target_properties(_protobuf_lite_library PROPERTIES IMPORTED_LOCATION ${CMAKE_BINARY_DIR}/lib/libprotobuf-lite${SHARED_LIB_EXT})

add_library(_protoc_library STATIC IMPORTED)
add_dependencies(_protoc_library protobuf)
set_target_properties(_protoc_library PROPERTIES IMPORTED_LOCATION ${CMAKE_BINARY_DIR}/lib/libprotoc${SHARED_LIB_EXT})

# Memory size configuration bits for stinger_core
set(STINGER_DEFAULT_VERTICES "(1L<<24)" CACHE STRING "Default number of vertices")
set(STINGER_DEFAULT_NUMETYPES "5" CACHE STRING "Default number of edge types")
Expand All @@ -125,8 +141,25 @@ set(STINGER_NAME_STR_MAX "255" CACHE STRING "Max string length in physmap")

configure_file(${CMAKE_SOURCE_DIR}/lib/stinger_core/inc/stinger_defs.h.in ${CMAKE_BINARY_DIR}/include/stinger_core/stinger_defs.h @ONLY)
configure_file(${CMAKE_SOURCE_DIR}/lib/stinger_core/inc/stinger_names.h.in ${CMAKE_BINARY_DIR}/include/stinger_core/stinger_names.h @ONLY)
#file(COPY ${CMAKE_SOURCE_DIR}/lib/stinger_core/inc/stinger_defs.h DESTINATION ${CMAKE_BINARY_DIR}/include/stinger_core/)
#file(COPY ${CMAKE_SOURCE_DIR}/lib/stinger_core/inc/stinger_names.h DESTINATION ${CMAKE_BINARY_DIR}/include/stinger_core/)

# Define a custom function for copying headers to the build directory
function(publish_headers header_list destination)
set(published_headers "")
foreach(header ${${header_list}})
get_filename_component(name ${header} NAME)
set(output "${destination}/${name}")
list(APPEND published_headers ${output})
add_custom_command(
OUTPUT ${output}
COMMAND ${CMAKE_COMMAND} -E make_directory ${destination}
COMMAND ${CMAKE_COMMAND} -E copy ${header} ${destination}
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
DEPENDS ${header}
)
endforeach()
# Overwrite the list of headers in the caller, so targets depend on the published version
set(${header_list} ${published_headers} PARENT_SCOPE)
endfunction()

include_directories("${CMAKE_BINARY_DIR}/include")
include_directories("${CMAKE_SOURCE_DIR}/external/googletest/include/")
Expand All @@ -138,11 +171,37 @@ add_subdirectory(src)
enable_testing()
add_test(NamesTest ${CMAKE_BINARY_DIR}/bin/stinger_names_test)
add_test(StingerCoreTest ${CMAKE_BINARY_DIR}/bin/stinger_core_test)
add_test(StingerBatchTest ${CMAKE_BINARY_DIR}/bin/stinger_batch_test)
add_test(StingerPhysmapTest ${CMAKE_BINARY_DIR}/bin/stinger_physmap_test)
add_test(StingerTraversalTest ${CMAKE_BINARY_DIR}/bin/stinger_traversal_test)
add_test(StingerPagerankTest ${CMAKE_BINARY_DIR}/bin/stinger_pagerank_test)
add_test(StingerAdamicAdarTest ${CMAKE_BINARY_DIR}/bin/stinger_adamic_adar_test)
add_test(StingerBetweennessTest ${CMAKE_BINARY_DIR}/bin/stinger_betweenness_test)
add_test(StingerHitsTest ${CMAKE_BINARY_DIR}/bin/stinger_hits_test)
add_test(StingerDiameterTest ${CMAKE_BINARY_DIR}/bin/stinger_diameter_test)
add_test(StingerIndependentSetsTest ${CMAKE_BINARY_DIR}/bin/stinger_independent_sets_test)
add_test(StingerShortestPathsTest ${CMAKE_BINARY_DIR}/bin/stinger_shortest_paths)

find_program(BASH bash REQUIRED)
add_test(
NAME ClientServerTest
COMMAND ${BASH} "${CMAKE_SOURCE_DIR}/src/tests/client_server_test/client_server_test.sh" "${CMAKE_BINARY_DIR}" "${STINGER_USE_TCP}"
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/util/
)

add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND} -E env CTEST_OUTPUT_ON_FAILURE=1 --verbose DEPENDS stinger_names_test stinger_core_test stinger_physmap_test stinger_traversal_test stinger_pagerank_test stinger_adamic_adar_test stinger_betweenness_test)

add_custom_target(check
COMMAND ${CMAKE_CTEST_COMMAND} -E env CTEST_OUTPUT_ON_FAILURE=1 --verbose
DEPENDS
stinger_names_test
stinger_core_test
stinger_batch_test
stinger_physmap_test
stinger_traversal_test
stinger_pagerank_test
stinger_adamic_adar_test
stinger_betweenness_test
stinger_hits_test
stinger_diameter_test
stinger_independent_sets_test
stinger_shortest_paths
)
4 changes: 2 additions & 2 deletions LICENSE
View file
@@ -1,7 +1,7 @@
Libraries in /external are the right of their respective owners.
Libraries in /external and /lib are the right of their respective owners.

STINGER
Copyright (c) 2009-2015 Georgia Institute of Technology
Copyright (c) 2009-2017 Georgia Institute of Technology
All rights reserved.

Redistribution and use in source and binary forms, with or without
Expand Down
37 changes: 11 additions & 26 deletions README.md
View file
@@ -1,7 +1,7 @@
STINGER
=======
[![Build Status (master)](https://travis-ci.org/stingergraph/stinger.png?branch=master)](https://travis-ci.org/stingergraph/stinger)
[![Build Status (dev)](https://travis-ci.org/stingergraph/stinger.png?branch=dev)](https://travis-ci.org/stginergraph/stinger)
[![Build Status (dev)](https://travis-ci.org/stingergraph/stinger.png?branch=dev)](https://travis-ci.org/stingergraph/stinger)

Learn more at [stingergraph.com](http://stingergraph.com).

Expand All @@ -13,8 +13,6 @@ STINGER is a package designed to support streaming graph analytics by using in-m
doc/ - Documentation
doxygen/ - Doxygen generated documentation
external/ - External dependencies packaged with STINGER
flask/ - Python Flask Relay Server for interacting with the JSON RPC server and STINGER server
html/ - Basic web pages that communicate with the JSON RPC server
lib/ - The STINGER library and dependencies
stinger_alg/ - Algorithm kernels that work on the STINGER data structure
stinger_core/ - The Core STINGER data structure
Expand All @@ -27,18 +25,20 @@ STINGER is a package designed to support streaming graph analytics by using in-m
streams/ - Binaries for streaming in new data
tools/ - Auxiliary tools
py/stinger - Python bindings to STINGER
standalone/ - Standalone binaries that use the STINGER core data structure
templates/json - Common templates for stream ingest
tests/ - Tests for the STINGER data structure and algorithms
SOURCEME.sh - File to be executed from out-of-source build directory to link the html/ web pages with the STINGER server
util/ -
flask/ - Python Flask Relay Server for interacting with the JSON RPC server and STINGER server
graph_explorer/ - Sample web pages for interacting with JSON RPC server
management/ - Web app for monitoring STINGER server
stingerctl - Control script for starting and stopping STINGER server, algorithms, and clients

Building
========

STINGER is built using [CMake](http://www.cmake.org). From the root of STINGER, first create a build directory:

mkdir build && cd build
. ../SOURCEME.sh

Then call CMake from that build directory to automatically configure the build and to create a Makefile:

Expand All @@ -52,15 +52,11 @@ Note: the -j flag is a multi-threaded build. Typically you should match the arg

All binary targets will be built and placed in build/bin. They are named according to the folder from which they were
built (so src/bin/server produces build/bin/stinger\_server, src/bin/clients/tools/json\_rpc\_server produces
build/bin/stinger\_json\_rpc\_server, etc.). If you ran SOURCEME.sh from the build directory as instructed above, the build/bin
directory is appended to your path.
build/bin/stinger\_json\_rpc\_server, etc.).

Executable Targets
==================

As indicated by the directory structure, there are three primary types of executable targets: clients, server, and standalone,
and subtypes in the case of clients.

The STINGER server maintains a STINGER graph in memory and can maintain multiple connections with client streams, algorithms, and monitors.

Client streams can send edges consisting of source, destination, weight, time, and type where some fields are optional
Expand All @@ -74,14 +70,10 @@ The server guarantees that all of an algorithm's dependencies will finish proces
Clients algorithms are required to provide a description string that indicates what data is stored and the type of the data.

Client tools are intended to be read-only, but are notified of all running algorithms and shared data. An example
of this kind of client is the JSON-RPC server (src/bin/clients/tools/json\_rpc\_server). This server provides access
of this kind of client is the JSON-RPC server (src/clients/tools/json\_rpc\_server). This server provides access
to shared algorithm data via JSON-RPC calls over HTTP. Additionally, some primitive operations are provided to support
selecting the top vertices as scored by a particular algorithm or obtaining the shortest paths between two vertices, for example.

Standalone executables are generally self-contained and use the STINGER
libraries for the graph structure and supporting functions. Most of the existing standalone executables demonstrate
a single streaming or static algorithm on a synthetic R-MAT graph and edge stream.

Running
=======

Expand All @@ -96,8 +88,8 @@ To run an example using the server and five terminals:
term5:build$ ./bin/stinger_rmat_edge_generator -n 100000 -x 10000

This will start a stream of R-MAT edges over 100,000 vertices in batches of 10,000 edges. A connected component labeling
and PageRank scoring will be maintained. The JSON-RPC server will host interactive web pages at
http://localhost:8088/full.html are powered by the live streaming analysis. The total memory usage of the dynamic graph is limited to 1 GiB.
and PageRank scoring will be maintained. The JSON-RPC server will host an endpoint at http://localhost:8088/json-rpc
The total memory usage of the dynamic graph is limited to 1 GiB.

Server Configuration
--------------------
Expand Down Expand Up @@ -228,13 +220,6 @@ This file would create edges between email addresses using the length field as t
cat emails.csv | ./bin/stinger_csv_stream template.csv

Please be aware that the CSV parser and the underlying code to parse CSV files does not currently trim whitespace, and does not treat quoted strings of any kind as quoted.

Using a Standalone Client
-------------------------
To create a toy R-MAT graph (256K vertices and 2M undirected edges) and run the insert-remove benchmark:

term1:build$ stinger_rmat_graph_generator -s 18 -e 8 -n 100000
term1:build$ stinger_insert_remove_benchmark -n 1 -b 100000 g.18.8.bin a.18.8.100000.bin


[![githalytics.com alpha](https://cruel-carlota.pagodabox.com/a7d82e7aa670122314238336dbbd7c89 "githalytics.com")](http://githalytics.com/robmccoll/stinger)
Expand All @@ -248,7 +233,7 @@ Runtime Issues
STINGER allocates and manages its own memory. When STINGER starts, it allocates one large block of memory (enough to hold its maximum size), and then manages its own memory allocation from that pool. The server version of STINGER does this in shared memory so that multiple processes can see the graph. Unfortunately, the error handling for memory allocations is not particularly user-friendly at the moment. Changing the way that this works is on the issues list (see https://github.com/robmccoll/stinger/issues/8).

- "Bus error" when running the server: The size of STINGER that the server is trying to allocate is too large for your memory. First try to increase the size of your /dev/shm (See below). If this does not work reduce the size of your STINGER and recompile.
- "XXX: eb pool exhausted" or "STINGER has run out of internal storage space" when running the server, standalone executables, or anything else using stinger\_core: you have run out of internal edge storage. Increase the size of STINGER and recompile.
- "XXX: eb pool exhausted" or "STINGER has run out of internal storage space" when running the server: you have run out of internal edge storage. Increase the size of STINGER and recompile.

See the section on STINGER server configuration to adjust the sizing of STINGER

Expand Down
23 changes: 23 additions & 0 deletions ReleaseNotes.md
View file
Expand Up @@ -4,6 +4,29 @@ Release Notes
Version
-------

These notes are for STINGER version 17.02

Notes
-----

- CMake fixes for TravisCI
- Added edge direction to STINGER
- Traversal macros refactored for reduced code
- Broken symlinks fixed
- Security and bug fixes
- Removed Cray MTA/XMT support
- New algorithm client for Streaming Connected Components
- New HTTP API endpoint for adding edge weight
- Fixed stingerctl script
- New algorithms for HITS, graph diameter, independent set, shortest path, and graph partitioning
- Changes to full/empty libraries for POWER8 support
- New parallel algorithm for inserting a batch of edges
- Use system Protocol Buffers library if installed
- Upgraded distributed of Protocol Buffers to 3.1.0

Version
-------

These notes are for STINGER version 15.10

Notes
Expand Down
9 changes: 0 additions & 9 deletions SOURCEME.sh
View file

This file was deleted.

0 comments on commit 149d5b5

Please sign in to comment.