Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

change code structure and try to clean wrapper definitions #8

Merged
merged 16 commits into from
May 7, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
16 changes: 16 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -72,3 +72,19 @@ dkms.conf
/x64/Debug
/packages
/Search/Search.vcxproj.user
/AnnService/IndexSearcher.vcxproj.user
/Wrappers/inc/SWIGTYPE_p_RemoteSearchResult.java
/Wrappers/inc/SWIGTYPE_p_QueryResult.java
/Wrappers/inc/SPTAGJNI.java
/Wrappers/inc/SPTAGClientJNI.java
/Wrappers/inc/SPTAGClient.py
/Wrappers/inc/SPTAGClient.java
/Wrappers/inc/SPTAG.py
/Wrappers/inc/SPTAG.java
/Wrappers/inc/CoreInterface_pwrap.cpp
/Wrappers/inc/CoreInterface_jwrap.cpp
/Wrappers/inc/ClientInterface_pwrap.cpp
/Wrappers/inc/ClientInterface_jwrap.cpp
/Wrappers/inc/AnnIndex.java
/Wrappers/inc/AnnClient.java
/AnnService.users - Copy.props
2 changes: 2 additions & 0 deletions AnnService.users.props
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@
<PropertyGroup Label="UserMacros">
<OutAppDir>$(SolutionDir)\$(Platform)\$(Configuration)\</OutAppDir>
<OutLibDir>$(SolutionDir)\$(Platform)\$(Configuration)\</OutLibDir>
<JavaIncDir></JavaIncDir>
<JavaLib></JavaLib>
</PropertyGroup>
</Project>

7 changes: 6 additions & 1 deletion AnnService/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ file(GLOB SRC_FILES ${PROJECT_SOURCE_DIR}/AnnService/src/Core/*.cpp ${PROJECT_SO
include_directories(${PROJECT_SOURCE_DIR}/AnnService)

add_library (SPTAGLib SHARED ${SRC_FILES} ${HDR_FILES})
target_link_libraries (SPTAGLib ${TBB_LIBRARIES})
add_library (SPTAGLibStatic STATIC ${SRC_FILES} ${HDR_FILES})
set_target_properties(SPTAGLibStatic PROPERTIES OUTPUT_NAME SPTAGLib)

Expand All @@ -30,7 +31,11 @@ file(GLOB BUILDER_FILES ${SRC_FILES} ${PROJECT_SOURCE_DIR}/AnnService/src/IndexB
add_executable (indexbuilder ${BUILDER_FILES} ${BUILDER_HDR_FILES})
target_link_libraries(indexbuilder ${Boost_LIBRARIES} ${TBB_LIBRARIES})

install(TARGETS SPTAGLib SPTAGLibStatic server client aggregator indexbuilder
file(GLOB SEARCHER_FILES ${SRC_FILES} ${PROJECT_SOURCE_DIR}/AnnService/src/IndexSearcher/*.cpp)
add_executable (indexsearcher ${SEARCHER_FILES} ${HDR_FILES})
target_link_libraries(indexsearcher ${Boost_LIBRARIES} ${TBB_LIBRARIES})

install(TARGETS SPTAGLib SPTAGLibStatic server client aggregator indexbuilder indexsearcher
RUNTIME DESTINATION bin
ARCHIVE DESTINATION lib
LIBRARY DESTINATION lib)
6 changes: 3 additions & 3 deletions Search/Search.vcxproj → AnnService/IndexSearcher.vcxproj
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,9 @@
<PropertyGroup Label="Globals">
<VCProjectVersion>15.0</VCProjectVersion>
<ProjectGuid>{97615D3B-9FA0-469E-B229-95A91A5087E0}</ProjectGuid>
<RootNamespace>Search</RootNamespace>
<RootNamespace>IndexSearcher</RootNamespace>
<WindowsTargetPlatformVersion>8.1</WindowsTargetPlatformVersion>
<ProjectName>Search</ProjectName>
<ProjectName>IndexSearcher</ProjectName>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<Import Project="$(SolutionDir)\AnnService.users.props" />
Expand Down Expand Up @@ -136,7 +136,7 @@
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ClCompile Include="main.cpp" />
<ClCompile Include="src\IndexSearcher\main.cpp" />
</ItemGroup>
<ItemGroup>
<None Include="packages.config">
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
</Filter>
</ItemGroup>
<ItemGroup>
<ClCompile Include="main.cpp">
<ClCompile Include="src\IndexSearcher\main.cpp">
<Filter>Source Files</Filter>
</ClCompile>
</ItemGroup>
Expand Down
4 changes: 2 additions & 2 deletions AnnService/inc/Core/BKT/ParameterDefinitionList.h
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@ DefineBKTParameter(m_pGraph.m_iTPTLeafSize, int, 2000L, "TPTLeafSize")
DefineBKTParameter(m_pGraph.m_numTopDimensionTPTSplit, int, 5L, "NumTopDimensionTpTreeSplit")

DefineBKTParameter(m_pGraph.m_iNeighborhoodSize, int, 32L, "NeighborhoodSize")
DefineBKTParameter(m_pGraph.m_iNeighborhoodScale, int, 16L, "GraphNeighborhoodScale")
DefineBKTParameter(m_pGraph.m_iCEFScale, int, 4L, "GraphCEFScale")
DefineBKTParameter(m_pGraph.m_iNeighborhoodScale, int, 2L, "GraphNeighborhoodScale")
DefineBKTParameter(m_pGraph.m_iCEFScale, int, 2L, "GraphCEFScale")
DefineBKTParameter(m_pGraph.m_iRefineIter, int, 0L, "RefineIterations")
DefineBKTParameter(m_pGraph.m_iCEF, int, 1000L, "CEF")
DefineBKTParameter(m_pGraph.m_iMaxCheckForRefineGraph, int, 10000L, "MaxCheckForRefineGraph")
Expand Down
2 changes: 2 additions & 0 deletions AnnService/inc/Core/Common/NeighborhoodGraph.h
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,7 @@ namespace SPTAG
for (int i = 0; i < m_iGraphSize; i++)
{
RefineNode<T>(index, i, false);
if (i % 1000 == 0) std::cout << "\rRefine 1 " << (i * 100 / m_iGraphSize) << "%";
}
std::cout << "Refine RNG, graph acc:" << GraphAccuracyEstimation(index, 100, idmap) << std::endl;

Expand All @@ -132,6 +133,7 @@ namespace SPTAG
for (int i = 0; i < m_iGraphSize; i++)
{
RefineNode<T>(index, i, false);
if (i % 1000 == 0) std::cout << "\rRefine 2 " << (i * 100 / m_iGraphSize) << "%";
}
std::cout << "Refine RNG, graph acc:" << GraphAccuracyEstimation(index, 100, idmap) << std::endl;

Expand Down
4 changes: 3 additions & 1 deletion AnnService/inc/Core/CommonDataStructure.h
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,9 @@ class ByteArray

std::uint8_t* Data() const;

std::size_t Length() const;
std::size_t Length() const;

void SetData(std::uint8_t* p_array, std::size_t p_length);

std::shared_ptr<std::uint8_t> DataHolder() const;

Expand Down
4 changes: 2 additions & 2 deletions AnnService/inc/Core/KDT/ParameterDefinitionList.h
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@ DefineKDTParameter(m_pGraph.m_iTPTLeafSize, int, 2000L, "TPTLeafSize")
DefineKDTParameter(m_pGraph.m_numTopDimensionTPTSplit, int, 5L, "NumTopDimensionTPTSplit")

DefineKDTParameter(m_pGraph.m_iNeighborhoodSize, int, 32L, "NeighborhoodSize")
DefineKDTParameter(m_pGraph.m_iNeighborhoodScale, int, 16L, "GraphNeighborhoodScale")
DefineKDTParameter(m_pGraph.m_iCEFScale, int, 4L, "GraphCEFScale")
DefineKDTParameter(m_pGraph.m_iNeighborhoodScale, int, 2L, "GraphNeighborhoodScale")
DefineKDTParameter(m_pGraph.m_iCEFScale, int, 2L, "GraphCEFScale")
DefineKDTParameter(m_pGraph.m_iRefineIter, int, 0L, "RefineIterations")
DefineKDTParameter(m_pGraph.m_iCEF, int, 1000L, "CEF")
DefineKDTParameter(m_pGraph.m_iMaxCheckForRefineGraph, int, 10000L, "MaxCheckForRefineGraph")
Expand Down
10 changes: 9 additions & 1 deletion AnnService/src/Core/CommonDataStructure.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,15 @@ ByteArray::Data() const
std::size_t
ByteArray::Length() const
{
return m_length;
return m_length;
}


void
ByteArray::SetData(std::uint8_t* p_array, std::size_t p_length)
{
m_data = p_array;
m_length = p_length;
}


Expand Down
12 changes: 8 additions & 4 deletions AnnService/src/IndexBuilder/main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -57,8 +57,8 @@ int main(int argc, char* argv[])

ErrorCode code;
if (options->m_inputFiles.find("BIN:") == 0) {
options->m_inputFiles = options->m_inputFiles.substr(4);
std::ifstream inputStream(options->m_inputFiles, std::ifstream::binary);
std::vector<std::string> files = SPTAG::Helper::StrUtils::SplitString(options->m_inputFiles.substr(4), ",");
std::ifstream inputStream(files[0], std::ifstream::binary);
if (!inputStream.is_open()) {
fprintf(stderr, "Failed to read input file.\n");
exit(1);
Expand All @@ -71,9 +71,13 @@ int main(int argc, char* argv[])
char* vecBuf = reinterpret_cast<char*>(vectorSet.Data());
inputStream.read(vecBuf, totalRecordVectorBytes);
inputStream.close();

std::shared_ptr<VectorSet> p_vectorSet(new BasicVectorSet(vectorSet, options->m_inputValueType, col, row));
code = indexBuilder->BuildIndex(p_vectorSet, nullptr);

std::shared_ptr<MetadataSet> p_metaSet = nullptr;
if (files.size() >= 3) {
p_metaSet.reset(new FileMetadataSet(files[1], files[2]));
}
code = indexBuilder->BuildIndex(p_vectorSet, p_metaSet);
indexBuilder->SaveIndex(options->m_outputFolder);
}
else {
Expand Down
File renamed without changes.
24 changes: 19 additions & 5 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -86,15 +86,29 @@ else()
message (FATAL_ERROR "Could not find Boost 1.67!")
endif()

find_library(TBB_LIBRARIES libtbb${CMAKE_SHARED_LIBRARY_SUFFIX})
if (TBB_LIBRARIES)
if (WIN32)
if (${CMAKE_SIZEOF_VOID_P} EQUAL "8")
set (TBB_LIBRARY_SUFFIX "lib/intel64/vc14")
else()
set (TBB_LIBRARY_SUFFIX "lib/ia32/vc14")
endif()

find_path(TBB_INCLUDE_DIRS tbb/tbb.h HINTS ${TBB_DIR} "C:/Program Files/Intel/TBB" PATH_SUFFIXES include)
find_library(TBB_LIBRARIES tbb${CMAKE_STATIC_LIBRARY_SUFFIX} HINTS ${TBB_DIR} "C:/Program Files/Intel/TBB" PATH_SUFFIXES ${TBB_LIBRARY_SUFFIX})
else()
find_path(TBB_INCLUDE_DIRS tbb/tbb.h HINTS ${TBB_DIR} "/usr/" PATH_SUFFIXES include)
find_library(TBB_LIBRARIES libtbb${CMAKE_SHARED_LIBRARY_SUFFIX} HINTS ${TBB_DIR} "/usr/")
endif()

if (TBB_INCLUDE_DIRS AND TBB_LIBRARIES)
include_directories (${TBB_INCLUDE_DIRS})
message (STATUS "Found TBB.")
message (STATUS "Library: ${TBB_LIBRARIES}")
message (STATUS "Include Path:" ${TBB_INCLUDE_DIRS})
message (STATUS "Library:" ${TBB_LIBRARIES})
else()
message (FATAL_ERROR "Could not find TBB!")
endif()

add_subdirectory (AnnService)
add_subdirectory (PythonWrapper)
add_subdirectory (Wrappers)
add_subdirectory (Test)
add_subdirectory (Search)
58 changes: 0 additions & 58 deletions PythonWrapper/CMakeLists.txt

This file was deleted.

1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ The searches in the trees and the graph are iteratively conducted.
* swig >= 3.0
* cmake >= 3.12.0
* boost >= 1.67.0
* tbb >= 4.2

### **Install**

Expand Down
48 changes: 37 additions & 11 deletions SPTAG.sln
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Server", "AnnService\Server
{C2BC5FDE-C853-4F3D-B7E4-2C9B5524DDF9} = {C2BC5FDE-C853-4F3D-B7E4-2C9B5524DDF9}
EndProjectSection
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "PythonCore", "PythonWrapper\PythonCore.vcxproj", "{AF31947C-0495-42FE-A1AD-8F0DA2A679C7}"
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "PythonCore", "Wrappers\PythonCore.vcxproj", "{AF31947C-0495-42FE-A1AD-8F0DA2A679C7}"
ProjectSection(ProjectDependencies) = postProject
{C2BC5FDE-C853-4F3D-B7E4-2C9B5524DDF9} = {C2BC5FDE-C853-4F3D-B7E4-2C9B5524DDF9}
EndProjectSection
Expand All @@ -33,7 +33,7 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Aggregator", "AnnService\Ag
{C2BC5FDE-C853-4F3D-B7E4-2C9B5524DDF9} = {C2BC5FDE-C853-4F3D-B7E4-2C9B5524DDF9}
EndProjectSection
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "PythonClient", "PythonWrapper\PythonClient.vcxproj", "{9B014CF6-E3FB-4BD4-B3B1-D26297BB31AA}"
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "PythonClient", "Wrappers\PythonClient.vcxproj", "{9B014CF6-E3FB-4BD4-B3B1-D26297BB31AA}"
ProjectSection(ProjectDependencies) = postProject
{F9A72303-6381-4C80-86FF-606A2F6F7B96} = {F9A72303-6381-4C80-86FF-606A2F6F7B96}
{C2BC5FDE-C853-4F3D-B7E4-2C9B5524DDF9} = {C2BC5FDE-C853-4F3D-B7E4-2C9B5524DDF9}
Expand All @@ -44,9 +44,27 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "IndexBuilder", "AnnService\
{C2BC5FDE-C853-4F3D-B7E4-2C9B5524DDF9} = {C2BC5FDE-C853-4F3D-B7E4-2C9B5524DDF9}
EndProjectSection
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "IndexSearcher", "AnnService\IndexSearcher.vcxproj", "{97615D3B-9FA0-469E-B229-95A91A5087E0}"
ProjectSection(ProjectDependencies) = postProject
{C2BC5FDE-C853-4F3D-B7E4-2C9B5524DDF9} = {C2BC5FDE-C853-4F3D-B7E4-2C9B5524DDF9}
EndProjectSection
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Test", "Test\Test.vcxproj", "{29A25655-CCF2-47F8-8BC8-DFE1B5CF993C}"
ProjectSection(ProjectDependencies) = postProject
{F9A72303-6381-4C80-86FF-606A2F6F7B96} = {F9A72303-6381-4C80-86FF-606A2F6F7B96}
{C2BC5FDE-C853-4F3D-B7E4-2C9B5524DDF9} = {C2BC5FDE-C853-4F3D-B7E4-2C9B5524DDF9}
EndProjectSection
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Search", "Search\Search.vcxproj", "{97615D3B-9FA0-469E-B229-95A91A5087E0}"
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "JavaCore", "Wrappers\JavaCore.vcxproj", "{93FEB26B-965E-4157-8BE5-052F5CA112BB}"
ProjectSection(ProjectDependencies) = postProject
{C2BC5FDE-C853-4F3D-B7E4-2C9B5524DDF9} = {C2BC5FDE-C853-4F3D-B7E4-2C9B5524DDF9}
EndProjectSection
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "JavaClient", "Wrappers\JavaClient.vcxproj", "{8866BF98-AA2E-450F-9F33-083E007CCA74}"
ProjectSection(ProjectDependencies) = postProject
{F9A72303-6381-4C80-86FF-606A2F6F7B96} = {F9A72303-6381-4C80-86FF-606A2F6F7B96}
{C2BC5FDE-C853-4F3D-B7E4-2C9B5524DDF9} = {C2BC5FDE-C853-4F3D-B7E4-2C9B5524DDF9}
EndProjectSection
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Expand Down Expand Up @@ -120,14 +138,6 @@ Global
{F492F794-E78B-4B1F-A556-5E045B9163D5}.Release|x64.Build.0 = Release|x64
{F492F794-E78B-4B1F-A556-5E045B9163D5}.Release|x86.ActiveCfg = Release|Win32
{F492F794-E78B-4B1F-A556-5E045B9163D5}.Release|x86.Build.0 = Release|Win32
{29A25655-CCF2-47F8-8BC8-DFE1B5CF993C}.Debug|x64.ActiveCfg = Debug|x64
{29A25655-CCF2-47F8-8BC8-DFE1B5CF993C}.Debug|x64.Build.0 = Debug|x64
{29A25655-CCF2-47F8-8BC8-DFE1B5CF993C}.Debug|x86.ActiveCfg = Debug|Win32
{29A25655-CCF2-47F8-8BC8-DFE1B5CF993C}.Debug|x86.Build.0 = Debug|Win32
{29A25655-CCF2-47F8-8BC8-DFE1B5CF993C}.Release|x64.ActiveCfg = Release|x64
{29A25655-CCF2-47F8-8BC8-DFE1B5CF993C}.Release|x64.Build.0 = Release|x64
{29A25655-CCF2-47F8-8BC8-DFE1B5CF993C}.Release|x86.ActiveCfg = Release|Win32
{29A25655-CCF2-47F8-8BC8-DFE1B5CF993C}.Release|x86.Build.0 = Release|Win32
{97615D3B-9FA0-469E-B229-95A91A5087E0}.Debug|x64.ActiveCfg = Debug|x64
{97615D3B-9FA0-469E-B229-95A91A5087E0}.Debug|x64.Build.0 = Debug|x64
{97615D3B-9FA0-469E-B229-95A91A5087E0}.Debug|x86.ActiveCfg = Debug|Win32
Expand All @@ -136,6 +146,22 @@ Global
{97615D3B-9FA0-469E-B229-95A91A5087E0}.Release|x64.Build.0 = Release|x64
{97615D3B-9FA0-469E-B229-95A91A5087E0}.Release|x86.ActiveCfg = Release|Win32
{97615D3B-9FA0-469E-B229-95A91A5087E0}.Release|x86.Build.0 = Release|Win32
{29A25655-CCF2-47F8-8BC8-DFE1B5CF993C}.Debug|x64.ActiveCfg = Debug|x64
{29A25655-CCF2-47F8-8BC8-DFE1B5CF993C}.Debug|x64.Build.0 = Debug|x64
{29A25655-CCF2-47F8-8BC8-DFE1B5CF993C}.Debug|x86.ActiveCfg = Debug|Win32
{29A25655-CCF2-47F8-8BC8-DFE1B5CF993C}.Debug|x86.Build.0 = Debug|Win32
{29A25655-CCF2-47F8-8BC8-DFE1B5CF993C}.Release|x64.ActiveCfg = Release|x64
{29A25655-CCF2-47F8-8BC8-DFE1B5CF993C}.Release|x64.Build.0 = Release|x64
{29A25655-CCF2-47F8-8BC8-DFE1B5CF993C}.Release|x86.ActiveCfg = Release|Win32
{29A25655-CCF2-47F8-8BC8-DFE1B5CF993C}.Release|x86.Build.0 = Release|Win32
{93FEB26B-965E-4157-8BE5-052F5CA112BB}.Debug|x64.ActiveCfg = Debug|x64
{93FEB26B-965E-4157-8BE5-052F5CA112BB}.Debug|x86.ActiveCfg = Debug|Win32
{93FEB26B-965E-4157-8BE5-052F5CA112BB}.Release|x64.ActiveCfg = Release|x64
{93FEB26B-965E-4157-8BE5-052F5CA112BB}.Release|x86.ActiveCfg = Release|Win32
{8866BF98-AA2E-450F-9F33-083E007CCA74}.Debug|x64.ActiveCfg = Debug|x64
{8866BF98-AA2E-450F-9F33-083E007CCA74}.Debug|x86.ActiveCfg = Debug|Win32
{8866BF98-AA2E-450F-9F33-083E007CCA74}.Release|x64.ActiveCfg = Release|x64
{8866BF98-AA2E-450F-9F33-083E007CCA74}.Release|x86.ActiveCfg = Release|Win32
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand Down
17 changes: 0 additions & 17 deletions Search/CMakeLists.txt

This file was deleted.

12 changes: 0 additions & 12 deletions Search/packages.config

This file was deleted.