Skip to content
/ open-source-search-engine Public

Nov 20 2017 -- A distributed open source search engine and spider/crawler written in C/C++ for Linux on Intel/AMD. From gigablast dot com, which has binaries for download. See the README.md file at the very bottom of this page for instructions.

License

Apache-2.0 license
1.5k stars 452 forks
Star
Notifications

gigablast/open-source-search-engine

master
Switch branches/tags

Name already in use

A tag already exists with the provided branch name. Many Git commands accept both tag and branch names, so creating this branch may cause unexpected behavior. Are you sure you want to create this branch?
3 branches 0 tags
Code

Latest commit

@gigablast
gigablast Merge pull request #194 from BxmvVa4v/devel3
6474e32 Sep 1, 2022
Merge pull request #194 from BxmvVa4v/devel3 
Const correctness in public hash interface and small code cleanup
6474e32

Git stats

Files

Permalink
Failed to load latest commit information.
Type
Name
Latest commit message
Commit time
antiword-dir
 
 
diffbot-widget
 
 
doxygen
 
 
html
 
 
script
 
 
ucdata
 
 
.gitignore
 
 
Abbreviations.cpp
 
 
Abbreviations.h
 
 
Accessdb.cpp
 
 
Accessdb.h
 
 
Address.cpp
 
 
Address.h
 
 
Ads.cpp
 
 
Ads.h
 
 
AdultBit.cpp
 
 
AdultBit.h
 
 
AutoBan.cpp
 
 
AutoBan.h
 
 
BigFile.cpp
 
 
BigFile.h
 
 
Bits.cpp
 
 
Bits.h
 
 
Blaster.cpp
 
 
Blaster.h
 
 
Cachedb.cpp
 
 
Cachedb.h
 
 
CatRec.cpp
 
 
CatRec.h
 
 
Catdb.cpp
 
 
Catdb.h
 
 
Categories.cpp
 
 
Categories.h
 
 
Clusterdb.cpp
 
 
Clusterdb.h
 
 
Collectiondb.cpp
 
 
Collectiondb.h
 
 
Conf.cpp
 
 
Conf.h
 
 
CountryCode.cpp
 
 
CountryCode.h
 
 
DailyMerge.cpp
 
 
DailyMerge.h
 
 
DataFeed.cpp
 
 
DataFeed.h
 
 
Datedb.cpp
 
 
Datedb.h
 
 
Dates.cpp
 
 
Dates.h
 
 
Diff.cpp
 
 
Diff.h
 
 
Dir.cpp
 
 
Dir.h
 
 
DiskPageCache.cpp
 
 
DiskPageCache.h
 
 
Dns.cpp
 
 
Dns.h
 
 
DnsProtocol.h
 
 
Domains.cpp
 
 
Domains.h
 
 
Entities.cpp
 
 
Entities.h
 
 
Errno.cpp
 
 
Errno.h
 
 
Events.h
 
 
Facebook.cpp
 
 
Facebook.h
 
 
File.cpp
 
 
File.h
 
 
Flags.cpp
 
 
Flags.h
 
 
GeoIP.c
 
 
GeoIP.h
 
 
GeoIPCity.c
 
 
GeoIPCity.h
 
 
GeoIP_internal.h
 
 
HashTable.cpp
 
 
HashTable.h
 
 
HashTableT.cpp
 
 
HashTableT.h
 
 
HashTableX.cpp
 
 
HashTableX.h
 
 
Highlight.cpp
 
 
Highlight.h
 
 
Hostdb.cpp
 
 
Hostdb.h
 
 
HttpMime.cpp
 
 
HttpMime.h
 
 
HttpRequest.cpp
 
 
HttpRequest.h
 
 
HttpServer.cpp
 
 
HttpServer.h
 
 
Images.cpp
 
 
Images.h
 
 
IndexList.cpp
 
 
IndexList.h
 
 
IndexReadInfo.cpp
 
 
IndexReadInfo.h
 
 
IndexTable.cpp
 
 
IndexTable.h
 
 
IndexTable2.cpp
 
 
IndexTable2.h
 
 
Indexdb.cpp
 
 
Indexdb.h
 
 
Iso8859.cpp
 
 
Iso8859.h
 
 
Json.cpp
 
 
Json.h
 
 
LICENSE
 
 
Lang.cpp
 
 
Lang.h
 
 
LangList.cpp
 
 
LangList.h
 
 
Language.cpp
 
 
Language.h
 
 
LanguageIdentifier.cpp
 
 
LanguageIdentifier.h
 
 
LanguagePages.cpp
 
 
LanguagePages.h
 
 
Linkdb.cpp
 
 
Linkdb.h
 
 
LinkedList.h
 
 
Log.cpp
 
 
Log.h
 
 
Loop.cpp
 
 
Loop.h
 
 
Make.depend
 
 
Makefile
 
 
Matches.cpp
 
 
Matches.h
 
 
Mem.cpp
 
 
Mem.h
 
 
MemPool.cpp
 
 
MemPool.h
 
 
MemPoolTree.cpp
 
 
MemPoolTree.h
 
 
MetaContainer.cpp
 
 
MetaContainer.h
 
 
Mime.cpp
 
 
Mime.h
 
 
Monitordb.cpp
 
 
Monitordb.h
 
 
Msg0.cpp
 
 
Msg0.h
 
 
Msg1.cpp
 
 
Msg1.h
 
 
Msg13.cpp
 
 
Msg13.h
 
 
Msg17.cpp
 
 
Msg17.h
 
 
Msg1f.cpp
 
 
Msg1f.h
 
 
Msg2.cpp
 
 
Msg2.h
 
 
Msg20.cpp
 
 
Msg20.h
 
 
Msg22.cpp
 
 
Msg22.h
 
 
Msg24.cpp
 
 
Msg28.cpp
 
 
Msg28.h
 
 
Msg2a.cpp
 
 
Msg2a.h
 
 
Msg2b.cpp
 
 
Msg2b.h
 
 
Msg3.cpp
 
 
Msg3.h
 
 
Msg30.cpp
 
 
Msg30.h
 
 
Msg35.cpp
 
 
Msg35.h
 
 
Msg36.cpp
 
 
Msg36.h
 
 
Msg37.cpp
 
 
Msg37.h
 
 
Msg39.cpp
 
 
Msg39.h
 
 
Msg3a.cpp
 
 
Msg3a.h
 
 
Msg3e.cpp
 
 
Msg3e.h
 
 
Msg4.cpp
 
 
Msg4.h
 
 
Msg40.cpp
 
 
Msg40.h
 
 
Msg40Cache.cpp
 
 
Msg40Cache.h
 
 
Msg42.cpp
 
 
Msg42.h
 
 
Msg5.cpp
 
 
Msg5.h
 
 
Msg51.cpp
 
 
Msg51.h
 
 
Msg6b.cpp
 
 
Msg6b.h
 
 
Msg8b.cpp
 
 
Msg8b.h
 
 
Msg9b.cpp
 
 
Msg9b.h
 
 
MsgC.cpp
 
 
MsgC.h
 
 
Msgaa.cpp
 
 
Msgaa.h
 
 
Msge0.cpp
 
 
Msge0.h
 
 
Msge1.cpp
 
 
Msge1.h
 
 
Multicast.cpp
 
 
Multicast.h
 
 
OldDiskPageCache.cpp
 
 
OldDiskPageCache.h
 
 
PageAddColl.cpp
 
 
PageAddUrl.cpp
 
 
PageBasic.cpp
 
 
PageCatdb.cpp
 
 
PageCrawlBot.cpp
 
 
PageCrawlBot.h
 
 
PageDirectory.cpp
 
 
PageEvents.cpp
 
 
PageGet.cpp
 
 
PageHosts.cpp
 
 
PageIndexdb.cpp
 
 
PageInject.cpp
 
 
PageInject.h
 
 
PageLogView.cpp
 
 
PageNetTest.cpp
 
 
PageNetTest.h
 
 
PageOverview.cpp
 
 
PageParser.cpp
 
 
PageParser.h
 
 
PagePerf.cpp
 
 
PageReindex.cpp
 
 
PageReindex.h
 
 
PageResults.cpp
 
 
PageResults.h
 
 
PageRoot.cpp
 
 
PageSockets.cpp
 
 
PageSpam.cpp
 
 
PageStats.cpp
 
 
PageStatsdb.cpp
 
 
PageSubmit.cpp
 
 
PageThesaurus.cpp
 
 
PageThreads.cpp
 
 
PageTitledb.cpp
 
 
PageTurk.cpp
 
 
PageTurk.h
 
 
Pages.cpp
 
 
Pages.h
 
 
Parms.cpp
 
 
Parms.h
 
 
Phrases.cpp
 
 
Phrases.h
 
 
PingServer.cpp
 
 
PingServer.h
 
 
Placedb.cpp
 
 
Placedb.h
 
 
Pops.cpp
 
 
Pops.h
 
 
Pos.cpp
 
 
Pos.h
 
 
Posdb.cpp
 
 
Posdb.h
 
 
PostQueryRerank.cpp
 
 
PostQueryRerank.h
 
 
Process.cpp
 
 
Process.h
 
 
Profiler.cpp
 
 
Profiler.h
 
 
Proxy.cpp
 
 
Proxy.h
 
 
Punycode.cpp
 
 
Punycode.h
 
 
QAClient.cpp
 
 
QAClient.h
 
 
Query.cpp
 
 
Query.h
 
 
README.md
 
 
Rdb.cpp
 
 
Rdb.h
 
 
RdbBase.cpp
 
 
RdbBase.h
 
 
RdbBuckets.cpp
 
 
RdbBuckets.h
 
 
RdbCache.cpp
 
 
RdbCache.h
 
 
RdbDump.cpp
 
 
RdbDump.h
 
 
RdbList.cpp
 
 
RdbList.h
 
 
RdbMap.cpp
 
 
RdbMap.h
 
 
RdbMem.cpp
 
 
RdbMem.h
 
 
RdbMerge.cpp
 
 
RdbMerge.h
 
 
RdbScan.cpp
 
 
RdbScan.h
 
 
RdbTree.cpp
 
 
RdbTree.h
 
 
Rebalance.cpp
 
 
Rebalance.h
 
 
Repair.cpp
 
 
Repair.h
 
 
RequestTable.cpp
 
 
RequestTable.h
 
 
Revdb.cpp
 
 
Revdb.h
 
 
S99gb
 
 
SafeBuf.cpp
 
 
SafeBuf.h
 
 
SafeList.h
 
 
Sanity.h
 
 
Scores.cpp
 
 
Scores.h
 
 
Scraper.cpp
 
 
Scraper.h
 
 
SearchInput.cpp
 
 
SearchInput.h
 
 
Sections.cpp
 
 
Sections.h
 
 
SiteGetter.cpp
 
 
SiteGetter.h
 
 
Speller.cpp
 
 
Speller.h
 
 
Spider.cpp
 
 
Spider.h
 
 
SpiderProxy.cpp
 
 
SpiderProxy.h
 
 
Stats.cpp
 
 
Stats.h
 
 
Statsdb.cpp
 
 
Statsdb.h
 
 
StopWords.cpp
 
 
StopWords.h
 
 
Strings.cpp
 
 
Strings.h
 
 
Summary.cpp
 
 
Summary.h
 
 
Syncdb.cpp
 
 
Syncdb.h
 
 
Synonyms.cpp
 
 
Synonyms.h
 
 
Tagdb.cpp
 
 
Tagdb.h
 
 
TcpServer.cpp
 
 
TcpServer.h
 
 
TcpSocket.h
 
 
Test.cpp
 
 
Test.h
 
 
Tfndb.cpp
 
 
Tfndb.h
 
 
Thesaurus.cpp
 
 
Thesaurus.h
 
 
Threads.cpp
 
 
Threads.h
 
 
Timedb.cpp
 
 
Timedb.h
 
 
Timer.h
 
 
Title.cpp
 
 
Title.h
 
 
Titledb.cpp
 
 
Titledb.h
 
 
TopTree.cpp
 
 
TopTree.h
 
 
TuringTest.cpp
 
 
TuringTest.h
 
 
Turkdb.cpp
 
 
UCNormalizer.cpp
 
 
UCNormalizer.h
 
 
UCPropTable.cpp
 
 
UCPropTable.h
 
 
UCWordIterator.cpp
 
 
UCWordIterator.h
 
 
UdpProtocol.h
 
 
UdpServer.cpp
 
 
UdpServer.h
 
 
UdpSlot.cpp
 
 
UdpSlot.h
 
 
Unicode.cpp
 
 
Unicode.h
 
 
UnicodeProperties.cpp
 
 
UnicodeProperties.h
 
 
Url.cpp
 
 
Url.h
 
 
Users.cpp
 
 
Users.h
 
 
ValidPointer.cpp
 
 
ValidPointer.h
 
 
Vector.cpp
 
 
Vector.h
 
 
Version.cpp
 
 
Version.h
 
 
Weights.cpp
 
 
Weights.h
 
 
Wiki.cpp
 
 
Wiki.h
 
 
Wiktionary.cpp
 
 
Wiktionary.h
 
 
Words.cpp
 
 
Words.h
 
 
Xml.cpp
 
 
Xml.h
 
 
XmlDoc.cpp
 
 
XmlDoc.h
 
 
XmlNode.cpp
 
 
XmlNode.h
 
 
addtest.cpp
 
 
animate.cpp
 
 
antiword
 
 
badcattable.dat
 
 
blaster2.cpp
 
 
bmptopnm
 
 
camsort.cpp
 
 
catcountry.dat
 
 
character-sets
 
 
check_unicode.cpp
 
 
control.deb
 
 
convert.cpp
 
 
copyright.head
 
 
copyright.tail
 
 
create_ucd_tables.cpp
 
 
dlstubs.c
 
 
dmozparse.cpp
 
 
dnstest.cpp
 
 
dumpcore.cpp
 
 
errnotest.cpp
 
 
fastIndexTable.cpp
 
 
fctypes.cpp
 
 
fctypes.h
 
 
filterquerylogs.cpp
 
 
gb-1.0.spec
 
 
gb-include.h
 
 
gb.deb.rules
 
 
gb.pem
 
 
gbfilter.cpp
 
 
gbtitletest.cpp
 
 
geneaology.cpp
 
 
generateSuperMergeCode.cpp
 
 
geo_ip_table.cpp
 
 
geo_ip_table.h
 
 
getsample.cpp
 
 
giftopnm
 
 
gigablast.cbp
 
 
gigablast.layout
 
 
hash.cpp
 
 
hash.h
 
 
hashtest.cpp
 
 
hashtest2.cpp
 
 
hashtest3.cpp
 
 
hosts.cpp
 
 
iana_charset.cpp
 
 
iana_charset.h
 
 
init.gb.conf
 
 
injectme3
 
 
injectmedemo
 
 
injector.cpp
 
 
iostream.h
 
 
ip.cpp
 
 
ip.h
 
 
ipconfig.cpp
 
 
jointest.cpp
 
 
jpegtopnm
 
 
keepalive.cpp
 
 
libjpeg.so.62
 
 
libm.a
 
 
libnetpbm.so.10
 
 
libpng12.so.0
 
 
libpthread.a
 
 
libstdc++.a
 
 
libtiff.so.4
 
 
linkspam.cpp
 
 
linkspam.h
 
 
looptest.cpp
 
 
main.cpp
 
 
malloc.c
 
 
matches2.cpp
 
 
matches2.h
 
 
membustest.cpp
 
 
memtest.cpp
 
 
mergetest.cpp
 
 
mixfile.cpp
 
 
mmseg.h
 
 
monitor.cpp
 
 
mysynonyms.txt
 
 
numwords.cpp
 
 
parse_iana_charsets.pl
 
 
pdftohtml
 
 
pngtopnm
 
 
pnmscale
 
 
porter.cpp
 
 
postalCodes.txt
 
 
ppmtojpeg
 
 
pstotext
 
 
qa.cpp
 
 
quarantine.cpp
 
 
rdbtest.cpp
 
 
rdbtest2.cpp
 
 
readRec.cpp
 
 
reindex2.cpp
 
 
rescue.cpp
 
 
rmbots.cpp
 
 
seektest.cpp
 
 
seo.h
 
 
sitelinks.txt
 
 
sleepandlog.cpp
 
 
sort.cpp
 
 
sort.h
 
 
streambuf.h
 
 
superMergeTest.cpp
 
 
supported_charsets.cpp
 
 
supported_charsets.txt
 
 
test2.cpp
 
 
test_convert.cpp
 
 
test_hash.cpp
 
 
test_norm.cpp
 
 
test_parser.cpp
 
 
test_parser2.cpp
 
 
test_unicode.cpp
 
 
testfloats.cpp
 
 
threadtest.cpp
 
 
thunder.cpp
 
 
tifftopnm
 
 
treetest.cpp
 
 
types.h
 
 
udptest.cpp
 
 
unifiedDict.txt
 
 
uniq2.cpp
 
 
urlinfo.cpp
 
 
wikititles.txt.part1
 
 
wikititles.txt.part2
 
 
wiktionary-buf.txt
 
 
wiktionary-lang.txt
 
 
wiktionary-syns.dat
 
 
open-source-search-engine RUNNING GIGABLAST CODE ARCHITECTURE CONTACT BUILD DEPENDENCIES On Debian (and derivatives) On RedHat/CentOS

README.md

open-source-search-engine

An open source web and enterprise search engine and spider/crawler. As can be seen on http://www.gigablast.com/ .

RUNNING GIGABLAST

See html/faq.html for all administrative documentation including the quick start instructions.

Alternatively, visit http://www.gigablast.com/faq.html

CODE ARCHITECTURE

See html/developer.html for all code documentation.

Alternatively, visit http://www.gigablast.com/developer.html

CONTACT

Contact me for feature requests or help in general. I will work for free for good use cases. mattdwells@hotmail.com.

BUILD DEPENDENCIES

On Debian (and derivatives)

sudo apt-get install make g++ libssl-dev zlib1g-dev

On RedHat/CentOS

sudo yum install gcc-c++ openssl-devel

About

Nov 20 2017 -- A distributed open source search engine and spider/crawler written in C/C++ for Linux on Intel/AMD. From gigablast dot com, which has binaries for download. See the README.md file at the very bottom of this page for instructions.

Resources

Readme

License

Apache-2.0 license

Stars

1.5k stars

Watchers

138 watching

Forks

452 forks
Report repository

Releases

No releases published

Packages

No packages published

Contributors 15

+ 4 contributors

Languages