Permalink
Browse files

implemented datasources

  • Loading branch information...
nl5887 committed Jan 17, 2017
1 parent 78a89bc commit 6fefd04add9f20776cd5083d8be5f6a33c4ceffa
Showing with 26,542 additions and 258 deletions.
  1. +27 −4 README.md
  2. +1 −5 cmd/cmd.go
  3. +21 −0 config.toml.sample
  4. +79 −0 server/config.go
  5. +70 −82 server/conn.go
  6. +124 −0 server/datasources/blockchain/blockchain.go
  7. +68 −49 server/datasources/es5/elasticsearchindexv5.go
  8. +2 −3 server/datasources/index.go
  9. +174 −0 server/datasources/twitter/twitter.go
  10. +76 −2 server/server.go
  11. +3 −0 vendor/github.com/BurntSushi/toml/COMPATIBLE
  12. +14 −0 vendor/github.com/BurntSushi/toml/COPYING
  13. +19 −0 vendor/github.com/BurntSushi/toml/Makefile
  14. +220 −0 vendor/github.com/BurntSushi/toml/README.md
  15. +509 −0 vendor/github.com/BurntSushi/toml/decode.go
  16. +121 −0 vendor/github.com/BurntSushi/toml/decode_meta.go
  17. +27 −0 vendor/github.com/BurntSushi/toml/doc.go
  18. +568 −0 vendor/github.com/BurntSushi/toml/encode.go
  19. +19 −0 vendor/github.com/BurntSushi/toml/encoding_types.go
  20. +18 −0 vendor/github.com/BurntSushi/toml/encoding_types_1.1.go
  21. +858 −0 vendor/github.com/BurntSushi/toml/lex.go
  22. +557 −0 vendor/github.com/BurntSushi/toml/parse.go
  23. +1 −0 vendor/github.com/BurntSushi/toml/session.vim
  24. +91 −0 vendor/github.com/BurntSushi/toml/type_check.go
  25. +242 −0 vendor/github.com/BurntSushi/toml/type_fields.go
  26. +20 −0 vendor/github.com/cenkalti/backoff/LICENSE
  27. +30 −0 vendor/github.com/cenkalti/backoff/README.md
  28. +66 −0 vendor/github.com/cenkalti/backoff/backoff.go
  29. +156 −0 vendor/github.com/cenkalti/backoff/exponential.go
  30. +46 −0 vendor/github.com/cenkalti/backoff/retry.go
  31. +79 −0 vendor/github.com/cenkalti/backoff/ticker.go
  32. +21 −0 vendor/github.com/dghubble/go-twitter/LICENSE
  33. +270 −0 vendor/github.com/dghubble/go-twitter/README.md
  34. +23 −0 vendor/github.com/dghubble/go-twitter/test
  35. +37 −0 vendor/github.com/dghubble/go-twitter/twitter/accounts.go
  36. +25 −0 vendor/github.com/dghubble/go-twitter/twitter/backoffs.go
  37. +88 −0 vendor/github.com/dghubble/go-twitter/twitter/demux.go
  38. +130 −0 vendor/github.com/dghubble/go-twitter/twitter/direct_messages.go
  39. +70 −0 vendor/github.com/dghubble/go-twitter/twitter/doc.go
  40. +104 −0 vendor/github.com/dghubble/go-twitter/twitter/entities.go
  41. +47 −0 vendor/github.com/dghubble/go-twitter/twitter/errors.go
  42. +71 −0 vendor/github.com/dghubble/go-twitter/twitter/favorites.go
  43. +73 −0 vendor/github.com/dghubble/go-twitter/twitter/followers.go
  44. +73 −0 vendor/github.com/dghubble/go-twitter/twitter/friends.go
  45. +55 −0 vendor/github.com/dghubble/go-twitter/twitter/friendships.go
  46. +63 −0 vendor/github.com/dghubble/go-twitter/twitter/search.go
  47. +264 −0 vendor/github.com/dghubble/go-twitter/twitter/statuses.go
  48. +110 −0 vendor/github.com/dghubble/go-twitter/twitter/stream_messages.go
  49. +56 −0 vendor/github.com/dghubble/go-twitter/twitter/stream_utils.go
  50. +326 −0 vendor/github.com/dghubble/go-twitter/twitter/streams.go
  51. +105 −0 vendor/github.com/dghubble/go-twitter/twitter/timelines.go
  52. +59 −0 vendor/github.com/dghubble/go-twitter/twitter/twitter.go
  53. +122 −0 vendor/github.com/dghubble/go-twitter/twitter/users.go
  54. +30 −0 vendor/github.com/dghubble/oauth1/CHANGES.md
  55. +21 −0 vendor/github.com/dghubble/oauth1/LICENSE
  56. +125 −0 vendor/github.com/dghubble/oauth1/README.md
  57. +265 −0 vendor/github.com/dghubble/oauth1/auther.go
  58. +172 −0 vendor/github.com/dghubble/oauth1/config.go
  59. +24 −0 vendor/github.com/dghubble/oauth1/context.go
  60. +97 −0 vendor/github.com/dghubble/oauth1/doc.go
  61. +36 −0 vendor/github.com/dghubble/oauth1/encode.go
  62. +12 −0 vendor/github.com/dghubble/oauth1/endpoint.go
  63. +62 −0 vendor/github.com/dghubble/oauth1/signer.go
  64. +23 −0 vendor/github.com/dghubble/oauth1/test
  65. +43 −0 vendor/github.com/dghubble/oauth1/token.go
  66. +65 −0 vendor/github.com/dghubble/oauth1/transport.go
  67. +52 −0 vendor/github.com/dghubble/sling/CHANGES.md
  68. +21 −0 vendor/github.com/dghubble/sling/LICENSE
  69. +273 −0 vendor/github.com/dghubble/sling/README.md
  70. +68 −0 vendor/github.com/dghubble/sling/body.go
  71. +179 −0 vendor/github.com/dghubble/sling/doc.go
  72. +383 −0 vendor/github.com/dghubble/sling/sling.go
  73. +23 −0 vendor/github.com/dghubble/sling/test
  74. +122 −110 vendor/github.com/dutchcoders/marija-web/bindata_gen.go
  75. +19 −0 vendor/github.com/gernest/mention/LICENCE
  76. +101 −0 vendor/github.com/gernest/mention/README.md
  77. +97 −0 vendor/github.com/gernest/mention/mention.go
  78. +31 −0 vendor/github.com/golang/protobuf/LICENSE
  79. +43 −0 vendor/github.com/golang/protobuf/proto/Makefile
  80. +229 −0 vendor/github.com/golang/protobuf/proto/clone.go
  81. +970 −0 vendor/github.com/golang/protobuf/proto/decode.go
  82. +1,362 −0 vendor/github.com/golang/protobuf/proto/encode.go
  83. +300 −0 vendor/github.com/golang/protobuf/proto/equal.go
  84. +587 −0 vendor/github.com/golang/protobuf/proto/extensions.go
  85. +898 −0 vendor/github.com/golang/protobuf/proto/lib.go
  86. +311 −0 vendor/github.com/golang/protobuf/proto/message_set.go
  87. +484 −0 vendor/github.com/golang/protobuf/proto/pointer_reflect.go
  88. +270 −0 vendor/github.com/golang/protobuf/proto/pointer_unsafe.go
  89. +872 −0 vendor/github.com/golang/protobuf/proto/properties.go
  90. +854 −0 vendor/github.com/golang/protobuf/proto/text.go
  91. +895 −0 vendor/github.com/golang/protobuf/proto/text_parser.go
  92. +27 −0 vendor/github.com/google/go-querystring/LICENSE
  93. +320 −0 vendor/github.com/google/go-querystring/query/encode.go
  94. +202 −0 vendor/github.com/qedus/blockchain/LICENSE
  95. +67 −0 vendor/github.com/qedus/blockchain/README.md
  96. +67 −0 vendor/github.com/qedus/blockchain/address.go
  97. +75 −0 vendor/github.com/qedus/blockchain/block.go
  98. +86 −0 vendor/github.com/qedus/blockchain/blockchain.go
  99. +140 −0 vendor/github.com/qedus/blockchain/transaction.go
  100. +168 −0 vendor/github.com/qedus/blockchain/wallet.go
  101. +3 −0 vendor/golang.org/x/oauth2/AUTHORS
  102. +31 −0 vendor/golang.org/x/oauth2/CONTRIBUTING.md
  103. +3 −0 vendor/golang.org/x/oauth2/CONTRIBUTORS
  104. +27 −0 vendor/golang.org/x/oauth2/LICENSE
  105. +65 −0 vendor/golang.org/x/oauth2/README.md
  106. +25 −0 vendor/golang.org/x/oauth2/client_appengine.go
  107. +76 −0 vendor/golang.org/x/oauth2/internal/oauth2.go
  108. +225 −0 vendor/golang.org/x/oauth2/internal/token.go
  109. +69 −0 vendor/golang.org/x/oauth2/internal/transport.go
  110. +341 −0 vendor/golang.org/x/oauth2/oauth2.go
  111. +158 −0 vendor/golang.org/x/oauth2/token.go
  112. +132 −0 vendor/golang.org/x/oauth2/transport.go
  113. +202 −0 vendor/google.golang.org/appengine/LICENSE
  114. +675 −0 vendor/google.golang.org/appengine/internal/api.go
  115. +159 −0 vendor/google.golang.org/appengine/internal/api_classic.go
  116. +116 −0 vendor/google.golang.org/appengine/internal/api_common.go
  117. +28 −0 vendor/google.golang.org/appengine/internal/app_id.go
  118. +133 −0 vendor/google.golang.org/appengine/internal/base/api_base.pb.go
  119. +33 −0 vendor/google.golang.org/appengine/internal/base/api_base.proto
  120. +2,778 −0 vendor/google.golang.org/appengine/internal/datastore/datastore_v3.pb.go
  121. +541 −0 vendor/google.golang.org/appengine/internal/datastore/datastore_v3.proto
  122. +14 −0 vendor/google.golang.org/appengine/internal/identity.go
  123. +27 −0 vendor/google.golang.org/appengine/internal/identity_classic.go
  124. +97 −0 vendor/google.golang.org/appengine/internal/identity_vm.go
  125. +110 −0 vendor/google.golang.org/appengine/internal/internal.go
  126. +899 −0 vendor/google.golang.org/appengine/internal/log/log_service.pb.go
  127. +150 −0 vendor/google.golang.org/appengine/internal/log/log_service.proto
  128. +15 −0 vendor/google.golang.org/appengine/internal/main.go
  129. +44 −0 vendor/google.golang.org/appengine/internal/main_vm.go
  130. +61 −0 vendor/google.golang.org/appengine/internal/metadata.go
  131. +56 −0 vendor/google.golang.org/appengine/internal/net.go
  132. +40 −0 vendor/google.golang.org/appengine/internal/regen.sh
  133. +231 −0 vendor/google.golang.org/appengine/internal/remote_api/remote_api.pb.go
  134. +44 −0 vendor/google.golang.org/appengine/internal/remote_api/remote_api.proto
  135. +107 −0 vendor/google.golang.org/appengine/internal/transaction.go
  136. +355 −0 vendor/google.golang.org/appengine/internal/urlfetch/urlfetch_service.pb.go
  137. +64 −0 vendor/google.golang.org/appengine/internal/urlfetch/urlfetch_service.proto
  138. +210 −0 vendor/google.golang.org/appengine/urlfetch/urlfetch.go
  139. +129 −3 vendor/vendor.json
View
@@ -44,14 +44,12 @@ $ brew tap dutchcoders/homebrew-marija
$ brew install marija
```
## Usage
There are a few steps you need to take before you can start.
* add your elasticsearch server to the configuration
* use the cloud icon to retrieve the indexes
* enable the index(es) you want to search in using the eye icon
* add your datasources to config.toml
* enable the datasources you want to search in using the eye icon
* use the refresh icon to refresh the list of available fields
* add the fields you want to use as nodes
@@ -60,6 +58,31 @@ There are a few steps you need to take before you can start.
You're all setup now, just type your queries and start exploring your data.
## Configuration
```
[datasource]
[datasource.elasticsearch]
type="elasticsearch"
url="http://127.0.0.1:9200/demo_index"
#username=
#password=
[datasource.twitter]
type="twitter"
consumer_key=""
consumer_secret=""
token=""
token_secret=""
[datasource.blockchain]
type="blockchain"
[[logging]]
output = "stdout"
level = "debug"
```
## Features
* work on multiple servers and indexes at the same time
View
@@ -10,11 +10,6 @@ import (
)
var Version = "0.1"
var format = logging.MustStringFormatter(
"%{color}%{time:15:04:05.000} %{shortfunc} ▶ %{level:.4s} %{id:03x}%{color:reset} %{message}",
)
var helpTemplate = `NAME:
{{.Name}} - {{.Usage}}
@@ -89,6 +84,7 @@ func New() *Cmd {
srvr := server.New(
server.Address(c.String("port")),
server.Path(c.String("path")),
server.Config(c.String("config")),
)
srvr.Run()
View
@@ -0,0 +1,21 @@
[datasource]
[datasource.elasticsearch]
type="elasticsearch"
url="http://127.0.0.1:9200/demo_index"
#username=
#password=
[datasource.twitter]
type="twitter"
consumer_key=""
consumer_secret=""
token=""
token_secret=""
[datasource.blockchain]
type="blockchain"
[[logging]]
output = "stdout"
level = "debug"
View
@@ -0,0 +1,79 @@
package server
import (
"io"
_ "log"
"os"
"fmt"
"github.com/BurntSushi/toml"
_ "github.com/dutchcoders/marija/server/datasources/blockchain"
_ "github.com/dutchcoders/marija/server/datasources/es5"
_ "github.com/dutchcoders/marija/server/datasources/twitter"
"github.com/op/go-logging"
)
func Debug2() func(*Server) {
return func(server *Server) {
server.debug = true
}
}
func Config(val string) func(*Server) {
return func(server *Server) {
if _, err := toml.DecodeFile(val, &server); err != nil {
panic(err)
}
fmt.Printf("Config: %#v\n", server.config)
logBackends := []logging.Backend{}
for _, log := range server.Logging {
var err error
var output io.Writer = os.Stdout
switch log.Output {
case "stdout":
output = os.Stdout
case "stderr":
output = os.Stderr
default:
output, err = os.OpenFile(os.ExpandEnv(log.Output), os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0660)
}
if err != nil {
panic(err)
}
backend := logging.NewLogBackend(output, "", 0)
backendFormatter := logging.NewBackendFormatter(backend, format)
backendLeveled := logging.AddModuleLevel(backendFormatter)
level, err := logging.LogLevel(log.Level)
if err != nil {
panic(err)
}
backendLeveled.SetLevel(level, "")
logBackends = append(logBackends, backendLeveled)
}
logging.SetBackend(logBackends...)
}
}
func Path(val string) func(*Server) {
return func(server *Server) {
server.path = val
}
}
func Address(addr string) func(*Server) {
return func(server *Server) {
server.address = addr
}
}
Oops, something went wrong.

0 comments on commit 6fefd04

Please sign in to comment.