Skip to content

Commit 87b66db

Browse files
valyalavalyala
committed
app/victoria-logs: initial code release
1 parent aeac39c commit 87b66db

82 files changed

Lines changed: 31486 additions & 1 deletion

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎Makefile‎

Lines changed: 61 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ include package/release/Makefile
2121

2222
all: \
2323
victoria-metrics-prod \
24+
victoria-logs-prod \
2425
vmagent-prod \
2526
vmalert-prod \
2627
vmauth-prod \
@@ -33,6 +34,7 @@ clean:
3334

3435
publish: docker-scan \
3536
publish-victoria-metrics \
37+
publish-victoria-logs \
3638
publish-vmagent \
3739
publish-vmalert \
3840
publish-vmauth \
@@ -42,6 +44,7 @@ publish: docker-scan \
4244

4345
package: \
4446
package-victoria-metrics \
47+
package-victoria-logs \
4548
package-vmagent \
4649
package-vmalert \
4750
package-vmauth \
@@ -178,6 +181,7 @@ publish-release:
178181

179182
release: \
180183
release-victoria-metrics \
184+
release-victoria-logs \
181185
release-vmutils
182186

183187
release-victoria-metrics: \
@@ -191,7 +195,6 @@ release-victoria-metrics: \
191195
release-victoria-metrics-openbsd-amd64 \
192196
release-victoria-metrics-windows-amd64
193197

194-
# adds i386 arch
195198
release-victoria-metrics-linux-386:
196199
GOOS=linux GOARCH=386 $(MAKE) release-victoria-metrics-goos-goarch
197200

@@ -238,6 +241,63 @@ release-victoria-metrics-windows-goarch: victoria-metrics-windows-$(GOARCH)-prod
238241
cd bin && rm -rf \
239242
victoria-metrics-windows-$(GOARCH)-prod.exe
240243

244+
release-victoria-logs: \
245+
release-victoria-logs-linux-386 \
246+
release-victoria-logs-linux-amd64 \
247+
release-victoria-logs-linux-arm \
248+
release-victoria-logs-linux-arm64 \
249+
release-victoria-logs-darwin-amd64 \
250+
release-victoria-logs-darwin-arm64 \
251+
release-victoria-logs-freebsd-amd64 \
252+
release-victoria-logs-openbsd-amd64 \
253+
release-victoria-logs-windows-amd64
254+
255+
release-victoria-logs-linux-386:
256+
GOOS=linux GOARCH=386 $(MAKE) release-victoria-logs-goos-goarch
257+
258+
release-victoria-logs-linux-amd64:
259+
GOOS=linux GOARCH=amd64 $(MAKE) release-victoria-logs-goos-goarch
260+
261+
release-victoria-logs-linux-arm:
262+
GOOS=linux GOARCH=arm $(MAKE) release-victoria-logs-goos-goarch
263+
264+
release-victoria-logs-linux-arm64:
265+
GOOS=linux GOARCH=arm64 $(MAKE) release-victoria-logs-goos-goarch
266+
267+
release-victoria-logs-darwin-amd64:
268+
GOOS=darwin GOARCH=amd64 $(MAKE) release-victoria-logs-goos-goarch
269+
270+
release-victoria-logs-darwin-arm64:
271+
GOOS=darwin GOARCH=arm64 $(MAKE) release-victoria-logs-goos-goarch
272+
273+
release-victoria-logs-freebsd-amd64:
274+
GOOS=freebsd GOARCH=amd64 $(MAKE) release-victoria-logs-goos-goarch
275+
276+
release-victoria-logs-openbsd-amd64:
277+
GOOS=openbsd GOARCH=amd64 $(MAKE) release-victoria-logs-goos-goarch
278+
279+
release-victoria-logs-windows-amd64:
280+
GOARCH=amd64 $(MAKE) release-victoria-logs-windows-goarch
281+
282+
release-victoria-logs-goos-goarch: victoria-logs-$(GOOS)-$(GOARCH)-prod
283+
cd bin && \
284+
tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-logs-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
285+
victoria-logs-$(GOOS)-$(GOARCH)-prod \
286+
&& sha256sum victoria-logs-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
287+
victoria-logs-$(GOOS)-$(GOARCH)-prod \
288+
| sed s/-$(GOOS)-$(GOARCH)-prod/-prod/ > victoria-logs-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
289+
cd bin && rm -rf victoria-logs-$(GOOS)-$(GOARCH)-prod
290+
291+
release-victoria-logs-windows-goarch: victoria-logs-windows-$(GOARCH)-prod
292+
cd bin && \
293+
zip victoria-logs-windows-$(GOARCH)-$(PKG_TAG).zip \
294+
victoria-logs-windows-$(GOARCH)-prod.exe \
295+
&& sha256sum victoria-logs-windows-$(GOARCH)-$(PKG_TAG).zip \
296+
victoria-logs-windows-$(GOARCH)-prod.exe \
297+
> victoria-logs-windows-$(GOARCH)-$(PKG_TAG)_checksums.txt
298+
cd bin && rm -rf \
299+
victoria-logs-windows-$(GOARCH)-prod.exe
300+
241301
release-vmutils: \
242302
release-vmutils-linux-386 \
243303
release-vmutils-linux-amd64 \

‎app/victoria-logs/Makefile‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
# All these commands must run from repository root.
2+
3+
victoria-logs:
4+
APP_NAME=victoria-logs $(MAKE) app-local
5+
6+
victoria-logs-race:
7+
APP_NAME=victoria-logs RACE=-race $(MAKE) app-local
8+
9+
victoria-logs-prod:
10+
APP_NAME=victoria-logs $(MAKE) app-via-docker
11+
12+
victoria-logs-pure-prod:
13+
APP_NAME=victoria-logs $(MAKE) app-via-docker-pure
14+
15+
victoria-logs-linux-amd64-prod:
16+
APP_NAME=victoria-logs $(MAKE) app-via-docker-linux-amd64
17+
18+
victoria-logs-linux-arm-prod:
19+
APP_NAME=victoria-logs $(MAKE) app-via-docker-linux-arm
20+
21+
victoria-logs-linux-arm64-prod:
22+
APP_NAME=victoria-logs $(MAKE) app-via-docker-linux-arm64
23+
24+
victoria-logs-linux-ppc64le-prod:
25+
APP_NAME=victoria-logs $(MAKE) app-via-docker-linux-ppc64le
26+
27+
victoria-logs-linux-386-prod:
28+
APP_NAME=victoria-logs $(MAKE) app-via-docker-linux-386
29+
30+
victoria-logs-darwin-amd64-prod:
31+
APP_NAME=victoria-logs $(MAKE) app-via-docker-darwin-amd64
32+
33+
victoria-logs-darwin-arm64-prod:
34+
APP_NAME=victoria-logs $(MAKE) app-via-docker-darwin-arm64
35+
36+
victoria-logs-freebsd-amd64-prod:
37+
APP_NAME=victoria-logs $(MAKE) app-via-docker-freebsd-amd64
38+
39+
victoria-logs-openbsd-amd64-prod:
40+
APP_NAME=victoria-logs $(MAKE) app-via-docker-openbsd-amd64
41+
42+
victoria-logs-windows-amd64-prod:
43+
APP_NAME=victoria-logs $(MAKE) app-via-docker-windows-amd64
44+
45+
package-victoria-logs:
46+
APP_NAME=victoria-logs $(MAKE) package-via-docker
47+
48+
package-victoria-logs-pure:
49+
APP_NAME=victoria-logs $(MAKE) package-via-docker-pure
50+
51+
package-victoria-logs-amd64:
52+
APP_NAME=victoria-logs $(MAKE) package-via-docker-amd64
53+
54+
package-victoria-logs-arm:
55+
APP_NAME=victoria-logs $(MAKE) package-via-docker-arm
56+
57+
package-victoria-logs-arm64:
58+
APP_NAME=victoria-logs $(MAKE) package-via-docker-arm64
59+
60+
package-victoria-logs-ppc64le:
61+
APP_NAME=victoria-logs $(MAKE) package-via-docker-ppc64le
62+
63+
package-victoria-logs-386:
64+
APP_NAME=victoria-logs $(MAKE) package-via-docker-386
65+
66+
publish-victoria-logs:
67+
APP_NAME=victoria-logs $(MAKE) publish-via-docker
68+
69+
victoria-logs-linux-amd64:
70+
APP_NAME=victoria-logs CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(MAKE) app-local-goos-goarch
71+
72+
victoria-logs-linux-arm:
73+
APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=arm $(MAKE) app-local-goos-goarch
74+
75+
victoria-logs-linux-arm64:
76+
APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(MAKE) app-local-goos-goarch
77+
78+
victoria-logs-linux-ppc64le:
79+
APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
80+
81+
victoria-logs-linux-s390x:
82+
APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
83+
84+
victoria-logs-linux-386:
85+
APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
86+
87+
victoria-logs-darwin-amd64:
88+
APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 $(MAKE) app-local-goos-goarch
89+
90+
victoria-logs-darwin-arm64:
91+
APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 $(MAKE) app-local-goos-goarch
92+
93+
victoria-logs-freebsd-amd64:
94+
APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
95+
96+
victoria-logs-openbsd-amd64:
97+
APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
98+
99+
victoria-logs-windows-amd64:
100+
GOARCH=amd64 APP_NAME=victoria-logs $(MAKE) app-local-windows-goarch
101+
102+
victoria-logs-pure:
103+
APP_NAME=victoria-logs $(MAKE) app-local-pure

‎app/victoria-logs/deployment/Dockerfile‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
ARG base_image
2+
FROM $base_image
3+
4+
EXPOSE 8428
5+
6+
ENTRYPOINT ["/victoria-logs-prod"]
7+
ARG src_binary
8+
COPY $src_binary ./victoria-logs-prod

‎app/victoria-logs/main.go‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
package main
2+
3+
import (
4+
"flag"
5+
"fmt"
6+
"net/http"
7+
"os"
8+
"time"
9+
10+
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert"
11+
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlselect"
12+
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
13+
"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
14+
"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
15+
"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
16+
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
17+
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
18+
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
19+
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
20+
"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
21+
"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
22+
)
23+
24+
var (
25+
httpListenAddr = flag.String("httpListenAddr", ":9428", "TCP address to listen for http connections. See also -httpListenAddr.useProxyProtocol")
26+
useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
27+
"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
28+
"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
29+
gogc = flag.Int("gogc", 100, "GOGC to use. See https://tip.golang.org/doc/gc-guide")
30+
)
31+
32+
func main() {
33+
// Write flags and help message to stdout, since it is easier to grep or pipe.
34+
flag.CommandLine.SetOutput(os.Stdout)
35+
flag.Usage = usage
36+
envflag.Parse()
37+
cgroup.SetGOGC(*gogc)
38+
buildinfo.Init()
39+
logger.Init()
40+
pushmetrics.Init()
41+
42+
logger.Infof("starting VictoriaLogs at %q...", *httpListenAddr)
43+
startTime := time.Now()
44+
45+
vlstorage.Init()
46+
vlselect.Init()
47+
vlinsert.Init()
48+
49+
go httpserver.Serve(*httpListenAddr, *useProxyProtocol, requestHandler)
50+
logger.Infof("started VictoriaLogs in %.3f seconds; see https://docs.victoriametrics.com/VictoriaLogs/", time.Since(startTime).Seconds())
51+
52+
sig := procutil.WaitForSigterm()
53+
logger.Infof("received signal %s", sig)
54+
55+
logger.Infof("gracefully shutting down webservice at %q", *httpListenAddr)
56+
startTime = time.Now()
57+
if err := httpserver.Stop(*httpListenAddr); err != nil {
58+
logger.Fatalf("cannot stop the webservice: %s", err)
59+
}
60+
logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
61+
62+
vlinsert.Stop()
63+
vlselect.Stop()
64+
vlstorage.Stop()
65+
66+
fs.MustStopDirRemover()
67+
68+
logger.Infof("the VictoriaLogs has been stopped in %.3f seconds", time.Since(startTime).Seconds())
69+
}
70+
71+
func requestHandler(w http.ResponseWriter, r *http.Request) bool {
72+
if r.URL.Path == "/" {
73+
if r.Method != http.MethodGet {
74+
return false
75+
}
76+
w.Header().Add("Content-Type", "text/html; charset=utf-8")
77+
fmt.Fprintf(w, "<h2>Single-node VictoriaLogs</h2></br>")
78+
fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/VictoriaLogs/'>https://docs.victoriametrics.com/VictoriaLogs/</a></br>")
79+
fmt.Fprintf(w, "Useful endpoints:</br>")
80+
httpserver.WriteAPIHelp(w, [][2]string{
81+
{"metrics", "available service metrics"},
82+
{"flags", "command-line flags"},
83+
})
84+
return true
85+
}
86+
if vlinsert.RequestHandler(w, r) {
87+
return true
88+
}
89+
if vlselect.RequestHandler(w, r) {
90+
return true
91+
}
92+
return false
93+
}
94+
95+
func usage() {
96+
const s = `
97+
victoria-logs is a log management and analytics service.
98+
99+
See the docs at https://docs.victoriametrics.com/VictoriaLogs/
100+
`
101+
flagutil.Usage(s)
102+
}

‎app/victoria-logs/multiarch/Dockerfile‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
# See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
2+
ARG certs_image
3+
ARG root_image
4+
FROM $certs_image as certs
5+
RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
6+
7+
FROM $root_image
8+
COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
9+
EXPOSE 8428
10+
ENTRYPOINT ["/victoria-logs-prod"]
11+
ARG TARGETARCH
12+
COPY victoria-logs-linux-${TARGETARCH}-prod ./victoria-logs-prod

‎app/vlinsert/elasticsearch/bulk_response.qtpl‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
{% stripspace %}
2+
3+
{% func BulkResponse(n int, tookMs int64) %}
4+
{
5+
"took":{%dl tookMs %},
6+
"errors":false,
7+
"items":[
8+
{% for i := 0; i < n; i++ %}
9+
{
10+
"create":{
11+
"status":201
12+
}
13+
}
14+
{% if i+1 < n %},{% endif %}
15+
{% endfor %}
16+
]
17+
}
18+
{% endfunc %}
19+
20+
{% endstripspace %}

0 commit comments

Comments
 (0)