Merge branch 'master' into master

.gitignore

@@ -1,3 +1,4 @@
+/.idea
 /log
 /tmp
 # dotenv

.travis.yml

@@ -1,5 +1,7 @@
 language: generic
 
+if: tag IS present
+
 install:
   # Installing helm
   - wget -q https://storage.googleapis.com/kubernetes-helm/helm-v2.6.2-linux-amd64.tar.gz
@@ -20,7 +22,7 @@ script:
   - helm repo index charts/
   - git add charts/
   - git commit --message "Add new version ${TRAVIS_TAG}"
-  - git remote add origin-pages https://${GITHUB_TOKEN}@github.com/flant/loghouse.git
+  - git remote add origin-pages https://${GITHUB_TOKEN}@github.com/${TRAVIS_REPO_SLUG}.git
   - git push --quiet --set-upstream origin-pages gh-pages
 after_success:
   - true

Gemfile

@@ -5,11 +5,13 @@ ruby '2.3.4'
 gem 'sinatra'
 gem 'sinatra-contrib'
 gem 'puma'
-gem 'clickhouse', github: 'archan937/clickhouse'
+gem 'clickhouse', git: 'https://github.com/archan937/clickhouse'
 gem 'parslet'
 gem 'activesupport'
 gem 'json'
+gem 'chronic'
 
 gem 'kubeclient'
 
 gem 'pry', group: :development
+gem 'dotenv', require: 'dotenv/load', group: :development

Gemfile.lock

@@ -1,5 +1,5 @@
 GIT
-  remote: git://github.com/archan937/clickhouse.git
+  remote: https://github.com/archan937/clickhouse
   revision: 6b00d0459d0633e0c7b755166b0e4e0dab044076
   specs:
     clickhouse (0.1.10)
@@ -15,20 +15,22 @@ GIT
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (5.1.4)
+    activesupport (5.1.5)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (~> 0.7)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
     addressable (2.5.2)
       public_suffix (>= 2.0.2, < 4.0)
-    backports (3.10.3)
+    backports (3.11.1)
+    chronic (0.10.2)
     coderay (1.1.1)
     concurrent-ruby (1.0.5)
     domain_name (0.5.20161021)
       unf (>= 0.0.5, < 1.0.0)
+    dotenv (2.2.1)
     erubis (2.7.0)
-    faraday (0.13.1)
+    faraday (0.14.0)
       multipart-post (>= 1.2, < 3)
     http (0.9.8)
       addressable (~> 2.3)
@@ -39,7 +41,8 @@ GEM
       domain_name (~> 0.5)
     http-form_data (1.0.1)
     http_parser.rb (0.6.0)
-    i18n (0.8.6)
+    i18n (0.9.5)
+      concurrent-ruby (~> 1.0)
     json (2.1.0)
     kubeclient (2.2.0)
       http (= 0.9.8)
@@ -51,44 +54,44 @@ GEM
     mime-types (3.1)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2016.0521)
-    minitest (5.10.3)
-    multi_json (1.12.2)
+    minitest (5.11.3)
+    multi_json (1.13.1)
     multipart-post (2.0.0)
-    mustermann (1.0.1)
+    mustermann (1.0.2)
     netrc (0.11.0)
     parslet (1.8.0)
     pond (0.2.0)
     pry (0.10.4)
       coderay (~> 1.1.0)
       method_source (~> 0.8.1)
       slop (~> 3.4)
-    public_suffix (3.0.0)
+    public_suffix (3.0.2)
     puma (3.10.0)
-    rack (2.0.3)
-    rack-protection (2.0.0)
+    rack (2.0.4)
+    rack-protection (2.0.1)
       rack
     recursive-open-struct (1.0.0)
     rest-client (2.0.0)
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 4.0)
       netrc (~> 0.8)
-    sinatra (2.0.0)
+    sinatra (2.0.1)
       mustermann (~> 1.0)
       rack (~> 2.0)
-      rack-protection (= 2.0.0)
+      rack-protection (= 2.0.1)
       tilt (~> 2.0)
-    sinatra-contrib (2.0.0)
+    sinatra-contrib (2.0.1)
       backports (>= 2.0)
       multi_json
       mustermann (~> 1.0)
-      rack-protection (= 2.0.0)
-      sinatra (= 2.0.0)
+      rack-protection (= 2.0.1)
+      sinatra (= 2.0.1)
       tilt (>= 1.3, < 3)
     slop (3.6.0)
     thor (0.20.0)
     thread_safe (0.3.6)
     tilt (2.0.8)
-    tzinfo (1.2.3)
+    tzinfo (1.2.5)
       thread_safe (~> 0.1)
     unf (0.1.4)
       unf_ext
@@ -99,7 +102,9 @@ PLATFORMS
 
 DEPENDENCIES
   activesupport
+  chronic
   clickhouse!
+  dotenv
   json
   kubeclient
   parslet
@@ -112,4 +117,4 @@ RUBY VERSION
    ruby 2.3.4p301
 
 BUNDLED WITH
-   1.15.4
+   1.16.0

README.md

@@ -1,7 +1,17 @@
+<p align="center">
+  <img src="https://cdn.rawgit.com/flant/loghouse/master/docs/logo.png" style="max-height:100%;" height="300">
+</p>
+
+___
+
+
 Ready to use log management solution for Kubernetes. Efficiently store big amounts of your logs (in [ClickHouse](https://github.com/yandex/ClickHouse) database), process them using a simple query language and monitor them online through web UI. Easy and quick to deploy in an already functioning Kubernetes cluster.
 
 Status is **alpha**. However we (Flant) use it in our production Kubernetes deployments since September, 2017.
 
+Some data may be dropped in alpha's updates. Be careful, when updating! All info will be published in release notes. 
+**Data's structure will be stable in beta version** (planned on April 2018).
+
 Loghouse-dashboard UI demo in action (~3 Mb):
 
 ![loghouse web UI](https://cdn.rawgit.com/flant/loghouse/master/docs/web-ui-animated.gif)
@@ -16,7 +26,7 @@ Loghouse-dashboard UI demo in action (~3 Mb):
 * Quickly & straightforward deployable to Kubernetes via Dockerfiles and Helm chart.
 * Web UI made cosy and powerful:
   * Papertrail-like user experience.
-  * Customizable time frames: from date to date / from now till given period (last hour, last day, etc).
+  * Customizable time frames: from date to date / from now till given period (last hour, last day, etc) / seek to specific time and show logs around it.
   * Infinite scrolling of older log entries.
   * Save your queries to use in future.
   * Basic permissions (limiting entries shown for users by specifying Kubernetes namespaces).
@@ -49,6 +59,8 @@ To install loghouse, you need to have [Helm](https://github.com/kubernetes/helm)
 
 Web UI (loghouse-dashboard) will be reachable via address specified in values.yaml config as ```loghouse_host```. You'll be prompted by basic authorization generated via htpasswd and configured in ```auth``` parameter of your values.yaml.
 
+> To clean old logs in cron, you can use a script in this [issue](https://github.com/flant/loghouse/issues/42).
+
 # Architecture
 
 ![loghouse architecture](https://cdn.rawgit.com/flant/loghouse/master/docs/architecture.png)

application.rb

@@ -3,6 +3,8 @@
 module Loghouse
   TIME_ZONE = ENV.fetch('TIME_ZONE') { 'Europe/Moscow' }
 
+  class UnauthenticatedError < StandardError; end
+
   # rubocop:disable Metrics/ClassLength
   class Application < Sinatra::Base
     configure do
@@ -13,9 +15,10 @@ class Application < Sinatra::Base
     end
 
     before do
-      Time.zone    = TIME_ZONE
-      User.current = self.class.development? ? 'admin' : user_from_header
-      @tab_queries = LoghouseQuery.all.first(10)
+      Time.zone          = TIME_ZONE
+      Chronic.time_class = Time.zone
+      User.current       = self.class.development? ? 'admin' : user_from_header
+      @tab_queries       = LoghouseQuery.all.first(10)
 
       if request.path_info.match(/.csv$/)
         request.accept.unshift('text/csv')
@@ -36,7 +39,7 @@ class Application < Sinatra::Base
                 end
 
       begin
-        @query.validate_query!
+        @query.validate!(name: false)
       rescue LoghouseQuery::BadFormat => e
         @error = "Bad query format: #{e}"
       rescue LoghouseQuery::BadTimeFormat => e
@@ -136,6 +139,27 @@ class Application < Sinatra::Base
       ''
     end
 
+    error UnauthenticatedError do |e|
+      @status = 401
+      @message = "Not Authenticated"
+
+      render_error
+    end
+
+    error User::PermissionsNotFound do |e|
+      @status = 403
+      @message = e.message
+
+      render_error
+    end
+
+    error do |e|
+      @status = 500
+      @message = 'Internal Server Error'
+
+      render_error
+    end
+
     helpers do
       def h(text)
         Rack::Utils.escape_html(text)
@@ -153,16 +177,23 @@ def version
     private
 
     def query_from_params
-      LoghouseQuery.new(name: params[:name], query: params[:query].to_s, namespaces: params[:namespaces],
-                        time_from: params[:time_from], time_to: params[:time_to])
+      LoghouseQuery.new(name: params[:name], query: params[:query].to_s, namespaces: params[:namespaces])
+                   .time_params(format: params[:time_format], from: params[:time_from], to: params[:time_to],
+                                seek_to: params[:seek_to])
+
     end
 
     def user_from_header
       auth_header = env['HTTP_AUTHORIZATION']
 
-      raise 'Unauthenticated' if auth_header.blank?
+      raise UnauthenticatedError if auth_header.blank?
 
       Base64.decode64(auth_header.gsub(/Basic /, '')).split(':').first
     end
+
+    def render_error
+      status @status
+      erb :error, layout: false
+    end
   end
 end

charts/loghouse/Chart.yaml

@@ -1,5 +1,5 @@
 name: loghouse
-version: 0.0.1
+version: 0.2.2
 description: Loghouse (Fluentd, Clickhouse, Tabix, Loghouse) for collect and prepare
              logs for Kubernetes cluster.
 keywords:

charts/loghouse/templates/_helpers.tpl

@@ -0,0 +1,10 @@
+{{/*
+CronJob api version
+*/}}
+{{- define "CronJob.apiVersion" -}}
+{{- if semverCompare ">=1.8" .Capabilities.KubeVersion.GitVersion -}}
+"batch/v1beta1"
+{{- else -}}
+"batch/v2alpha1"
+{{- end -}}
+{{- end -}}

charts/loghouse/templates/clickhouse/clickhouse-configmap.yaml

@@ -60,7 +60,7 @@ data:
         </profiles>
         <users>
             <default>
-                <password_sha256_hex>{{ .Values.clickhouse_pass_sha256 }}</password_sha256_hex>
+                <password_sha256_hex>{{ .Values.clickhouse_pass_original | sha256sum }}</password_sha256_hex>
                 <networks incl="networks" replace="replace">
                     <ip>::/0</ip>
                 </networks>

charts/loghouse/templates/clickhouse/clickhouse.yaml

@@ -28,7 +28,7 @@ spec:
           mountPath: /var/lib/clickhouse
       containers:
       - name: clickhouse
-        image: flant/loghouse-clickhouse:{{ default .Chart.Version .Values.version }}
+        image: flant/loghouse-clickhouse:{{ .Chart.Version }}
         imagePullPolicy: {{ .Values.imagePullPolicy }}
         ports:
         - name: http

charts/loghouse/templates/fluentd/fluentd-configmap.yaml

@@ -88,7 +88,7 @@ data:
         # static fields
         source                 "kubernetes"
         namespace              ${record["kubernetes"]["namespace_name"]}
-        host                   ${record["kubernetes"]["host"]}
+        host                   ${record["kubernetes"]["host"] || ENV["K8S_NODE_NAME"]}
         pod_name               ${record["kubernetes"]["pod_name"]}
         container_name         ${record["kubernetes"]["container_name"]}
         stream                 ${record["stream"]}

charts/loghouse/templates/fluentd/fluentd.yaml

@@ -19,7 +19,7 @@ spec:
       serviceAccountName: fluentd
       containers:
       - name: fluentd
-        image: flant/loghouse-fluentd:{{ default .Chart.Version .Values.version }}
+        image: flant/loghouse-fluentd:{{ .Chart.Version }}
         imagePullPolicy: {{ .Values.imagePullPolicy }}
         env:
         - name: FLUENTD_ARGS
@@ -36,6 +36,10 @@ spec:
           value: {{ .Values.clickhouse.db | quote }}
         - name: K8S_LOGS_TABLE
           value: {{ .Values.clickhouse.table | quote }}
+        - name: K8S_NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
         resources:
           limits:
             memory: {{ .Values.fluentd.resources.limits.memory }}

charts/loghouse/templates/loghouse/loghouse-cronjob.yaml

@@ -1,4 +1,4 @@
-apiVersion: batch/v1beta1
+apiVersion: {{ template "CronJob.apiVersion" $ }}
 kind: CronJob
 metadata:
   name: {{ .Chart.Name }}-tables
@@ -16,10 +16,10 @@ spec:
           initContainers:
           - name: wait-clickhouse
             image: alpine:3.6
-            command: ['/bin/sh', '-c', 'while ! getent ahostsv4 clickhouse; do sleep 1; done']
+            command: [ '/bin/sh', '-c', 'while ! nc -z clickhouse 8123; do sleep 1; done' ]
           containers:
             - name: cron
-              image: flant/loghouse-dashboard:{{ default .Chart.Version .Values.version }}
+              image: flant/loghouse-dashboard:{{ .Chart.Version }}
               imagePullPolicy: {{ .Values.imagePullPolicy }}
               command: ['/bin/bash', '-l', '-c', 'rake create_logs_tables']
               env:

charts/loghouse/templates/loghouse/loghouse-ingress.yaml

@@ -5,6 +5,11 @@ metadata:
   name: loghouse
   namespace: {{ .Release.Namespace }}
   annotations:
+{{- if .Values.ingress.annotations }}
+{{- range .Values.ingress.annotations }}
+    {{ . }}
+{{- end }}
+{{- else }}
     kubernetes.io/ingress.class: "nginx"
 {{- if .Values.ingress.enable_https }}
     kubernetes.io/tls-acme: "true"
@@ -14,6 +19,7 @@ metadata:
     ingress.kubernetes.io/force-ssl-redirect: "false"
     ingress.kubernetes.io/ssl-redirect: "false"
 {{- end }}
+{{- end }}
 spec:
   rules:
   - host: {{ .Values.ingress.loghouse.host }}

charts/loghouse/templates/loghouse/loghouse-init-db.yaml

@@ -16,10 +16,10 @@ spec:
       initContainers:
       - name: wait-clickhouse
         image: alpine:3.6
-        command: [ '/bin/sh', '-c', 'while ! getent ahostsv4 clickhouse; do sleep 1; done' ]
+        command: [ '/bin/sh', '-c', 'while ! nc -z clickhouse 8123; do sleep 1; done' ]
       containers:
       - name: init
-        image: flant/loghouse-fluentd:{{ default .Chart.Version .Values.version }}
+        image: flant/loghouse-fluentd:{{ .Chart.Version }}
         imagePullPolicy: {{ .Values.imagePullPolicy }}
         command: ['/bin/bash', '-l', '-c', 'clickhouse-client --host=${CLICKHOUSE_URL} --port=9000 --user=${CLICKHOUSE_USERNAME} --password=${CLICKHOUSE_PASSWORD} --query="CREATE DATABASE ${CLICKHOUSE_DATABASE};"']
         env: