logstash+kibana+rabbitmq+elasticsearch

Install logstash rabbitmq-server elasticsearch from repository.
Download kibana from its site and unpack to your webroot folder.
I'll add nginx logs to logstash.

Nginx

set ngixn logs format in nginx.conf:

1
2
log_format logstash '$http_host ' '$remote_addr [$time_local] ' '"$request" $status $body_bytes_sent ' '"$http_referer" "$http_user_agent" ' '$request_time ' '$upstream_response_time ' '$upstream_addr';
access_log /var/log/nginx/access.log logstash;

Logstash

add in /etc/logstash/conf.d 2 files,
logstash-nginx:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
input {
tcp {
type => "nginx_agupcom"
data_timeout => 10
mode => "server"
host => "127.0.0.1"
port => 14001
}
}

filter {
if [type] == "nginx_agupcom" {
grok {
match => [
"message", "%{IPORHOST:http_host} %{IPORHOST:clientip} \[%{HTTPDATE:timestamp}\] \"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})\" %{NUMBER:response} (?:%{NUMBER:bytes}|-) %{QS:referrer} %{QS:agent} %{NUMBER:request_time:float} %{NUMBER:upstream_time:float} %{HOSTPORT:upstream_addr}",
"message", "%{IPORHOST:http_host} %{IPORHOST:clientip} \[%{HTTPDATE:timestamp}\] \"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})\" %{NUMBER:response} (?:%{NUMBER:bytes}|-) %{QS:referrer} %{QS:agent} %{NUMBER:request_time:float}"
]
}
date {
match => [ "timestamp" , "dd/MMM/YYYY:HH:mm:ss Z" ]
}
geoip {
source => "clientip"
}
}
}

output {
rabbitmq {
exchange => "logstash_agupcom"
exchange_type => direct
host => "127.0.0.1" # string (required)
key => "logstash_agupcom"
workers => 2
durable => true
persistent => true
}
}

nginx_agupcom — name of my log type
logstash_agupcom name of queue in rabbitmq.

elastic-nginx:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
input {
rabbitmq {
host => "127.0.0.1"
queue => "logstash_agupcom"
durable => true
key => "logstash_agupcom"
exchange => "logstash_agupcom"
threads => 2
prefetch_count => 50
}
}

output {
elasticsearch {
index => "logstash-%{+YYYY.MM.dd}-%{type}"
protocol => "transport"
cluster => "elastic"
workers => "2"
host => "127.0.0.1"
}
}

Elasticsearch

egrep -v '^#|^$' /etc/elasticsearch/elasticsearch.yml
cluster.name: elastic
node.name: «localhost»
node.master: true
node.data: true

Run applications

service ngixn reload
service elasticsearch restart
service rabbitmq-server restart
service logstash restart

Kibana

Kibana nginx config:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
server {
listen 80 ;

server_name kibana.yourhostname.here;
access_log /var/log/nginx/access.log logstash;

allow 127.0.0.1;
allow 172.16.0.0/24;
deny all;
add_header X-Frame-Options DENY;
add_header X-Content-Type-Options nosniff;
add_header X-XSS-Protection "1; mode=block;";
add_header X-Content-Security-Policy "allow 'self';";
add_header X-WebKit-CSP "allow 'self';";

proxy_cache off;

auth_basic "Unauthorized";
auth_basic_user_file /etc/nginx/htpasswd; # path to htpasswd with login:pass for current kibana instance, just because kibana doesn't have own authorization

if ( $http_user_agent ~* (nmap|nikto|wikto|sf|sqlmap|bsqlbf|w3af|acunetix|havij|appscan) ) {
return 403;
}

if ( $http_referer ~* (babes|forsale|girl|jewelry|love|nudit|organic|poker|porn|sex|teen) ) {
return 403;
}

location / {
root /var/www/kibana;
index index.html index.htm;
}
location ~ ^/_aliases$ {
proxy_pass http://127.0.0.1:9200;
proxy_read_timeout 90;
}
location ~ ^/.*/_aliases$ {
proxy_pass http://127.0.0.1:9200;
proxy_read_timeout 90;
}
location ~ ^/_nodes$ {
proxy_pass http://127.0.0.1:9200;
proxy_read_timeout 90;
}
location ~ ^/.*/_search$ {
proxy_pass http://127.0.0.1:9200;
proxy_read_timeout 90;
}
location ~ ^/.*/_mapping {
proxy_pass http://127.0.0.1:9200;
proxy_read_timeout 90;
}

# Password protected end points
location ~ ^/kibana-int/dashboard/.*$ {
proxy_pass http://127.0.0.1:9200;
proxy_read_timeout 90;
limit_except GET {
proxy_pass http://127.0.0.1:9200;
}
}
location ~ ^/kibana-int/temp.*$ {
proxy_pass http://127.0.0.1:9200;
proxy_read_timeout 90;
limit_except GET {
proxy_pass http://127.0.0.1:9200;
}
}
}

fix kibana config.js:
replace elasticsearch: «http://»+window.location.hostname+":9200",
with elasticsearch: «http://kibana.yourhostname.here:80»,
because your kibana behind nginx.