141
file:
path: /etc/logstash/aws_opsgenie/
state: directory
owner: logstash
group: logstash
mode: 0775
- name: Creates directory aws_nginx
file:
path: /etc/logstash/aws_nginx/
state: directory
owner: logstash
group: logstash
mode: 0775
- name: Create Logstash filebeat pipeline configuration files.
template:
src: "{{ item }}.j2"
dest: "/etc/logstash/aws_filebeat/{{ item }}"
owner: logstash
group: logstash
mode: 0644
with_items:
- 01-beats-input.conf
- 11-nginx.conf
- 31-elasticsearch-filebeat-output.conf
notify: restart logstash
- name: Create Logstash filebeat pipeline configuration files.
template:
src: "{{ item }}.j2"
dest: "/etc/logstash/aws_logstash/{{ item }}"
owner: logstash
group: logstash
mode: 0644
with_items:
- 01-cloudwatch-logs-input.conf
- 30-elasticsearch-logstash-output.conf
notify: restart logstash
- name: Create CloudWatch Nginx pipeline configuration files.
template:
src: "{{ item }}.j2"
dest: "/etc/logstash/aws_nginx/{{ item }}"
owner: logstash
group: logstash
mode: 0644
with_items:
- 04-cloudwatch-nginx-logs-input.conf
- 34-elasticsearch-logstash-nginx-output.conf
- 14-nginx-logstash.conf
notify: restart logstash
- name: Create Logstash filebeat pipeline configuration files.
142
template:
src: "{{ item }}.j2"
dest: "/etc/logstash/aws_opsgenie/{{ item }}"
owner: logstash
group: logstash
mode: 0644
with_items:
- 01-cloudwatch-logs-input.conf
- 02-opsgenie-output.conf
- 03-opsgenie-filter.conf
notify: restart logstash
- name: Create Logstash pipeline and configuration files.
template:
src: "{{ item }}.j2"
dest: "/etc/logstash/{{ item }}"
owner: logstash
group: logstash
mode: 0644
with_items:
- logstash.yml
- pipelines.yml
notify: restart logstash
Сбор логов из CloudWatch
input {
cloudwatch_logs {
start_position => "end"
log_group => ["{{ logstash.log_group_nginx }}"]
access_key_id => "{{ logstash.aws_access_key_id }}"
secret_access_key => "{{ logstash.aws_secret_access_key }}"
log_group_prefix => "true"
region => "{{ logstash.region }}"
}
}
Темплейт обработки логов Nginx
filter {
if [event][module] == "nginx" {
if [fileset][name] == "access" {
grok {
match => { "message" => ["%{IPORHOST:remote_ip} - %{DATA:user_name}
\[%{HTTPDATE:access_time}\] \"%{DATA:referrer}\" \"%{WORD:http_method} %{DATA:url}
HTTP/%{NUMBER:http_version}\" %{NUMBER:response_code} %{NUMBER:body_sent_bytes}
\"%{DATA:agent}\" \"-\" \[%{BASE10NUM:request_time}
%{BASE10NUM:upstream_response_time} %{GREEDYDATA:pipe}\] \[%{NUMBER:connection}
%{NUMBER:connection_requests}"] }
remove_field => "message"
}
mutate {
143
add_field => { "read_timestamp" => "%{@timestamp}" }
}
date {
match => [ "[nginx][access][time]", "dd/MMM/YYYY:H:m:s Z" ]
remove_field => "[nginx][access][time]"
}
useragent {
source => "[nginx][access][agent]"
target => "[nginx][access][user_agent]"
remove_field => "[nginx][access][agent]"
}
geoip {
source => "[nginx][access][remote_ip]"
target => "[nginx][access][geoip]"
}
}
else if [fileset][name] == "error" {
grok {
match => { "message" => ["%{DATA:[nginx][error][time]}
\[%{DATA:[nginx][error][level]}\]
%{NUMBER:[nginx][error][pid]}#%{NUMBER:[nginx][error][tid]}:
(\*%{NUMBER:[nginx][error][connection_id]} )?%{GREEDYDATA:[nginx][error][message]}"] }
remove_field => "message"
}
mutate {
rename => { "@timestamp" => "read_timestamp" }
}
date {
match => [ "[nginx][error][time]", "YYYY/MM/dd H:m:s" ]
remove_field => "[nginx][error][time]"
}
}
}
}
Фильтрация и обработка логов для отправки в OpsGenie
filter {
mutate{
add_field => {
"alias" => "%{[cloudwatch_logs][event_id]}"
"opsgenieAction" => "create"
"description" => "%{stack_trace}"
"tags" => ["{{ logstash.profile }}", "%{[cloudwatch_logs][ingestion_time]}",
"%{[cloudwatch_logs][event_id]}"]
"[details][organizationId]"=> "%{organizationId}"
"[details][channelId]" => "%{channelId}"
"[details][messageId]" => "%{messageId}"
"[details][flowId]" => "%{flowId}"
"[details][externalPropertyId]" => "%{externalPropertyId}"
"[details][repositoryId]" => "%{repositoryId}"
"[details][propertyCode]" => "%{propertyCode}"
"[details][chainCode]" => "%{chainCode}"
"[details][outboundChannelId]" => "%{outboundChannelId}"
144
"[details][recordType]" => "%{recordType}"
"[details][cloudWatchTime]" => "%{[cloudwatch_logs][ingestion_time]}"
"[details][logger_name]" => "%{[logger_name]}"
"priority" => "{{ logstash.priority }}"
"source" => "{{ logstash.profile }}"
"entity" => "%{app}"
}
}
ruby {
code => "event.set('teams', [{'name' => '{{ logstash.team }}'}])"
}
}
Отправка логов в систему алертинга OpsGenie:
output {
if [level] == 'ERROR' {
opsgenie {
apiKey => '{{ logstash.token }}'
}
}
}
Отправка логов в базу данных ElasticSearch
output {
if [level] in ["DEBUG","TRACE"] {
amazon_es {
hosts => ["{{ logstash.output_hosts }}"]
region => "{{ logstash.region }}"
aws_access_key_id => "{{ logstash.elastic_key_id | default(logstash.aws_access_key_id) }}"
aws_secret_access_key => "{{ logstash.elastic_secret_access_key |
default(logstash.aws_secret_access_key) }}"
index => "logstash-debug-%{+YYYY.MM.dd}"
}
} else {
amazon_es {
hosts => ["{{ logstash.output_hosts }}"]
region => "{{ logstash.region }}"
aws_access_key_id => "{{ logstash.elastic_key_id | default(logstash.aws_access_key_id) }}"
aws_secret_access_key => "{{ logstash.elastic_secret_access_key |
default(logstash.aws_secret_access_key) }}"
index => "logstash-info-%{+YYYY.MM.dd}"
}
}
}
Пайплайны обработки данных внутри Logstash
- pipeline.id: aws_logstash
path.config: "/etc/logstash/aws_logstash/*.conf"
145
- pipeline.id: aws_filebeat
path.config: "/etc/logstash/aws_filebeat/*.conf"
- pipeline.id: aws_opsgenie
path.config: "/etc/logstash/aws_opsgenie/*.conf"
- pipeline.id: aws_nginx
path.config: "/etc/logstash/aws_nginx/*.conf"
Установка Curator для автоматического удаления логов по
расписанию
---
- include: setup-RedHat.yml
when: ansible_os_family == 'RedHat'
- include: setup-Debian.yml
when: ansible_os_family == 'Debian'
- name: Install Elasticsearch Curator and required dependencies.
pip:
name:
- elasticsearch-curator
- argparse
- name: Creates directory aws_opsgenie
file:
path: /var/log/curator_logs/
state: directory
owner: logstash
group: logstash
mode: 0775
- name: Configure cron jobs for Elasticsearch Curator.
cron:
name: "{{ item.name }}"
job: "{{ item.job }}"
minute: "{{ item.minute | default('*') }}"
hour: "{{ item.hour | default('*') }}"
day: "{{ item.day | default('*') }}"
weekday: "{{ item.weekday | default('*') }}"
month: "{{ item.month | default('*') }}"
with_items: "{{ elasticsearch_curator_cron_jobs }}"
- name: Create directory curator
file:
path: /etc/curator/
state: directory
owner: logstash
group: logstash
mode: 0775
- name: Creates Logs directory curator
file:
path: /var/log/curator_logs/
state: directory
146
owner: logstash
group: logstash
mode: 0775
- name: Create Curator configuration files.
template:
src: "{{ item }}.j2"
dest: "/etc/curator/{{ item }}"
owner: logstash
group: logstash
mode: 0644
with_items:
- curator_cluster_config.yml
- delete30DaysOldDebugIndices.yml
- delete180DaysOldInfoIndices.yml
ignore_errors: yes
Настройки Curator
---
# Remember, leave a key empty if there is no value. None will be a string,
# not a Python "NoneType"
client:
hosts:
- {{ logstash.output_hosts }}
port: 443
url_prefix:
use_ssl: False
certificate:
client_cert:
client_key:
aws_key: "{{ logstash.elastic_key_id | default(logstash.aws_access_key_id) }}"
aws_secret_key: "{{ logstash.elastic_secret_access_key |
default(logstash.aws_secret_access_key) }}"
aws_region: {{ logstash.region }}
ssl_no_validate: False
http_auth:
timeout: 30
master_only: False
logging:
loglevel: INFO
logfile:
logformat: default
blacklist: ['elasticsearch', 'urllib3']
Удаление индексов старше 180 дней
actions:
1:
action: delete_indices
description: >-
"Delete info indices older than 180 days (based on index name), for (logstash-info)."
options:
timeout_override: 300
147
continue_if_exception: True
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: regex
value: '^(logstash-info|cwl-).*$'
exclude:
- filtertype: age
source: name
direction: older
timestring: '%Y.%m.%d'
unit: days
unit_count: {{ logstash.unit_count_info }}
Запуск системы мониторинга с помощью Docker Compose
version: '3.7'
volumes:
prometheus_data: {}
grafana_data: {}
networks:
monitor-net:
driver: overlay
name: inbound
traefik:
driver: overlay
name: traefik
services:
traefik:
image: traefik:v1.7.12-alpine
command:
- "--logLevel=DEBUG"
- "--api"
- "--metrics"
- "--metrics.prometheus.buckets=0.1,0.3,1.2,5.0"
- "--docker"
- "--docker.swarmMode"
- "--docker.domain=docker.localhost"
- "--docker.watch"
networks:
- traefik
- monitor-net
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- ./hapicloud_cert/:/etc/certs/
ports:
- target: 80
published: 80
mode: host
- target: 443
published: 443
mode: host
- target: 8080
published: 8080
protocol: tcp
mode: ingress
deploy:
148
mode: global
placement:
constraints:
- node.role == manager
update_config:
parallelism: 1
delay: 10s
restart_policy:
condition: on-failure
prometheus:
image: prom/prometheus
volumes:
- ./prometheus/:/etc/prometheus/
- prometheus_data:/prometheus
command:
- '--config.file=/etc/prometheus/prometheus.yml'
- '--storage.tsdb.path=/prometheus'
- '--web.console.libraries=/usr/share/prometheus/console_libraries'
- '--web.console.templates=/usr/share/prometheus/consoles'
networks:
- monitor-net
deploy:
placement:
constraints:
- node.role==manager
labels:
- "traefik.frontend.rule=Host:prometheus.hapicloud.io"
- "traefik.backend=prometheus"
- "traefik.port=9090"
- "traefik.docker.network=traefik"
restart_policy:
condition: on-failure
node-exporter:
image: prom/node-exporter
volumes:
- /proc:/host/proc:ro
- /sys:/host/sys:ro
- /:/rootfs:ro
command:
- '--path.procfs=/host/proc'
- '--path.sysfs=/host/sys'
- --collector.filesystem.ignored-mount-points
-
"^/(sys|proc|dev|host|etc|rootfs/var/lib/docker/containers|rootfs/var/lib/docker
/overlay2|rootfs/run/docker/netns|rootfs/var/lib/docker/aufs)($$|/)"
ports:
- 9100:9100
networks:
- monitor-net
deploy:
mode: global
restart_policy:
condition: on-failure
alertmanager:
image: prom/alertmanager
ports:
- 9093:9093
volumes:
- "./alertmanager/:/etc/alertmanager/"
networks:
- monitor-net
149
command:
- '--config.file=/etc/alertmanager/config.yml'
- '--storage.path=/alertmanager'
deploy:
placement:
constraints:
- node.role==manager
labels:
- "traefik.frontend.rule=Host:alertmanager.hapicloud.io"
- "traefik.backend=alertmanager"
- "traefik.port=9093"
- "traefik.docker.network=traefik"
restart_policy:
condition: on-failure
cadvisor:
image: google/cadvisor
volumes:
- /:/rootfs:ro
- /var/run:/var/run:rw
- /sys:/sys:ro
- /var/lib/docker/:/var/lib/docker:ro
ports:
- 8088:8088
networks:
- monitor-net
deploy:
mode: global
restart_policy:
condition: on-failure
labels:
- "traefik.frontend.rule=Host:cadvisor.hapicloud.io"
- "traefik.backend=cadvisor"
- "traefik.port=8088"
- "traefik.docker.network=traefik"
grafana:
image: grafana/grafana
logging:
driver: journald
options:
tag: grafana
depends_on:
- prometheus
volumes:
- grafana_data:/var/lib/grafana
- ./grafana/provisioning/:/etc/grafana/provisioning/
- ./grafana/notifiers:/etc/grafana/provisioning/notifiers/
env_file:
- ./grafana/config.monitoring
networks:
- monitor-net
user: "104"
deploy:
placement:
constraints:
- node.role==manager
labels:
- "traefik.frontend.rule=Host:grafana.hapicloud.io"
- "traefik.backend=grafana"
- "traefik.port=3000"
- "traefik.docker.network=traefik"
restart_policy:
condition: on-failure
~