commit 0d91f1ca86274767b36ea21d80e49657ec52d37c Author: Joan López de la Franca Beltran Date: Tue Apr 12 16:51:30 2022 +0200 Contents for lecture 1-1 diff --git a/1-1-grafana-loki-vs-elk/docker-compose.yml b/1-1-grafana-loki-vs-elk/docker-compose.yml new file mode 100644 index 0000000..36ecb23 --- /dev/null +++ b/1-1-grafana-loki-vs-elk/docker-compose.yml @@ -0,0 +1,81 @@ +version: "3.9" + +services: + elasticsearch: + image: "docker.elastic.co/elasticsearch/elasticsearch:8.1.0" + ports: + - "9200:9200" + - "9300:9300" + environment: + - network.host=0.0.0.0 + - discovery.type=single-node + - bootstrap.memory_lock=true + - xpack.security.enabled=false + - ELASTIC_PASSWORD=elastic + # Used to avoid huge memory consumption by Elasticsearch. + # Otherwise, you need to extend the Docker limits + # to avoid continuous container crashes due to lack of memory. + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + ulimits: + memlock: + soft: -1 + hard: -1 + + logstash: + image: "logstash" + command: --log.level=debug --enable-local-plugin-development + ports: + - "9600:9600" + environment: + - ELASTIC_PASSWORD=elastic + volumes: + - ./logstash.conf:/usr/share/logstash/pipeline/logstash.conf + depends_on: + - elasticsearch + + filebeat: + image: "docker.elastic.co/beats/filebeat:8.1.0" + environment: + - ELASTIC_PASSWORD=elastic + volumes: + - ./filebeat.yml:/usr/share/filebeat/filebeat.yml + - logs_dir:/logs + depends_on: + - elasticsearch + + kibana: + image: "docker.elastic.co/kibana/kibana:8.1.0" + ports: + - "5601:5601" + environment: + - ELASTIC_PASSWORD=elastic + depends_on: + - elasticsearch + + grafana: + image: "grafana/grafana:8.4.2" + ports: + - "3000:3000" + volumes: + - ./provisioning:/etc/grafana/provisioning + + loki: + image: "grafana/loki:2.4.2" + ports: + - "3100:3100" + + promtail: + image: "grafana/promtail:2.4.2" + volumes: + - logs_dir:/tmp/log + - ./promtail.yml:/etc/promtail/config.yaml + command: -config.file=/etc/promtail/config.yaml + + flog: + image: "mingrammer/flog:0.4.3" + command: -f json -t log -o /tmp/log/json.log -d 200ms -w -l + volumes: + - logs_dir:/tmp/log + +volumes: + logs_dir: \ No newline at end of file diff --git a/1-1-grafana-loki-vs-elk/filebeat.yml b/1-1-grafana-loki-vs-elk/filebeat.yml new file mode 100644 index 0000000..a6a6655 --- /dev/null +++ b/1-1-grafana-loki-vs-elk/filebeat.yml @@ -0,0 +1,11 @@ +logging.level: info +logging.to_files: false + +filebeat.inputs: + - type: log + enabled: true + paths: + - '/logs/*.log' + +output.logstash: + hosts: ["logstash:5044"] \ No newline at end of file diff --git a/1-1-grafana-loki-vs-elk/logstash.conf b/1-1-grafana-loki-vs-elk/logstash.conf new file mode 100644 index 0000000..33e704e --- /dev/null +++ b/1-1-grafana-loki-vs-elk/logstash.conf @@ -0,0 +1,18 @@ +input { + beats { + port => "5044" + } +} + +filter { + json { + source => "message" + target => "fields" + } +} + +output { + elasticsearch { + hosts => [ "elasticsearch:9200" ] + } +} \ No newline at end of file diff --git a/1-1-grafana-loki-vs-elk/promtail.yml b/1-1-grafana-loki-vs-elk/promtail.yml new file mode 100644 index 0000000..b237340 --- /dev/null +++ b/1-1-grafana-loki-vs-elk/promtail.yml @@ -0,0 +1,11 @@ +clients: + - url: http://loki:3100/loki/api/v1/push + +scrape_configs: + - job_name: flog + static_configs: + - targets: + - localhost + labels: + job: flogs + __path__: /tmp/log/*.log \ No newline at end of file diff --git a/1-1-grafana-loki-vs-elk/provisioning/datasources/loki.yml b/1-1-grafana-loki-vs-elk/provisioning/datasources/loki.yml new file mode 100644 index 0000000..fd2fc24 --- /dev/null +++ b/1-1-grafana-loki-vs-elk/provisioning/datasources/loki.yml @@ -0,0 +1,8 @@ +apiVersion: 1 + +datasources: + - name: Loki + type: loki + url: http://loki:3100 + isDefault: true + editable: true