diff --git a/active/container_elk/.env b/active/container_elk/.env new file mode 100644 index 0000000..19507aa --- /dev/null +++ b/active/container_elk/.env @@ -0,0 +1,41 @@ +# Project namespace (defaults to the current folder name if not set) +#COMPOSE_PROJECT_NAME=myproject + + +# Password for the 'elastic' user (at least 6 characters) +ELASTIC_PASSWORD=changeme + + +# Password for the 'kibana_system' user (at least 6 characters) +KIBANA_PASSWORD=changeme + + +# Version of Elastic products +STACK_VERSION=8.7.1 + + +# Set the cluster name +CLUSTER_NAME=docker-cluster + + +# Set to 'basic' or 'trial' to automatically start the 30-day trial +LICENSE=basic +#LICENSE=trial + + +# Port to expose Elasticsearch HTTP API to the host +ES_PORT=9200 + + +# Port to expose Kibana to the host +KIBANA_PORT=5601 + + +# Increase or decrease based on the available host memory (in bytes) +ES_MEM_LIMIT=1073741824 +KB_MEM_LIMIT=1073741824 +LS_MEM_LIMIT=1073741824 + + +# SAMPLE Predefined Key only to be used in POC environments +ENCRYPTION_KEY=c34d38b3a14956121ff2170e5030b471551370178f43e5626eec58b04a30fae2 \ No newline at end of file diff --git a/active/container_elk/elk-compose.yaml b/active/container_elk/elk-compose.yaml new file mode 100644 index 0000000..974a554 --- /dev/null +++ b/active/container_elk/elk-compose.yaml @@ -0,0 +1,219 @@ +version: "3.8" + + +volumes: + certs: + driver: local + esdata01: + driver: local + kibanadata: + driver: local + metricbeatdata01: + driver: local + filebeatdata01: + driver: local + logstashdata01: + driver: local + + +networks: + default: + name: elastic + external: false + + +services: + setup: + image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION} + volumes: + - certs:/usr/share/elasticsearch/config/certs + user: "0" + command: > + bash -c ' + if [ x${ELASTIC_PASSWORD} == x ]; then + echo "Set the ELASTIC_PASSWORD environment variable in the .env file"; + exit 1; + elif [ x${KIBANA_PASSWORD} == x ]; then + echo "Set the KIBANA_PASSWORD environment variable in the .env file"; + exit 1; + fi; + if [ ! -f config/certs/ca.zip ]; then + echo "Creating CA"; + bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip; + unzip config/certs/ca.zip -d config/certs; + fi; + if [ ! -f config/certs/certs.zip ]; then + echo "Creating certs"; + echo -ne \ + "instances:\n"\ + " - name: es01\n"\ + " dns:\n"\ + " - es01\n"\ + " - localhost\n"\ + " ip:\n"\ + " - 127.0.0.1\n"\ + " - name: kibana\n"\ + " dns:\n"\ + " - kibana\n"\ + " - localhost\n"\ + " ip:\n"\ + " - 127.0.0.1\n"\ + > config/certs/instances.yml; + bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key; + unzip config/certs/certs.zip -d config/certs; + fi; + echo "Setting file permissions" + chown -R root:root config/certs; + find . -type d -exec chmod 750 \{\} \;; + find . -type f -exec chmod 640 \{\} \;; + echo "Waiting for Elasticsearch availability"; + until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done; + echo "Setting kibana_system password"; + until curl -s -X POST --cacert config/certs/ca/ca.crt -u "elastic:${ELASTIC_PASSWORD}" -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done; + echo "All done!"; + ' + healthcheck: + test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"] + interval: 1s + timeout: 5s + retries: 120 + + es01: + depends_on: + setup: + condition: service_healthy + image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION} + labels: + co.elastic.logs/module: elasticsearch + volumes: + - certs:/usr/share/elasticsearch/config/certs + - esdata01:/usr/share/elasticsearch/data + ports: + - ${ES_PORT}:9200 + environment: + - node.name=es01 + - cluster.name=${CLUSTER_NAME} + - discovery.type=single-node + - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} + - bootstrap.memory_lock=true + - xpack.security.enabled=true + - xpack.security.http.ssl.enabled=true + - xpack.security.http.ssl.key=certs/es01/es01.key + - xpack.security.http.ssl.certificate=certs/es01/es01.crt + - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt + - xpack.security.transport.ssl.enabled=true + - xpack.security.transport.ssl.key=certs/es01/es01.key + - xpack.security.transport.ssl.certificate=certs/es01/es01.crt + - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt + - xpack.security.transport.ssl.verification_mode=certificate + - xpack.license.self_generated.type=${LICENSE} + mem_limit: ${ES_MEM_LIMIT} + ulimits: + memlock: + soft: -1 + hard: -1 + healthcheck: + test: + [ + "CMD-SHELL", + "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'", + ] + interval: 10s + timeout: 10s + retries: 120 + + kibana: + depends_on: + es01: + condition: service_healthy + image: docker.elastic.co/kibana/kibana:${STACK_VERSION} + labels: + co.elastic.logs/module: kibana + volumes: + - certs:/usr/share/kibana/config/certs + - kibanadata:/usr/share/kibana/data + ports: + - ${KIBANA_PORT}:5601 + environment: + - SERVERNAME=kibana + - ELASTICSEARCH_HOSTS=https://es01:9200 + - ELASTICSEARCH_USERNAME=kibana_system + - ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD} + - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt + - XPACK_SECURITY_ENCRYPTIONKEY=${ENCRYPTION_KEY} + - XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY=${ENCRYPTION_KEY} + - XPACK_REPORTING_ENCRYPTIONKEY=${ENCRYPTION_KEY} + mem_limit: ${KB_MEM_LIMIT} + healthcheck: + test: + [ + "CMD-SHELL", + "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'", + ] + interval: 10s + timeout: 10s + retries: 120 + + metricbeat01: + depends_on: + es01: + condition: service_healthy + kibana: + condition: service_healthy + image: docker.elastic.co/beats/metricbeat:${STACK_VERSION} + user: root + volumes: + - certs:/usr/share/metricbeat/certs + - metricbeatdata01:/usr/share/metricbeat/data + - "./metricbeat.yaml:/usr/share/metricbeat/metricbeat.yml:ro" + - "/var/run/docker.sock:/var/run/docker.sock:ro" + - "/sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro" + - "/proc:/hostfs/proc:ro" + - "/:/hostfs:ro" + environment: + - ELASTIC_USER=elastic + - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} + - ELASTIC_HOSTS=https://es01:9200 + - KIBANA_HOSTS=http://kibana:5601 + - LOGSTASH_HOSTS=http://logstash01:9600 + + filebeat01: + depends_on: + es01: + condition: service_healthy + image: docker.elastic.co/beats/filebeat:${STACK_VERSION} + user: root + volumes: + - certs:/usr/share/filebeat/certs + - filebeatdata01:/usr/share/filebeat/data + - "./filebeat_ingest_data/:/usr/share/filebeat/ingest_data/" + - "./filebeat.yaml:/usr/share/filebeat/filebeat.yml:ro" + - "/var/lib/docker/containers:/var/lib/docker/containers:ro" + - "/var/run/docker.sock:/var/run/docker.sock:ro" + environment: + - ELASTIC_USER=elastic + - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} + - ELASTIC_HOSTS=https://es01:9200 + - KIBANA_HOSTS=http://kibana:5601 + - LOGSTASH_HOSTS=http://logstash01:9600 + + logstash01: + depends_on: + es01: + condition: service_healthy + kibana: + condition: service_healthy + image: docker.elastic.co/logstash/logstash:${STACK_VERSION} + labels: + co.elastic.logs/module: logstash + user: root + volumes: + - certs:/usr/share/logstash/certs + - logstashdata01:/usr/share/logstash/data + - "./logstash_ingest_data/:/usr/share/logstash/ingest_data/" + - "./logstash.conf:/usr/share/logstash/pipeline/logstash.conf:ro" + environment: + - xpack.monitoring.enabled=false + - ELASTIC_USER=elastic + - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} + - ELASTIC_HOSTS=https://es01:9200 \ No newline at end of file diff --git a/active/container_elk/elk.md b/active/container_elk/elk.md new file mode 100644 index 0000000..5059948 --- /dev/null +++ b/active/container_elk/elk.md @@ -0,0 +1,14 @@ +# Elk Stack + +## Install + + + +```bash +# Copy over the files +scp -rp active/container_elk/. elk:elk +# SSH into the host +ssh -t elk "cd elk ; bash --login" +# Run the services +docker compose -f elk-compose.yaml up +``` \ No newline at end of file diff --git a/active/container_elk/filebeat.yaml b/active/container_elk/filebeat.yaml new file mode 100644 index 0000000..21cfedb --- /dev/null +++ b/active/container_elk/filebeat.yaml @@ -0,0 +1,29 @@ +filebeat.inputs: +- type: filestream + id: default-filestream + paths: + - ingest_data/*.log + + +filebeat.autodiscover: + providers: + - type: docker + hints.enabled: true + + +processors: + - add_docker_metadata: ~ + + +setup.kibana: + host: ${KIBANA_HOSTS} + username: ${ELASTIC_USER} + password: ${ELASTIC_PASSWORD} + + +output.elasticsearch: + hosts: ${ELASTIC_HOSTS} + username: ${ELASTIC_USER} + password: ${ELASTIC_PASSWORD} + ssl.enabled: true + ssl.certificate_authorities: "certs/ca/ca.crt" \ No newline at end of file diff --git a/active/container_elk/logstash.conf b/active/container_elk/logstash.conf new file mode 100644 index 0000000..06619f2 --- /dev/null +++ b/active/container_elk/logstash.conf @@ -0,0 +1,24 @@ +input { + file { + #https://www.elastic.co/guide/en/logstash/current/plugins-inputs-file.html + #default is TAIL which assumes more data will come into the file. + #change to mode => "read" if the file is a compelte file. by default, the file will be removed once reading is complete -- backup your files if you need them. + mode => "tail" + path => "/usr/share/logstash/ingest_data/*" + } +} + + +filter { +} + + +output { + elasticsearch { + index => "logstash-%{+YYYY.MM.dd}" + hosts=> "${ELASTIC_HOSTS}" + user=> "${ELASTIC_USER}" + password=> "${ELASTIC_PASSWORD}" + cacert=> "certs/ca/ca.crt" + } +} \ No newline at end of file diff --git a/active/container_elk/metricbeat.yaml b/active/container_elk/metricbeat.yaml new file mode 100644 index 0000000..605e7bb --- /dev/null +++ b/active/container_elk/metricbeat.yaml @@ -0,0 +1,62 @@ +metricbeat.config.modules: + path: ${path.config}/modules.d/*.yml + reload.enabled: false + + +metricbeat.modules: +- module: elasticsearch + xpack.enabled: true + period: 10s + hosts: ${ELASTIC_HOSTS} + ssl.certificate_authorities: "certs/ca/ca.crt" + ssl.certificate: "certs/es01/es01.crt" + ssl.key: "certs/es01/es01.key" + username: ${ELASTIC_USER} + password: ${ELASTIC_PASSWORD} + ssl.enabled: true + + +- module: logstash + xpack.enabled: true + period: 10s + hosts: ${LOGSTASH_HOSTS} + + +- module: kibana + metricsets: + - stats + period: 10s + hosts: ${KIBANA_HOSTS} + username: ${ELASTIC_USER} + password: ${ELASTIC_PASSWORD} + xpack.enabled: true + + +- module: docker + metricsets: + - "container" + - "cpu" + - "diskio" + - "healthcheck" + - "info" + #- "image" + - "memory" + - "network" + hosts: ["unix:///var/run/docker.sock"] + period: 10s + enabled: true + + +processors: + - add_host_metadata: ~ + - add_docker_metadata: ~ + + +output.elasticsearch: + hosts: ${ELASTIC_HOSTS} + username: ${ELASTIC_USER} + password: ${ELASTIC_PASSWORD} + ssl: + certificate: "certs/es01/es01.crt" + certificate_authorities: "certs/ca/ca.crt" + key: "certs/es01/es01.key" \ No newline at end of file