diff --git a/docker/docker-compose-elk-basic.yml b/docker/docker-compose-helk-elastic-basic.yml similarity index 67% rename from docker/docker-compose-elk-basic.yml rename to docker/docker-compose-helk-elastic-basic.yml index e45b643d..4b7eed30 100644 --- a/docker/docker-compose-elk-basic.yml +++ b/docker/docker-compose-helk-elastic-basic.yml @@ -1,16 +1,23 @@ -version: '3' +version: '3.5' services: helk-elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:6.3.2 + image: docker.elastic.co/elasticsearch/elasticsearch:6.4.0 container_name: helk-elasticsearch + secrets: + - source: elasticsearch.yml + target: /usr/share/elasticsearch/config/elasticsearch.yml volumes: - - ./helk-elasticsearch/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml - esdata:/usr/share/elasticsearch/data - ./helk-elasticsearch/scripts:/usr/share/elasticsearch/scripts entrypoint: /usr/share/elasticsearch/scripts/elasticsearch-entrypoint.sh environment: - - "ES_JAVA_OPTS=-Xms4g -Xmx4g" + - cluster.name=helk-cluster + - node.name=helk-1 + - bootstrap.memory_lock=true + - discovery.zen.minimum_master_nodes=1 + - discovery.type=single-node + - "ES_JAVA_OPTS=-Xms2g -Xmx2g" ulimits: memlock: soft: -1 @@ -18,13 +25,13 @@ services: restart: always networks: helk: - aliases: - - helk_elasticsearch.hunt.local helk-logstash: - image: docker.elastic.co/logstash/logstash:6.3.2 + image: docker.elastic.co/logstash/logstash:6.4.0 container_name: helk-logstash + secrets: + - source: logstash.yml + target: /usr/share/logstash/config/logstash.yml volumes: - - ./helk-logstash/logstash.yml:/usr/share/logstash/config/logstash.yml - ./helk-logstash/pipeline:/usr/share/logstash/pipeline - ./helk-logstash/output_templates:/usr/share/logstash/output_templates - ./helk-logstash/enrichments/cti:/usr/share/logstash/cti @@ -32,18 +39,20 @@ services: environment: - "LS_JAVA_OPTS=-Xms1g -Xmx1g" entrypoint: /usr/share/logstash/scripts/logstash-entrypoint.sh + ports: + - "5044:5044" restart: always depends_on: - - helk-elasticsearch + - helk-zookeeper networks: helk: - aliases: - - helk_logstash.hunt.local helk-kibana: - image: docker.elastic.co/kibana/kibana:6.3.2 + image: docker.elastic.co/kibana/kibana:6.4.0 container_name: helk-kibana + secrets: + - source: kibana.yml + target: /usr/share/kibana/config/kibana.yml volumes: - - ./helk-kibana/kibana.yml:/usr/share/kibana/config/kibana.yml - ./helk-kibana/dashboards:/usr/share/kibana/dashboards - ./helk-kibana/scripts:/usr/share/kibana/scripts entrypoint: /usr/share/kibana/scripts/kibana-entrypoint.sh @@ -52,13 +61,13 @@ services: - helk-elasticsearch networks: helk: - aliases: - - helk_kibana.hunt.local helk-nginx: - image: cyb3rward0g/helk-nginx:0.0.6 + image: cyb3rward0g/helk-nginx:0.0.7 container_name: helk-nginx + secrets: + - source: htpasswd.users + target: /etc/nginx/htpasswd.users volumes: - - ./helk-nginx/htpasswd.users:/etc/nginx/htpasswd.users - ./helk-nginx/default:/etc/nginx/sites-available/default - ./helk-nginx/scripts/:/opt/helk/scripts/ entrypoint: /opt/helk/scripts/nginx-entrypoint.sh @@ -68,22 +77,19 @@ services: restart: always depends_on: - helk-kibana + - helk-jupyter networks: helk: - aliases: - - helk_nginx.hunt.local helk-jupyter: - image: cyb3rward0g/helk-jupyter:0.0.4 + image: cyb3rward0g/helk-jupyter:0.0.5 container_name: helk-jupyter restart: always depends_on: - - helk-nginx + - helk-elasticsearch networks: helk: - aliases: - - helk_jupyter.hunt.local helk-spark-master: - image: cyb3rward0g/helk-spark-master:2.3.1-a + image: cyb3rward0g/helk-spark-master:2.3.1-b container_name: helk-spark-master environment: - SPARK_MASTER_PORT=7077 @@ -95,44 +101,34 @@ services: - helk-elasticsearch networks: helk: - aliases: - - helk_spark_master.hunt.local helk-spark-worker: - image: cyb3rward0g/helk-spark-worker:2.3.1-a + image: cyb3rward0g/helk-spark-worker:2.3.1-b container_name: helk-spark-worker environment: - SPARK_MASTER=spark://helk-spark-master:7077 - SPARK_WORKER_MEMORY=512m - SPARK_WORKER_WEBUI_PORT=8081 - SPARK_WORKER_PORT=42950 - ports: - - "8081:8081" restart: always depends_on: - helk-spark-master networks: helk: - aliases: - - helk_spark_worker.hunt.local helk-spark-worker2: - image: cyb3rward0g/helk-spark-worker:2.3.1-a + image: cyb3rward0g/helk-spark-worker:2.3.1-b container_name: helk-spark-worker2 environment: - SPARK_MASTER=spark://helk-spark-master:7077 - SPARK_WORKER_MEMORY=512m - SPARK_WORKER_WEBUI_PORT=8082 - SPARK_WORKER_PORT=42951 - ports: - - "8082:8082" restart: always depends_on: - helk-spark-master networks: helk: - aliases: - - helk_spark_worker2.hunt.local helk-zookeeper: - image: cyb3rward0g/helk-zookeeper:1.1.1 + image: cyb3rward0g/helk-zookeeper:2.0.0-a container_name: helk-zookeeper ports: - "2181:2181" @@ -141,10 +137,8 @@ services: - helk-kibana networks: helk: - aliases: - - helk_zookeeper.hunt.local helk-kafka-broker: - image: cyb3rward0g/helk-kafka-broker:1.1.1 + image: cyb3rward0g/helk-kafka-broker:2.0.0-b container_name: helk-kafka-broker restart: always depends_on: @@ -156,14 +150,13 @@ services: - REPLICATION_FACTOR=2 - ADVERTISED_LISTENER=HOSTIP - ZOOKEEPER_NAME=helk-zookeeper + - KAFKA_CREATE_TOPICS=True ports: - "9092:9092" networks: helk: - aliases: - - helk_kafka_broker.hunt.local helk-kafka-broker2: - image: cyb3rward0g/helk-kafka-broker:1.1.1 + image: cyb3rward0g/helk-kafka-broker:2.0.0-b container_name: helk-kafka-broker2 restart: always depends_on: @@ -175,25 +168,26 @@ services: - REPLICATION_FACTOR=2 - ADVERTISED_LISTENER=HOSTIP - ZOOKEEPER_NAME=helk-zookeeper + - KAFKA_CREATE_TOPICS=True ports: - "9093:9093" networks: helk: - aliases: - - helk_kafka_broker2.hunt.local - helk-sigma: - image: thomaspatzke/helk-sigma - container_name: helk-sigma - depends_on: - - helk-kibana - networks: - helk: - aliases: - - helk_sigma.hunt.local + networks: helk: driver: bridge volumes: esdata: - driver: local \ No newline at end of file + driver: local + +secrets: + elasticsearch.yml: + file: ./helk-elasticsearch/elasticsearch.yml + logstash.yml: + file: ./helk-logstash/logstash.yml + kibana.yml: + file: ./helk-kibana/kibana.yml + htpasswd.users: + file: ./helk-nginx/htpasswd.users \ No newline at end of file diff --git a/docker/docker-compose-elk-trial.yml b/docker/docker-compose-helk-elastic-trial.yml similarity index 68% rename from docker/docker-compose-elk-trial.yml rename to docker/docker-compose-helk-elastic-trial.yml index 10b8eba2..a78b3e07 100644 --- a/docker/docker-compose-elk-trial.yml +++ b/docker/docker-compose-helk-elastic-trial.yml @@ -1,16 +1,24 @@ -version: '3' +version: '3.5' services: helk-elasticsearch: - build: helk-elasticsearch/trial/ + image: docker.elastic.co/elasticsearch/elasticsearch:6.4.0 container_name: helk-elasticsearch + secrets: + - source: elasticsearch.yml + target: /usr/share/elasticsearch/config/elasticsearch.yml volumes: - - ./helk-elasticsearch/trial/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml - esdata:/usr/share/elasticsearch/data - ./helk-elasticsearch/scripts:/usr/share/elasticsearch/scripts entrypoint: /usr/share/elasticsearch/scripts/elasticsearch-entrypoint.sh environment: - - "ES_JAVA_OPTS=-Xms4g -Xmx4g" + - cluster.name=helk-cluster + - node.name=helk-1 + - bootstrap.memory_lock=true + - discovery.zen.minimum_master_nodes=1 + - discovery.type=single-node + - "ES_JAVA_OPTS=-Xms2g -Xmx2g" + - ELASTIC_PASSWORD=elasticpassword ulimits: memlock: soft: -1 @@ -18,13 +26,13 @@ services: restart: always networks: helk: - aliases: - - helk_elasticsearch.hunt.local helk-logstash: - image: docker.elastic.co/logstash/logstash:6.3.2 + image: docker.elastic.co/logstash/logstash:6.4.0 container_name: helk-logstash + secrets: + - source: logstash.yml + target: /usr/share/logstash/config/logstash.yml volumes: - - ./helk-logstash/trial/logstash.yml:/usr/share/logstash/config/logstash.yml - ./helk-logstash/trial/pipeline:/usr/share/logstash/pipeline - ./helk-logstash/output_templates:/usr/share/logstash/output_templates - ./helk-logstash/enrichments/cti:/usr/share/logstash/cti @@ -32,18 +40,20 @@ services: environment: - "LS_JAVA_OPTS=-Xms1g -Xmx1g" entrypoint: /usr/share/logstash/scripts/logstash-entrypoint.sh + ports: + - "5044:5044" restart: always depends_on: - - helk-kibana + - helk-zookeeper networks: helk: - aliases: - - helk_logstash.hunt.local helk-kibana: - image: docker.elastic.co/kibana/kibana:6.3.2 + image: docker.elastic.co/kibana/kibana:6.4.0 container_name: helk-kibana + secrets: + - source: kibana.yml + target: /usr/share/kibana/config/kibana.yml volumes: - - ./helk-kibana/trial/kibana.yml:/usr/share/kibana/config/kibana.yml - ./helk-kibana/dashboards:/usr/share/kibana/dashboards - ./helk-kibana/trial/scripts:/usr/share/kibana/scripts entrypoint: /usr/share/kibana/scripts/kibana-entrypoint.sh @@ -52,10 +62,8 @@ services: - helk-elasticsearch networks: helk: - aliases: - - helk_kibana.hunt.local helk-nginx: - image: cyb3rward0g/helk-nginx:0.0.6 + image: cyb3rward0g/helk-nginx:0.0.7 container_name: helk-nginx volumes: - ./helk-nginx/trial/default:/etc/nginx/sites-available/default @@ -67,22 +75,19 @@ services: restart: always depends_on: - helk-kibana + - helk-jupyter networks: helk: - aliases: - - helk_nginx.hunt.local helk-jupyter: - image: cyb3rward0g/helk-jupyter:0.0.4 + image: cyb3rward0g/helk-jupyter:0.0.5 container_name: helk-jupyter restart: always depends_on: - - helk-nginx + - helk-elasticsearch networks: helk: - aliases: - - helk_jupyter.hunt.local helk-spark-master: - image: cyb3rward0g/helk-spark-master:2.3.1-a + image: cyb3rward0g/helk-spark-master:2.3.1-b container_name: helk-spark-master environment: - SPARK_MASTER_PORT=7077 @@ -94,56 +99,44 @@ services: - helk-elasticsearch networks: helk: - aliases: - - helk_spark_master.hunt.local helk-spark-worker: - image: cyb3rward0g/helk-spark-worker:2.3.1-a + image: cyb3rward0g/helk-spark-worker:2.3.1-b container_name: helk-spark-worker environment: - SPARK_MASTER=spark://helk-spark-master:7077 - SPARK_WORKER_MEMORY=512m - SPARK_WORKER_WEBUI_PORT=8081 - SPARK_WORKER_PORT=42950 - ports: - - "8081:8081" restart: always depends_on: - helk-spark-master networks: helk: - aliases: - - helk_spark_worker.hunt.local helk-spark-worker2: - image: cyb3rward0g/helk-spark-worker:2.3.1-a + image: cyb3rward0g/helk-spark-worker:2.3.1-b container_name: helk-spark-worker2 environment: - SPARK_MASTER=spark://helk-spark-master:7077 - SPARK_WORKER_MEMORY=512m - SPARK_WORKER_WEBUI_PORT=8082 - SPARK_WORKER_PORT=42951 - ports: - - "8082:8082" restart: always depends_on: - helk-spark-master networks: helk: - aliases: - - helk_spark_worker2.hunt.local helk-zookeeper: - image: cyb3rward0g/helk-zookeeper:1.1.1 + image: cyb3rward0g/helk-zookeeper:2.0.0-a container_name: helk-zookeeper ports: - "2181:2181" restart: always depends_on: - - helk-elasticsearch + - helk-kibana networks: helk: - aliases: - - helk_zookeeper.hunt.local helk-kafka-broker: - image: cyb3rward0g/helk-kafka-broker:1.1.1 + image: cyb3rward0g/helk-kafka-broker:2.0.0-b container_name: helk-kafka-broker restart: always depends_on: @@ -155,14 +148,13 @@ services: - REPLICATION_FACTOR=2 - ADVERTISED_LISTENER=HOSTIP - ZOOKEEPER_NAME=helk-zookeeper + - KAFKA_CREATE_TOPICS=True ports: - "9092:9092" networks: helk: - aliases: - - helk_kafka_broker.hunt.local helk-kafka-broker2: - image: cyb3rward0g/helk-kafka-broker:1.1.1 + image: cyb3rward0g/helk-kafka-broker:2.0.0-b container_name: helk-kafka-broker2 restart: always depends_on: @@ -174,21 +166,12 @@ services: - REPLICATION_FACTOR=2 - ADVERTISED_LISTENER=HOSTIP - ZOOKEEPER_NAME=helk-zookeeper + - KAFKA_CREATE_TOPICS=True ports: - "9093:9093" networks: helk: - aliases: - - helk_kafka_broker.hunt.local - helk-sigma: - image: thomaspatzke/helk-sigma - container_name: helk-sigma - depends_on: - - helk-kibana - networks: - helk: - aliases: - - helk_sigma.hunt.local + networks: helk: driver: bridge @@ -196,4 +179,12 @@ networks: volumes: esdata: driver: local + +secrets: + elasticsearch.yml: + file: ./helk-elasticsearch/trial/elasticsearch.yml + logstash.yml: + file: ./helk-logstash/trial/logstash.yml + kibana.yml: + file: ./helk-kibana/trial/kibana.yml diff --git a/docker/helk-base/Dockerfile b/docker/helk-base/Dockerfile index 3b142f98..a3caf7a2 100644 --- a/docker/helk-base/Dockerfile +++ b/docker/helk-base/Dockerfile @@ -3,7 +3,7 @@ # Author: Roberto Rodriguez (@Cyb3rWard0g) # License: GPL-3.0 -FROM phusion/baseimage:0.10.1 +FROM phusion/baseimage:0.11 LABEL maintainer="Roberto Rodriguez @Cyb3rWard0g" LABEL description="Dockerfile HELK Base Image.." diff --git a/docker/helk-elasticsearch/Dockerfile b/docker/helk-elasticsearch/Dockerfile index 46e59989..6c16bff2 100644 --- a/docker/helk-elasticsearch/Dockerfile +++ b/docker/helk-elasticsearch/Dockerfile @@ -1,12 +1,12 @@ # HELK script: HELK Elasticsearch Dockerfile # HELK build Stage: Alpha -# HELK ELK version: 6.3.2 +# HELK ELK version: 6.4.0 # Author: Roberto Rodriguez (@Cyb3rWard0g) # License: GPL-3.0 # References: # https://cyberwardog.blogspot.com/2017/02/setting-up-pentesting-i-mean-threat_98.html -FROM docker.elastic.co/elasticsearch/elasticsearch:6.3.2 +FROM docker.elastic.co/elasticsearch/elasticsearch:6.4.0 LABEL maintainer="Roberto Rodriguez @Cyb3rWard0g" LABEL description="Dockerfile base for the HELK Elasticsearch." \ No newline at end of file diff --git a/docker/helk-elasticsearch/elasticsearch.yml b/docker/helk-elasticsearch/elasticsearch.yml index 175ccfb5..23a526f8 100644 --- a/docker/helk-elasticsearch/elasticsearch.yml +++ b/docker/helk-elasticsearch/elasticsearch.yml @@ -14,13 +14,13 @@ # # Use a descriptive name for your cluster: # -cluster.name: helk-elk +#cluster.name: helk-elk # # ------------------------------------ Node ------------------------------------ # # Use a descriptive name for the node: # -node.name: helk-1 +#node.name: helk-1 # # Add custom attributes to the node: # @@ -40,7 +40,7 @@ node.name: helk-1 # # Lock the memory on startup: # -bootstrap.memory_lock: true +#bootstrap.memory_lock: true # # Make sure that the heap size is set to about half the memory available # on the system and that the owner of the process is allowed to use this @@ -74,8 +74,8 @@ network.host: 0.0.0.0 # minimum_master_nodes need to be explicitly set when bound on a public IP # set to 1 to allow single node clusters # Details: https://github.com/elastic/elasticsearch/pull/17288 -discovery.zen.minimum_master_nodes: 1 -discovery.type: single-node +#discovery.zen.minimum_master_nodes: 1 +#discovery.type: single-node # # For more information, consult the zen discovery module documentation. # diff --git a/docker/helk-elasticsearch/trial/Dockerfile b/docker/helk-elasticsearch/trial/Dockerfile deleted file mode 100644 index d906c24c..00000000 --- a/docker/helk-elasticsearch/trial/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -# HELK script: HELK Elasticsearch Dockerfile -# HELK build Stage: Alpha -# HELK ELK version: 6.3.2 -# Author: Roberto Rodriguez (@Cyb3rWard0g) -# License: GPL-3.0 - -# References: -# https://cyberwardog.blogspot.com/2017/02/setting-up-pentesting-i-mean-threat_98.html - -FROM docker.elastic.co/elasticsearch/elasticsearch:6.3.2 -LABEL maintainer="Roberto Rodriguez @Cyb3rWard0g" -LABEL description="Dockerfile base for the HELK Elasticsearch." - -RUN /usr/share/elasticsearch/bin/elasticsearch-keystore create -RUN printf "elasticpassword" | /usr/share/elasticsearch/bin/elasticsearch-keystore add "bootstrap.password" \ No newline at end of file diff --git a/docker/helk-elasticsearch/trial/elasticsearch.yml b/docker/helk-elasticsearch/trial/elasticsearch.yml index a47971be..02afa3c3 100644 --- a/docker/helk-elasticsearch/trial/elasticsearch.yml +++ b/docker/helk-elasticsearch/trial/elasticsearch.yml @@ -14,13 +14,13 @@ # # Use a descriptive name for your cluster: # -cluster.name: helk-elk +#cluster.name: helk-elk # # ------------------------------------ Node ------------------------------------ # # Use a descriptive name for the node: # -node.name: helk-1 +#node.name: helk-1 # # Add custom attributes to the node: # @@ -40,7 +40,7 @@ node.name: helk-1 # # Lock the memory on startup: # -bootstrap.memory_lock: true +#bootstrap.memory_lock: true # # Make sure that the heap size is set to about half the memory available # on the system and that the owner of the process is allowed to use this @@ -74,8 +74,8 @@ network.host: 0.0.0.0 # minimum_master_nodes need to be explicitly set when bound on a public IP # set to 1 to allow single node clusters # Details: https://github.com/elastic/elasticsearch/pull/17288 -discovery.zen.minimum_master_nodes: 1 -discovery.type: single-node +#discovery.zen.minimum_master_nodes: 1 +#discovery.type: single-node # # For more information, consult the zen discovery module documentation. # diff --git a/docker/helk-jupyter/Dockerfile b/docker/helk-jupyter/Dockerfile index 7e851c61..9531b354 100644 --- a/docker/helk-jupyter/Dockerfile +++ b/docker/helk-jupyter/Dockerfile @@ -3,16 +3,16 @@ # Author: Roberto Rodriguez (@Cyb3rWard0g) # License: GPL-3.0 -FROM cyb3rward0g/helk-spark-base:2.3.1 +FROM cyb3rward0g/helk-spark-base:2.3.1-a LABEL maintainer="Roberto Rodriguez @Cyb3rWard0g" LABEL description="Dockerfile base for HELK Jupyter." ENV DEBIAN_FRONTEND noninteractive -USER root - # *********** Installing Prerequisites *************** # -qq : No output except for errors +RUN echo "[HELK-DOCKER-INSTALLATION-INFO] Updating Ubuntu base image.." \ + && apt-get update -qq RUN echo "[HELK-DOCKER-INSTALLATION-INFO] Extracting templates from packages.." \ && apt-get install -qqy --no-install-recommends \ curl python3-pip python3-dev python-tk unzip python3-setuptools \ @@ -30,14 +30,17 @@ RUN curl -sL https://deb.nodesource.com/setup_8.x | sudo -E bash - RUN apt-get install -y --no-install-recommends nodejs # *********** Installing HELK python packages *************** -RUN pip3 install pandas \ - jupyter \ - jupyterlab==0.33.4 \ - jupyterhub==0.9.1 +RUN pip3 install \ + pandas==0.23.4 \ + altair==2.2.2 \ + vega_datasets \ + jupyter==1.0.0 \ + jupyterlab==0.34.1 \ + jupyterhub==0.9.2 # *********** Installing Jupyter Lab Extension - JupyterHub *************** RUN npm install -g configurable-http-proxy -RUN jupyter labextension install @jupyterlab/hub-extension@0.10.0 +RUN jupyter labextension install @jupyterlab/hub-extension@0.11.0 # *********** Creating the Jupyter directories *************** RUN bash -c 'mkdir -pv /opt/helk/{es-hadoop,jupyter,jupyterhub}' @@ -60,6 +63,11 @@ RUN wget https://artifacts.elastic.co/downloads/elasticsearch-hadoop/elasticsear && unzip -j /opt/helk/es-hadoop/*.zip -d /opt/helk/es-hadoop/ \ && rm /opt/helk/es-hadoop/*.zip +# *********** Download Graphframes *************** +#ENV GRAPHFRAMES_VERSION=0.6.0 +#RUN wget -qO- https://github.com/graphframes/graphframes/archive/release-${GRAPHFRAMES_VERSION}.tar.gz | sudo tar xvz -C /opt/helk/graphframes/ --strip-components=1 \ +# && mv /opt/helk/graphframes/python/graphframes /opt/helk/spark/python/pyspark/graphframes + EXPOSE 8000 # *********** RUN HELK *************** WORKDIR ${JUPYTER_DIR} diff --git a/docker/helk-jupyter/spark/spark-defaults.conf b/docker/helk-jupyter/spark/spark-defaults.conf index ba8f2ba6..56d429c1 100644 --- a/docker/helk-jupyter/spark/spark-defaults.conf +++ b/docker/helk-jupyter/spark/spark-defaults.conf @@ -25,7 +25,8 @@ spark.executor.logs.rolling.strategy spark.executor.logs.rolling.time.interval spark.jars /opt/helk/es-hadoop/elasticsearch-hadoop-6.3.2.jar # Comma-separated list of Maven coordinates of jars to include on the driver and executor classpaths. # The coordinates should be groupId:artifactId:version. -spark.jars.packages graphframes:graphframes:0.5.0-spark2.1-s_2.11,org.apache.spark:spark-sql-kafka-0-10_2.11:2.3.0,databricks:spark-sklearn:0.2.3 +spark.jars.packages graphframes:graphframes:0.6.0-spark2.3-s_2.11,org.apache.spark:spark-sql-kafka-0-10_2.11:2.3.1,databricks:spark-sklearn:0.2.3 +#spark.jars.packages org.apache.spark:spark-sql-kafka-0-10_2.11:2.3.1,databricks:spark-sklearn:0.2.3 # ************ Spark UI **************** # Base directory in which Spark events are logged diff --git a/docker/helk-kafka-base/Dockerfile b/docker/helk-kafka-base/Dockerfile index 5ebdecfc..8902fb51 100644 --- a/docker/helk-kafka-base/Dockerfile +++ b/docker/helk-kafka-base/Dockerfile @@ -3,7 +3,7 @@ # Author: Roberto Rodriguez (@Cyb3rWard0g) # License: GPL-3.0 -FROM cyb3rward0g/helk-base:0.0.1 +FROM cyb3rward0g/helk-base:0.0.2 LABEL maintainer="Roberto Rodriguez @Cyb3rWard0g" LABEL description="Dockerfile base for the HELK Kafka." @@ -24,10 +24,18 @@ RUN apt-get -qy clean \ RUN bash -c 'mkdir -pv /opt/helk/kafka' # *********** Install Kafka *************** -ENV KAFKA_VERSION=1.1.1 +ENV KAFKA_VERSION=2.0.0 ENV KAFKA_LOGS_PATH=/var/log/kafka ENV KAFKA_CONSOLE_LOG=/var/log/kafka/helk-kafka.log -ENV KAFKA_HOME=/opt/helk/kafka/kafka_2.11-${KAFKA_VERSION} +ENV KAFKA_HOME=/opt/helk/kafka -RUN wget -qO- http://mirrors.ocf.berkeley.edu/apache/kafka/${KAFKA_VERSION}/kafka_2.11-${KAFKA_VERSION}.tgz | sudo tar xvz -C /opt/helk/kafka/ \ - && mkdir -v $KAFKA_LOGS_PATH \ No newline at end of file +RUN wget -qO- http://mirrors.ocf.berkeley.edu/apache/kafka/${KAFKA_VERSION}/kafka_2.11-${KAFKA_VERSION}.tgz | sudo tar xvz -C /opt/helk/kafka/ --strip-components=1 \ + && mkdir -v $KAFKA_LOGS_PATH + +# ********* Adding Kafka User ************* +ENV KAFKA_GID=910 +ENV KAFKA_UID=910 +ENV KAFKA_USER=kafkauser +RUN groupadd -g ${KAFKA_GID} ${KAFKA_USER} \ + && useradd -u ${KAFKA_UID} -g ${KAFKA_GID} -d ${KAFKA_HOME} --no-create-home -s /bin/bash ${KAFKA_USER} \ + && chown -R ${KAFKA_USER}:${KAFKA_USER} ${KAFKA_HOME} ${KAFKA_LOGS_PATH} \ No newline at end of file diff --git a/docker/helk-kafka-broker/Dockerfile b/docker/helk-kafka-broker/Dockerfile index 99ccd259..f5263b6c 100644 --- a/docker/helk-kafka-broker/Dockerfile +++ b/docker/helk-kafka-broker/Dockerfile @@ -3,21 +3,25 @@ # Author: Roberto Rodriguez (@Cyb3rWard0g) # License: GPL-3.0 -FROM cyb3rward0g/helk-kafka-base:1.1.1 +FROM cyb3rward0g/helk-kafka-base:2.0.0-a LABEL maintainer="Roberto Rodriguez @Cyb3rWard0g" LABEL description="Dockerfile base for the HELK Kafka Broker." ENV DEBIAN_FRONTEND noninteractive +# *********** Configure Kafka Broker *************** ENV KAFKA_SCRIPT=$KAFKA_HOME/bin/kafka-server-start.sh ENV KAFKA_CONFIG=$KAFKA_HOME/config/server.properties -# *********** Configure Kafka Broker *************** RUN mv $KAFKA_CONFIG ${KAFKA_HOME}/config/backup_server.properties -ADD server.properties ${KAFKA_HOME}/config/ -ADD scripts/kafka-entrypoint.sh /opt/helk/scripts/ -RUN chmod +x /opt/helk/scripts/kafka-entrypoint.sh +COPY server.properties ${KAFKA_HOME}/config/ +COPY scripts /opt/helk/kafka/scripts +RUN chmod +x /opt/helk/kafka/scripts/kafka-entrypoint.sh +RUN chmod +x /opt/helk/kafka/scripts/kafka-create-topics.sh + +USER ${KAFKA_USER} EXPOSE $KAFKA_BROKER_PORT -WORKDIR "/opt/helk/scripts/" -ENTRYPOINT ["./kafka-entrypoint.sh"] \ No newline at end of file +WORKDIR "/opt/helk/kafka/scripts/" +ENTRYPOINT ["./kafka-entrypoint.sh"] +CMD ["/bin/bash","-c","$KAFKA_SCRIPT $KAFKA_CONFIG"] \ No newline at end of file diff --git a/docker/helk-kafka-broker/scripts/kafka-create-topics.sh b/docker/helk-kafka-broker/scripts/kafka-create-topics.sh new file mode 100755 index 00000000..fdf1ca78 --- /dev/null +++ b/docker/helk-kafka-broker/scripts/kafka-create-topics.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# HELK script: kafka-create-topics.sh +# HELK script description: creates kafka topics +# HELK build Stage: Alpha +# Author: Roberto Rodriguez (@Cyb3rWard0g) +# License: GPL-3.0 + +# *********** Configuring Kafka ************** +if [[ -z "$KAFKA_CREATE_TOPICS" ]]; then + echo "[HELK-DOCKER-INSTALLATION-INFO] No topics will be created" + exit 0 +fi + +if [[ ! -z "$REPLICATION_FACTOR" ]]; then + echo "[HELK-DOCKER-INSTALLATION-INFO] Setting replication factor for topics to $REPLICATION_FACTOR" +else + REPLICATION_FACTOR=1 +fi + +if [[ ! -z "$ZOOKEEPER_NAME" ]]; then + echo "[HELK-DOCKER-INSTALLATION-INFO] Setting Zookeeper name to $ZOOKEEPER_NAME" +else + ZOOKEEPER_NAME=localhost +fi + +# *********** Waiting for Kafka broker to be up *************** +echo "[HELK-DOCKER-INSTALLATION-INFO] Checking to see if Kafka broker is up..." +while [[ "$(curl -sm5 $KAFKA_BROKER_NAME:$KAFKA_BROKER_PORT -o /dev/null; echo $?)" != 56 ]] ; do + echo "[HELK-DOCKER-INSTALLATION-INFO] Kafka broker $KAFKA_BROKER_NAME is not available yet" + sleep 1 +done + +echo "[HELK-DOCKER-INSTALLATION-INFO] Kafka is up now..." +echo "[HELK-DOCKER-INSTALLATION-INFO] Giving kakfa some time to connect to Zookeeper..." +sleep 10 + +# *********** Creating Kafka Topics************** +declare -a temas=("winlogbeat" "sysmontransformed" "securitytransformed") + +for t in ${temas[@]}; do + echo "[HELK-DOCKER-INSTALLATION-INFO] Creating Kafka ${t} Topic.." + ${KAFKA_HOME}/bin/kafka-topics.sh --create --zookeeper ${ZOOKEEPER_NAME}:2181 --replication-factor ${REPLICATION_FACTOR} --partitions 1 --topic ${t} --if-not-exists +done + +wait diff --git a/docker/helk-kafka-broker/scripts/kafka-entrypoint.sh b/docker/helk-kafka-broker/scripts/kafka-entrypoint.sh index 39e02ca5..7ec17cc7 100755 --- a/docker/helk-kafka-broker/scripts/kafka-entrypoint.sh +++ b/docker/helk-kafka-broker/scripts/kafka-entrypoint.sh @@ -18,28 +18,7 @@ else exit 1 fi -if [[ ! -z "$REPLICATION_FACTOR" ]]; then - echo "[HELK-DOCKER-INSTALLATION-INFO] Setting replication factor for topics to $REPLICATION_FACTOR" -else - REPLICATION_FACTOR=1 -fi - -if [[ ! -z "$ZOOKEEPER_NAME" ]]; then - echo "[HELK-DOCKER-INSTALLATION-INFO] Setting Zookeeper name to $ZOOKEEPER_NAME" -else - ZOOKEEPER_NAME=localhost -fi - -# *********** Starting Kafka ************** -exec $KAFKA_SCRIPT $KAFKA_CONFIG >> $KAFKA_CONSOLE_LOG 2>&1 & -sleep 30 - -# *********** Creating Kafka Topics************** -declare -a temas=("winlogbeat" "sysmontransformed" "securitytransformed") - -for t in ${temas[@]}; do - echo "[HELK-DOCKER-INSTALLATION-INFO] Creating Kafka ${t} Topic.." - ${KAFKA_HOME}/bin/kafka-topics.sh --create --zookeeper ${ZOOKEEPER_NAME}:2181 --replication-factor ${REPLICATION_FACTOR} --partitions 1 --topic ${t} --if-not-exists -done +./kafka-create-topics.sh & +unset KAFKA_CREATE_TOPICS -tail -f $KAFKA_CONSOLE_LOG +exec "$@" diff --git a/docker/helk-kibana/Dockerfile b/docker/helk-kibana/Dockerfile index a4d35264..6a89d896 100644 --- a/docker/helk-kibana/Dockerfile +++ b/docker/helk-kibana/Dockerfile @@ -1,12 +1,12 @@ # HELK script: HELK Kibana Dockerfile # HELK build Stage: Alpha -# HELK ELK version: 6.3.2 +# HELK ELK version: 6.4.0 # Author: Roberto Rodriguez (@Cyb3rWard0g) # License: GPL-3.0 # References: # https://cyberwardog.blogspot.com/2017/02/setting-up-pentesting-i-mean-threat_98.html -FROM docker.elastic.co/kibana/kibana:6.3.2 +FROM docker.elastic.co/kibana/kibana:6.4.0 LABEL maintainer="Roberto Rodriguez @Cyb3rWard0g" LABEL description="Dockerfile base for the HELK Kibana." \ No newline at end of file diff --git a/docker/helk-kibana/scripts/kibana-entrypoint.sh b/docker/helk-kibana/scripts/kibana-entrypoint.sh index 9250bd35..fdb0e94f 100755 --- a/docker/helk-kibana/scripts/kibana-entrypoint.sh +++ b/docker/helk-kibana/scripts/kibana-entrypoint.sh @@ -6,6 +6,11 @@ # Author: Roberto Rodriguez (@Cyb3rWard0g) # License: GPL-3.0 +# *********** Install Plugins ********************* +#echo "[HELK-DOCKER-INSTALLATION-INFO] Installing Kibana-Canvas.." +#NODE_OPTIONS="--max-old-space-size=4096" +#kibana-plugin install https://download.elastic.co/kibana/canvas/kibana-canvas-0.1.2174.zip + # *********** Start Kibana services *************** echo "[HELK-DOCKER-INSTALLATION-INFO] Waiting for elasticsearch URI to be accessible.." until curl -s helk-elasticsearch:9200 -o /dev/null; do diff --git a/docker/helk-kibana/trial/scripts/kibana-entrypoint.sh b/docker/helk-kibana/trial/scripts/kibana-entrypoint.sh index 766e061c..da1ee8fe 100755 --- a/docker/helk-kibana/trial/scripts/kibana-entrypoint.sh +++ b/docker/helk-kibana/trial/scripts/kibana-entrypoint.sh @@ -6,10 +6,14 @@ # Author: Roberto Rodriguez (@Cyb3rWard0g) # License: GPL-3.0 -ELASTICSEARCH_ACCESS=http://elastic:"elasticpassword"@helk-elasticsearch:9200 +# *********** Install Plugins ********************* +#echo "[HELK-DOCKER-INSTALLATION-INFO] Installing Kibana-Canvas.." +#NODE_OPTIONS="--max-old-space-size=4096" +#kibana-plugin install https://download.elastic.co/kibana/canvas/kibana-canvas-0.1.2174.zip # *********** Check if Elasticsearch is up *************** echo "[HELK-DOCKER-INSTALLATION-INFO] Waiting for elasticsearch URI to be accessible.." +ELASTICSEARCH_ACCESS=http://elastic:"elasticpassword"@helk-elasticsearch:9200 until curl -s $ELASTICSEARCH_ACCESS -o /dev/null; do sleep 1 done diff --git a/docker/helk-logstash/Dockerfile b/docker/helk-logstash/Dockerfile index 136c8df3..f1d1001e 100644 --- a/docker/helk-logstash/Dockerfile +++ b/docker/helk-logstash/Dockerfile @@ -1,6 +1,6 @@ # HELK script: HELK Logstash Dockerfile # HELK build Stage: Alpha -# HELK ELK version: 6.3.2 +# HELK ELK version: 6.4.0 # Author: Roberto Rodriguez (@Cyb3rWard0g) # License: GPL-3.0 @@ -8,6 +8,6 @@ # https://cyberwardog.blogspot.com/2017/02/setting-up-pentesting-i-mean-threat_98.html # https://github.com/spujadas/elk-docker/blob/master/Dockerfile -FROM docker.elastic.co/logstash/logstash:6.3.2 +FROM docker.elastic.co/logstash/logstash:6.4.0 LABEL maintainer="Roberto Rodriguez @Cyb3rWard0g" LABEL description="Dockerfile base for the HELK Logstash." \ No newline at end of file diff --git a/docker/helk-logstash/scripts/logstash-entrypoint.sh b/docker/helk-logstash/scripts/logstash-entrypoint.sh index a1b6d882..76445633 100755 --- a/docker/helk-logstash/scripts/logstash-entrypoint.sh +++ b/docker/helk-logstash/scripts/logstash-entrypoint.sh @@ -34,6 +34,9 @@ done # ********** Install Plugin ***************** echo "[HELK-DOCKER-INSTALLATION-INFO] Installing Logstash plugins.." logstash-plugin install logstash-filter-prune +# Current FIX to https://discuss.elastic.co/t/kafka-output-plugin-java-lang-long-error/145398 +logstash-plugin remove logstash-output-kafka +logstash-plugin install --version 7.1.1 logstash-output-kafka # ********** Starting Logstash ***************** echo "[HELK-DOCKER-INSTALLATION-INFO] Running docker-entrypoint script.." diff --git a/docker/helk-logstash/trial/scripts/logstash-entrypoint.sh b/docker/helk-logstash/trial/scripts/logstash-entrypoint.sh index a639c28c..2fa5a1d4 100755 --- a/docker/helk-logstash/trial/scripts/logstash-entrypoint.sh +++ b/docker/helk-logstash/trial/scripts/logstash-entrypoint.sh @@ -36,6 +36,9 @@ done # ********** Install Plugin ***************** echo "[HELK-DOCKER-INSTALLATION-INFO] Installing Logstash plugins.." logstash-plugin install logstash-filter-prune +# Current FIX to https://discuss.elastic.co/t/kafka-output-plugin-java-lang-long-error/145398 +logstash-plugin remove logstash-output-kafka +logstash-plugin install --version 7.1.1 logstash-output-kafka # ********** Starting Logstash ***************** echo "[HELK-DOCKER-INSTALLATION-INFO] Running docker-entrypoint script.." diff --git a/docker/helk-nginx/Dockerfile b/docker/helk-nginx/Dockerfile index d7f2fd71..e8b0d27b 100644 --- a/docker/helk-nginx/Dockerfile +++ b/docker/helk-nginx/Dockerfile @@ -7,7 +7,7 @@ # https://cyberwardog.blogspot.com/2017/02/setting-up-pentesting-i-mean-threat_98.html # https://github.com/spujadas/elk-docker/blob/master/Dockerfile -FROM cyb3rward0g/helk-base:0.0.1 +FROM cyb3rward0g/helk-base:0.0.2 LABEL maintainer="Roberto Rodriguez @Cyb3rWard0g" LABEL description="Dockerfile base for the HELK Nginx." diff --git a/docker/helk-spark-base/Dockerfile b/docker/helk-spark-base/Dockerfile index 36846ebf..25c83ec2 100644 --- a/docker/helk-spark-base/Dockerfile +++ b/docker/helk-spark-base/Dockerfile @@ -3,7 +3,7 @@ # Author: Roberto Rodriguez (@Cyb3rWard0g) # License: GPL-3.0 -FROM cyb3rward0g/helk-base:0.0.1 +FROM cyb3rward0g/helk-base:0.0.2 LABEL maintainer="Roberto Rodriguez @Cyb3rWard0g" LABEL description="Dockerfile base for HELK Spark." diff --git a/docker/helk-spark-master/Dockerfile b/docker/helk-spark-master/Dockerfile index 561c6b54..b866f08c 100644 --- a/docker/helk-spark-master/Dockerfile +++ b/docker/helk-spark-master/Dockerfile @@ -3,7 +3,7 @@ # Author: Roberto Rodriguez (@Cyb3rWard0g) # License: GPL-3.0 -FROM cyb3rward0g/helk-spark-base:2.3.1 +FROM cyb3rward0g/helk-spark-base:2.3.1-a LABEL maintainer="Roberto Rodriguez @Cyb3rWard0g" LABEL description="Dockerfile base for HELK Spark Master." diff --git a/docker/helk-spark-worker/Dockerfile b/docker/helk-spark-worker/Dockerfile index 639f2a42..6c0c99c3 100644 --- a/docker/helk-spark-worker/Dockerfile +++ b/docker/helk-spark-worker/Dockerfile @@ -3,7 +3,7 @@ # Author: Roberto Rodriguez (@Cyb3rWard0g) # License: GPL-3.0 -FROM cyb3rward0g/helk-spark-base:2.3.1 +FROM cyb3rward0g/helk-spark-base:2.3.1-a LABEL maintainer="Roberto Rodriguez @Cyb3rWard0g" LABEL description="Dockerfile base for HELK Spark Worker." diff --git a/docker/helk-zookeeper/Dockerfile b/docker/helk-zookeeper/Dockerfile index 603122ff..c2c55dd8 100644 --- a/docker/helk-zookeeper/Dockerfile +++ b/docker/helk-zookeeper/Dockerfile @@ -3,7 +3,7 @@ # Author: Roberto Rodriguez (@Cyb3rWard0g) # License: GPL-3.0 -FROM cyb3rward0g/helk-kafka-base:1.1.1 +FROM cyb3rward0g/helk-kafka-base:2.0.0-a LABEL maintainer="Roberto Rodriguez @Cyb3rWard0g" LABEL description="Dockerfile base for the HELK Kafka Zookeeper." @@ -13,7 +13,8 @@ ENV DEBIAN_FRONTEND noninteractive RUN bash -c 'mkdir -pv /opt/helk/zookeeper' # *********** ConfigureZookeeper *************** -ENV ZOO_CONF_DIR=/opt/helk/zookeeper/conf \ +ENV ZOO_HOME=/opt/helk/zookeeper \ + ZOO_CONF_DIR=/opt/helk/zookeeper/conf \ ZOO_DATA_DIR=/opt/helk/zookeeper/data \ ZOO_DATA_LOG_DIR=/opt/helk/zookeeper/datalog \ ZOO_PORT=2181 \ @@ -26,11 +27,14 @@ ENV ZOO_CONF_DIR=/opt/helk/zookeeper/conf \ # *********** Configure zookeeper *************** RUN mkdir -p "$ZOO_CONF_DIR" "$ZOO_DATA_LOG_DIR" "$ZOO_DATA_DIR" "$ZOO_LOGS_PATH" -ADD scripts/zookeeper-entrypoint.sh /opt/helk/scripts/ -RUN chmod +x /opt/helk/scripts/zookeeper-entrypoint.sh +COPY scripts /opt/helk/zookeeper/scripts +RUN chmod +x /opt/helk/zookeeper/scripts/zookeeper-entrypoint.sh -VOLUME ["$ZOO_DATA_DIR", "$ZOO_DATA_LOG_DIR"] +RUN chown -R ${KAFKA_USER}:${KAFKA_USER} ${ZOO_HOME} ${ZOO_LOGS_PATH} + +USER ${KAFKA_USER} EXPOSE $ZOO_PORT 2888 3888 -WORKDIR "/opt/helk/scripts/" -ENTRYPOINT ["./zookeeper-entrypoint.sh"] \ No newline at end of file +WORKDIR "/opt/helk/zookeeper/scripts/" +ENTRYPOINT ["./zookeeper-entrypoint.sh"] +CMD ["/bin/bash","-c","/opt/helk/kafka/bin/zookeeper-server-start.sh /opt/helk/zookeeper/conf/zookeeper.properties"] \ No newline at end of file diff --git a/docker/helk-zookeeper/scripts/zookeeper-entrypoint.sh b/docker/helk-zookeeper/scripts/zookeeper-entrypoint.sh index 2f9e165b..d7218d70 100755 --- a/docker/helk-zookeeper/scripts/zookeeper-entrypoint.sh +++ b/docker/helk-zookeeper/scripts/zookeeper-entrypoint.sh @@ -25,10 +25,4 @@ if [ ! -f $ZOO_CONF_DIR/zookeeper.properties ]; then done fi -ln -sf /dev/stdout $ZOO_LOGS_FILE - -echo "[HELK-DOCKER-INSTALLATION-INFO] Starting Zookeeper.." -KAFKA_SCRIPT_ZOOKEEPER=$KAFKA_HOME/bin/zookeeper-server-start.sh -KAFKA_CONFIG_ZOOKEEPER="$ZOO_CONF_DIR/zookeeper.properties" - -exec $KAFKA_SCRIPT_ZOOKEEPER $KAFKA_CONFIG_ZOOKEEPER >> $ZOO_LOGS_FILE 2>&1 +exec "$@" \ No newline at end of file diff --git a/docker/helk_install.sh b/docker/helk_install.sh index dbe95a55..eb1ce76d 100755 --- a/docker/helk_install.sh +++ b/docker/helk_install.sh @@ -29,7 +29,7 @@ check_min_requirements(){ if [ "$systemKernel" == "Linux" ]; then AVAILABLE_MEMORY=$(awk '/MemAvailable/{printf "%.f", $2/1024/1024}' /proc/meminfo) AVAILABLE_DISK=$(df -m | awk '$NF=="/"{printf "%.f\t\t", $4 / 1024}') - if [ "${AVAILABLE_MEMORY}" -ge "12" ] && [ "${AVAILABLE_DISK}" -ge "30" ]; then + if [ "${AVAILABLE_MEMORY}" -ge "11" ] && [ "${AVAILABLE_DISK}" -ge "30" ]; then echo "[HELK-INSTALLATION-INFO] Available Memory: $AVAILABLE_MEMORY" echo "[HELK-INSTALLATION-INFO] Available Disk: $AVAILABLE_DISK" else @@ -68,8 +68,8 @@ install_curl(){ install_helk(){ # ****** Building & running HELK *********** echo "[HELK-INSTALLATION-INFO] Building & running HELK via docker-compose" - echo "[HELK-INSTALLATION-INFO] Using docker-compose-elk-${license_choice}.yml file" - docker-compose -f docker-compose-elk-${license_choice}.yml up --build -d >> $LOGFILE 2>&1 + echo "[HELK-INSTALLATION-INFO] Using docker-compose-helk-elastic-${subscription_choice}.yml file" + docker-compose -f docker-compose-helk-elastic-${subscription_choice}.yml up --build -d >> $LOGFILE 2>&1 ERROR=$? if [ $ERROR -ne 0 ]; then echoerror "Could not run HELK via docker-compose (Error Code: $ERROR)." @@ -132,19 +132,19 @@ set_helk_ip(){ host_ip="${ip_choice:-$host_ip}" } -set_helk_license(){ +set_helk_subscription(){ # *********** Accepting Defaults or Allowing user to set HELK IP *************** - local license_input - read -t 30 -p "[HELK-INSTALLATION-INFO] Set HELK License. Default value is basic: " -e -i "basic" license_input - license_choice=${license_input:-"basic"} - # *********** Validating License Input *************** - case $license_choice in + local subscription_input + read -t 30 -p "[HELK-INSTALLATION-INFO] Set HELK elastic subscription (basic or trial). Default value is basic: " -e -i "basic" subscription_input + subscription_choice=${subscription_input:-"basic"} + # *********** Validating subscription Input *************** + case $subscription_choice in basic) ;; trial) ;; *) - echo "[HELK-INSTALLATION-ERROR] Not a valid license. Valid Options: basic or trial" + echo "[HELK-INSTALLATION-ERROR] Not a valid subscription. Valid Options: basic or trial" exit 1 ;; esac @@ -152,7 +152,7 @@ set_helk_license(){ prepare_helk(){ echo "[HELK-INSTALLATION-INFO] HELK IP set to ${host_ip}" - echo "[HELK-INSTALLATION-INFO] HELK License set to ${license_choice}" + echo "[HELK-INSTALLATION-INFO] HELK elastic subscription set to ${subscription_choice}" if [ "$systemKernel" == "Linux" ]; then # Reference: https://get.docker.com/ echo "[HELK-INSTALLATION-INFO] Checking distribution list and version" @@ -210,7 +210,17 @@ prepare_helk(){ # *********** Check if docker is installed *************** if [ -x "$(command -v docker)" ]; then echo "[HELK-INSTALLATION-INFO] Docker already installed" - + echo "[HELK-INSTALLATION-INFO] Making sure you assigned enough disk space to the current Docker base directory" + AVAILABLE_DOCKER_DISK=$(df -m $(docker info --format '{{.DockerRootDir}}') | awk '$1 ~ /\//{printf "%.f\t\t", $4 / 1024}') + if [ "${AVAILABLE_DOCKER_DISK}" -ge "30" ]; then + echo "[HELK-INSTALLATION-INFO] Available Docker Disk: $AVAILABLE_DOCKER_DISK" + else + echo "[HELK-INSTALLATION-ERROR] YOU DO NOT HAVE ENOUGH DOCKER DISK SPACE ASSIGNED" + echo "[HELK-INSTALLATION-ERROR] Available Docker Disk: $AVAILABLE_DOCKER_DISK" + echo "[HELK-INSTALLATION-ERROR] Check the requirements section in our installation Wiki" + echo "[HELK-INSTALLATION-ERROR] Installation Wiki: https://github.com/Cyb3rWard0g/HELK/wiki/Installation" + exit 1 + fi else echo "[HELK-INSTALLATION-INFO] Docker is not installed" @@ -251,7 +261,7 @@ prepare_helk(){ fi echo "[HELK-INSTALLATION-INFO] Setting KAFKA ADVERTISED_LISTENER value..." # ****** Setting KAFKA ADVERTISED_LISTENER environment variable *********** - sed -i "s/ADVERTISED_LISTENER=HOSTIP/ADVERTISED_LISTENER=$host_ip/g" docker-compose-elk-${license_choice}.yml + sed -i "s/ADVERTISED_LISTENER=HOSTIP/ADVERTISED_LISTENER=$host_ip/g" docker-compose-helk-elastic-${subscription_choice}.yml } @@ -262,8 +272,8 @@ show_banner(){ echo "** HELK - THE HUNTING ELK **" echo "** **" echo "** Author: Roberto Rodriguez (@Cyb3rWard0g) **" - echo "** HELK build version: v0.1.2-alpha08062018 **" - echo "** HELK ELK version: 6.3.2 **" + echo "** HELK build version: v0.1.3-alpha08242018 **" + echo "** HELK ELK version: 6.4.0 **" echo "** License: GPL-3.0 **" echo "**********************************************" echo " " @@ -283,6 +293,7 @@ show_final_information(){ echo "HELK JUPYTERHUB URL: http://${host_ip}/jupyter" echo "HELK JUPYTERHUB USER:PWD : hunter1:hunter1P@ssw0rd!" echo "HELK JUPYTERHUB USER:PWD : hunter2:hunter2P@ssw0rd!" + echo "HELK JUPYTERHUB USER:PWD : hunter3:hunter3P@ssw0rd!" echo "HELK SPARK MASTER UI: http://${host_ip}:8080" echo " " echo "IT IS HUNTING SEASON!!!!!" @@ -296,7 +307,7 @@ manual_install(){ check_min_requirements get_host_ip set_helk_ip - set_helk_license + set_helk_subscription prepare_helk install_helk sleep 180 @@ -317,13 +328,13 @@ usage(){ echo "Usage: $0 [option...]" >&2 echo echo " -i set HELKs IP address" - echo " -l set HELKs License (basic or trial)" + echo " -l set HELKs subscription (basic or trial)" echo " -q quiet -> not output to the console" echo echo "Examples:" echo " $0 Install HELK manually" - echo " $0 -i 192.168.64.131 -l basic Install HELK with an IP address set and basic License" - echo " $0 -i 192.168.64.131 -l trial -q Install HELK with an IP address set and trial License without sending output to the console" + echo " $0 -i 192.168.64.131 -l basic Install HELK with an IP address set and basic subscription" + echo " $0 -i 192.168.64.131 -l trial -q Install HELK with an IP address set and trial subscription without sending output to the console" echo exit 1 } @@ -339,7 +350,7 @@ while getopts ":i:l:q" opt; do quiet="TRUE" ;; l ) - license_choice=$OPTARG + subscription_choice=$OPTARG ;; \? ) echo "[HELK-INSTALLATION-ERROR] Invalid option: $OPTARG" 1>&2 @@ -356,7 +367,7 @@ if [ $# -gt 0 ]; then echo "[HELK-INSTALLATION-ERROR] Invalid option" usage fi -if [ -z "$host_ip" ] && [ -z "$quiet" ] && [ -z "$license_choice" ]; then +if [ -z "$host_ip" ] && [ -z "$quiet" ] && [ -z "$subscription_choice" ]; then manual_install else if [[ "$host_ip" =~ ^[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*$ ]]; then @@ -366,14 +377,14 @@ else usage fi done - # *********** Validating License Input *************** - case $license_choice in + # *********** Validating subscription Input *************** + case $subscription_choice in basic) ;; trial) ;; *) - echo "[HELK-INSTALLATION-ERROR] Not a valid license. Valid Options: basic or trial" + echo "[HELK-INSTALLATION-ERROR] Not a valid subscription. Valid Options: basic or trial" usage ;; esac