diff --git a/Dockerfile b/Dockerfile index ca1e78b..e2374ca 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,30 +1,46 @@ -FROM phusion/baseimage:0.11 as build +FROM alpine:3.8 as base LABEL maintainer="Denys Zhdanov " -RUN export DEBIAN_FRONTEND=noninteractive \ - && apt-get -y update \ - && apt-get -y upgrade \ - && apt-get -y install \ +RUN true \ + && apk add --no-cache \ + cairo \ + collectd \ + collectd-disk \ + collectd-nginx \ + findutils \ + librrd \ + memcached \ + nginx \ + nodejs \ + py3-pyldap \ + redis \ + runit \ + sqlite \ + && rm -rf \ + /etc/nginx/conf.d/default.conf \ + && mkdir -p \ + /var/log/carbon \ + /var/log/graphite + +FROM base as build +LABEL maintainer="Denys Zhdanov " + +RUN true \ + && apk add --update \ + alpine-sdk \ git \ - libcairo2-dev \ libffi-dev \ - librrd-dev \ - nginx \ - pkg-config \ - python3-cairo \ + pkgconfig \ + py3-cairo \ + py3-pip \ + py3-pyldap \ + py3-virtualenv \ + py-rrd \ python3-dev \ - python3-ldap \ - python3-pip \ - python3-rrdtool \ - sqlite3 \ + rrdtool-dev \ wget \ - && rm -rf /var/lib/apt/lists/* - -# fix python dependencies (LTS Django) -RUN python3 -m pip install --upgrade virtualenv virtualenv-tools \ && virtualenv /opt/graphite \ && . /opt/graphite/bin/activate \ - && python3 -m pip install --upgrade pip \ && pip3 install \ django==1.11.15 \ django-statsd-mozilla \ @@ -35,47 +51,48 @@ RUN python3 -m pip install --upgrade virtualenv virtualenv-tools \ rrdtool ARG version=1.1.4 -ARG whisper_version=${version} -ARG carbon_version=${version} -ARG graphite_version=${version} - -ARG whisper_repo=https://github.com/graphite-project/whisper.git -ARG carbon_repo=https://github.com/graphite-project/carbon.git -ARG graphite_repo=https://github.com/graphite-project/graphite-web.git - -ARG statsd_version=v0.8.0 - -ARG statsd_repo=https://github.com/etsy/statsd.git # install whisper -RUN git clone -b ${whisper_version} --depth 1 ${whisper_repo} /usr/local/src/whisper -WORKDIR /usr/local/src/whisper -RUN . /opt/graphite/bin/activate && python3 ./setup.py install +ARG whisper_version=${version} +ARG whisper_repo=https://github.com/graphite-project/whisper.git +RUN git clone -b ${whisper_version} --depth 1 ${whisper_repo} /usr/local/src/whisper \ + && cd /usr/local/src/whisper \ + && . /opt/graphite/bin/activate \ + && python3 ./setup.py install # install carbon -RUN git clone -b ${carbon_version} --depth 1 ${carbon_repo} /usr/local/src/carbon -WORKDIR /usr/local/src/carbon -RUN . /opt/graphite/bin/activate && pip3 install -r requirements.txt \ - && python3 ./setup.py install +ARG carbon_version=${version} +ARG carbon_repo=https://github.com/graphite-project/carbon.git +RUN . /opt/graphite/bin/activate \ + && git clone -b ${carbon_version} --depth 1 ${carbon_repo} /usr/local/src/carbon \ + && cd /usr/local/src/carbon \ + && pip3 install -r requirements.txt \ + && python3 ./setup.py install # install graphite -RUN git clone -b ${graphite_version} --depth 1 ${graphite_repo} /usr/local/src/graphite-web -WORKDIR /usr/local/src/graphite-web -RUN . /opt/graphite/bin/activate && pip3 install -r requirements.txt \ - && python3 ./setup.py install +ARG graphite_version=${version} +ARG graphite_repo=https://github.com/graphite-project/graphite-web.git +RUN . /opt/graphite/bin/activate \ + && git clone -b ${graphite_version} --depth 1 ${graphite_repo} /usr/local/src/graphite-web \ + && cd /usr/local/src/graphite-web \ + && pip3 install -r requirements.txt \ + && python3 ./setup.py install + +# install statsd (as we have to use this ugly way) +ARG statsd_version=8d5363cb109cc6363661a1d5813e0b96787c4411 +ARG statsd_repo=https://github.com/etsy/statsd.git +RUN git init /opt/statsd \ + && git -C /opt/statsd remote add origin "${statsd_repo}" \ + && git -C /opt/statsd fetch origin "${statsd_version}" \ + && git -C /opt/statsd checkout "${statsd_version}" # fixing RRD support (see https://github.com/graphite-project/docker-graphite-statsd/issues/63) RUN sed -i \ 's/return os.path.realpath(fs_path)/return os.path.realpath(fs_path).decode("utf-8")/' \ /opt/graphite/webapp/graphite/readers/rrd.py -# installing nodejs 6 -WORKDIR /opt -RUN wget https://nodejs.org/download/release/v6.14.4/node-v6.14.4-linux-x64.tar.gz && \ - tar -xvpzf node-v6.14.4-linux-x64.tar.gz && rm node-v6.14.4-linux-x64.tar.gz && mv node-v6.14.4-linux-x64 nodejs - -# install statsd -RUN git clone -b ${statsd_version} ${statsd_repo} /opt/statsd +COPY conf/opt/graphite/conf/ /opt/defaultconf/graphite/ +COPY conf/opt/graphite/webapp/graphite/local_settings.py /opt/defaultconf/graphite/local_settings.py # config graphite COPY conf/opt/graphite/conf/*.conf /opt/graphite/conf/ @@ -85,56 +102,21 @@ RUN mkdir -p /var/log/graphite/ \ && PYTHONPATH=/opt/graphite/webapp /opt/graphite/bin/django-admin.py collectstatic --noinput --settings=graphite.settings # config statsd -COPY conf/opt/statsd/config_*.js /opt/statsd/ +COPY conf/opt/statsd/config/ /opt/defaultconf/statsd/config/ -FROM phusion/baseimage:0.11 as production +FROM base as production LABEL maintainer="Denys Zhdanov " ENV STATSD_INTERFACE udp -# choose a timezone at build-time -# use `--build-arg CONTAINER_TIMEZONE=Europe/Brussels` in `docker build` -ARG CONTAINER_TIMEZONE - -RUN if [ ! -z "${CONTAINER_TIMEZONE}" ]; \ - then ln -sf /usr/share/zoneinfo/$CONTAINER_TIMEZONE /etc/localtime && \ - dpkg-reconfigure -f noninteractive tzdata; \ - fi - -RUN export DEBIAN_FRONTEND=noninteractive \ - && apt-get update --fix-missing \ - && apt-get -y upgrade \ - && apt-get install --yes --no-install-recommends \ - collectd \ - expect \ - git \ - libcairo2 \ - librrd-dev \ - memcached \ - nginx \ - python3-ldap \ - python3-pip \ - redis \ - sqlite3 \ - && apt-get clean \ - && apt-get autoremove --yes \ - && rm -rf \ - /var/lib/apt/lists/* \ - /etc/nginx/sites-enabled/default \ - && mkdir -p \ - /var/log/carbon \ - /var/log/graphite - COPY conf / -COPY conf /etc/graphite-statsd/conf/ # copy /opt from build image COPY --from=build /opt /opt -RUN /usr/local/bin/django_admin_init.exp # defaults EXPOSE 80 2003-2004 2023-2024 8080 8125 8125/udp 8126 -VOLUME ["/opt/graphite/conf", "/opt/graphite/storage", "/opt/graphite/webapp/graphite/functions/custom", "/etc/nginx", "/opt/statsd", "/etc/logrotate.d", "/var/log", "/var/lib/redis"] +VOLUME ["/opt/graphite/conf", "/opt/graphite/storage", "/opt/graphite/webapp/graphite/functions/custom", "/etc/nginx", "/opt/statsd/config", "/etc/logrotate.d", "/var/log", "/var/lib/redis"] -CMD ["/sbin/my_init"] +ENTRYPOINT ["/entrypoint"] diff --git a/README.md b/README.md index cf577c2..f3704a4 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ Host | Container | Service 2024 | 2024 | [carbon aggregator - pickle](http://graphite.readthedocs.io/en/latest/carbon-daemons.html#carbon-aggregator-py) 8080 | 8080 | Graphite internal gunicorn port (without Nginx proxying). 8125 | 8125 | [statsd](https://github.com/etsy/statsd/blob/master/docs/server.md) -8126 | 8126 | [statsd admin](https://github.com/etsy/statsd/blob/v0.7.2/docs/admin_interface.md) +8126 | 8126 | [statsd admin](https://github.com/etsy/statsd/blob/master/docs/admin_interface.md) By default, statsd listens on the UDP port 8125. If you want it to listen on the TCP port 8125 instead, you can set the environment variable `STATSD_INTERFACE` to `tcp` when running the container. @@ -63,7 +63,7 @@ DOCKER ASSIGNED | /opt/graphite/conf | graphite config DOCKER ASSIGNED | /opt/graphite/storage | graphite stats storage DOCKER ASSIGNED | /opt/graphite/webapp/graphite/functions/custom | graphite custom functions dir DOCKER ASSIGNED | /etc/nginx | nginx config -DOCKER ASSIGNED | /opt/statsd | statsd config +DOCKER ASSIGNED | /opt/statsd/config | statsd config DOCKER ASSIGNED | /etc/logrotate.d | logrotate config DOCKER ASSIGNED | /var/log | log files DOCKER ASSIGNED | /var/lib/redis | Redis TagDB data (optional) @@ -179,7 +179,7 @@ be sure to delete the old whisper files under `/opt/graphite/storage/whisper/`. --- **Important:** Ensure your Statsd flush interval is at least as long as the highest-resolution retention. -For example, if `/opt/statsd/config.js` looks like this. +For example, if `/opt/statsd/config/udp.js` looks like this. ``` flushInterval: 10000 @@ -218,12 +218,12 @@ docker run -d\ --restart=always\ -v /path/to/graphite/configs:/opt/graphite/conf\ -v /path/to/graphite/data:/opt/graphite/storage\ - -v /path/to/statsd:/opt/statsd\ + -v /path/to/statsd_config:/opt/statsd/config\ graphiteapp/graphite-statsd ``` **Note**: The container will initialize properly if you mount empty volumes at - `/opt/graphite/conf`, `/opt/graphite/storage`, or `/opt/statsd`. + `/opt/graphite/conf`, `/opt/graphite/storage`, or `/opt/statsd/config`. ## Memcached config diff --git a/conf/entrypoint b/conf/entrypoint new file mode 100755 index 0000000..93a1181 --- /dev/null +++ b/conf/entrypoint @@ -0,0 +1,9 @@ +#!/bin/sh + +. /opt/graphite/bin/activate + +# Ensure /usr/local/bin is in PATH. On some +# weird platforms,this might not be the case +PATH="${PATH}:/usr/local/bin" + +runsvdir -P /etc/service diff --git a/conf/etc/my_init.d/01_conf_init.sh b/conf/etc/my_init.d/01_conf_init.sh deleted file mode 100755 index f533bd5..0000000 --- a/conf/etc/my_init.d/01_conf_init.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -conf_dir=/etc/graphite-statsd/conf - -# auto setup graphite with default configs if /opt/graphite/conf is missing -# needed for the use case when a docker host volume is mounted at an of the following: -# - /opt/graphite/conf -# - /opt/graphite/storage - -function folder_empty() { - [ -z "$(find "${1}" -mindepth 1 -not -name "lost+found" -print -quit)" ] -} - -if folder_empty /var/log/graphite; then - mkdir -p /var/log/graphite - touch /var/log/syslog -fi - -if folder_empty /opt/graphite/conf; then - cp -R $conf_dir/opt/graphite/conf/*.conf /opt/graphite/conf/ -fi - -if folder_empty /opt/graphite/webapp/graphite; then - cp $conf_dir/opt/graphite/webapp/graphite/local_settings.py /opt/graphite/webapp/graphite/local_settings.py -fi - -if folder_empty /opt/graphite/storage; then - mkdir -p /opt/graphite/storage/whisper - /usr/local/bin/django_admin_init.exp -fi - -if folder_empty /opt/graphite/webapp/graphite/functions/custom; then - touch /opt/graphite/webapp/graphite/functions/custom/__init__.py -fi - -# auto setup statsd with default config if /opt/statsd is missing -# needed for the use case when a docker host volume is mounted at an of the following: -# - /opt/statsd -if folder_empty /opt/statsd; then - git clone -b v0.8.0 https://github.com/etsy/statsd.git /opt/statsd && \ - cp $conf_dir/opt/statsd/config_*.js /opt/statsd/ -fi diff --git a/conf/etc/nginx/nginx.conf b/conf/etc/nginx/nginx.conf index bc50cd1..80e0c16 100644 --- a/conf/etc/nginx/nginx.conf +++ b/conf/etc/nginx/nginx.conf @@ -1,4 +1,4 @@ -user www-data; +user nginx; worker_processes 4; pid /run/nginx.pid; daemon off; diff --git a/conf/etc/service/carbon-aggregator/run b/conf/etc/service/carbon-aggregator/run index 2021f7f..4219cfb 100755 --- a/conf/etc/service/carbon-aggregator/run +++ b/conf/etc/service/carbon-aggregator/run @@ -1,5 +1,4 @@ -#!/bin/bash +#!/bin/sh rm -f /opt/graphite/storage/carbon-aggregator-a.pid -source /opt/graphite/bin/activate -exec python3 /opt/graphite/bin/carbon-aggregator.py start --debug 2>&1 >> /var/log/carbon-aggregator.log +exec python3 /opt/graphite/bin/carbon-aggregator.py start --debug 2>&1 | tee -a /var/log/carbon-aggregator.log diff --git a/conf/etc/service/carbon/run b/conf/etc/service/carbon/run index 81be778..c079873 100755 --- a/conf/etc/service/carbon/run +++ b/conf/etc/service/carbon/run @@ -1,5 +1,4 @@ -#!/bin/bash +#!/bin/sh rm -f /opt/graphite/storage/carbon-cache-a.pid -source /opt/graphite/bin/activate -exec python3 /opt/graphite/bin/carbon-cache.py start --debug 2>&1 >> /var/log/carbon.log +exec python3 /opt/graphite/bin/carbon-cache.py start --debug 2>&1 | tee -a /var/log/carbon.log diff --git a/conf/etc/service/collectd/run b/conf/etc/service/collectd/run index 57db6a3..a4af505 100755 --- a/conf/etc/service/collectd/run +++ b/conf/etc/service/collectd/run @@ -1,4 +1,10 @@ -#!/bin/bash +#!/bin/sh -[[ -n ${COLLECTD} ]] || exit 1 -exec /usr/sbin/collectd -f -C /etc/collectd/collectd.conf \ No newline at end of file +[ -n "${COLLECTD}" ] || exit 0 + +[ -d /etc/collectd/collectd.conf.d ] || mkdir -p /etc/collectd/collectd.conf.d +if folder_empty /etc/collectd/collectd.conf.d; then + touch /etc/collectd/collectd.conf.d/do_not_spill_warning_about_folder_is_empty.conf +fi + +exec /usr/sbin/collectd -f -C /etc/collectd/collectd.conf diff --git a/conf/etc/service/graphite/run b/conf/etc/service/graphite/run index 5f153c7..acd53be 100755 --- a/conf/etc/service/graphite/run +++ b/conf/etc/service/graphite/run @@ -1,13 +1,49 @@ -#!/bin/bash +#!/bin/sh + +if folder_empty /var/log/graphite; then + mkdir -p /var/log/graphite + touch /var/log/syslog +fi + +if folder_empty /opt/graphite/conf; then + cp /opt/defaultconf/graphite/conf/*.conf /opt/graphite/conf/ +fi + +if folder_empty /opt/graphite/webapp/graphite; then + cp /opt/defaultconf/graphite/local_settings.py /opt/graphite/webapp/graphite/local_settings.py +fi + +if folder_empty /opt/graphite/storage; then + mkdir -p /opt/graphite/storage/whisper + + export PYTHONPATH=/opt/graphite/webapp + export DJANGO_SETTINGS_MODULE=graphite.settings + + /opt/graphite/bin/django-admin.py makemigrations + /opt/graphite/bin/django-admin.py migrate auth + /opt/graphite/bin/django-admin.py migrate --run-syncdb + /opt/graphite/bin/django-admin.py createsuperuser + --noinput \ + --email root.graphite@mailinator.com \ + --username root <<- ENDINPUT + root + root + ENDINPUT +fi + +if folder_empty /opt/graphite/webapp/graphite/functions/custom; then + touch /opt/graphite/webapp/graphite/functions/custom/__init__.py +fi -sv start nginx || exit 1 -source /opt/graphite/bin/activate export GRAPHITE_WSGI_PROCESSES=${GRAPHITE_WSGI_PROCESSES:-4} export GRAPHITE_WSGI_THREADS=${GRAPHITE_WSGI_THREADS:-2} export GRAPHITE_WSGI_REQUEST_TIMEOUT=${GRAPHITE_WSGI_REQUEST_TIMEOUT:-65} export GRAPHITE_WSGI_REQUEST_LINE=${GRAPHITE_WSGI_REQUEST_LINE:-0} export GRAPHITE_WSGI_MAX_REQUESTS=${GRAPHITE_WSGI_MAX_REQUESTS:-1000} -export PYTHONPATH=/opt/graphite/webapp && \ +export PYTHONPATH=/opt/graphite/webapp + +sv start nginx || exit 1 + exec /opt/graphite/bin/gunicorn wsgi --preload --pythonpath=/opt/graphite/webapp/graphite \ --workers=${GRAPHITE_WSGI_PROCESSES} \ --threads=${GRAPHITE_WSGI_THREADS} \ @@ -15,4 +51,4 @@ exec /opt/graphite/bin/gunicorn wsgi --preload --pythonpath=/opt/graphite/webapp --max-requests=${GRAPHITE_WSGI_MAX_REQUESTS} \ --timeout=${GRAPHITE_WSGI_REQUEST_TIMEOUT} \ --bind=0.0.0.0:8080 \ - --log-file=/var/log/gunicorn.log \ No newline at end of file + --log-file=/var/log/gunicorn.log diff --git a/conf/etc/service/nginx/run b/conf/etc/service/nginx/run index 7691c03..fdeca60 100755 --- a/conf/etc/service/nginx/run +++ b/conf/etc/service/nginx/run @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh mkdir -p /var/log/nginx exec /usr/sbin/nginx -c /etc/nginx/nginx.conf diff --git a/conf/etc/service/redis/run b/conf/etc/service/redis/run index 2aff14f..b63003f 100755 --- a/conf/etc/service/redis/run +++ b/conf/etc/service/redis/run @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh -[[ -n ${REDIS_TAGDB} ]] || exit 1 -exec /usr/bin/redis-server /etc/redis/redis.conf \ No newline at end of file +[ -n "${REDIS_TAGDB}" ] || exit 0 +exec /usr/bin/redis-server /etc/redis/redis.conf diff --git a/conf/etc/service/statsd/run b/conf/etc/service/statsd/run index 00e555e..571530f 100755 --- a/conf/etc/service/statsd/run +++ b/conf/etc/service/statsd/run @@ -1,4 +1,7 @@ -#!/bin/bash +#!/bin/sh -exec /opt/nodejs/bin/node /opt/statsd/stats.js /opt/statsd/config_$STATSD_INTERFACE.js >> /var/log/statsd.log 2>&1 +if folder_empty /opt/statsd/config/; then + cp /opt/defaultconf/statsd/config/*.js /opt/statsd/config/ +fi +exec node /opt/statsd/stats.js /opt/statsd/config/$STATSD_INTERFACE.js 2>&1 | tee -a /var/log/statsd.log diff --git a/conf/opt/statsd/config_tcp.js b/conf/opt/statsd/config/tcp.js similarity index 100% rename from conf/opt/statsd/config_tcp.js rename to conf/opt/statsd/config/tcp.js diff --git a/conf/opt/statsd/config_udp.js b/conf/opt/statsd/config/udp.js similarity index 100% rename from conf/opt/statsd/config_udp.js rename to conf/opt/statsd/config/udp.js diff --git a/conf/usr/local/bin/django_admin_init.exp b/conf/usr/local/bin/django_admin_init.exp deleted file mode 100755 index e28b512..0000000 --- a/conf/usr/local/bin/django_admin_init.exp +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env expect - -set timeout -1 -spawn /usr/local/bin/manage.sh - -#expect "Would you like to create one now" { -# send "yes\r" -#} - -expect "Username (leave blank to use 'root'):" { - send "root\r" -} - -expect "Email address:" { - send "root.graphite@mailinator.com\r" -} - -expect "Password:" { - send "root\r" -} - -expect "Password *:" { - send "root\r" -} - -expect "Superuser created successfully" diff --git a/conf/usr/local/bin/folder_empty b/conf/usr/local/bin/folder_empty new file mode 100755 index 0000000..d88d138 --- /dev/null +++ b/conf/usr/local/bin/folder_empty @@ -0,0 +1,3 @@ +#!/bin/sh + +[ -z "$(find "${1}" -mindepth 1 -not -name "lost+found" -print -quit)" ] diff --git a/conf/usr/local/bin/manage.sh b/conf/usr/local/bin/manage.sh deleted file mode 100755 index aaf52e0..0000000 --- a/conf/usr/local/bin/manage.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -# Django 1.11 -PYTHONPATH=/opt/graphite/webapp /opt/graphite/bin/django-admin.py makemigrations --settings=graphite.settings -PYTHONPATH=/opt/graphite/webapp /opt/graphite/bin/django-admin.py migrate auth --settings=graphite.settings -PYTHONPATH=/opt/graphite/webapp /opt/graphite/bin/django-admin.py migrate --run-syncdb --settings=graphite.settings -PYTHONPATH=/opt/graphite/webapp /opt/graphite/bin/django-admin.py createsuperuser --settings=graphite.settings - -# Django 1.8 -# PYTHONPATH=/opt/graphite/webapp django-admin.py syncdb --settings=graphite.settings -# Django 1.6 -# PYTHONPATH=/opt/graphite/webapp django-admin.py update_users --settings=graphite.settings