first commit

README.md

@@ -1,2 +1,15 @@
-# fluent-pilot
-Collect logs in docker containers
+Build
+=====
+
+```
+cd images
+./build.sh
+```
+
+Run
+===
+
+```
+docker run --rm -it -v /var/run/docker.sock:/var/run/docker.sock -v /:/host  -e FLUENTD_OUTPUT=elasticsearch -e ELASTICSEARCH_HOST=127.0.0.1 -e ELASTICSEARCH_PORT=9200 --net host pilot
+```
+

build.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -e
+
+VERSION=0.1
+GIT_SHA=$(git rev-parse --short HEAD || echo "GitNotFound")
+export CGO_ENABLED=0 GOOS=linux GOARCH=amd64
+go build -o bin/pilot ./main.go

docker-images/Dockerfile

@@ -0,0 +1,23 @@
+FROM alpine:3.5
+
+RUN apk update && apk upgrade && \  
+  apk add ruby-json ruby-irb && \
+  apk add build-base ruby-dev && \
+  apk add ca-certificates wget && \
+  gem install fluentd -v "~> 0.12.0" --no-ri --no-rdoc && \
+  gem install fluent-plugin-elasticsearch --no-ri --no-rdoc && \
+  apk del build-base ruby-dev && \
+  rm -rf /root/.gem
+
+COPY plugins/ /etc/fluentd/plugins/
+
+COPY fluentd.conf /etc/fluentd/
+VOLUME /etc/fluentd/conf.d
+
+COPY pilot fluentd.tpl entrypoint config.default /pilot/
+VOLUME /pilot/pos
+
+EXPOSE 24224
+#CMD exec fluentd -c /fluentd/etc/$FLUENTD_CONF -p /fluentd/plugins $FLUENTD_OPT
+CMD /pilot/entrypoint
+

docker-images/build.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -e
+cd $(dirname $0)
+(
+cd ..
+./build.sh
+cp bin/pilot docker-images/
+)
+
+docker build -t pilot .
+

docker-images/config.default

@@ -0,0 +1,62 @@
+#!/bin/sh
+
+set -e
+cd $(dirname $0)
+
+assert_not_empty(){
+    arg=$1
+    shift
+    if [ -z "$arg" ]; then
+        echo "$@"
+        exit 1
+    fi
+}
+
+es(){
+assert_not_empty "$ELASTICSEARCH_HOST" "ELASTICSEARCH_HOST required"
+assert_not_empty "$ELASTICSEARCH_PORT" "ELASTICSEARCH_PORT required"
+
+cat >> /etc/fluentd/fluentd.conf << EOF
+<match **>
+@type elasticsearch
+hosts $ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT
+reconnect_on_error true
+target_index_key target
+index_name _default_index
+flush_interval 5
+type_name fluentd
+</match>
+EOF
+}
+
+default(){
+echo "use default output"
+cat >> /etc/fluentd/fluentd.conf << EOF
+<match **>
+@type stdout
+</match>
+EOF
+}
+
+graylog(){
+assert_not_empty "$GRAYLOG_HOST" "GRAYLOG_HOST required"
+assert_not_empty "$GRAYLOG_PORT" "GRAYLOG_PORT required"
+cat >> /etc/fluentd/fluentd.conf << EOF
+<match **>
+type gelf
+host $GRAYLOG_HOST
+port $GRAYLOG_PORT
+flush_interval 3s
+</match>
+EOF
+}
+
+case "$FLUENTD_OUTPUT" in
+    elasticsearch)
+        es;;
+    graylog)
+        graylog;;
+    *)
+        default
+esac
+

docker-images/elasticsearch_index_template.rb

@@ -0,0 +1,37 @@
+module Fluent::ElasticsearchIndexTemplate
+
+  def get_template(template_file)
+    if !File.exists?(template_file)
+      raise "If you specify a template_name you must specify a valid template file (checked '#{template_file}')!"
+    end
+    file_contents = IO.read(template_file).gsub(/\n/,'')
+    JSON.parse(file_contents)
+  end
+
+  def template_exists?(name)
+    client.indices.get_template(:name => name)
+    return true
+  rescue Elasticsearch::Transport::Transport::Errors::NotFound
+    return false
+  end
+
+  def template_put(name, template)
+    client.indices.put_template(:name => name, :body => template)
+  end
+
+  def template_install(name, template_file)
+    if !template_exists?(name)
+      template_put(name, get_template(template_file))
+      log.info("Template configured, but no template installed. Installed '#{name}' from #{template_file}.")
+    else
+      log.info("Template configured and already installed.")
+    end
+  end
+
+  def templates_hash_install (templates)
+    templates.each do |key, value|
+      template_install(key, value)
+    end
+  end
+
+end

docker-images/entrypoint

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+set -x
+cd $(dirname $0)
+config=config.default
+if [ -f "config.custom" ]; then
+    config=config.custom
+fi
+
+chmod +x $config
+lsof
+fuser $config
+
+if ! ./$config; then
+    echo "failed generate config with $config"
+    exit 1
+fi
+
+#/usr/bin/fluentd -c /etc/fluentd/fluentd.conf -d /var/run/fluentd.pid --log /var/log/fluentd.log
+#/usr/bin/fluentd -c /etc/fluentd/fluentd.conf -d /var/run/fluentd.pid 
+/pilot/pilot -base /host -template /pilot/fluentd.tpl -log-level debug
+

docker-images/fluentd.conf

@@ -0,0 +1,2 @@
+@include conf.d/*.conf
+

docker-images/fluentd.tpl

@@ -0,0 +1,27 @@
+{{range .configList}}
+<source>
+  @type tail
+  tag {{ $.containerId }}.{{ .Name }}
+  path {{ .HostDir }}/{{ .File }}
+  format {{ .Format }}
+  pos_file /pilot/pos/fluentd.pos
+  refresh_interval 5
+  keep_time_key
+</source>
+
+<filter {{ $.containerId }}.{{ .Name }}>
+@type record_transformer
+<record>
+host "#{Socket.gethostname}"
+{{range $key, $value := .Tags}}
+{{ $key }} {{ $value }}
+{{end}}
+
+{{if not .Tags.target}}target {{if $.source.Application}}{{ $.source.App }}-{{end}}{{ .Name }}{{end}}
+
+{{if $.source.Application}}docker_app {{ $.source.App }} {{end}}
+{{if $.source.Service}}docker_service {{ $.source.Service }} {{end}}
+{{if $.source.Container}}docker_container {{ $.source.Container }} {{end}}
+</record>
+</filter>
+{{end}}