From 04d59373e8362d29e4b6d4549d2eb54293cd1f18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=AD=E6=9E=97=E5=8D=8E?= Date: Thu, 30 Mar 2017 16:41:01 +0800 Subject: [PATCH 1/3] ignore binary --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 0cdefe08..23831231 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,4 @@ _testmain.go *.test *.prof bin/ +docker-image/pilot From 126c95bf7ec281c1a4f11e77f0463d9b1bbc9e0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=AD=E6=9E=97=E5=8D=8E?= Date: Wed, 12 Apr 2017 10:53:36 +0800 Subject: [PATCH 2/3] add support for create log store --- .gitignore | 2 +- docker-images/Dockerfile | 2 +- docker-images/config.default | 13 ++-- docker-images/plugins/out_aliyun_sls.rb | 84 +++++++++++++++++-------- 4 files changed, 69 insertions(+), 32 deletions(-) diff --git a/.gitignore b/.gitignore index 23831231..1f02af10 100644 --- a/.gitignore +++ b/.gitignore @@ -23,4 +23,4 @@ _testmain.go *.test *.prof bin/ -docker-image/pilot +docker-images/pilot diff --git a/docker-images/Dockerfile b/docker-images/Dockerfile index 7e776f7a..4aee2714 100644 --- a/docker-images/Dockerfile +++ b/docker-images/Dockerfile @@ -7,7 +7,7 @@ RUN apk update && apk upgrade && \ gem install fluentd -v "~> 0.12.0" --no-ri --no-rdoc && \ gem install fluent-plugin-elasticsearch --no-ri --no-rdoc && \ gem install gelf -v "~> 3.0.0" --no-ri --no-rdoc && \ - gem install aliyun_sls_sdk -v ">=0.0.5" --no-ri --no-rdoc && \ + gem install aliyun_sls_sdk -v ">=0.0.9" --no-ri --no-rdoc && \ apk del build-base ruby-dev && \ rm -rf /root/.gem diff --git a/docker-images/config.default b/docker-images/config.default index 8beb6bef..64ada3bc 100644 --- a/docker-images/config.default +++ b/docker-images/config.default @@ -78,11 +78,14 @@ cat >> $FLUENTD_CONFIG << EOF @type aliyun_sls -project $ALIYUNSLS_PROJECT -region_endpoint $ALIYUNSLS_REGION_ENDPOINT -access_key_id $ALIYUNSLS_ACCESS_KEY_ID -access_key_secret $ALIYUNSLS_ACCESS_KEY_SECRET -ssl_verify $SSL_VERIFY +project $ALIYUNSLS_PROJECT +region_endpoint $ALIYUNSLS_REGION_ENDPOINT +access_key_id $ALIYUNSLS_ACCESS_KEY_ID +access_key_secret $ALIYUNSLS_ACCESS_KEY_SECRET +ssl_verify $SSL_VERIFY +need_create_logstore $ALIYUNSLS_NEED_CREATE_LOGSTORE +create_logstore_ttl $CREATE_LOGSTORE_TTL +create_logstore_shard_count $CREATE_LOGSTORE_SHARD_COUNT flush_interval 3s diff --git a/docker-images/plugins/out_aliyun_sls.rb b/docker-images/plugins/out_aliyun_sls.rb index ff200a1b..dcd3cf5c 100644 --- a/docker-images/plugins/out_aliyun_sls.rb +++ b/docker-images/plugins/out_aliyun_sls.rb @@ -8,11 +8,15 @@ class AliyunSlsOutput < BufferedOutput config_param :access_key_id, :string, :default => nil config_param :access_key_secret, :string, :default => nil config_param :ssl_verify, :bool, :default => false + config_param :need_create_logstore, :bool, :default => false + config_param :create_logstore_ttl, :integer, :default => 1 + config_param :create_logstore_shard_count, :integer, :default => 2 def initialize super require "aliyun_sls_sdk/protobuf" - require "aliyun_sls_sdk/connection" + require "aliyun_sls_sdk" + @log_store_created = false end def configure(conf) @@ -38,7 +42,40 @@ def format(tag, time, record) def client @topic = `hostname`.strip - @_sls_con ||= AliyunSlsSdk::Connection.new(@project, @region_endpoint, @access_key_id, @access_key_secret) + @_sls_con ||= AliyunSlsSdk::LogClient.new(@region_endpoint, @access_key_id, @access_key_secret, @ssl_verify) + end + + def createLogStore(logstore_name) + begin + getStoreResp = client.get_logstore(@project, logstore_name) + rescue AliyunSlsSdk::LogException => e + if e.errorCode == "LogStoreNotExist" + retries = 2 + begin + createLogStoreResp = logClient.create_logstore(@project, logstore_name, @create_logstore_ttl, @create_logstore_shard_count) + rescue AliyunSlsSdk::LogException => e + if e.errorCode == "LogstoreAlreadyExist" + log.warn "logstore #{logstore_name} already exist" + else + raise + end + rescue => e + if retries > 0 + log.warn "Error caught when creating logs store: #{e}" + retries -= 1 + retry + end + end + end + end + end + + def getLogItem(record) + contents = {} + record.each { |k, v| + contents[k] = v + } + AliyunSlsSdk::LogItem.new(nil, contents) end def write(chunk) @@ -46,21 +83,34 @@ def write(chunk) chunk.msgpack_each do |tag, time, record| if record and record["@target"] logStoreName = record["@target"] + if not @log_store_created + if @need_create_logstore + createLogStore(logStoreName) + @log_store_created = true + else + @log_store_created = true + end + end + record.delete("@target") if not log_list_hash[logStoreName] - log_list = AliyunSlsSdk::Protobuf::LogGroup.new(:logs => [], :topic => @topic, :source => @source) - log_list_hash[logStoreName] = log_list + logItems = [] + log_list_hash[logStoreName] = logItems end - log = AliyunSlsSdk::Protobuf::Log.new(:time => Time.now.to_i, :contents => []) - pack_log_item(log_list_hash[logStoreName], log, record) + log_list_hash[logStoreName] << getLogItem(record) else log.warn "no @target key in record: #{record}, tag: #{tag}, time: #{time}" end end - log_list_hash.each do |storeName, log_list| + + puts log_list_hash.to_s + log_list_hash.each do |storeName, logitems| + puts storeName + puts logitems + putLogRequest = AliyunSlsSdk::PutLogsRequest.new(@project, storeName, @topic, nil, logitems, nil, true) retries = 2 begin - client.puts_logs(storeName, log_list, @ssl_verify) - rescue Exception => e + client.put_logs(putLogRequest) + rescue => e if retries > 0 log.warn "\tCaught in puts logs: #{e}" client.http.shutdown @@ -72,21 +122,5 @@ def write(chunk) end end end - - private - - def pack_log_item(log_list, log, record) - pack_hash_log_item(log, record) - log_list.logs << log - end - - def pack_hash_log_item(log, hash) - if hash - hash.each { |k, v| - log_item = AliyunSlsSdk::Protobuf::Log::Content.new(:key => k, :value => v || "") - log.contents << log_item - } - end - end end end From 139e1c9fdb7619686e38d4afa86300414ae83028 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=AD=E6=9E=97=E5=8D=8E?= Date: Wed, 12 Apr 2017 16:11:42 +0800 Subject: [PATCH 3/3] add support for create logstore --- docker-images/config.default | 8 +-- docker-images/plugins/out_aliyun_sls.rb | 73 +++++++++++-------------- docs/output/aliyun_sls.md | 15 +++-- 3 files changed, 46 insertions(+), 50 deletions(-) diff --git a/docker-images/config.default b/docker-images/config.default index 64ada3bc..d2f83074 100644 --- a/docker-images/config.default +++ b/docker-images/config.default @@ -82,10 +82,10 @@ project $ALIYUNSLS_PROJECT region_endpoint $ALIYUNSLS_REGION_ENDPOINT access_key_id $ALIYUNSLS_ACCESS_KEY_ID access_key_secret $ALIYUNSLS_ACCESS_KEY_SECRET -ssl_verify $SSL_VERIFY -need_create_logstore $ALIYUNSLS_NEED_CREATE_LOGSTORE -create_logstore_ttl $CREATE_LOGSTORE_TTL -create_logstore_shard_count $CREATE_LOGSTORE_SHARD_COUNT +ssl_verify ${SSL_VERIFY:-false} +need_create_logstore ${ALIYUNSLS_NEED_CREATE_LOGSTORE:-false} +create_logstore_ttl ${ALIYUNSLS_CREATE_LOGSTORE_TTL:-1} +create_logstore_shard_count ${ALIYUNSLS_CREATE_LOGSTORE_SHARD_COUNT:-2} flush_interval 3s diff --git a/docker-images/plugins/out_aliyun_sls.rb b/docker-images/plugins/out_aliyun_sls.rb index dcd3cf5c..b49cfff1 100644 --- a/docker-images/plugins/out_aliyun_sls.rb +++ b/docker-images/plugins/out_aliyun_sls.rb @@ -46,26 +46,20 @@ def client end def createLogStore(logstore_name) + retries = 2 begin - getStoreResp = client.get_logstore(@project, logstore_name) + createLogStoreResp = client.create_logstore(@project, logstore_name, @create_logstore_ttl, @create_logstore_shard_count) rescue AliyunSlsSdk::LogException => e - if e.errorCode == "LogStoreNotExist" - retries = 2 - begin - createLogStoreResp = logClient.create_logstore(@project, logstore_name, @create_logstore_ttl, @create_logstore_shard_count) - rescue AliyunSlsSdk::LogException => e - if e.errorCode == "LogstoreAlreadyExist" - log.warn "logstore #{logstore_name} already exist" - else - raise - end - rescue => e - if retries > 0 - log.warn "Error caught when creating logs store: #{e}" - retries -= 1 - retry - end - end + if e.errorCode == "LogStoreAlreadyExist" + log.warn "logstore #{logstore_name} already exist" + else + raise + end + rescue => e + if retries > 0 + log.warn "Error caught when creating logs store: #{e}" + retries -= 1 + retry end end end @@ -83,18 +77,9 @@ def write(chunk) chunk.msgpack_each do |tag, time, record| if record and record["@target"] logStoreName = record["@target"] - if not @log_store_created - if @need_create_logstore - createLogStore(logStoreName) - @log_store_created = true - else - @log_store_created = true - end - end record.delete("@target") if not log_list_hash[logStoreName] - logItems = [] - log_list_hash[logStoreName] = logItems + log_list_hash[logStoreName] = [] end log_list_hash[logStoreName] << getLogItem(record) else @@ -102,23 +87,31 @@ def write(chunk) end end - puts log_list_hash.to_s log_list_hash.each do |storeName, logitems| - puts storeName - puts logitems putLogRequest = AliyunSlsSdk::PutLogsRequest.new(@project, storeName, @topic, nil, logitems, nil, true) - retries = 2 + retries = 0 begin client.put_logs(putLogRequest) - rescue => e - if retries > 0 - log.warn "\tCaught in puts logs: #{e}" - client.http.shutdown - @_sls_con = nil - retries -= 1 - retry + rescue => e + if e.instance_of?(AliyunSlsSdk::LogException) && e.errorCode == "LogStoreNotExist" && @need_create_logstore + createLogStore(storeName) + # wait up to 60 seconds to create the logstore + if retries < 3 + retries += 1 + sleep(10 * retries) + retry + end + else + log.warn "\tCaught in puts logs: #{e.message}" + if retries < 3 + client.http.shutdown + @_sls_con = nil + retries += 1 + sleep(1 * retries) + retry + end + log.error "Could not puts logs to aliyun sls: #{e.message}" end - log.error "Could not puts logs to aliyun sls: #{e}" end end end diff --git a/docs/output/aliyun_sls.md b/docs/output/aliyun_sls.md index bd00a5f3..6f1feb4c 100644 --- a/docs/output/aliyun_sls.md +++ b/docs/output/aliyun_sls.md @@ -2,12 +2,15 @@ #### Environment variables for image `fluentd` -- FLUENTD_OUTPUT=aliyun_sls # specify your output plugin name -- ALIYUNSLS_PROJECT=test-fluentd # specify your aliyun sls project name -- ALIYUNSLS_REGION_ENDPOINT=cn-hangzhou.log.aliyuncs.com # specify your region root endpoint -- ALIYUNSLS_ACCESS_KEY_ID="your aliyun access key id" -- ALIYUNSLS_ACCESS_KEY_SECRET="your aliyun access key secret" -- SSL_VERIFY="true" # use `https` scheme to access aliyun sls service, default is false. +- FLUENTD_OUTPUT=aliyun_sls # Required, specify your output plugin name +- ALIYUNSLS_PROJECT=test-fluentd # Required, specify your aliyun sls project name +- ALIYUNSLS_REGION_ENDPOINT=cn-hangzhou.log.aliyuncs.com # Required, specify your region root endpoint +- ALIYUNSLS_ACCESS_KEY_ID="your aliyun access key id" # Required +- ALIYUNSLS_ACCESS_KEY_SECRET="your aliyun access key secret" # Required +- SSL_VERIFY="true" # Optional, use `https` scheme to access aliyun sls service, default is false. +- ALIYUNSLS_NEED_CREATE_LOGSTORE="true" # Optional, when set `true`, logstore will be created if not exist, default is false. +- ALIYUNSLS_CREATE_LOGSTORE_TTL=2 # Optional, used when ALIYUNSLS_NEED_CREATE_LOGSTORE set `true` and as param when creating log store, set the logging data time to live in days default is 1 day to live +- ALIYUNSLS_CREATE_LOGSTORE_SHARD_COUNT=2 # Optional, used when ALIYUNSLS_NEED_CREATE_LOGSTORE set `true` and as param when creating log store, set the shard count, default is 2 shard count #### Labels for your `app image` whose log will streamed to aliyun sls