add support for create logstore

docker-images/config.default

@@ -82,10 +82,10 @@ project              $ALIYUNSLS_PROJECT
 region_endpoint      $ALIYUNSLS_REGION_ENDPOINT
 access_key_id        $ALIYUNSLS_ACCESS_KEY_ID
 access_key_secret    $ALIYUNSLS_ACCESS_KEY_SECRET
-ssl_verify           $SSL_VERIFY
-need_create_logstore $ALIYUNSLS_NEED_CREATE_LOGSTORE
-create_logstore_ttl  $CREATE_LOGSTORE_TTL
-create_logstore_shard_count $CREATE_LOGSTORE_SHARD_COUNT
+ssl_verify           ${SSL_VERIFY:-false}
+need_create_logstore ${ALIYUNSLS_NEED_CREATE_LOGSTORE:-false}
+create_logstore_ttl  ${ALIYUNSLS_CREATE_LOGSTORE_TTL:-1}
+create_logstore_shard_count ${ALIYUNSLS_CREATE_LOGSTORE_SHARD_COUNT:-2}
 
 flush_interval 3s
 </match>

docker-images/plugins/out_aliyun_sls.rb

@@ -46,26 +46,20 @@ def client
     end
 
     def createLogStore(logstore_name)
+      retries = 2
       begin
-        getStoreResp = client.get_logstore(@project, logstore_name)
+        createLogStoreResp = client.create_logstore(@project, logstore_name, @create_logstore_ttl, @create_logstore_shard_count)
       rescue AliyunSlsSdk::LogException => e
-        if e.errorCode == "LogStoreNotExist"
-          retries = 2
-          begin
-            createLogStoreResp = logClient.create_logstore(@project, logstore_name, @create_logstore_ttl, @create_logstore_shard_count)
-          rescue AliyunSlsSdk::LogException => e
-            if e.errorCode == "LogstoreAlreadyExist"
-              log.warn "logstore #{logstore_name} already exist"
-            else
-              raise
-            end
-          rescue => e
-            if retries > 0
-              log.warn "Error caught when creating logs store: #{e}"
-              retries -= 1
-              retry
-            end
-          end
+        if e.errorCode == "LogStoreAlreadyExist"
+          log.warn "logstore #{logstore_name} already exist"
+        else
+          raise
+        end
+      rescue => e
+        if retries > 0
+          log.warn "Error caught when creating logs store: #{e}"
+          retries -= 1
+          retry
         end
       end
     end
@@ -83,42 +77,41 @@ def write(chunk)
       chunk.msgpack_each do |tag, time, record|
         if record and record["@target"]
           logStoreName = record["@target"]
-          if not @log_store_created
-            if @need_create_logstore
-              createLogStore(logStoreName)
-              @log_store_created = true
-            else
-              @log_store_created = true
-            end
-          end
           record.delete("@target")
           if not log_list_hash[logStoreName]
-            logItems = []
-            log_list_hash[logStoreName] = logItems
+            log_list_hash[logStoreName] = []
           end
           log_list_hash[logStoreName] << getLogItem(record)
         else
           log.warn "no @target key in record: #{record}, tag: #{tag}, time: #{time}"
         end
       end
 
-      puts log_list_hash.to_s
       log_list_hash.each do |storeName, logitems|
-        puts storeName
-        puts logitems
         putLogRequest = AliyunSlsSdk::PutLogsRequest.new(@project, storeName, @topic, nil, logitems, nil, true)
-        retries = 2
+        retries = 0
         begin
           client.put_logs(putLogRequest)
-        rescue => e
-          if retries > 0
-            log.warn "\tCaught in puts logs: #{e}"
-            client.http.shutdown
-            @_sls_con = nil
-            retries -= 1
-            retry
+        rescue  => e
+          if e.instance_of?(AliyunSlsSdk::LogException) && e.errorCode == "LogStoreNotExist" && @need_create_logstore
+            createLogStore(storeName)
+            # wait up to 60 seconds to create the logstore
+            if retries < 3
+              retries += 1
+              sleep(10 * retries)
+              retry
+            end
+          else
+            log.warn "\tCaught in puts logs: #{e.message}"
+            if retries < 3
+              client.http.shutdown
+              @_sls_con = nil
+              retries += 1
+              sleep(1 * retries)
+              retry
+            end
+            log.error "Could not puts logs to aliyun sls: #{e.message}"
           end
-          log.error "Could not puts logs to aliyun sls: #{e}"
         end
       end
     end

docs/output/aliyun_sls.md

@@ -2,12 +2,15 @@
 
 #### Environment variables for image `fluentd`
 
-- FLUENTD_OUTPUT=aliyun_sls  # specify your output plugin name
-- ALIYUNSLS_PROJECT=test-fluentd  # specify your aliyun sls project name
-- ALIYUNSLS_REGION_ENDPOINT=cn-hangzhou.log.aliyuncs.com  # specify your region root endpoint
-- ALIYUNSLS_ACCESS_KEY_ID="your aliyun access key id"
-- ALIYUNSLS_ACCESS_KEY_SECRET="your aliyun access key secret"
-- SSL_VERIFY="true" # use `https` scheme to access aliyun sls service, default is false.
+- FLUENTD_OUTPUT=aliyun_sls  # Required, specify your output plugin name
+- ALIYUNSLS_PROJECT=test-fluentd  # Required, specify your aliyun sls project name
+- ALIYUNSLS_REGION_ENDPOINT=cn-hangzhou.log.aliyuncs.com  # Required, specify your region root endpoint
+- ALIYUNSLS_ACCESS_KEY_ID="your aliyun access key id"     # Required 
+- ALIYUNSLS_ACCESS_KEY_SECRET="your aliyun access key secret"  # Required 
+- SSL_VERIFY="true" # Optional, use `https` scheme to access aliyun sls service, default is false.
+- ALIYUNSLS_NEED_CREATE_LOGSTORE="true" # Optional, when set `true`, logstore will be created if not exist, default is false.
+- ALIYUNSLS_CREATE_LOGSTORE_TTL=2  # Optional, used when ALIYUNSLS_NEED_CREATE_LOGSTORE set `true` and as param when creating log store, set the logging data time to live in days default is 1 day to live
+- ALIYUNSLS_CREATE_LOGSTORE_SHARD_COUNT=2 # Optional, used when ALIYUNSLS_NEED_CREATE_LOGSTORE set `true` and as param when creating log store, set the shard count, default is 2 shard count 
 
 #### Labels for your `app image` whose log will streamed to aliyun sls