Add backoff strategy on audit extractions

fluentd/scripts/dump_activities.rb

@@ -1,12 +1,19 @@
 require 'json'
 require 'date'
 require_relative './parse_entities'
+require_relative './dump_utils'
 
 def get_audit_activities
   interval_time = extract_activity_interval
   datetime_from = DateTime.now - (interval_time + 1.0)/(24*60)
-  datetime_to = DateTime.now - 1.0/(24*60)
-  output = `sdm audit activities -j -e --from "#{datetime_from.to_s}" --to "#{datetime_to.to_s}"`
+  error_file_path = "/var/log/sdm-audit-activities-errors.log"
+  output = backoff_retry(-> () {
+    datetime_to = DateTime.now - 1.0/(24*60)
+    `sdm audit activities -j -e --from "#{datetime_from.to_s}" --to "#{datetime_to.to_s}" 2> #{error_file_path}`
+  }, "activities", error_file_path)
+  unless output
+    return
+  end
   output.split("\n")
 end
 
@@ -24,5 +31,8 @@ def print_activities(activities)
 end
 
 activities = get_audit_activities
+unless activities
+  return
+end
 parsed_activities = parse_activities(activities)
 print_activities(parsed_activities)

fluentd/scripts/dump_sdm_entity.rb

@@ -1,5 +1,6 @@
 require 'json'
 require 'date'
+require_relative './dump_utils'
 
 ENTITY_TYPES = {
   "resources" => "resource",
@@ -12,7 +13,13 @@ def entity_name_argument
 end
 
 def get_audit_rows
-  output = `sdm audit #{entity_name_argument} -j`
+  error_file_path = "/var/log/sdm-audit-#{entity_name_argument}-errors.log"
+  output = backoff_retry(-> () {
+    `sdm audit #{entity_name_argument} -j 2> #{error_file_path}`
+  }, entity_name_argument, error_file_path)
+  unless output
+    return
+  end
   output.split("\n")
 end
 

fluentd/scripts/dump_utils.rb

@@ -0,0 +1,29 @@
+
+require 'json'
+
+DEFAULT_RETRY_INTERVAL = 2
+BACKOFF_MAX_RETRIES = 3
+
+def backoff_retry(exec_fn, entity, error_file_path)
+  retries = 0
+  begin
+    result = exec_fn.()
+    errors = ""
+    if File.exist?(error_file_path)
+      errors = File.read(error_file_path)
+    end
+    if errors != ""
+      raise StandardError.new errors.chomp
+    end
+    result
+  rescue StandardError => e
+    if retries < BACKOFF_MAX_RETRIES
+      puts "#{JSON.generate({"error_message" => "#{e.to_s} (retrying for the #{retries + 1} time)"})}"
+      sleep DEFAULT_RETRY_INTERVAL ** retries
+      retries += 1
+      retry
+    else
+      puts "#{JSON.generate({"error_message" => "Aborting audit #{entity} extraction - the retrials count was exceeded"})}"
+    end
+  end
+end

test/plugin/test_input_filter_sdm_decode_chunk_events.rb

@@ -3,8 +3,8 @@
 require 'test/unit'
 require 'fluent/test'
 require 'fluent/test/helpers'
-require 'fluent/test/driver/output'
 require_relative '../helper'
+require_relative '../stdout_utils'
 require_relative '../../fluentd/plugins/filter_sdm_decode_chunk_events'
 
 class TestSDMDecodeChunkEventsFilter < Test::Unit::TestCase
@@ -166,13 +166,3 @@ def sample_decoded_chunk_log
     ]
   }
 end
-
-def omit_stdout(&block)
-  original_stdout = $stdout
-  $stdout = StringIO.new
-  begin
-    yield
-  ensure
-    $stdout = original_stdout
-  end
-end

test/scripts/test_dump_utils.rb

@@ -0,0 +1,91 @@
+
+require 'test/unit'
+require 'fluent/test'
+require 'fluent/test/helpers'
+require_relative '../../fluentd/scripts/dump_utils'
+require_relative '../stdout_utils'
+
+FILE_PATH = "#{`pwd`.chomp}/sdm-audit-test-errors.log"
+
+class TestBackoffRetry < Test::Unit::TestCase
+  def cleanup
+    super
+    if File.exist?(FILE_PATH)
+      File.delete(FILE_PATH)
+    end
+  end
+
+  def teardown
+    super
+    if File.exist?(FILE_PATH)
+      File.delete(FILE_PATH)
+    end
+  end
+
+  def test_when_run_successfully
+    calls_count = 0
+    result = nil
+    omit_stdout do
+      result = backoff_retry(-> () {
+        calls_count += 1
+        mock_success
+      }, "test", FILE_PATH)
+    end
+    assert_true(result)
+    assert_true(File.exist?(FILE_PATH))
+    assert_empty(File.read(FILE_PATH))
+    assert_equal(1, calls_count)
+  end
+
+  def test_when_fails_and_dont_exceed_retry
+    calls_count = 0
+    result = nil
+    omit_stdout do
+      result = backoff_retry(-> () {
+        calls_count += 1
+        mock_fail_once
+      }, "test", FILE_PATH)
+    end
+    assert_true(result)
+    assert_true(File.exist?(FILE_PATH))
+    assert_empty(File.read(FILE_PATH))
+    assert_equal(2, calls_count)
+  end
+
+  def test_when_exceed_retry
+    calls_count = 0
+    result = nil
+    omit_stdout do
+      result = backoff_retry(-> () {
+        calls_count += 1
+        mock_fail
+      }, "test", FILE_PATH)
+    end
+    assert_nil(result)
+    assert_true(File.exist?(FILE_PATH))
+    assert_not_empty(File.read(FILE_PATH))
+    assert_equal(4, calls_count)
+  end
+
+  private
+  def mock_success
+    File.new(FILE_PATH, "w+")
+    true
+  end
+
+  def mock_fail_once
+    if File.exist?(FILE_PATH)
+      mock_success
+    else
+      mock_fail
+      nil
+    end
+  end
+
+  def mock_fail
+    File.write(FILE_PATH, "w") do |file|
+      file.write("error")
+    end
+    nil
+  end
+end

test/stdout_utils.rb

@@ -0,0 +1,12 @@
+
+require 'fluent/test/driver/output'
+
+def omit_stdout
+  original_stdout = $stdout
+  $stdout = StringIO.new
+  begin
+    yield
+  ensure
+    $stdout = original_stdout
+  end
+end