Report even on inspec runtime exceptions (#430)

* Catch InSpec runtime exceptions and make failed report

Signed-off-by: Alex Pop <apop@chef.io>

* Add tests for failed min report

Signed-off-by: Alex Pop <apop@chef.io>

* Fix lint errors for the spec

Signed-off-by: Alex Pop <apop@chef.io>

.gitignore

@@ -44,6 +44,8 @@ vendor/
 .coverage/
 .zero-knife.rb
 Policyfile.lock.json
+chef_guid
+local-mode-cache/
 
 # vagrant stuff
 .vagrant/

files/default/handler/audit_report.rb

@@ -198,13 +198,33 @@ def call(opts, profiles)
           Chef::Log.debug "Audit Report #{r}"
           r
         else
-          Chef::Log.warn 'No audit tests are defined.'
-          {}
+          failed_report('No audit tests are defined.')
         end
       rescue Inspec::FetcherFailure => e
-        Chef::Log.error e.message
-        Chef::Log.error "We cannot fetch all profiles: #{tests}. Please make sure you're authenticated and the server is reachable."
-        {}
+        err = "Cannot fetch all profiles: #{tests}. Please make sure you're authenticated and the server is reachable. #{e.message}"
+        failed_report(err)
+      rescue => e
+        failed_report(e.message)
+      end
+
+      # In case InSpec raises a runtime exception without providing a valid report,
+      # we make one up and add two new fields to it: `status` and `status_message`
+      def failed_report(err)
+        Chef::Log.error "InSpec has raised a runtime exception. Generating a minimal failed report."
+        Chef::Log.error err
+        {
+          "platform": {
+            "name": "unknown",
+            "release": "unknown"
+          },
+          "profiles": [],
+          "statistics": {
+            "duration": 0.0000001
+          },
+          "version": "4.22.0",
+          "status": "failed",
+          "status_message": err
+        }
       end
 
       # extracts relevant node data

spec/unit/report/audit_report_spec.rb

@@ -152,21 +152,51 @@ def set_inspec_backend_cache(status = false)
       opts = { 'report' => true, 'format' => 'json', 'output' => '/dev/null' }
       path = File.expand_path('../../../data/mock_profile.rb', __FILE__)
       profiles = [{ 'name': 'example', 'path': path }]
-      # we cirumwent the default load mechanisms, therefore we have to require inspec
+      # we circumvent the default load mechanisms, therefore we have to require inspec
       require 'inspec'
       report = @audit_report.call(opts, profiles)
       expected_report = /^.*profiles.*controls.*version.*statistics.*duration.*controls.*/
       expect(report.to_json).to match(expected_report)
     end
+
     it 'given a profile, returns a json-min report' do
       require 'inspec'
       opts = { 'report' => true, 'format' => 'json-min', 'output' => '/dev/null' }
       path = File.expand_path('../../../data/mock_profile.rb', __FILE__)
       profiles = [{ 'name': 'example', 'path': path }]
-      # we cirumwent the default load mechanisms, therefore we have to require inspec
+      # we circumvent the default load mechanisms, therefore we have to require inspec
       require 'inspec'
       report = @audit_report.call(opts, profiles)
       expect(report[:controls].length).to eq(2)
     end
+
+    it 'given an unfetchable profile, returns a min failed report' do
+      require 'inspec'
+      opts = { 'report' => true, 'format' => 'json-automate', 'output' => '/dev/null' }
+      profiles = [{ 'name': 'example', 'path': '/tmp/missing-in-action-profile' }]
+      # we circumvent the default load mechanisms, therefore we have to require inspec
+      require 'inspec'
+      report = @audit_report.call(opts, profiles)
+      expect(report[:profiles].length).to eq(0)
+      expect(report[:status]).to eq('failed')
+      expect(report[:platform]).to eq({ 'name': 'unknown', 'release': 'unknown' })
+      expected_status_message = /^Cannot fetch all profiles.*does not exist$/
+      expect(report[:status_message]).to match(expected_status_message)
+    end
+
+    it 'given a bad InSpec config, returns a min failed report' do
+      require 'inspec'
+      opts = { 'backend' => 'ssh', 'report' => true, 'format' => 'json-automate', 'output' => '/dev/null' }
+      path = File.expand_path('../../../data/mock_profile.rb', __FILE__)
+      profiles = [{ 'name': 'example', 'path': path }]
+      # we circumvent the default load mechanisms, therefore we have to require inspec
+      require 'inspec'
+      report = @audit_report.call(opts, profiles)
+      expect(report[:profiles].length).to eq(0)
+      expect(report[:status]).to eq('failed')
+      expect(report[:platform]).to eq({ 'name': 'unknown', 'release': 'unknown' })
+      expected_status_message = /^Client error. can't connect.*/
+      expect(report[:status_message]).to match(expected_status_message)
+    end
   end
 end