Add metrics for VM states

src/bosh-director/lib/bosh/director/metrics_collector.rb

@@ -42,6 +42,33 @@ def initialize(config)
           labels: %i[name],
           docstring: 'Number of unresponsive agents per deployment',
         )
+
+        @unhealthy_agents = Prometheus::Client.registry.gauge(
+          :bosh_unhealthy_agents,
+          labels: %i[name],
+          docstring: 'Number of unhealthy agents (job_state == running AND number_of_processes == 0) per deployment',
+        )
+        @total_available_agents = Prometheus::Client.registry.gauge(
+          :bosh_total_available_agents,
+          labels: %i[name],
+          docstring: 'Number of total available agents (all agents, no criteria) per deployment',
+        )
+        @failing_instances = Prometheus::Client.registry.gauge(
+          :bosh_failing_instances,
+          labels: %i[name],
+          docstring: 'Number of failing instances (job_state == "failing") per deployment',
+        )
+        @stopped_instances = Prometheus::Client.registry.gauge(
+          :bosh_stopped_instances,
+          labels: %i[name],
+          docstring: 'Number of instances (job_state == "stopped") per deployment',
+        )
+
+        @unknown_instances = Prometheus::Client.registry.gauge(
+          :bosh_unknown_instances,
+          labels: %i[name],
+          docstring: 'Number of instances with unknown job_state per deployment',
+        )
         @scheduler = Rufus::Scheduler.new
       end
 
@@ -136,6 +163,98 @@ def populate_vm_metrics
         removed_deployments.each do |deployment|
           @unresponsive_agents.set(0, labels: { name: deployment })
         end
+
+        # Fetch and populate unhealthy_agents metrics
+        response_unhealthy = Net::HTTP.get_response("127.0.0.1", "/unhealthy_agents", @config.health_monitor_port)
+        return unless response_unhealthy.is_a?(Net::HTTPSuccess)
+
+        unhealthy_agent_counts = JSON.parse(response_unhealthy.body)
+        return unless unhealthy_agent_counts.is_a?(Hash)
+
+        existing_unhealthy_deployment_names = @unhealthy_agents.values.map do |key, _|
+          key[:name]
+        end
+
+        unhealthy_agent_counts.each do |deployment, count|
+          @unhealthy_agents.set(count, labels: { name: deployment })
+        end
+
+        removed_unhealthy_deployments = existing_unhealthy_deployment_names - unhealthy_agent_counts.keys
+        removed_unhealthy_deployments.each do |deployment|
+          @unhealthy_agents.set(0, labels: { name: deployment })
+        end
+
+        # Fetch and populate total_available_agents metrics
+        response_total = Net::HTTP.get_response('127.0.0.1', '/total_available_agents', @config.health_monitor_port)
+        if response_total.is_a?(Net::HTTPSuccess)
+          total_agent_counts = JSON.parse(response_total.body) rescue nil
+          if total_agent_counts.is_a?(Hash)
+            existing_total_deployment_names = @total_available_agents.values.map { |key, _| key[:name] }
+
+            total_agent_counts.each do |deployment, count|
+              @total_available_agents.set(count, labels: { name: deployment })
+            end
+
+            removed_total_deployments = existing_total_deployment_names - total_agent_counts.keys
+            removed_total_deployments.each do |deployment|
+              @total_available_agents.set(0, labels: { name: deployment })
+            end
+          end
+        end
+
+        # Fetch and populate failing_instances metrics
+        response_failing = Net::HTTP.get_response('127.0.0.1', '/failing_instances', @config.health_monitor_port)
+        if response_failing.is_a?(Net::HTTPSuccess)
+          failing_counts = JSON.parse(response_failing.body) rescue nil
+          if failing_counts.is_a?(Hash)
+            existing_failing_deployment_names = @failing_instances.values.map { |key, _| key[:name] }
+
+            failing_counts.each do |deployment, count|
+              @failing_instances.set(count, labels: { name: deployment })
+            end
+
+            removed_failing_deployments = existing_failing_deployment_names - failing_counts.keys
+            removed_failing_deployments.each do |deployment|
+              @failing_instances.set(0, labels: { name: deployment })
+            end
+          end
+        end
+
+        # Fetch and populate stopped_instances metrics
+        response_stopped = Net::HTTP.get_response('127.0.0.1', '/stopped_instances', @config.health_monitor_port)
+        if response_stopped.is_a?(Net::HTTPSuccess)
+          stopped_counts = JSON.parse(response_stopped.body) rescue nil
+          if stopped_counts.is_a?(Hash)
+            existing_stopped_deployment_names = @stopped_instances.values.map { |key, _| key[:name] }
+
+            stopped_counts.each do |deployment, count|
+              @stopped_instances.set(count, labels: { name: deployment })
+            end
+
+            removed_stopped_deployments = existing_stopped_deployment_names - stopped_counts.keys
+            removed_stopped_deployments.each do |deployment|
+              @stopped_instances.set(0, labels: { name: deployment })
+            end
+          end
+        end
+
+        # Fetch and populate unknown_instances metrics
+        response_unknown = Net::HTTP.get_response('127.0.0.1', '/unknown_instances', @config.health_monitor_port)
+        if response_unknown.is_a?(Net::HTTPSuccess)
+          unknown_counts = JSON.parse(response_unknown.body) rescue nil
+          if unknown_counts.is_a?(Hash)
+            existing_unknown_deployment_names = @unknown_instances.values.map { |key, _| key[:name] }
+
+            unknown_counts.each do |deployment, count|
+              @unknown_instances.set(count, labels: { name: deployment })
+            end
+
+            removed_unknown_deployments = existing_unknown_deployment_names - unknown_counts.keys
+            removed_unknown_deployments.each do |deployment|
+              @unknown_instances.set(0, labels: { name: deployment })
+            end
+          end
+        end
       end
 
       def populate_network_metrics

src/bosh-director/spec/unit/bosh/director/metrics_collector_spec.rb

@@ -34,7 +34,17 @@ def tick
       allow(resurrector_manager).to receive(:pause_for_all?).and_return(false, true, false)
       allow(Api::ConfigManager).to receive(:deploy_config_enabled?).and_return(true, false)
       stub_request(:get, /unresponsive_agents/)
-        .to_return(status: 200, body: JSON.dump('flaky_deployment' => 1, 'good_deployment' => 0))
+        .to_return(status: 200, body: JSON.dump("flaky_deployment" => 1, "good_deployment" => 0))
+      stub_request(:get, /unhealthy_agents/)
+        .to_return(status: 200, body: JSON.dump("flaky_deployment" => 2, "good_deployment" => 0))
+        stub_request(:get, /total_available_agents/)
+          .to_return(status: 200, body: JSON.dump("flaky_deployment" => 3, "good_deployment" => 2))
+        stub_request(:get, /failing_instances/)
+          .to_return(status: 200, body: JSON.dump("flaky_deployment" => 1, "good_deployment" => 0))
+        stub_request(:get, /stopped_instances/)
+          .to_return(status: 200, body: JSON.dump("flaky_deployment" => 0, "good_deployment" => 0))
+        stub_request(:get, /unknown_instances/)
+          .to_return(status: 200, body: JSON.dump("flaky_deployment" => 0, "good_deployment" => 1))
     end
 
     after do
@@ -44,6 +54,11 @@ def tick
       Prometheus::Client.registry.unregister(:bosh_networks_dynamic_ips_total)
       Prometheus::Client.registry.unregister(:bosh_networks_dynamic_free_ips_total)
       Prometheus::Client.registry.unregister(:bosh_unresponsive_agents)
+      Prometheus::Client.registry.unregister(:bosh_unhealthy_agents)
+      Prometheus::Client.registry.unregister(:bosh_total_available_agents)
+      Prometheus::Client.registry.unregister(:bosh_failing_instances)
+      Prometheus::Client.registry.unregister(:bosh_stopped_instances)
+      Prometheus::Client.registry.unregister(:bosh_unknown_instances)
     end
 
     describe '#prep' do
@@ -293,46 +308,89 @@ def tick
           expect(metric.get(labels: { name: 'good_deployment' })).to eq(0)
         end
 
-        context 'when the health monitor returns a non 200 response' do
+        it "emits the number of unhealthy agents for each deployment" do
+          metrics_collector.start
+          metric = Prometheus::Client.registry.get(:bosh_unhealthy_agents)
+          expect(metric.get(labels: { name: "flaky_deployment" })).to eq(2)
+          expect(metric.get(labels: { name: "good_deployment" })).to eq(0)
+        end
+
+        it 'emits the number of total available agents for each deployment' do
+          metrics_collector.start
+          metric = Prometheus::Client.registry.get(:bosh_total_available_agents)
+          expect(metric.get(labels: { name: 'flaky_deployment' })).to eq(3)
+          expect(metric.get(labels: { name: 'good_deployment' })).to eq(2)
+        end
+
+        it 'emits the failing/stopped/unknown instance metrics for each deployment' do
+          metrics_collector.start
+          metric = Prometheus::Client.registry.get(:bosh_failing_instances)
+          expect(metric.get(labels: { name: 'flaky_deployment' })).to eq(1)
+          expect(metric.get(labels: { name: 'good_deployment' })).to eq(0)
+
+          metric = Prometheus::Client.registry.get(:bosh_stopped_instances)
+          expect(metric.get(labels: { name: 'flaky_deployment' })).to eq(0)
+          expect(metric.get(labels: { name: 'good_deployment' })).to eq(0)
+
+          metric = Prometheus::Client.registry.get(:bosh_unknown_instances)
+          expect(metric.get(labels: { name: 'flaky_deployment' })).to eq(0)
+          expect(metric.get(labels: { name: 'good_deployment' })).to eq(1)
+        end
+
+        context "when the health monitor returns a non 200 response" do
           before do
-            stub_request(:get, '127.0.0.1:12345/unresponsive_agents')
+            stub_request(:get, "127.0.0.1:12345/unresponsive_agents")
+              .to_return(status: 404)
+            stub_request(:get, "127.0.0.1:12345/unhealthy_agents")
               .to_return(status: 404)
           end
 
           it 'does not emit the vm metrics' do
             metrics_collector.start
             metric = Prometheus::Client.registry.get(:bosh_unresponsive_agents)
             expect(metric.values).to be_empty
+            metric = Prometheus::Client.registry.get(:bosh_unhealthy_agents)
+            expect(metric.values).to be_empty
           end
         end
 
         context 'when the health monitor returns a non-json response' do
           before do
-            stub_request(:get, '127.0.0.1:12345/unresponsive_agents')
-              .to_return(status: 200, body: JSON.dump('bad response'))
+            stub_request(:get, "127.0.0.1:12345/unresponsive_agents")
+              .to_return(status: 200, body: JSON.dump("bad response"))
+            stub_request(:get, "127.0.0.1:12345/unhealthy_agents")
+              .to_return(status: 200, body: JSON.dump("bad response"))
           end
 
           it 'does not emit the vm metrics' do
             metrics_collector.start
             metric = Prometheus::Client.registry.get(:bosh_unresponsive_agents)
             expect(metric.values).to be_empty
+            metric = Prometheus::Client.registry.get(:bosh_unhealthy_agents)
+            expect(metric.values).to be_empty
           end
         end
 
         context 'when a deployment is deleted after metrics are gathered' do
           before do
             stub_request(:get, /unresponsive_agents/)
-              .to_return(status: 200, body: JSON.dump('flaky_deployment' => 1, 'good_deployment' => 0))
+              .to_return(status: 200, body: JSON.dump("flaky_deployment" => 1, "good_deployment" => 0))
+            stub_request(:get, /unhealthy_agents/)
+              .to_return(status: 200, body: JSON.dump("flaky_deployment" => 2, "good_deployment" => 0))
             metrics_collector.start
 
             stub_request(:get, /unresponsive_agents/)
-              .to_return(status: 200, body: JSON.dump('good_deployment' => 0))
+              .to_return(status: 200, body: JSON.dump("good_deployment" => 0))
+            stub_request(:get, /unhealthy_agents/)
+              .to_return(status: 200, body: JSON.dump("good_deployment" => 0))
             scheduler.tick
           end
 
           it 'resets the metrics for the deleted deployment' do
             metric = Prometheus::Client.registry.get(:bosh_unresponsive_agents)
-            expect(metric.get(labels: { name: 'flaky_deployment' })).to eq(0)
+            expect(metric.get(labels: { name: "flaky_deployment" })).to eq(0)
+            metric = Prometheus::Client.registry.get(:bosh_unhealthy_agents)
+            expect(metric.get(labels: { name: "flaky_deployment" })).to eq(0)
           end
         end
       end

src/bosh-monitor/lib/bosh/monitor/agent.rb

@@ -3,6 +3,8 @@ class Agent
     attr_reader   :id
     attr_reader   :discovered_at
     attr_accessor :updated_at
+    attr_accessor :job_state
+    attr_accessor :number_of_processes
 
     ATTRIBUTES = %i[deployment job index instance_id cid].freeze
 

src/bosh-monitor/lib/bosh/monitor/api_controller.rb

@@ -40,5 +40,45 @@ def initialize(heartbeat_interval = 1)
         status(503)
       end
     end
+
+    get "/unhealthy_agents" do
+      if @instance_manager.director_initial_deployment_sync_done
+        JSON.generate(@instance_manager.unhealthy_agents)
+      else
+        status(503)
+      end
+    end
+
+    get '/total_available_agents' do
+      if @instance_manager.director_initial_deployment_sync_done
+        JSON.generate(@instance_manager.total_available_agents)
+      else
+        status(503)
+      end
+    end
+
+    get '/failing_instances' do
+      if @instance_manager.director_initial_deployment_sync_done
+        JSON.generate(@instance_manager.failing_instances)
+      else
+        status(503)
+      end
+    end
+
+    get '/stopped_instances' do
+      if @instance_manager.director_initial_deployment_sync_done
+        JSON.generate(@instance_manager.stopped_instances)
+      else
+        status(503)
+      end
+    end
+
+    get '/unknown_instances' do
+      if @instance_manager.director_initial_deployment_sync_done
+        JSON.generate(@instance_manager.unknown_instances)
+      else
+        status(503)
+      end
+    end
   end
 end

src/bosh-monitor/lib/bosh/monitor/events/heartbeat.rb

@@ -65,7 +65,7 @@ def to_s
       end
 
       def to_hash
-        {
+        result = {
           kind: 'heartbeat',
           id: @id,
           timestamp: @timestamp.to_i,
@@ -79,6 +79,10 @@ def to_hash
           teams: @teams,
           metrics: @metrics.map(&:to_hash),
         }
+        # Include number_of_processes if present in attributes
+        result[:number_of_processes] = @attributes["number_of_processes"] if @attributes.key?("number_of_processes")
+
+        result
       end
 
       def to_json(*_args)