From ff1db1f2c7bc4523565b8798d0f593bc680f2ce5 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 11 Jun 2026 22:18:35 +0200 Subject: [PATCH 1/5] Collect per-object table/index size, usage, and locking stats (#1103) Collection layer for both apps. A daily collector captures per-table and per-index size, growth, index usage, and locking/contention from sys.dm_db_partition_stats, sys.dm_db_index_usage_stats, and sys.dm_db_index_operational_stats (stable 2016+, Azure SQL DB, and MI). Dashboard: collect.index_object_stats table (02 + 06 ensure block + NC index), collector proc (55), schedule row + master dispatch (04/42). Dynamic retention (43) picks the table up automatically by collection_time. Lite: RemoteCollectorService.IndexObjectStats.cs (per-DB cross-database sp_executesql on-prem; per-database connect on Azure), DuckDB schema v29 + archival registration in BOTH ArchivableTables arrays, schedule default. Cumulative usage/locking counters carry sqlserver_start_time as the reset boundary so deltas are computed safely in the read layer. Verified live on SQL 2016: 148-149 rows across 6 databases, unused/write-only index signal works. Co-Authored-By: Claude Opus 4.8 (1M context) --- Lite/Database/DuckDbInitializer.cs | 4 +- Lite/Database/Schema.cs | 61 ++ Lite/Services/ArchiveService.cs | 1 + ...RemoteCollectorService.IndexObjectStats.cs | 578 +++++++++++++++++ Lite/Services/RemoteCollectorService.cs | 1 + Lite/Services/ScheduleManager.cs | 1 + install/02_create_tables.sql | 98 +++ install/04_create_schedule_table.sql | 1 + install/06_ensure_collection_table.sql | 69 ++- install/42_scheduled_master_collector.sql | 4 + install/55_collect_index_object_stats.sql | 581 ++++++++++++++++++ 11 files changed, 1396 insertions(+), 3 deletions(-) create mode 100644 Lite/Services/RemoteCollectorService.IndexObjectStats.cs create mode 100644 install/55_collect_index_object_stats.sql diff --git a/Lite/Database/DuckDbInitializer.cs b/Lite/Database/DuckDbInitializer.cs index 78b7f699..ffa4e164 100644 --- a/Lite/Database/DuckDbInitializer.cs +++ b/Lite/Database/DuckDbInitializer.cs @@ -97,7 +97,7 @@ public void Dispose() /// /// Current schema version. Increment this when schema changes require table rebuilds. /// - internal const int CurrentSchemaVersion = 28; + internal const int CurrentSchemaVersion = 29; private readonly string _archivePath; @@ -116,7 +116,7 @@ public DuckDbInitializer(string databasePath, ILogger? logger "query_snapshots", "cpu_utilization_stats", "file_io_stats", "memory_stats", "memory_clerks", "memory_pressure_events", "tempdb_stats", "perfmon_stats", "deadlocks", "blocked_process_reports", "memory_grant_stats", "waiting_tasks", - "running_jobs", "database_size_stats", "server_properties", + "running_jobs", "database_size_stats", "index_object_stats", "server_properties", "session_stats", "server_config", "database_config", "database_scoped_config", "trace_flags", "config_alert_log", "collection_log" diff --git a/Lite/Database/Schema.cs b/Lite/Database/Schema.cs index 3c2b8c95..99707ca2 100644 --- a/Lite/Database/Schema.cs +++ b/Lite/Database/Schema.cs @@ -639,6 +639,65 @@ vlf_count INTEGER public const string CreateDatabaseSizeStatsIndex = @" CREATE INDEX IF NOT EXISTS idx_database_size_stats_time ON database_size_stats(server_id, collection_time)"; + // Per-table/per-index size, usage, and locking stats. Sizes are point-in-time; + // usage/locking counters are cumulative (reset boundary in sqlserver_start_time). + // Column order MUST match the appender in RemoteCollectorService.IndexObjectStats.cs + // and the data columns in install/02_create_tables.sql (Dashboard parity). + public const string CreateIndexObjectStatsTable = @" +CREATE TABLE IF NOT EXISTS index_object_stats ( + collection_id BIGINT PRIMARY KEY, + collection_time TIMESTAMP NOT NULL, + server_id INTEGER NOT NULL, + server_name VARCHAR NOT NULL, + sqlserver_start_time TIMESTAMP, + database_name VARCHAR NOT NULL, + database_id INTEGER NOT NULL, + schema_name VARCHAR NOT NULL, + object_id INTEGER NOT NULL, + table_name VARCHAR NOT NULL, + index_id INTEGER NOT NULL, + index_name VARCHAR, + index_type_desc VARCHAR, + is_unique BOOLEAN, + is_primary_key BOOLEAN, + is_filtered BOOLEAN, + partition_count INTEGER, + reserved_mb DECIMAL(19,2), + used_mb DECIMAL(19,2), + in_row_data_mb DECIMAL(19,2), + lob_data_mb DECIMAL(19,2), + row_overflow_mb DECIMAL(19,2), + total_rows BIGINT, + user_seeks BIGINT, + user_scans BIGINT, + user_lookups BIGINT, + user_updates BIGINT, + last_user_seek TIMESTAMP, + last_user_scan TIMESTAMP, + last_user_lookup TIMESTAMP, + last_user_update TIMESTAMP, + leaf_insert_count BIGINT, + leaf_update_count BIGINT, + leaf_delete_count BIGINT, + range_scan_count BIGINT, + singleton_lookup_count BIGINT, + row_lock_count BIGINT, + row_lock_wait_count BIGINT, + row_lock_wait_in_ms BIGINT, + page_lock_count BIGINT, + page_lock_wait_count BIGINT, + page_lock_wait_in_ms BIGINT, + index_lock_promotion_attempt_count BIGINT, + index_lock_promotion_count BIGINT, + page_latch_wait_count BIGINT, + page_latch_wait_in_ms BIGINT, + page_io_latch_wait_count BIGINT, + page_io_latch_wait_in_ms BIGINT +)"; + + public const string CreateIndexObjectStatsIndex = @" +CREATE INDEX IF NOT EXISTS idx_index_object_stats_object ON index_object_stats(server_id, database_name, object_id, index_id, collection_time)"; + public const string CreateServerPropertiesTable = @" CREATE TABLE IF NOT EXISTS server_properties ( collection_id BIGINT PRIMARY KEY, @@ -762,6 +821,7 @@ public static IEnumerable GetAllTableStatements() yield return CreateTraceFlagsTable; yield return CreateRunningJobsTable; yield return CreateDatabaseSizeStatsTable; + yield return CreateIndexObjectStatsTable; yield return CreateServerPropertiesTable; yield return CreateSessionStatsTable; yield return CreateAlertLogTable; @@ -795,6 +855,7 @@ public static IEnumerable GetAllIndexStatements() yield return CreateTraceFlagsIndex; yield return CreateRunningJobsIndex; yield return CreateDatabaseSizeStatsIndex; + yield return CreateIndexObjectStatsIndex; yield return CreateServerPropertiesIndex; yield return CreateSessionStatsIndex; yield return CreateDismissedArchiveAlertsIndex; diff --git a/Lite/Services/ArchiveService.cs b/Lite/Services/ArchiveService.cs index 1a03fff2..bfa83800 100644 --- a/Lite/Services/ArchiveService.cs +++ b/Lite/Services/ArchiveService.cs @@ -75,6 +75,7 @@ internal static readonly (string Table, string TimeColumn)[] ArchivableTables = ("waiting_tasks", "collection_time"), ("running_jobs", "collection_time"), ("database_size_stats", "collection_time"), + ("index_object_stats", "collection_time"), ("server_properties", "collection_time"), ("session_stats", "collection_time"), ("server_config", "capture_time"), diff --git a/Lite/Services/RemoteCollectorService.IndexObjectStats.cs b/Lite/Services/RemoteCollectorService.IndexObjectStats.cs new file mode 100644 index 00000000..782ade43 --- /dev/null +++ b/Lite/Services/RemoteCollectorService.IndexObjectStats.cs @@ -0,0 +1,578 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor Lite. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Threading; +using System.Threading.Tasks; +using DuckDB.NET.Data; +using Microsoft.Data.SqlClient; +using Microsoft.Extensions.Logging; +using PerformanceMonitorLite.Models; + +namespace PerformanceMonitorLite.Services; + +public partial class RemoteCollectorService +{ + /// + /// Collects per-table and per-index size, usage, and locking statistics for growth + /// trending, unused-index detection, and contention analysis. + /// Size columns are absolute point-in-time values; usage and locking counters are + /// cumulative (reset on instance restart / DB detach / AUTO_CLOSE) - sqlserver_start_time + /// carries the reset boundary so deltas can be computed safely in the read layer. + /// All three DMVs are database-scoped: on-prem iterates databases via a cursor + cross-DB + /// sp_executesql into a #temp; Azure SQL DB connects to each database individually. + /// In-Memory OLTP (Hekaton) objects are not represented by these DMVs. + /// + private async Task CollectIndexObjectStatsAsync(ServerConnection server, CancellationToken cancellationToken) + { + var serverStatus = _serverManager.GetConnectionStatus(server.Id); + bool isAzureSqlDb = serverStatus?.SqlEngineEdition == 5; + + string onPremQuery = @" +SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; +SET NOCOUNT ON; + +DECLARE + @sqlserver_start_time datetime2(7) = + (SELECT osi.sqlserver_start_time FROM sys.dm_os_sys_info AS osi), + @db_name sysname, + @sql nvarchar(MAX); + +CREATE TABLE #ios +( + database_name sysname NOT NULL, + database_id int NOT NULL, + schema_name sysname NOT NULL, + object_id int NOT NULL, + table_name sysname NOT NULL, + index_id int NOT NULL, + index_name sysname NULL, + index_type_desc nvarchar(60) NULL, + is_unique bit NULL, + is_primary_key bit NULL, + is_filtered bit NULL, + partition_count int NULL, + reserved_mb decimal(19,2) NULL, + used_mb decimal(19,2) NULL, + in_row_data_mb decimal(19,2) NULL, + lob_data_mb decimal(19,2) NULL, + row_overflow_mb decimal(19,2) NULL, + total_rows bigint NULL, + user_seeks bigint NULL, + user_scans bigint NULL, + user_lookups bigint NULL, + user_updates bigint NULL, + last_user_seek datetime2(7) NULL, + last_user_scan datetime2(7) NULL, + last_user_lookup datetime2(7) NULL, + last_user_update datetime2(7) NULL, + leaf_insert_count bigint NULL, + leaf_update_count bigint NULL, + leaf_delete_count bigint NULL, + range_scan_count bigint NULL, + singleton_lookup_count bigint NULL, + row_lock_count bigint NULL, + row_lock_wait_count bigint NULL, + row_lock_wait_in_ms bigint NULL, + page_lock_count bigint NULL, + page_lock_wait_count bigint NULL, + page_lock_wait_in_ms bigint NULL, + index_lock_promotion_attempt_count bigint NULL, + index_lock_promotion_count bigint NULL, + page_latch_wait_count bigint NULL, + page_latch_wait_in_ms bigint NULL, + page_io_latch_wait_count bigint NULL, + page_io_latch_wait_in_ms bigint NULL +); + +DECLARE db_cursor CURSOR LOCAL FAST_FORWARD FOR + SELECT + d.name + FROM sys.databases AS d + WHERE d.state_desc = N'ONLINE' + AND d.database_id > 0 + AND HAS_DBACCESS(d.name) = 1 + /*EXCLUSION_FILTER_CURSOR*/ + ORDER BY + d.name; + +OPEN db_cursor; +FETCH NEXT FROM db_cursor INTO @db_name; + +WHILE @@FETCH_STATUS = 0 +BEGIN + BEGIN TRY + SET @sql = N'EXECUTE ' + QUOTENAME(@db_name) + N'.sys.sp_executesql N'' +INSERT #ios +( + database_name, database_id, schema_name, object_id, table_name, index_id, index_name, + index_type_desc, is_unique, is_primary_key, is_filtered, partition_count, reserved_mb, + used_mb, in_row_data_mb, lob_data_mb, row_overflow_mb, total_rows, user_seeks, user_scans, + user_lookups, user_updates, last_user_seek, last_user_scan, last_user_lookup, last_user_update, + leaf_insert_count, leaf_update_count, leaf_delete_count, range_scan_count, singleton_lookup_count, + row_lock_count, row_lock_wait_count, row_lock_wait_in_ms, page_lock_count, page_lock_wait_count, + page_lock_wait_in_ms, index_lock_promotion_attempt_count, index_lock_promotion_count, + page_latch_wait_count, page_latch_wait_in_ms, page_io_latch_wait_count, page_io_latch_wait_in_ms +) +SELECT + DB_NAME(), DB_ID(), s.name, o.object_id, o.name, i.index_id, i.name, i.type_desc, + i.is_unique, i.is_primary_key, i.has_filter, ps.partition_count, + CONVERT(decimal(19,2), ps.reserved_pages * 8.0 / 1024.0), + CONVERT(decimal(19,2), ps.used_pages * 8.0 / 1024.0), + CONVERT(decimal(19,2), ps.in_row_pages * 8.0 / 1024.0), + CONVERT(decimal(19,2), ps.lob_pages * 8.0 / 1024.0), + CONVERT(decimal(19,2), ps.row_overflow_pages * 8.0 / 1024.0), + ps.total_rows, us.user_seeks, us.user_scans, us.user_lookups, us.user_updates, + us.last_user_seek, us.last_user_scan, us.last_user_lookup, us.last_user_update, + os.leaf_insert_count, os.leaf_update_count, os.leaf_delete_count, os.range_scan_count, + os.singleton_lookup_count, os.row_lock_count, os.row_lock_wait_count, os.row_lock_wait_in_ms, + os.page_lock_count, os.page_lock_wait_count, os.page_lock_wait_in_ms, + os.index_lock_promotion_attempt_count, os.index_lock_promotion_count, + os.page_latch_wait_count, os.page_latch_wait_in_ms, os.page_io_latch_wait_count, + os.page_io_latch_wait_in_ms +FROM sys.indexes AS i +JOIN sys.objects AS o + ON o.object_id = i.object_id +JOIN sys.schemas AS s + ON s.schema_id = o.schema_id +LEFT JOIN +( + SELECT + dps.object_id, dps.index_id, + partition_count = COUNT_BIG(*), + reserved_pages = SUM(dps.reserved_page_count), + used_pages = SUM(dps.used_page_count), + in_row_pages = SUM(dps.in_row_data_page_count), + lob_pages = SUM(dps.lob_used_page_count), + row_overflow_pages = SUM(dps.row_overflow_used_page_count), + total_rows = SUM(dps.row_count) + FROM sys.dm_db_partition_stats AS dps + GROUP BY dps.object_id, dps.index_id +) AS ps + ON ps.object_id = i.object_id AND ps.index_id = i.index_id +LEFT JOIN sys.dm_db_index_usage_stats AS us + ON us.database_id = DB_ID() AND us.object_id = i.object_id AND us.index_id = i.index_id +LEFT JOIN +( + SELECT + ios.object_id, ios.index_id, + leaf_insert_count = SUM(ios.leaf_insert_count), + leaf_update_count = SUM(ios.leaf_update_count), + leaf_delete_count = SUM(ios.leaf_delete_count), + range_scan_count = SUM(ios.range_scan_count), + singleton_lookup_count = SUM(ios.singleton_lookup_count), + row_lock_count = SUM(ios.row_lock_count), + row_lock_wait_count = SUM(ios.row_lock_wait_count), + row_lock_wait_in_ms = SUM(ios.row_lock_wait_in_ms), + page_lock_count = SUM(ios.page_lock_count), + page_lock_wait_count = SUM(ios.page_lock_wait_count), + page_lock_wait_in_ms = SUM(ios.page_lock_wait_in_ms), + index_lock_promotion_attempt_count = SUM(ios.index_lock_promotion_attempt_count), + index_lock_promotion_count = SUM(ios.index_lock_promotion_count), + page_latch_wait_count = SUM(ios.page_latch_wait_count), + page_latch_wait_in_ms = SUM(ios.page_latch_wait_in_ms), + page_io_latch_wait_count = SUM(ios.page_io_latch_wait_count), + page_io_latch_wait_in_ms = SUM(ios.page_io_latch_wait_in_ms) + FROM sys.dm_db_index_operational_stats(DB_ID(), NULL, NULL, NULL) AS ios + GROUP BY ios.object_id, ios.index_id +) AS os + ON os.object_id = i.object_id AND os.index_id = i.index_id +WHERE o.is_ms_shipped = 0 +AND o.type IN (N''''U'''', N''''V'''') +OPTION(RECOMPILE);'';'; + + EXECUTE sys.sp_executesql @sql; + END TRY + BEGIN CATCH + END CATCH; + + FETCH NEXT FROM db_cursor INTO @db_name; +END; + +CLOSE db_cursor; +DEALLOCATE db_cursor; + +SELECT + sqlserver_start_time = @sqlserver_start_time, + x.database_name, + x.database_id, + x.schema_name, + x.object_id, + x.table_name, + x.index_id, + x.index_name, + x.index_type_desc, + x.is_unique, + x.is_primary_key, + x.is_filtered, + x.partition_count, + x.reserved_mb, + x.used_mb, + x.in_row_data_mb, + x.lob_data_mb, + x.row_overflow_mb, + x.total_rows, + x.user_seeks, + x.user_scans, + x.user_lookups, + x.user_updates, + x.last_user_seek, + x.last_user_scan, + x.last_user_lookup, + x.last_user_update, + x.leaf_insert_count, + x.leaf_update_count, + x.leaf_delete_count, + x.range_scan_count, + x.singleton_lookup_count, + x.row_lock_count, + x.row_lock_wait_count, + x.row_lock_wait_in_ms, + x.page_lock_count, + x.page_lock_wait_count, + x.page_lock_wait_in_ms, + x.index_lock_promotion_attempt_count, + x.index_lock_promotion_count, + x.page_latch_wait_count, + x.page_latch_wait_in_ms, + x.page_io_latch_wait_count, + x.page_io_latch_wait_in_ms +FROM #ios AS x +ORDER BY + x.reserved_mb DESC;"; + + var (exclusionClause, _) = BuildDatabaseExclusionFilter(server.ExcludedDatabases, "d.name"); + onPremQuery = onPremQuery.Replace("/*EXCLUSION_FILTER_CURSOR*/", exclusionClause); + + const string azureSqlDbQuery = @" +SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; + +SELECT + sqlserver_start_time = (SELECT osi.sqlserver_start_time FROM sys.dm_os_sys_info AS osi), + database_name = DB_NAME(), + database_id = DB_ID(), + schema_name = s.name, + object_id = o.object_id, + table_name = o.name, + index_id = i.index_id, + index_name = i.name, + index_type_desc = i.type_desc, + is_unique = i.is_unique, + is_primary_key = i.is_primary_key, + is_filtered = i.has_filter, + partition_count = ps.partition_count, + reserved_mb = CONVERT(decimal(19,2), ps.reserved_pages * 8.0 / 1024.0), + used_mb = CONVERT(decimal(19,2), ps.used_pages * 8.0 / 1024.0), + in_row_data_mb = CONVERT(decimal(19,2), ps.in_row_pages * 8.0 / 1024.0), + lob_data_mb = CONVERT(decimal(19,2), ps.lob_pages * 8.0 / 1024.0), + row_overflow_mb = CONVERT(decimal(19,2), ps.row_overflow_pages * 8.0 / 1024.0), + total_rows = ps.total_rows, + user_seeks = us.user_seeks, + user_scans = us.user_scans, + user_lookups = us.user_lookups, + user_updates = us.user_updates, + last_user_seek = us.last_user_seek, + last_user_scan = us.last_user_scan, + last_user_lookup = us.last_user_lookup, + last_user_update = us.last_user_update, + leaf_insert_count = os.leaf_insert_count, + leaf_update_count = os.leaf_update_count, + leaf_delete_count = os.leaf_delete_count, + range_scan_count = os.range_scan_count, + singleton_lookup_count = os.singleton_lookup_count, + row_lock_count = os.row_lock_count, + row_lock_wait_count = os.row_lock_wait_count, + row_lock_wait_in_ms = os.row_lock_wait_in_ms, + page_lock_count = os.page_lock_count, + page_lock_wait_count = os.page_lock_wait_count, + page_lock_wait_in_ms = os.page_lock_wait_in_ms, + index_lock_promotion_attempt_count = os.index_lock_promotion_attempt_count, + index_lock_promotion_count = os.index_lock_promotion_count, + page_latch_wait_count = os.page_latch_wait_count, + page_latch_wait_in_ms = os.page_latch_wait_in_ms, + page_io_latch_wait_count = os.page_io_latch_wait_count, + page_io_latch_wait_in_ms = os.page_io_latch_wait_in_ms +FROM sys.indexes AS i +JOIN sys.objects AS o + ON o.object_id = i.object_id +JOIN sys.schemas AS s + ON s.schema_id = o.schema_id +LEFT JOIN +( + SELECT + dps.object_id, dps.index_id, + partition_count = COUNT_BIG(*), + reserved_pages = SUM(dps.reserved_page_count), + used_pages = SUM(dps.used_page_count), + in_row_pages = SUM(dps.in_row_data_page_count), + lob_pages = SUM(dps.lob_used_page_count), + row_overflow_pages = SUM(dps.row_overflow_used_page_count), + total_rows = SUM(dps.row_count) + FROM sys.dm_db_partition_stats AS dps + GROUP BY dps.object_id, dps.index_id +) AS ps + ON ps.object_id = i.object_id AND ps.index_id = i.index_id +LEFT JOIN sys.dm_db_index_usage_stats AS us + ON us.database_id = DB_ID() AND us.object_id = i.object_id AND us.index_id = i.index_id +LEFT JOIN +( + SELECT + ios.object_id, ios.index_id, + leaf_insert_count = SUM(ios.leaf_insert_count), + leaf_update_count = SUM(ios.leaf_update_count), + leaf_delete_count = SUM(ios.leaf_delete_count), + range_scan_count = SUM(ios.range_scan_count), + singleton_lookup_count = SUM(ios.singleton_lookup_count), + row_lock_count = SUM(ios.row_lock_count), + row_lock_wait_count = SUM(ios.row_lock_wait_count), + row_lock_wait_in_ms = SUM(ios.row_lock_wait_in_ms), + page_lock_count = SUM(ios.page_lock_count), + page_lock_wait_count = SUM(ios.page_lock_wait_count), + page_lock_wait_in_ms = SUM(ios.page_lock_wait_in_ms), + index_lock_promotion_attempt_count = SUM(ios.index_lock_promotion_attempt_count), + index_lock_promotion_count = SUM(ios.index_lock_promotion_count), + page_latch_wait_count = SUM(ios.page_latch_wait_count), + page_latch_wait_in_ms = SUM(ios.page_latch_wait_in_ms), + page_io_latch_wait_count = SUM(ios.page_io_latch_wait_count), + page_io_latch_wait_in_ms = SUM(ios.page_io_latch_wait_in_ms) + FROM sys.dm_db_index_operational_stats(DB_ID(), NULL, NULL, NULL) AS ios + GROUP BY ios.object_id, ios.index_id +) AS os + ON os.object_id = i.object_id AND os.index_id = i.index_id +WHERE o.is_ms_shipped = 0 +AND o.type IN (N'U', N'V') +ORDER BY + reserved_mb DESC +OPTION(RECOMPILE);"; + + var serverId = GetServerId(server); + var serverName = GetServerNameForStorage(server); + var collectionTime = DateTime.UtcNow; + var rowsCollected = 0; + _lastSqlMs = 0; + _lastDuckDbMs = 0; + + var rows = new List(); + var sqlSw = Stopwatch.StartNew(); + + if (isAzureSqlDb) + { + var databases = await GetAzureDatabaseListAsync(server, cancellationToken); + foreach (var dbName in databases) + { + try + { + using var dbConn = await OpenAzureDatabaseConnectionAsync(server, dbName, cancellationToken); + using var cmd = new SqlCommand(azureSqlDbQuery, dbConn); + cmd.CommandTimeout = CommandTimeoutSeconds; + using var reader = await cmd.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + rows.Add(ReadIndexObjectStatRow(reader)); + } + } + catch (Exception ex) + { + _logger?.LogDebug("Skipping database '{Database}' for index/object stats: {Error}", dbName, ex.Message); + } + } + } + else + { + using var sqlConnection = await CreateConnectionAsync(server, cancellationToken); + using var command = new SqlCommand(onPremQuery, sqlConnection); + command.CommandTimeout = CommandTimeoutSeconds; + var (_, exclusionParams) = BuildDatabaseExclusionFilter(server.ExcludedDatabases, "d.name"); + foreach (var p in exclusionParams) command.Parameters.Add(p); + + using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + rows.Add(ReadIndexObjectStatRow(reader)); + } + } + sqlSw.Stop(); + + var duckSw = Stopwatch.StartNew(); + using (var duckConnection = _duckDb.CreateConnection()) + { + await duckConnection.OpenAsync(cancellationToken); + using (var appender = duckConnection.CreateAppender("index_object_stats")) + { + foreach (var r in rows) + { + appender.CreateRow() + .AppendValue(GenerateCollectionId()) + .AppendValue(collectionTime) + .AppendValue(serverId) + .AppendValue(serverName) + .AppendValue(r.SqlServerStartTime) + .AppendValue(r.DatabaseName) + .AppendValue(r.DatabaseId) + .AppendValue(r.SchemaName) + .AppendValue(r.ObjectId) + .AppendValue(r.TableName) + .AppendValue(r.IndexId) + .AppendValue(r.IndexName) + .AppendValue(r.IndexTypeDesc) + .AppendValue(r.IsUnique) + .AppendValue(r.IsPrimaryKey) + .AppendValue(r.IsFiltered) + .AppendValue(r.PartitionCount) + .AppendValue(r.ReservedMb) + .AppendValue(r.UsedMb) + .AppendValue(r.InRowDataMb) + .AppendValue(r.LobDataMb) + .AppendValue(r.RowOverflowMb) + .AppendValue(r.TotalRows) + .AppendValue(r.UserSeeks) + .AppendValue(r.UserScans) + .AppendValue(r.UserLookups) + .AppendValue(r.UserUpdates) + .AppendValue(r.LastUserSeek) + .AppendValue(r.LastUserScan) + .AppendValue(r.LastUserLookup) + .AppendValue(r.LastUserUpdate) + .AppendValue(r.LeafInsertCount) + .AppendValue(r.LeafUpdateCount) + .AppendValue(r.LeafDeleteCount) + .AppendValue(r.RangeScanCount) + .AppendValue(r.SingletonLookupCount) + .AppendValue(r.RowLockCount) + .AppendValue(r.RowLockWaitCount) + .AppendValue(r.RowLockWaitInMs) + .AppendValue(r.PageLockCount) + .AppendValue(r.PageLockWaitCount) + .AppendValue(r.PageLockWaitInMs) + .AppendValue(r.IndexLockPromotionAttemptCount) + .AppendValue(r.IndexLockPromotionCount) + .AppendValue(r.PageLatchWaitCount) + .AppendValue(r.PageLatchWaitInMs) + .AppendValue(r.PageIoLatchWaitCount) + .AppendValue(r.PageIoLatchWaitInMs) + .EndRow(); + rowsCollected++; + } + } + } + duckSw.Stop(); + + _lastSqlMs = sqlSw.ElapsedMilliseconds; + _lastDuckDbMs = duckSw.ElapsedMilliseconds; + + _logger?.LogDebug("Collected {RowCount} index/object stat rows for server '{Server}'", rowsCollected, server.DisplayName); + return rowsCollected; + } + + private static IndexObjectStatRow ReadIndexObjectStatRow(SqlDataReader reader) + { + long? L(int i) => reader.IsDBNull(i) ? null : Convert.ToInt64(reader.GetValue(i)); + int? I(int i) => reader.IsDBNull(i) ? null : Convert.ToInt32(reader.GetValue(i)); + decimal? D(int i) => reader.IsDBNull(i) ? null : reader.GetDecimal(i); + DateTime? T(int i) => reader.IsDBNull(i) ? null : reader.GetDateTime(i); + bool? B(int i) => reader.IsDBNull(i) ? null : (bool?)(Convert.ToInt32(reader.GetValue(i)) == 1); + + return new IndexObjectStatRow + { + SqlServerStartTime = T(0), + DatabaseName = reader.GetString(1), + DatabaseId = Convert.ToInt32(reader.GetValue(2)), + SchemaName = reader.GetString(3), + ObjectId = Convert.ToInt32(reader.GetValue(4)), + TableName = reader.GetString(5), + IndexId = Convert.ToInt32(reader.GetValue(6)), + IndexName = reader.IsDBNull(7) ? null : reader.GetString(7), + IndexTypeDesc = reader.IsDBNull(8) ? null : reader.GetString(8), + IsUnique = B(9), + IsPrimaryKey = B(10), + IsFiltered = B(11), + PartitionCount = I(12), + ReservedMb = D(13), + UsedMb = D(14), + InRowDataMb = D(15), + LobDataMb = D(16), + RowOverflowMb = D(17), + TotalRows = L(18), + UserSeeks = L(19), + UserScans = L(20), + UserLookups = L(21), + UserUpdates = L(22), + LastUserSeek = T(23), + LastUserScan = T(24), + LastUserLookup = T(25), + LastUserUpdate = T(26), + LeafInsertCount = L(27), + LeafUpdateCount = L(28), + LeafDeleteCount = L(29), + RangeScanCount = L(30), + SingletonLookupCount = L(31), + RowLockCount = L(32), + RowLockWaitCount = L(33), + RowLockWaitInMs = L(34), + PageLockCount = L(35), + PageLockWaitCount = L(36), + PageLockWaitInMs = L(37), + IndexLockPromotionAttemptCount = L(38), + IndexLockPromotionCount = L(39), + PageLatchWaitCount = L(40), + PageLatchWaitInMs = L(41), + PageIoLatchWaitCount = L(42), + PageIoLatchWaitInMs = L(43) + }; + } + + private sealed class IndexObjectStatRow + { + public DateTime? SqlServerStartTime { get; set; } + public string DatabaseName { get; set; } = ""; + public int DatabaseId { get; set; } + public string SchemaName { get; set; } = ""; + public int ObjectId { get; set; } + public string TableName { get; set; } = ""; + public int IndexId { get; set; } + public string? IndexName { get; set; } + public string? IndexTypeDesc { get; set; } + public bool? IsUnique { get; set; } + public bool? IsPrimaryKey { get; set; } + public bool? IsFiltered { get; set; } + public int? PartitionCount { get; set; } + public decimal? ReservedMb { get; set; } + public decimal? UsedMb { get; set; } + public decimal? InRowDataMb { get; set; } + public decimal? LobDataMb { get; set; } + public decimal? RowOverflowMb { get; set; } + public long? TotalRows { get; set; } + public long? UserSeeks { get; set; } + public long? UserScans { get; set; } + public long? UserLookups { get; set; } + public long? UserUpdates { get; set; } + public DateTime? LastUserSeek { get; set; } + public DateTime? LastUserScan { get; set; } + public DateTime? LastUserLookup { get; set; } + public DateTime? LastUserUpdate { get; set; } + public long? LeafInsertCount { get; set; } + public long? LeafUpdateCount { get; set; } + public long? LeafDeleteCount { get; set; } + public long? RangeScanCount { get; set; } + public long? SingletonLookupCount { get; set; } + public long? RowLockCount { get; set; } + public long? RowLockWaitCount { get; set; } + public long? RowLockWaitInMs { get; set; } + public long? PageLockCount { get; set; } + public long? PageLockWaitCount { get; set; } + public long? PageLockWaitInMs { get; set; } + public long? IndexLockPromotionAttemptCount { get; set; } + public long? IndexLockPromotionCount { get; set; } + public long? PageLatchWaitCount { get; set; } + public long? PageLatchWaitInMs { get; set; } + public long? PageIoLatchWaitCount { get; set; } + public long? PageIoLatchWaitInMs { get; set; } + } +} diff --git a/Lite/Services/RemoteCollectorService.cs b/Lite/Services/RemoteCollectorService.cs index 72c17b22..30697285 100644 --- a/Lite/Services/RemoteCollectorService.cs +++ b/Lite/Services/RemoteCollectorService.cs @@ -501,6 +501,7 @@ failed first attempt self-heals instead of staying broken until a manual "trace_flags" => await CollectTraceFlagsAsync(server, cancellationToken), "running_jobs" => await CollectRunningJobsAsync(server, cancellationToken), "database_size_stats" => await CollectDatabaseSizeStatsAsync(server, cancellationToken), + "index_object_stats" => await CollectIndexObjectStatsAsync(server, cancellationToken), "server_properties" => await CollectServerPropertiesAsync(server, cancellationToken), "session_stats" => await CollectSessionStatsAsync(server, cancellationToken), _ => throw new ArgumentException($"Unknown collector: {collectorName}") diff --git a/Lite/Services/ScheduleManager.cs b/Lite/Services/ScheduleManager.cs index 223c032c..020f5834 100644 --- a/Lite/Services/ScheduleManager.cs +++ b/Lite/Services/ScheduleManager.cs @@ -755,6 +755,7 @@ private static List GetDefaultSchedules() new() { Name = "trace_flags", Enabled = true, FrequencyMinutes = 0, RetentionDays = 30, Description = "Active trace flags via DBCC TRACESTATUS (on-load only)" }, new() { Name = "running_jobs", Enabled = true, FrequencyMinutes = 5, RetentionDays = 7, Description = "Currently running SQL Agent jobs with duration comparison" }, new() { Name = "database_size_stats", Enabled = true, FrequencyMinutes = 60, RetentionDays = 90, Description = "Database file sizes for growth trending and capacity planning" }, + new() { Name = "index_object_stats", Enabled = true, FrequencyMinutes = 1440, RetentionDays = 90, Description = "Per-object table/index size, usage, and locking stats for growth, unused-index, and contention analysis (daily collection)" }, new() { Name = "server_properties", Enabled = true, FrequencyMinutes = 0, RetentionDays = 365, Description = "Server edition, licensing, CPU/memory hardware metadata (on-load only)" }, new() { Name = "session_stats", Enabled = true, FrequencyMinutes = 5, RetentionDays = 30, Description = "Per-application session counts from sys.dm_exec_sessions" } }; diff --git a/install/02_create_tables.sql b/install/02_create_tables.sql index 4e56f859..8b680a26 100644 --- a/install/02_create_tables.sql +++ b/install/02_create_tables.sql @@ -1475,6 +1475,84 @@ BEGIN PRINT 'Created collect.database_size_stats table'; END; +/* +Index/Object Statistics Table (FinOps) +Per-table and per-index size, usage, and locking stats for growth trending, +unused-index detection, and contention analysis. Size columns are absolute +point-in-time values; usage and locking counters are cumulative (reset on +instance restart / DB detach / AUTO_CLOSE) - sqlserver_start_time carries the +reset boundary so deltas can be computed safely in the read layer. +*/ +IF OBJECT_ID(N'collect.index_object_stats', N'U') IS NULL +BEGIN + CREATE TABLE + collect.index_object_stats + ( + collection_id bigint IDENTITY NOT NULL, + collection_time datetime2(7) NOT NULL + DEFAULT SYSDATETIME(), + sqlserver_start_time datetime2(7) NULL, + database_name sysname NOT NULL, + database_id integer NOT NULL, + schema_name sysname NOT NULL, + object_id integer NOT NULL, + table_name sysname NOT NULL, + index_id integer NOT NULL, + index_name sysname NULL, + index_type_desc nvarchar(60) NULL, + is_unique bit NULL, + is_primary_key bit NULL, + is_filtered bit NULL, + partition_count integer NULL, + reserved_mb decimal(19,2) NULL, + used_mb decimal(19,2) NULL, + in_row_data_mb decimal(19,2) NULL, + lob_data_mb decimal(19,2) NULL, + row_overflow_mb decimal(19,2) NULL, + total_rows bigint NULL, + user_seeks bigint NULL, + user_scans bigint NULL, + user_lookups bigint NULL, + user_updates bigint NULL, + last_user_seek datetime2(7) NULL, + last_user_scan datetime2(7) NULL, + last_user_lookup datetime2(7) NULL, + last_user_update datetime2(7) NULL, + leaf_insert_count bigint NULL, + leaf_update_count bigint NULL, + leaf_delete_count bigint NULL, + range_scan_count bigint NULL, + singleton_lookup_count bigint NULL, + row_lock_count bigint NULL, + row_lock_wait_count bigint NULL, + row_lock_wait_in_ms bigint NULL, + page_lock_count bigint NULL, + page_lock_wait_count bigint NULL, + page_lock_wait_in_ms bigint NULL, + index_lock_promotion_attempt_count bigint NULL, + index_lock_promotion_count bigint NULL, + page_latch_wait_count bigint NULL, + page_latch_wait_in_ms bigint NULL, + page_io_latch_wait_count bigint NULL, + page_io_latch_wait_in_ms bigint NULL, + /*Analysis helpers - computed columns*/ + total_reads AS + ( + ISNULL(user_seeks, 0) + + ISNULL(user_scans, 0) + + ISNULL(user_lookups, 0) + ), + CONSTRAINT + PK_index_object_stats + PRIMARY KEY CLUSTERED + (collection_time, collection_id) + WITH + (DATA_COMPRESSION = PAGE) + ); + + PRINT 'Created collect.index_object_stats table'; +END; + /* Server Properties Table (FinOps) */ @@ -1594,5 +1672,25 @@ BEGIN PRINT 'Created collect.query_store_data.IX_query_store_data_id_lookup index'; END; +IF NOT EXISTS +( + SELECT + 1/0 + FROM sys.indexes + WHERE object_id = OBJECT_ID(N'collect.index_object_stats') + AND name = N'IX_index_object_stats_object_lookup' +) +BEGIN + SET @index_sql = N' + CREATE NONCLUSTERED INDEX + IX_index_object_stats_object_lookup + ON collect.index_object_stats + (database_name, object_id, index_id, collection_time DESC) + WITH + (SORT_IN_TEMPDB = ON, DATA_COMPRESSION = PAGE' + @online_option + N');'; + EXEC sys.sp_executesql @index_sql; + PRINT 'Created collect.index_object_stats.IX_index_object_stats_object_lookup index'; +END; + PRINT 'All collection tables created successfully'; GO diff --git a/install/04_create_schedule_table.sql b/install/04_create_schedule_table.sql index 80c8ece4..32e9859b 100644 --- a/install/04_create_schedule_table.sql +++ b/install/04_create_schedule_table.sql @@ -76,6 +76,7 @@ FROM (N'waiting_tasks_collector', 1, 1, 2, 30, N'Currently waiting tasks - blocking chains and wait analysis'), (N'running_jobs_collector', 1, 1, 2, 7, N'Currently running SQL Agent jobs with historical duration comparison'), (N'database_size_stats_collector', 1, 60, 10, 90, N'Database file sizes for growth trending and capacity planning'), + (N'index_object_stats_collector', 1, 1440, 15, 90, N'Per-object table/index size, usage, and locking stats for growth, unused-index, and contention analysis (daily collection)'), (N'server_properties_collector', 1, 1440, 5, 365, N'Server edition, licensing, CPU/memory hardware metadata for license audit') ) AS v (collector_name, enabled, frequency_minutes, max_duration_minutes, retention_days, description) WHERE NOT EXISTS diff --git a/install/06_ensure_collection_table.sql b/install/06_ensure_collection_table.sql index e78e5bb9..10e93401 100644 --- a/install/06_ensure_collection_table.sql +++ b/install/06_ensure_collection_table.sql @@ -1133,6 +1133,73 @@ BEGIN (DATA_COMPRESSION = PAGE) ); + END; + ELSE IF @table_name = N'index_object_stats' + BEGIN + CREATE TABLE + collect.index_object_stats + ( + collection_id bigint IDENTITY NOT NULL, + collection_time datetime2(7) NOT NULL + DEFAULT SYSDATETIME(), + sqlserver_start_time datetime2(7) NULL, + database_name sysname NOT NULL, + database_id integer NOT NULL, + schema_name sysname NOT NULL, + object_id integer NOT NULL, + table_name sysname NOT NULL, + index_id integer NOT NULL, + index_name sysname NULL, + index_type_desc nvarchar(60) NULL, + is_unique bit NULL, + is_primary_key bit NULL, + is_filtered bit NULL, + partition_count integer NULL, + reserved_mb decimal(19,2) NULL, + used_mb decimal(19,2) NULL, + in_row_data_mb decimal(19,2) NULL, + lob_data_mb decimal(19,2) NULL, + row_overflow_mb decimal(19,2) NULL, + total_rows bigint NULL, + user_seeks bigint NULL, + user_scans bigint NULL, + user_lookups bigint NULL, + user_updates bigint NULL, + last_user_seek datetime2(7) NULL, + last_user_scan datetime2(7) NULL, + last_user_lookup datetime2(7) NULL, + last_user_update datetime2(7) NULL, + leaf_insert_count bigint NULL, + leaf_update_count bigint NULL, + leaf_delete_count bigint NULL, + range_scan_count bigint NULL, + singleton_lookup_count bigint NULL, + row_lock_count bigint NULL, + row_lock_wait_count bigint NULL, + row_lock_wait_in_ms bigint NULL, + page_lock_count bigint NULL, + page_lock_wait_count bigint NULL, + page_lock_wait_in_ms bigint NULL, + index_lock_promotion_attempt_count bigint NULL, + index_lock_promotion_count bigint NULL, + page_latch_wait_count bigint NULL, + page_latch_wait_in_ms bigint NULL, + page_io_latch_wait_count bigint NULL, + page_io_latch_wait_in_ms bigint NULL, + total_reads AS + ( + ISNULL(user_seeks, 0) + + ISNULL(user_scans, 0) + + ISNULL(user_lookups, 0) + ), + CONSTRAINT + PK_index_object_stats + PRIMARY KEY CLUSTERED + (collection_time, collection_id) + WITH + (DATA_COMPRESSION = PAGE) + ); + END; ELSE IF @table_name = N'server_properties' BEGIN @@ -1172,7 +1239,7 @@ BEGIN END; ELSE BEGIN - SET @error_message = N'Unknown table name: ' + @table_name + N'. Valid table names are: wait_stats, query_stats, memory_stats, memory_pressure_events, deadlock_xml, blocked_process_xml, procedure_stats, query_snapshots, query_store_data, trace_analysis, default_trace_events, file_io_stats, memory_grant_stats, cpu_scheduler_stats, memory_clerks_stats, perfmon_stats, cpu_utilization_stats, blocking_deadlock_stats, latch_stats, spinlock_stats, tempdb_stats, plan_cache_stats, session_stats, waiting_tasks, running_jobs, database_size_stats, server_properties'; + SET @error_message = N'Unknown table name: ' + @table_name + N'. Valid table names are: wait_stats, query_stats, memory_stats, memory_pressure_events, deadlock_xml, blocked_process_xml, procedure_stats, query_snapshots, query_store_data, trace_analysis, default_trace_events, file_io_stats, memory_grant_stats, cpu_scheduler_stats, memory_clerks_stats, perfmon_stats, cpu_utilization_stats, blocking_deadlock_stats, latch_stats, spinlock_stats, tempdb_stats, plan_cache_stats, session_stats, waiting_tasks, running_jobs, database_size_stats, index_object_stats, server_properties'; RAISERROR(@error_message, 16, 1); RETURN; END; diff --git a/install/42_scheduled_master_collector.sql b/install/42_scheduled_master_collector.sql index 48df380a..c1a07b2e 100644 --- a/install/42_scheduled_master_collector.sql +++ b/install/42_scheduled_master_collector.sql @@ -325,6 +325,10 @@ BEGIN BEGIN EXECUTE collect.database_size_stats_collector @debug = @debug; END; + ELSE IF @collector_name = N'index_object_stats_collector' + BEGIN + EXECUTE collect.index_object_stats_collector @debug = @debug; + END; ELSE IF @collector_name = N'server_properties_collector' BEGIN EXECUTE collect.server_properties_collector @debug = @debug; diff --git a/install/55_collect_index_object_stats.sql b/install/55_collect_index_object_stats.sql new file mode 100644 index 00000000..67f92024 --- /dev/null +++ b/install/55_collect_index_object_stats.sql @@ -0,0 +1,581 @@ +/* +Copyright 2026 Darling Data, LLC +https://www.erikdarling.com/ + +*/ + +SET ANSI_NULLS ON; +SET ANSI_PADDING ON; +SET ANSI_WARNINGS ON; +SET ARITHABORT ON; +SET CONCAT_NULL_YIELDS_NULL ON; +SET QUOTED_IDENTIFIER ON; +SET NUMERIC_ROUNDABORT OFF; +SET IMPLICIT_TRANSACTIONS OFF; +SET STATISTICS TIME, IO OFF; +GO + +USE PerformanceMonitor; +GO + +/******************************************************************************* +Collector: index_object_stats_collector +Purpose: Captures per-table and per-index size, usage, and locking statistics + for growth trending, unused-index detection, and contention analysis. +Collection Type: Point-in-time snapshot for sizes; cumulative counters for + usage/locking (deltas derived in the read layer using + sqlserver_start_time as the reset boundary). +Target Table: collect.index_object_stats +Frequency: Every 1440 minutes (daily) - object grain is high volume. +Dependencies: sys.dm_db_partition_stats, sys.dm_db_index_usage_stats, + sys.dm_db_index_operational_stats, sys.indexes, sys.objects, sys.schemas +Notes: All three DMVs are database-scoped, so the full join executes inside each + database's context via dynamic SQL. Azure SQL DB (engine edition 5) + collects only the connected database (no cross-database enumeration). + In-Memory OLTP (Hekaton) objects are not represented by these DMVs. +*******************************************************************************/ + +IF OBJECT_ID(N'collect.index_object_stats_collector', N'P') IS NULL +BEGIN + EXECUTE(N'CREATE PROCEDURE collect.index_object_stats_collector AS RETURN 138;'); +END; +GO + +ALTER PROCEDURE + collect.index_object_stats_collector +( + @debug bit = 0 /*Print debugging information*/ +) +WITH RECOMPILE +AS +BEGIN + SET NOCOUNT ON; + SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; + + DECLARE + @rows_collected bigint = 0, + @start_time datetime2(7) = SYSDATETIME(), + @sqlserver_start_time datetime2(7), + @error_message nvarchar(4000), + @engine_edition integer = + CONVERT(integer, SERVERPROPERTY(N'EngineEdition')); + + BEGIN TRY + /* + Ensure target table exists + */ + IF OBJECT_ID(N'collect.index_object_stats', N'U') IS NULL + BEGIN + INSERT INTO + config.collection_log + ( + collection_time, + collector_name, + collection_status, + rows_collected, + duration_ms, + error_message + ) + VALUES + ( + @start_time, + N'index_object_stats_collector', + N'TABLE_MISSING', + 0, + 0, + N'Table collect.index_object_stats does not exist, calling ensure procedure' + ); + + EXECUTE config.ensure_collection_table + @table_name = N'index_object_stats', + @debug = @debug; + + IF OBJECT_ID(N'collect.index_object_stats', N'U') IS NULL + BEGIN + RAISERROR(N'Table collect.index_object_stats still missing after ensure procedure', 16, 1); + RETURN; + END; + END; + + /* + Reset boundary for cumulative usage/locking counters + */ + SELECT + @sqlserver_start_time = osi.sqlserver_start_time + FROM sys.dm_os_sys_info AS osi; + + /* + Azure SQL DB: single database scope (no cross-database enumeration) + */ + IF @engine_edition = 5 + BEGIN + INSERT INTO + collect.index_object_stats + ( + collection_time, + sqlserver_start_time, + database_name, + database_id, + schema_name, + object_id, + table_name, + index_id, + index_name, + index_type_desc, + is_unique, + is_primary_key, + is_filtered, + partition_count, + reserved_mb, + used_mb, + in_row_data_mb, + lob_data_mb, + row_overflow_mb, + total_rows, + user_seeks, + user_scans, + user_lookups, + user_updates, + last_user_seek, + last_user_scan, + last_user_lookup, + last_user_update, + leaf_insert_count, + leaf_update_count, + leaf_delete_count, + range_scan_count, + singleton_lookup_count, + row_lock_count, + row_lock_wait_count, + row_lock_wait_in_ms, + page_lock_count, + page_lock_wait_count, + page_lock_wait_in_ms, + index_lock_promotion_attempt_count, + index_lock_promotion_count, + page_latch_wait_count, + page_latch_wait_in_ms, + page_io_latch_wait_count, + page_io_latch_wait_in_ms + ) + SELECT + collection_time = @start_time, + sqlserver_start_time = @sqlserver_start_time, + database_name = DB_NAME(), + database_id = DB_ID(), + schema_name = s.name, + object_id = o.object_id, + table_name = o.name, + index_id = i.index_id, + index_name = i.name, + index_type_desc = i.type_desc, + is_unique = i.is_unique, + is_primary_key = i.is_primary_key, + is_filtered = i.has_filter, + partition_count = ps.partition_count, + reserved_mb = CONVERT(decimal(19,2), ps.reserved_pages * 8.0 / 1024.0), + used_mb = CONVERT(decimal(19,2), ps.used_pages * 8.0 / 1024.0), + in_row_data_mb = CONVERT(decimal(19,2), ps.in_row_pages * 8.0 / 1024.0), + lob_data_mb = CONVERT(decimal(19,2), ps.lob_pages * 8.0 / 1024.0), + row_overflow_mb = CONVERT(decimal(19,2), ps.row_overflow_pages * 8.0 / 1024.0), + total_rows = ps.total_rows, + user_seeks = us.user_seeks, + user_scans = us.user_scans, + user_lookups = us.user_lookups, + user_updates = us.user_updates, + last_user_seek = us.last_user_seek, + last_user_scan = us.last_user_scan, + last_user_lookup = us.last_user_lookup, + last_user_update = us.last_user_update, + leaf_insert_count = os.leaf_insert_count, + leaf_update_count = os.leaf_update_count, + leaf_delete_count = os.leaf_delete_count, + range_scan_count = os.range_scan_count, + singleton_lookup_count = os.singleton_lookup_count, + row_lock_count = os.row_lock_count, + row_lock_wait_count = os.row_lock_wait_count, + row_lock_wait_in_ms = os.row_lock_wait_in_ms, + page_lock_count = os.page_lock_count, + page_lock_wait_count = os.page_lock_wait_count, + page_lock_wait_in_ms = os.page_lock_wait_in_ms, + index_lock_promotion_attempt_count = os.index_lock_promotion_attempt_count, + index_lock_promotion_count = os.index_lock_promotion_count, + page_latch_wait_count = os.page_latch_wait_count, + page_latch_wait_in_ms = os.page_latch_wait_in_ms, + page_io_latch_wait_count = os.page_io_latch_wait_count, + page_io_latch_wait_in_ms = os.page_io_latch_wait_in_ms + FROM sys.indexes AS i + JOIN sys.objects AS o + ON o.object_id = i.object_id + JOIN sys.schemas AS s + ON s.schema_id = o.schema_id + LEFT JOIN + ( + SELECT + dps.object_id, + dps.index_id, + partition_count = COUNT_BIG(*), + reserved_pages = SUM(dps.reserved_page_count), + used_pages = SUM(dps.used_page_count), + in_row_pages = SUM(dps.in_row_data_page_count), + lob_pages = SUM(dps.lob_used_page_count), + row_overflow_pages = SUM(dps.row_overflow_used_page_count), + total_rows = SUM(dps.row_count) + FROM sys.dm_db_partition_stats AS dps + GROUP BY + dps.object_id, + dps.index_id + ) AS ps + ON ps.object_id = i.object_id + AND ps.index_id = i.index_id + LEFT JOIN sys.dm_db_index_usage_stats AS us + ON us.database_id = DB_ID() + AND us.object_id = i.object_id + AND us.index_id = i.index_id + LEFT JOIN + ( + SELECT + ios.object_id, + ios.index_id, + leaf_insert_count = SUM(ios.leaf_insert_count), + leaf_update_count = SUM(ios.leaf_update_count), + leaf_delete_count = SUM(ios.leaf_delete_count), + range_scan_count = SUM(ios.range_scan_count), + singleton_lookup_count = SUM(ios.singleton_lookup_count), + row_lock_count = SUM(ios.row_lock_count), + row_lock_wait_count = SUM(ios.row_lock_wait_count), + row_lock_wait_in_ms = SUM(ios.row_lock_wait_in_ms), + page_lock_count = SUM(ios.page_lock_count), + page_lock_wait_count = SUM(ios.page_lock_wait_count), + page_lock_wait_in_ms = SUM(ios.page_lock_wait_in_ms), + index_lock_promotion_attempt_count = SUM(ios.index_lock_promotion_attempt_count), + index_lock_promotion_count = SUM(ios.index_lock_promotion_count), + page_latch_wait_count = SUM(ios.page_latch_wait_count), + page_latch_wait_in_ms = SUM(ios.page_latch_wait_in_ms), + page_io_latch_wait_count = SUM(ios.page_io_latch_wait_count), + page_io_latch_wait_in_ms = SUM(ios.page_io_latch_wait_in_ms) + FROM sys.dm_db_index_operational_stats(DB_ID(), NULL, NULL, NULL) AS ios + GROUP BY + ios.object_id, + ios.index_id + ) AS os + ON os.object_id = i.object_id + AND os.index_id = i.index_id + WHERE o.is_ms_shipped = 0 + AND o.type IN (N'U', N'V') + OPTION(RECOMPILE); + + SET @rows_collected = ROWCOUNT_BIG(); + END; + ELSE + BEGIN + /* + On-prem / Azure MI / AWS RDS: cursor over all online databases. + All three DMVs are database-scoped, so the entire join runs inside + each database's context and inserts into the fully-qualified target. + */ + DECLARE + @db_name sysname, + @db_id integer, + @sql nvarchar(max), + @exec_sql nvarchar(max); + + DECLARE db_cursor CURSOR LOCAL FAST_FORWARD FOR + SELECT + d.name, + d.database_id + FROM sys.databases AS d + WHERE d.state = 0 /*ONLINE only - skip RESTORING (mirroring/AG secondary)*/ + AND d.database_id > 0 + AND HAS_DBACCESS(d.name) = 1 + AND NOT EXISTS + ( + SELECT + 1/0 + FROM config.collector_database_exclusions AS e + WHERE e.database_name = d.name + ) + ORDER BY + d.database_id; + + OPEN db_cursor; + FETCH NEXT FROM db_cursor INTO @db_name, @db_id; + + WHILE @@FETCH_STATUS = 0 + BEGIN + BEGIN TRY + SET @sql = N' + INSERT INTO + PerformanceMonitor.collect.index_object_stats + ( + collection_time, + sqlserver_start_time, + database_name, + database_id, + schema_name, + object_id, + table_name, + index_id, + index_name, + index_type_desc, + is_unique, + is_primary_key, + is_filtered, + partition_count, + reserved_mb, + used_mb, + in_row_data_mb, + lob_data_mb, + row_overflow_mb, + total_rows, + user_seeks, + user_scans, + user_lookups, + user_updates, + last_user_seek, + last_user_scan, + last_user_lookup, + last_user_update, + leaf_insert_count, + leaf_update_count, + leaf_delete_count, + range_scan_count, + singleton_lookup_count, + row_lock_count, + row_lock_wait_count, + row_lock_wait_in_ms, + page_lock_count, + page_lock_wait_count, + page_lock_wait_in_ms, + index_lock_promotion_attempt_count, + index_lock_promotion_count, + page_latch_wait_count, + page_latch_wait_in_ms, + page_io_latch_wait_count, + page_io_latch_wait_in_ms + ) + SELECT + collection_time = @start_time, + sqlserver_start_time = @sqlserver_start_time, + database_name = DB_NAME(), + database_id = DB_ID(), + schema_name = s.name, + object_id = o.object_id, + table_name = o.name, + index_id = i.index_id, + index_name = i.name, + index_type_desc = i.type_desc, + is_unique = i.is_unique, + is_primary_key = i.is_primary_key, + is_filtered = i.has_filter, + partition_count = ps.partition_count, + reserved_mb = CONVERT(decimal(19,2), ps.reserved_pages * 8.0 / 1024.0), + used_mb = CONVERT(decimal(19,2), ps.used_pages * 8.0 / 1024.0), + in_row_data_mb = CONVERT(decimal(19,2), ps.in_row_pages * 8.0 / 1024.0), + lob_data_mb = CONVERT(decimal(19,2), ps.lob_pages * 8.0 / 1024.0), + row_overflow_mb = CONVERT(decimal(19,2), ps.row_overflow_pages * 8.0 / 1024.0), + total_rows = ps.total_rows, + user_seeks = us.user_seeks, + user_scans = us.user_scans, + user_lookups = us.user_lookups, + user_updates = us.user_updates, + last_user_seek = us.last_user_seek, + last_user_scan = us.last_user_scan, + last_user_lookup = us.last_user_lookup, + last_user_update = us.last_user_update, + leaf_insert_count = os.leaf_insert_count, + leaf_update_count = os.leaf_update_count, + leaf_delete_count = os.leaf_delete_count, + range_scan_count = os.range_scan_count, + singleton_lookup_count = os.singleton_lookup_count, + row_lock_count = os.row_lock_count, + row_lock_wait_count = os.row_lock_wait_count, + row_lock_wait_in_ms = os.row_lock_wait_in_ms, + page_lock_count = os.page_lock_count, + page_lock_wait_count = os.page_lock_wait_count, + page_lock_wait_in_ms = os.page_lock_wait_in_ms, + index_lock_promotion_attempt_count = os.index_lock_promotion_attempt_count, + index_lock_promotion_count = os.index_lock_promotion_count, + page_latch_wait_count = os.page_latch_wait_count, + page_latch_wait_in_ms = os.page_latch_wait_in_ms, + page_io_latch_wait_count = os.page_io_latch_wait_count, + page_io_latch_wait_in_ms = os.page_io_latch_wait_in_ms + FROM sys.indexes AS i + JOIN sys.objects AS o + ON o.object_id = i.object_id + JOIN sys.schemas AS s + ON s.schema_id = o.schema_id + LEFT JOIN + ( + SELECT + dps.object_id, + dps.index_id, + partition_count = COUNT_BIG(*), + reserved_pages = SUM(dps.reserved_page_count), + used_pages = SUM(dps.used_page_count), + in_row_pages = SUM(dps.in_row_data_page_count), + lob_pages = SUM(dps.lob_used_page_count), + row_overflow_pages = SUM(dps.row_overflow_used_page_count), + total_rows = SUM(dps.row_count) + FROM sys.dm_db_partition_stats AS dps + GROUP BY + dps.object_id, + dps.index_id + ) AS ps + ON ps.object_id = i.object_id + AND ps.index_id = i.index_id + LEFT JOIN sys.dm_db_index_usage_stats AS us + ON us.database_id = DB_ID() + AND us.object_id = i.object_id + AND us.index_id = i.index_id + LEFT JOIN + ( + SELECT + ios.object_id, + ios.index_id, + leaf_insert_count = SUM(ios.leaf_insert_count), + leaf_update_count = SUM(ios.leaf_update_count), + leaf_delete_count = SUM(ios.leaf_delete_count), + range_scan_count = SUM(ios.range_scan_count), + singleton_lookup_count = SUM(ios.singleton_lookup_count), + row_lock_count = SUM(ios.row_lock_count), + row_lock_wait_count = SUM(ios.row_lock_wait_count), + row_lock_wait_in_ms = SUM(ios.row_lock_wait_in_ms), + page_lock_count = SUM(ios.page_lock_count), + page_lock_wait_count = SUM(ios.page_lock_wait_count), + page_lock_wait_in_ms = SUM(ios.page_lock_wait_in_ms), + index_lock_promotion_attempt_count = SUM(ios.index_lock_promotion_attempt_count), + index_lock_promotion_count = SUM(ios.index_lock_promotion_count), + page_latch_wait_count = SUM(ios.page_latch_wait_count), + page_latch_wait_in_ms = SUM(ios.page_latch_wait_in_ms), + page_io_latch_wait_count = SUM(ios.page_io_latch_wait_count), + page_io_latch_wait_in_ms = SUM(ios.page_io_latch_wait_in_ms) + FROM sys.dm_db_index_operational_stats(DB_ID(), NULL, NULL, NULL) AS ios + GROUP BY + ios.object_id, + ios.index_id + ) AS os + ON os.object_id = i.object_id + AND os.index_id = i.index_id + WHERE o.is_ms_shipped = 0 + AND o.type IN (N''U'', N''V'') + OPTION(RECOMPILE);'; + + SET @exec_sql = QUOTENAME(@db_name) + N'.sys.sp_executesql'; + + EXECUTE @exec_sql + @sql, + N'@start_time datetime2(7), @sqlserver_start_time datetime2(7)', + @start_time = @start_time, + @sqlserver_start_time = @sqlserver_start_time; + + SET @rows_collected = @rows_collected + ROWCOUNT_BIG(); + END TRY + BEGIN CATCH + /* + Log per-database errors but continue with remaining databases + */ + IF @debug = 1 + BEGIN + DECLARE @db_error_message nvarchar(4000) = ERROR_MESSAGE(); + RAISERROR(N'Error collecting index/object stats for database [%s]: %s', 0, 1, @db_name, @db_error_message) WITH NOWAIT; + END; + END CATCH; + + FETCH NEXT FROM db_cursor INTO @db_name, @db_id; + END; + + CLOSE db_cursor; + DEALLOCATE db_cursor; + END; + + /* + Debug output + */ + IF @debug = 1 + BEGIN + RAISERROR(N'Collected %I64d index/object stat rows', 0, 1, @rows_collected) WITH NOWAIT; + + SELECT TOP (20) + ios.database_name, + ios.schema_name, + ios.table_name, + ios.index_name, + ios.index_type_desc, + ios.reserved_mb, + ios.total_rows, + ios.total_reads, + ios.user_updates, + ios.row_lock_wait_in_ms, + ios.index_lock_promotion_count + FROM collect.index_object_stats AS ios + WHERE ios.collection_time = @start_time + ORDER BY + ios.reserved_mb DESC; + END; + + /* + Log successful collection + */ + INSERT INTO + config.collection_log + ( + collector_name, + collection_status, + rows_collected, + duration_ms + ) + VALUES + ( + N'index_object_stats_collector', + N'SUCCESS', + @rows_collected, + DATEDIFF(MILLISECOND, @start_time, SYSDATETIME()) + ); + + END TRY + BEGIN CATCH + IF @@TRANCOUNT > 0 + BEGIN + ROLLBACK TRANSACTION; + END; + + /* + Clean up cursor if open + */ + IF CURSOR_STATUS(N'local', N'db_cursor') >= 0 + BEGIN + CLOSE db_cursor; + DEALLOCATE db_cursor; + END; + + SET @error_message = ERROR_MESSAGE(); + + /* + Log the error + */ + INSERT INTO + config.collection_log + ( + collector_name, + collection_status, + duration_ms, + error_message + ) + VALUES + ( + N'index_object_stats_collector', + N'ERROR', + DATEDIFF(MILLISECOND, @start_time, SYSDATETIME()), + @error_message + ); + + RAISERROR(N'Error in index/object stats collector: %s', 16, 1, @error_message); + END CATCH; +END; +GO + +PRINT 'Index/object stats collector created successfully'; +PRINT 'Captures per-table/per-index size, usage, and locking stats for trending and contention analysis'; +PRINT 'Use: EXECUTE collect.index_object_stats_collector @debug = 1;'; +GO From 42b3452cef21f1a96cb63605faea62daa9d96a30 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 11 Jun 2026 22:56:48 +0200 Subject: [PATCH 2/5] Add object/index read layer + FinOps UI tabs (#1103) Read layer (both apps): GetObjectSizeGrowthAsync (per-table size + 7d/30d growth + daily rate, indexes rolled up), GetIndexUsageAsync (seeks/scans/ lookups/updates + Unused/Write-only/Active classification), GetIndexLockingAsync (row/page lock waits, escalations, latch waits, top contended). Dashboard reads collect.index_object_stats; Lite reads v_index_object_stats. Dashboard growth queries validated live on SQL 2016. UI: three FinOps sub-tabs in both apps - "Object Sizes & Growth", "Index Usage", "Locking & Contention" - mirroring the existing Storage Growth grid pattern (sortable columns, column filters on identity columns, refresh, count indicators). Co-Authored-By: Claude Opus 4.8 (1M context) --- Dashboard/Controls/FinOpsContent.xaml | 180 ++++++++ Dashboard/Controls/FinOpsContent.xaml.cs | 70 +++ .../DatabaseService.FinOps.IndexObjects.cs | 427 ++++++++++++++++++ Lite/Controls/FinOpsTab.xaml | 177 ++++++++ Lite/Controls/FinOpsTab.xaml.cs | 82 ++++ .../LocalDataService.FinOps.IndexObjects.cs | 319 +++++++++++++ 6 files changed, 1255 insertions(+) create mode 100644 Dashboard/Services/DatabaseService.FinOps.IndexObjects.cs create mode 100644 Lite/Services/LocalDataService.FinOps.IndexObjects.cs diff --git a/Dashboard/Controls/FinOpsContent.xaml b/Dashboard/Controls/FinOpsContent.xaml index 157e1eeb..b0b31c9d 100644 --- a/Dashboard/Controls/FinOpsContent.xaml +++ b/Dashboard/Controls/FinOpsContent.xaml @@ -913,6 +913,186 @@ + + + + + + + + + +