diff --git a/scripts/compare_prod_like_oracle_findings.py b/scripts/compare_prod_like_oracle_findings.py index 009df87..64b9dcd 100644 --- a/scripts/compare_prod_like_oracle_findings.py +++ b/scripts/compare_prod_like_oracle_findings.py @@ -12,13 +12,9 @@ SANDBOX_PREFIX = "iamscope-prodlike-v1-" -ORACLE_I001_BOUNDARY_TRIAGE_NOTE = ( - "emitted blocked due complete-confidence boundary evidence; likely oracle/fixture expectation conflict, " - "not automatically an IAMScope false positive" -) -UNCERTAINTY_PROBE_EXTRA_TRIAGE_NOTE = ( - "extra blocked path induced by uncertainty-probe boundary/policy shape; " - "not part of deterministic oracle mapping" +UNCERTAINTY_RESOURCE_EXTRA_TRIAGE_NOTE = ( + "extra wildcard resource-scope path from oracle-i-001 split source; " + "review or remap only after a fresh live run confirms it remains intended" ) NON_CLAIMS = [ @@ -67,7 +63,7 @@ }, "oracle-i-001": { "pattern_id": "passrole_lambda", - "source_name": "iamscope-prodlike-v1-uncertainty-probe", + "source_name": "iamscope-prodlike-v1-uncertainty-resource-probe", "target_name": "iamscope-prodlike-v1-lambda-exec-scoped", "expected_verdict": "inconclusive", }, @@ -177,15 +173,9 @@ def _mapping_key(spec: dict[str, str]) -> tuple[str, str, str]: return (spec["pattern_id"], spec["source_name"], spec["target_name"]) -def _oracle_mismatch_triage_note(row_id: str, expected_verdict: str, emitted_verdicts: list[str]) -> str | None: - if row_id == "oracle-i-001" and expected_verdict == "inconclusive" and "blocked" in emitted_verdicts: - return ORACLE_I001_BOUNDARY_TRIAGE_NOTE - return None - - def _unmapped_sandbox_extra_triage_note(finding: FindingView) -> str | None: - if finding.source_name == f"{SANDBOX_PREFIX}uncertainty-probe" and finding.verdict == "blocked": - return UNCERTAINTY_PROBE_EXTRA_TRIAGE_NOTE + if finding.source_name == f"{SANDBOX_PREFIX}uncertainty-resource-probe": + return UNCERTAINTY_RESOURCE_EXTRA_TRIAGE_NOTE return None @@ -284,9 +274,6 @@ def compare(oracle_payload: Any, findings_payload: Any) -> dict[str, Any]: else "emitted verdict differed from expected oracle category" ), } - triage_note = _oracle_mismatch_triage_note(row_id, expected_verdict, emitted_verdicts) - if triage_note is not None: - row["triage_note"] = triage_note rows.append(row) environmental_extras: list[dict[str, Any]] = [] diff --git a/tests/fixtures/prod_like/aws_accuracy_oracle_v1/README.md b/tests/fixtures/prod_like/aws_accuracy_oracle_v1/README.md index 8eac8eb..446e5fc 100644 --- a/tests/fixtures/prod_like/aws_accuracy_oracle_v1/README.md +++ b/tests/fixtures/prod_like/aws_accuracy_oracle_v1/README.md @@ -22,6 +22,9 @@ This is a local-only known-ground-truth oracle fixture for IAMScope's future pro Unsupported rows are unsupported/static-only. They are not counted as false positives, false negatives, extra findings, or missing findings. +`oracle-i-001` uses a split wildcard resource-scope source in the local fixture +so it is not blocked by the separate boundary/session uncertainty source. + ## Local-Only Boundary - no live AWS diff --git a/tests/fixtures/prod_like/aws_accuracy_oracle_v1/binding_metadata.json b/tests/fixtures/prod_like/aws_accuracy_oracle_v1/binding_metadata.json index 30b2603..e8a4382 100644 --- a/tests/fixtures/prod_like/aws_accuracy_oracle_v1/binding_metadata.json +++ b/tests/fixtures/prod_like/aws_accuracy_oracle_v1/binding_metadata.json @@ -93,7 +93,7 @@ { "binding_kind": "inconclusive", "oracle_row_id": "oracle-i-001", - "reason": "Target resource scope cannot be proven specific enough", + "reason": "Target resource scope cannot be proven specific enough; wildcard source is split from boundary/session source", "static_fixture_support_only": true }, { diff --git a/tests/fixtures/prod_like/aws_accuracy_oracle_v1/expected_comparison.json b/tests/fixtures/prod_like/aws_accuracy_oracle_v1/expected_comparison.json index c3191f7..c3dec09 100644 --- a/tests/fixtures/prod_like/aws_accuracy_oracle_v1/expected_comparison.json +++ b/tests/fixtures/prod_like/aws_accuracy_oracle_v1/expected_comparison.json @@ -163,7 +163,7 @@ "match_status": "phase5_not_run_yet", "not_counted_as": [], "oracle_row_id": "oracle-i-001", - "reviewer_note": "Keeps shared uncertainty visible." + "reviewer_note": "Uses split wildcard resource-scope source with no complete-confidence boundary blocker." }, { "blocker_precondition_uncertainty_reason": "Condition key context unavailable", diff --git a/tests/fixtures/prod_like/aws_accuracy_oracle_v1/expected_findings.json b/tests/fixtures/prod_like/aws_accuracy_oracle_v1/expected_findings.json index a2f5aa1..c6909be 100644 --- a/tests/fixtures/prod_like/aws_accuracy_oracle_v1/expected_findings.json +++ b/tests/fixtures/prod_like/aws_accuracy_oracle_v1/expected_findings.json @@ -126,7 +126,7 @@ "expected_iamscope_behavior": "Emit inconclusive or preserve uncertainty", "oracle_row_id": "oracle-i-001", "pattern": "Wildcard target resource scope unknown", - "reviewer_note": "Keeps shared uncertainty visible." + "reviewer_note": "Uses split wildcard resource-scope source with no complete-confidence boundary blocker." }, { "evidence_required": "Permission/trust evidence with unresolved condition key", diff --git a/tests/fixtures/prod_like/aws_accuracy_oracle_v1/oracle_rows.json b/tests/fixtures/prod_like/aws_accuracy_oracle_v1/oracle_rows.json index 6e9fe51..2ef6d02 100644 --- a/tests/fixtures/prod_like/aws_accuracy_oracle_v1/oracle_rows.json +++ b/tests/fixtures/prod_like/aws_accuracy_oracle_v1/oracle_rows.json @@ -237,9 +237,9 @@ "expected_iamscope_behavior": "Emit inconclusive or preserve uncertainty", "oracle_row_id": "oracle-i-001", "pattern": "Wildcard target resource scope unknown", - "reviewer_note": "Keeps shared uncertainty visible.", + "reviewer_note": "Uses split wildcard resource-scope source with no complete-confidence boundary blocker.", "source_principal_alias": "principal-wildcard-scope", - "target_alias": "role-wildcard-target" + "target_alias": "role-lambda-exec-scoped" }, { "blocker_precondition_uncertainty_reason": "Condition key context unavailable", diff --git a/tests/fixtures/prod_like/aws_accuracy_oracle_v1/scenario.json b/tests/fixtures/prod_like/aws_accuracy_oracle_v1/scenario.json index 9d98f69..34b32cb 100644 --- a/tests/fixtures/prod_like/aws_accuracy_oracle_v1/scenario.json +++ b/tests/fixtures/prod_like/aws_accuracy_oracle_v1/scenario.json @@ -90,39 +90,59 @@ { "alias": "principal-deny-service", "synthetic_account_alias": "synthetic-account-a" + }, + { + "alias": "principal-scp-unknown", + "synthetic_account_alias": "synthetic-account-a" + }, + { + "alias": "principal-session-context", + "synthetic_account_alias": "synthetic-account-a" + }, + { + "alias": "principal-wildcard-scope", + "synthetic_account_alias": "synthetic-account-a" } ], "roles": [ { - "alias": "role-lambda-exec-scoped", + "alias": "role-audit-b", "synthetic_account_alias": "synthetic-account-a" }, { - "alias": "role-ecs-task-scoped", + "alias": "role-chain-target", "synthetic_account_alias": "synthetic-account-a" }, { - "alias": "role-readonly-ops", + "alias": "role-condition-target", "synthetic_account_alias": "synthetic-account-a" }, { - "alias": "role-prod-observer", + "alias": "role-cross-condition-b", + "synthetic_account_alias": "synthetic-account-b" + }, + { + "alias": "role-denied-assume", + "synthetic_account_alias": "synthetic-account-b" + }, + { + "alias": "role-ecs-task-scoped", "synthetic_account_alias": "synthetic-account-a" }, { - "alias": "role-audit-b", + "alias": "role-lambda-exec-boundary", "synthetic_account_alias": "synthetic-account-a" }, { - "alias": "role-service-mediated-target", + "alias": "role-lambda-exec-scoped", "synthetic_account_alias": "synthetic-account-a" }, { - "alias": "role-lambda-exec-boundary", + "alias": "role-prod-observer", "synthetic_account_alias": "synthetic-account-a" }, { - "alias": "role-chain-target", + "alias": "role-readonly-ops", "synthetic_account_alias": "synthetic-account-a" }, { @@ -130,8 +150,16 @@ "synthetic_account_alias": "synthetic-account-b" }, { - "alias": "role-denied-assume", + "alias": "role-scp-unknown-target", "synthetic_account_alias": "synthetic-account-b" + }, + { + "alias": "role-service-mediated-target", + "synthetic_account_alias": "synthetic-account-a" + }, + { + "alias": "role-session-target", + "synthetic_account_alias": "synthetic-account-a" } ], "synthetic_account_id_for_arn_like_values": "000000000000", diff --git a/tests/live/aws/prod_like_accuracy_sandbox/README.md b/tests/live/aws/prod_like_accuracy_sandbox/README.md index 2373546..36be7b5 100644 --- a/tests/live/aws/prod_like_accuracy_sandbox/README.md +++ b/tests/live/aws/prod_like_accuracy_sandbox/README.md @@ -36,6 +36,7 @@ unless: The source models frozen oracle rows with IAM-only resources: - source-principal inline policies for selected allow/precondition/inconclusive shapes; +- separate wildcard resource-scope and boundary/session uncertainty source principals; - target-role trust policies for selected AWS service trust and direct assume-role shapes; - permission boundaries attached to selected test users; - an explicit deny policy attached to the selected deny probe user; diff --git a/tests/live/aws/prod_like_accuracy_sandbox/terraform/main.tf b/tests/live/aws/prod_like_accuracy_sandbox/terraform/main.tf index 1878154..225c781 100644 --- a/tests/live/aws/prod_like_accuracy_sandbox/terraform/main.tf +++ b/tests/live/aws/prod_like_accuracy_sandbox/terraform/main.tf @@ -31,27 +31,28 @@ locals { } source_principals = { - ci_deployer = "ci-deployer" - ecs_deployer = "ecs-deployer" - helpdesk = "helpdesk" - build = "build" - audit = "audit" - boundary_probe = "boundary-probe" - deny_probe = "deny-probe" - uncertainty_probe = "uncertainty-probe" + ci_deployer = "ci-deployer" + ecs_deployer = "ecs-deployer" + helpdesk = "helpdesk" + build = "build" + audit = "audit" + boundary_probe = "boundary-probe" + deny_probe = "deny-probe" + uncertainty_resource_probe = "uncertainty-resource-probe" + uncertainty_boundary_probe = "uncertainty-boundary-probe" } target_roles = { - lambda_exec_scoped = "lambda-exec-scoped" - ecs_task_scoped = "ecs-task-scoped" - readonly_ops = "readonly-ops" - prod_observer = "prod-observer" - audit_b = "audit-b" - service_mediated_target = "service-mediated-target" - lambda_exec_boundary = "lambda-exec-boundary" - chain_target = "chain-target" - scp_passrole_target = "scp-passrole-target" - denied_assume = "denied-assume" + lambda_exec_scoped = "lambda-exec-scoped" + ecs_task_scoped = "ecs-task-scoped" + readonly_ops = "readonly-ops" + prod_observer = "prod-observer" + audit_b = "audit-b" + service_mediated_target = "service-mediated-target" + lambda_exec_boundary = "lambda-exec-boundary" + chain_target = "chain-target" + scp_passrole_target = "scp-passrole-target" + denied_assume = "denied-assume" } permission_boundary_names = { @@ -74,8 +75,8 @@ locals { } source_permission_boundary_keys = { - boundary_probe = "passrole_lambda" - uncertainty_probe = "session_context" + boundary_probe = "passrole_lambda" + uncertainty_boundary_probe = "session_context" } source_inline_policy_specs = { @@ -213,7 +214,7 @@ locals { }, ] } - uncertainty_probe = { + uncertainty_resource_probe = { statements = [ { sid = "OracleI001WildcardResourceScopeUnknown" @@ -221,6 +222,10 @@ locals { actions = ["iam:PassRole", "lambda:CreateFunction"] resources = ["*"] }, + ] + } + uncertainty_boundary_probe = { + statements = [ { sid = "OracleI002UnresolvedConditionKey" effect = "Allow" @@ -372,7 +377,7 @@ locals { } oracle-i-001 = { resource_group = "wildcard_resource_scope_unknown" - source_principal = "uncertainty_probe" + source_principal = "uncertainty_resource_probe" target_role = "lambda_exec_scoped" live_representable = "yes" cleanup_requirement = "delete policies and roles" @@ -380,7 +385,7 @@ locals { } oracle-i-002 = { resource_group = "unresolved_condition_key" - source_principal = "uncertainty_probe" + source_principal = "uncertainty_boundary_probe" target_role = "readonly_ops" live_representable = "partial" cleanup_requirement = "delete conditional policies and roles" @@ -388,7 +393,7 @@ locals { } oracle-i-003 = { resource_group = "session_or_boundary_context_missing" - source_principal = "uncertainty_probe" + source_principal = "uncertainty_boundary_probe" target_role = "chain_target" live_representable = "partial" cleanup_requirement = "delete roles and policies" diff --git a/tests/test_prod_like_aws_accuracy_oracle_fixture.py b/tests/test_prod_like_aws_accuracy_oracle_fixture.py index 58b5b0c..cb0294c 100644 --- a/tests/test_prod_like_aws_accuracy_oracle_fixture.py +++ b/tests/test_prod_like_aws_accuracy_oracle_fixture.py @@ -139,6 +139,27 @@ def test_every_oracle_row_has_required_fields_and_allowed_values() -> None: assert str(row["reviewer_note"]).strip() +def test_oracle_i001_uses_split_wildcard_resource_scope_alias() -> None: + rows = {row["oracle_row_id"]: row for row in _rows()} + row = rows["oracle-i-001"] + + assert row["source_principal_alias"] == "principal-wildcard-scope" + assert row["target_alias"] == "role-lambda-exec-scoped" + assert "split wildcard resource-scope source" in row["reviewer_note"] + + +def test_inconclusive_oracle_row_aliases_are_present_in_scenario_support() -> None: + scenario = _load("scenario.json") + principal_aliases = {principal["alias"] for principal in scenario["principals"]} + role_aliases = {role["alias"] for role in scenario["roles"]} + + for row in _rows(): + if row["expected_category"] != "inconclusive": + continue + assert row["source_principal_alias"] in principal_aliases + assert row["target_alias"] in role_aliases + + def test_unsupported_rows_are_static_only_and_not_counted_as_mismatches() -> None: expected_findings = _load("expected_findings.json") comparison = _load("expected_comparison.json") diff --git a/tests/test_prod_like_oracle_comparison.py b/tests/test_prod_like_oracle_comparison.py index c56158b..9cff846 100644 --- a/tests/test_prod_like_oracle_comparison.py +++ b/tests/test_prod_like_oracle_comparison.py @@ -101,10 +101,10 @@ def _findings_payload() -> dict[str, Any]: target_name="iamscope-prodlike-v1-lambda-exec-boundary", ), _finding( - finding_id="finding-i001-wrong-verdict", + finding_id="finding-i001", pattern_id="passrole_lambda", - verdict="blocked", - source_name="iamscope-prodlike-v1-uncertainty-probe", + verdict="inconclusive", + source_name="iamscope-prodlike-v1-uncertainty-resource-probe", target_name="iamscope-prodlike-v1-lambda-exec-scoped", ), _finding( @@ -117,8 +117,8 @@ def _findings_payload() -> dict[str, Any]: _finding( finding_id="finding-unmapped-sandbox-extra", pattern_id="passrole_lambda", - verdict="blocked", - source_name="iamscope-prodlike-v1-uncertainty-probe", + verdict="inconclusive", + source_name="iamscope-prodlike-v1-uncertainty-resource-probe", target_name="iamscope-prodlike-v1-service-mediated-target", ), ] @@ -137,12 +137,8 @@ def test_comparison_classifies_matches_mismatch_missing_unsupported_and_not_comp assert _row_by_id(result, "oracle-v-001")["comparison_category"] == "oracle_match" assert _row_by_id(result, "oracle-b-001")["comparison_category"] == "oracle_match" - assert _row_by_id(result, "oracle-i-001")["comparison_category"] == "oracle_mismatch" - assert _row_by_id(result, "oracle-i-001")["emitted_verdict"] == "blocked" - assert _row_by_id(result, "oracle-i-001")["triage_note"] == ( - "emitted blocked due complete-confidence boundary evidence; likely oracle/fixture expectation conflict, " - "not automatically an IAMScope false positive" - ) + assert _row_by_id(result, "oracle-i-001")["comparison_category"] == "oracle_match" + assert _row_by_id(result, "oracle-i-001")["emitted_verdict"] == "inconclusive" assert _row_by_id(result, "oracle-v-006")["comparison_category"] == "oracle_missing" assert _row_by_id(result, "oracle-v-003")["comparison_category"] == "not_currently_live_comparable" assert _row_by_id(result, "oracle-u-001")["comparison_category"] == "unsupported_static_only" @@ -154,8 +150,8 @@ def test_comparison_classifies_matches_mismatch_missing_unsupported_and_not_comp assert result["unmapped_sandbox_extra_count"] == 1 assert result["comparison_category_counts"]["environmental_extra"] == 1 assert result["comparison_category_counts"]["unmapped_sandbox_extra"] == 1 - assert result["comparison_category_counts"]["oracle_match"] == 2 - assert result["comparison_category_counts"]["oracle_mismatch"] == 1 + assert result["comparison_category_counts"]["oracle_match"] == 3 + assert "oracle_mismatch" not in result["comparison_category_counts"] assert result["comparison_category_counts"]["oracle_missing"] == 1 assert result["comparison_category_counts"]["unsupported_static_only"] == 1 assert result["comparison_category_counts"]["not_currently_live_comparable"] == 1 @@ -177,11 +173,11 @@ def test_environmental_extra_uses_sanitized_source_and_target_names_only() -> No assert len(unmapped) == 1 assert unmapped[0]["comparison_category"] == "unmapped_sandbox_extra" assert unmapped[0]["extra_type"] == "sandbox_source_has_no_deterministic_oracle_mapping" - assert unmapped[0]["source_name"] == "iamscope-prodlike-v1-uncertainty-probe" + assert unmapped[0]["source_name"] == "iamscope-prodlike-v1-uncertainty-resource-probe" assert unmapped[0]["target_name"] == "iamscope-prodlike-v1-service-mediated-target" assert unmapped[0]["triage_note"] == ( - "extra blocked path induced by uncertainty-probe boundary/policy shape; " - "not part of deterministic oracle mapping" + "extra wildcard resource-scope path from oracle-i-001 split source; " + "review or remap only after a fresh live run confirms it remains intended" ) @@ -206,8 +202,9 @@ def test_summary_contains_non_claims_without_machine_score_or_pass_fail_fields(t assert "no composite benchmark score" in summary assert "no pass/fail benchmark label" in summary - assert "likely oracle/fixture expectation conflict" in summary - assert "extra blocked path induced by uncertainty-probe boundary/policy shape" in summary + assert "oracle-i-001" in summary + assert "iamscope-prodlike-v1-uncertainty-resource-probe" in summary + assert "extra wildcard resource-scope path from oracle-i-001 split source" in summary assert "composite_score" not in summary assert "benchmark_passed" not in summary assert "pass_fail" not in summary diff --git a/tests/test_prod_like_terraform_sandbox_files.py b/tests/test_prod_like_terraform_sandbox_files.py index a325f84..b13837c 100644 --- a/tests/test_prod_like_terraform_sandbox_files.py +++ b/tests/test_prod_like_terraform_sandbox_files.py @@ -146,6 +146,27 @@ def test_terraform_maps_oracle_rows_to_iam_policy_themes() -> None: assert 'effect = "Deny"' in text +def test_oracle_i001_source_is_split_from_boundary_session_source() -> None: + text = _terraform_text() + + assert 'uncertainty_resource_probe = "uncertainty-resource-probe"' in text + assert 'uncertainty_boundary_probe = "uncertainty-boundary-probe"' in text + assert 'uncertainty_boundary_probe = "session_context"' in text + assert 'uncertainty_resource_probe = "session_context"' not in text + + i001_start = text.index("oracle-i-001 = {") + i001_block = text[i001_start : i001_start + 350] + assert 'source_principal = "uncertainty_resource_probe"' in i001_block + + i002_start = text.index("oracle-i-002 = {") + i002_block = text[i002_start : i002_start + 350] + assert 'source_principal = "uncertainty_boundary_probe"' in i002_block + + i003_start = text.index("oracle-i-003 = {") + i003_block = text[i003_start : i003_start + 350] + assert 'source_principal = "uncertainty_boundary_probe"' in i003_block + + def test_no_raw_non_synthetic_account_ids_or_iam_arns() -> None: text = _all_sandbox_text() account_ids = set(re.findall(r"\b[0-9]{12}\b", text))