diff --git a/TODO.md b/TODO.md index ff8536f12..c4cdd0cf8 100644 --- a/TODO.md +++ b/TODO.md @@ -26,3 +26,4 @@ - [x] Slice 16: Extract tree-traversal duplication in transform.go - [x] Slice 17: Align test method naming - [x] Slice 18: Fail evaluation when rehydration errors occur (instead of silently swallowing them) +- [x] Slice 19: Add Long descriptions, Example blocks, and docs feedback (policy contract hint, snyk trail example) diff --git a/cmd/kosli/evaluate.go b/cmd/kosli/evaluate.go index cb444ad80..c06895299 100644 --- a/cmd/kosli/evaluate.go +++ b/cmd/kosli/evaluate.go @@ -6,13 +6,22 @@ import ( "github.com/spf13/cobra" ) -const evaluateDesc = `All Kosli evaluate commands.` +const evaluateShortDesc = `Evaluate Kosli trail data against OPA/Rego policies.` + +// Backtick breaks (`"` + "`x`" + `"`) are needed to embed markdown +// inline code spans inside raw string literals. +const evaluateLongDesc = evaluateShortDesc + ` +Fetch trail data from Kosli and evaluate it against custom policies written +in Rego, the policy language used by Open Policy Agent (OPA). +The policy must use ` + "`package policy`" + ` and define an ` + "`allow`" + ` rule. +An optional ` + "`violations`" + ` rule (a set of strings) can provide human-readable denial reasons. +The command exits with code 0 when allowed and code 1 when denied.` func newEvaluateCmd(out io.Writer) *cobra.Command { cmd := &cobra.Command{ Use: "evaluate", - Short: evaluateDesc, - Long: evaluateDesc, + Short: evaluateShortDesc, + Long: evaluateLongDesc, } // Add subcommands diff --git a/cmd/kosli/evaluateTrail.go b/cmd/kosli/evaluateTrail.go index 68692e89d..ba4d5e52c 100644 --- a/cmd/kosli/evaluateTrail.go +++ b/cmd/kosli/evaluateTrail.go @@ -6,7 +6,40 @@ import ( "github.com/spf13/cobra" ) -const evaluateTrailDesc = `Evaluate a trail against a policy.` +const evaluateTrailShortDesc = `Evaluate a trail against a policy.` + +const evaluateTrailLongDesc = evaluateTrailShortDesc + ` +Fetch a single trail from Kosli and evaluate it against a Rego policy using OPA. +The trail data is passed to the policy as ` + "`input.trail`" + `. + +Use ` + "`--attestations`" + ` to enrich the input with detailed attestation data +(e.g. pull request approvers, scan results). Use ` + "`--show-input`" + ` to inspect the +full data structure available to the policy. Use ` + "`--output json`" + ` for structured output.` + +const evaluateTrailExample = ` +# evaluate a trail against a policy: +kosli evaluate trail yourTrailName \ + --policy yourPolicyFile.rego \ + --flow yourFlowName \ + --api-token yourAPIToken \ + --org yourOrgName + +# evaluate a trail with attestation enrichment: +kosli evaluate trail yourTrailName \ + --policy yourPolicyFile.rego \ + --flow yourFlowName \ + --attestations pull-request \ + --api-token yourAPIToken \ + --org yourOrgName + +# evaluate a trail and show the policy input data: +kosli evaluate trail yourTrailName \ + --policy yourPolicyFile.rego \ + --flow yourFlowName \ + --show-input \ + --output json \ + --api-token yourAPIToken \ + --org yourOrgName` type evaluateTrailOptions struct { commonEvaluateOptions @@ -15,10 +48,11 @@ type evaluateTrailOptions struct { func newEvaluateTrailCmd(out io.Writer) *cobra.Command { o := new(evaluateTrailOptions) cmd := &cobra.Command{ - Use: "trail TRAIL-NAME", - Short: evaluateTrailDesc, - Long: evaluateTrailDesc, - Args: cobra.ExactArgs(1), + Use: "trail TRAIL-NAME", + Short: evaluateTrailShortDesc, + Long: evaluateTrailLongDesc, + Example: evaluateTrailExample, + Args: cobra.ExactArgs(1), PreRunE: func(cmd *cobra.Command, args []string) error { err := RequireGlobalFlags(global, []string{"Org", "ApiToken"}) if err != nil { diff --git a/cmd/kosli/evaluateTrails.go b/cmd/kosli/evaluateTrails.go index 78c3e3046..ff3ac0a1e 100644 --- a/cmd/kosli/evaluateTrails.go +++ b/cmd/kosli/evaluateTrails.go @@ -6,7 +6,41 @@ import ( "github.com/spf13/cobra" ) -const evaluateTrailsDesc = `Evaluate multiple trails against a policy.` +const evaluateTrailsShortDesc = `Evaluate multiple trails against a policy.` + +const evaluateTrailsLongDesc = evaluateTrailsShortDesc + ` +Fetch multiple trails from Kosli and evaluate them together against a Rego policy using OPA. +The trail data is passed to the policy as ` + "`input.trails`" + ` (an array), unlike +` + "`evaluate trail`" + ` which passes ` + "`input.trail`" + ` (a single object). + +Use ` + "`--attestations`" + ` to enrich the input with detailed attestation data +(e.g. pull request approvers, scan results). Use ` + "`--show-input`" + ` to inspect the +full data structure available to the policy. Use ` + "`--output json`" + ` for structured output.` + +const evaluateTrailsExample = ` +# evaluate multiple trails against a policy: +kosli evaluate trails yourTrailName1 yourTrailName2 \ + --policy yourPolicyFile.rego \ + --flow yourFlowName \ + --api-token yourAPIToken \ + --org yourOrgName + +# evaluate trails with attestation enrichment: +kosli evaluate trails yourTrailName1 yourTrailName2 \ + --policy yourPolicyFile.rego \ + --flow yourFlowName \ + --attestations pull-request \ + --api-token yourAPIToken \ + --org yourOrgName + +# evaluate trails with JSON output and show the policy input: +kosli evaluate trails yourTrailName1 yourTrailName2 \ + --policy yourPolicyFile.rego \ + --flow yourFlowName \ + --show-input \ + --output json \ + --api-token yourAPIToken \ + --org yourOrgName` type evaluateTrailsOptions struct { commonEvaluateOptions @@ -15,10 +49,11 @@ type evaluateTrailsOptions struct { func newEvaluateTrailsCmd(out io.Writer) *cobra.Command { o := new(evaluateTrailsOptions) cmd := &cobra.Command{ - Use: "trails TRAIL-NAME [TRAIL-NAME...]", - Short: evaluateTrailsDesc, - Long: evaluateTrailsDesc, - Args: cobra.MinimumNArgs(1), + Use: "trails TRAIL-NAME [TRAIL-NAME...]", + Short: evaluateTrailsShortDesc, + Long: evaluateTrailsLongDesc, + Example: evaluateTrailsExample, + Args: cobra.MinimumNArgs(1), PreRunE: func(cmd *cobra.Command, args []string) error { err := RequireGlobalFlags(global, []string{"Org", "ApiToken"}) if err != nil { diff --git a/docs.kosli.com/content/tutorials/evaluate_trails_with_opa.md b/docs.kosli.com/content/tutorials/evaluate_trails_with_opa.md index dc71a9f45..628faf800 100644 --- a/docs.kosli.com/content/tutorials/evaluate_trails_with_opa.md +++ b/docs.kosli.com/content/tutorials/evaluate_trails_with_opa.md @@ -2,7 +2,6 @@ title: "Evaluate trails with OPA policies" bookCollapseSection: false weight: 509 -draft: true summary: "Learn how to use kosli evaluate trail and kosli evaluate trails to check your Kosli trails against custom OPA/Rego policies. This tutorial walks through writing a policy that verifies pull requests have been approved." --- @@ -19,6 +18,7 @@ To follow this tutorial, you need to: * [Install Kosli CLI](/getting_started/install/). * [Get a Kosli API token](/getting_started/service-accounts/). * Set the `KOSLI_API_TOKEN` environment variable to your token: + ```shell {.command} export KOSLI_API_TOKEN= ``` @@ -52,14 +52,18 @@ allow if { Let's break down what this policy does: -- **`package policy`** — every evaluate policy must use the `policy` package. -- **`import rego.v1`** — use Rego v1 syntax (the `if`/`contains` keywords). -- **`default allow = false`** — trails are denied unless explicitly allowed. -- **`violations`** — a set of messages describing why the policy failed. The rule iterates over trails, then over pull requests within the `pull-request` attestation, looking for PRs where `approvers` is empty. -- **`allow`** — trails are allowed only when there are no violations. +* **`package policy`** — every evaluate policy must use the `policy` package. +* **`import rego.v1`** — use Rego v1 syntax (the `if`/`contains` keywords). +* **`default allow = false`** — trails are denied unless explicitly allowed. +* **`violations`** — a set of messages describing why the policy failed. The rule iterates over trails, then over pull requests within the `pull-request` attestation, looking for PRs where `approvers` is empty. +* **`allow`** — trails are allowed only when there are no violations. {{}} -The policy contract requires `package policy` and an `allow` rule. The `violations` rule is optional but recommended — it provides human-readable reasons when a trail is denied. +**Policy contract** — these are Kosli-specific conventions, not OPA built-ins: + +* **`package policy`** — required. Kosli queries `data.policy.*` to find your rules. +* **`allow`** — required. Must evaluate to a **boolean**. Kosli exits with code 0 when `true`, code 1 when `false`. +* **`violations`** — optional but recommended. Must be a **set of strings**, where each string is a human-readable reason the policy failed. Kosli displays these when `allow` is `false`. {{}} ## Step 3: Evaluate multiple trails @@ -105,7 +109,9 @@ RESULT: ALLOWED ## Step 4: Evaluate a single trail -To evaluate just one trail, use `kosli evaluate trail` (singular). The data is passed to the policy as `input.trail` instead of `input.trails`, so you need a slightly different policy. Save this as `pr-approved-single.rego`: +The `kosli evaluate trail` (singular) command evaluates facts within a single trail — a different use case from comparing across multiple trails. For example, you might check that a snyk container scan found no high-severity vulnerabilities. + +Save this as `snyk-no-high-vulns.rego`: ```rego package policy @@ -115,9 +121,11 @@ import rego.v1 default allow = false violations contains msg if { - some pr in input.trail.compliance_status.attestations_statuses["pull-request"].pull_requests - count(pr.approvers) == 0 - msg := sprintf("trail '%v': pull-request %v has no approvers", [input.trail.name, pr.url]) + some name, artifact in input.trail.compliance_status.artifacts_statuses + snyk := artifact.attestations_statuses["snyk-container-scan"] + some result in snyk.processed_snyk_results.results + result.high_count > 0 + msg := sprintf("artifact '%v': snyk container scan found %d high severity vulnerabilities", [name, result.high_count]) } allow if { @@ -125,21 +133,28 @@ allow if { } ``` +This policy iterates over every artifact in the trail, looks up its `snyk-container-scan` attestation, and checks whether any result has a non-zero `high_count`. + +Use `--attestations` to enrich only the snyk data (faster than fetching all attestation details). +The value uses the format `artifact-name.attestation-type` — here, `dashboard` is the artifact name and `snyk-container-scan` is the attestation name: + ```shell {.command} kosli evaluate trail \ - --policy pr-approved-single.rego \ + --policy snyk-no-high-vulns.rego \ --org cyber-dojo \ --flow dashboard-ci \ - 9978a1ca82c273a68afaa85fc37dd60d1e394f84 + --attestations dashboard.snyk-container-scan \ + 44ca5fa2630947cf375fdbda10972a4bedaaaba3 ``` ```plaintext {.light-console} -RESULT: DENIED -VIOLATIONS: trail '9978a1ca82c273a68afaa85fc37dd60d1e394f84': pull-request https://github.com/cyber-dojo/dashboard/pull/344 has no approvers +RESULT: ALLOWED ``` +The trail has zero high-severity vulnerabilities, so the policy allows it. + {{}} -When writing a policy for `kosli evaluate trail`, reference `input.trail` (a single object) instead of `input.trails` (an array). You can write one policy that handles both by checking for both keys, or keep separate policies for each command. +When writing a policy for `kosli evaluate trail`, reference `input.trail` (a single object). For `kosli evaluate trails`, reference `input.trails` (an array). The data shapes differ, so use separate policies for each command. {{}} ## Step 5: Explore the policy input with --show-input @@ -148,24 +163,26 @@ When writing policies, it helps to see exactly what data is available. Use `--sh ```shell {.command} kosli evaluate trail \ - --policy pr-approved-single.rego \ + --policy snyk-no-high-vulns.rego \ --org cyber-dojo \ --flow dashboard-ci \ + --attestations dashboard.snyk-container-scan \ --show-input \ --output json \ - 9978a1ca82c273a68afaa85fc37dd60d1e394f84 + 44ca5fa2630947cf375fdbda10972a4bedaaaba3 ``` This outputs the evaluation result along with the complete `input` object. You can pipe it through `jq` to explore the structure: ```shell {.command} kosli evaluate trail \ - --policy pr-approved-single.rego \ + --policy snyk-no-high-vulns.rego \ --org cyber-dojo \ --flow dashboard-ci \ + --attestations dashboard.snyk-container-scan \ --show-input \ --output json \ - 9978a1ca82c273a68afaa85fc37dd60d1e394f84 2>/dev/null | jq '.input.trail.compliance_status | keys' + 44ca5fa2630947cf375fdbda10972a4bedaaaba3 2>/dev/null | jq '.input.trail.compliance_status | keys' ``` ```plaintext {.light-console} @@ -211,13 +228,14 @@ audit record in Kosli that captures the policy, the full evaluation report, and violations. This step requires write access to your Kosli org. The examples below use variables -you'd set in your CI/CD pipeline: +you'd set in your CI/CD pipeline. +In your own pipeline you'd use your own policy file — here we use `my-policy.rego` as a placeholder: ```shell {.command} # Run the evaluation and save the full JSON report to a file # (|| true prevents the step from failing when the policy denies) kosli evaluate trail "$TRAIL_NAME" \ - --policy pr-approved-single.rego \ + --policy my-policy.rego \ --org "$KOSLI_ORG" \ --flow "$FLOW_NAME" \ --show-input \ @@ -236,18 +254,18 @@ kosli attest generic \ --trail "$TRAIL_NAME" \ --org "$KOSLI_ORG" \ --compliant="$is_compliant" \ - --attachments pr-approved-single.rego,eval-report.json \ + --attachments my-policy.rego,eval-report.json \ --user-data eval-violations.json ``` This creates a generic attestation on the trail with: -- **`--compliant`** set based on whether the policy allowed or denied — read directly +* **`--compliant`** set based on whether the policy allowed or denied — read directly from the JSON report rather than relying on the exit code, which avoids issues with `set -e` in CI environments like GitHub Actions -- **`--attachments`** containing the Rego policy (for reproducibility) and the full +* **`--attachments`** containing the Rego policy (for reproducibility) and the full JSON evaluation report (including the input data the policy evaluated) -- **`--user-data`** containing the violations, which appear in the Kosli UI as +* **`--user-data`** containing the violations, which appear in the Kosli UI as structured metadata on the attestation {{}}