diff --git a/acceptance/bin/retry.py b/acceptance/bin/retry.py new file mode 100755 index 00000000000..3df80351eb6 --- /dev/null +++ b/acceptance/bin/retry.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Retry a command until it succeeds and its output matches expectations. + +Usage: retry.py [--until SUBSTR] [--until-not SUBSTR] CMD [ARGS...] + +Retries CMD up to 5 times (configurable via RETRY_MAX_ATTEMPTS env var), +sleeping RETRY_INTERVAL_MS milliseconds (default 500) between attempts. +An attempt is considered successful when the command exits with code 0 and: + --until SUBSTR SUBSTR appears in stdout + --until-not SUBSTR SUBSTR does not appear in stdout +""" + +import argparse +import os +import subprocess +import sys +import time + + +def main(): + parser = argparse.ArgumentParser(prog="retry.py") + parser.add_argument("--until") + parser.add_argument("--until-not") + parser.add_argument("cmd", nargs=argparse.REMAINDER) + args = parser.parse_args() + if not args.cmd: + parser.error("no command given") + until = args.until + until_not = args.until_not + argv = args.cmd + + interval = float(os.environ.get("RETRY_INTERVAL_MS", "500")) / 1000.0 + max_attempts = int(os.environ.get("RETRY_MAX_ATTEMPTS", "5")) + + result = subprocess.run(argv, capture_output=True) + for _ in range(1, max_attempts): + success = ( + result.returncode == 0 + and (until is None or until.encode() in result.stdout) + and (until_not is None or until_not.encode() not in result.stdout) + ) + if success: + break + time.sleep(interval) + result = subprocess.run(argv, capture_output=True) + + sys.stdout.buffer.write(result.stdout) + sys.stderr.buffer.write(result.stderr) + sys.exit(result.returncode) + + +if __name__ == "__main__": + main() diff --git a/acceptance/bundle/resources/dashboards/detect-change/script b/acceptance/bundle/resources/dashboards/detect-change/script index 36649c77cad..71cd75ab974 100644 --- a/acceptance/bundle/resources/dashboards/detect-change/script +++ b/acceptance/bundle/resources/dashboards/detect-change/script @@ -31,8 +31,9 @@ $CLI lakeview get "${DASHBOARD_ID}" | jq '{display_name,page_display_name: (.ser title "Make an out of band modification to the dashboard and confirm that it is detected:\n" RESOURCE_ID=$($CLI workspace get-status "${DASHBOARD_PATH}" | jq -r '.resource_id') DASHBOARD_JSON="{\"serialized_dashboard\": \"{}\", \"warehouse_id\": \"$TEST_DEFAULT_WAREHOUSE_ID\"}" -$CLI lakeview update "${RESOURCE_ID}" --json "${DASHBOARD_JSON}" | jq '{lifecycle_state}' -echo "$($CLI lakeview get "$DASHBOARD_ID" | jq -r '.etag'):ETAG_2" >> ACC_REPLS +UPDATE_RESP=$($CLI lakeview update "${RESOURCE_ID}" --json "${DASHBOARD_JSON}") +echo "$UPDATE_RESP" | jq '{lifecycle_state}' +echo "$(echo "$UPDATE_RESP" | jq -r '.etag'):ETAG_2" >> ACC_REPLS title "Try to redeploy the bundle and confirm that the out of band modification is detected:" trace $CLI bundle plan diff --git a/acceptance/bundle/resources/dashboards/publish-failure-stale-content/script b/acceptance/bundle/resources/dashboards/publish-failure-stale-content/script index 500d87c5d76..cc7551e9b68 100644 --- a/acceptance/bundle/resources/dashboards/publish-failure-stale-content/script +++ b/acceptance/bundle/resources/dashboards/publish-failure-stale-content/script @@ -12,7 +12,8 @@ unset MSYS_NO_PATHCONV trace $CLI bundle deploy replace_ids.py DASHBOARD_ID=$($CLI bundle summary --output json | jq -r '.resources.dashboards.dashboard1.id') -add_repl.py "$($CLI lakeview get $DASHBOARD_ID | jq -r '.etag')" ETAG_1 +ETAG_1=$($CLI lakeview get $DASHBOARD_ID | jq -r '.etag') +add_repl.py "$ETAG_1" ETAG_1 trace $CLI lakeview get $DASHBOARD_ID | jq '{display_name, etag}' trace $CLI lakeview get-published $DASHBOARD_ID | jq '{display_name}' trace $CLI bundle plan -o json | gron.py | grep -E "etag|published" @@ -28,7 +29,8 @@ update_file.py databricks.yml "my dashboard" "my dashboard renamed" # SaveState is only called on success, so state retains the pre-PATCH etag. errcode trace $CLI bundle deploy trace print_requests.py //lakeview/dashboards -add_repl.py "$($CLI lakeview get $DASHBOARD_ID | jq -r '.etag')" ETAG_2 +# The PATCH bumped the remote etag to ETAG_2; retry until it is visible (eventual consistency). +add_repl.py "$(retry.py --until-not "$ETAG_1" $CLI lakeview get $DASHBOARD_ID | jq -r '.etag')" ETAG_2 trace $CLI lakeview get $DASHBOARD_ID | jq '{display_name, etag}' trace $CLI lakeview get-published $DASHBOARD_ID | jq '{display_name}' trace $CLI bundle plan -o json | gron.py | grep -E "etag|published"