From f4412bfaf95618b164d914cc5fd6312a8a9dda1e Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Thu, 12 Feb 2026 00:24:01 -0800 Subject: [PATCH 1/6] Add one more metric for state sync --- go.work.sum | 54 +++++++++++++++++++++++++++ sei-cosmos/storev2/rootmulti/store.go | 1 + 2 files changed, 55 insertions(+) diff --git a/go.work.sum b/go.work.sum index da90763093..c19394b3d8 100644 --- a/go.work.sum +++ b/go.work.sum @@ -6,6 +6,7 @@ cel.dev/expr v0.16.1/go.mod h1:AsGA5zb3WruAEQeQng1RZdGEXmBj0jvMWh6l5SnNuC8= cel.dev/expr v0.20.0 h1:OunBvVCfvpWlt4dN7zg3FM6TDkzOePe1+foGJ9AXeeI= cel.dev/expr v0.20.0/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw= cloud.google.com/go v0.98.0/go.mod h1:ua6Ush4NALrHk5QXDWnjvZHN93OuF0HfuEPq9I1X0cM= +cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys= cloud.google.com/go v0.116.0 h1:B3fRrSDkLRt5qSHWe40ERJvhvnQwdZiHu0bJOpldweE= cloud.google.com/go v0.116.0/go.mod h1:cEPSRWPzZEswwdr9BxE6ChEn01dWlTaF05LiC2Xs70U= cloud.google.com/go v0.121.1 h1:S3kTQSydxmu1JfLRLpKtxRPA7rSrYPRPEUmL/PavVUw= @@ -248,6 +249,7 @@ cloud.google.com/go/gkemulticloud v1.4.1 h1:SvVD2nJTGScEDYygIQ5dI14oFYhgtJx8Hazk cloud.google.com/go/gkemulticloud v1.4.1/go.mod h1:KRvPYcx53bztNwNInrezdfNF+wwUom8Y3FuJBwhvFpQ= cloud.google.com/go/gkemulticloud v1.5.3 h1:334aZmOzIt3LVBpguCof8IHaLaftcZlx+L0TGBukYkY= cloud.google.com/go/gkemulticloud v1.5.3/go.mod h1:KPFf+/RcfvmuScqwS9/2MF5exZAmXSuoSLPuaQ98Xlk= +cloud.google.com/go/grafeas v0.2.0 h1:CYjC+xzdPvbV65gi6Dr4YowKcmLo045pm18L0DhdELM= cloud.google.com/go/grafeas v0.3.11 h1:CobnwnyeY1j1Defi5vbEircI+jfrk3ci5m004ZjiFP4= cloud.google.com/go/grafeas v0.3.11/go.mod h1:dcQyG2+T4tBgG0MvJAh7g2wl/xHV2w+RZIqivwuLjNg= cloud.google.com/go/grafeas v0.3.15 h1:lBjwKmhpiqOAFaE0xdqF8CqO74a99s8tUT5mCkBBxPs= @@ -439,6 +441,7 @@ cloud.google.com/go/speech v1.25.2 h1:rKOXU9LAZTOYHhRNB4gZDekNjJx21TktQpetBa5IzO cloud.google.com/go/speech v1.25.2/go.mod h1:KPFirZlLL8SqPaTtG6l+HHIFHPipjbemv4iFg7rTlYs= cloud.google.com/go/speech v1.27.1 h1:+OktATNlQc+4WH78OrQadIP4CzXb9mBucdDGCO1NrlI= cloud.google.com/go/speech v1.27.1/go.mod h1:efCfklHFL4Flxcdt9gpEMEJh9MupaBzw3QiSOVeJ6ck= +cloud.google.com/go/storage v1.29.0 h1:6weCgzRvMg7lzuUurI4697AqIRPU1SvzHhynwpW31jI= cloud.google.com/go/storage v1.49.0 h1:zenOPBOWHCnojRd9aJZAyQXBYqkJkdQS42dxL55CIMw= cloud.google.com/go/storage v1.49.0/go.mod h1:k1eHhhpLvrPjVGfo0mOUPEJ4Y2+a/Hv5PiwehZI9qGU= cloud.google.com/go/storage v1.53.0 h1:gg0ERZwL17pJ+Cz3cD2qS60w1WMDnwcm5YPAIQBHUAw= @@ -559,6 +562,7 @@ github.com/RoaringBitmap/roaring v1.2.2/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/ github.com/Shopify/goreferrer v0.0.0-20220729165902-8cddb4f5de06 h1:KkH3I3sJuOLP3TjA/dfr4NAY8bghDwnXiU7cTKxQqo0= github.com/Shopify/sarama v1.19.0 h1:9oksLxC6uxVPHPVYUmq6xhr1BOF/hHobWH2UzO67z1s= github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= +github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= github.com/VictoriaMetrics/metrics v1.23.1 h1:/j8DzeJBxSpL2qSIdqnRFLvQQhbJyJbbEi22yMm7oL0= github.com/VictoriaMetrics/metrics v1.23.1/go.mod h1:rAr/llLpEnAdTehiNlUxKgnjcOuROSzpw0GvjpEbvFc= github.com/Zilliqa/gozilliqa-sdk v1.2.1-0.20201201074141-dd0ecada1be6 h1:1d9pzdbkth4D9AX6ndKSl7of3UTV0RYl3z64U2dXMGo= @@ -627,6 +631,7 @@ github.com/apache/arrow/go/v15 v15.0.2/go.mod h1:DGXsR3ajT524njufqf95822i+KTh+ye github.com/apache/thrift v0.16.0 h1:qEy6UW60iVOlUy+b9ZR0d5WzUWYGOo4HfopoyBaNmoY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e h1:QEF07wC0T1rKkctt1RINW/+RMTVmiwxETico2l3gxJA= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6 h1:G1bPvciwNyF7IUmKXNt9Ak3m6u9DE1rF+RmtIkBpVdA= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310 h1:BUAU3CGlLvorLI26FmByPp2eC2qla6E1Tw+scpcg/to= github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI= github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a h1:pv34s756C4pEXnjgPfGYgdhg/ZdajGhyOvzx8k+23nw= github.com/aws/aws-lambda-go v1.13.3 h1:SuCy7H3NLyp+1Mrfp+m80jcbi9KYWAs9/BXwppwRDzY= @@ -673,12 +678,14 @@ github.com/cenkalti/backoff/v5 v5.0.2 h1:rIfFVxEf1QsI7E1ZHfp/B4DF/6QBAUhmgkxc0H7 github.com/cenkalti/backoff/v5 v5.0.2/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g= github.com/checkpoint-restore/go-criu/v5 v5.0.0 h1:TW8f/UvntYoVDMN1K2HlT82qH1rb0sOjpGw3m6Ym+i4= +github.com/checkpoint-restore/go-criu/v5 v5.3.0 h1:wpFFOoomK3389ue2lAb0Boag6XPht5QYpipxmSNL4d8= github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764= github.com/chzyer/logex v1.2.0 h1:+eqR0HfOetur4tgnC8ftU5imRnhi4te+BadWS95c5AM= github.com/chzyer/readline v1.5.0 h1:lSwwFrbNviGePhkewF1az4oLmcwqCZijQ2/Wi3BGHAI= github.com/chzyer/test v0.0.0-20210722231415-061457976a23 h1:dZ0/VyGgQdVGAss6Ju0dt5P0QltE0SFY5Woh6hbIfiQ= github.com/cilium/ebpf v0.6.2 h1:iHsfF/t4aW4heW2YKfeHrVPGdtYTL4C4KocpM8KTSnI= +github.com/cilium/ebpf v0.7.0 h1:1k/q3ATgxSXRdrmPfH8d7YK0GfqVsEKZAX9dQZvs56k= github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible h1:C29Ae4G5GtYyYMm1aztcyj/J5ckgJm2zwdDajFbx1NY= github.com/circonus-labs/circonusllhist v0.1.3 h1:TJH+oke8D16535+jHExHj4nQvzlZrj7ug5D7I/orNUA= @@ -688,6 +695,7 @@ github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec h1:EdRZT3IeKQmfCSrgo github.com/cloudflare/cloudflare-go v0.114.0 h1:ucoti4/7Exo0XQ+rzpn1H+IfVVe++zgiM+tyKtf0HUA= github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe h1:QQ3GSy+MqSHxm/d8nCtnAiZdYFd45cYZPs8vOOIYKfk= github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b h1:ACGZRIr7HsgBKHsueQ1yM4WaVaXh21ynwqsF8M8tXhA= github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 h1:QVw89YDxXxEe+l8gU8ETbOasdwEV+avkR75ZzsVV9WI= github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= @@ -698,25 +706,31 @@ github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 h1:sDMmm+q/3+Bu github.com/consensys/bavard v0.1.31-0.20250406004941-2db259e4b582 h1:dTlIwEdFQmldzFf5F6bbTcYWhvnAgZai2g8eq3Wwxqg= github.com/consensys/bavard v0.1.31-0.20250406004941-2db259e4b582/go.mod h1:k/zVjHHC4B+PQy1Pg7fgvG3ALicQw540Crag8qx+dZs= github.com/containerd/console v1.0.2 h1:Pi6D+aZXM+oUw1czuKgH5IJ+y0jhYcwBJfx5/Ghn9dE= +github.com/containerd/console v1.0.3 h1:lIr7SlA5PxZyMV30bDW0MGbiOPXwc63yRuCP0ARubLw= github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= github.com/coreos/bbolt v1.3.2 h1:wZwiHHUieZCquLkDL0B8UhzreNWsPHooDAG3q34zk0s= github.com/coreos/etcd v3.3.10+incompatible h1:jFneRYjIvLMLhDLCzuTuU4rSJUjRplcJQ7pD7MnhC04= github.com/coreos/go-etcd v2.0.0+incompatible h1:bXhRBIXoTm9BYHS3gE0TtQuyNZyeEMux2sDi4oo5YOo= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a h1:W8b4lQ4tFF21aspRGoBuCNV6V2fFJBF+pm1J6OY8Lys= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/crate-crypto/go-eth-kzg v1.3.0 h1:05GrhASN9kDAidaFJOda6A4BEvgvuXbazXg/0E3OOdI= github.com/crate-crypto/go-eth-kzg v1.3.0/go.mod h1:J9/u5sWfznSObptgfa92Jq8rTswn6ahQWEuiLHOjCUI= github.com/creachadair/command v0.0.0-20220426235536-a748effdf6a1 h1:r626P+s8TKpQaHwIaLmA2nGetfIVYEhtqcs3g2U1dC8= github.com/creachadair/command v0.0.0-20220426235536-a748effdf6a1/go.mod h1:bAM+qFQb/KwWyCc9MLC4U1jvn3XyakqP5QRkds5T6cY= +github.com/creachadair/command v0.2.0 h1:qTA9cMMhZePAxFoNdnk6F6nn94s1qPndIg9hJbqI9cA= github.com/creachadair/command v0.2.0/go.mod h1:j+Ar+uYnFsHpkMeV9kGj6lJ45y9u2xqtg8FYy6cm+0o= github.com/creack/pty v1.1.9 h1:uDmaGzcdjhF4i/plgjmEsriH11Y0o7RKapEf/LDaM3w= +github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= github.com/cristalhq/acmd v0.8.1 h1:mtFp/cbeJNY5jokF9zPz5mRllGHropRrOkOVxeGS6FI= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c h1:/ovYnF02fwL0kvspmy9AuyKg1JhdTRUgPw4nUxd9oZM= github.com/cyphar/filepath-securejoin v0.2.2 h1:jCwT2GTP+PY5nBz3c/YL5PAIbusElVrPujOBSCj8xRg= +github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/decred/dcrd/lru v1.0.0 h1:Kbsb1SFDsIlaupWPwsPp+dkxiBY1frcS07PCPgotKz8= github.com/deepmap/oapi-codegen v1.6.0 h1:w/d1ntwh91XI0b/8ja7+u5SvA4IFfM0UNNLmiDR1gg0= @@ -726,6 +740,7 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954 h1:RMLoZVzv4GliuWafOuPuQDKSm1SJph7uCRnnS61JAn4= github.com/djherbis/atime v1.1.0 h1:rgwVbP/5by8BvvjBNrbh64Qz33idKT3pSnMSJsxhi0g= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= +github.com/docker/docker v25.0.6+incompatible h1:5cPwbwriIcsua2REJe8HqQV+6WlWc1byg2QSXzBxBGg= github.com/docker/docker v25.0.6+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815 h1:bWDMxwH3px2JBh6AyO7hdCn/PkvCZXii8TGj7sbtEbQ= github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0 h1:C7t6eeMaEQVy6e8CarIhscYQlNmw5e3G36y7l7Y21Ao= @@ -739,11 +754,13 @@ github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 h1:clC1lXBpe2kTj2VHdaI github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU= github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.10.1/go.mod h1:AY7fTTXNdv/aJ2O5jwpxAPOWUZ7hQAEvzN5Pf27BkQQ= +github.com/envoyproxy/go-control-plane v0.10.3 h1:xdCVXxEe0Y3FQith+0cj2irwZudqGYvecuLB1HtdexY= github.com/envoyproxy/go-control-plane v0.13.1 h1:vPfJZCkob6yTMEgS+0TwfTUfbHjfy/6vOJ8hUWX/uXE= github.com/envoyproxy/go-control-plane v0.13.1/go.mod h1:X45hY0mufo6Fd0KW3rqsGvQMw58jvjymeCzBU3mWyHw= github.com/envoyproxy/go-control-plane/envoy v1.32.4 h1:jb83lalDRZSpPWW2Z7Mck/8kXZ5CQAFYVjQcdVIr83A= github.com/envoyproxy/go-control-plane/envoy v1.32.4/go.mod h1:Gzjc5k8JcJswLjAx1Zm+wSYE20UrLtt7JZMWiWQXQEw= github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws= +github.com/envoyproxy/protoc-gen-validate v0.9.1 h1:PS7VIOgmSVhWUEeZwTe7z7zouA22Cr590PzXKbZHOVY= github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew= github.com/envoyproxy/protoc-gen-validate v1.1.0 h1:tntQDh69XqOCOZsDz0lVJQez/2L6Uu2PdjCQwWCJ3bM= github.com/envoyproxy/protoc-gen-validate v1.1.0/go.mod h1:sXRDRVmzEbkM7CVcM06s9shE/m23dg3wzjl0UWqJ2q4= @@ -809,6 +826,7 @@ github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlnd github.com/go-zookeeper/zk v1.0.2 h1:4mx0EYENAdX/B/rbunjlt5+4RTA/a9SMHBRuSKdGxPM= github.com/go-zookeeper/zk v1.0.2/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw= github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA= +github.com/godbus/dbus/v5 v5.0.6 h1:mkgN1ofwASrYnJ5W6U/BxG15eXXXjirgZc7CLqkcaro= github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/googleapis v1.4.1 h1:1Yx4Myt7BxzvUr5ldGSbwYiZG6t9wGBZ+8/fX3Wvtq0= github.com/gogo/status v1.1.0 h1:+eIkrewn5q6b30y+g/BJINVVdi2xH7je5MPJ3ZPK3JA= @@ -828,6 +846,7 @@ github.com/google/go-pkcs11 v0.3.0 h1:PVRnTgtArZ3QQqTGtbtjtnIkzl2iY2kt24yqbrf7td github.com/google/go-pkcs11 v0.3.0/go.mod h1:6eQoGcuNJpa7jnd5pMGdkSaQpNDYvPlXWMcjXXThLlY= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= +github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA= @@ -871,12 +890,15 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1ns github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 h1:X5VWvz21y3gzm9Nw/kaUeku/1+uBhcekkmy4IkffJww= github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1/go.mod h1:Zanoh4+gvIgluNqcfMVTJueD4wSS5hT7zTt4Mrutd90= github.com/guptarohit/asciigraph v0.5.5 h1:ccFnUF8xYIOUPPY3tmdvRyHqmn1MYI9iv1pLKX+/ZkQ= +github.com/hashicorp/consul/api v1.10.1 h1:MwZJp86nlnL+6+W1Zly4JUuVn9YHhMggBirMpHGD7kw= github.com/hashicorp/consul/api v1.10.1/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= github.com/hashicorp/consul/api v1.11.0/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= github.com/hashicorp/consul/api v1.12.0 h1:k3y1FYv6nuKyNTqj6w9gXOx5r5CfLj/k/euUeBXj1OY= +github.com/hashicorp/consul/sdk v0.3.0 h1:UOxjlb4xVNF93jak1mzzoBatyFju9nrkxpVwIp/QqxQ= github.com/hashicorp/consul/sdk v0.8.0 h1:OJtKBtEjboEZvG6AOUdh4Z1Zbyu0WcxQ0qatRrZHTVU= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.16.2 h1:K4ev2ib4LdQETX5cSZBG0DVLk1jwGqSPXBjdah3veNs= github.com/hashicorp/go-hclog v0.16.2/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-hclog v1.0.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-hclog v1.2.0 h1:La19f8d7WIlm4ogzNHB0JGqs5AUDAZ2UfCY4sJXcJdM= @@ -888,11 +910,15 @@ github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sL github.com/hashicorp/go-syslog v1.0.0 h1:KaodqZuhUoZereWVIYmpUgZysurB1kBLX2j0MwMrUAE= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1 h1:sNCoNyDEvN1xa+X0baata4RdcpKwcMS6DH+xwfqPgjw= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/logutils v1.0.0 h1:dLEQVugN8vlakKOUE3ihGLTZJRB4j+M2cdTm/ORI65Y= +github.com/hashicorp/mdns v1.0.0 h1:WhIgCr5a7AaVH6jPUwjtRuuE7/RDufnUvzIr48smyxs= github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= github.com/hashicorp/mdns v1.0.4 h1:sY0CMhFmjIPDMlTB+HfymFHCaYLhgifZ0QhjaYKD/UQ= +github.com/hashicorp/memberlist v0.1.3 h1:EmmoJme1matNzb+hMpDuR/0sbJSUisxyqBGG676r31M= github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/memberlist v0.3.0 h1:8+567mCcFDnS5ADl7lrpxPMWiFCElyUEeW0gtj34fMA= +github.com/hashicorp/serf v0.9.5 h1:EBWvyu9tcRszt3Bxp3KNssBMP1KuHWyO51lz9+786iM= github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= github.com/hashicorp/serf v0.9.7 h1:hkdgbqizGQHuU5IPqYM1JdSMV8nKfpuOnZYXssk9muY= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= @@ -901,10 +927,12 @@ github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq github.com/hudl/fargo v1.4.0 h1:ZDDILMbB37UlAVLlWcJ2Iz1XuahZZTDZfdCKeclfq2s= github.com/hudl/fargo v1.4.0/go.mod h1:9Ai6uvFy5fQNq6VPKtg+Ceq1+eTY4nKUlR2JElEOcDo= github.com/hydrogen18/memlistener v1.0.0 h1:JR7eDj8HD6eXrc5fWLbSUnfcQFL06PYvCc0DKQnWfaU= +github.com/iancoleman/strcase v0.2.0 h1:05I4QRnGpI0m37iZQRuskXh+w77mr6Z41lwQzuHLwW0= github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSASxEI= github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2 h1:rcanfLhLDA8nozr/K289V1zcntHr3V+SHlXwzz1ZI2g= github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ= +github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= github.com/imkira/go-interpol v1.1.0 h1:KIiKr0VSG2CUW1hl1jpiyuzuJeKUUpC8iM1AIE7N1Vk= github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= @@ -931,6 +959,7 @@ github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkr github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= +github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.2.0 h1:J2SLSdy7HgElq8ekSl2Mxh6vrRNFxqbXGenYH2I02Vs= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/txtarfs v0.0.0-20210218200122-0702f000015a h1:8NZHLa6Gp0hW6xJ0c3F1Kse7dJw30fOcDzHuF9sLbnE= @@ -1005,6 +1034,8 @@ github.com/miekg/pkcs11 v1.0.3 h1:iMwmD7I5225wv84WxIG/bmxz9AXjWvTWIbM/TYHvWtw= github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI= github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= +github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= +github.com/mitchellh/cli v1.0.0 h1:iGBIsUe3+HZ/AD/Vd7DErOt5sU9fa8Uj7A2s1aggv1Y= github.com/mitchellh/cli v1.1.0 h1:tEElEatulEHDeedTxwckzyYMA5c86fbmNIUL1hBIiTg= github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ= github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc= @@ -1017,6 +1048,7 @@ github.com/mitchellh/reflectwalk v1.0.1 h1:FVzMWA5RllMAKIdUSC8mdWo3XtwoecrH79BY7 github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY= github.com/mmcloughlin/profile v0.1.1 h1:jhDmAqPyebOsVDOCICJoINoLb/AnLBaUw58nFzxWS2w= github.com/moby/sys/mountinfo v0.4.1 h1:1O+1cHA1aujwEwwVMa2Xm2l+gIpUHyd3+D+d7LZh1kM= +github.com/moby/sys/mountinfo v0.5.0 h1:2Ks8/r6lopsxWi9m58nlwjaeSzUX9iiL1vj5qB/9ObI= github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= @@ -1027,6 +1059,7 @@ github.com/mozilla/scribe v0.0.0-20180711195314-fb71baf557c1 h1:29NKShH4TWd3lxCD github.com/mozilla/tls-observatory v0.0.0-20210609171429-7bc42856d2e5 h1:0KqC6/sLy7fDpBdybhVkkv4Yz+PmB7c9Dz9z3dLW804= github.com/mozilla/tls-observatory v0.0.0-20210609171429-7bc42856d2e5/go.mod h1:FUqVoUPHSEdDR0MnFM3Dh8AU0pZHLXUD127SAJGER/s= github.com/mrunalp/fileutils v0.5.0 h1:NKzVxiH7eSk+OQ4M+ZYW1K6h27RUV3MI6NUTsHhU6Z4= +github.com/mrunalp/fileutils v0.5.1 h1:F+S7ZlNKnrwHfSwdlgNSkKo67ReVf8o9fel6C3dkm/Q= github.com/mrunalp/fileutils v0.5.1/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= @@ -1054,6 +1087,7 @@ github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0C github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 h1:3snG66yBm59tKhhSPQrQ/0bCrv1LQbKt40LnUPiUxdc= github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/selinux v1.8.2 h1:c4ca10UMgRcvZ6h0K4HtS15UaVSBEaE+iln2LVpAuGc= +github.com/opencontainers/selinux v1.10.0 h1:rAiKF8hTcgLI3w0DHm6i0ylVVcOrlgR1kK99DRLDhyU= github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492 h1:lM6RxxfUMrYL/f8bWEUqdXrANWtrL7Nndbm9iFN0DlU= github.com/opentracing/basictracer-go v1.0.0 h1:YyUAhaEfjoWXclZVJ9sGoNct7j4TVk7lZWlQw5UXuoo= @@ -1115,11 +1149,15 @@ github.com/pion/udp v0.1.1 h1:8UAPvyqmsxK8oOjloDk4wUt63TzFe9WEJkg5lChlj7o= github.com/pion/udp v0.1.1/go.mod h1:6AFo+CMdKQm7UiA0eUPA8/eVCTx8jBIITLZHc9DWX5M= github.com/pion/webrtc/v3 v3.1.42 h1:wJEQFIXVanptnQcHOLTuIo4AtGB2+mG2x4OhIhnITOA= github.com/pion/webrtc/v3 v3.1.42/go.mod h1:ffD9DulDrPxyWvDPUIPAOSAWx9GUlOExiJPf7cCcMLA= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A= +github.com/pkg/profile v1.2.1 h1:F++O52m40owAmADcojzM+9gyjmMOY/T4oYJkgFDH8RE= +github.com/pkg/sftp v1.13.1 h1:I2qBYMChEhIjOgazfJmV3/mZM256btk6wkCDRmW7JYs= github.com/pkg/sftp v1.13.7 h1:uv+I3nNJvlKZIQGSr8JVQLNHFU9YhhNpvC14Y6KgmSM= github.com/pkg/sftp v1.13.7/go.mod h1:KMKI0t3T6hfA+lTR/ssZdunHo+uwq7ghoN09/FSu3DY= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= +github.com/posener/complete v1.1.1 h1:ccV59UEOTzVDnDUEFdT95ZzHVZ+5+158q8+SJb2QV5w= github.com/posener/complete v1.2.3 h1:NP0eAhjcjImqslEwo/1hq7gpajME0fTLTezBKDqfXqo= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= @@ -1160,16 +1198,19 @@ github.com/sagikazarmark/crypt v0.6.0/go.mod h1:U8+INwJo3nBv1m6A/8OBXAq7Jnpspk5A github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da h1:p3Vo3i64TCLY7gIfzeQaUJ+kppEO5WQG3cL8iE8tGHU= github.com/sanity-io/litter v1.5.5 h1:iE+sBxPBzoK6uaEP5Lt3fHNgpKcHXc/A2HGETy0uJQo= +github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4= github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY= github.com/schollz/closestmatch v2.1.0+incompatible h1:Uel2GXEpJqOWBrlyI+oY9LTiyyjYS17cCYRqP13/SHk= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= github.com/seccomp/libseccomp-golang v0.9.1 h1:NJjM5DNFOs0s3kYE1WUOr6G8V97sdt46rlXTMfXGWBo= +github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646 h1:RpforrEYXWkmGwJHIGnLZ3tTWStkjVVstwzNGqxX2Ds= github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= github.com/segmentio/fasthash v1.0.3 h1:EI9+KE1EwvMLBWwjpRDc+fEM+prwxDYbslddQGtrmhM= github.com/sei-protocol/go-ethereum v1.15.7-sei-12 h1:3Wj5nU7X0+mKcDho6mwf0leVytQWmNEq6xFv9Wr+HOs= github.com/sei-protocol/go-ethereum v1.15.7-sei-12/go.mod h1:+S9k+jFzlyVTNcYGvqFhzN/SFhI6vA+aOY4T5tLSPL0= +github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/shirou/gopsutil/v3 v3.22.9/go.mod h1:bBYl1kjgEJpWpxeHmLI+dVHWtyAwfcmSBLDsp2TNT8A= github.com/shirou/gopsutil/v3 v3.23.2 h1:PAWSuiAszn7IhPMBtXsbSCafej7PqUOvY6YywlQUExU= @@ -1191,6 +1232,7 @@ github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4 github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v1.3.0/go.mod h1:BrRVncBjOJa/eUcVVm9CE+oC6as8k+VYr4NY7WCi9V4= github.com/spf13/cobra v1.6.0/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM= github.com/spiffe/go-spiffe/v2 v2.5.0 h1:N2I01KCUkv1FAjZXJMwh95KK1ZIQLYbPfhaxw8WS0hE= @@ -1211,10 +1253,12 @@ github.com/tdewolff/test v1.0.7 h1:8Vs0142DmPFW/bQeHRP3MV19m1gvndjUb1sn8yy74LM= github.com/tidwall/sjson v1.1.4 h1:bTSsPLdAYF5QNLSwYsKfBKKTnlGbIuhqL3CpRsjzGhg= github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk= github.com/tklauser/numcpus v0.4.0/go.mod h1:1+UI3pD8NW14VMwdgJNJ1ESk2UnwhAnz5hMwiKKqXCQ= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966 h1:j6JEOq5QWFker+d7mFQYOhjTZonQ7YkLTHm56dbn+yM= github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce h1:fb190+cK2Xz/dvi9Hv8eCYJYvIGUTN2/KLq1pT6CjEc= github.com/torquem-ch/mdbx-go v0.27.5 h1:bbhXQGFCmoxbRDXKYEJwxSOOTeBKwoD4pFBUpK9+V1g= github.com/torquem-ch/mdbx-go v0.27.5/go.mod h1:T2fsoJDVppxfAPTLd1svUgH1kpPmeXdPESmroSHcL1E= +github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926 h1:G3dpKMzFDjgEh2q1Z7zUUtKa8ViPtH+ocF0bE0g00O8= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ= github.com/tyler-smith/go-bip39 v1.0.2 h1:+t3w+KwLXO6154GNJY+qUtIxLTmFjfUmpguQT1OlOT8= github.com/uptrace/opentelemetry-go-extra/otelgorm v0.3.2 h1:Jjn3zoRz13f8b1bR6LrXWglx93Sbh4kYfwgmPju3E2k= @@ -1269,16 +1313,21 @@ go.einride.tech/aip v0.68.0 h1:4seM66oLzTpz50u4K1zlJyOXQ3tCzcJN7I22tKkjipw= go.einride.tech/aip v0.68.0/go.mod h1:7y9FF8VtPWqpxuAxl0KQWqaULxW4zFIesD6zF5RIHHg= go.einride.tech/aip v0.68.1 h1:16/AfSxcQISGN5z9C5lM+0mLYXihrHbQ1onvYTr93aQ= go.einride.tech/aip v0.68.1/go.mod h1:XaFtaj4HuA3Zwk9xoBtTWgNubZ0ZZXv9BZJCkuKuWbg= +go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738 h1:VcrIfasaLFkyjk6KNlXQSzO+B0fZcnECiDrKJsfxka0= go.etcd.io/etcd v0.0.0-20200513171258-e048e166ab9c h1:/RwRVN9EdXAVtdHxP7Ndn/tfmM9/goiwU0QTnLBgS4w= +go.etcd.io/etcd/api/v3 v3.5.0 h1:GsV3S+OfZEOCNXdtNkBSR7kgLobAa/SO6tCxRa0GAYw= go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/api/v3 v3.5.4 h1:OHVyt3TopwtUQ2GKdd5wu3PmmipR4FTwCqoEjSyRdIc= go.etcd.io/etcd/api/v3 v3.5.4/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A= +go.etcd.io/etcd/client/pkg/v3 v3.5.0 h1:2aQv6F436YnN7I4VbI8PPYrBhu+SmrTaADcf8Mi/6PU= go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/pkg/v3 v3.5.4 h1:lrneYvz923dvC14R54XcA7FXoZ3mlGZAgmwhfm7HqOg= go.etcd.io/etcd/client/pkg/v3 v3.5.4/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v2 v2.305.0 h1:ftQ0nOOHMcbMS3KIaDQ0g5Qcd6bhaBrQT6b89DfwLTs= go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs= go.etcd.io/etcd/client/v2 v2.305.4 h1:Dcx3/MYyfKcPNLpR4VVQUP5KgYrBeJtktBwEKkw08Ao= go.etcd.io/etcd/client/v2 v2.305.4/go.mod h1:Ud+VUwIi9/uQHOMA+4ekToJ12lTxlv0zB/+DHwTGEbU= +go.etcd.io/etcd/client/v3 v3.5.0 h1:62Eh0XOro+rDwkrypAGDfgmNh5Joq+z+W9HZdlXMzek= go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0= go.etcd.io/etcd/client/v3 v3.5.4 h1:p83BUL3tAYS0OT/r0qglgc3M1JjhM0diV8DSWAhVXv4= go.etcd.io/etcd/client/v3 v3.5.4/go.mod h1:ZaRkVgBZC+L+dLCjTcF1hRXpgZXQPOvnA/Ak/gq3kiY= @@ -1390,6 +1439,7 @@ golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/telemetry v0.0.0-20251008203120-078029d740a8 h1:LvzTn0GQhWuvKH/kVRS3R3bVAsdQWI7hvfLHGgh9+lU= golang.org/x/telemetry v0.0.0-20251008203120-078029d740a8/go.mod h1:Pi4ztBfryZoJEkyFTI5/Ocsu2jXyDr6iSdgJiYE/uwE= +golang.org/x/telemetry v0.0.0-20260109210033-bd525da824e2 h1:O1cMQHRfwNpDfDJerqRoE2oD+AFlyid87D40L/OkkJo= golang.org/x/telemetry v0.0.0-20260109210033-bd525da824e2/go.mod h1:b7fPSJ0pKZ3ccUh8gnTONJxhn3c/PS6tyzQvyqw4iA8= golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= @@ -1407,6 +1457,7 @@ golang.org/x/tools v0.15.0/go.mod h1:hpksKq4dtpQWS1uQ61JkdqWM3LscIS6Slf+VVkm+wQk golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= golang.org/x/tools v0.37.0/go.mod h1:MBN5QPQtLMHVdvsbtarmTNukZDdgwdwlO5qGacAzF0w= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.11.0 h1:f1IJhK4Km5tBJmaiJXtk/PkL4cdVX6J+tGiM187uT5E= @@ -1419,6 +1470,7 @@ google.golang.org/api v0.237.0/go.mod h1:cOVEm2TpdAGHL2z+UwyS+kmlGr3bVWQQ6sYEqkK google.golang.org/api v0.239.0/go.mod h1:cOVEm2TpdAGHL2z+UwyS+kmlGr3bVWQQ6sYEqkKje50= google.golang.org/api v0.247.0 h1:tSd/e0QrUlLsrwMKmkbQhYVa109qIintOls2Wh6bngc= google.golang.org/api v0.247.0/go.mod h1:r1qZOPmxXffXg6xS5uhx16Fa/UFY8QU/K4bfKrnvovM= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= @@ -1458,6 +1510,7 @@ google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojt google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= google.golang.org/protobuf v1.36.7/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= +gopkg.in/cheggaaa/pb.v1 v1.0.25 h1:Ev7yu1/f6+d+b3pi5vPdRPc6nNtP1umSfcWiEfRqv6I= gopkg.in/cheggaaa/pb.v1 v1.0.28 h1:n1tBJnnK2r7g9OW2btFH91V92STTUevLXYFb8gy9EMk= gopkg.in/errgo.v2 v2.1.0 h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8= gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= @@ -1476,6 +1529,7 @@ gorm.io/driver/postgres v1.6.0 h1:2dxzU8xJ+ivvqTRph34QX+WrRaJlmfyPqXmoGVjMBa4= gorm.io/driver/postgres v1.6.0/go.mod h1:vUw0mrGgrTK+uPHEhAdV4sfFELrByKVGnaVRkXDhtWo= gorm.io/gorm v1.30.1 h1:lSHg33jJTBxs2mgJRfRZeLDG+WZaHYCk3Wtfl6Ngzo4= gorm.io/gorm v1.30.1/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE= +gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= k8s.io/api v0.35.0 h1:iBAU5LTyBI9vw3L5glmat1njFK34srdLmktWwLTprlY= k8s.io/api v0.35.0/go.mod h1:AQ0SNTzm4ZAczM03QH42c7l3bih1TbAXYo0DkF8ktnA= diff --git a/sei-cosmos/storev2/rootmulti/store.go b/sei-cosmos/storev2/rootmulti/store.go index 1a9e532a1c..e997232cd2 100644 --- a/sei-cosmos/storev2/rootmulti/store.go +++ b/sei-cosmos/storev2/rootmulti/store.go @@ -854,6 +854,7 @@ func (rs *Store) Snapshot(height uint64, protoWriter protoio.Writer) error { keySizePerStore[currentStoreName] += int64(len(item.Key)) valueSizePerStore[currentStoreName] += int64(len(item.Value)) numKeysPerStore[currentStoreName] += 1 + telemetry.IncrCounter(1, "state_sync", "num_keys_exported") case string: if err := protoWriter.WriteMsg(&snapshottypes.SnapshotItem{ Item: &snapshottypes.SnapshotItem_Store{ From b26dca2e78c9d7c6200a74d68c27df7cf92587a2 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Thu, 12 Feb 2026 08:37:59 -0800 Subject: [PATCH 2/6] Add more metrics for snapshot creation --- sei-db/state_db/sc/memiavl/db.go | 11 ++++++-- sei-db/state_db/sc/memiavl/metrics.go | 36 ++++++++++++++++++++++++- sei-db/state_db/sc/memiavl/multitree.go | 1 + sei-db/state_db/sc/memiavl/snapshot.go | 33 +---------------------- 4 files changed, 46 insertions(+), 35 deletions(-) diff --git a/sei-db/state_db/sc/memiavl/db.go b/sei-db/state_db/sc/memiavl/db.go index 6be6192477..285168e467 100644 --- a/sei-db/state_db/sc/memiavl/db.go +++ b/sei-db/state_db/sc/memiavl/db.go @@ -654,6 +654,7 @@ func (db *DB) Commit() (version int64, _err error) { // Rewrite tree snapshot if applicable db.rewriteIfApplicable(v) db.tryTruncateWAL() + otelMetrics.CurrentSnapshotHeight.Record(context.Background(), db.MultiTree.SnapshotVersion()) return v, nil } @@ -740,7 +741,10 @@ func (db *DB) RewriteSnapshot(ctx context.Context) error { writeStart := time.Now() err := db.WriteSnapshotWithRateLimit(ctx, path, db.snapshotWriterPool, db.opts.SnapshotWriteRateMBps) writeElapsed := time.Since(writeStart).Seconds() - + otelMetrics.SnapshotRewriteLatency.Record( + context.Background(), + writeElapsed, + ) if err != nil { db.logger.Error("snapshot write failed, cleaning up temporary directory", "tmpDir", tmpDir, @@ -884,7 +888,7 @@ func (db *DB) rewriteSnapshotBackground() error { defer close(ch) startTime := time.Now() cloned.logger.Info("start rewriting snapshot", "version", cloned.Version()) - + otelMetrics.SnapshotCreationCount.Add(context.Background(), 1) rewriteStart := time.Now() if err := cloned.RewriteSnapshot(ctx); err != nil { cloned.logger.Error("failed to rewrite snapshot", "error", err, "elapsed", time.Since(rewriteStart).Seconds()) @@ -928,6 +932,7 @@ func (db *DB) rewriteSnapshotBackground() error { cloned.logger.Info("loaded multitree after snapshot", "elapsed", time.Since(loadStart).Seconds()) // do a best effort catch-up, will do another final catch-up in main thread. + replayStartTime := time.Now() if wal := db.GetWAL(); wal != nil { catchupStart := time.Now() if err := mtree.Catchup(ctx, wal, db.walIndexDelta, 0); err != nil { @@ -939,6 +944,8 @@ func (db *DB) rewriteSnapshotBackground() error { } ch <- snapshotResult{mtree: mtree} + replayTimeElapsed := time.Since(replayStartTime).Seconds() + otelMetrics.CatchupReplayLatency.Record(context.Background(), replayTimeElapsed) totalElapsed := time.Since(startTime).Seconds() cloned.logger.Info("snapshot rewrite process completed", "duration_sec", totalElapsed, "duration_min", totalElapsed/60) otelMetrics.SnapshotCreationLatency.Record( diff --git a/sei-db/state_db/sc/memiavl/metrics.go b/sei-db/state_db/sc/memiavl/metrics.go index 0baefaa2c1..c0fbfc0c62 100644 --- a/sei-db/state_db/sc/memiavl/metrics.go +++ b/sei-db/state_db/sc/memiavl/metrics.go @@ -11,6 +11,12 @@ var ( otelMetrics = struct { RestartLatency metric.Float64Histogram SnapshotCreationLatency metric.Float64Histogram + SnapshotRewriteLatency metric.Float64Histogram + SnapshotCreationCount metric.Int64Counter + SnapshotPruneLatency metric.Float64Histogram + CatchupReplayLatency metric.Float64Histogram + CatchupReplayNumBlocks metric.Int64Counter + CurrentSnapshotHeight metric.Int64Gauge CommitLatency metric.Float64Histogram ApplyChangesetLatency metric.Float64Histogram NumOfKVPairs metric.Int64Counter @@ -24,9 +30,37 @@ var ( )), SnapshotCreationLatency: must(meter.Float64Histogram( "memiavl_snapshot_creation_latency", - metric.WithDescription("Time taken to create memiavl snapshot"), + metric.WithDescription("Total time taken to create memiavl snapshot + replay"), metric.WithUnit("s"), )), + SnapshotRewriteLatency: must(meter.Float64Histogram( + "memiavl_snapshot_rewrite_latency", + metric.WithDescription("Time taken to write to the new memiavl snapshot"), + metric.WithUnit("s"), + )), + SnapshotCreationCount: must(meter.Int64Counter( + "memiavl_snapshot_creation_count", + metric.WithDescription("Total num of times memiavl snapshot creation happens"), + metric.WithUnit("By"))), + SnapshotPruneLatency: must(meter.Float64Histogram( + "memiavl_snapshot_prune_latency", + metric.WithDescription("Time taken to prune memiavl snapshot"), + metric.WithUnit("s"), + )), + CatchupReplayLatency: must(meter.Float64Histogram( + "memiavl_snapshot_catchup_replay_latency", + metric.WithDescription("Time taken to catchup and replay after snapshot rewrite"), + metric.WithUnit("s"), + )), + CatchupReplayNumBlocks: must(meter.Int64Counter( + "memiavl_snapshot_catchup_replay_num_blocks", + metric.WithDescription("Num of blocks memIAVL has replayed after snapshot creation"), + metric.WithUnit("{count}"), + )), + CurrentSnapshotHeight: must(meter.Int64Gauge( + "memiavl_current_snapshot_height", + metric.WithDescription("Current snapshot height"), + )), CommitLatency: must(meter.Float64Histogram( "memiavl_commit_latency", metric.WithDescription("Time taken to commit"), diff --git a/sei-db/state_db/sc/memiavl/multitree.go b/sei-db/state_db/sc/memiavl/multitree.go index dff9954f32..88c2727619 100644 --- a/sei-db/state_db/sc/memiavl/multitree.go +++ b/sei-db/state_db/sc/memiavl/multitree.go @@ -438,6 +438,7 @@ func (t *MultiTree) Catchup(ctx context.Context, stream wal.ChangelogWAL, delta t.lastCommitInfo.Version = entry.Version t.lastCommitInfo.StoreInfos = []proto.StoreInfo{} replayCount++ + otelMetrics.CatchupReplayNumBlocks.Add(context.Background(), 1) if replayCount%1000 == 0 { t.logger.Info(fmt.Sprintf("Replayed %d changelog entries\n", replayCount)) } diff --git a/sei-db/state_db/sc/memiavl/snapshot.go b/sei-db/state_db/sc/memiavl/snapshot.go index 9c4b90eeb5..2c88372dd4 100644 --- a/sei-db/state_db/sc/memiavl/snapshot.go +++ b/sei-db/state_db/sc/memiavl/snapshot.go @@ -680,15 +680,7 @@ type snapshotWriter struct { wg sync.WaitGroup // Wait for all writer goroutines // Pipeline metrics for each channel - maxKvFill int - maxLeafFill int - maxBranchFill int - kvFillSum int64 - leafFillSum int64 - branchFillSum int64 - kvFillCount int64 - leafFillCount int64 - branchFillCount int64 + lastMetricsReport time.Time } @@ -862,22 +854,6 @@ func (w *snapshotWriter) writeKeyValueDirect(key, value []byte) error { // writeLeaf sends leaf and KV write operations to the pipeline func (w *snapshotWriter) writeLeaf(version uint32, key, value, hash []byte) error { - // Track channel fill metrics for all channels - kvFill := len(w.kvChan) - leafFill := len(w.leafChan) - - if kvFill > w.maxKvFill { - w.maxKvFill = kvFill - } - if leafFill > w.maxLeafFill { - w.maxLeafFill = leafFill - } - - atomic.AddInt64(&w.kvFillSum, int64(kvFill)) - atomic.AddInt64(&w.kvFillCount, 1) - atomic.AddInt64(&w.leafFillSum, int64(leafFill)) - atomic.AddInt64(&w.leafFillCount, 1) - // Calculate key offset BEFORE sending to KV channel keyOffset := w.kvsOffset keyLen := uint32(len(key)) //nolint:gosec @@ -940,13 +916,6 @@ func (w *snapshotWriter) writeLeafDirect(version uint32, keyLen uint32, keyOffse // writeBranch sends a branch write operation to the pipeline func (w *snapshotWriter) writeBranch(version, size uint32, height, preTrees uint8, keyLeaf uint32, hash []byte) error { - // Track channel fill metrics - branchFill := len(w.branchChan) - if branchFill > w.maxBranchFill { - w.maxBranchFill = branchFill - } - atomic.AddInt64(&w.branchFillSum, int64(branchFill)) - atomic.AddInt64(&w.branchFillCount, 1) // Make copy of hash since we're sending to another goroutine hashCopy := make([]byte, len(hash)) From d629834adeee5d94c2a3c0a51dbd049c468fc856 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Thu, 12 Feb 2026 08:41:38 -0800 Subject: [PATCH 3/6] Add prune latency metrics --- sei-db/state_db/sc/memiavl/db.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sei-db/state_db/sc/memiavl/db.go b/sei-db/state_db/sc/memiavl/db.go index 285168e467..25ec77f06b 100644 --- a/sei-db/state_db/sc/memiavl/db.go +++ b/sei-db/state_db/sc/memiavl/db.go @@ -522,7 +522,9 @@ func (db *DB) pruneSnapshots() { db.logger.Info("pruneSnapshots started") startTime := time.Now() defer func() { - db.logger.Info("pruneSnapshots completed", "duration_sec", fmt.Sprintf("%.2fs", time.Since(startTime).Seconds())) + pruneLatency := time.Since(startTime).Seconds() + otelMetrics.SnapshotPruneLatency.Record(context.Background(), pruneLatency) + db.logger.Info("pruneSnapshots completed", "duration_sec", fmt.Sprintf("%.2fs", pruneLatency)) }() currentVersion, err := currentVersion(db.dir) From 3a9ac255c20f3d25f54c2a5fd67468e248a2dd44 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Thu, 12 Feb 2026 08:45:26 -0800 Subject: [PATCH 4/6] Add pruning count metrics --- sei-db/state_db/sc/memiavl/db.go | 4 +++- sei-db/state_db/sc/memiavl/metrics.go | 8 +++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/sei-db/state_db/sc/memiavl/db.go b/sei-db/state_db/sc/memiavl/db.go index 25ec77f06b..ff37195c68 100644 --- a/sei-db/state_db/sc/memiavl/db.go +++ b/sei-db/state_db/sc/memiavl/db.go @@ -546,10 +546,12 @@ func (db *DB) pruneSnapshots() { } name := snapshotName(version) - db.logger.Info("prune snapshot", "name", name) if err := atomicRemoveDir(filepath.Join(db.dir, name)); err != nil { db.logger.Error("failed to prune snapshot", "err", err) + } else { + db.logger.Info("successfully pruned snapshot", "name", name) + otelMetrics.SnapshotPruneCount.Add(context.Background(), 1) } return false, nil diff --git a/sei-db/state_db/sc/memiavl/metrics.go b/sei-db/state_db/sc/memiavl/metrics.go index c0fbfc0c62..163ac4c581 100644 --- a/sei-db/state_db/sc/memiavl/metrics.go +++ b/sei-db/state_db/sc/memiavl/metrics.go @@ -14,6 +14,7 @@ var ( SnapshotRewriteLatency metric.Float64Histogram SnapshotCreationCount metric.Int64Counter SnapshotPruneLatency metric.Float64Histogram + SnapshotPruneCount metric.Int64Counter CatchupReplayLatency metric.Float64Histogram CatchupReplayNumBlocks metric.Int64Counter CurrentSnapshotHeight metric.Int64Gauge @@ -41,12 +42,17 @@ var ( SnapshotCreationCount: must(meter.Int64Counter( "memiavl_snapshot_creation_count", metric.WithDescription("Total num of times memiavl snapshot creation happens"), - metric.WithUnit("By"))), + metric.WithUnit("{count}"))), SnapshotPruneLatency: must(meter.Float64Histogram( "memiavl_snapshot_prune_latency", metric.WithDescription("Time taken to prune memiavl snapshot"), metric.WithUnit("s"), )), + SnapshotPruneCount: must(meter.Int64Counter( + "memiavl_snapshot_prune_count", + metric.WithDescription("Total number of snapshots pruned successfully"), + metric.WithUnit("{count}"), + )), CatchupReplayLatency: must(meter.Float64Histogram( "memiavl_snapshot_catchup_replay_latency", metric.WithDescription("Time taken to catchup and replay after snapshot rewrite"), From c52b76a51fbbfd0b16a3b3f65cc48e39e9caeaee Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Thu, 12 Feb 2026 09:21:19 -0800 Subject: [PATCH 5/6] Address comments --- sei-db/state_db/sc/memiavl/db.go | 33 ++++++++++-------- sei-db/state_db/sc/memiavl/metrics.go | 50 +++++++++++++-------------- 2 files changed, 44 insertions(+), 39 deletions(-) diff --git a/sei-db/state_db/sc/memiavl/db.go b/sei-db/state_db/sc/memiavl/db.go index ae99635ac5..f5c09f3313 100644 --- a/sei-db/state_db/sc/memiavl/db.go +++ b/sei-db/state_db/sc/memiavl/db.go @@ -468,7 +468,10 @@ func (db *DB) checkBackgroundSnapshotRewrite() error { if result.mtree == nil { // background snapshot rewrite failed + otelMetrics.SnapshotRewriteCount.Add(context.Background(), 1, metric.WithAttributes(attribute.String("success", "false"))) return fmt.Errorf("background snapshot rewriting failed: %w", result.err) + } else { + otelMetrics.SnapshotRewriteCount.Add(context.Background(), 1, metric.WithAttributes(attribute.String("success", "true"))) } // wait for potential pending writes to finish, to make sure we catch up to latest state. @@ -485,11 +488,15 @@ func (db *DB) checkBackgroundSnapshotRewrite() error { } // catchup the remaining entries in rlog + startTime := time.Now() if wal := db.GetWAL(); wal != nil { if err := result.mtree.Catchup(context.Background(), wal, db.walIndexDelta, 0); err != nil { return fmt.Errorf("catchup failed: %w", err) } } + replayElapsedTime := time.Since(startTime).Seconds() + otelMetrics.CatchupBeforeReloadLatency.Record(context.Background(), replayElapsedTime) + db.logger.Info("successfully replayed and caught up before switching to new memiavl snapshot", "version", db.MultiTree.Version(), "latency_sec", replayElapsedTime) // do the switch if err := db.reloadMultiTree(result.mtree); err != nil { @@ -546,14 +553,16 @@ func (db *DB) pruneSnapshots() { if err := atomicRemoveDir(filepath.Join(db.dir, name)); err != nil { db.logger.Error("failed to prune snapshot", "err", err) + otelMetrics.SnapshotPruneCount.Add(context.Background(), 1, metric.WithAttributes(attribute.String("success", "false"))) } else { db.logger.Info("successfully pruned snapshot", "name", name) - otelMetrics.SnapshotPruneCount.Add(context.Background(), 1) + otelMetrics.SnapshotPruneCount.Add(context.Background(), 1, metric.WithAttributes(attribute.String("success", "true"))) } return false, nil }); err != nil { db.logger.Error("fail to prune snapshots", "err", err) + otelMetrics.SnapshotPruneCount.Add(context.Background(), 1, metric.WithAttributes(attribute.String("success", "false"))) return } } @@ -742,10 +751,7 @@ func (db *DB) RewriteSnapshot(ctx context.Context) error { writeStart := time.Now() err := db.WriteSnapshotWithRateLimit(ctx, path, db.snapshotWriterPool, db.opts.SnapshotWriteRateMBps) writeElapsed := time.Since(writeStart).Seconds() - otelMetrics.SnapshotRewriteLatency.Record( - context.Background(), - writeElapsed, - ) + if err != nil { db.logger.Error("snapshot write failed, cleaning up temporary directory", "tmpDir", tmpDir, @@ -848,6 +854,7 @@ func (db *DB) rewriteIfApplicable(height int64) { if err := db.rewriteSnapshotBackground(); err != nil { db.logger.Error("failed to rewrite snapshot in background", "err", err) + otelMetrics.SnapshotRewriteCount.Add(context.Background(), 1, metric.WithAttributes(attribute.String("success", "false"))) } } } @@ -889,7 +896,6 @@ func (db *DB) rewriteSnapshotBackground() error { defer close(ch) startTime := time.Now() cloned.logger.Info("start rewriting snapshot", "version", cloned.Version()) - otelMetrics.SnapshotCreationCount.Add(context.Background(), 1) rewriteStart := time.Now() if err := cloned.RewriteSnapshot(ctx); err != nil { cloned.logger.Error("failed to rewrite snapshot", "error", err, "elapsed", time.Since(rewriteStart).Seconds()) @@ -922,7 +928,6 @@ func (db *DB) rewriteSnapshotBackground() error { cloned.logger.Info("loaded multitree after snapshot", "elapsed", time.Since(loadStart).Seconds()) // do a best effort catch-up, will do another final catch-up in main thread. - replayStartTime := time.Now() if wal := db.GetWAL(); wal != nil { catchupStart := time.Now() if err := mtree.Catchup(ctx, wal, db.walIndexDelta, 0); err != nil { @@ -930,17 +935,17 @@ func (db *DB) rewriteSnapshotBackground() error { ch <- snapshotResult{err: err} return } - cloned.logger.Info("finished best-effort catchup", "version", cloned.Version(), "latest", mtree.Version(), "elapsed", time.Since(catchupStart).Seconds()) + catchupElapsed := time.Since(catchupStart).Seconds() + otelMetrics.CatchupAfterRewriteLatency.Record(context.Background(), catchupElapsed) + cloned.logger.Info("finished best-effort catchup after snapshot rewrite", "version", cloned.Version(), "latest", mtree.Version(), "elapsed", catchupElapsed) } ch <- snapshotResult{mtree: mtree} - replayTimeElapsed := time.Since(replayStartTime).Seconds() - otelMetrics.CatchupReplayLatency.Record(context.Background(), replayTimeElapsed) - totalElapsed := time.Since(startTime).Seconds() - cloned.logger.Info("snapshot rewrite process completed", "duration_sec", totalElapsed, "duration_min", totalElapsed/60) - otelMetrics.SnapshotCreationLatency.Record( + totalRewriteElapsed := time.Since(startTime).Seconds() + cloned.logger.Info("snapshot rewrite process completed", "duration_sec", totalRewriteElapsed, "duration_min", totalRewriteElapsed/60) + otelMetrics.SnapshotRewriteLatency.Record( context.Background(), - totalElapsed, + totalRewriteElapsed, ) }() diff --git a/sei-db/state_db/sc/memiavl/metrics.go b/sei-db/state_db/sc/memiavl/metrics.go index 163ac4c581..4c4f5f586d 100644 --- a/sei-db/state_db/sc/memiavl/metrics.go +++ b/sei-db/state_db/sc/memiavl/metrics.go @@ -9,39 +9,34 @@ var ( meter = otel.Meter("seidb_memiavl") otelMetrics = struct { - RestartLatency metric.Float64Histogram - SnapshotCreationLatency metric.Float64Histogram - SnapshotRewriteLatency metric.Float64Histogram - SnapshotCreationCount metric.Int64Counter - SnapshotPruneLatency metric.Float64Histogram - SnapshotPruneCount metric.Int64Counter - CatchupReplayLatency metric.Float64Histogram - CatchupReplayNumBlocks metric.Int64Counter - CurrentSnapshotHeight metric.Int64Gauge - CommitLatency metric.Float64Histogram - ApplyChangesetLatency metric.Float64Histogram - NumOfKVPairs metric.Int64Counter - MemNodeTotalSize metric.Int64Gauge - NumOfMemNode metric.Int64Gauge + RestartLatency metric.Float64Histogram + SnapshotRewriteLatency metric.Float64Histogram + SnapshotRewriteCount metric.Int64Counter + SnapshotPruneLatency metric.Float64Histogram + SnapshotPruneCount metric.Int64Counter + CatchupAfterRewriteLatency metric.Float64Histogram + CatchupBeforeReloadLatency metric.Float64Histogram + CatchupReplayNumBlocks metric.Int64Counter + CurrentSnapshotHeight metric.Int64Gauge + CommitLatency metric.Float64Histogram + ApplyChangesetLatency metric.Float64Histogram + NumOfKVPairs metric.Int64Counter + MemNodeTotalSize metric.Int64Gauge + NumOfMemNode metric.Int64Gauge }{ RestartLatency: must(meter.Float64Histogram( "memiavl_restart_latency", metric.WithDescription("Time taken to restart the memiavl database"), metric.WithUnit("s"), )), - SnapshotCreationLatency: must(meter.Float64Histogram( - "memiavl_snapshot_creation_latency", - metric.WithDescription("Total time taken to create memiavl snapshot + replay"), - metric.WithUnit("s"), - )), SnapshotRewriteLatency: must(meter.Float64Histogram( "memiavl_snapshot_rewrite_latency", metric.WithDescription("Time taken to write to the new memiavl snapshot"), metric.WithUnit("s"), )), - SnapshotCreationCount: must(meter.Int64Counter( - "memiavl_snapshot_creation_count", - metric.WithDescription("Total num of times memiavl snapshot creation happens"), + SnapshotRewriteCount: must(meter.Int64Counter( + "memiavl_snapshot_rewrite_count", + metric.WithDescription("Total num of times memiavl snapshot rewrite attempts"), metric.WithUnit("{count}"))), SnapshotPruneLatency: must(meter.Float64Histogram( "memiavl_snapshot_prune_latency", @@ -50,14 +45,19 @@ var ( )), SnapshotPruneCount: must(meter.Int64Counter( "memiavl_snapshot_prune_count", - metric.WithDescription("Total number of snapshots pruned successfully"), + metric.WithDescription("Total number of snapshot prune attempts"), metric.WithUnit("{count}"), )), - CatchupReplayLatency: must(meter.Float64Histogram( - "memiavl_snapshot_catchup_replay_latency", + CatchupAfterRewriteLatency: must(meter.Float64Histogram( + "memiavl_snapshot_catchup_after_rewrite_latency", metric.WithDescription("Time taken to catchup and replay after snapshot rewrite"), metric.WithUnit("s"), )), + CatchupBeforeReloadLatency: must(meter.Float64Histogram( + "memiavl_snapshot_catchup_before_reload_latency", + metric.WithDescription("Time taken to catchup and replay before switch to new snapshot"), + metric.WithUnit("s"), + )), CatchupReplayNumBlocks: must(meter.Int64Counter( "memiavl_snapshot_catchup_replay_num_blocks", metric.WithDescription("Num of blocks memIAVL has replayed after snapshot creation"), From ab38167ffd36f4f01160b5048614e78da69cde73 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Fri, 13 Feb 2026 11:02:50 -0800 Subject: [PATCH 6/6] Address comments --- sei-db/state_db/sc/memiavl/db.go | 14 +++++++------- sei-db/state_db/sc/memiavl/db_test.go | 4 ++-- sei-db/state_db/sc/memiavl/metrics.go | 14 +++++++------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/sei-db/state_db/sc/memiavl/db.go b/sei-db/state_db/sc/memiavl/db.go index f5c09f3313..9497751ec9 100644 --- a/sei-db/state_db/sc/memiavl/db.go +++ b/sei-db/state_db/sc/memiavl/db.go @@ -468,10 +468,10 @@ func (db *DB) checkBackgroundSnapshotRewrite() error { if result.mtree == nil { // background snapshot rewrite failed - otelMetrics.SnapshotRewriteCount.Add(context.Background(), 1, metric.WithAttributes(attribute.String("success", "false"))) + otelMetrics.NumSnapshotRewriteAttempts.Add(context.Background(), 1, metric.WithAttributes(attribute.String("success", "false"))) return fmt.Errorf("background snapshot rewriting failed: %w", result.err) } else { - otelMetrics.SnapshotRewriteCount.Add(context.Background(), 1, metric.WithAttributes(attribute.String("success", "true"))) + otelMetrics.NumSnapshotRewriteAttempts.Add(context.Background(), 1, metric.WithAttributes(attribute.String("success", "true"))) } // wait for potential pending writes to finish, to make sure we catch up to latest state. @@ -553,16 +553,16 @@ func (db *DB) pruneSnapshots() { if err := atomicRemoveDir(filepath.Join(db.dir, name)); err != nil { db.logger.Error("failed to prune snapshot", "err", err) - otelMetrics.SnapshotPruneCount.Add(context.Background(), 1, metric.WithAttributes(attribute.String("success", "false"))) + otelMetrics.NumSnapshotPruneAttempts.Add(context.Background(), 1, metric.WithAttributes(attribute.String("success", "false"))) } else { db.logger.Info("successfully pruned snapshot", "name", name) - otelMetrics.SnapshotPruneCount.Add(context.Background(), 1, metric.WithAttributes(attribute.String("success", "true"))) + otelMetrics.NumSnapshotPruneAttempts.Add(context.Background(), 1, metric.WithAttributes(attribute.String("success", "true"))) } return false, nil }); err != nil { db.logger.Error("fail to prune snapshots", "err", err) - otelMetrics.SnapshotPruneCount.Add(context.Background(), 1, metric.WithAttributes(attribute.String("success", "false"))) + otelMetrics.NumSnapshotPruneAttempts.Add(context.Background(), 1, metric.WithAttributes(attribute.String("success", "false"))) return } } @@ -664,7 +664,7 @@ func (db *DB) Commit() (version int64, _err error) { // Rewrite tree snapshot if applicable db.rewriteIfApplicable(v) db.tryTruncateWAL() - otelMetrics.CurrentSnapshotHeight.Record(context.Background(), db.MultiTree.SnapshotVersion()) + otelMetrics.CurrentSnapshotHeight.Record(context.Background(), db.SnapshotVersion()) return v, nil } @@ -854,7 +854,7 @@ func (db *DB) rewriteIfApplicable(height int64) { if err := db.rewriteSnapshotBackground(); err != nil { db.logger.Error("failed to rewrite snapshot in background", "err", err) - otelMetrics.SnapshotRewriteCount.Add(context.Background(), 1, metric.WithAttributes(attribute.String("success", "false"))) + otelMetrics.NumSnapshotRewriteAttempts.Add(context.Background(), 1, metric.WithAttributes(attribute.String("success", "false"))) } } } diff --git a/sei-db/state_db/sc/memiavl/db_test.go b/sei-db/state_db/sc/memiavl/db_test.go index c1058db3c0..41231852b9 100644 --- a/sei-db/state_db/sc/memiavl/db_test.go +++ b/sei-db/state_db/sc/memiavl/db_test.go @@ -203,7 +203,7 @@ func TestSnapshotTriggerOnIntervalDiff(t *testing.T) { return db.snapshotRewriteChan != nil }, 100*time.Millisecond, 10*time.Millisecond, "rewrite should not start at height %d", i) // snapshot version should remain 0 until rewrite - require.EqualValues(t, 0, db.MultiTree.SnapshotVersion()) + require.EqualValues(t, 0, db.SnapshotVersion()) } // Wait for minimum time interval to elapse (1 second + buffer) @@ -223,7 +223,7 @@ func TestSnapshotTriggerOnIntervalDiff(t *testing.T) { }, 5*time.Second, 100*time.Millisecond) // After completion, snapshot version should be 5 - require.EqualValues(t, 5, db.MultiTree.SnapshotVersion()) + require.EqualValues(t, 5, db.SnapshotVersion()) require.NoError(t, db.Close()) } diff --git a/sei-db/state_db/sc/memiavl/metrics.go b/sei-db/state_db/sc/memiavl/metrics.go index 4c4f5f586d..ee8037e718 100644 --- a/sei-db/state_db/sc/memiavl/metrics.go +++ b/sei-db/state_db/sc/memiavl/metrics.go @@ -11,9 +11,9 @@ var ( otelMetrics = struct { RestartLatency metric.Float64Histogram SnapshotRewriteLatency metric.Float64Histogram - SnapshotRewriteCount metric.Int64Counter + NumSnapshotRewriteAttempts metric.Int64Counter SnapshotPruneLatency metric.Float64Histogram - SnapshotPruneCount metric.Int64Counter + NumSnapshotPruneAttempts metric.Int64Counter CatchupAfterRewriteLatency metric.Float64Histogram CatchupBeforeReloadLatency metric.Float64Histogram CatchupReplayNumBlocks metric.Int64Counter @@ -34,17 +34,17 @@ var ( metric.WithDescription("Time taken to write to the new memiavl snapshot"), metric.WithUnit("s"), )), - SnapshotRewriteCount: must(meter.Int64Counter( - "memiavl_snapshot_rewrite_count", - metric.WithDescription("Total num of times memiavl snapshot rewrite attempts"), + NumSnapshotRewriteAttempts: must(meter.Int64Counter( + "memiavl_num_snapshot_rewrite_attempts", + metric.WithDescription("Total num of memiavl snapshot rewrite attempts"), metric.WithUnit("{count}"))), SnapshotPruneLatency: must(meter.Float64Histogram( "memiavl_snapshot_prune_latency", metric.WithDescription("Time taken to prune memiavl snapshot"), metric.WithUnit("s"), )), - SnapshotPruneCount: must(meter.Int64Counter( - "memiavl_snapshot_prune_count", + NumSnapshotPruneAttempts: must(meter.Int64Counter( + "memiavl_num_snapshot_prune_attempts", metric.WithDescription("Total number of snapshot prune attempts"), metric.WithUnit("{count}"), )),