diff --git a/Cargo.lock b/Cargo.lock index 23c409f..50ab4b5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,15 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 - -[[package]] -name = "addr2line" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" -dependencies = [ - "gimli", -] +version = 4 [[package]] name = "adler" @@ -131,23 +122,28 @@ checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.118", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "attohttpc" version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f77d243921b0979fbbd728dd2d5162e68ac8252976797c24eb5b3a6af9090dc" dependencies = [ - "http", + "http 0.2.12", "log", - "native-tls", - "rustls", + "rustls 0.21.12", "serde", "serde_json", "url", - "webpki-roots", + "webpki-roots 0.25.4", ] [[package]] @@ -156,7 +152,7 @@ version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" dependencies = [ - "hermit-abi 0.1.19", + "hermit-abi", "libc", "winapi", ] @@ -179,7 +175,7 @@ dependencies = [ "quick-xml", "rust-ini", "serde", - "thiserror", + "thiserror 1.0.63", "time", "url", ] @@ -190,29 +186,20 @@ version = "0.25.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42fed2b9fca70f2908268d057a607f2a906f47edbf856ea8587de9038d264e22" dependencies = [ - "thiserror", + "thiserror 1.0.63", ] [[package]] -name = "backtrace" -version = "0.3.73" +name = "base64" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" -dependencies = [ - "addr2line", - "cc", - "cfg-if", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", -] +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] name = "base64" -version = "0.21.7" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "base64ct" @@ -297,6 +284,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "chrono" version = "0.4.38" @@ -364,7 +357,7 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" dependencies = [ - "getrandom", + "getrandom 0.2.15", "once_cell", "tiny-keccak", ] @@ -381,15 +374,23 @@ version = "9.0.0" dependencies = [ "advisory-lock", "anyhow", + "base64 0.22.1", + "chrono", "dotenv", "env_logger", "fs_extra", + "hmac", "inotify", "log", + "reqwest", "rust-s3", - "thiserror", + "serde_json", + "sha2", + "thiserror 1.0.63", "tokio", "tokio-cron-scheduler", + "tokio-util", + "urlencoding", ] [[package]] @@ -404,7 +405,7 @@ dependencies = [ "libcrio", "log", "log4rs", - "rand", + "rand 0.8.5", "serde", "serde_json", "tinytemplate", @@ -679,7 +680,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.118", ] [[package]] @@ -729,15 +730,44 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "wasip2", + "wasm-bindgen", ] [[package]] -name = "gimli" -version = "0.29.0" +name = "h2" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" +checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http 1.4.2", + "indexmap 2.4.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] [[package]] name = "hashbrown" @@ -766,12 +796,6 @@ dependencies = [ "libc", ] -[[package]] -name = "hermit-abi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" - [[package]] name = "hex" version = "0.4.3" @@ -818,6 +842,16 @@ dependencies = [ "itoa", ] +[[package]] +name = "http" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +dependencies = [ + "bytes", + "itoa", +] + [[package]] name = "http-body" version = "0.4.6" @@ -825,7 +859,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", - "http", + "http 0.2.12", + "pin-project-lite", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http 1.4.2", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http 1.4.2", + "http-body 1.0.1", "pin-project-lite", ] @@ -857,19 +914,55 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "http", - "http-body", + "http 0.2.12", + "http-body 0.4.6", "httparse", "httpdate", "itoa", "pin-project-lite", - "socket2", + "socket2 0.5.7", "tokio", "tower-service", "tracing", "want", ] +[[package]] +name = "hyper" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "h2", + "http 1.4.2", + "http-body 1.0.1", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http 1.4.2", + "hyper 1.6.0", + "hyper-util", + "rustls 0.23.40", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots 1.0.7", +] + [[package]] name = "hyper-tls" version = "0.5.0" @@ -877,12 +970,36 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ "bytes", - "hyper", + "hyper 0.14.30", "native-tls", "tokio", "tokio-native-tls", ] +[[package]] +name = "hyper-util" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c6995591a8f1380fcb4ba966a252a4b29188d51d2b89e3a252f5305be65aea8" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http 1.4.2", + "http-body 1.0.1", + "hyper 1.6.0", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2 0.6.4", + "tokio", + "tower-service", + "tracing", +] + [[package]] name = "iana-time-zone" version = "0.1.60" @@ -967,6 +1084,12 @@ dependencies = [ "generic-array", ] +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + [[package]] name = "is_terminal_polyfill" version = "1.70.1" @@ -990,18 +1113,19 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.70" +version = "0.3.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" +checksum = "2964e92d1d9dc3364cae4d718d93f227e3abb088e747d92e0395bfdedf1c12ca" dependencies = [ + "once_cell", "wasm-bindgen", ] [[package]] name = "libc" -version = "0.2.156" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5f43f184355eefb8d17fc948dbecf6c13be3c141f20d834ae842193a448c72a" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libcrio" @@ -1062,17 +1186,23 @@ dependencies = [ "log-mdc", "once_cell", "parking_lot", - "rand", + "rand 0.8.5", "serde", "serde-value", "serde_json", "serde_yaml", - "thiserror", + "thiserror 1.0.63", "thread-id", "typemap-ors", "winapi", ] +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + [[package]] name = "maybe-async" version = "0.2.10" @@ -1081,7 +1211,7 @@ checksum = "5cf92c10c7e361d6b99666ec1c6f9805b0bea2c3bd8c78dc6fe98ac5bd78db11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.118", ] [[package]] @@ -1096,15 +1226,6 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" -[[package]] -name = "minidom" -version = "0.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f45614075738ce1b77a1768912a60c0227525971b03e09122a05b8a34a2a6278" -dependencies = [ - "rxml", -] - [[package]] name = "minimal-lexical" version = "0.2.1" @@ -1122,14 +1243,13 @@ dependencies = [ [[package]] name = "mio" -version = "1.0.2" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" dependencies = [ - "hermit-abi 0.3.9", "libc", "wasi", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -1173,7 +1293,7 @@ checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.118", ] [[package]] @@ -1185,15 +1305,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "object" -version = "0.36.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b64972346851a39438c60b341ebc01bba47464ae329e55cf343eb93964efd9" -dependencies = [ - "memchr", -] - [[package]] name = "once_cell" version = "1.19.0" @@ -1223,7 +1334,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.118", ] [[package]] @@ -1299,7 +1410,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7676374caaee8a325c9e7a2ae557f216c5563a171d6997b0ef8a65af35147700" dependencies = [ "base64ct", - "rand_core", + "rand_core 0.6.4", "subtle", ] @@ -1356,9 +1467,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" dependencies = [ "unicode-ident", ] @@ -1373,6 +1484,61 @@ dependencies = [ "serde", ] +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls 0.23.40", + "socket2 0.5.7", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +dependencies = [ + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand 0.9.4", + "ring", + "rustc-hash", + "rustls 0.23.40", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2 0.5.7", + "tracing", + "windows-sys 0.59.0", +] + [[package]] name = "quote" version = "1.0.36" @@ -1382,6 +1548,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + [[package]] name = "rand" version = "0.8.5" @@ -1389,8 +1561,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha", - "rand_core", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", ] [[package]] @@ -1400,7 +1582,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", ] [[package]] @@ -1409,7 +1601,16 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom", + "getrandom 0.2.15", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", ] [[package]] @@ -1450,6 +1651,48 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-core", + "futures-util", + "h2", + "http 1.4.2", + "http-body 1.0.1", + "http-body-util", + "hyper 1.6.0", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls 0.23.40", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots 1.0.7", +] + [[package]] name = "ring" version = "0.17.8" @@ -1458,7 +1701,7 @@ checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" dependencies = [ "cc", "cfg-if", - "getrandom", + "getrandom 0.2.15", "libc", "spin", "untrusted", @@ -1484,19 +1727,18 @@ dependencies = [ "async-trait", "aws-creds", "aws-region", - "base64", + "base64 0.21.7", "bytes", "cfg-if", "futures", "hex", "hmac", - "http", - "hyper", + "http 0.2.12", + "hyper 0.14.30", "hyper-tls", "log", "maybe-async", "md5", - "minidom", "native-tls", "percent-encoding", "quick-xml", @@ -1504,7 +1746,7 @@ dependencies = [ "serde_derive", "serde_json", "sha2", - "thiserror", + "thiserror 1.0.63", "time", "tokio", "tokio-native-tls", @@ -1513,10 +1755,10 @@ dependencies = [ ] [[package]] -name = "rustc-demangle" -version = "0.1.24" +name = "rustc-hash" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" [[package]] name = "rustix" @@ -1539,10 +1781,34 @@ checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" dependencies = [ "log", "ring", - "rustls-webpki", + "rustls-webpki 0.101.7", "sct", ] +[[package]] +name = "rustls" +version = "0.23.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki 0.103.13", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +dependencies = [ + "web-time", + "zeroize", +] + [[package]] name = "rustls-webpki" version = "0.101.7" @@ -1554,21 +1820,21 @@ dependencies = [ ] [[package]] -name = "rxml" -version = "0.9.1" +name = "rustls-webpki" +version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a98f186c7a2f3abbffb802984b7f1dfd65dac8be1aafdaabbca4137f53f0dff7" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ - "bytes", - "rxml_validation", - "smartstring", + "ring", + "rustls-pki-types", + "untrusted", ] [[package]] -name = "rxml_validation" -version = "0.9.1" +name = "rustversion" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22a197350ece202f19a166d1ad6d9d6de145e1d2a8ef47db299abe164dbd7530" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" [[package]] name = "ryu" @@ -1651,7 +1917,7 @@ checksum = "24008e81ff7613ed8e5ba0cfaf24e2c2f1e5b8a0495711e44fcd4882fca62bcf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.118", ] [[package]] @@ -1666,6 +1932,18 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "serde_yaml" version = "0.9.34+deprecated" @@ -1723,24 +2001,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] -name = "smartstring" -version = "1.0.1" +name = "socket2" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29" +checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" dependencies = [ - "autocfg", - "static_assertions", - "version_check", + "libc", + "windows-sys 0.52.0", ] [[package]] name = "socket2" -version = "0.5.7" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -1749,12 +2026,6 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" -[[package]] -name = "static_assertions" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" - [[package]] name = "strsim" version = "0.10.0" @@ -1780,15 +2051,24 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.74" +version = "2.0.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fceb41e3d546d0bd83421d3409b1460cc7444cd389341a4c880fe7a042cb3d7" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + [[package]] name = "tempfile" version = "3.12.0" @@ -1823,7 +2103,16 @@ version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.63", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", ] [[package]] @@ -1834,7 +2123,18 @@ checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.118", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] @@ -1914,18 +2214,17 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.39.2" +version = "1.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daa4fb1bc778bd6f04cbfc4bb2d06a7396a8f299dc33ea1900cedaa316f467b1" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" dependencies = [ - "backtrace", "bytes", "libc", "mio", "pin-project-lite", - "socket2", + "socket2 0.6.4", "tokio-macros", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -1945,13 +2244,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.4.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.118", ] [[package]] @@ -1964,6 +2263,16 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls 0.23.40", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.15" @@ -1975,6 +2284,58 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags 2.6.0", + "bytes", + "futures-util", + "http 1.4.2", + "http-body 1.0.1", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + [[package]] name = "tower-service" version = "0.3.3" @@ -2000,7 +2361,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.118", ] [[package]] @@ -2086,6 +2447,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf8parse" version = "0.2.2" @@ -2098,7 +2465,7 @@ version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" dependencies = [ - "getrandom", + "getrandom 0.2.15", "serde", ] @@ -2129,37 +2496,45 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + [[package]] name = "wasm-bindgen" -version = "0.2.93" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" +checksum = "0bf938a0bacb0469e83c1e148908bd7d5a6010354cf4fb73279b7447422e3a89" dependencies = [ "cfg-if", "once_cell", + "rustversion", "wasm-bindgen-macro", + "wasm-bindgen-shared", ] [[package]] -name = "wasm-bindgen-backend" -version = "0.2.93" +name = "wasm-bindgen-futures" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" +checksum = "cc7ec4f8827a71586374db3e87abdb5a2bb3a15afed140221307c3ec06b1f63b" dependencies = [ - "bumpalo", - "log", - "once_cell", - "proc-macro2", - "quote", - "syn 2.0.74", - "wasm-bindgen-shared", + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.93" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" +checksum = "eeff24f84126c0ec2db7a449f0c2ec963c6a49efe0698c4242929da037ca28ed" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2167,22 +2542,58 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.93" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" +checksum = "9d08065faf983b2b80a79fd87d8254c409281cf7de75fc4b773019824196c904" dependencies = [ + "bumpalo", "proc-macro2", "quote", - "syn 2.0.74", - "wasm-bindgen-backend", + "syn 2.0.118", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.93" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" +checksum = "5fd04d9e306f1907bd13c6361b5c6bfc7b3b3c095ed3f8a9246390f8dbdee129" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-streams" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e072d4e72f700fb3443d8fe94a39315df013eef1104903cdb0a2abd322bbecd" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f2dfbb17949fa2088e5d39408c48368947b86f7834484e87b73de55bc14d97d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] [[package]] name = "webpki-roots" @@ -2190,6 +2601,15 @@ version = "0.25.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" +[[package]] +name = "webpki-roots" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "winapi" version = "0.3.9" @@ -2240,6 +2660,12 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + [[package]] name = "windows-sys" version = "0.52.0" @@ -2258,6 +2684,15 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-targets" version = "0.52.6" @@ -2322,6 +2757,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + [[package]] name = "zerocopy" version = "0.7.35" @@ -2340,9 +2781,15 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.118", ] +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + [[package]] name = "zip" version = "0.6.6" diff --git a/README.md b/README.md index 7228b23..1ae6bd0 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Core Dump Handler -This helm chart is designed to deploy functionality that automatically saves core dumps from most public cloud kubernetes service providers and private kubernetes instances to an S3 compatible storage service. +This helm chart is designed to deploy functionality that automatically saves core dumps from most public cloud kubernetes service providers and private kubernetes instances to an object storage service. [![build status](https://github.com/ibm/core-dump-handler/workflows/CI/badge.svg)](https://github.com/ibm/core-dump-handler/actions) [![Docker Repository on Quay](https://quay.io/repository/icdh/core-dump-handler/status "Docker Repository on Quay")](https://quay.io/repository/icdh/core-dump-handler) @@ -99,7 +99,7 @@ This chart aims to tackle the problems surrounding core dumps by leveraging comm The chart deploys two processes: -1. The **agent** manages the updating of `/proc/sys/kernel/*` configuration, deploys the composer service and uploads the core dumps zipfile created by the composer to an object storage instance. +1. The **agent** manages the updating of `/proc/sys/kernel/*` configuration, deploys the composer service and uploads the core dumps zipfile created by the composer to an object storage instance. Native S3-compatible and Azure Blob Storage uploads are supported. 2. The **composer** handles the processing of a core dump and creating runtime, container coredump and image JSON documents from CRICTL and inserting them into a single zip file. The zip file is stored on the local file system of the node for the agent to upload. @@ -205,6 +205,11 @@ or run the helm install command with the settings S3_SECRET=XXXX S3_BUCKET_NAME=XXXX S3_REGION=XXXX + + # Or for Azure Blob Storage + STORAGE_PROVIDER=azure + AZURE_STORAGE_CONNECTION_STRING=DefaultEndpointsProtocol=https;AccountName=XXXX;AccountKey=XXXX;EndpointSuffix=core.windows.net + AZURE_STORAGE_CONTAINER_NAME=core-dumps ``` 1. Change directory to the integration folder and run the test diff --git a/charts/core-dump-handler/README.md b/charts/core-dump-handler/README.md index 026be63..a9f966d 100644 --- a/charts/core-dump-handler/README.md +++ b/charts/core-dump-handler/README.md @@ -1,12 +1,12 @@ # Core Dump Handler -This helm chart is designed to deploy functionality that automatically saves core dumps from any public cloud kubernetes service provider or [RedHat OpenShift Kubernetes Service](https://cloud.ibm.com/kubernetes/catalog/create?platformType=openshift) to an S3 compatible storage service. +This helm chart is designed to deploy functionality that automatically saves core dumps from any public cloud kubernetes service provider or [RedHat OpenShift Kubernetes Service](https://cloud.ibm.com/kubernetes/catalog/create?platformType=openshift) to an object storage service. ## Prerequisites The [Helm](https://helm.sh/) cli to run the chart -An [S3 Protocol Compatible](https://en.wikipedia.org/wiki/Amazon_S3) object storage solution. +An object storage solution reachable from the cluster. Native S3-compatible uploads and Azure Blob Storage uploads are supported. A [CRIO](https://cri-o.io/) compatible container runtime on the kubernetes hosts. If you service provider uses something else we will willingly recieve patches to support them. @@ -22,6 +22,30 @@ helm install core-dump-handler . --create-namespace --namespace observe \ Where the `--set` options are configuration for your S3 protocol compatible provider +Azure Blob Storage is auto-detected when `daemonset.azureClientId` is present. The agent prioritizes managed identity authentication; if the client ID is not set, it falls back to connection string / account key. + +**With Azure Workload Identity (recommended):** + +``` +helm install core-dump-handler . --create-namespace --namespace observe \ +--set daemonset.azureClientId=00000000-0000-0000-0000-000000000000 \ +--set daemonset.azureTenantId=00000000-0000-0000-0000-000000000000 \ +--set daemonset.azureStorageContainerName=core-dumps \ +--set daemonset.azureStorageBlobEndpoint=https://myaccount.blob.core.windows.net \ +--set serviceAccount.annotations.azure\.workload\.identity/client-id=00000000-0000-0000-0000-000000000000 \ +--set daemonset.podLabels.azure\.workload\.identity/use=true +``` + +Note: `storageProvider` is automatically set to Azure when a client ID is provided. + +**Fallback with connection string (if managed identity is not available):** + +``` +helm install core-dump-handler . --create-namespace --namespace observe \ +--set daemonset.azureStorageConnectionString='DefaultEndpointsProtocol=https;AccountName=XXX;AccountKey=XXX;EndpointSuffix=core.windows.net' \ +--set daemonset.azureStorageContainerName=core-dumps +``` + For the following providers an additional option of values should be provided using the `--values` flag e.g. @@ -33,6 +57,9 @@ helm install core-dump-handler . --create-namespace --namespace observe \ ``` + + + @@ -234,6 +261,13 @@ The agent pod has the following environment variables and these are all set by t e.g. --set S3_REGION=host.mycloud.com See https://github.com/IBM/core-dump-handler/issues/124 for further discussion. +* STORAGE_PROVIDER - Optional explicit backend selector. Supported values are `s3` and `azure`. When omitted, the agent auto-detects Azure if Azure storage settings are present, otherwise it uses S3. +* AZURE_STORAGE_CONNECTION_STRING - Azure Blob Storage connection string. When set, the agent derives account and endpoint settings from it. +* AZURE_STORAGE_ACCOUNT_NAME - Azure storage account name when not using a connection string. +* AZURE_STORAGE_ACCOUNT_KEY - Azure storage account key when not using a connection string. +* AZURE_STORAGE_CONTAINER_NAME - Blob container that receives the uploaded archives. +* AZURE_STORAGE_BLOB_ENDPOINT - Blob service endpoint, for example `https://myaccount.blob.core.windows.net`. +* AZURE_STORAGE_BLOB_PREFIX - Optional virtual folder prefix prepended to each uploaded blob name. * VENDOR - Some older hosts may require targeted builds for the composer. default(Default) - A RHEL8 build @@ -258,6 +292,18 @@ The following secrets are configurable and map to the corresponding environment key: s3Region + key: azureStorageConnectionString + + key: azureStorageAccountName + + key: azureStorageAccountKey + + key: azureStorageContainerName + + key: azureStorageBlobEndpoint + + key: azureStorageBlobPrefix + ### Values General @@ -322,14 +368,21 @@ Daemonset * useINotify: Maps to the USE_INOTIFY environment variable (Default false) * DeployCrioConfig: Maps to the DEPLOY_CRIO_CONFIG enviroment variable (Default false) * includeCrioExe: Maps to the DEPLOY_CRIO_EXE enviroment variable (Default false) -* manageStoreSecret: Defines if the chart will be responsible for creating the S3 environment variables. +* manageStoreSecret: Defines if the chart will be responsible for creating the storage environment variables. Set to false if you are using an external secrets managment system (Default true) +* storageProvider : Maps to the STORAGE_PROVIDER enviroment variable. Use `azure` to force Azure Blob Storage, `s3` to force S3, or leave empty for autodetection. * s3AccessKey : Maps to the S3_ACCESS_KEY enviroment variable * s3Secret : Maps to the S3_SECRET enviroment variable * s3BucketName : Maps to the S3_BUCKET_NAME enviroment variable * 3Region : Maps to the S3_REGION enviroment variable +* azureStorageConnectionString : Maps to the AZURE_STORAGE_CONNECTION_STRING environment variable +* azureStorageAccountName : Maps to the AZURE_STORAGE_ACCOUNT_NAME environment variable +* azureStorageAccountKey : Maps to the AZURE_STORAGE_ACCOUNT_KEY environment variable +* azureStorageContainerName : Maps to the AZURE_STORAGE_CONTAINER_NAME environment variable +* azureStorageBlobEndpoint : Maps to the AZURE_STORAGE_BLOB_ENDPOINT environment variable +* azureStorageBlobPrefix : Maps to the AZURE_STORAGE_BLOB_PREFIX environment variable * extraEnvVars: Option for passing additional configuration to the agent such as endpoint properties. * envFrom: Array of [EnvFromSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.25/#envfromsource-v1-core) to inject into main container. * sidecarContainers: Array of [Container](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.25/#container-v1-core) to define as part of the pod. diff --git a/charts/core-dump-handler/templates/daemonset.yaml b/charts/core-dump-handler/templates/daemonset.yaml index f61f473..666d6a2 100644 --- a/charts/core-dump-handler/templates/daemonset.yaml +++ b/charts/core-dump-handler/templates/daemonset.yaml @@ -14,6 +14,9 @@ spec: metadata: labels: name: {{ .Values.daemonset.label }} + {{- with .Values.daemonset.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} annotations: kubectl.kubernetes.io/default-container: "coredump-container" {{- with .Values.daemonset.podAnnotations }} @@ -75,6 +78,16 @@ spec: value: {{ .Values.daemonset.suidDumpable | quote }} - name: DEPLOY_CRIO_EXE value: {{ .Values.daemonset.includeCrioExe | quote }} + - name: STORAGE_PROVIDER + value: {{ .Values.daemonset.storageProvider | quote }} + {{- if .Values.daemonset.azureClientId }} + - name: AZURE_CLIENT_ID + value: {{ .Values.daemonset.azureClientId | quote }} + {{- end }} + {{- if .Values.daemonset.azureTenantId }} + - name: AZURE_TENANT_ID + value: {{ .Values.daemonset.azureTenantId | quote }} + {{- end }} {{- if .Values.daemonset.manageStoreSecret }} - name: S3_ACCESS_KEY valueFrom: @@ -93,11 +106,49 @@ spec: secretKeyRef: name: s3config key: s3BucketName + optional: true - name: S3_REGION valueFrom: secretKeyRef: name: s3config key: s3Region + optional: true + - name: AZURE_STORAGE_CONNECTION_STRING + valueFrom: + secretKeyRef: + name: s3config + key: azureStorageConnectionString + optional: true + - name: AZURE_STORAGE_ACCOUNT_NAME + valueFrom: + secretKeyRef: + name: s3config + key: azureStorageAccountName + optional: true + - name: AZURE_STORAGE_ACCOUNT_KEY + valueFrom: + secretKeyRef: + name: s3config + key: azureStorageAccountKey + optional: true + - name: AZURE_STORAGE_CONTAINER_NAME + valueFrom: + secretKeyRef: + name: s3config + key: azureStorageContainerName + optional: true + - name: AZURE_STORAGE_BLOB_ENDPOINT + valueFrom: + secretKeyRef: + name: s3config + key: azureStorageBlobEndpoint + optional: true + - name: AZURE_STORAGE_BLOB_PREFIX + valueFrom: + secretKeyRef: + name: s3config + key: azureStorageBlobPrefix + optional: true {{- end }} - name: VENDOR value: {{ .Values.daemonset.vendor }} diff --git a/charts/core-dump-handler/templates/secrets.yaml b/charts/core-dump-handler/templates/secrets.yaml index 9bcf205..27dba00 100644 --- a/charts/core-dump-handler/templates/secrets.yaml +++ b/charts/core-dump-handler/templates/secrets.yaml @@ -11,6 +11,28 @@ stringData: {{- if .Values.daemonset.s3AccessKey }} s3AccessKey: {{ .Values.daemonset.s3AccessKey }} {{- end }} +{{- if .Values.daemonset.s3BucketName }} s3BucketName: {{ .Values.daemonset.s3BucketName }} +{{- end }} +{{- if .Values.daemonset.s3Region }} s3Region: {{ .Values.daemonset.s3Region }} {{- end }} +{{- if .Values.daemonset.azureStorageConnectionString }} + azureStorageConnectionString: {{ .Values.daemonset.azureStorageConnectionString }} +{{- end }} +{{- if .Values.daemonset.azureStorageAccountName }} + azureStorageAccountName: {{ .Values.daemonset.azureStorageAccountName }} +{{- end }} +{{- if .Values.daemonset.azureStorageAccountKey }} + azureStorageAccountKey: {{ .Values.daemonset.azureStorageAccountKey }} +{{- end }} +{{- if .Values.daemonset.azureStorageContainerName }} + azureStorageContainerName: {{ .Values.daemonset.azureStorageContainerName }} +{{- end }} +{{- if .Values.daemonset.azureStorageBlobEndpoint }} + azureStorageBlobEndpoint: {{ .Values.daemonset.azureStorageBlobEndpoint }} +{{- end }} +{{- if .Values.daemonset.azureStorageBlobPrefix }} + azureStorageBlobPrefix: {{ .Values.daemonset.azureStorageBlobPrefix }} +{{- end }} +{{- end }} diff --git a/charts/core-dump-handler/values.azure.yaml b/charts/core-dump-handler/values.azure.yaml new file mode 100644 index 0000000..ea17bb9 --- /dev/null +++ b/charts/core-dump-handler/values.azure.yaml @@ -0,0 +1,19 @@ +# Azure Blob Storage with Managed Identity (recommended) +# +# When AZURE_CLIENT_ID is set, the agent automatically selects Azure Blob Storage +# and authenticates using the managed identity. No need to set storageProvider. + +daemonset: + manageStoreSecret: true + azureClientId: "00000000-0000-0000-0000-000000000000" + azureTenantId: "00000000-0000-0000-0000-000000000000" + azureStorageContainerName: core-dumps + azureStorageBlobEndpoint: "https://XXXX.blob.core.windows.net" + + # Required for Azure Workload Identity on AKS. + podLabels: + azure.workload.identity/use: "true" + +serviceAccount: + annotations: + azure.workload.identity/client-id: "00000000-0000-0000-0000-000000000000" \ No newline at end of file diff --git a/charts/core-dump-handler/values.schema.json b/charts/core-dump-handler/values.schema.json index 0cac93d..5315557 100644 --- a/charts/core-dump-handler/values.schema.json +++ b/charts/core-dump-handler/values.schema.json @@ -174,18 +174,61 @@ "properties": { "manageStoreSecret": { "const": true + }, + "storageProvider": { + "const": "azure" } }, "required": [ - "manageStoreSecret" + "manageStoreSecret", + "storageProvider" + ] + }, + "then": { + "required": [ + "azureStorageContainerName" + ], + "anyOf": [ + { + "required": [ + "azureStorageConnectionString" + ] + }, + { + "required": [ + "azureStorageAccountName", + "azureStorageAccountKey", + "azureStorageBlobEndpoint" + ] + } ] + } + }, + { + "if": { + "properties": { + "manageStoreSecret": { + "const": true + } + }, + "required": [ + "manageStoreSecret" + ], + "not": { + "properties": { + "storageProvider": { + "const": "azure" + } + }, + "required": [ + "storageProvider" + ] + } }, "then": { "required": [ - "s3AccessKey", "s3BucketName", - "s3Region", - "s3Secret" + "s3Region" ] } }, @@ -215,6 +258,12 @@ "type": "string" } }, + "podLabels": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, "hostDirectory": { "type": "string" }, @@ -230,6 +279,15 @@ "vendor": { "type": "string" }, + "storageProvider": { + "type": "string" + }, + "azureClientId": { + "type": "string" + }, + "azureTenantId": { + "type": "string" + }, "interval": { "type": "integer", "minimum": 1000 @@ -270,6 +328,24 @@ "s3Region": { "type": "string" }, + "azureStorageConnectionString": { + "type": "string" + }, + "azureStorageAccountName": { + "type": "string" + }, + "azureStorageAccountKey": { + "type": "string" + }, + "azureStorageContainerName": { + "type": "string" + }, + "azureStorageBlobEndpoint": { + "type": "string" + }, + "azureStorageBlobPrefix": { + "type": "string" + }, "extraEnvVars": { "type": "string" }, diff --git a/charts/core-dump-handler/values.yaml b/charts/core-dump-handler/values.yaml index 05f8213..a2e612c 100644 --- a/charts/core-dump-handler/values.yaml +++ b/charts/core-dump-handler/values.yaml @@ -35,6 +35,7 @@ composer: daemonset: podAnnotations: {} + podLabels: {} name: "core-dump-handler" label: "core-dump-ds" hostDirectory: "/var/mnt/core-dump-handler" @@ -50,12 +51,21 @@ daemonset: useINotify: true deployCrioConfig: false includeCrioExe: false - # S3 access + # Storage access manageStoreSecret: true + storageProvider: "" + azureClientId: "" + azureTenantId: "" s3AccessKey: XXX s3Secret: XXX s3BucketName: XXX s3Region: XXX + azureStorageConnectionString: "" + azureStorageAccountName: "" + azureStorageAccountKey: "" + azureStorageContainerName: "" + azureStorageBlobEndpoint: "" + azureStorageBlobPrefix: "" extraEnvVars: "" envFrom: [] sidecarContainers: [] diff --git a/core-dump-agent/Cargo.toml b/core-dump-agent/Cargo.toml index 817b716..d65db2b 100644 --- a/core-dump-agent/Cargo.toml +++ b/core-dump-agent/Cargo.toml @@ -9,30 +9,25 @@ resolver = "2" [dependencies] anyhow = "1.0.86" +base64 = "0.22.1" +chrono = { version = "0.4.38", default-features = false, features = ["clock"] } dotenv = "0.15.0" env_logger = "0.11.5" +hmac = "0.12.1" log = "0.4.22" +reqwest = { version = "0.12.9", default-features = false, features = ["http2", "rustls-tls", "stream"] } advisory-lock = "0.3.0" +sha2 = "0.10.8" +serde_json = "1.0.125" tokio-cron-scheduler = "0.11.0" tokio = { version = "1", features = ["macros", "rt-multi-thread", "fs"] } -inotify = "0.10" +tokio-util = { version = "0.7.12", features = ["io"] } +rust-s3 = { version = "0.34.0", default-features = false, features = ["tokio-rustls-tls"] } thiserror = "1.0.63" +urlencoding = "2.1.3" -[target.x86_64-unknown-linux-musl.dependencies.rust-s3] -version = "0.34.0" -default-features = false -features = ["tokio-rustls-tls"] - -[target.aarch64-unknown-linux-musl.dependencies.rust-s3] -version = "0.34.0" -default-features = false -features = ["tokio-rustls-tls"] - -[target.x86_64-unknown-linux-gnu.dependencies.rust-s3] -version = "0.34.0" - -[target.aarch64-unknown-linux-gnu.dependencies.rust-s3] -version = "0.34.0" +[target.'cfg(unix)'.dependencies] +inotify = "0.10" [dev-dependencies] fs_extra = "1.2" diff --git a/core-dump-agent/src/main.rs b/core-dump-agent/src/main.rs index 8394edb..912af64 100644 --- a/core-dump-agent/src/main.rs +++ b/core-dump-agent/src/main.rs @@ -2,12 +2,21 @@ extern crate dotenv; extern crate s3; use advisory_lock::{AdvisoryFileLock, FileLockMode}; +use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; +use base64::Engine; +use chrono::Utc; use env_logger::Env; -use inotify::{EventMask, Inotify, WatchMask}; +use hmac::{Hmac, Mac}; use log::{error, info, warn}; +use reqwest::header::{ + AUTHORIZATION, CONTENT_LENGTH, HeaderMap, HeaderValue, +}; +use reqwest::Client; use s3::bucket::Bucket; use s3::creds::Credentials; use s3::region::Region; +use sha2::Sha256; +use std::collections::HashMap; use std::env; use std::fs; use std::fs::File; @@ -19,14 +28,43 @@ use std::process::Command; use std::time::Duration; use thiserror::Error; use tokio_cron_scheduler::{Job, JobScheduler}; +use tokio_util::io::ReaderStream; + +#[cfg(unix)] +use inotify::{EventMask, Inotify, WatchMask}; + +type HmacSha256 = Hmac; + +enum StorageBackend { + S3(Bucket), + Azure(AzureBlobClient), +} -#[allow(dead_code)] -struct Storage { - name: String, - region: Region, - credentials: Credentials, - bucket: String, - location_supported: bool, +struct UploadResult { + provider: &'static str, + status_code: u16, + uploaded_bytes: u64, +} + +struct AzureBlobClient { + client: Client, + auth: AzureAuth, + container_name: String, + blob_endpoint: String, + blob_prefix: String, +} + +enum AzureAuth { + SharedKey { + account_name: String, + account_key: String, + }, + ManagedIdentity { + client_id: String, + tenant_id: Option, + federated_token_file: Option, + authority_host: String, + }, } #[derive(Error, Debug)] @@ -39,6 +77,7 @@ const BIN_PATH: &str = "/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin"; const CDC_NAME: &str = "cdc"; static DEFAULT_BASE_DIR: &str = "/var/mnt/core-dump-handler"; static DEFAULT_CORE_DIR: &str = "/var/mnt/core-dump-handler/cores"; +const AZURE_BLOB_API_VERSION: &str = "2023-11-03"; static DEFAULT_SUID_DUMPABLE: &str = "2"; @@ -82,16 +121,16 @@ async fn main() -> Result<(), anyhow::Error> { if pattern == "sweep" { let file = std::env::args().nth(2).unwrap_or_default(); if !file.is_empty() { - let bucket = match get_bucket() { + let storage = match get_storage_backend() { Ok(v) => v, Err(e) => { - error!("Bucket creation failed in sweep: {}", e); + error!("Storage client creation failed in sweep: {}", e); process::exit(1); } }; let p = Path::new(&file); info!("Uploading {}", file); - process_file(p, &bucket).await; + process_file(p, &storage).await; } else { info!("Uploading all content in {}", core_dir_command); run_polling_agent().await; @@ -222,6 +261,7 @@ async fn main() -> Result<(), anyhow::Error> { } } + #[cfg(unix)] if use_inotify == "true" { info!("INotify Starting..."); let inotify_task = tokio::spawn(async move { @@ -255,10 +295,10 @@ async fn main() -> Result<(), anyhow::Error> { if event.mask.contains(EventMask::ISDIR) { warn!("Unknown Directory created: {:?}", event.name); } else { - let bucket = match get_bucket() { + let storage = match get_storage_backend() { Ok(v) => v, Err(e) => { - error!("Bucket creation failed in event: {}", e); + error!("Storage client creation failed in event: {}", e); continue; } }; @@ -270,7 +310,7 @@ async fn main() -> Result<(), anyhow::Error> { s.to_str().unwrap_or_default() ); let p = Path::new(&file); - process_file(p, &bucket).await + process_file(p, &storage).await } None => { continue; @@ -284,15 +324,20 @@ async fn main() -> Result<(), anyhow::Error> { inotify_task.await?; } + #[cfg(not(unix))] + if use_inotify == "true" { + warn!("USE_INOTIFY is not supported on this platform; using polling instead"); + } + Ok(()) } -async fn process_file(zip_path: &Path, bucket: &Bucket) { +async fn process_file(zip_path: &Path, storage: &StorageBackend) { info!("Uploading: {}", zip_path.display()); let f = File::open(zip_path).expect("no file found"); - match f.try_lock(FileLockMode::Shared) { + match AdvisoryFileLock::try_lock(&f, FileLockMode::Shared) { Ok(_) => { /* If we can lock then we are ok */ } Err(e) => { let l_inotify = env::var("USE_INOTIFY") @@ -324,12 +369,12 @@ async fn process_file(zip_path: &Path, bucket: &Bucket) { } }; - let mut fasync = tokio::fs::File::open(zip_path) + let fasync = tokio::fs::File::open(zip_path) .await .expect("file was removed"); - let code = match bucket - .put_object_stream(&mut fasync, upload_file_name) + let upload_result = match storage + .upload(upload_file_name, metadata.len(), fasync) .await { Ok(v) => v, @@ -346,12 +391,252 @@ async fn process_file(zip_path: &Path, bucket: &Bucket) { } }; info!( - "S3 Returned: status_code: {} uploaded_bytes: {}", - code.status_code(), - code.uploaded_bytes() + "{} returned: status_code: {} uploaded_bytes: {}", + upload_result.provider, + upload_result.status_code, + upload_result.uploaded_bytes ); } +impl StorageBackend { + async fn upload( + &self, + upload_file_name: &str, + content_length: u64, + file: tokio::fs::File, + ) -> Result { + match self { + StorageBackend::S3(bucket) => { + let mut file = file; + let response = bucket.put_object_stream(&mut file, upload_file_name).await?; + Ok(UploadResult { + provider: "S3", + status_code: response.status_code(), + uploaded_bytes: response.uploaded_bytes() as u64, + }) + } + StorageBackend::Azure(client) => client.upload(upload_file_name, content_length, file).await, + } + } +} + +impl AzureBlobClient { + async fn upload( + &self, + upload_file_name: &str, + content_length: u64, + file: tokio::fs::File, + ) -> Result { + let blob_name = self.blob_name(upload_file_name); + let request_url = self.blob_url(&blob_name); + let request_date = Utc::now().format("%a, %d %b %Y %H:%M:%S GMT").to_string(); + let authorization = self.authorization_header(&blob_name, content_length, &request_date).await?; + + let mut headers = HeaderMap::new(); + headers.insert("x-ms-blob-type", HeaderValue::from_static("BlockBlob")); + headers.insert( + "x-ms-date", + HeaderValue::from_str(&request_date).map_err(|error| anyhow::Error::msg(error.to_string()))?, + ); + headers.insert("x-ms-version", HeaderValue::from_static(AZURE_BLOB_API_VERSION)); + headers.insert(CONTENT_LENGTH, HeaderValue::from(content_length)); + headers.insert( + AUTHORIZATION, + HeaderValue::from_str(&authorization).map_err(|error| anyhow::Error::msg(error.to_string()))?, + ); + + let response = self + .client + .put(request_url) + .headers(headers) + .body(reqwest::Body::wrap_stream(ReaderStream::new(file))) + .send() + .await?; + + let status = response.status(); + if !status.is_success() { + let body = response.text().await.unwrap_or_default(); + anyhow::bail!("Azure Blob upload failed with {}: {}", status, body); + } + + Ok(UploadResult { + provider: "Azure Blob", + status_code: status.as_u16(), + uploaded_bytes: content_length, + }) + } + + fn blob_name(&self, upload_file_name: &str) -> String { + if self.blob_prefix.is_empty() { + upload_file_name.to_string() + } else { + format!("{}/{}", self.blob_prefix, upload_file_name) + } + } + + fn blob_url(&self, blob_name: &str) -> String { + let encoded_blob_name = blob_name + .split('/') + .map(urlencoding::encode) + .collect::>() + .join("/"); + format!( + "{}/{}/{}", + self.blob_endpoint.trim_end_matches('/'), + self.container_name, + encoded_blob_name + ) + } + + async fn authorization_header( + &self, + blob_name: &str, + content_length: u64, + request_date: &str, + ) -> Result { + match &self.auth { + AzureAuth::SharedKey { + account_name, + account_key, + } => { + let canonicalized_headers = format!( + "x-ms-blob-type:BlockBlob\nx-ms-date:{}\nx-ms-version:{}", + request_date, AZURE_BLOB_API_VERSION + ); + let canonicalized_resource = format!( + "/{}/{}/{}", + account_name, self.container_name, blob_name + ); + let string_to_sign = format!( + "PUT\n\n\n{}\n\n\n\n\n\n\n\n\n{}\n{}", + content_length, canonicalized_headers, canonicalized_resource + ); + + let decoded_key = BASE64_STANDARD.decode(account_key.as_bytes())?; + let mut mac = HmacSha256::new_from_slice(&decoded_key)?; + mac.update(string_to_sign.as_bytes()); + let signature = BASE64_STANDARD.encode(mac.finalize().into_bytes()); + Ok(format!("SharedKey {}:{}", account_name, signature)) + } + AzureAuth::ManagedIdentity { + client_id, + tenant_id, + federated_token_file, + authority_host, + } => { + let token = if let Some(token_file) = federated_token_file { + let tenant_id = tenant_id.as_ref().ok_or_else(|| { + anyhow::anyhow!( + "AZURE_TENANT_ID is required when AZURE_FEDERATED_TOKEN_FILE is set" + ) + })?; + self.exchange_federated_token( + authority_host, + tenant_id, + client_id, + token_file, + ) + .await? + } else { + self.get_imds_token(client_id).await? + }; + + Ok(format!("Bearer {}", token)) + } + } + } + + async fn exchange_federated_token( + &self, + authority_host: &str, + tenant_id: &str, + client_id: &str, + token_file: &str, + ) -> Result { + let assertion = fs::read_to_string(token_file)?; + let url = format!( + "{}/{}/oauth2/v2.0/token", + authority_host.trim_end_matches('/'), + tenant_id + ); + + let response = self + .client + .post(url) + .header("Content-Type", "application/x-www-form-urlencoded") + .body(format!( + "client_id={}&scope={}&grant_type=client_credentials&client_assertion_type=urn:ietf:params:oauth:client-assertion-type:jwt-bearer&client_assertion={}", + urlencoding::encode(client_id), + urlencoding::encode("https://storage.azure.com/.default"), + urlencoding::encode(assertion.trim()) + )) + .send() + .await?; + + let status = response.status(); + let body = response.text().await?; + if !status.is_success() { + anyhow::bail!("Azure federated token exchange failed with {}: {}", status, body); + } + + let value: serde_json::Value = serde_json::from_str(&body)?; + value + .get("access_token") + .and_then(|v| v.as_str()) + .map(|v| v.to_string()) + .ok_or_else(|| anyhow::anyhow!("Azure federated token response did not contain access_token")) + } + + async fn get_imds_token(&self, client_id: &str) -> Result { + let mut url = reqwest::Url::parse( + "http://169.254.169.254/metadata/identity/oauth2/token", + )?; + url.query_pairs_mut() + .append_pair("api-version", "2018-02-01") + .append_pair("resource", "https://storage.azure.com/") + .append_pair("client_id", client_id); + + let response = self + .client + .get(url) + .header("Metadata", "true") + .send() + .await?; + + let status = response.status(); + let body = response.text().await?; + if !status.is_success() { + anyhow::bail!("Azure IMDS token request failed with {}: {}", status, body); + } + + let value: serde_json::Value = serde_json::from_str(&body)?; + value + .get("access_token") + .and_then(|v| v.as_str()) + .map(|v| v.to_string()) + .ok_or_else(|| anyhow::anyhow!("Azure IMDS token response did not contain access_token")) + } +} + +fn get_storage_backend() -> Result { + let provider = env::var("STORAGE_PROVIDER") + .unwrap_or_else(|_| String::new()) + .to_lowercase(); + + if provider == "azure" || (provider.is_empty() && has_azure_config()) { + return Ok(StorageBackend::Azure(get_azure_blob_client()?)); + } + + Ok(StorageBackend::S3(get_bucket()?)) +} + +fn has_azure_config() -> bool { + env::var("AZURE_CLIENT_ID").is_ok() + || env::var("AZURE_STORAGE_CONNECTION_STRING").is_ok() + || (env::var("AZURE_STORAGE_ACCOUNT_NAME").is_ok() + && env::var("AZURE_STORAGE_CONTAINER_NAME").is_ok()) +} + fn get_bucket() -> Result { let s3_access_key = env::var("S3_ACCESS_KEY").unwrap_or_default(); let s3_secret = env::var("S3_SECRET").unwrap_or_default(); @@ -385,24 +670,104 @@ fn get_bucket() -> Result { ) }?; - let s3 = Storage { - name: "aws".into(), - region, - credentials, - bucket: s3_bucket_name, - location_supported: false, + Ok(Bucket::new(&s3_bucket_name, region, credentials)?.with_path_style()) +} + +fn get_azure_blob_client() -> Result { + let connection_string = env::var("AZURE_STORAGE_CONNECTION_STRING").unwrap_or_default(); + let mut blob_endpoint = env::var("AZURE_STORAGE_BLOB_ENDPOINT").unwrap_or_default(); + let mut account_name = env::var("AZURE_STORAGE_ACCOUNT_NAME").unwrap_or_default(); + let mut account_key = env::var("AZURE_STORAGE_ACCOUNT_KEY").unwrap_or_default(); + let client_id = env::var("AZURE_CLIENT_ID").unwrap_or_default(); + let tenant_id = env::var("AZURE_TENANT_ID").ok(); + let federated_token_file = env::var("AZURE_FEDERATED_TOKEN_FILE").ok(); + let authority_host = env::var("AZURE_AUTHORITY_HOST") + .unwrap_or_else(|_| "https://login.microsoftonline.com".to_string()); + + if !connection_string.is_empty() { + let values = parse_connection_string(&connection_string); + if account_name.is_empty() { + account_name = values.get("AccountName").cloned().unwrap_or_default(); + } + if account_key.is_empty() { + account_key = values.get("AccountKey").cloned().unwrap_or_default(); + } + if blob_endpoint.is_empty() { + blob_endpoint = values.get("BlobEndpoint").cloned().unwrap_or_default(); + if blob_endpoint.is_empty() { + let endpoint_suffix = values + .get("EndpointSuffix") + .cloned() + .unwrap_or_else(|| "core.windows.net".to_string()); + let default_protocol = values + .get("DefaultEndpointsProtocol") + .cloned() + .unwrap_or_else(|| "https".to_string()); + if !account_name.is_empty() { + blob_endpoint = format!( + "{}://{}.blob.{}", + default_protocol, account_name, endpoint_suffix + ); + } + } + } + } + + let container_name = env::var("AZURE_STORAGE_CONTAINER_NAME")?; + let blob_prefix = env::var("AZURE_STORAGE_BLOB_PREFIX") + .unwrap_or_default() + .trim_matches('/') + .to_string(); + + let auth = if !client_id.is_empty() { + if blob_endpoint.is_empty() { + anyhow::bail!( + "Azure managed identity requires AZURE_STORAGE_BLOB_ENDPOINT or AZURE_STORAGE_CONNECTION_STRING to derive it" + ); + } + + AzureAuth::ManagedIdentity { + client_id, + tenant_id, + federated_token_file, + authority_host, + } + } else { + if account_name.is_empty() || account_key.is_empty() || blob_endpoint.is_empty() { + anyhow::bail!( + "Azure Storage requires AZURE_STORAGE_CONTAINER_NAME and either AZURE_CLIENT_ID or AZURE_STORAGE_CONNECTION_STRING / AZURE_STORAGE_ACCOUNT_NAME / AZURE_STORAGE_ACCOUNT_KEY / AZURE_STORAGE_BLOB_ENDPOINT" + ); + } + + AzureAuth::SharedKey { + account_name, + account_key, + } }; - Ok(Bucket::new(&s3.bucket, s3.region, s3.credentials)?.with_path_style()) + + if blob_endpoint.is_empty() { + anyhow::bail!( + "Azure Storage requires AZURE_STORAGE_BLOB_ENDPOINT or AZURE_STORAGE_CONNECTION_STRING to derive it" + ); + } + + Ok(AzureBlobClient { + client: Client::builder().build()?, + auth, + container_name, + blob_endpoint, + blob_prefix, + }) } async fn run_polling_agent() { let core_location = env::var("CORE_DIR").unwrap_or_else(|_| DEFAULT_CORE_DIR.to_string()); info!("Executing Agent with location : {}", core_location); - let bucket = match get_bucket() { + let storage = match get_storage_backend() { Ok(v) => v, Err(e) => { - error!("Bucket Creation Failed: {}", e); + error!("Storage client creation failed: {}", e); return; } }; @@ -418,7 +783,36 @@ async fn run_polling_agent() { info!("Dir Content {:?}", paths); for zip_path in paths { - process_file(&zip_path, &bucket).await; + process_file(&zip_path, &storage).await; + } +} + +fn parse_connection_string(connection_string: &str) -> HashMap { + connection_string + .split(';') + .filter_map(|entry| { + let (key, value) = entry.split_once('=')?; + Some((key.trim().to_string(), value.trim().to_string())) + }) + .collect() +} + +#[cfg(test)] +mod unit_tests { + use super::parse_connection_string; + + #[test] + fn parses_azure_connection_string() { + let values = parse_connection_string( + "DefaultEndpointsProtocol=https;AccountName=testacct;AccountKey=dGVzdA==;EndpointSuffix=core.windows.net", + ); + + assert_eq!(values.get("AccountName"), Some(&"testacct".to_string())); + assert_eq!(values.get("AccountKey"), Some(&"dGVzdA==".to_string())); + assert_eq!( + values.get("EndpointSuffix"), + Some(&"core.windows.net".to_string()) + ); } } diff --git a/core-dump-composer/src/events.rs b/core-dump-composer/src/events.rs index 82bae9c..94a44c1 100644 --- a/core-dump-composer/src/events.rs +++ b/core-dump-composer/src/events.rs @@ -87,9 +87,9 @@ impl CoreEvent { pub fn write_event(&self, eventlocation: &str) -> Result<(), anyhow::Error> { let full_path = format!("{}/{}-event.json", eventlocation, self.uuid); let file = File::create(full_path)?; - file.lock(FileLockMode::Exclusive)?; + AdvisoryFileLock::lock(&file, FileLockMode::Exclusive)?; serde_json::to_writer(&file, &self)?; - file.unlock()?; + AdvisoryFileLock::unlock(&file)?; Ok(()) } } diff --git a/core-dump-composer/src/main.rs b/core-dump-composer/src/main.rs index d1bfb2d..8ea2fc6 100644 --- a/core-dump-composer/src/main.rs +++ b/core-dump-composer/src/main.rs @@ -131,7 +131,7 @@ fn handle(mut cc: config::CoreConfig) -> Result<(), anyhow::Error> { process::exit(1); } }; - file.lock(FileLockMode::Exclusive)?; + AdvisoryFileLock::lock(&file, FileLockMode::Exclusive)?; let mut zip = ZipWriter::new(&file); debug!(
ProviderProductValues
MicrosoftAKS with Azure Blob Storagevalues.azure.yaml
AWSEKSvalues.aws.yaml