From 516c3adcd1f9b6dc7d8a8956428f7a5b9a8be548 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 27 Nov 2024 14:26:49 +1100 Subject: [PATCH 1/9] master-libvirt: saveArtifactsStep for deb/rpm upgrade --- master-libvirt/master.cfg | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/master-libvirt/master.cfg b/master-libvirt/master.cfg index 2838d542f..90be25f88 100644 --- a/master-libvirt/master.cfg +++ b/master-libvirt/master.cfg @@ -198,12 +198,27 @@ def getPAMTestStep(): command=["./pam-test.sh"], ) +def getSaveArtifactsStep(): + return steps.DirectoryUpload( + name="save mariadb log files", + doStepIf=hasFailed, + workersrc="/home/buildbot/logs/", + masterdest=util.Interpolate( + "/srv/buildbot/packages/" + + "%(prop:tarbuildnum)s" + + "/logs/" + + "%(prop:buildername)s" + ), + ) + + # FACTORY ## f_deb_install f_deb_install = util.BuildFactory() f_deb_install.addStep(getScript("deb-install.sh")) f_deb_install.addStep(getDebInstallStep()) +f_deb_install.addStep(getSaveArtifactsStep()) f_deb_install.addStep(getScript("pam-test.sh")) f_deb_install.addStep(getPAMTestStep()) @@ -212,11 +227,13 @@ f_deb_upgrade = util.BuildFactory() f_deb_upgrade.addStep(getMajorVersionStep()) f_deb_upgrade.addStep(getScript("deb-upgrade.sh")) f_deb_upgrade.addStep(getDebUpgradeStep()) +f_deb_upgrade.addStep(getSaveArtifactsStep()) ## f_rpm_install f_rpm_install = util.BuildFactory() f_rpm_install.addStep(getScript("rpm-install.sh")) f_rpm_install.addStep(getRpmInstallStep()) +f_rpm_install.addStep(getSaveArtifactsStep()) f_rpm_install.addStep(getScript("pam-test.sh")) f_rpm_install.addStep(getPAMTestStep()) @@ -225,6 +242,7 @@ f_rpm_upgrade = util.BuildFactory() f_rpm_upgrade.addStep(getMajorVersionStep()) f_rpm_upgrade.addStep(getScript("rpm-upgrade.sh")) f_rpm_upgrade.addStep(getRpmUpgradeStep()) +f_rpm_upgrade.addStep(getSaveArtifactsStep()) ####### WORKERS and BUILDERS From 4ced66e1a6ec150576d3829d0fea85e0d440bec6 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 27 Nov 2024 14:30:31 +1100 Subject: [PATCH 2/9] bash_lib: change get_columnstore-logs to save_failure_logs This makes it more generic and expands the columnstore saving to the required information. * service logs * columnstore data dir (tar.bz2) * /tmp/columnstore_tmp_files (appended to log file) More information like the mariadb.service journal and coredumps are saved. --- scripts/bash_lib.sh | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/scripts/bash_lib.sh b/scripts/bash_lib.sh index b0f3bf144..0a4a0b33d 100644 --- a/scripts/bash_lib.sh +++ b/scripts/bash_lib.sh @@ -437,22 +437,33 @@ upgrade_test_type() { esac } -get_columnstore_logs() { +save_failure_logs() { + local logdir=../logs/ + local logfile=$logdir/$test_mode.log + mkdir -p $logdir + bb_log_err "Previous step failed, saving mariadb logs" if [[ $test_mode == "columnstore" ]]; then bb_log_info "storing Columnstore logs in columnstore_logs" set +ex - # It is done in such a weird way, because Columnstore currently makes its logs hard to read - # //TEMP this is fragile and weird (test that /var/log/mariadb/columnstore exist) - for f in $(sudo ls /var/log/mariadb/columnstore | xargs); do - f=/var/log/mariadb/columnstore/$f - echo "----------- $f -----------" >>/home/buildbot/columnstore_logs - sudo cat "$f" 1>>/home/buildbot/columnstore_logs 2>&1 + for f in $(sudo find /tmp/columnstore_tmp_files /var/log/mariadb/columnstore -type f); do + echo "----------- $f -----------" >>"$logfile" + sudo cat "$f" 1>>"$logfile" 2>&1 done - for f in /tmp/columnstore_tmp_files/*; do - echo "----------- $f -----------" >>/home/buildbot/columnstore_logs - sudo cat "$f" | sudo tee -a /home/buildbot/columnstore_logs 2>&1 + for s in mcs-writeengineserver mcs-controllernode.service mcs-ddlproc.service mcs-dmlproc.service \ + mcs-loadbrm.service mcs-primproc.service mcs-workernode@1.service; do + echo "----------- $s -----------" >>"$logfile" + sudo journalctl -u "$s" | tee -a "$logfile" 2>&1 done + if [ -d /var/lib/columnstore ]; then + tar -Jcvf $logdir/columnstore.tar.bz2 /var/lib/columnstore + fi + fi + echo "----------- mariadb.service -----------" >>"$logfile" + sudo journalctl -u mariadb.service | tee -a "$logfile" 2>&1 + if [ -f "$logfile" ]; then + bzip2 "$logfile" fi + sudo find /var/lib/systemd/coredump/ -type -f -exec mv {} $logdir \; } check_mariadb_server_and_create_structures() { From 36ef633afe5473125debbed61e27c60ff957a75a Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 27 Nov 2024 15:13:55 +1100 Subject: [PATCH 3/9] bash_lib: do not change set +-e during check_mariadb_server_and_{create,verify}_structures These are handled by rpm/deb-install/upgrade. --- scripts/bash_lib.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scripts/bash_lib.sh b/scripts/bash_lib.sh index 0a4a0b33d..7ee975fdc 100644 --- a/scripts/bash_lib.sh +++ b/scripts/bash_lib.sh @@ -468,7 +468,6 @@ save_failure_logs() { check_mariadb_server_and_create_structures() { # All the commands below should succeed - set -e sudo mariadb -e "CREATE DATABASE db" sudo mariadb -e "CREATE TABLE db.t_innodb(a1 SERIAL, c1 CHAR(8)) ENGINE=InnoDB; INSERT INTO db.t_innodb VALUES (1,'foo'),(2,'bar')" sudo mariadb -e "CREATE TABLE db.t_myisam(a2 SERIAL, c2 CHAR(8)) ENGINE=MyISAM; INSERT INTO db.t_myisam VALUES (1,'foo'),(2,'bar')" @@ -484,14 +483,12 @@ check_mariadb_server_and_create_structures() { exit 1 fi fi - set +e } check_mariadb_server_and_verify_structures() { # Print "have_xx" capabilitites for the new server sudo mariadb -e "select 'Stat' t, variable_name name, variable_value val from information_schema.global_status where variable_name like '%have%' union select 'Vars' t, variable_name name, variable_value val from information_schema.global_variables where variable_name like '%have%' order by t, name" # All the commands below should succeed - set -e sudo mariadb -e "select @@version, @@version_comment" sudo mariadb -e "SHOW TABLES IN db" sudo mariadb -e "SELECT * FROM db.t_innodb; INSERT INTO db.t_innodb VALUES (3,'foo'),(4,'bar')" @@ -511,7 +508,6 @@ check_mariadb_server_and_verify_structures() { exit 1 fi fi - set +e } control_mariadb_server() { From 638214dacf8010a22a90ab357d2e163d7801a91b Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 27 Nov 2024 14:36:02 +1100 Subject: [PATCH 4/9] deb: add trap for save_failure logs when we start to process the new packages in CI. --- scripts/deb-install.sh | 6 ++++++ scripts/deb-upgrade.sh | 3 +++ 2 files changed, 9 insertions(+) diff --git a/scripts/deb-install.sh b/scripts/deb-install.sh index 1de387bee..0cc466d99 100755 --- a/scripts/deb-install.sh +++ b/scripts/deb-install.sh @@ -69,6 +69,9 @@ fi # apt get update may be running in the background (Ubuntu start). apt_get_update +# set -e already set at start of script +trap save_failure_logs ERR + sudo sh -c "DEBIAN_FRONTEND=noninteractive MYSQLD_STARTUP_TIMEOUT=180 \ apt-get install -y $package_list $columnstore_package_list" @@ -120,6 +123,8 @@ sudo mariadb --verbose -e "create database test; \ grant all on *.* to galera;" sudo mariadb -e "select @@version" bb_log_info "test for MDEV-18563, MDEV-18526" + +# disabling -e so save_failure_logs won't have an effect while having setting up for next test set +e control_mariadb_server stop @@ -134,6 +139,7 @@ for p in /bin /sbin /usr/bin /usr/sbin /usr/local/bin /usr/local/sbin; do fi done sudo mariadb-install-db --no-defaults --user=mysql --plugin-maturity=unknown + set +e ## Install mariadb-test for further use # sudo sh -c "DEBIAN_FRONTEND=noninteractive MYSQLD_STARTUP_TIMEOUT=180 apt-get install -y mariadb-test" diff --git a/scripts/deb-upgrade.sh b/scripts/deb-upgrade.sh index f25f348e0..9d204382f 100755 --- a/scripts/deb-upgrade.sh +++ b/scripts/deb-upgrade.sh @@ -164,6 +164,9 @@ deb_setup_bb_galera_artifacts_mirror deb_setup_bb_artifacts_mirror apt_get_update +# now we upgrade, this is what we should save +trap save_failure_logs ERR +set -e # Install the new packages if ! sudo sh -c "DEBIAN_FRONTEND=noninteractive MYSQLD_STARTUP_TIMEOUT=180 \ apt-get -o Dpkg::Options::=--force-confnew install --allow-unauthenticated -y $package_list"; then From 0ee903441b7d2cf0306afda9460ec25120e9fa5f Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 27 Nov 2024 14:37:47 +1100 Subject: [PATCH 5/9] bash_lib: remove explict call to (remove) get_columnstore_logs As we going be in set -e mode when this is called, the trap will handle the saving of the error information. --- scripts/bash_lib.sh | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/scripts/bash_lib.sh b/scripts/bash_lib.sh index 7ee975fdc..c5d2911ac 100644 --- a/scripts/bash_lib.sh +++ b/scripts/bash_lib.sh @@ -478,10 +478,7 @@ check_mariadb_server_and_create_structures() { sudo mariadb -e "CREATE PROCEDURE db.p() SELECT * FROM db.v_merge" sudo mariadb -e "CREATE FUNCTION db.f() RETURNS INT DETERMINISTIC RETURN 1" if [[ $test_mode == "columnstore" ]]; then - if ! sudo mariadb -e "CREATE TABLE db.t_columnstore(a INT, c VARCHAR(8)) ENGINE=ColumnStore; SHOW CREATE TABLE db.t_columnstore; INSERT INTO db.t_columnstore VALUES (1,'foo'),(2,'bar')"; then - get_columnstore_logs - exit 1 - fi + sudo mariadb -e "CREATE TABLE db.t_columnstore(a INT, c VARCHAR(8)) ENGINE=ColumnStore; SHOW CREATE TABLE db.t_columnstore; INSERT INTO db.t_columnstore VALUES (1,'foo'),(2,'bar')" fi } @@ -503,10 +500,7 @@ check_mariadb_server_and_verify_structures() { sudo mariadb -e "SELECT db.f()" if [[ $test_mode == "columnstore" ]]; then - if ! sudo mariadb -e "SELECT * FROM db.t_columnstore; INSERT INTO db.t_columnstore VALUES (3,'foo'),(4,'bar')"; then - get_columnstore_logs - exit 1 - fi + sudo mariadb -e "SELECT * FROM db.t_columnstore; INSERT INTO db.t_columnstore VALUES (3,'foo'),(4,'bar')" fi } From 5d44adae22b37d3cbf965c867f8ba253646755b2 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 27 Nov 2024 14:39:28 +1100 Subject: [PATCH 6/9] deb-install: remove duplicate columnstore error routine --- scripts/deb-install.sh | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/scripts/deb-install.sh b/scripts/deb-install.sh index 0cc466d99..a86cd3aca 100755 --- a/scripts/deb-install.sh +++ b/scripts/deb-install.sh @@ -84,22 +84,6 @@ if [[ $systemdCapability == "yes" ]]; then if ! sudo systemctl status mariadb --no-pager; then sudo journalctl -xe --no-pager bb_log_warn "mariadb service isn't running properly after installation" - if echo "$package_list" | grep -q columnstore; then - bb_log_info "It is likely to be caused by ColumnStore" - bb_log_info "problems upon installation, getting the logs" - set +e - # It is done in such a weird way, because Columnstore currently makes its - # logs hard to read - for f in $(sudo ls /var/log/mariadb/columnstore | xargs); do - f=/var/log/mariadb/columnstore/$f - echo "----------- $f -----------" - sudo cat "$f" - done - for f in /tmp/columnstore_tmp_files/*; do - echo "----------- $f -----------" - sudo cat "$f" - done - fi bb_log_err "mariadb service didn't start properly after installation" exit 1 fi From 87ac98902666c376f7167b209b5894071849aa15 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 27 Nov 2024 14:42:10 +1100 Subject: [PATCH 7/9] deb-upgrade: replace explicit columnstore save logs --- scripts/deb-upgrade.sh | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/scripts/deb-upgrade.sh b/scripts/deb-upgrade.sh index 9d204382f..3ef68860d 100755 --- a/scripts/deb-upgrade.sh +++ b/scripts/deb-upgrade.sh @@ -130,12 +130,8 @@ fi # To avoid confusing errors in further logic, do an explicit check # whether the service is up and running if [[ $systemdCapability == "yes" ]]; then - if ! sudo systemctl status mariadb --no-pager; then - sudo journalctl -xe --no-pager - get_columnstore_logs - bb_log_err "mariadb service didn't start properly after installation" - exit 1 - fi + bb_log_info "Ensure mariadb.service is running" + sudo systemctl status mariadb --no-pager fi if [[ $test_mode == "all" ]]; then From 03821e3c7086b540375f25e567368f826671f482 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 27 Nov 2024 14:46:03 +1100 Subject: [PATCH 8/9] rpm-upgrade: add trap for error handling --- scripts/rpm-upgrade.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/rpm-upgrade.sh b/scripts/rpm-upgrade.sh index 991f1ad7c..c28e7ccd6 100755 --- a/scripts/rpm-upgrade.sh +++ b/scripts/rpm-upgrade.sh @@ -194,6 +194,12 @@ fi rpm_setup_bb_galera_artifacts_mirror rpm_setup_bb_artifacts_mirror + +# Any of the below steps could fail +# This is where the new packages are processed from +trap save_failure_logs ERR +set -e + if [[ $test_type == "major" ]]; then # major upgrade (remove then install) echo "$package_list" | xargs sudo "$pkg_cmd" "$pkg_cmd_options" install From 59f9a2f448cbac086ec84d1184a5aced4e22e1c8 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 27 Nov 2024 14:18:59 +1100 Subject: [PATCH 9/9] rpm-install: use save_failure_logs trap --- scripts/rpm-install.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/rpm-install.sh b/scripts/rpm-install.sh index 0c7a3dec2..aef85bd54 100755 --- a/scripts/rpm-install.sh +++ b/scripts/rpm-install.sh @@ -58,6 +58,10 @@ set -u sh -c 'g=/usr/lib*/galera*/libgalera_smm.so; echo -e "[galera]\nwsrep_provider=$g"' | sudo tee /etc/my.cnf.d/galera.cnf + +# Any of the below steps could fail +trap save_failure_logs ERR +set -e case "$systemdCapability" in yes) if ! sudo systemctl start mariadb; then