diff --git a/master-libvirt/master.cfg b/master-libvirt/master.cfg index 2838d542f..90be25f88 100644 --- a/master-libvirt/master.cfg +++ b/master-libvirt/master.cfg @@ -198,12 +198,27 @@ def getPAMTestStep(): command=["./pam-test.sh"], ) +def getSaveArtifactsStep(): + return steps.DirectoryUpload( + name="save mariadb log files", + doStepIf=hasFailed, + workersrc="/home/buildbot/logs/", + masterdest=util.Interpolate( + "/srv/buildbot/packages/" + + "%(prop:tarbuildnum)s" + + "/logs/" + + "%(prop:buildername)s" + ), + ) + + # FACTORY ## f_deb_install f_deb_install = util.BuildFactory() f_deb_install.addStep(getScript("deb-install.sh")) f_deb_install.addStep(getDebInstallStep()) +f_deb_install.addStep(getSaveArtifactsStep()) f_deb_install.addStep(getScript("pam-test.sh")) f_deb_install.addStep(getPAMTestStep()) @@ -212,11 +227,13 @@ f_deb_upgrade = util.BuildFactory() f_deb_upgrade.addStep(getMajorVersionStep()) f_deb_upgrade.addStep(getScript("deb-upgrade.sh")) f_deb_upgrade.addStep(getDebUpgradeStep()) +f_deb_upgrade.addStep(getSaveArtifactsStep()) ## f_rpm_install f_rpm_install = util.BuildFactory() f_rpm_install.addStep(getScript("rpm-install.sh")) f_rpm_install.addStep(getRpmInstallStep()) +f_rpm_install.addStep(getSaveArtifactsStep()) f_rpm_install.addStep(getScript("pam-test.sh")) f_rpm_install.addStep(getPAMTestStep()) @@ -225,6 +242,7 @@ f_rpm_upgrade = util.BuildFactory() f_rpm_upgrade.addStep(getMajorVersionStep()) f_rpm_upgrade.addStep(getScript("rpm-upgrade.sh")) f_rpm_upgrade.addStep(getRpmUpgradeStep()) +f_rpm_upgrade.addStep(getSaveArtifactsStep()) ####### WORKERS and BUILDERS diff --git a/scripts/bash_lib.sh b/scripts/bash_lib.sh index b0f3bf144..c5d2911ac 100644 --- a/scripts/bash_lib.sh +++ b/scripts/bash_lib.sh @@ -437,27 +437,37 @@ upgrade_test_type() { esac } -get_columnstore_logs() { +save_failure_logs() { + local logdir=../logs/ + local logfile=$logdir/$test_mode.log + mkdir -p $logdir + bb_log_err "Previous step failed, saving mariadb logs" if [[ $test_mode == "columnstore" ]]; then bb_log_info "storing Columnstore logs in columnstore_logs" set +ex - # It is done in such a weird way, because Columnstore currently makes its logs hard to read - # //TEMP this is fragile and weird (test that /var/log/mariadb/columnstore exist) - for f in $(sudo ls /var/log/mariadb/columnstore | xargs); do - f=/var/log/mariadb/columnstore/$f - echo "----------- $f -----------" >>/home/buildbot/columnstore_logs - sudo cat "$f" 1>>/home/buildbot/columnstore_logs 2>&1 + for f in $(sudo find /tmp/columnstore_tmp_files /var/log/mariadb/columnstore -type f); do + echo "----------- $f -----------" >>"$logfile" + sudo cat "$f" 1>>"$logfile" 2>&1 done - for f in /tmp/columnstore_tmp_files/*; do - echo "----------- $f -----------" >>/home/buildbot/columnstore_logs - sudo cat "$f" | sudo tee -a /home/buildbot/columnstore_logs 2>&1 + for s in mcs-writeengineserver mcs-controllernode.service mcs-ddlproc.service mcs-dmlproc.service \ + mcs-loadbrm.service mcs-primproc.service mcs-workernode@1.service; do + echo "----------- $s -----------" >>"$logfile" + sudo journalctl -u "$s" | tee -a "$logfile" 2>&1 done + if [ -d /var/lib/columnstore ]; then + tar -Jcvf $logdir/columnstore.tar.bz2 /var/lib/columnstore + fi + fi + echo "----------- mariadb.service -----------" >>"$logfile" + sudo journalctl -u mariadb.service | tee -a "$logfile" 2>&1 + if [ -f "$logfile" ]; then + bzip2 "$logfile" fi + sudo find /var/lib/systemd/coredump/ -type -f -exec mv {} $logdir \; } check_mariadb_server_and_create_structures() { # All the commands below should succeed - set -e sudo mariadb -e "CREATE DATABASE db" sudo mariadb -e "CREATE TABLE db.t_innodb(a1 SERIAL, c1 CHAR(8)) ENGINE=InnoDB; INSERT INTO db.t_innodb VALUES (1,'foo'),(2,'bar')" sudo mariadb -e "CREATE TABLE db.t_myisam(a2 SERIAL, c2 CHAR(8)) ENGINE=MyISAM; INSERT INTO db.t_myisam VALUES (1,'foo'),(2,'bar')" @@ -468,19 +478,14 @@ check_mariadb_server_and_create_structures() { sudo mariadb -e "CREATE PROCEDURE db.p() SELECT * FROM db.v_merge" sudo mariadb -e "CREATE FUNCTION db.f() RETURNS INT DETERMINISTIC RETURN 1" if [[ $test_mode == "columnstore" ]]; then - if ! sudo mariadb -e "CREATE TABLE db.t_columnstore(a INT, c VARCHAR(8)) ENGINE=ColumnStore; SHOW CREATE TABLE db.t_columnstore; INSERT INTO db.t_columnstore VALUES (1,'foo'),(2,'bar')"; then - get_columnstore_logs - exit 1 - fi + sudo mariadb -e "CREATE TABLE db.t_columnstore(a INT, c VARCHAR(8)) ENGINE=ColumnStore; SHOW CREATE TABLE db.t_columnstore; INSERT INTO db.t_columnstore VALUES (1,'foo'),(2,'bar')" fi - set +e } check_mariadb_server_and_verify_structures() { # Print "have_xx" capabilitites for the new server sudo mariadb -e "select 'Stat' t, variable_name name, variable_value val from information_schema.global_status where variable_name like '%have%' union select 'Vars' t, variable_name name, variable_value val from information_schema.global_variables where variable_name like '%have%' order by t, name" # All the commands below should succeed - set -e sudo mariadb -e "select @@version, @@version_comment" sudo mariadb -e "SHOW TABLES IN db" sudo mariadb -e "SELECT * FROM db.t_innodb; INSERT INTO db.t_innodb VALUES (3,'foo'),(4,'bar')" @@ -495,12 +500,8 @@ check_mariadb_server_and_verify_structures() { sudo mariadb -e "SELECT db.f()" if [[ $test_mode == "columnstore" ]]; then - if ! sudo mariadb -e "SELECT * FROM db.t_columnstore; INSERT INTO db.t_columnstore VALUES (3,'foo'),(4,'bar')"; then - get_columnstore_logs - exit 1 - fi + sudo mariadb -e "SELECT * FROM db.t_columnstore; INSERT INTO db.t_columnstore VALUES (3,'foo'),(4,'bar')" fi - set +e } control_mariadb_server() { diff --git a/scripts/deb-install.sh b/scripts/deb-install.sh index 1de387bee..a86cd3aca 100755 --- a/scripts/deb-install.sh +++ b/scripts/deb-install.sh @@ -69,6 +69,9 @@ fi # apt get update may be running in the background (Ubuntu start). apt_get_update +# set -e already set at start of script +trap save_failure_logs ERR + sudo sh -c "DEBIAN_FRONTEND=noninteractive MYSQLD_STARTUP_TIMEOUT=180 \ apt-get install -y $package_list $columnstore_package_list" @@ -81,22 +84,6 @@ if [[ $systemdCapability == "yes" ]]; then if ! sudo systemctl status mariadb --no-pager; then sudo journalctl -xe --no-pager bb_log_warn "mariadb service isn't running properly after installation" - if echo "$package_list" | grep -q columnstore; then - bb_log_info "It is likely to be caused by ColumnStore" - bb_log_info "problems upon installation, getting the logs" - set +e - # It is done in such a weird way, because Columnstore currently makes its - # logs hard to read - for f in $(sudo ls /var/log/mariadb/columnstore | xargs); do - f=/var/log/mariadb/columnstore/$f - echo "----------- $f -----------" - sudo cat "$f" - done - for f in /tmp/columnstore_tmp_files/*; do - echo "----------- $f -----------" - sudo cat "$f" - done - fi bb_log_err "mariadb service didn't start properly after installation" exit 1 fi @@ -120,6 +107,8 @@ sudo mariadb --verbose -e "create database test; \ grant all on *.* to galera;" sudo mariadb -e "select @@version" bb_log_info "test for MDEV-18563, MDEV-18526" + +# disabling -e so save_failure_logs won't have an effect while having setting up for next test set +e control_mariadb_server stop @@ -134,6 +123,7 @@ for p in /bin /sbin /usr/bin /usr/sbin /usr/local/bin /usr/local/sbin; do fi done sudo mariadb-install-db --no-defaults --user=mysql --plugin-maturity=unknown + set +e ## Install mariadb-test for further use # sudo sh -c "DEBIAN_FRONTEND=noninteractive MYSQLD_STARTUP_TIMEOUT=180 apt-get install -y mariadb-test" diff --git a/scripts/deb-upgrade.sh b/scripts/deb-upgrade.sh index f25f348e0..3ef68860d 100755 --- a/scripts/deb-upgrade.sh +++ b/scripts/deb-upgrade.sh @@ -130,12 +130,8 @@ fi # To avoid confusing errors in further logic, do an explicit check # whether the service is up and running if [[ $systemdCapability == "yes" ]]; then - if ! sudo systemctl status mariadb --no-pager; then - sudo journalctl -xe --no-pager - get_columnstore_logs - bb_log_err "mariadb service didn't start properly after installation" - exit 1 - fi + bb_log_info "Ensure mariadb.service is running" + sudo systemctl status mariadb --no-pager fi if [[ $test_mode == "all" ]]; then @@ -164,6 +160,9 @@ deb_setup_bb_galera_artifacts_mirror deb_setup_bb_artifacts_mirror apt_get_update +# now we upgrade, this is what we should save +trap save_failure_logs ERR +set -e # Install the new packages if ! sudo sh -c "DEBIAN_FRONTEND=noninteractive MYSQLD_STARTUP_TIMEOUT=180 \ apt-get -o Dpkg::Options::=--force-confnew install --allow-unauthenticated -y $package_list"; then diff --git a/scripts/rpm-install.sh b/scripts/rpm-install.sh index 0c7a3dec2..aef85bd54 100755 --- a/scripts/rpm-install.sh +++ b/scripts/rpm-install.sh @@ -58,6 +58,10 @@ set -u sh -c 'g=/usr/lib*/galera*/libgalera_smm.so; echo -e "[galera]\nwsrep_provider=$g"' | sudo tee /etc/my.cnf.d/galera.cnf + +# Any of the below steps could fail +trap save_failure_logs ERR +set -e case "$systemdCapability" in yes) if ! sudo systemctl start mariadb; then diff --git a/scripts/rpm-upgrade.sh b/scripts/rpm-upgrade.sh index 991f1ad7c..c28e7ccd6 100755 --- a/scripts/rpm-upgrade.sh +++ b/scripts/rpm-upgrade.sh @@ -194,6 +194,12 @@ fi rpm_setup_bb_galera_artifacts_mirror rpm_setup_bb_artifacts_mirror + +# Any of the below steps could fail +# This is where the new packages are processed from +trap save_failure_logs ERR +set -e + if [[ $test_type == "major" ]]; then # major upgrade (remove then install) echo "$package_list" | xargs sudo "$pkg_cmd" "$pkg_cmd_options" install