-
-
Notifications
You must be signed in to change notification settings - Fork 2k
MDEV-36025: backup taken from a replica with optimistic parallel replication fails to restore most of the time #4888
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Open
hemantdangi-gc
wants to merge
1
commit into
10.11
Choose a base branch
from
10.11_MDEV-36025
base: 10.11
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
Open
Changes from all commits
Commits
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -79,6 +79,9 @@ Street, Fifth Floor, Boston, MA 02110-1335 USA | |
| #include <row0quiesce.h> | ||
| #include <srv0start.h> | ||
| #include "trx0sys.h" | ||
| #include "trx0trx.h" | ||
| #include "trx0roll.h" | ||
| #include <buf0flu.h> | ||
| #include <buf0dblwr.h> | ||
| #include <buf0flu.h> | ||
| #include "ha_innodb.h" | ||
|
|
@@ -151,6 +154,8 @@ my_bool xtrabackup_help; | |
| my_bool xtrabackup_export; | ||
| my_bool ignored_option; | ||
|
|
||
| my_bool xtrabackup_rollback_xa; | ||
|
|
||
| longlong xtrabackup_use_memory; | ||
|
|
||
| uint opt_protocol; | ||
|
|
@@ -1354,6 +1359,7 @@ enum options_xtrabackup | |
| OPT_XTRA_BACKUP, | ||
| OPT_XTRA_PREPARE, | ||
| OPT_XTRA_EXPORT, | ||
| OPT_XTRA_ROLLBACK_XA, | ||
| OPT_XTRA_PRINT_PARAM, | ||
| OPT_XTRA_USE_MEMORY, | ||
| OPT_XTRA_THROTTLE, | ||
|
|
@@ -1477,6 +1483,13 @@ struct my_option xb_client_options[]= { | |
| "create files to import to another database when prepare.", | ||
| (G_PTR *) &xtrabackup_export, (G_PTR *) &xtrabackup_export, 0, GET_BOOL, | ||
| NO_ARG, 0, 0, 0, 0, 0, 0}, | ||
| {"rollback-xa", OPT_XTRA_ROLLBACK_XA, | ||
| "Rollback prepared XA transactions on --prepare. Enabled by default; " | ||
| "use --skip-rollback-xa to disable. " | ||
| "After preparing target directory with this option " | ||
| "it can no longer be a base for incremental backup.", | ||
| (G_PTR *) &xtrabackup_rollback_xa, (G_PTR *) &xtrabackup_rollback_xa, 0, | ||
| GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0}, | ||
| {"print-param", OPT_XTRA_PRINT_PARAM, | ||
| "print parameter of mysqld needed for copyback.", | ||
| (G_PTR *) &xtrabackup_print_param, (G_PTR *) &xtrabackup_print_param, 0, | ||
|
|
@@ -6900,8 +6913,26 @@ static bool xtrabackup_prepare_func(char** argv) | |
| if (!ok) goto cleanup; | ||
| } | ||
|
|
||
| /* Prevent incompatible combination of --rollback_xa and --export | ||
| options. These options cannot be used together because: | ||
| - --export sets SRV_OPERATION_RESTORE_EXPORT which makes the redo | ||
| log mapping read-only for consistency during table export | ||
| - --rollback_xa requires write access to the log system to modify | ||
| transaction state during XA rollback | ||
| The combination creates mmap state inconsistency in InnoDB's MTR | ||
| system, leading to crash. | ||
| */ | ||
| if (xtrabackup_rollback_xa && xtrabackup_export) { | ||
| msg("mariabackup: ERROR: --rollback_xa and --export options cannot " | ||
| "be used together. This combination causes internal mmap state " | ||
| "inconsistency leading to crashes."); | ||
| goto error; | ||
| } | ||
|
|
||
| srv_operation = xtrabackup_export | ||
| ? SRV_OPERATION_RESTORE_EXPORT : SRV_OPERATION_RESTORE; | ||
| ? SRV_OPERATION_RESTORE_EXPORT | ||
| : (xtrabackup_rollback_xa ? SRV_OPERATION_RESTORE_ROLLBACK_XA | ||
| : SRV_OPERATION_RESTORE); | ||
|
|
||
| if (innodb_init_param()) { | ||
| goto error; | ||
|
|
@@ -6920,12 +6951,46 @@ static bool xtrabackup_prepare_func(char** argv) | |
| srv_max_dirty_pages_pct_lwm = srv_max_buf_pool_modified_pct; | ||
| } | ||
|
|
||
| if (xtrabackup_rollback_xa) | ||
| srv_fast_shutdown = 0; | ||
|
|
||
| recv_sys.recovery_on = false; | ||
| if (innodb_init()) { | ||
| goto error; | ||
| } | ||
|
|
||
| ut_ad(!fil_system.freeze_space_list); | ||
| if (xtrabackup_rollback_xa) { | ||
| /* Roll back recovered prepared XA transactions. | ||
| The backup does not contain the binary log needed | ||
| to resolve them. (MDEV-36025) */ | ||
| XID *xid_list = | ||
| (XID *) my_malloc(PSI_NOT_INSTRUMENTED, | ||
| MAX_XID_LIST_SIZE * sizeof(XID), | ||
| MYF(0)); | ||
| if (!xid_list) { | ||
| msg("Can't allocate memory for XID list"); | ||
| ok = false; | ||
| goto cleanup; | ||
| } | ||
| ut_ad(recv_no_log_write); | ||
| ut_d(recv_no_log_write = false); | ||
| int got; | ||
| while ((got = trx_recover_for_mysql(xid_list, | ||
| MAX_XID_LIST_SIZE)) > 0) { | ||
| for (int i = 0; i < got; i++) { | ||
| trx_t *trx = trx_get_trx_by_xid(&xid_list[i]); | ||
| if (trx) { | ||
| trx_rollback_for_mysql(trx); | ||
| trx->free(); | ||
| msg("Rolled back prepared XA transaction"); | ||
| } | ||
| } | ||
| } | ||
| my_free(xid_list); | ||
| ut_d(recv_no_log_write = true); | ||
| } | ||
|
|
||
| ut_ad(!fil_system.freeze_space_list); | ||
|
|
||
| corrupted_pages.read_from_file(MB_CORRUPTED_PAGES_FILE); | ||
| if (xtrabackup_incremental) | ||
|
|
@@ -6982,9 +7047,21 @@ static bool xtrabackup_prepare_func(char** argv) | |
| else if (ok) xb_write_galera_info(xtrabackup_incremental); | ||
| #endif | ||
|
|
||
| innodb_shutdown(); | ||
| /* Without buf_flush_sync(), the rolled-back changes would exist only | ||
| in the buffer pool and be lost on shutdown, leaving the data files in | ||
| an inconsistent state. | ||
| In the innodb_preshutdown(), the condition was updated to include | ||
| SRV_OPERATION_RESTORE_ROLLBACK_XA so it waits for transactions when | ||
| srv_fast_shutdown == 0. The innodb_preshutdown() is called by | ||
| innodb_shutdown(), which will wait for any active transactions to | ||
| finish and shut down purge and undo background sources for | ||
| SRV_OPERATION_RESTORE_ROLLBACK_XA. */ | ||
| if (xtrabackup_rollback_xa) | ||
| buf_flush_sync_batch(0, false); | ||
|
|
||
| innodb_shutdown(); | ||
|
|
||
| innodb_free_param(); | ||
| innodb_free_param(); | ||
|
Comment on lines
-6985
to
+7064
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The comment is rather confusing. Why can’t we invoke the higher-level function |
||
|
|
||
| /* output to metadata file */ | ||
| if (ok) { | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,41 @@ | ||
| CALL mtr.add_suppression("Found 1 prepared XA transactions"); | ||
| RESET MASTER; | ||
| CREATE TABLE t1 (a INT) ENGINE=INNODB; | ||
| XA START 'test1'; | ||
| INSERT t1 VALUES (10); | ||
| XA END 'test1'; | ||
| XA PREPARE 'test1'; | ||
| XA RECOVER; | ||
| formatID gtrid_length bqual_length data | ||
| 1 5 0 test1 | ||
| # xtrabackup backup | ||
| XA ROLLBACK 'test1'; | ||
| # xtrabackup prepare and rollback prepared XA | ||
| # shutdown server | ||
| # remove datadir | ||
| # xtrabackup move back | ||
| # restart | ||
| XA RECOVER; | ||
| formatID gtrid_length bqual_length data | ||
| 1 5 0 test1 | ||
| # xtrabackup prepare and DO NOT rollback prepared XA | ||
| # shutdown server | ||
| # remove datadir | ||
| # xtrabackup move back | ||
| # restart | ||
| XA RECOVER; | ||
| formatID gtrid_length bqual_length data | ||
| 1 5 0 test1 | ||
| XA ROLLBACK 'test1'; | ||
| # xtrabackup prepare for export and rollback prepared XA (should fail) | ||
| # The above command should fail with error about incompatible options | ||
| # xtrabackup prepare for export and DO NOT rollback prepared XA | ||
| # shutdown server | ||
| # remove datadir | ||
| # xtrabackup move back | ||
| # restart | ||
| XA RECOVER; | ||
| formatID gtrid_length bqual_length data | ||
| 1 5 0 test1 | ||
| XA ROLLBACK 'test1'; | ||
| DROP TABLE t1; |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,75 @@ | ||
| # | ||
| # Optionally rollback prepared XA when backup is prepared | ||
| # | ||
| --source include/have_innodb.inc | ||
| --source include/have_binlog_format_mixed.inc | ||
|
|
||
| CALL mtr.add_suppression("Found 1 prepared XA transactions"); | ||
|
|
||
| RESET MASTER; | ||
|
|
||
| let targetdir1=$MYSQLTEST_VARDIR/tmp/backup1; | ||
| let targetdir2=$MYSQLTEST_VARDIR/tmp/backup2; | ||
| let targetdir3=$MYSQLTEST_VARDIR/tmp/backup3; | ||
| let targetdir4=$MYSQLTEST_VARDIR/tmp/backup4; | ||
|
|
||
| CREATE TABLE t1 (a INT) ENGINE=INNODB; | ||
| XA START 'test1'; | ||
| INSERT t1 VALUES (10); | ||
| XA END 'test1'; | ||
| XA PREPARE 'test1'; | ||
| XA RECOVER; | ||
|
|
||
| --echo # xtrabackup backup | ||
| --disable_result_log | ||
| exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir1; | ||
| --enable_result_log | ||
|
|
||
| perl; | ||
| use lib "lib"; | ||
| use My::Handles { suppress_init_messages => 1 }; | ||
| use My::File::Path; | ||
| copytree($ENV{'targetdir1'}, $ENV{'targetdir2'}); | ||
| copytree($ENV{'targetdir1'}, $ENV{'targetdir3'}); | ||
| copytree($ENV{'targetdir1'}, $ENV{'targetdir4'}); | ||
| EOF | ||
|
|
||
| XA ROLLBACK 'test1'; | ||
|
|
||
| --echo # xtrabackup prepare and rollback prepared XA | ||
| --disable_result_log | ||
| exec $XTRABACKUP --prepare --rollback_xa --target-dir=$targetdir1; | ||
| --let $targetdir = $targetdir1 | ||
| --source include/restart_and_restore.inc | ||
| --enable_result_log | ||
| XA RECOVER; | ||
|
|
||
| --echo # xtrabackup prepare and DO NOT rollback prepared XA | ||
| --disable_result_log | ||
| exec $XTRABACKUP --prepare --target-dir=$targetdir2; | ||
| --let $targetdir = $targetdir2 | ||
| --source include/restart_and_restore.inc | ||
| --enable_result_log | ||
| XA RECOVER; | ||
| XA ROLLBACK 'test1'; | ||
|
|
||
| --echo # xtrabackup prepare for export and rollback prepared XA (should fail) | ||
| --disable_result_log | ||
| --error 1 | ||
| exec $XTRABACKUP --prepare --rollback_xa --export --target-dir=$targetdir3; | ||
| --echo # The above command should fail with error about incompatible options | ||
|
|
||
| --echo # xtrabackup prepare for export and DO NOT rollback prepared XA | ||
| --disable_result_log | ||
| exec $XTRABACKUP --prepare --export --target-dir=$targetdir4 --skip-rollback-xa; | ||
| --let $targetdir = $targetdir4 | ||
| --source include/restart_and_restore.inc | ||
| --enable_result_log | ||
| XA RECOVER; | ||
| XA ROLLBACK 'test1'; | ||
|
|
||
| DROP TABLE t1; | ||
| rmdir $targetdir1; | ||
| rmdir $targetdir2; | ||
| rmdir $targetdir3; | ||
| rmdir $targetdir4; |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
30 changes: 30 additions & 0 deletions
30
mysql-test/suite/mariabackup/xa_prepared_on_restore.result
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,30 @@ | ||
| call mtr.add_suppression("Can't init tc log"); | ||
| call mtr.add_suppression("Found .* prepared transactions!"); | ||
| call mtr.add_suppression("Aborting"); | ||
| CREATE TABLE t1 (a INT) ENGINE=INNODB; | ||
| SET GLOBAL innodb_flush_log_at_trx_commit=1; | ||
| INSERT INTO t1 VALUES (0); | ||
| FLUSH TABLES; | ||
| connect con1,localhost,root,,; | ||
| SET debug_sync='ha_commit_trans_after_prepare WAIT_FOR go'; | ||
| INSERT INTO t1 VALUES (1); | ||
| connect con2,localhost,root,,; | ||
| SET debug_sync='ha_commit_trans_after_prepare WAIT_FOR go'; | ||
| INSERT INTO t1 VALUES (2); | ||
| connect con3,localhost,root,,; | ||
| SET debug_sync='ha_commit_trans_after_prepare WAIT_FOR go'; | ||
| INSERT INTO t1 VALUES (3); | ||
| connect con4,localhost,root,,; | ||
| SET debug_sync='ha_commit_trans_after_prepare WAIT_FOR go'; | ||
| INSERT INTO t1 VALUES (4); | ||
| connect con5,localhost,root,,; | ||
| SET debug_sync='ha_commit_trans_after_prepare WAIT_FOR go'; | ||
| INSERT INTO t1 VALUES (5); | ||
| connection default; | ||
| # Kill the server | ||
| FOUND 1 /Found .* prepared transactions!/ in mysqld.1.err | ||
| # restart | ||
| SELECT * FROM t1; | ||
| a | ||
| 0 | ||
| DROP TABLE t1; |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I don’t think it is acceptable to change the default behaviour in stable release series. The need to change a large number of existing tests in a stable release series should be a warning sign to any reviewer.
Furthermore, I don’t think it is acceptable to break incremental backup by default, in any release.