MDEV-36436 Assertion "unexpected references" == 0 after ALTER IGNORE TABLE

include/my_base.h

@@ -225,6 +225,8 @@ enum ha_extra_function {
   HA_EXTRA_STARTING_ORDERED_INDEX_SCAN,
   /** Start writing rows during ALTER TABLE...ALGORITHM=COPY. */
   HA_EXTRA_BEGIN_ALTER_COPY,
+  /** Start writing rows during ALTER IGNORE TABLE..ALGORITHM=COPY */
+  HA_EXTRA_BEGIN_ALTER_IGNORE_COPY,
   /** Finish writing rows during ALTER TABLE...ALGORITHM=COPY. */
   HA_EXTRA_END_ALTER_COPY,
   /** Abort of writing rows during ALTER TABLE..ALGORITHM=COPY or

mysql-test/suite/innodb/r/innodb-alter-nullable.result

@@ -78,8 +78,8 @@ ALTER IGNORE TABLE t1 ADD UNIQUE(c);
 affected rows: 3
 info: Records: 3  Duplicates: 1  Warnings: 0
 ALTER IGNORE TABLE t1 ADD PRIMARY KEY(c);
-affected rows: 2
-info: Records: 2  Duplicates: 0  Warnings: 1
+affected rows: 3
+info: Records: 3  Duplicates: 1  Warnings: 1
 Warnings:
 Warning	1265	Data truncated for column 'c' at row 1
 SELECT * FROM t1;

sql/ha_partition.cc

@@ -9492,6 +9492,7 @@ int ha_partition::extra(enum ha_extra_function operation)
   case HA_EXTRA_BEGIN_ALTER_COPY:
   case HA_EXTRA_END_ALTER_COPY:
   case HA_EXTRA_ABORT_ALTER_COPY:
+  case HA_EXTRA_BEGIN_ALTER_IGNORE_COPY:
     DBUG_RETURN(loop_partitions(extra_cb, &operation));
   default:
   {

sql/sql_insert.cc

@@ -5165,8 +5165,13 @@ select_create::prepare(List<Item> &_values, SELECT_LEX_UNIT *u)
       !table->s->long_unique_table)
   {
     table->file->ha_start_bulk_insert((ha_rows) 0);
-    if (thd->lex->duplicates == DUP_ERROR && !thd->lex->ignore)
-      table->file->extra(HA_EXTRA_BEGIN_ALTER_COPY);
+    if (thd->lex->duplicates == DUP_ERROR)
+    {
+      static_assert(int{HA_EXTRA_BEGIN_ALTER_IGNORE_COPY} ==
+		    int{HA_EXTRA_BEGIN_ALTER_COPY} + 1, "");
+      table->file->extra(ha_extra_function(int{HA_EXTRA_BEGIN_ALTER_COPY} +
+                                           thd->lex->ignore));
+    }
     table->file->extra(HA_EXTRA_WRITE_CACHE);
   }
   thd->abort_on_warning= !info.ignore && thd->is_strict_mode();

sql/sql_table.cc

@@ -12351,8 +12351,9 @@ copy_data_between_tables(THD *thd, TABLE *from, TABLE *to, bool ignore,
 
   thd->progress.max_counter= from->file->records();
   time_to_report_progress= MY_HOW_OFTEN_TO_WRITE/10;
-  if (!ignore) /* for now, InnoDB needs the undo log for ALTER IGNORE */
-    to->file->extra(HA_EXTRA_BEGIN_ALTER_COPY);
+  static_assert(int{HA_EXTRA_BEGIN_ALTER_IGNORE_COPY} ==
+                int{HA_EXTRA_BEGIN_ALTER_COPY} + 1, "");
+  to->file->extra(ha_extra_function(int{HA_EXTRA_BEGIN_ALTER_COPY} + ignore));
 
   while (likely(!(error= info.read_record())))
   {
@@ -12455,6 +12456,9 @@ copy_data_between_tables(THD *thd, TABLE *from, TABLE *to, bool ignore,
 
   if (error > 0 && !from->s->tmp_table)
   {
+    /* Abort the operation which invoked extra(HA_EXTRA_BEGIN_ALTER_COPY)
+    or extra(HA_EXTRA_BEGIN_ALTER_IGNORE_COPY) */
+    to->file->extra(HA_EXTRA_ABORT_ALTER_COPY);
     /* We are going to drop the temporary table */
     to->file->extra(HA_EXTRA_PREPARE_FOR_DROP);
   }
@@ -12466,7 +12470,7 @@ copy_data_between_tables(THD *thd, TABLE *from, TABLE *to, bool ignore,
     error= 1;
   }
   bulk_insert_started= 0;
-  if (!ignore && error <= 0)
+  if (error <= 0)
   {
     int alt_error= to->file->extra(HA_EXTRA_END_ALTER_COPY);
     if (alt_error > 0)

storage/innobase/btr/btr0cur.cc

@@ -2286,7 +2286,7 @@ btr_cur_ins_lock_and_undo(
 	      || dict_index_is_clust(index)
 	      || (flags & BTR_CREATE_FLAG));
 	ut_ad((flags & BTR_NO_UNDO_LOG_FLAG)
-	      || !index->table->skip_alter_undo);
+	      || index->table->skip_alter_undo != dict_table_t::NO_UNDO);
 
 	ut_ad(mtr->is_named_space(index->table->space));
 

storage/innobase/handler/ha_innodb.cc

@@ -111,6 +111,7 @@ extern my_bool opt_readonly;
 #include "ut0mem.h"
 #include "row0ext.h"
 #include "mariadb_stats.h"
+#include "trx0undo.h"
 simple_thread_local ha_handler_stats *mariadb_stats;
 
 #include "lz4.h"
@@ -15884,9 +15885,13 @@ ha_innobase::extra(
 			break;
 		}
 		goto stmt_boundary;
+	case HA_EXTRA_BEGIN_ALTER_IGNORE_COPY:
 	case HA_EXTRA_BEGIN_ALTER_COPY:
 		trx = check_trx_exists(thd);
-		m_prebuilt->table->skip_alter_undo = 1;
+		m_prebuilt->table->skip_alter_undo =
+			(operation == HA_EXTRA_BEGIN_ALTER_COPY)
+			? dict_table_t::NO_UNDO
+			: dict_table_t::IGNORE_UNDO;
 		if (m_prebuilt->table->is_temporary()
 		    || !m_prebuilt->table->versioned_by_id()) {
 			break;
@@ -15905,7 +15910,15 @@ ha_innobase::extra(
 			they could cause a severe performance regression. */
 			break;
 		}
-		m_prebuilt->table->skip_alter_undo = 0;
+
+		if (m_prebuilt->table->skip_alter_undo ==
+			dict_table_t::IGNORE_UNDO) {
+			trx->reset_and_truncate_undo();
+		}
+
+		m_prebuilt->table->skip_alter_undo =
+			dict_table_t::NORMAL_UNDO;
+
 		if (dberr_t err= trx->bulk_insert_apply<TRX_DDL_BULK>()) {
 			trx->rollback();
 			return convert_error_code_to_mysql(
@@ -15935,10 +15948,15 @@ ha_innobase::extra(
 		alter_stats_rebuild(m_prebuilt->table, thd);
 		break;
 	case HA_EXTRA_ABORT_ALTER_COPY:
+		/* Abort the ALTER COPY operation. For ALTER IGNORE,
+		this prevents undo logs from being added to the
+		purge thread, allowing proper cleanup of the
+		temporary undo state. */
 		if (m_prebuilt->table->skip_alter_undo &&
 		    !m_prebuilt->table->is_temporary()) {
 			trx = check_trx_exists(ha_thd());
-			m_prebuilt->table->skip_alter_undo = 0;
+			m_prebuilt->table->skip_alter_undo =
+				dict_table_t::NORMAL_UNDO;
 			trx->rollback();
 			return(HA_ERR_ROLLBACK);
 		}

storage/innobase/include/dict0mem.h

@@ -2222,12 +2222,21 @@ struct dict_table_t {
 	Use DICT_TF2_FLAG_IS_SET() to parse this flag. */
 	unsigned				flags2:DICT_TF2_BITS;
 
-	/** TRUE if the table is an intermediate table during copy alter
-	operation or a partition/subpartition which is required for copying
-	data and skip the undo log for insertion of row in the table.
-	This variable will be set and unset during extra(), or during the
+	/** Undo log handling modes for ALTER [IGNORE] TABLE...ALGORITHM=COPY */
+	static constexpr unsigned	NORMAL_UNDO = 0;
+	/** Never writes row-level undo log records */
+	static constexpr unsigned	NO_UNDO = 1;
+	/** For ALTER IGNORE TABLE...ALGORITHM=COPY, this enables rewriting
+	old insert undo blocks to maintain only the latest insert undo log. */
+	static constexpr unsigned	IGNORE_UNDO = 2;
+
+	/** Mode for handling undo logs during ALTER TABLE...ALGORITHM=COPY operations.
+	This will not be consulted in ha_innobase::inplace_alter_table();
+	Set during copy alter operations or partition/subpartition operations.
+	When set, controls undo log behavior for row operations in the table.
+	This variable is set and unset during extra(), or during the
 	process of altering partitions */
-	unsigned                                skip_alter_undo:1;
+	unsigned                                skip_alter_undo:2;
 
 	/*!< whether this is in a single-table tablespace and the .ibd
 	file is missing or page decryption failed and page is corrupted */

storage/innobase/include/trx0trx.h

@@ -1205,6 +1205,9 @@ struct trx_t : ilist_node<>
     return bulk_insert == type ? bulk_insert_apply_low(): DB_SUCCESS;
   }
 
+  /** Reset the undo no and remove the undo log from transaction */
+  void reset_and_truncate_undo();
+
 private:
   /** Apply the buffered bulk inserts. */
   dberr_t bulk_insert_apply_low();

storage/innobase/include/trx0undo.h

@@ -271,6 +271,10 @@ struct trx_undo_t {
 	uint16_t	top_offset;	/*!< offset of the latest undo record,
 					i.e., the topmost element in the undo
 					log if we think of it as a stack */
+	/** For ALTER IGNORE: store previous undo record info to
+	rewrite undo chain */
+	uint16_t	old_offset;	/*!< previous undo record offset
+					for ALTER IGNORE */
 	undo_no_t	top_undo_no;	/*!< undo number of the latest record
 					(IB_ID_MAX if the undo log is empty) */
 	buf_block_t*	guess_block;	/*!< guess for the buffer block where

storage/innobase/row/row0ins.cc

@@ -2781,7 +2781,13 @@ row_ins_clust_index_entry_low(
 		if (!index->table->n_rec_locks
 		    && !index->table->versioned()
 		    && !index->table->is_temporary()
-		    && !index->table->has_spatial_index()) {
+		    && !index->table->has_spatial_index()
+		    /* Prevent ALTER IGNORE from using bulk insert
+		    optimization. since it requires to write undo log
+		    for each row operation that's incompatible with
+		    bulk operations */
+		    && index->table->skip_alter_undo !=
+		         dict_table_t::IGNORE_UNDO) {
 
 			ut_ad(!index->table->skip_alter_undo);
 			trx->bulk_insert = TRX_DML_BULK;
@@ -3341,7 +3347,7 @@ row_ins_clust_index_entry(
 	   skip the undo log and record lock checking for
 	   insertion operation.
 	*/
-	if (index->table->skip_alter_undo) {
+	if (index->table->skip_alter_undo == dict_table_t::NO_UNDO) {
 		flags |= BTR_NO_UNDO_LOG_FLAG | BTR_NO_LOCKING_FLAG;
 	}
 

storage/innobase/row/row0uins.cc

@@ -452,7 +452,7 @@ static bool row_undo_ins_parse_undo_rec(undo_node_t* node, bool dict_locked)
 		node->table = NULL;
 		return false;
 	} else {
-		ut_ad(!node->table->skip_alter_undo);
+		ut_ad(node->table->skip_alter_undo != dict_table_t::NO_UNDO);
 		clust_index = dict_table_get_first_index(node->table);
 
 		if (clust_index != NULL) {

storage/innobase/row/row0undo.cc

@@ -171,7 +171,7 @@ row_undo_search_clust_to_pcur(
 	rec_offs*	offsets		= offsets_;
 	rec_offs_init(offsets_);
 
-	ut_ad(!node->table->skip_alter_undo);
+	ut_ad(node->table->skip_alter_undo != dict_table_t::NO_UNDO);
 
 	mtr_start(&mtr);
 

storage/innobase/trx/trx0rec.cc

@@ -1912,9 +1912,29 @@ trx_undo_report_row_operation(
 		ut_ad(!trx->read_only);
 		ut_ad(trx->id);
 		pundo = &trx->rsegs.m_redo.undo;
+		const bool empty{!*pundo};
 		rseg = trx->rsegs.m_redo.rseg;
 		undo_block = trx_undo_assign_low<false>(trx, rseg, pundo,
 							&mtr, &err);
+                /* For ALTER IGNORE, implement undo log rewriting
+		to maintain only the latest insert undo record.
+		This allows easy rollback of the last inserted row
+		on duplicate key errors. Before writing a new undo
+		record, rewind the undo log to the previous record
+		position, effectively discarding all intermediate
+		undo records and keeping only the most recent one. */
+		if (!empty && (*pundo)->old_offset <= (*pundo)->top_offset &&
+		    index->table->skip_alter_undo ==
+			dict_table_t::IGNORE_UNDO) {
+
+			(*pundo)->top_offset = (*pundo)->old_offset;
+			(*pundo)->top_undo_no = 0;
+			mtr.write<2>(
+			  *undo_block,
+			  undo_block->page.frame + TRX_UNDO_PAGE_HDR +
+			  TRX_UNDO_PAGE_FREE, (*pundo)->old_offset);
+			trx->undo_no= 0;
+		}
 	}
 
 	trx_undo_t*	undo	= *pundo;
@@ -2001,6 +2021,8 @@ trx_undo_report_row_operation(
 			/* Success */
 			undo->top_page_no = undo_block->page.id().page_no();
 			mtr.commit();
+
+			undo->old_offset = offset;
 			undo->top_offset  = offset;
 			undo->top_undo_no = trx->undo_no++;
 			undo->guess_block = undo_block;

storage/innobase/trx/trx0trx.cc

@@ -2269,3 +2269,12 @@ trx_set_rw_mode(
 		trx->read_view.set_creator_trx_id(trx->id);
 	}
 }
+
+/** Reset the undo no and remove the undo log from transaction */
+void trx_t::reset_and_truncate_undo()
+{
+	ut_ad(undo_no <= 1);
+	ut_ad(mod_tables.size() == 1 || undo_no == 0);
+	undo_no= 0;
+	trx_undo_try_truncate(*this);
+}

storage/mroonga/ha_mroonga.cpp

@@ -561,6 +561,9 @@ static const char *mrn_inspect_extra_function(enum ha_extra_function operation)
   case HA_EXTRA_ABORT_ALTER_COPY:
     inspected = "HA_EXTRA_ABORT_ALTER_COPY";
     break;
+  case HA_EXTRA_BEGIN_ALTER_IGNORE_COPY:
+    inspected = "HA_EXTRA_BEGIN_ALTER_IGNORE_COPY";
+    break;
 #ifdef MRN_HAVE_HA_EXTRA_EXPORT
   case HA_EXTRA_EXPORT:
     inspected = "HA_EXTRA_EXPORT";