MDEV-5673: Crash while parallel dropping multiple tables under heavy load

Improve long semaphore wait output to include all semaphore waits
and try to find out if there is a sequence of waiters.

storage/innobase/sync/sync0arr.c

@@ -468,7 +468,9 @@ void
 sync_array_cell_print(
 /*==================*/
 	FILE*		file,		/*!< in: file where to print */
-	sync_cell_t*	cell)	/*!< in: sync cell */
+	sync_cell_t*	cell,		/*!< in: sync cell */
+	os_thread_id_t* reserver)	/*!< out: write reserver or
+					0 */
 {
 	mutex_t*	mutex = NULL;
 	rw_lock_t*	rwlock = NULL;
@@ -526,6 +528,8 @@ sync_array_cell_print(
 				writer == RW_LOCK_EX
 				? " exclusive\n"
 				: " wait exclusive\n");
+
+			*reserver = rwlock->writer_thread;
 		}
 
 		fprintf(file,
@@ -549,7 +553,6 @@ sync_array_cell_print(
 	}
 }
 
-#ifdef UNIV_SYNC_DEBUG
 /******************************************************************//**
 Looks for a cell with the given thread id.
 @return	pointer to cell or NULL if not found */
@@ -577,6 +580,8 @@ sync_array_find_thread(
 	return(NULL);	/* Not found */
 }
 
+#ifdef UNIV_SYNC_DEBUG
+
 /******************************************************************//**
 Recursion step for deadlock detection.
 @return	TRUE if deadlock detected */
@@ -948,6 +953,7 @@ sync_array_print_long_waits(
 
 		double	diff;
 		void*	wait_object;
+		os_thread_id_t reserver=0;
 
 		cell = sync_array_get_nth_cell(sync_primary_wait_array, i);
 
@@ -963,8 +969,29 @@ sync_array_print_long_waits(
 		if (diff > SYNC_ARRAY_TIMEOUT) {
 			fputs("InnoDB: Warning: a long semaphore wait:\n",
 			      stderr);
-			sync_array_cell_print(stderr, cell);
+			sync_array_cell_print(stderr, cell, &reserver);
 			noticed = TRUE;
+		} else {
+			fputs("InnoDB: Warning: semaphore wait:\n",
+			      stderr);
+			sync_array_cell_print(stderr, cell, &reserver);
+		}
+
+		/* Try to output cell information for writer recursive way */
+		while (reserver != 0) {
+			sync_cell_t* reserver_wait;
+
+			reserver_wait = sync_array_find_thread(sync_primary_wait_array, reserver);
+
+			if (reserver_wait &&
+			    reserver_wait->wait_object != NULL &&
+			    reserver_wait->waiting) {
+				fputs("InnoDB: Warning: Writer thread is waiting this semaphore:\n",
+				      stderr);
+				sync_array_cell_print(stderr, reserver_wait, &reserver);
+			} else {
+				reserver = 0;
+			}
 		}
 
 		if (diff > fatal_timeout) {
@@ -1026,6 +1053,7 @@ sync_array_output_info(
 	sync_cell_t*	cell;
 	ulint		count;
 	ulint		i;
+	os_thread_id_t  r;
 
 	fprintf(file,
 		"OS WAIT ARRAY INFO: reservation count %ld, signal count %ld\n",
@@ -1039,7 +1067,7 @@ sync_array_output_info(
 
 		if (cell->wait_object != NULL) {
 			count++;
-			sync_array_cell_print(file, cell);
+			sync_array_cell_print(file, cell, &r);
 		}
 
 		i++;

storage/xtradb/sync/sync0arr.c

@@ -468,7 +468,9 @@ void
 sync_array_cell_print(
 /*==================*/
 	FILE*		file,		/*!< in: file where to print */
-	sync_cell_t*	cell)	/*!< in: sync cell */
+	sync_cell_t*	cell,		/*!< in: sync cell */
+	os_thread_id_t* reserver)	/*!< out: write reserver or
+					0 */
 {
 	mutex_t*	mutex = NULL;
 	rw_lock_t*	rwlock = NULL;
@@ -538,6 +540,8 @@ sync_array_cell_print(
 					writer == RW_LOCK_EX
 					? " exclusive\n"
 					: " wait exclusive\n");
+
+				*reserver = rwlock->writer_thread;
 			}
 
 			fprintf(file,
@@ -573,7 +577,6 @@ sync_array_cell_print(
 	}
 }
 
-#ifdef UNIV_SYNC_DEBUG
 /******************************************************************//**
 Looks for a cell with the given thread id.
 @return	pointer to cell or NULL if not found */
@@ -601,6 +604,8 @@ sync_array_find_thread(
 	return(NULL);	/* Not found */
 }
 
+#ifdef UNIV_SYNC_DEBUG
+
 /******************************************************************//**
 Recursion step for deadlock detection.
 @return	TRUE if deadlock detected */
@@ -972,6 +977,7 @@ sync_array_print_long_waits(
 
 		double	diff;
 		void*	wait_object;
+		os_thread_id_t reserver=0;
 
 		cell = sync_array_get_nth_cell(sync_primary_wait_array, i);
 
@@ -987,8 +993,29 @@ sync_array_print_long_waits(
 		if (diff > SYNC_ARRAY_TIMEOUT) {
 			fputs("InnoDB: Warning: a long semaphore wait:\n",
 			      stderr);
-			sync_array_cell_print(stderr, cell);
+			sync_array_cell_print(stderr, cell, &reserver);
 			noticed = TRUE;
+		} else {
+			fputs("InnoDB: Warning: semaphore wait:\n",
+			      stderr);
+			sync_array_cell_print(stderr, cell, &reserver);
+		}
+
+		/* Try to output cell information for writer recursive way */
+		while (reserver != 0) {
+			sync_cell_t* reserver_wait;
+
+			reserver_wait = sync_array_find_thread(sync_primary_wait_array, reserver);
+
+			if (reserver_wait &&
+			    reserver_wait->wait_object != NULL &&
+			    reserver_wait->waiting) {
+				fputs("InnoDB: Warning: Writer thread is waiting this semaphore:\n",
+				      stderr);
+				sync_array_cell_print(stderr, reserver_wait, &reserver);
+			} else {
+				reserver = 0;
+			}
 		}
 
 		if (diff > fatal_timeout) {
@@ -1050,6 +1077,7 @@ sync_array_output_info(
 	sync_cell_t*	cell;
 	ulint		count;
 	ulint		i;
+	os_thread_id_t  r;
 
 	fprintf(file,
 		"OS WAIT ARRAY INFO: reservation count %ld, signal count %ld\n",
@@ -1063,7 +1091,7 @@ sync_array_output_info(
 
 		if (cell->wait_object != NULL) {
 			count++;
-			sync_array_cell_print(file, cell);
+			sync_array_cell_print(file, cell, &r);
 		}
 
 		i++;