MDEV-26010: Assertion lsn > 2 failed in buf_pool_t::get_oldest_modification

In commit 22b62edaed (MDEV-25113)
we introduced a race condition. buf_LRU_free_page() would read
buf_page_t::oldest_modification() as 0 and assume that
buf_page_t::list can be used (for attaching the block to the
buf_pool.free list). In the observed race condition,
buf_pool_t::delete_from_flush_list() had cleared the field,
and buf_pool_t::delete_from_flush_list_low() was executing
concurrently with buf_LRU_block_free_non_file_page(),
which resulted in buf_pool.flush_list.end becoming corrupted.

buf_pool_t::delete_from_flush_list(), buf_flush_relocate_on_flush_list():
First remove the block from buf_pool.flush_list, and only then
invoke buf_page_t::clear_oldest_modification(), to ensure that
reading oldest_modification()==0 really implies that the block
no longer is in buf_pool.flush_list.

storage/innobase/buf/buf0flu.cc

@@ -208,10 +208,10 @@ void buf_pool_t::insert_into_flush_list(buf_block_t *block, lsn_t lsn)
 @param clear   whether to invoke buf_page_t::clear_oldest_modification() */
 void buf_pool_t::delete_from_flush_list(buf_page_t *bpage, bool clear)
 {
-  if (clear)
-    bpage->clear_oldest_modification();
   delete_from_flush_list_low(bpage);
   stat.flush_list_bytes-= bpage->physical_size();
+  if (clear)
+    bpage->clear_oldest_modification();
 #ifdef UNIV_DEBUG
   buf_flush_validate_skip();
 #endif /* UNIV_DEBUG */
@@ -309,10 +309,10 @@ buf_flush_relocate_on_flush_list(
 		the bpage from the flush list. */
 		buf_pool.flush_hp.adjust(bpage);
 
-		bpage->clear_oldest_modification();
-
 		prev = UT_LIST_GET_PREV(list, bpage);
 		UT_LIST_REMOVE(buf_pool.flush_list, bpage);
+
+		bpage->clear_oldest_modification();
 	} else {
 		/* bpage was removed from buf_pool.flush_list
 		since we last checked, and before we acquired

storage/innobase/include/buf0buf.h

@@ -944,7 +944,7 @@ public:
   lsn_t oldest_modification() const { return oldest_modification_; }
   /** Set oldest_modification when adding to buf_pool.flush_list */
   inline void set_oldest_modification(lsn_t lsn);
-  /** Clear oldest_modification when removing from buf_pool.flush_list */
+  /** Clear oldest_modification after removing from buf_pool.flush_list */
   inline void clear_oldest_modification();
   /** Note that a block is no longer dirty, while not removing
   it from buf_pool.flush_list */
@@ -2246,10 +2246,11 @@ inline void buf_page_t::set_oldest_modification(lsn_t lsn)
 {
   mysql_mutex_assert_owner(&buf_pool.flush_list_mutex);
   ut_ad(oldest_modification() <= 1);
+  ut_ad(lsn > 2);
   oldest_modification_= lsn;
 }
 
-/** Clear oldest_modification when removing from buf_pool.flush_list */
+/** Clear oldest_modification after removing from buf_pool.flush_list */
 inline void buf_page_t::clear_oldest_modification()
 {
   mysql_mutex_assert_owner(&buf_pool.flush_list_mutex);