Mysql主从复制—gtid集合信息的变更时机(包含gtid初始化)

Posted 2023-02-23 翔之天空

 

参考：主从32讲的 2、GTID相关信息的变更时机

          主从32讲的 3、GTID模块初始化和参数simple recovery

          mysql 5.7 Gtid内部学习(五) mysql.gtid_executed表/gtid_executed变量/gtid_purged变量的更改时机 ：https://www.jianshu.com/p/905d7e89a305

          Mysql 5.7 Gtid内部学习(六) Mysql启动初始化Gtid模块：https://www.jianshu.com/p/fc836446cde0

 

版本：5.7.24

 

此文只是对上面的参考文章 做了一个整理

 

 

 

--代码中 各种gtid集合的注释：

  /**
    The set of GTIDs that existed in some previously purged binary log.
    This is always a subset of executed_gtids.
  */
  Gtid_set lost_gtids;

  /*
    The set of GTIDs that has been executed and
    stored into gtid_executed table.
  */
  Gtid_set executed_gtids;

  /*
    The set of GTIDs that exists only in gtid_executed table, not in
    binlog files.
  */
  Gtid_set gtids_only_in_table;

  /* The previous GTIDs in the last binlog. */
  Gtid_set previous_gtids_logged;

  /// The set of GTIDs that are owned by some thread.
  Owned_gtids owned_gtids;






--保存gtid集合到mysql.gtid_executed表中的方法：Gtid_state::save_gtids_of_last_binlog_into_table

--调用关系：在rotate时候(切换binlog) 需要更新mysql.gtid_executed表的数据
MYSQL_BIN_LOG::rotate
|-new_file_without_locking
  |-MYSQL_BIN_LOG::new_file_impl
    |-Gtid_state::save_gtids_of_last_binlog_into_table


int Gtid_state::save_gtids_of_last_binlog_into_table(bool on_rotation)

  DBUG_ENTER("Gtid_state::save_gtids_of_last_binlog_into_table");
  int ret= 0;
  /*
    Use local Sid_map, so that we don't need a lock while inserting
    into the table.
  */
  Sid_map sid_map(NULL);
  Gtid_set logged_gtids_last_binlog(&sid_map, NULL);
  // Allocate some intervals on stack to reduce allocation.
  static const int PREALLOCATED_INTERVAL_COUNT= 64;
  Gtid_set::Interval iv[PREALLOCATED_INTERVAL_COUNT];
  logged_gtids_last_binlog.add_interval_memory(PREALLOCATED_INTERVAL_COUNT, iv);
  /*
    logged_gtids_last_binlog= executed_gtids - previous_gtids_logged -
                              gtids_only_in_table
  */
  global_sid_lock->wrlock();
  ret= (logged_gtids_last_binlog.add_gtid_set(&executed_gtids) !=               //将当前执行过(executed_gtids)的gtid集合加入logged_gtids_last_binlog集合
        RETURN_STATUS_OK);
  if (!ret)
  
    logged_gtids_last_binlog.remove_gtid_set(&previous_gtids_logged);           //将不在当前binlog文件中的gtid(previous_gtids_logged)在logged_gtids_last_binlog集合中剔除掉
    logged_gtids_last_binlog.remove_gtid_set(&gtids_only_in_table);             //将只在mysql.gtid_executed表(不在binlog文件里)中的gtid在logged_gtids_last_binlog集合中剔除掉，剔除后logged_gtids_last_binlog集合只剩gtid_executed变量的集合
    if (!logged_gtids_last_binlog.is_empty() ||                                 //logged_gtids_last_binlog不为空或者要切换binlog日志时
        mysql_bin_log.is_rotating_caused_by_incident)
    
      /* Prepare previous_gtids_logged for next binlog on binlog rotation */
      if (on_rotation)
        ret= previous_gtids_logged.add_gtid_set(&logged_gtids_last_binlog);     //如果时要切换binlog日志时，需要用logged_gtids_last_binlog更新previous_gtids_logged的集合
      global_sid_lock->unlock();
      /* Save set of GTIDs of the last binlog into gtid_executed table */
      if (!ret)
        ret= save(&logged_gtids_last_binlog);                                   //用logged_gtids_last_binlog更新mysql.gtid_executed表
    
    else
      global_sid_lock->unlock();
  
  else
    global_sid_lock->unlock();
  DBUG_RETURN(ret);







--提交时更新mysql.gtid_executed表信息的方法 commit_owned_gtids

int commit_owned_gtids(THD *thd, bool all, bool *need_clear_owned_gtid_ptr)

  DBUG_ENTER("commit_owned_gtids(...)");
  int error= 0;
  if ((!opt_bin_log || (thd->slave_thread && !opt_log_slave_updates)) &&
      (all || !thd->in_multi_stmt_transaction_mode()) &&
      !thd->is_operating_gtid_table_implicitly &&
      !thd->is_operating_substatement_implicitly)
  
    /*
      If the binary log is disabled for this thread (either by
      log_bin=0 or sql_log_bin=0 or by log_slave_updates=0 for a
      slave thread), then the statement will not be written to
      the binary log. In this case, we should save its GTID into
      mysql.gtid_executed table and @@GLOBAL.GTID_EXECUTED as it
      did when binlog is enabled.
    */
    if (thd->owned_gtid.sidno > 0)
    
      error= gtid_state->save(thd);                                           //更新mysql.gtid_executed表
      *need_clear_owned_gtid_ptr= true;
    
    else if (thd->owned_gtid.sidno == THD::OWNED_SIDNO_ANONYMOUS)
      *need_clear_owned_gtid_ptr= true;
  
  else
  
    *need_clear_owned_gtid_ptr= false;
  
  DBUG_RETURN(error);






--更新gtid_executed变量和gtid_purged变量等gtid集合的方法 Gtid_state::update_gtids_impl_own_gtid

--调用关系，主库时在ordered_commit的commit阶段 更新gtid_executed变量的值，所以时实时更新的。
           但从库的时候没有ordered_commit步骤，所以直接在Gtid_state::update_on_commit方法中调用
MYSQL_BIN_LOG::ordered_commit
|-MYSQL_BIN_LOG::finish_commit
  |-Gtid_state::update_on_commit
    |-Gtid_state::update_gtids_impl
      |-Gtid_state::update_gtids_impl_own_gtid



void Gtid_state::update_gtids_impl_own_gtid(THD *thd, bool is_commit)

  if (is_commit)
  
    DBUG_ASSERT(!executed_gtids.contains_gtid(thd->owned_gtid));
    DBUG_EXECUTE_IF(
      "rpl_gtid_update_on_commit_simulate_out_of_memory",
      DBUG_SET("+d,rpl_gtid_get_free_interval_simulate_out_of_memory"););
    /*
      Any session adds transaction owned GTID into global executed_gtids.

      If binlog is disabled, we report @@GLOBAL.GTID_PURGED from
      executed_gtids, since @@GLOBAL.GTID_PURGED and @@GLOBAL.GTID_EXECUTED
      are always same, so we did not save gtid into lost_gtids for every
      transaction for improving performance.

      If binlog is enabled and log_slave_updates is disabled, slave
      SQL thread or slave worker thread adds transaction owned GTID
      into global executed_gtids, lost_gtids and gtids_only_in_table.
    */
    executed_gtids._add_gtid(thd->owned_gtid);                                 //更新gtid_executed变量
    thd->rpl_thd_ctx.session_gtids_ctx().
      notify_after_gtid_executed_update(thd);
    if (thd->slave_thread && opt_bin_log && !opt_log_slave_updates)            //如果是从库并且没有开log_slave_updates参数，也会实时的更新gtid_purged变量的。
    
      lost_gtids._add_gtid(thd->owned_gtid);                                   //更新gtid_purged变量
      gtids_only_in_table._add_gtid(thd->owned_gtid);
    
  
......








--清除binlog操作方法 MYSQL_BIN_LOG::purge_logs

int MYSQL_BIN_LOG::purge_logs(const char *to_log,bool included,bool need_lock_index,bool need_update_threads,ulonglong *decrease_log_space,bool auto_purge)

......
  // Update gtid_state->lost_gtids
  if (!is_relay_log)
  
    global_sid_lock->wrlock();
    error= init_gtid_sets(NULL,
                          const_cast<Gtid_set *>(gtid_state->get_lost_gtids()),    //gtid_state->get_lost_gtids() 传给init_gtid_sets， init_gtid_sets做正向查找获得
                          opt_master_verify_checksum,
                          false/*false=don't need lock*/,
                          NULL/*trx_parser*/, NULL/*gtid_partial_trx*/);
    global_sid_lock->unlock();
    if (error)
      goto err;
  
......








--清空GTID信息的方法 Gtid_state::clear

int Gtid_state::clear(THD *thd)

  DBUG_ENTER("Gtid_state::clear()");
  int ret= 0;
  // the wrlock implies that no other thread can hold any of the mutexes
  sid_lock->assert_some_wrlock();
  lost_gtids.clear();                                            //清空gtid_purged变量
  executed_gtids.clear();                                        //清空gtid_executed变量
  gtids_only_in_table.clear();                                   //清空only in table Gtid set           
  previous_gtids_logged.clear();                                 //清空 previous gtids logged Gtid set  
  /* Reset gtid_executed table. */
  if ((ret= gtid_table_persistor->reset(thd)) == 1)              //清空mysql.gtid_executed表
  
    /*
      Gtid table is not ready to be used, so failed to
      open it. Ignore the error.
    */
    thd->clear_error();
    ret= 0;
  
  next_free_gno= 1;
  DBUG_RETURN(ret);






--在做set global gtid_purged的时候，要把gtid_purged的信息 加入到 执行过的gtid集合中。 方法 Gtid_state::add_lost_gtids

enum_return_status Gtid_state::add_lost_gtids(const Gtid_set *gtid_set)

......
  if (save(gtid_set))                                                     //将set gtid_purge的值加入到mysql.gtid_executed表中
    RETURN_REPORTED_ERROR;
  PROPAGATE_REPORTED_ERROR(gtids_only_in_table.add_gtid_set(gtid_set));   //将set gtid_purge的值加入到gtids_only_in_table中
  PROPAGATE_REPORTED_ERROR(lost_gtids.add_gtid_set(gtid_set));            //将set gtid_purge的值加入到gtid_purge变量中
  PROPAGATE_REPORTED_ERROR(executed_gtids.add_gtid_set(gtid_set));        //将set gtid_purge的值加入到gtid_executed变量中
  lock_sidnos(gtid_set);
  broadcast_sidnos(gtid_set);
  unlock_sidnos(gtid_set);
  DBUG_RETURN(RETURN_STATUS_OK);







--mysql的server层 初始化时，各种gtid集合的初始化如下：

1：mysql.gtid_executed表：从mysql.gtid_executed持久表读取数据，然后将只在binlog存在而在mysql.gtid_executed表中不存在的gtid集合加入mysql.gtid_executed表中，这样mysql.gtid_executed表就为执行过的全部gtid集合。
2：gtid_executed变量： 从mysql.gtid_executed持久表读取数据， 然后将只在binlog存在而在gtid_executed变量中不存在的gtid集合加入gtid_executed变量中，这样gtid_executed变量就为执行过的全部gtid集合。
3： gtids_only_in_table：只在mysql.gtid_executed表中存在而在binlog不存在的gtid集合，gtids_only_in_table= executed_gtids - gtids_in_binlog
4： gtid_purged变量：lost_gtids = gtids_only_in_table + purged_gtids_from_binlog;
5： previous_gtids_logged：等于gtids_in_binlog的gtid集合



int mysqld_main(int argc, char **argv)
#endif

......
// Initialize executed_gtids from mysql.gtid_executed table.
  if (gtid_state->read_gtid_executed_from_table() == -1)                                //从mysql.gtid_executed表读取数据
    unireg_abort(1);
  if (opt_bin_log)
  
    /*
      Initialize GLOBAL.GTID_EXECUTED and GLOBAL.GTID_PURGED from
      gtid_executed table and binlog files during server startup.
    */
    Gtid_set *executed_gtids=
      const_cast<Gtid_set *>(gtid_state->get_executed_gtids());                        //建立gtid_executed变量的指针
    Gtid_set *lost_gtids=
      const_cast<Gtid_set *>(gtid_state->get_lost_gtids());                            //建立gtid_purged变量的指针
    Gtid_set *gtids_only_in_table=
      const_cast<Gtid_set *>(gtid_state->get_gtids_only_in_table());                   //建立gtids_only_in_table的指针
    Gtid_set *previous_gtids_logged=
      const_cast<Gtid_set *>(gtid_state->get_previous_gtids_logged());                 //建立previous_gtids_logged的指针

    Gtid_set purged_gtids_from_binlog(global_sid_map, global_sid_lock);                //定义purged_gtids_from_binlog临时变量用于存储从binlog中扫描到已经丢弃的Gtid事务。
    Gtid_set gtids_in_binlog(global_sid_map, global_sid_lock);                         //定义gtids_in_binlog中间变量binlog中包含的所有Gtid事务包括丢弃的。
    Gtid_set gtids_in_binlog_not_in_table(global_sid_map, global_sid_lock);            //定义gtids_in_binlog_not_in_table中间变量没有存放在表中而在binlog中存在过的Gtid事务，显然主库包含这样一个集合，因为主库的gtids_in_binlog>gtids_only_in_table，而从库同样也不包含这样一个集合因为从库的全部Gtid事务都在表中。


    if (mysql_bin_log.init_gtid_sets(&gtids_in_binlog,                                 //读取binlog文件。反向读取获得 gtids_in_binlog然后正向读取获得 purged_gtids_from_binlog，并且这里正向读取purged_gtids_from_binlog将会受到binlog_gtid_simple_recovery参数的影响，参考《Mysql参数01_参数最佳实践》章节。
                                     &purged_gtids_from_binlog,
                                     opt_master_verify_checksum,
                                     true/*true=need lock*/,
                                     NULL/*trx_parser*/,
                                     NULL/*gtid_partial_trx*/,
                                     true/*is_server_starting*/))


      unireg_abort(MYSQLD_ABORT_EXIT);
    global_sid_lock->wrlock();
    purged_gtids_from_binlog.dbug_print("purged_gtids_from_binlog");
    gtids_in_binlog.dbug_print("gtids_in_binlog");


    if (!gtids_in_binlog.is_empty() &&                                                 //如果gtids_in_binlog不为空 且 executed_gtids集合不是gtids_in_binlog集合的子集
        !gtids_in_binlog.is_subset(executed_gtids))
    
      gtids_in_binlog_not_in_table.add_gtid_set(&gtids_in_binlog);                     //把gtids_in_binlog集合加入到gtids_in_binlog_not_in_table集合中。
      if (!executed_gtids->is_empty())                                                 //如果executed_gtids集合不为空。
        gtids_in_binlog_not_in_table.remove_gtid_set(executed_gtids);                  //还要剔除掉执行过的gtid集合executed_gtids，这样gtids_in_binlog_not_in_table就正确了，是只在binlog中存在 而表中没有的gtid集合。
      /*
        Save unsaved GTIDs into gtid_executed table, in the following
        four cases:
          1. the upgrade case.
          2. the case that a slave is provisioned from a backup of
             the master and the slave is cleaned by RESET MASTER
             and RESET SLAVE before this.
          3. the case that no binlog rotation happened from the
             last RESET MASTER on the server before it crashes.
          4. The set of GTIDs of the last binlog is not saved into the
             gtid_executed table if server crashes, so we save it into
             gtid_executed table and executed_gtids during recovery
             from the crash.
      */
      if (gtid_state->save(&gtids_in_binlog_not_in_table) == -1)                        //这里将gtids_in_binlog_not_in_table这个Gtid集合存储到mysql.gtid_executed表中完成修正。
      
        global_sid_lock->unlock();
        unireg_abort(MYSQLD_ABORT_EXIT);
      
      executed_gtids->add_gtid_set(&gtids_in_binlog_not_in_table);                      //gtid_executed变量中加入这个gtids_in_binlog_not_in_table集合，完成了gtid_executed变量的修正。
    /* gtids_only_in_table= executed_gtids - gtids_in_binlog */
    if (gtids_only_in_table->add_gtid_set(executed_gtids) !=                            //把gtid_executed变量的gtid集合加入到gtids_only_in_table的集合
        RETURN_STATUS_OK)
    
      global_sid_lock->unlock();
      unireg_abort(MYSQLD_ABORT_EXIT);
    
    gtids_only_in_table->remove_gtid_set(&gtids_in_binlog);                             //此时gtids_only_in_table的集合只需要剔除在binlog中有的gtid集合即可。
    /*
      lost_gtids = executed_gtids -
                   (gtids_in_binlog - purged_gtids_from_binlog)
                 = gtids_only_in_table + purged_gtids_from_binlog;
    */
    DBUG_ASSERT(lost_gtids->is_empty());
    if (lost_gtids->add_gtid_set(gtids_only_in_table) != RETURN_STATUS_OK ||            //lost_gtids的集合 加入gtids_only_in_table和purged_gtids_from_binlog 集合即可。
        lost_gtids->add_gtid_set(&purged_gtids_from_binlog) !=
        RETURN_STATUS_OK)
    
      global_sid_lock->unlock();
      unireg_abort(MYSQLD_ABORT_EXIT);
    
    /* Prepare previous_gtids_logged for next binlog */
    if (previous_gtids_logged->add_gtid_set(&gtids_in_binlog) !=                        //gtids_in_binlog的集合 加入previous_gtids_logged即可。
        RETURN_STATUS_OK)
    
      global_sid_lock->unlock();
      unireg_abort(MYSQLD_ABORT_EXIT);