将数据一次性加载到DataSet与逐行从DB内读取的性能比较

Posted 2020-12-01 千年海岩

在开发中经常会遇到将当前处理批次的数据与数据库内的数据做校验的情况，通过有两种处理方式：

1、将待校验的数据一次性加载到DataSet，再将待校验的数据与DataSet内的数据逐行比较。

2、待校验数据在使用时，才从DB取出那一条数据进行比对。

第一种方法要每次在DataSet中查找数据，第二种直接定位到具体数据，两种方式在数据量为20W条记录的，第一种的处理时长是第二种的10倍。

 

        string strConnect = "Data Source=localhost;Initial Catalog=cmsw;Integrated Security=True; Application Name=pgq";
        DataSet dsCode = new DataSet();
        using (SqlConnection sqlConn = new SqlConnection(strConnect))
        {
           sqlConn.Open();
           SqlDataAdapter sda = new SqlDataAdapter("SELECT top 200000 TRANCODE from JobRecord  ", sqlConn);
           sda.Fill(dsCode, "Test");
           sqlConn.Close();
        }

        DateTime dt1 = DateTime.Now;

        foreach (DataRow dr in dsCode.Tables[0].Rows)
        {
           using (SqlConnection sqlConn = new SqlConnection(strConnect))
           {


               string code = dr["TRANCODE"].ToString();
               SqlCommand cmd = new SqlCommand("SELECT *  from JobRecord  WHERE TRANCODE=@TRANCODE ", sqlConn);
               cmd.Parameters.AddWithValue("TRANCODE", code);
               sqlConn.Open();
               SqlDataReader dataRead = cmd.ExecuteReader();
               if (dataRead.Read())
               {
                   Console.WriteLine("One Cycle:{0}",dataRead["TRANCODE"]);
               }
               sqlConn.Close();
           }
        }
        DateTime dt2 = DateTime.Now;
        TimeSpan ts1 = dt2 - dt1;
        Console.Clear();

        Console.WriteLine("Total MilSeconds:{0}", ts1.Milliseconds);

        DataSet dsTest2 = new DataSet();
        using (SqlConnection sqlConn = new SqlConnection(strConnect))
        {
           sqlConn.Open();
           SqlDataAdapter sda = new SqlDataAdapter("SELECT top 200000 * from JobRecord  ", sqlConn);
           sda.Fill(dsTest2, "dsTest2");
           sqlConn.Close();
        }
        int serialNo = 0;
        while (serialNo < 200000)
        {
           Random rand = new Random();
           int index = rand.Next(1, 200000);
           var code = dsTest2.Tables[0].AsEnumerable()
               .Where(r => r.Field<string>("TRANCODE") == dsCode.Tables[0].Rows[index]["TRANCODE"].ToString())
               .Select(r => r.Field<string>("TRANCODE")).ElementAtOrDefault(0);
           Console.WriteLine("Scend Cycle:{0}",code);
           serialNo++;
        }

        DateTime dt3 = DateTime.Now;
        TimeSpan ts2 = dt3 - dt2;
        Console.WriteLine("First Result, TotalSeconds:{0};
Second Result TotalSeconds:{1}", ts1.TotalSeconds, ts2.TotalSeconds);
        Console.ReadLine();