优化复杂查询
Posted
技术标签:
【中文标题】优化复杂查询【英文标题】：Optimize A Complex Query 【发布时间】：2019-04-09 12:36:30 【问题描述】：下面的过程执行需要 9 秒，而主表只有 30000 条记录。
ALTER PROC [dbo].[TransactionReport_NewWork] @StartDate datetime = NULL, @EndDate datetime = NULL
, @Mid varchar(50) = NULL -- varchar(5000)=null
, @BatchNo varchar(50) = NULL,
@AuthId varchar(50) = NULL, @RRN varchar(50) = NULL --varchar(500)=null
, @CardNo varchar(50) = NULL, @PageNo int = NULL, @PageSize int = 10, @ReportType int = NULL, @ReportSubType int = NULL,
@IsOrder varchar(10) = NULL,
@CurrencyCode varchar(10) = NULL
, @InvoiceNo varchar(50) = NULL
, @AppCode varchar(5) = NULL
, @ProductCodes StringValues READONLY, --varchar(10)=null,
@TIDS StringValues READONLY
, @SettlementStatus IntegerValues READONLY -- varchar(50)=null
, @TransactionType IntegerValues READONLY
, @ReportValues StringValues READONLY
AS
SET @PageNo = (CASE
WHEN @PageNo = 0 THEN 1
WHEN @PageNo IS NULL THEN 1
ELSE @PageNo
END)
SET @PageSize = (CASE
WHEN @PageSize = 0 THEN 2147483647
ELSE @PageSize
END)
SELECT
x.* --into #TransLogDetail
FROM (SELECT TOP 100 PERCENT
ROW_NUMBER() OVER (
--order by a.TransactionDateTime desc
ORDER BY
CASE
WHEN j.ProductName IS NOT NULL THEN j.ProductCode
END ASC,
CASE
WHEN j.ProductName IS NULL THEN a.TransactionDateTime
END DESC,
a.TransactionDateTime DESC
) SNo,
b.MerchantName,
a.var64_42 MID,
a.var64_41 TID,
(CASE
WHEN c.TransactionType = 'SALE' THEN d.TransactionStatus
WHEN c.TransactionType = 'MOTO' THEN 'MOTO'
--when c.TransactionType ='LOYALTY_POINTS_REDEMPTION' then c.TransactionType
ELSE d.TransactionStatus
END)
AS TransactionType,
a.var64_60 BatchNo,
(CASE
WHEN
a.var64_54 IS NOT NULL THEN CAST(CONVERT(decimal, a.var64_04) / 100 AS numeric(18, 2)) - CAST(ISNULL(CONVERT(decimal, a.var64_54) / 100, 0) AS numeric(18, 2))
ELSE CAST(CONVERT(decimal, a.var64_04) / 100 AS numeric(18, 2))
END) Amount,
(CASE
WHEN c.TransactionType = 'MOTO' THEN SUBSTRING(a.var64_02, 1, 6) + '******' + SUBSTRING(a.var64_02, 13, 4)
ELSE
--substring(a.var64_35,1,6) +'******'+ substring(a.var64_35,13,4)
SUBSTRING(a.var64_02, 1, 6) + '******' + SUBSTRING(a.var64_02, 13, 4)
END)
CardNumber,
a.var64_38 AuthID,
a.var64_37 RRN,
a.var64_62 InvoiceNo,
CONVERT(datetime, STUFF(STUFF((CONVERT(varchar, DATEPART(YEAR, GETDATE())) + a.var64_13 + ' ' + a.var64_12), 12, 0, ':'), 15, 0, ':'))
TransactionDateTime,
e.SettlementStatus,
a.var64_35 Track2,
a.var64_11 Stan,
h.Description AS POSEntryMode,
g.Description AS POSConditionCode,
(CASE
WHEN a.var64_54 IS NULL THEN NULL
ELSE CAST(ISNULL(CONVERT(decimal, a.var64_54) / 100, 0) AS numeric(18, 2))
END) Tip,
a.var64_55 BatchData,
a.OrderNo,
a.var64_48 KSN
--, convert(varchar,a.TransactionDateTime,120) TransactionDateTime_Web
,
STUFF(STUFF((CONVERT(varchar, DATEPART(YEAR, GETDATE())) + a.var64_13 + ' ' + a.var64_12), 12, 0, ':'), 15, 0, ':') TransactionDateTime_Web
--, isnull( i.Code +'('+isnull(i.Symbol,'')+')', N'PKR(?)') as CurrencyCode
,
ISNULL(i.Code, 'PKR') AS CurrencyCode,
a.var64_02 Pan,
j.ProductName,
(CASE
WHEN a.var64_28 = '' THEN 0
ELSE CAST(ISNULL(CONVERT(decimal, a.var64_28) / 100, 0) AS numeric(18, 2))
END) AS ProductPrice,
(CASE
WHEN a.var64_28 = '' THEN 0
ELSE CAST(ISNULL(CONVERT(decimal, RIGHT(a.var64_61, 12)) / 100, 0) AS numeric(18, 2))
END) ProductQuantity,
--a.var64_28 as ProductPrice
--,a.var64_61 as ProductQuantity,
(CASE
WHEN LEN(a.var64_63) >= 57 THEN CAST(ISNULL(CONVERT(decimal, SUBSTRING(a.var64_63, 1, 12)) / 100, 0) AS numeric(18, 2)) +
CAST(ISNULL(CONVERT(decimal, SUBSTRING(a.var64_63, 43, 12)) / 100, 0) AS numeric(18, 2))
ELSE 0
END) TotalDiscount,
TotalRecords = COUNT(*) OVER (),
TotalPages = CAST(CEILING(COUNT(*) OVER () / (@PageSize * 1.0)) AS int)
--substring(a.var64_63,1,12)+'Part2'+substring(a.var64_63,43,12) TotalDiscount
FROM TransactionResponseLog a
LEFT JOIN Merchant b
ON a.var64_42 = b.mid
AND b.isactive = 1
LEFT JOIN GatewayTransactionType c
ON a.TransactionTypeID = c.TransactionTypeID
LEFT JOIN TransactionStatus d
ON a.TransactionStatusID = d.TransactionStatusId
LEFT JOIN SettlementStatus e
ON a.SettlementStatusID = e.SettlementStatusId
LEFT JOIN Association f
ON
--substring(a.var64_35,1,1)
SUBSTRING(a.var64_02, 1, 1)
= f.PaymentAssocationCode
LEFT JOIN POSConditionCode g
ON a.var64_25 = g.Code
LEFT JOIN POSEntryMode h
ON a.var64_22 = h.Code
LEFT JOIN CurrencyCode i
ON i.IsoCode = ISNULL(a.var64_49, '0586')
LEFT JOIN ProuctWithRequestId j
ON a.TransRequestID = j.TransRequestID
WHERE a.var64_42 = ISNULL(@MID, a.var64_42)
--and a.var64_49 =isnull(@CurrencyCode,a.var64_49)
AND
(
-------------For Currency Check
------- For Currency Check Is Not Null
((
--a.var64_49 =@CurrencyCode
i.IsoCode = @CurrencyCode
)
AND (@CurrencyCode IS NOT NULL))
OR
----For All Transactions
((@CurrencyCode IS NULL)
AND (1 = 1))
)
AND (
-------------For InvoiceNo Check
--select * from TerminalSequence
------- For InvoiceNo Check Is Not Null
((a.var64_62 = @InvoiceNo)
AND (@InvoiceNo IS NOT NULL)
AND a.var64_60 = (SELECT
dbo.fn_LPAD(a.BatchNo, 6, '0')
FROM TerminalSequence a
WHERE a.TID IN (SELECT
*
FROM @ReportValues)
AND a.AppCode = @AppCode)
)
OR
----For All Transactions
((@InvoiceNo IS NULL)
)
)
AND (
((j.ProductCode IN (SELECT
*
FROM @ProductCodes)
)
AND (@ProductCodesCount <> 0))
OR ((@ProductCodesCount = 0))
)
AND (
((a.var64_41 IN (SELECT
*
FROM @TIDS)
)
AND (@TIdCount <> 0))
OR ((a.var64_41 = a.var64_41)
AND (@TIdCount = 0))
)
AND (
------- For Gateway Transactions Other Than Moto
((@TranTypeExMotoCount <> 0)
AND (a.TransactionStatusId IN (SELECT
*
FROM @TransactionTypeExMoto)
)
AND (a.TransactionTypeId <> @MotoId-- ( select * from @TransactionTypeMoto )
)
)
OR
------- For Gateway Moto Transaction
((@TranTypeMotoCount <> 0)
AND (a.TransactionTypeId = @MotoId))
--- For All Transactions
OR
((a.TransactionStatusId = a.TransactionStatusId)
AND (@TranTypeCount = 0))
)
AND CONVERT(date, a.TransactionDateTime) BETWEEN ISNULL(CONVERT(date, @StartDate), a.TransactionDateTime)
AND ISNULL(CONVERT(date, @EndDate), a.TransactionDateTime)
AND (
((a.SettlementStatusId IN (SELECT
*
FROM @SettlementStatus)
)
AND (@SettlementStatusCount <> 0))
OR ((a.SettlementStatusId = a.SettlementStatusId)
AND (@SettlementStatusCount = 0))
)
AND a.var64_38 = ISNULL(@AuthID, a.var64_38)
AND a.var64_37 = ISNULL(@RRN, a.var64_37)
--and substring(a.var64_35,1,16) =isnull(@CardNo,substring(a.var64_35,1,16))
AND
(
(
(c.TransactionType IN ('SALE', 'REFUND', 'SETTLEMENT', 'LOYALTY_POINTS_REDEMPTION'))
AND
--( substring(a.var64_35,1,16) =isnull(@CardNo,substring(a.var64_35,1,16)) )
(a.var64_02 = ISNULL(@CardNo, a.var64_02)
)
)
OR ((c.TransactionType = 'MOTO')
AND (a.var64_02 = ISNULL(@CardNo, a.var64_02)))
)
--and a. = isnull(@BatchNo,a.var64_60)
AND
(
-------------For BatchNo Check
------- For BatchNo Check Is Not Null
((a.var64_60 = @BatchNo)
AND (@BatchNo IS NOT NULL))
OR
----For BatchNo Check Is Null
((@BatchNo IS NULL)
AND (1 = 1)))
AND a.var64_39 = '00'
AND c.TransactionType IN ('SALE', 'MOTO', 'PUSH_QR_SALE', 'REFUND', 'SETTLEMENT', 'LOYALTY_POINTS_REDEMPTION')
AND (
-------------For Transactions Based On Order
------- For Is Order 1 Get Only Order Transactions
((@IsOrder = 1)
AND (a.OrderNo IS NOT NULL))
------- For Is Order 0 Get Other Transactions Than Order Transactions
OR
((@IsOrder = 0)
AND (a.OrderNo IS NULL))
OR
----For All Transactions
((@IsOrder IS NULL)
AND (1 = 1))
)
AND
----Start Of First And
(
(
@ReportTypeVar = 'TO_DATE'
AND (
((CONVERT(varchar, a.TransactionDateTime, 106) IN (SELECT
*
FROM @ReportValues)
)
AND (@ReportValCount <> 0))
OR ((CONVERT(varchar, a.TransactionDateTime, 106) = CONVERT(varchar, a.TransactionDateTime, 106))
AND (@ReportValCount = 0))
)
)
OR (
@ReportTypeVar = 'WEEKDAY'
AND (
((DATENAME(WEEKDAY, a.TransactionDateTime) IN (SELECT
*
FROM @ReportValues)
)
AND (@ReportValCount <> 0))
OR ((DATENAME(WEEKDAY, a.TransactionDateTime) = DATENAME(WEEKDAY, a.TransactionDateTime))
AND (@ReportValCount = 0))
)
)
OR (
@ReportTypeVar = 'MONTH'
AND (
((DATENAME(MONTH, a.TransactionDateTime) IN (SELECT
*
FROM @ReportValues)
)
AND (@ReportValCount <> 0))
OR ((DATENAME(MONTH, a.TransactionDateTime) = DATENAME(MONTH, a.TransactionDateTime))
AND (@ReportValCount = 0))
)
)
OR (
@ReportTypeVar = 'QUARTER'
AND (
((DATENAME(QUARTER, a.TransactionDateTime) IN (SELECT
*
FROM @ReportValues)
)
AND (@ReportValCount <> 0))
OR ((DATENAME(QUARTER, a.TransactionDateTime) = DATENAME(QUARTER, a.TransactionDateTime))
AND (@ReportValCount = 0))
)
)
OR (
@ReportTypeVar = 'DayOfMonth'
AND (
((DATENAME(DAY, a.TransactionDateTime) IN (SELECT
*
FROM @ReportValues)
)
AND (@ReportValCount <> 0))
OR ((DATENAME(DAY, a.TransactionDateTime) = DATENAME(DAY, a.TransactionDateTime))
AND (@ReportValCount = 0))
)
)
OR (
@ReportTypeVar = 'TID_WISE'
AND (
((a.var64_41 IN (SELECT
*
FROM @ReportValues)
)
AND (@ReportValCount <> 0))
OR ((a.var64_41 = a.var64_41)
AND (@ReportValCount = 0))
)
)
OR (
@ReportTypeVar = 'BATCHNO'
AND (
((a.var64_60 IN (SELECT
*
FROM @ReportValues)
)
AND (@ReportValCount <> 0))
OR ((1 = 1)
AND (@ReportValCount = 0))
)
)
OR (
@ReportTypeVar = 'ASSOCIATION'
AND (
((f.AssociationName IN (SELECT
*
FROM @ReportValues)
)
AND (@ReportValCount <> 0))
OR ((f.AssociationName = f.AssociationName)
AND (@ReportValCount = 0))
)
)
)
----End Of First And
ORDER BY CASE
WHEN j.ProductName IS NOT NULL THEN j.ProductCode
END ASC,
CASE
WHEN j.ProductName IS NULL THEN a.TransactionDateTime
END DESC, a.TransactionDateTime DESC) x
ORDER BY x.sno ASC
OFFSET ((@PageNo - 1) * @PageSize) ROWS
FETCH NEXT @PageSize ROWS ONLY;
**The result of execution plan is below.**
表“工作台”。扫描计数 3，逻辑读取 55959，物理读取 0，预读读取 1158，lob 逻辑读取 0，lob 物理读取 0，lob 预读读取 0。 表“工作文件”。扫描计数 0，逻辑读取 0，物理读取 0，预读读取 0，lob 逻辑读取 0，lob 物理读取 0，lob 预读读取 0。
表'TransactionRequestLog'。扫描计数 1，逻辑读取 2979，物理读取 0，预读读取 0，lob 逻辑读取 0，lob 物理读取 0，lob 预读读取 0。
表'ProductHierarchy'。扫描计数 1，逻辑读取 1，物理读取 0，预读读取 0，lob 逻辑读取 0，lob 物理读取 0，lob 预读读取 0。
表'POSConditionCode'。扫描计数 1，逻辑读取 19452，物理读取 0，预读读取 0，lob 逻辑读取 0，lob 物理读取 0，lob 预读读取 0。 表“#B1653DF7”。扫描计数 1，逻辑读取 19452，物理读取 0，预读读取 0，lob 逻辑读取 0，lob 物理读取 0，lob 预读读取 0。
表'关联'。扫描计数 1，逻辑读取 19452，物理读取 0，预读读取 0，lob 逻辑读取 0，lob 物理读取 0，lob 预读读取 0。
表'结算状态'。扫描计数 1，逻辑读取 38905，物理读取 0，预读读取 0，lob 逻辑读取 0，lob 物理读取 0，lob 预读读取 0。
表'TransactionStatus'。扫描计数 1，逻辑读取 38905，物理读取 0，预读读取 0，lob 逻辑读取 0，lob 物理读取 0，lob 预读读取 0。
表'商人'。扫描计数 1，逻辑读取 1750680，物理读取 0，预读读取 0，lob 逻辑读取 0，lob 物理读取 0，lob 预读读取 0。 表“#B2596230”。扫描计数 1，逻辑读取 19453，物理读取 0，预读读取 0，lob 逻辑读取 0，lob 物理读取 0，lob 预读读取 0。 表“#A586BBB9”。扫描计数 1，逻辑读取 19454，物理读取 0，预读读取 0，lob 逻辑读取 0，lob 物理读取 0，lob 预读读取 0。 表“工作台”。扫描计数 0，逻辑读取 0，物理读取 0，预读读取 0，lob 逻辑读取 0，lob 物理读取 0，lob 预读读取 0。
表'transactionresponselog'。扫描计数 1，逻辑读取 2110，物理读取 0，预读读取 0，lob 逻辑读取 0，lob 物理读取 0，lob 预读读取 0。
表'GatewayTransactionType'。扫描计数 1，逻辑读取 1，物理读取 0，预读读取 0，lob 逻辑读取 0，lob 物理读取 0，lob 预读读取 0。
表'POSEntryMode'。扫描计数 1，逻辑读取 1，物理读取 0，预读读取 0，lob 逻辑读取 0，lob 物理读取 0，lob 预读读取 0。
表'CurrencyCode'。扫描计数 1，逻辑读取 5，物理读取 0，预读读取 0，lob 逻辑读取 0，lob 物理读取 0，lob 预读读取 0。
除了transactionresponselog和transactionrequestlog之外的所有表都是master表。
【问题讨论】：
请使用 pastetheplan.com，它更方便，并且会提供更多信息供我们调试。另外，您使用的是什么 sql-server 版本？ 在您进行一些主要格式化之前，您甚至无法假装对其进行优化。你怎么能指望任何人拿起 300 行的这种热乱七八糟的东西并理解它？ 您还应该查看this，然后再继续使用痛苦的别名。我实际上试图格式化并放弃了。实在是太草率了。 @SeanLange 我已经格式化了代码，你现在可以查看了.. 格式化的想法是让它清晰易读。把所有东西都排在左边是没有帮助的。我估计要花 2 个小时才能把它格式化成可用的东西。那就是把东西排好，去掉所有多余的空间和东西。删除数十组额外的括号。这与您最初发布的原因所面临的挑战无关。在咨询情况下，我估计这需要大约 20-40 小时来解决这个问题。那就是我手中的数据库。在论坛上这样做太过分了。 【参考方案1】：这是一个包罗万象的查询，有很多关于它的文章，Gail Shaw 已经写了好几次关于它们的文章，就像在她的文章 How to Confuse the SQL Server Query Optimizer 中一样。没有简单的方法来优化这个查询。我的建议是使用动态 SQL 来简化条件。您似乎也只分享了整个过程的一部分。相信动态 SQL 是可行的方法而不是每次都重新编译的原因是查询非常复杂，并且在完全优化之前可能会超时。 你的一些列检查是这样写的：
AND (
-------------For BatchNo Check
------- For BatchNo Check Is Not Null
(
(a.var64_60 = @BatchNo)
AND (@BatchNo IS NOT NULL)
)
OR
----For BatchNo Check Is Null
(
(@BatchNo IS NULL)
AND (1 = 1)
)
)
可以简化成这样：
AND (a.var64_60 = @BatchNo OR @BatchNo IS NULL)
这类似于比较表值参数中的多个值。
AND (j.ProductCode IN ( SELECT * FROM @ProductCodes) OR @ProductCodesCount = 0)
使用动态代码还将简化引擎的@ReportTypeVar 条件。
请注意，即使使用动态 SQL，您也需要能够对查询进行参数化以防止任何类型的 SQL 注入。
【讨论】：
以上是关于优化复杂查询的主要内容，如果未能解决你的问题，请参考以下文章