SQL query with “not in” is very slow

Question

I have the following SQL query which is very slow. How can I write the script differently?

select 
    pws_name 
from 
    pws_asset ass
join 
    Account acc on acc.AccountId = ass.pws_AccountId
where 
    acc.AccountNumber  in ('188012', '172146', '214727', '13636', '201194', '280294', '34328')
    and ass.pws_name not in ('1018684', '1018784', '1019584', '1019784', '1019884', '1070838', '1277139', '1277339'.........)

Answer 1

Kindly follow below steps, that will help in query performance.

Step 1: DECLARE two variables

DECLARE @AccNumList VARCHAR(4000)
DECLARE @PwsNameList VARCHAR(4000)

SET @AccNumList = '188012,172146,214727,13636,201194,280294,34328'
SET @PwsNameList = '1018684,1018784,1019584,1019784,1019884,1070838,1277139,1277339'

Step 2: Create two different temporary tables.
1 for account numbers

Create table #tblAcNum(AccountNumber VARCHAR(50))

2 for pws_name not needed

Create table #tblPwsNameNotNeeded(pws_name VARCHAR(50))

Step 3: Add records in above two tables which are used in IN and NOT IN.
Please check this Split csv string using XML in SQL Server for reference.

INSERT INTO #tblAcNum(AccountNumber)
SELECT
l.value('.','VARCHAR(50)') AcNum
FROM
(
SELECT CAST('<a>' + REPLACE(@AccNumList,',','</a><a>') + '</a>' AS XML) AcNumXML
) x
CROSS APPLY x.AcNumXML.nodes('a') Split(l)

INSERT INTO #tblPwsNameNotNeeded(pws_name)
SELECT
l.value('.','VARCHAR(50)') pws_name
FROM
(
SELECT CAST('<a>' + REPLACE(@PwsNameList,',','</a><a>') + '</a>' AS XML) PwsNameXML
) x
CROSS APPLY x.PwsNameXML.nodes('a') Split(l)

Step 3: INNER JOIN #tblAcNum table with account table with accountnumber column

Step 4: Use NOT EXISTS() function for pws_name not needed like below

WHERE NOT EXISTS
(
SELECT 1
FROM #tblPwsNameNotNeeded pn
Where pn.pws_name = ass.pws_name
)

Step 5: Drop temporary tables after your select query.

DROP TABLE tblAcNum;
DROP TABLE #tblPwsNameNotNeeded;

Please check below query.

DECLARE @AccNumList VARCHAR(4000)
DECLARE @PwsNameList VARCHAR(4000)

SET @AccNumList = '188012,172146,214727,13636,201194,280294,34328'
SET @PwsNameList = '1018684,1018784,1019584,1019784,1019884,1070838,1277139,1277339'

Create table #tblAcNum(AccountNumber VARCHAR(50))

Create table #tblPwsNameNotNeeded(pws_name VARCHAR(50))

INSERT INTO #tblAcNum(AccountNumber)
SELECT
l.value('.','VARCHAR(50)') AcNum
FROM
(
SELECT CAST('<a>' + REPLACE(@AccNumList,',','</a><a>') + '</a>' AS XML) AcNumXML
) x
CROSS APPLY x.AcNumXML.nodes('a') Split(l)

INSERT INTO #tblPwsNameNotNeeded(pws_name)
SELECT
l.value('.','VARCHAR(50)') pws_name
FROM
(
SELECT CAST('<a>' + REPLACE(@PwsNameList,',','</a><a>') + '</a>' AS XML) PwsNameXML
) x
CROSS APPLY x.PwsNameXML.nodes('a') Split(l)

select 
ass.pws_name 
from pws_asset ass 
join Account acc on acc.AccountId = ass.pws_AccountId 
INNER JOIN #tblAcNum an ON an.AccountNumber = acc.AccountNumber
WHERE NOT EXISTS
(
SELECT 1
FROM #tblPwsNameNotNeeded pn
Where pn.pws_name = ass.pws_name
)

DROP TABLE tblAcNum;
DROP TABLE #tblPwsNameNotNeeded;

Answer 2

Try this:

; with cte_excludepws as
(select AccountId from pws_asset where pws_name not in ('1018684', '1018784', '1019584', '1019784', '1019884', '1070838', '1277139', '1277339'.........))
select 
    pws_name 
from 
    pws_asset ass
join 
    Account acc on acc.AccountId = ass.pws_AccountId
where ass.AccountId not in (select AccountId from cte_excludepws)
and acc.AccountNumber  in ('188012', '172146', '214727', '13636', '201194', '280294', '34328')

Alternatively if you can - take the AccountID's into a temporary table instead of cte and create an index on it.

Answer 3

First, be sure that the account numbers are really strings. If they are numbers, drop the single quotes!

Then, for this query

select a.pws_name 
from pws_asset a join 
     Account ac 
     on ac.AccountId = a.pws_AccountId
where ac.AccountNumber  in ('188012', '172146', '214727', '13636', '201194', '280294', '34328') and
      a.pws_name not in ('1018684', '1018784', '1019584', '1019784', '1019884', '1070838', '1277139', '1277339'.........);

I would recommend indexes on: account(accountNumber, AccountId) and pws_asset(pws_AccountId, pws_name) .