异常值检测并在完整的 dataframe 中替换它们

Question

def outliers(column, creditCardData):

creditCardData[column].describe()


zscore = (creditCardData[column] -
creditCardData[column].mean())/creditCardData[column].std()
no_of_out = sum(zscore > 3)
print('No of outliers: ', no_of_out)

upper_f = creditCardData[column].mean() + 3*creditCardData[column].std()
lower_f = creditCardData[column].mean() - 3*creditCardData[column].std()

no_of_out_up = sum(creditCardData[column]>upper_f)
no_of_out_lo = sum(creditCardData[column]<lower_f)

print('Removing outliers____________')

creditCardData[column][creditCardData[column]>upper_f] = upper_f
creditCardData[column][creditCardData[column]<lower_f] = lower_f

no_of_out_up = sum(creditCardData[column]>upper)
no_of_out_lo = sum(creditCardData[column]<lower)

print('Null values: ', creditCardData[column].isnull().sum())


outliers('PURCHASES', creditCardData)




outliers('ONEOFF_PURCHASES',creditCardData)
No of outliers:  422
Removing outliers____________
Null values:  0
<ipython-input-137-83ef36d41cf4>:15: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame

The outliers are not replacing. can anyone help me on this? built a function for detecting outliers in features using z score method and trying to fix the issue by replacing the outliers with upper limit. I couldn't achieve the output for this function. So, could you help me in this? and for z score detection threshold for upper limit is 3 and -3 for lower limit. Built on Dataset CreditCardData Help me or guide me through the problem i am facing here!

Answer 1

def Outliers(col_name, creditCardData):

mean = creditCardData[col_name].mean()
std = creditCardData[col_name].std()

upper = mean + 3 * std
lower = mean - 3 * std

print('Upper bound: ', upper)
print('Lower bound: ', lower, '\n')

no_of_out_up = sum(creditCardData[col_name]>upper)
no_of_out_lo = sum(creditCardData[col_name]<lower)

print('No of outliers above upperbound: ', no_of_out_up)
print('No of outliers below lowerbound: ', no_of_out_lo, '\n')

print('Removing outliers____________\n')
creditCardData[col_name][creditCardData[col_name]>upper] = upper
creditCardData[col_name][creditCardData[col_name]<lower] = lower

no_of_out_up = sum(creditCardData[col_name]>upper)
no_of_out_lo = sum(creditCardData[col_name]<lower)

print('No of outliers above upperbound: ', no_of_out_up)
print('No of outliers below lowerbound: ', no_of_out_lo, '\n')