如何在熊猫数据帧上优化带有条件的双循环？

Question

我有这两个数据框：

df = pd.DataFrame({'Points':[0,1,2,3],'Axis1':[1,2,2,3], 'Axis2':[4,2,3,0],'ClusterId':[1,2,2,3]})
df
   Points  Axis1  Axis2  ClusterId
0       0      1      4          1
1       1      2      2          2
2       2      2      3          2
3       3      3      0          3

Neighbour = pd.DataFrame()
Neighbour['Points'] = df['Points']
Neighbour['Closest'] = np.nan
Neighbour['Distance'] = np.nan

Neighbour
   Points  Closest  Distance
0       0      NaN       NaN
1       1      NaN       NaN
2       2      NaN       NaN
3       3      NaN       NaN

我希望Closest 列包含不在同一个集群中的最近点（df 中的 ClusterId），基于以下距离函数，应用于 Axis1 和 Axis2 ：

def distance(x1,y1,x2,y2):
    dist = sqrt((x1-x2)**2 + (y1-y2)**2)
    return dist 

我希望距离列包含点与其最近点之间的距离。

以下脚本有效，但我认为这确实不是在 Python 中执行的最佳方法：

for i in range(len(Neighbour['Points'])): 
    bestD = -1 #best distance
    #bestP for best point
    for ii in range(len(Neighbour['Points'])): 
        if df.loc[i,"ClusterId"] != df.loc[ii,"ClusterId"]: #if not share the same cluster
            dist = distance(df.iloc[i,1],df.iloc[i,2],df.iloc[ii,1],df.iloc[ii,2])
            if dist < bestD or bestD == -1:
                bestD = dist
                bestP = Neighbour.iloc[ii,0]
    Neighbour.loc[i,'Closest'] = bestP
    Neighbour.loc[i,'Distance'] = bestD

Neighbour
   Points  Closest  Distance
0       0      2.0  1.414214
1       1      0.0  2.236068
2       2      0.0  1.414214
3       3      1.0  2.236068

是否有更有效的方法来填充 Closest 和 Distance 列（尤其是在没有 for 循环的情况下）？ 使用 map 和 reduce 可能是一个合适的场合，但我真的不知道如何。

Answer 1

要计算距离，您可以在scipy.spatial.distance.cdist的底层 ndarray 上使用 scipy.spatial.distance.cdist。 这可能比您的双循环更快。

>>> import numpy as np
>>> from scipy.spatial.distance import cdist

>>> distance_matrix = cdist(df.values[:, 1:3], df.values[:, 1:3], 'euclidean')
>>> distance_matrix
array([[0.        , 2.23606798, 1.41421356, 4.47213595],
       [2.23606798, 0.        , 1.        , 2.23606798],
       [1.41421356, 1.        , 0.        , 3.16227766],
       [4.47213595, 2.23606798, 3.16227766, 0.        ]])
>>> np.fill_diagonal(distance_matrix, np.inf) # set diagonal to inf so minimum isn't distance(x, x) = 0
>>> distance_matrix
array([[       inf, 2.23606798, 1.41421356, 4.47213595],
       [2.23606798,        inf, 1.        , 2.23606798],
       [1.41421356, 1.        ,        inf, 3.16227766],
       [4.47213595, 2.23606798, 3.16227766,        inf]])

为了加快速度，您还可以检查pdist函数而不是 cdist，当您有 50_000 行时，它占用的内存更少。
还有KDTree旨在找到一个点的最近邻居。

然后你可以使用np.argmin来获得最近的距离，并检查最近的点是否在集群中，像这样（虽然我没有尝试过）：

for i in range(len(Neighbour['Points'])):
    same_cluster = True
    while same_cluster:
        index_min = np.argmin(distance_matrix[i])
        same_cluster = (df.loc[i,"ClusterId"] == df.loc[index_min,"ClusterId"])
        if same_cluster:
            distance_matrix[i][index_min] = np.inf
    Neighbour.loc[i,'Closest'] = index_min
    Neighbour.loc[i,'Distance'] = distance_matrix[i][index_min]

Answer 2

要完成@politinsa 的回答，以下脚本允许比较两种方法的性能：

from sklearn.datasets import make_moons
from sklearn.utils import check_random_state
import numpy as np
import timeit
import pandas as pd
from math import sqrt
from scipy.spatial.distance import cdist

def distance(x1,y1,x2,y2):
    dist = sqrt((x1-x2)**2 + (y1-y2)**2)
    return dist 

X,y = make_moons(n_samples=1000, noise=0.1)
W = list(range(1000))
rs = check_random_state(0)
Z = rs.randint(0, 10, size=(1000,))
df = pd.DataFrame(dict(Points=W, Axis1=X[:,0], Axis2=X[:,1],ClusterId=Z))
Neighbour = pd.DataFrame()
Neighbour['Points'] = df['Points']
Neighbour['Closest'] = np.nan
Neighbour['Distance'] = np.nan

start = timeit.default_timer()

for i in range(len(Neighbour['Points'])): 
    bestD = -1 #best distance
    for ii in range(len(Neighbour['Points'])): 
        if df.loc[i,"ClusterId"] != df.loc[ii,"ClusterId"]: #if not share the same cluster
            dist = distance(df.iloc[i,1],df.iloc[i,2],df.iloc[ii,1],df.iloc[ii,2])
            if dist < bestD or bestD == -1:
                bestD = dist
                bestP = Neighbour.iloc[ii,0]
    Neighbour.loc[i,'Closest'] = int(bestP)
    Neighbour.loc[i,'Distance'] = bestD

stop = timeit.default_timer()
print('Time initial script: ', stop - start)

start = timeit.default_timer()

distance_matrix = cdist(df.values[:, 1:3], df.values[:, 1:3])
np.fill_diagonal(distance_matrix, np.inf) # set diagonal to inf so minimum isn't distance(x, x) = 0
for i in range(len(Neighbour['Points'])):
    same_cluster = True
    while same_cluster:
        index_min = np.argmin(distance_matrix[i])
        same_cluster = (df.loc[i,"ClusterId"] == df.loc[index_min,"ClusterId"])
        if same_cluster:
            distance_matrix[i][index_min] = np.inf
    Neighbour.loc[i,'Closest'] = index_min
    Neighbour.loc[i,'Distance'] = distance_matrix[i][index_min]
stop = timeit.default_timer()
print('Time @politinsa\'s script: ', stop - start) 

输出（以秒为单位）：

Time initial script:  70.62462342600003
Time @politinsa's script:  0.6489833670000235

Answer 3

您可以先创建笛卡尔积，然后使用以下距离函数相应地应用新列作为距离

def distance(row):
    x1 = row.Axis1_x
    y1 = row.Axis2_x
    x2 = row.Axis1_y
    y2 = row.Axis2_y
    dist = math.sqrt((x1-x2)**2 + (y1-y2)**2)
    return dist


df = pd.DataFrame({'Points':[0,1,2,3],'Axis1':[1,2,2,3], 'Axis2':[4,2,3,0],'ClusterId':[1,2,2,3]})
df['join_key'] = '12345'
df = df.merge(df, how='outer', on='join_key')
df['distance'] = df.apply(distance, axis=1)
df = df.drop(columns=['join_key'])

所以你会看到像下面这样的笛卡尔 df

从现在开始，您将看到每个点到每个点的距离。 我认为最难的部分是这个。 如果有帮助，请告诉我。