Pandas automatic JOIN between two pandas dataframe

Question

Anyone knows how to find potential join between two pandas dataframe?

eg. I want to search for potential match between columns of these datasets

I have developed something like that

from tqdm import tqdm
import numpy as np

def _check_na(df, col, threshold=0.9):
    return sum(df[col].isna()) > (1- threshold) * df.shape[0]

def _check_dtype(df1, df2, c1, c2):
    
    try:
        df1[c1] = df1[c1].astype(df2.dtypes[c2])
        return True
    except:
        return False

def _fast_intersect(df1, df2, c1, c2, perc=0.2, maxrows=1000):
    n = min(df1.shape[0], df2.shape[0])
    
    c=1
    if n>maxrows:
        c = len(np.intersect1d(df1[c1].sample(n= maxrows, random_state=0).astype(str).values, df2[c2].astype(str)) )
    
    if (c>0) or n<=maxrows:
            return len(np.intersect1d(df1[c1].astype(str).values, df2[c2].astype(str).values))
    
    return 0

def possible_join(df1, df2, threshold=0.8, verbose=0):
    
    n = min(df1.shape[0], df2.shape[0])
    
    for c1 in tqdm(df1.columns):
        
        if _check_na(df1, c1):
            if verbose>1:
                print('WARN: too many na',c1)
            continue
        
        for c2 in df2.columns:
            

            if not(_check_dtype(df1, df2, c1, c2)):
                if verbose>1:
                    print('WARN: incompatible type',c1, df1.dtypes[c1],c2, df2.dtypes[c2])
                continue
            
            if _check_na(df2, c2):
                if verbose>1:
                    print('WARN: too many na',c2)
                continue
                
            c = _fast_intersect(df1, df2, c1, c2)
            print(c)

            if (c>= threshold * n):
                print ('** MATCH **:', c1,'-',c2,':', c, c/n, '%')
            elif (c>= threshold * n /2):
                print ('** LOW MATCH **:',c1,'-',c2,':', c, c/n, '%')
                

_check_dtype check for potentila match between data types

_check_na excludes columns with a lot of NaN

_fast_intersect tries a fast join with only few data, if it passes look for full join.

but is is too slow!

test

d = {'col11': [1, 2], 'col21': ['3', '4']}
df1 = pd.DataFrame(data=d)

d = {'col12': [1, 3], 'col22': [3, 4]}
df2 = pd.DataFrame(data=d)

possible_join(df1, df2, threshold=0.6, verbose=2)

the result is

col21 and col22 with 100% full match

col11 and col12 with 50% low match

Answer 1

use this peace of code in _fast_intersect :

first_df = df1.sample(n= int(n * perc), random_state=0)
second_df = df2
return pd.merge(first_df, second_df, how="inner", on="c2").shape[0]