xxxxxxxxxx
import pandas as pd
# Drop all duplicates in the DataFrame
df = df.drop_duplicates()
# Drop all duplicates in a specific column of the DataFrame
df = df.drop_duplicates(subset = "column")
# Drop all duplicate pairs in DataFrame
df = df.drop_duplicates(subset = ["column", "column2"])
# Display DataFrame
print(df)
xxxxxxxxxx
# Remove by index
df = df[df.index.duplicated(keep='first')]
# Other methods to remove duplicates
import pandas as pd
df = df.drop_duplicates()
df = df.drop_duplicates(subset = "column")
df = df.drop_duplicates(subset = ["column", "column2"])
xxxxxxxxxx
idx = pd.Index(['lama', 'cow', 'lama', 'beetle', 'lama', 'hippo'])
idx.drop_duplicates(keep='first')
Index(['lama', 'cow', 'beetle', 'hippo'], dtype='object')
idx.drop_duplicates(keep='last')
Index(['cow', 'beetle','lamb', 'hippo'], dtype='object')
idx.drop_duplicates(keep='False')
Index(['cow', 'beetle','hippo'], dtype='object')
xxxxxxxxxx
# Most optimized way to remove duplicates with pandas
import pandas as pd
# Remove all duplicates from the DataFrame
df.drop_duplicates(inplace=True)
# Remove duplicates based on a specific column
df.drop_duplicates(subset="column", inplace=True)
# Remove duplicates based on multiple columns
df.drop_duplicates(subset=["column", "column2"], inplace=True)
# Display DataFrame
print(df)