You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
38 lines
945 B
Python
38 lines
945 B
Python
def dedup_list(data, check_keys):
|
|
"""
|
|
Remove duplicate dictionaries from list.
|
|
"""
|
|
seen = set()
|
|
out = []
|
|
|
|
dup_count = 0
|
|
for x in data:
|
|
dedupeKey = tuple(x[k] for k in check_keys if k in x)
|
|
if dedupeKey in seen:
|
|
dup_count += 1
|
|
continue
|
|
if dup_count > 0:
|
|
out.append({"type": "control", "hidden": dup_count})
|
|
dup_count = 0
|
|
out.append(x)
|
|
seen.add(dedupeKey)
|
|
if dup_count > 0:
|
|
out.append({"type": "control", "hidden": dup_count})
|
|
return out
|
|
|
|
|
|
# from random import randint
|
|
# from timeit import timeit
|
|
# entries = 10000
|
|
# a = [
|
|
# {'ts': "sss", 'msg': randint(1, 2), str(randint(1, 2)): \
|
|
# randint(1, 2)} for x in range(entries)
|
|
# ]
|
|
# kk = ["msg", "nick"]
|
|
# call = lambda: dedup_list(a, kk)
|
|
# #print(timeit(call, number=10))
|
|
# print(dedup_list(a, kk))
|
|
|
|
# # sh-5.1$ python helpers.py
|
|
# # 1.0805372429895215
|