Stephan Heunis
484b6741cd
This script only works for a particular issue in projects. It does not generalize. It also asks for much improvement. Not sure if this commit should remain in the history, but committing the script nonetheless for the provenance.
60 lines
1.6 KiB
Python
60 lines
1.6 KiB
Python
import json
|
|
from pathlib import Path
|
|
|
|
# Helpers
|
|
debug = False
|
|
root_path = Path(__file__).resolve().parent.parent
|
|
|
|
def dict_in_list(mydict, mylist):
|
|
""""""
|
|
for l in mylist:
|
|
if is_equal(l, mydict):
|
|
return True
|
|
return False
|
|
|
|
def is_equal(dict1, dict2):
|
|
if not set(dict1.keys()) == set(dict2.keys()):
|
|
return False
|
|
|
|
if 'roles' not in dict1:
|
|
if dict1['object'] == dict2['object']:
|
|
return True
|
|
else:
|
|
return False
|
|
else:
|
|
if set(dict1['roles']) == set(dict2['roles']) and dict1['object'] == dict2['object']:
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
|
|
# Script starts
|
|
projects = {}
|
|
with open(Path(root_path / 'records' / 'TRR379Project-local.jsonl')) as fobj:
|
|
for line in fobj:
|
|
rec = json.loads(line)
|
|
projects[rec['pid']] = rec
|
|
|
|
new_projects = projects
|
|
|
|
for p in projects.values():
|
|
if debug:
|
|
print(f'\nCurrent project: {p["pid"]}')
|
|
attributed_to = []
|
|
if debug:
|
|
print(f'- nr of contributions: {len(p['attributed_to'])}')
|
|
for thing in p['attributed_to']:
|
|
if dict_in_list(thing, attributed_to):
|
|
if debug:
|
|
print('- found duplicate:')
|
|
print(thing)
|
|
else:
|
|
attributed_to.append(thing)
|
|
if debug:
|
|
print(f'- nr of deduplicated contributions: {len(attributed_to)}')
|
|
print(f'- deduplicated contributions: {attributed_to}')
|
|
|
|
new_projects[p['pid']]['attributed_to'] = attributed_to
|
|
|
|
for p in new_projects:
|
|
print(json.dumps(new_projects[p], ensure_ascii=False)) |