-
Notifications
You must be signed in to change notification settings - Fork 9
/
scirepeval_tasks.jsonl
22 lines (22 loc) · 5.37 KB
/
scirepeval_tasks.jsonl
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
{"name":"Biomimicry","type":"classification","data":{"meta":{"name":"allenai/scirepeval","config":"biomimicry"},"test":{"name":"allenai/scirepeval_test","config":"biomimicry"}},"metrics":["f1"],"few_shot":[{"sample_size":64,"iterations":50},{"sample_size":16,"iterations":100}]}
{"name":"DRSM","type":"classification","data":{"meta":{"name":"allenai/scirepeval","config":"drsm"},"test":{"name":"allenai/scirepeval_test","config":"drsm"}},"metrics":["f1_macro"],"few_shot":[{"sample_size":64,"iterations":50},{"sample_size":24,"iterations":100}]}
{"name":"RELISH","type":"proximity","data":{"meta":{"name":"allenai/scirepeval","config":"relish"},"test":{"name":"allenai/scirepeval_test","config":"relish"}},"metrics":["ndcg"]}
{"name":"NFCorpus","type":"adhoc_search","data":{"meta":{"name":"allenai/scirepeval","config":"nfcorpus"},"test":{"name":"allenai/scirepeval_test","config":"nfcorpus"}},"metrics":["ndcg"]}
{"name":"TREC-CoVID","type":"adhoc_search","data":{"meta":{"name":"allenai/scirepeval","config":"trec_covid"},"test":{"name":"allenai/scirepeval_test","config":"trec_covid"}},"metrics":["ndcg"]}
{"name":"Paper-Reviewer Matching","type":"proximity","data":{"meta":{"name":"allenai/scirepeval","config":"paper_reviewer_matching"},"test":{"name":"allenai/scirepeval_test","config":"paper_reviewer_matching"},"reviewers":{"name":"allenai/scirepeval_test","config":"reviewers"}},"metrics":["P_5", "P_10"]}
{"name":"Peer Review Score","type":"regression","data":{"meta":{"name":"allenai/scirepeval","config":"peer_review_score_hIndex"},"test":{"name":"allenai/scirepeval_test","config":"peer_review_score"}},"embeddings":{"save":"embeddings/peer_review_score_hIndex.jsonl"},"metrics":["kendalltau"]}
{"name":"Max hIndex","type":"regression","data":{"meta":{"name":"allenai/scirepeval","config":"peer_review_score_hIndex"},"test":{"name":"allenai/scirepeval_test","config":"hIndex"}},"embeddings":{"load":"embeddings/peer_review_score_hIndex.jsonl"},"metrics":["kendalltau"]}
{"name":"Tweet Mentions","type":"regression","data":{"meta":{"name":"allenai/scirepeval","config":"tweet_mentions"},"test":{"name":"allenai/scirepeval_test","config":"tweet_mentions"}},"metrics":["kendalltau"]}
{"name":"SciDocs MAG","type":"classification","data":{"meta":{"name":"allenai/scirepeval","config":"scidocs_mag_mesh"},"test":{"name":"allenai/scirepeval_test","config":"scidocs_mag"}},"embeddings":{"save":"embeddings/scidocs_mag_mesh.jsonl"},"metrics":["f1_macro"]}
{"name":"SciDocs MeSH","type":"classification","data":{"meta":{"name":"allenai/scirepeval","config":"scidocs_mag_mesh"},"test":{"name":"allenai/scirepeval_test","config":"scidocs_mesh"}},"embeddings":{"load":"embeddings/scidocs_mag_mesh.jsonl"},"metrics":["f1_macro"]}
{"name":"SciDocs Cite","type":"proximity","data":{"simple_format":true, "meta":{"name":"allenai/scirepeval","config":"scidocs_view_cite_read"},"test":{"name":"allenai/scirepeval_test","config":"scidocs_cite"}},"embeddings":{"save":"embeddings/scidocs_view_cite_read.jsonl"},"metrics":["map","ndcg"]}
{"name":"SciDocs CoView","type":"proximity","data":{"simple_format":true, "meta":{"name":"allenai/scirepeval","config":"scidocs_view_cite_read"},"test":{"name":"allenai/scirepeval_test","config":"scidocs_view"}},"embeddings":{"load":"embeddings/scidocs_view_cite_read.jsonl"},"metrics":["map","ndcg"]}
{"name":"SciDocs CoCite","type":"proximity","data":{"simple_format":true, "meta":{"name":"allenai/scirepeval","config":"scidocs_view_cite_read"},"test":{"name":"allenai/scirepeval_test","config":"scidocs_cocite"}},"embeddings":{"load":"embeddings/scidocs_view_cite_read.jsonl"},"metrics":["map","ndcg"]}
{"name":"SciDocs CoRead","type":"proximity","data":{"simple_format":true, "meta":{"name":"allenai/scirepeval","config":"scidocs_view_cite_read"},"test":{"name":"allenai/scirepeval_test","config":"scidocs_read"}},"embeddings":{"load":"embeddings/scidocs_view_cite_read.jsonl"},"metrics":["map","ndcg"]}
{"name":"Same Author Detection","type":"proximity","data":{"meta":{"name":"allenai/scirepeval","config":"same_author"},"test":{"name":"allenai/scirepeval_test","config":"same_author"}},"metrics":["map"]}
{"name":"Highly Influential Citations","type":"proximity","data":{"meta":{"name":"allenai/scirepeval","config":"high_influence_cite"},"test":{"name":"allenai/scirepeval_test","config":"high_influence_cite"}},"metrics":["map"]}
{"name":"Search","type":"adhoc_search","data":{"meta":{"name":"allenai/scirepeval","config":"search"},"test":{"name":"allenai/scirepeval_test","config":"search"}},"fields":["title","abstract","venue","year"],"metrics":["ndcg"]}
{"name":"Citation Count","type":"regression","data":{"meta":{"name":"allenai/scirepeval","config":"cite_count"},"test":{"name":"allenai/scirepeval_test","config":"cite_count"}},"metrics":["kendalltau"]}
{"name":"Publication Year","type":"regression","data":{"meta":{"name":"allenai/scirepeval","config":"pub_year"},"test":{"name":"allenai/scirepeval_test","config":"pub_year"}},"metrics":["kendalltau"]}
{"name":"Fields of study","type":"classification","data":{"meta":{"name":"allenai/scirepeval","config":"fos"},"test":{"name":"allenai/scirepeval_test","config":"fos"}},"metrics":["f1_macro"],"few_shot":[{"sample_size":10,"iterations":50},{"sample_size":5,"iterations":100}],"multi_label":true}
{"name":"MeSH","type":"classification","data":{"meta":{"name":"allenai/scirepeval","config":"mesh_descriptors"},"test":{"name":"allenai/scirepeval_test","config":"mesh_descriptors"}},"metrics":["f1_macro"]}