AI/CodeT5
1
0
Fork 0
mirror of https://github.com/salesforce/CodeT5.git synced 2024-10-01 06:35:38 -04:00
Code Issues Packages Projects Releases Wiki Activity

Update utils.py

Browse Source
This commit is contained in:
WANG Yue 2021-11-12 11:29:17 +08:00 committed by GitHub
parent d27512d23b
commit 5bb41e21b0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 1 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
utils.py
View File

@ -64,7 +64,6 @@ def load_and_cache_clone_data(args, filename, pool, tokenizer, split_tag, is_sam
logger.info("Create cache data into %s", cache_fn)
tuple_examples = [(example, idx, tokenizer, args) for idx, example in enumerate(examples)]
features = pool.map(convert_clone_examples_to_features, tqdm(tuple_examples, total=len(tuple_examples)))
# features = [convert_clone_examples_to_features(x) for x in tuple_examples]
all_source_ids = torch.tensor([f.source_ids for f in features], dtype=torch.long)
all_labels = torch.tensor([f.label for f in features], dtype=torch.long)
data = TensorDataset(all_source_ids, all_labels)
@ -149,7 +148,7 @@ def get_filenames(data_root, task, sub_task, split=''):
def read_examples(filename, data_num, task):
read_example_dict = {
'summarize': read_summarize_examples, # read_summarize_examples， read_summarize_indent_examples
'summarize': read_summarize_examples,
'refine': read_refine_examples,
'translate': read_translate_examples,
'concode': read_concode_examples,