Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

W&B: Fix for 4360 #4388

Merged
merged 35 commits into from
Aug 11, 2021
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
d0d89b6
Improve docstrings and run names
AyushExel Jul 27, 2021
a283de6
default wandb login prompt with timeout
AyushExel Jul 27, 2021
33ab468
Merge branch 'ultralytics:master' into master
AyushExel Jul 27, 2021
78d1ed0
return key
AyushExel Jul 27, 2021
bba130b
Merge branch 'master' of https://github.com/AyushExel/yolov5
AyushExel Jul 27, 2021
725bd4a
Update api_key check logic
AyushExel Jul 27, 2021
bb372f5
Properly support zipped dataset feature
AyushExel Jul 28, 2021
27c06f0
update docstring
AyushExel Jul 28, 2021
3cafca4
merge master
AyushExel Jul 28, 2021
e64525a
Revert tuorial change
AyushExel Jul 28, 2021
0a67b4a
extend changes to log_dataset
AyushExel Jul 28, 2021
80eb6aa
add run name
AyushExel Jul 28, 2021
3b039be
bug fix
AyushExel Jul 28, 2021
31f04a2
bug fix
AyushExel Jul 28, 2021
2482fb3
Update comment
AyushExel Jul 28, 2021
6012e3f
fix import check
AyushExel Jul 28, 2021
66e36ef
remove unused import
AyushExel Jul 28, 2021
aa58d9e
Hardcore .yaml file extension
AyushExel Jul 28, 2021
7225c19
Merge branch 'ultralytics:master' into fix_dataset_check
AyushExel Jul 28, 2021
dfacf7b
reduce code
glenn-jocher Jul 28, 2021
3593b48
Merge branch 'ultralytics:master' into fix_dataset_check
AyushExel Jul 28, 2021
ab11132
Reformat using pycharm
AyushExel Jul 28, 2021
5d61242
Remove redundant try catch
AyushExel Jul 28, 2021
0a43ca7
More refactoring and bug fixes
AyushExel Jul 28, 2021
4830a54
merge master
AyushExel Jul 28, 2021
570381d
retry
AyushExel Jul 28, 2021
28d7cba
Reformat using pycharm
AyushExel Jul 28, 2021
c72f59f
respect LOGGERS include list
AyushExel Jul 28, 2021
23ef166
Merge branch 'fix_dataset_check' of https://github.com/AyushExel/yolo…
AyushExel Jul 28, 2021
e0e41c9
Merge branch 'ultralytics:master' into fix_dataset_check
AyushExel Jul 28, 2021
2b664dd
Merge branch 'ultralytics:master' into fix_dataset_check
AyushExel Jul 30, 2021
c6c5f3e
Merge branch 'ultralytics:master' into fix_dataset_check
AyushExel Aug 1, 2021
180aed6
Merge branch 'ultralytics:master' into fix_dataset_check
AyushExel Aug 11, 2021
e0d419a
Fix
AyushExel Aug 11, 2021
61b9e1d
fix
AyushExel Aug 11, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Reformat using pycharm
  • Loading branch information
AyushExel committed Jul 28, 2021
commit ab111322a9f9235b7a559779b057b52c182299c9
4 changes: 2 additions & 2 deletions utils/loggers/__init__.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
# YOLOv5 experiment logging utils
import torch
import warnings
from threading import Thread

import torch
from torch.utils.tensorboard import SummaryWriter

from utils.general import colorstr, emojis
Expand All @@ -14,6 +13,7 @@

try:
import wandb

assert hasattr(wandb, '__version__') # verify package import not local dir
except (ImportError, AssertionError):
wandb = None
Expand Down
1 change: 0 additions & 1 deletion utils/loggers/wandb/log_dataset.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import argparse

import yaml

from wandb_utils import WandbLogger
Expand Down
3 changes: 1 addition & 2 deletions utils/loggers/wandb/sweep.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import sys
from pathlib import Path

import wandb
from pathlib import Path

FILE = Path(__file__).absolute()
sys.path.append(FILE.parents[2].as_posix()) # add utils/ to path
Expand Down
21 changes: 10 additions & 11 deletions utils/loggers/wandb/wandb_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,9 @@
import logging
import os
import sys
import yaml
from contextlib import contextmanager
from pathlib import Path

import yaml
from tqdm import tqdm

FILE = Path(__file__).absolute()
Expand Down Expand Up @@ -150,21 +149,21 @@ def __init__(self, opt, run_id, job_type='Training'):
if not opt.resume:
if opt.upload_dataset:
self.wandb_artifact_data_dict = self.check_and_upload_dataset(opt)
elif opt.data.endswith('_wandb.yaml'): # When dataset is W&B artifact

elif opt.data.endswith('_wandb.yaml'): # When dataset is W&B artifact
with open(opt.data, encoding='ascii', errors='ignore') as f:
data_dict = yaml.safe_load(f)
self.data_dict = data_dict
else: # Local .yaml dataset file or .zip file
else: # Local .yaml dataset file or .zip file
self.data_dict = check_dataset(opt.data)

self.setup_training(opt)
# write data_dict to config. useful for resuming from artifacts
if not self.wandb_artifact_data_dict:
self.wandb_artifact_data_dict = self.data_dict
self.wandb_run.config.update({'data_dict': self.wandb_artifact_data_dict},
allow_val_change=True)
allow_val_change=True)

if self.job_type == 'Dataset Creation':
self.data_dict = self.check_and_upload_dataset(opt)

Expand All @@ -179,7 +178,7 @@ def check_and_upload_dataset(self, opt):
Updated dataset info dictionary where local dataset paths are replaced by WAND_ARFACT_PREFIX links.
"""
assert wandb, 'Install wandb to upload dataset'
config_path = self.log_dataset_artifact(opt.data ,
config_path = self.log_dataset_artifact(opt.data,
opt.single_cls,
'YOLOv5' if opt.project == 'runs/train' else Path(opt.project).stem)
print("Created dataset config file ", config_path)
Expand Down Expand Up @@ -216,7 +215,7 @@ def setup_training(self, opt):
opt.artifact_alias)
self.val_artifact_path, self.val_artifact = self.download_dataset_artifact(data_dict.get('val'),
opt.artifact_alias)

if self.train_artifact_path is not None:
train_path = Path(self.train_artifact_path) / 'data/images/'
data_dict['train'] = str(train_path)
Expand Down Expand Up @@ -326,7 +325,7 @@ def log_dataset_artifact(self, data_file, single_cls, project, overwrite_config=
if data.get('val'):
data['val'] = WANDB_ARTIFACT_PREFIX + str(Path(project) / 'val')
path = Path(data_file).stem
path = (path if overwrite_config else path + '_wandb') + '.yaml' # updated data.yaml path
path = (path if overwrite_config else path + '_wandb') + '.yaml' # updated data.yaml path
data.pop('download', None)
data.pop('path', None)
with open(path, 'w') as f:
Expand Down