I'm getting this error using PyTorch ddp for tensorboard's add_scalars (add scalar works fine). Is there something I can do?
-- Process 1 terminated with the following error:
Traceback (most recent call last):
File "/scratch/staff/brm512/anaconda3/envs/ln1/lib/python3.7/site-packages/torch/multiprocessing/spawn.py", line 19, in _wrap
fn(i, *args)
File "/scratch/staff/brm512/anaconda3/envs/ln1/lib/python3.7/site-packages/pytorch_lightning/trainer/ddp_mixin.py", line 181, in ddp_train
self.run_pretrain_routine(model)
File "/scratch/staff/brm512/anaconda3/envs/ln1/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 471, in run_pretrain_routine
self.train()
File "/scratch/staff/brm512/anaconda3/envs/ln1/lib/python3.7/site-packages/pytorch_lightning/trainer/train_loop_mixin.py", line 60, in train
self.run_training_epoch()
File "/scratch/staff/brm512/anaconda3/envs/ln1/lib/python3.7/site-packages/pytorch_lightning/trainer/train_loop_mixin.py", line 114, in run_training_epoch
self.run_evaluation(test=self.testing)
File "/scratch/staff/brm512/anaconda3/envs/ln1/lib/python3.7/site-packages/pytorch_lightning/trainer/evaluation_loop_mixin.py", line 130, in run_evaluation
test)
File "/scratch/staff/brm512/anaconda3/envs/ln1/lib/python3.7/site-packages/pytorch_lightning/trainer/evaluation_loop_mixin.py", line 74, in evaluate
eval_results = model.validation_end(outputs)
File "/home/userfs/b/brm512/experiments/HomographyNet/lightning_module.py", line 547, in validation_end
self.logger.experiment.add_scalars('losses', {'train loss': self.loss_meter_training.avg, 'val loss':self.loss_meter_validation.avg} , self.epoch_nb)
File "/scratch/staff/brm512/anaconda3/envs/ln1/lib/python3.7/site-packages/torch/utils/tensorboard/writer.py", line 363, in add_scalars
fw_logdir = self._get_file_writer().get_logdir()
AttributeError: 'TTDummyFileWriter' object has no attribute 'get_logdir'
I'm getting this error using PyTorch ddp for tensorboard's add_scalars (add scalar works fine). Is there something I can do?
-- Process 1 terminated with the following error:
Traceback (most recent call last):
File "/scratch/staff/brm512/anaconda3/envs/ln1/lib/python3.7/site-packages/torch/multiprocessing/spawn.py", line 19, in _wrap
fn(i, *args)
File "/scratch/staff/brm512/anaconda3/envs/ln1/lib/python3.7/site-packages/pytorch_lightning/trainer/ddp_mixin.py", line 181, in ddp_train
self.run_pretrain_routine(model)
File "/scratch/staff/brm512/anaconda3/envs/ln1/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 471, in run_pretrain_routine
self.train()
File "/scratch/staff/brm512/anaconda3/envs/ln1/lib/python3.7/site-packages/pytorch_lightning/trainer/train_loop_mixin.py", line 60, in train
self.run_training_epoch()
File "/scratch/staff/brm512/anaconda3/envs/ln1/lib/python3.7/site-packages/pytorch_lightning/trainer/train_loop_mixin.py", line 114, in run_training_epoch
self.run_evaluation(test=self.testing)
File "/scratch/staff/brm512/anaconda3/envs/ln1/lib/python3.7/site-packages/pytorch_lightning/trainer/evaluation_loop_mixin.py", line 130, in run_evaluation
test)
File "/scratch/staff/brm512/anaconda3/envs/ln1/lib/python3.7/site-packages/pytorch_lightning/trainer/evaluation_loop_mixin.py", line 74, in evaluate
eval_results = model.validation_end(outputs)
File "/home/userfs/b/brm512/experiments/HomographyNet/lightning_module.py", line 547, in validation_end
self.logger.experiment.add_scalars('losses', {'train loss': self.loss_meter_training.avg, 'val loss':self.loss_meter_validation.avg} , self.epoch_nb)
File "/scratch/staff/brm512/anaconda3/envs/ln1/lib/python3.7/site-packages/torch/utils/tensorboard/writer.py", line 363, in add_scalars
fw_logdir = self._get_file_writer().get_logdir()
AttributeError: 'TTDummyFileWriter' object has no attribute 'get_logdir'