From c7e1483e01b9edad97d1294ccfcf8ade197c497b Mon Sep 17 00:00:00 2001
From: Yandi <yandirzm@gmail.com>
Date: Sun, 22 Jan 2023 00:25:51 +0100
Subject: [PATCH] Viewing vanishing gradients
---
losses.py | 3 ++-
main.py | 17 +++++++++++++++--
train.py | 16 +++++++++++++---
3 files changed, 30 insertions(+), 6 deletions(-)
diff --git a/losses.py b/losses.py
index 35b896d..435d124 100644
--- a/losses.py
+++ b/losses.py
@@ -7,4 +7,5 @@ class RMSLELoss(nn.Module):
self.mse = nn.MSELoss()
def forward(self, pred, actual):
- return torch.sqrt(self.mse(torch.log(pred + 1), torch.log(actual + 1)))
\ No newline at end of file
+ resized_actual = actual.view([actual.shape[0], actual.shape[1],1])
+ return torch.sqrt(self.mse(torch.log(torch.add(pred,1)), torch.log(torch.add(resized_actual, 1))))
\ No newline at end of file
diff --git a/main.py b/main.py
index cca6083..9e9d6c2 100644
--- a/main.py
+++ b/main.py
@@ -8,6 +8,7 @@ import optimizers
import torch
import logging
import torch.optim
+import torch.nn as nn
def optimizer(cfg, model):
result = {"Adam" : torch.optim.Adam(model.parameters())}
@@ -35,7 +36,7 @@ if __name__ == "__main__":
use_cuda,
valid_ratio,
overwrite_index=True,
- max_num_samples=max_num_samples,
+ max_num_samples=max_num_samples
)
if use_cuda :
@@ -43,17 +44,29 @@ if __name__ == "__main__":
else :
device = toch.device('cpu')
- model = model.build_model(cfg, 18)
+ #model = model.build_model(cfg, 18)
+
+ model = nn.Sequential(
+ nn.Linear(18,1,False),
+ nn.ReLU()
+ )
model = model.to(device)
+ for param in list(model.parameters()):
+ param = 1
+
f_loss = losses.RMSLELoss()
optimizer = optimizer(cfg, model)
+ #optimizer = torch.optim.Adam((model.parameters()), lr = 10000)
+
for t in range(cfg["Training"]["Epochs"]):
print("Epoch {}".format(t))
train(model, train_loader, f_loss, optimizer, device)
+
+ print(list(model.parameters())[0].grad)
val_loss = test.test(model, valid_loader, f_loss, device)
print(" Validation : Loss : {:.4f}".format(val_loss))
diff --git a/train.py b/train.py
index 11c9e2e..489959f 100644
--- a/train.py
+++ b/train.py
@@ -27,10 +27,20 @@ def train(model, loader, f_loss, optimizer, device):
outputs = model(inputs)
loss = f_loss(outputs, targets)
+ print("Loss")
+ print(loss)
+
+ print("outputs")
+ print(outputs)
+
+ print("targets")
+ print(targets)
+
# Backward and optimize
optimizer.zero_grad()
loss.backward()
- optimizer.step()
- print(model.regressor.weight)
- print(model.regressor.bias)
\ No newline at end of file
+ print("GRads")
+ print(list(model.parameters())[0].grad)
+
+ optimizer.step()
\ No newline at end of file
--
GitLab