Skip to content
Snippets Groups Projects
Commit deda3a60 authored by Yandi's avatar Yandi
Browse files

preparing for batch job rnn 73 days

parent 7151c71a
No related branches found
No related tags found
1 merge request!1Master into main
Hide whitespace changes
Inline Side-by-side
Showing
with 403 additions and 9 deletions
create_submission.py
+ 1
− 1
View file @ deda3a60
...@@ -37,7 +37,7 @@ def create_submission(model, transform, device): ...@@ -37,7 +37,7 @@ def create_submission(model, transform, device):
# is not using the temporal context, this is here arbitrarily chosen # is not using the temporal context, this is here arbitrarily chosen
# However, note that it must be a divisor of the total number of days # However, note that it must be a divisor of the total number of days
# in the 2017 year , either 1, 5, 73 or 365 # in the 2017 year , either 1, 5, 73 or 365
num_days = 365 num_days = 73
num_workers = 7 num_workers = 7
use_cuda = torch.cuda.is_available() use_cuda = torch.cuda.is_available()
......
job.py
+ 3
− 1
View file @ deda3a60
...@@ -75,7 +75,9 @@ parser.add_argument("--model_name", ...@@ -75,7 +75,9 @@ parser.add_argument("--model_name",
# Ensure the log directory exists # Ensure the log directory exists
os.system("mkdir -p logslurms") os.system("mkdir -p logslurms")
args = parser.parse_args()
time_wall = {"no_limit": "48:00:00","hour" : "1:00:00", "half" : "0:00:00", "quarter" : "0:00:15"} time_wall = {"no_limit": "48:00:00","hour" : "1:00:00", "half" : "0:00:00", "quarter" : "0:00:15"}
# Launch the batch jobs # Launch the batch jobs
submit_job(makejob(commit_id, "cnn", 1, args.user, time_wall[args.time_wall])) submit_job(makejob(commit_id, args.model_name, 1, args.user, time_wall[args.time_wall]))
\ No newline at end of file \ No newline at end of file
logs/RNN_9/best_model.pt 0 → 100644
+ 0
− 0
View file @ deda3a60
File added
logs/main_unit_test.log
+ 363
− 0
View file @ deda3a60
...@@ -943,3 +943,366 @@ INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and ...@@ -943,3 +943,366 @@ INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - The train fold has 36097 samples INFO:root: - The train fold has 36097 samples
INFO:root: - The valid fold has 9047 samples INFO:root: - The valid fold has 9047 samples
INFO:root:= Dataloaders for mean and standard deviation
INFO:root: - Dataset creation
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Generating the index
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - Loaded a dataset with 45144 samples
INFO:root: - Splitting the data in training and validation sets
INFO:root:Generating the subset files from 45144 samples
INFO:root: - Subset dataset
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - The train fold has 36172 samples
INFO:root:= Dataloaders
INFO:root: - Dataset creation
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Generating the index
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - Loaded a dataset with 45144 samples
INFO:root: - Splitting the data in training and validation sets
INFO:root:Generating the subset files from 45144 samples
INFO:root: - Subset dataset
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - The train fold has 36304 samples
INFO:root: - The valid fold has 8840 samples
INFO:root:= Dataloaders for mean and standard deviation
INFO:root: - Dataset creation
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Generating the index
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - Loaded a dataset with 45144 samples
INFO:root: - Splitting the data in training and validation sets
INFO:root:Generating the subset files from 45144 samples
INFO:root: - Subset dataset
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - The train fold has 36173 samples
INFO:root:= Dataloaders
INFO:root: - Dataset creation
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Generating the index
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - Loaded a dataset with 45144 samples
INFO:root: - Splitting the data in training and validation sets
INFO:root:Generating the subset files from 45144 samples
INFO:root: - Subset dataset
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - The train fold has 35937 samples
INFO:root: - The valid fold has 9207 samples
INFO:root:= Dataloaders for mean and standard deviation
INFO:root: - Dataset creation
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Generating the index
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - Loaded a dataset with 45144 samples
INFO:root: - Splitting the data in training and validation sets
INFO:root:Generating the subset files from 45144 samples
INFO:root: - Subset dataset
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - The train fold has 36163 samples
INFO:root:= Dataloaders
INFO:root: - Dataset creation
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Generating the index
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - Loaded a dataset with 45144 samples
INFO:root: - Splitting the data in training and validation sets
INFO:root:Generating the subset files from 45144 samples
INFO:root: - Subset dataset
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - The train fold has 36186 samples
INFO:root: - The valid fold has 8958 samples
INFO:root:= Dataloaders for mean and standard deviation
INFO:root: - Dataset creation
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Generating the index
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - Loaded a dataset with 45144 samples
INFO:root: - Splitting the data in training and validation sets
INFO:root:Generating the subset files from 45144 samples
INFO:root: - Subset dataset
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - The train fold has 36232 samples
INFO:root:= Dataloaders
INFO:root: - Dataset creation
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Generating the index
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - Loaded a dataset with 45144 samples
INFO:root: - Splitting the data in training and validation sets
INFO:root:Generating the subset files from 45144 samples
INFO:root: - Subset dataset
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - The train fold has 36137 samples
INFO:root: - The valid fold has 9007 samples
INFO:root:= Dataloaders for mean and standard deviation
INFO:root: - Dataset creation
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Generating the index
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - Loaded a dataset with 45144 samples
INFO:root: - Splitting the data in training and validation sets
INFO:root:Generating the subset files from 45144 samples
INFO:root: - Subset dataset
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - The train fold has 36187 samples
INFO:root:= Dataloaders
INFO:root: - Dataset creation
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Generating the index
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - Loaded a dataset with 45144 samples
INFO:root: - Splitting the data in training and validation sets
INFO:root:Generating the subset files from 45144 samples
INFO:root: - Subset dataset
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - The train fold has 36114 samples
INFO:root: - The valid fold has 9030 samples
INFO:root:= Dataloaders for mean and standard deviation
INFO:root: - Dataset creation
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Generating the index
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - Loaded a dataset with 45144 samples
INFO:root: - Splitting the data in training and validation sets
INFO:root:Generating the subset files from 45144 samples
INFO:root: - Subset dataset
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - The train fold has 36068 samples
INFO:root:= Dataloaders
INFO:root: - Dataset creation
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Generating the index
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - Loaded a dataset with 45144 samples
INFO:root: - Splitting the data in training and validation sets
INFO:root:Generating the subset files from 45144 samples
INFO:root: - Subset dataset
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - The train fold has 36081 samples
INFO:root: - The valid fold has 9063 samples
INFO:root:= Dataloaders for mean and standard deviation
INFO:root: - Dataset creation
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Generating the index
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - Loaded a dataset with 45144 samples
INFO:root: - Splitting the data in training and validation sets
INFO:root:Generating the subset files from 45144 samples
INFO:root: - Subset dataset
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - The train fold has 35978 samples
INFO:root:= Dataloaders
INFO:root: - Dataset creation
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Generating the index
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - Loaded a dataset with 45144 samples
INFO:root: - Splitting the data in training and validation sets
INFO:root:Generating the subset files from 45144 samples
INFO:root: - Subset dataset
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - The train fold has 36240 samples
INFO:root: - The valid fold has 8904 samples
INFO:root:= Dataloaders for mean and standard deviation
INFO:root: - Dataset creation
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Generating the index
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - Loaded a dataset with 45144 samples
INFO:root: - Splitting the data in training and validation sets
INFO:root:Generating the subset files from 45144 samples
INFO:root: - Subset dataset
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - The train fold has 36126 samples
INFO:root:= Dataloaders
INFO:root: - Dataset creation
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Generating the index
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - Loaded a dataset with 45144 samples
INFO:root: - Splitting the data in training and validation sets
INFO:root:Generating the subset files from 45144 samples
INFO:root: - Subset dataset
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - The train fold has 36102 samples
INFO:root: - The valid fold has 9042 samples
INFO:root:= Dataloaders for mean and standard deviation
INFO:root: - Dataset creation
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Generating the index
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - Loaded a dataset with 45144 samples
INFO:root: - Splitting the data in training and validation sets
INFO:root:Generating the subset files from 45144 samples
INFO:root: - Subset dataset
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - The train fold has 36141 samples
INFO:root:= Dataloaders
INFO:root: - Dataset creation
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Generating the index
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - Loaded a dataset with 45144 samples
INFO:root: - Splitting the data in training and validation sets
INFO:root:Generating the subset files from 45144 samples
INFO:root: - Subset dataset
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - The train fold has 36175 samples
INFO:root: - The valid fold has 8969 samples
INFO:root:= Dataloaders for mean and standard deviation
INFO:root: - Dataset creation
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Generating the index
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - Loaded a dataset with 45144 samples
INFO:root: - Splitting the data in training and validation sets
INFO:root:Generating the subset files from 45144 samples
INFO:root: - Subset dataset
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - The train fold has 36208 samples
INFO:root:= Dataloaders
INFO:root: - Dataset creation
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Generating the index
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - Loaded a dataset with 45144 samples
INFO:root: - Splitting the data in training and validation sets
INFO:root:Generating the subset files from 45144 samples
INFO:root: - Subset dataset
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - The train fold has 36066 samples
INFO:root: - The valid fold has 9078 samples
INFO:root:= Dataloaders for mean and standard deviation
INFO:root: - Dataset creation
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Generating the index
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - Loaded a dataset with 45144 samples
INFO:root: - Splitting the data in training and validation sets
INFO:root:Generating the subset files from 45144 samples
INFO:root: - Subset dataset
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - The train fold has 36021 samples
INFO:root:= Dataloaders
INFO:root: - Dataset creation
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Generating the index
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - Loaded a dataset with 45144 samples
INFO:root: - Splitting the data in training and validation sets
INFO:root:Generating the subset files from 45144 samples
INFO:root: - Subset dataset
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - The train fold has 36116 samples
INFO:root: - The valid fold has 9028 samples
INFO:root:= Dataloaders for mean and standard deviation
INFO:root: - Dataset creation
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Generating the index
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - Loaded a dataset with 45144 samples
INFO:root: - Splitting the data in training and validation sets
INFO:root:Generating the subset files from 45144 samples
INFO:root: - Subset dataset
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - The train fold has 36234 samples
INFO:root:= Dataloaders
INFO:root: - Dataset creation
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Generating the index
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - Loaded a dataset with 45144 samples
INFO:root: - Splitting the data in training and validation sets
INFO:root:Generating the subset files from 45144 samples
INFO:root: - Subset dataset
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - The train fold has 36174 samples
INFO:root: - The valid fold has 8970 samples
INFO:root:= Dataloaders for mean and standard deviation
INFO:root: - Dataset creation
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Generating the index
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - Loaded a dataset with 45144 samples
INFO:root: - Splitting the data in training and validation sets
INFO:root:Generating the subset files from 45144 samples
INFO:root: - Subset dataset
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - The train fold has 36101 samples
INFO:root:= Dataloaders for mean and standard deviation
INFO:root: - Dataset creation
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Generating the index
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - Loaded a dataset with 45144 samples
INFO:root: - Splitting the data in training and validation sets
INFO:root:Generating the subset files from 45144 samples
INFO:root: - Subset dataset
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - The train fold has 36128 samples
INFO:root:= Dataloaders
INFO:root: - Dataset creation
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Generating the index
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - Loaded a dataset with 45144 samples
INFO:root: - Splitting the data in training and validation sets
INFO:root:Generating the subset files from 45144 samples
INFO:root: - Subset dataset
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root:The loaded dataset contains 25 latitudes, 37 longitudes, 28 depths and 2222 time points
INFO:root:Loading the index from sub_2CMEMS-MEDSEA-2010-2016-training.nc.bin_index.idx
INFO:root: - The train fold has 36035 samples
INFO:root: - The valid fold has 9109 samples
main.py
+ 5
− 4
View file @ deda3a60
...@@ -33,18 +33,19 @@ if __name__ == "__main__": ...@@ -33,18 +33,19 @@ if __name__ == "__main__":
parser.add_argument( parser.add_argument(
"--rootDir", "--rootDir",
default=None,
help="Directory in which the log files will be stored" help="Directory in which the log files will be stored"
) )
args = parser.parse_args() args = parser.parse_args()
config_file = open("config.yml")
cfg = yaml.load(config_file)
rootDir = cfg["LogDir"] if eval(args.rootDir) != None else args.rootDir rootDir = args.rootDir if args.rootDir != None else cfg["LogDir"]
logging.basicConfig(filename= rootDir + 'main_unit_test.log', level=logging.INFO) logging.basicConfig(filename= rootDir + 'main_unit_test.log', level=logging.INFO)
config_file = open("config.yml")
cfg = yaml.load(config_file)
use_cuda = torch.cuda.is_available() use_cuda = torch.cuda.is_available()
trainpath = cfg["Dataset"]["_DEFAULT_TRAIN_FILEPATH"] trainpath = cfg["Dataset"]["_DEFAULT_TRAIN_FILEPATH"]
...@@ -71,7 +72,7 @@ if __name__ == "__main__": ...@@ -71,7 +72,7 @@ if __name__ == "__main__":
MAX = eval(cfg["ApproximativeMaxi"]) MAX = eval(cfg["ApproximativeMaxi"])
MIN = eval(cfg["ApproximativeMini"]) MIN = eval(cfg["ApproximativeMini"])
else : else :
MEAN, STD, MAX, MIN = get_stats_train_dataset(trainpath, MEAN, STD, MAX, MIN = dataloader.get_stats_train_dataset(trainpath,
num_days, num_days,
batch_size, batch_size,
num_workers, num_workers,
......
model.py
+ 27
− 0
View file @ deda3a60
...@@ -3,6 +3,7 @@ import torch ...@@ -3,6 +3,7 @@ import torch
import torch.nn as nn import torch.nn as nn
import torch.nn.functional as F import torch.nn.functional as F
from torch.autograd import Function from torch.autograd import Function
from torch.autograd import Variable
import torch.nn as nn import torch.nn as nn
...@@ -35,6 +36,32 @@ class LinearRegression(nn.Module): ...@@ -35,6 +36,32 @@ class LinearRegression(nn.Module):
def forward(self, x): def forward(self, x):
return self.regressor(x) return self.regressor(x)
# Recurrent Neural Networks
class RNN(nn.Module):
def __init__(self, cfg, input_size):
super(RNN, self).__init__()
self.hidden_size = cfg["RNN"]["HiddenSize"]
self.num_layers = cfg["RNN"]["NumLayers"]
# RNN
self.rnn = nn.RNN(input_size, self.hidden_size, self.num_layers, batch_first=True, nonlinearity='relu')
self.fc = nn.Linear(self.hidden_size, 1)
def forward(self, x):
use_cuda = torch.cuda.is_available()
if use_cuda :
device = torch.device('cuda')
else :
device = toch.device('cpu')
# Initialize hidden state with zeros
h0 = torch.zeros(self.num_layers, x.size(0), self.hidden_size).to(device)
# One time step
out, hn = self.rnn(x, h0)
out = self.fc(out)
print(out.shape)
return out
# Bidirectional LSTM # Bidirectional LSTM
class BidirectionalLSTM(nn.Module): class BidirectionalLSTM(nn.Module):
def __init__(self, cfg, input_size): def __init__(self, cfg, input_size):
......
train.py
+ 1
− 0
View file @ deda3a60
...@@ -31,6 +31,7 @@ def train(args, model, loader, f_loss, optimizer, device, log_interval = 100): ...@@ -31,6 +31,7 @@ def train(args, model, loader, f_loss, optimizer, device, log_interval = 100):
# Compute the forward pass through the network up to the loss # Compute the forward pass through the network up to the loss
outputs = model(inputs) outputs = model(inputs)
print(targets.shape)
loss = f_loss(outputs, targets) loss = f_loss(outputs, targets)
# Backward and optimize # Backward and optimize
......
train_indices.subset
+ 0
− 0
View file @ deda3a60
No preview for this file type
valid_indices.subset
+ 0
− 0
View file @ deda3a60
No preview for this file type
wandb/debug-internal.log
+ 1
− 1
View file @ deda3a60
run-20230130_164435-f5w8sswr/logs/debug-internal.log run-20230131_200825-ytr4zy6r/logs/debug-internal.log
\ No newline at end of file \ No newline at end of file
wandb/debug.log
+ 1
− 1
View file @ deda3a60
run-20230130_164435-f5w8sswr/logs/debug.log run-20230131_200825-ytr4zy6r/logs/debug.log
\ No newline at end of file \ No newline at end of file
wandb/latest-run
+ 1
− 1
View file @ deda3a60
run-20230130_164435-f5w8sswr run-20230131_200825-ytr4zy6r
\ No newline at end of file \ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment