PyTorch Introduction

Components of Network Training

1. Dataset
2. Network
3. Optimizer
4. Training Loop

from pathlib import Path

import torch
from torch.utils.data import Dataset


class MyDataset(Dataset):
    def __init__(self): ...
        
    def __len__(self): ...
    
    def __getitem__(self, index: int): ...

import torch
from torch.utils.data import Dataset


class InMemoryDataset(Dataset):
    def __init__(self, dataset_root_folder: Path):
        samples = np.load(dataest_root_folder / 'samples.npy')
        
    def __len__(self):
        return len(self.samples)
    
    def __getitem__(self, index: int):
        return self.samples[index]

from PIL import Image
import torch
from torch.utils.data import Dataset


class OutOfMemoryDataset(Dataset):
    def __init__(self, dataset_root_folder: Path, transforms):
        self.transforms = transforms
        
        self.image_files = list(dataset_root_folder.iterdir())
        
    def __len__(self):
        return len(self.image_files)
    
    def __getitem__(self, index: int):
        image = Image.load(self.images_files[index])
        return self.transforms(image)

Dataset + DataLoader

Dataset: produces sample by index

Dataloader: incupsulates Dataset to generate stream of butchs of samples

from torch.utils.data import DataLoader

dataset = MyDataset()
dataloader = DataLoader(
    dataset, 
    batch_size=16, 
    shuffle=True, 
    num_workers=8, 
)

DataLoader and python trees

1. Tensor of shape [A1, ..., Ak] -> batch-tensor [BATCH_SIZE, A_1, ..., Ak]
2. Tuple/list of tensors -> tuple/list of batch-tensors
3. Dict -> recursively values of Dict

Good idea to return sample as dicts:

{
    'image': image_tensor,
    'mask': mask_tensor,
    'label': label,
}

Network architeture: nn.Module

Layers and Networks are indistinguishable:

1. They are transformations from input tensors to output tensors
2. They have weights (trainable, non-trainable and semi-trainable parameters)
3. They have hyperarameters

from torch import nn


class LayerOrNetwork(nn.Module):
    def __init__(self, hyperparameters):
        super().__init__()
        ...
        
    def forward(self, x):
        ...
        return y

class SimpleConv(nn.Module):
    def __init__(self):
        super().__init__()
        
        self.conv1 = nn.Conv2d(in_channels=3, out_channels=64, kernel_size=3)
        self.relu1 = nn.ReLU()
        self.conv2 = nn.Conv2d(in_channels=64, out_channels=64, kernel_size=3)
        self.conv3 = nn.Conv2d(in_channels=64, out_channels=64, kernel_size=3)
        self.pooling = nn.AdaptiveAvgPool2d(output_size=1)
        self.linear = nn.Linear(64, 1)
        
    def forward(self, x):
        x = self.conv1(x)
        x = self.relu1(x)
        
        x = self.conv2(x)
        x = nn.functional.relu(x)
        
        x = self.conv3(x)
        x = nn.functional.relu(x)
        
        x = self.pooling(x)
        x = x.squeeze(-1).squeeze(-1)
        
        x = self.linear(x)
        
        return x

Training Loop

Instantiate dataset + dataloader
Instantiate neural network

while True:
    Generate batch
    Pass input tensors through batch
    Compare network output with targets and compute loss
    Calculate gradients of loss w.r.t. network parameters
    Update network weights

network             
# network.submodule1.weight1
# network.submodule1.submodule2.weight2

x, y                # input tensor and output tensor
y_hat = network(x)  # network output
loss = ((y - y_hat)**2).mean()

loss.backward()
# network.submodule1.weight1.grad <- d loss / d weight1
# network.submodule1.weight2.grad <- d loss / d weight2

#initialization
optimizer = torch.optim.SGD(network.parameters(), lr=1e-1)

...

#inside training loop:
while True:
    ...
    loss = ...
    
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()

dataset = MyDataset()
dataloader = DataLoader(dataset, batch_size=16, shuffle=True, num_workers=8)

network = SimpleConv()

optimizer = torch.optim.SGD(network.parameters(), lr=1e-1)

while True:
    for batch in dataloader:
        images, targets = batch
        
        network_outputs = network(images)
        
        loss = ((targets - network_outputs)**2).mean()
        
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        
        print(loss.item())

Task.

Train neural network to square 1-dimensional input tensor $x \rightarrow x^2$

1. Implement Dataset that would generate sample pairs $(x, x^2), x \in [-3; 3]$
2. Implement multi-layer perceptron.
3. Create dataset, dataloader, network instance and optimizer
4. Implement training loop and train your neural network

Layer Examples. Linear

torch.nn.Linear: [..., C_in] -> [..., C_out]

import torch

linear = torch.nn.Linear(
    in_features=10, 
    out_features=20, 
    bias=True
)

x_in = torch.randn(16, 10)
print(linear(x_in).shape)

x_in = torch.randn(16, 3, 3, 3, 10)
print(linear(x_in).shape)

torch.Size([16, 20])
torch.Size([16, 3, 3, 3, 20])

Layer Examples. Conv

torch.nn.Conv2d: [BS, C_in, H, W] -> [BS, C_out, H', W']

import torch

conv = torch.nn.Conv2d(
    in_channels=10, 
    out_channels=20,
    kernel_size=3,
)

x_in = torch.randn(16, 10, 32, 32)
print(conv(x_in).shape)

conv = torch.nn.Conv2d(
    in_channels=10, 
    out_channels=20,
    kernel_size=3,
    padding=1,
)
x_in = torch.randn(16, 10, 32, 32)
print(conv(x_in).shape)

# Does not work:
# x_in = torch.randn(16, 2, 10, 32, 32)
# print(conv(x_in).shape)

torch.Size([16, 20, 30, 30])
torch.Size([16, 20, 32, 32])

Layer Examples. Pooling

torch.nn.MaxPool2d: [BS, C, H, W] -> [BS, C, H', W']

import torch

pool = torch.nn.MaxPool2d(
    kernel_size=2,
)

x_in = torch.randn(16, 10, 32, 32)
print(pool(x_in).shape)

torch.Size([16, 10, 16, 16])

Layer Examples. BatchNorm2d

torch.nn.BatchNorm2d: [BS, C, H, W] -> [BS, C, H, W]

import torch

batch_norm = torch.nn.BatchNorm2d(10)

x_in = torch.randn(16, 10, 32, 32)
print(batch_norm(x_in).shape)

torch.Size([16, 10, 32, 32])