# Custom Training Loop
There are five steps to creating a training loop
1. __Define__ the network
2. __Prepare__ the training data pipeline
3. __Specify__ Loss and Optimizer
4. __Train__ the model using loss and optimizer
a. Loop through epochs, loop through batches
b. For each batch do forward prop within gradient tape
c. Calculate gradients and update trainable parameters
d. Save losses per batch; update metrics using `update_state`
5. __Test__ the model
## Low Level handling of metrics
1. Call `metric.update_state()` to accumulate the metric statistics after each batch
2. Call `metric.result()` to get current value of metric for display
3. Call `meteric.reset_state()` to reset metric value for next epoch
## Simple Example
```python
class Model():
def __init__(self):
self.w = tf.Variable(5.0)
self.b = tf.Variable(0.0)
def __call__(self, x):
return self.w * x + self.b
def loss(y_true, y_pred):
return tf.reduce_mean(tf.square(y_true - y_pred))
def train(model, inputs, outputs, leanring_rate):
with tf.GradientTape() as tape:
current_loss = loss(outputs, model(inputs))
dw, db = tape.gradient(current_loss, [model.w, model.b])
model.w.assign_sub(learning_rate * dw)
model.b.assign_sub(learning_rate * db)
epochs = 20
for epoch in epochs:
train(model, inputs, outputs, learning_rate=0.1)
```
## Custom Training Loop for Fashion MNIST
```python
# 1. Define the model
def base_model():
inputs = tf.keras.Input(shape=(784,), name='clothing')
x = tf.keras.layers.Dense(64, activation='relu', name='dense_1')(inputs)
x = tf.keras.layers.Dense(64, activation='relu', name='dense_2')(x)
outputs = tf.keras.layers.Dense(10, activaiton='sigmoid', name='predictiond')(x)
model = tf.keras.Model(inputs=inputs, outputs=outputs)
return model
# 2. Prepare the trining pipeline
train_data = tfds.load("fashion_mnist", split='train')
test_data = tfds.load('fashion_mnist', split='test')
def format_image(data):
image = data['image']
image = tf.reshape(image,[-1])
image = image /255.0
return image, data['label']
train_data = train_data.map(format_image)
test_data = test_data.map(format_image)
batch_size = 64
train = train_data.shuffle(buffer_size=1024).batch(batch_size)
test = test_data.batch(batch_size=batch_size)
# 3. Define Loss and Optimizer
loss = tf.keras.loss.SparseCategoricalCrossentropy()
optmizer = tf.keras.optimizers.Adam()
# Define metrics
train_acc_metric = tf.keras.metrics.SparseCategoricalAccuracy()
val_acc_metric = tf.keras.metrics.SparseCategoricalAccuracy()
def train_data_for_one_epoch():
losses = []
for step, (x_batch_train, y_batch_train) in enumerate(train_dataset):
logits, loss_value = apply_gradient(optimizer, model, x_batch_train, y_batch_train)
losses.append(loss_value)
train_acc_metric.update_state(y_batch_trian, logits)
return losses
def apply_gradient(optimizer,model, x, y):
with tf.GradientTape() as tape:
logits = model(x)
loss_value = loss_object(y, logits)
gradients = tape.gradient(loss_value, model.trainable_weights)
optimizer.apply_gradinets(zip(gradients, model.trainable_weights))
return logits, loss_value
def perform_validation():
losses = []
for x_val, y_val in test_dataset:
val_logits = model(x_val)
val_loss = loss_object(y_val, val_logits)
losses.append(val_loss)
val_acc_metric.update_state(y_val, val_logits)
return losses
# 4. Define the training loop
# 4.1 Initialize the model
model = base_model()
# 4.2 Iterate over epochs
EPOCHS = 20
for epochs in range(EPOCHS):
# 4.2.1 Calculate Loss
# 4.2.2 Update weights using optimizer
losses_train = train_data_for_one_epoch()
train_acc = tran_acc_metric.result()
# 4.2.3 Calculate loss on validation step
losses_val = perform_validation()
val_acc = val_acc_metric.result()
# 4.2.4 Caculate and display metrics
losses_train_mean = np.mean(losses_train)
losses_val_mean = np.mean(losses_val)
# Display
print(f"\n Epoch {epoch}: Train loss: {float(np.mean(losses_train))}")
# Reset metric states
train_acc = tran_acc_metric.reset_states()
val_acc_metric.reset_states()
# 5. Test the model
# Make predictions
# Calculate loss
# Calculate metrics
```
#