See these notes for more prose on these examples.
import numpy as np
def example1_forward_backward(x, y, z):
"""
We implement the forward and
backward (gradient) pass for the function
f(x, y, z) = (x+y)z
"""
result = {"f_forward": None,
"df_dx": None,
"df_dy": None,
"df_dz": None}
# Forward pass
a = x + y
f = a*z
result["f_forward"] = f
# Backward pass (in reverse order from forward pass)
# Local partial derivatives on edges of computation graph
df_dz = a
df_da = z
da_dx = 1
da_dy = 1
# Chain rule for final outputs
df_dx = df_da*da_dx
df_dy = df_da*da_dy
result['df_dx'] = df_dx
result['df_dy'] = df_dy
result['df_dz'] = df_dz
return result
example1_forward_backward(-2, 5, -4)
{'f_forward': -12, 'df_dx': -4, 'df_dy': -4, 'df_dz': 3}
def logistic(u):
return 1/(1 + np.exp(-1*u))
logistic(30)
0.9999999999999065
logistic(-1)
0.2689414213699951
def example2_forward_backward(x, y):
"""
We implement the forward and
backward (gradient) pass for the function
f(x, y, z) = (x + sigma(y))(x+y)^{-2}
"""
result = {"f_forward": None,
"df_dx": None,
"df_dy": None}
# Forward pass
a = logistic(y)
b = x + a
c = x + y
d = c**(-2)
f = b*d
result["f_forward"] = f
# Backward pass (in reverse order from forward pass)
# Local partial derivatives on edges of computation graph
df_db = d
df_dd = b
db_dx = 1
db_da = 1
da_dy = (1 - logistic(y))*logistic(y)
dc_dx = 1
dd_dc = -2*c**(-3)
dc_dy = 1
# Chain rule for final outputs
df_dx = df_db*db_dx + df_dd*dd_dc*dc_dx
df_dy = df_db*db_da*da_dy + df_dd*dd_dc*dc_dy
result['df_dx'] = df_dx
result['df_dy'] = df_dy
return result
example2_forward_backward(1, -2)
{'f_forward': 1.1192029220221176, 'df_dx': 3.238405844044235, 'df_dy': 2.3433994294477416}