Backprop (forward and backward passes)

See these notes for more prose on these examples.

import numpy as np

Computation Graph 1

def example1_forward_backward(x, y, z): 
    """
    We implement the forward and 
    backward (gradient) pass for the function
    
    f(x, y, z) = (x+y)z
    """
    result = {"f_forward": None, 
            "df_dx": None,
            "df_dy": None, 
            "df_dz": None}
    
    # Forward pass 
    a = x + y
    f = a*z
    result["f_forward"] = f
    
    # Backward pass (in reverse order from forward pass) 
    # Local partial derivatives on edges of computation graph 
    df_dz = a 
    df_da = z
    da_dx = 1
    da_dy = 1 
    # Chain rule for final outputs 
    df_dx = df_da*da_dx
    df_dy = df_da*da_dy
    
    result['df_dx'] = df_dx
    result['df_dy'] = df_dy
    result['df_dz'] = df_dz
    return result 

example1_forward_backward(-2, 5, -4)

{'f_forward': -12, 'df_dx': -4, 'df_dy': -4, 'df_dz': 3}

Computation Graph 2

def logistic(u): 
    return 1/(1 + np.exp(-1*u))

logistic(30)

0.9999999999999065

logistic(-1)

0.2689414213699951

def example2_forward_backward(x, y): 
    """
    We implement the forward and 
    backward (gradient) pass for the function
    
    f(x, y, z) = (x + sigma(y))(x+y)^{-2}
    """
    result = {"f_forward": None, 
            "df_dx": None,
            "df_dy": None}
    
    # Forward pass 
    a = logistic(y)
    b = x + a 
    c = x + y
    d = c**(-2)
    f = b*d
    result["f_forward"] = f
    
    # Backward pass (in reverse order from forward pass) 
    # Local partial derivatives on edges of computation graph  
    df_db = d
    df_dd = b 
    db_dx = 1
    db_da = 1 
    da_dy = (1 - logistic(y))*logistic(y)
    dc_dx = 1
    dd_dc = -2*c**(-3)
    dc_dy = 1 
    
    # Chain rule for final outputs 
    df_dx = df_db*db_dx + df_dd*dd_dc*dc_dx 
    df_dy = df_db*db_da*da_dy + df_dd*dd_dc*dc_dy
    
    result['df_dx'] = df_dx
    result['df_dy'] = df_dy
    return result 

example2_forward_backward(1, -2)

{'f_forward': 1.1192029220221176,
 'df_dx': 3.238405844044235,
 'df_dy': 2.3433994294477416}