240615

week1/betaversion/lab_utils_uni.py

@@ -0,0 +1,398 @@
+""" 
+lab_utils_uni.py
+    routines used in Course 1, Week2, labs1-3 dealing with single variables (univariate)
+"""
+import numpy as np
+import matplotlib.pyplot as plt
+from matplotlib.ticker import MaxNLocator
+from matplotlib.gridspec import GridSpec
+from matplotlib.colors import LinearSegmentedColormap
+from ipywidgets import interact
+from lab_utils_common import compute_cost
+from lab_utils_common import dlblue, dlorange, dldarkred, dlmagenta, dlpurple, dlcolors
+
+plt.style.use('./deeplearning.mplstyle')
+n_bin = 5
+dlcm = LinearSegmentedColormap.from_list(
+        'dl_map', dlcolors, N=n_bin)
+
+##########################################################
+# Plotting Routines
+##########################################################
+
+def plt_house_x(X, y,f_wb=None, ax=None):
+    ''' plot house with aXis '''
+    if not ax:
+        fig, ax = plt.subplots(1,1)
+    ax.scatter(X, y, marker='x', c='r', label="Actual Value")
+
+    ax.set_title("Housing Prices")
+    ax.set_ylabel('Price (in 1000s of dollars)')
+    ax.set_xlabel(f'Size (1000 sqft)')
+    if f_wb is not None:
+        ax.plot(X, f_wb,  c=dlblue, label="Our Prediction")
+    ax.legend()
+
+
+def mk_cost_lines(x,y,w,b, ax):
+    ''' makes vertical cost lines'''
+    cstr = "cost = (1/m)*("
+    ctot = 0
+    label = 'cost for point'
+    addedbreak = False
+    for p in zip(x,y):
+        f_wb_p = w*p[0]+b
+        c_p = ((f_wb_p - p[1])**2)/2
+        c_p_txt = c_p
+        ax.vlines(p[0], p[1],f_wb_p, lw=3, color=dlpurple, ls='dotted', label=label)
+        label='' #just one
+        cxy = [p[0], p[1] + (f_wb_p-p[1])/2]
+        ax.annotate(f'{c_p_txt:0.0f}', xy=cxy, xycoords='data',color=dlpurple,
+            xytext=(5, 0), textcoords='offset points')
+        cstr += f"{c_p_txt:0.0f} +"
+        if len(cstr) > 38 and addedbreak is False:
+            cstr += "\n"
+            addedbreak = True
+        ctot += c_p
+    ctot = ctot/(len(x))
+    cstr = cstr[:-1] + f") = {ctot:0.0f}"
+    ax.text(0.15,0.02,cstr, transform=ax.transAxes, color=dlpurple)
+
+##########
+# Cost lab
+##########
+
+
+def plt_intuition(x_train, y_train):
+
+    w_range = np.array([200-200,200+200])
+    tmp_b = 100
+
+    w_array = np.arange(*w_range, 5)
+    cost = np.zeros_like(w_array)
+    for i in range(len(w_array)):
+        tmp_w = w_array[i]
+        cost[i] = compute_cost(x_train, y_train, tmp_w, tmp_b)
+
+    @interact(w=(*w_range,10),continuous_update=False)
+    def func( w=150):
+        f_wb = np.dot(x_train, w) + tmp_b
+
+        fig, ax = plt.subplots(1, 2, constrained_layout=True, figsize=(8,4))
+        fig.canvas.toolbar_position = 'bottom'
+
+        mk_cost_lines(x_train, y_train, w, tmp_b, ax[0])
+        plt_house_x(x_train, y_train, f_wb=f_wb, ax=ax[0])
+
+        ax[1].plot(w_array, cost)
+        cur_cost = compute_cost(x_train, y_train, w, tmp_b)
+        ax[1].scatter(w,cur_cost, s=100, color=dldarkred, zorder= 10, label= f"cost at w={w}")
+        ax[1].hlines(cur_cost, ax[1].get_xlim()[0],w, lw=4, color=dlpurple, ls='dotted')
+        ax[1].vlines(w, ax[1].get_ylim()[0],cur_cost, lw=4, color=dlpurple, ls='dotted')
+        ax[1].set_title("Cost vs. w, (b fixed at 100)")
+        ax[1].set_ylabel('Cost')
+        ax[1].set_xlabel('w')
+        ax[1].legend(loc='upper center')
+        fig.suptitle(f"Minimize Cost: Current Cost = {cur_cost:0.0f}", fontsize=12)
+        plt.show()
+
+# this is the 2D cost curve with interactive slider
+def plt_stationary(x_train, y_train):
+    # setup figure
+    fig = plt.figure( figsize=(9,8))
+    #fig = plt.figure(constrained_layout=True,  figsize=(12,10))
+    fig.set_facecolor('#ffffff') #white
+    fig.canvas.toolbar_position = 'top'
+    #gs = GridSpec(2, 2, figure=fig, wspace = 0.01)
+    gs = GridSpec(2, 2, figure=fig)
+    ax0 = fig.add_subplot(gs[0, 0])
+    ax1 = fig.add_subplot(gs[0, 1])
+    ax2 = fig.add_subplot(gs[1, :],  projection='3d')
+    ax = np.array([ax0,ax1,ax2])
+
+    #setup useful ranges and common linspaces
+    w_range = np.array([200-300.,200+300])
+    b_range = np.array([50-300., 50+300])
+    b_space  = np.linspace(*b_range, 100)
+    w_space  = np.linspace(*w_range, 100)
+
+    # get cost for w,b ranges for contour and 3D
+    tmp_b,tmp_w = np.meshgrid(b_space,w_space)
+    z=np.zeros_like(tmp_b)
+    for i in range(tmp_w.shape[0]):
+        for j in range(tmp_w.shape[1]):
+            z[i,j] = compute_cost(x_train, y_train, tmp_w[i][j], tmp_b[i][j] )
+            if z[i,j] == 0: z[i,j] = 1e-6
+
+    w0=200;b=-100    #initial point
+    ### plot model w cost ###
+    f_wb = np.dot(x_train,w0) + b
+    mk_cost_lines(x_train,y_train,w0,b,ax[0])
+    plt_house_x(x_train, y_train, f_wb=f_wb, ax=ax[0])
+
+    ### plot contour ###
+    CS = ax[1].contour(tmp_w, tmp_b, np.log(z),levels=12, linewidths=2, alpha=0.7,colors=dlcolors)
+    ax[1].set_title('Cost(w,b)')
+    ax[1].set_xlabel('w', fontsize=10)
+    ax[1].set_ylabel('b', fontsize=10)
+    ax[1].set_xlim(w_range) ; ax[1].set_ylim(b_range)
+    cscat  = ax[1].scatter(w0,b, s=100, color=dlblue, zorder= 10, label="cost with \ncurrent w,b")
+    chline = ax[1].hlines(b, ax[1].get_xlim()[0],w0, lw=4, color=dlpurple, ls='dotted')
+    cvline = ax[1].vlines(w0, ax[1].get_ylim()[0],b, lw=4, color=dlpurple, ls='dotted')
+    ax[1].text(0.5,0.95,"Click to choose w,b",  bbox=dict(facecolor='white', ec = 'black'), fontsize = 10,
+                transform=ax[1].transAxes, verticalalignment = 'center', horizontalalignment= 'center')
+
+    #Surface plot of the cost function J(w,b)
+    ax[2].plot_surface(tmp_w, tmp_b, z,  cmap = dlcm, alpha=0.3, antialiased=True)
+    ax[2].plot_wireframe(tmp_w, tmp_b, z, color='k', alpha=0.1)
+    plt.xlabel("$w$")
+    plt.ylabel("$b$")
+    ax[2].zaxis.set_rotate_label(False)
+    ax[2].xaxis.set_pane_color((1.0, 1.0, 1.0, 0.0))
+    ax[2].yaxis.set_pane_color((1.0, 1.0, 1.0, 0.0))
+    ax[2].zaxis.set_pane_color((1.0, 1.0, 1.0, 0.0))
+    ax[2].set_zlabel("J(w, b)\n\n", rotation=90)
+    plt.title("Cost(w,b) \n [You can rotate this figure]", size=12)
+    ax[2].view_init(30, -120)
+
+    return fig,ax, [cscat, chline, cvline]
+
+
+#https://matplotlib.org/stable/users/event_handling.html
+class plt_update_onclick:
+    def __init__(self, fig, ax, x_train,y_train, dyn_items):
+        self.fig = fig
+        self.ax = ax
+        self.x_train = x_train
+        self.y_train = y_train
+        self.dyn_items = dyn_items
+        self.cid = fig.canvas.mpl_connect('button_press_event', self)
+
+    def __call__(self, event):
+        if event.inaxes == self.ax[1]:
+            ws = event.xdata
+            bs = event.ydata
+            cst = compute_cost(self.x_train, self.y_train, ws, bs)
+
+            # clear and redraw line plot
+            self.ax[0].clear()
+            f_wb = np.dot(self.x_train,ws) + bs
+            mk_cost_lines(self.x_train,self.y_train,ws,bs,self.ax[0])
+            plt_house_x(self.x_train, self.y_train, f_wb=f_wb, ax=self.ax[0])
+
+            # remove lines and re-add on countour plot and 3d plot
+            for artist in self.dyn_items:
+                artist.remove()
+
+            a = self.ax[1].scatter(ws,bs, s=100, color=dlblue, zorder= 10, label="cost with \ncurrent w,b")
+            b = self.ax[1].hlines(bs, self.ax[1].get_xlim()[0],ws, lw=4, color=dlpurple, ls='dotted')
+            c = self.ax[1].vlines(ws, self.ax[1].get_ylim()[0],bs, lw=4, color=dlpurple, ls='dotted')
+            d = self.ax[1].annotate(f"Cost: {cst:.0f}", xy= (ws, bs), xytext = (4,4), textcoords = 'offset points',
+                               bbox=dict(facecolor='white'), size = 10)
+
+            #Add point in 3D surface plot
+            e = self.ax[2].scatter3D(ws, bs,cst , marker='X', s=100)
+
+            self.dyn_items = [a,b,c,d,e]
+            self.fig.canvas.draw()
+
+
+def soup_bowl():
+    """ Create figure and plot with a 3D projection"""
+    fig = plt.figure(figsize=(8,8))
+
+    #Plot configuration
+    ax = fig.add_subplot(111, projection='3d')
+    ax.xaxis.set_pane_color((1.0, 1.0, 1.0, 0.0))
+    ax.yaxis.set_pane_color((1.0, 1.0, 1.0, 0.0))
+    ax.zaxis.set_pane_color((1.0, 1.0, 1.0, 0.0))
+    ax.zaxis.set_rotate_label(False)
+    ax.view_init(45, -120)
+
+    #Useful linearspaces to give values to the parameters w and b
+    w = np.linspace(-20, 20, 100)
+    b = np.linspace(-20, 20, 100)
+
+    #Get the z value for a bowl-shaped cost function
+    z=np.zeros((len(w), len(b)))
+    j=0
+    for x in w:
+        i=0
+        for y in b:
+            z[i,j] = x**2 + y**2
+            i+=1
+        j+=1
+
+    #Meshgrid used for plotting 3D functions
+    W, B = np.meshgrid(w, b)
+
+    #Create the 3D surface plot of the bowl-shaped cost function
+    ax.plot_surface(W, B, z, cmap = "Spectral_r", alpha=0.7, antialiased=False)
+    ax.plot_wireframe(W, B, z, color='k', alpha=0.1)
+    ax.set_xlabel("$w$")
+    ax.set_ylabel("$b$")
+    ax.set_zlabel("$J(w,b)$", rotation=90)
+    ax.set_title("$J(w,b)$\n [You can rotate this figure]", size=15)
+
+    plt.show()
+
+def inbounds(a,b,xlim,ylim):
+    xlow,xhigh = xlim
+    ylow,yhigh = ylim
+    ax, ay = a
+    bx, by = b
+    if (ax > xlow and ax < xhigh) and (bx > xlow and bx < xhigh) \
+        and (ay > ylow and ay < yhigh) and (by > ylow and by < yhigh):
+        return True
+    return False
+
+def plt_contour_wgrad(x, y, hist, ax, w_range=[-100, 500, 5], b_range=[-500, 500, 5],
+                contours = [0.1,50,1000,5000,10000,25000,50000],
+                      resolution=5, w_final=200, b_final=100,step=10 ):
+    b0,w0 = np.meshgrid(np.arange(*b_range),np.arange(*w_range))
+    z=np.zeros_like(b0)
+    for i in range(w0.shape[0]):
+        for j in range(w0.shape[1]):
+            z[i][j] = compute_cost(x, y, w0[i][j], b0[i][j] )
+
+    CS = ax.contour(w0, b0, z, contours, linewidths=2,
+                   colors=[dlblue, dlorange, dldarkred, dlmagenta, dlpurple])
+    ax.clabel(CS, inline=1, fmt='%1.0f', fontsize=10)
+    ax.set_xlabel("w");  ax.set_ylabel("b")
+    ax.set_title('Contour plot of cost J(w,b), vs b,w with path of gradient descent')
+    w = w_final; b=b_final
+    ax.hlines(b, ax.get_xlim()[0],w, lw=2, color=dlpurple, ls='dotted')
+    ax.vlines(w, ax.get_ylim()[0],b, lw=2, color=dlpurple, ls='dotted')
+
+    base = hist[0]
+    for point in hist[0::step]:
+        edist = np.sqrt((base[0] - point[0])**2 + (base[1] - point[1])**2)
+        if(edist > resolution or point==hist[-1]):
+            if inbounds(point,base, ax.get_xlim(),ax.get_ylim()):
+                plt.annotate('', xy=point, xytext=base,xycoords='data',
+                         arrowprops={'arrowstyle': '->', 'color': 'r', 'lw': 3},
+                         va='center', ha='center')
+            base=point
+    return
+
+
+def plt_divergence(p_hist, J_hist, x_train,y_train):
+
+    x=np.zeros(len(p_hist))
+    y=np.zeros(len(p_hist))
+    v=np.zeros(len(p_hist))
+    for i in range(len(p_hist)):
+        x[i] = p_hist[i][0]
+        y[i] = p_hist[i][1]
+        v[i] = J_hist[i]
+
+    fig = plt.figure(figsize=(12,5))
+    plt.subplots_adjust( wspace=0 )
+    gs = fig.add_gridspec(1, 5)
+    fig.suptitle(f"Cost escalates when learning rate is too large")
+    #===============
+    #  First subplot
+    #===============
+    ax = fig.add_subplot(gs[:2], )
+
+    # Print w vs cost to see minimum
+    fix_b = 100
+    w_array = np.arange(-70000, 70000, 1000)
+    cost = np.zeros_like(w_array)
+
+    for i in range(len(w_array)):
+        tmp_w = w_array[i]
+        cost[i] = compute_cost(x_train, y_train, tmp_w, fix_b)
+
+    ax.plot(w_array, cost)
+    ax.plot(x,v, c=dlmagenta)
+    ax.set_title("Cost vs w, b set to 100")
+    ax.set_ylabel('Cost')
+    ax.set_xlabel('w')
+    ax.xaxis.set_major_locator(MaxNLocator(2))
+
+    #===============
+    # Second Subplot
+    #===============
+
+    tmp_b,tmp_w = np.meshgrid(np.arange(-35000, 35000, 500),np.arange(-70000, 70000, 500))
+    z=np.zeros_like(tmp_b)
+    for i in range(tmp_w.shape[0]):
+        for j in range(tmp_w.shape[1]):
+            z[i][j] = compute_cost(x_train, y_train, tmp_w[i][j], tmp_b[i][j] )
+
+    ax = fig.add_subplot(gs[2:], projection='3d')
+    ax.plot_surface(tmp_w, tmp_b, z,  alpha=0.3, color=dlblue)
+    ax.xaxis.set_major_locator(MaxNLocator(2))
+    ax.yaxis.set_major_locator(MaxNLocator(2))
+
+    ax.set_xlabel('w', fontsize=16)
+    ax.set_ylabel('b', fontsize=16)
+    ax.set_zlabel('\ncost', fontsize=16)
+    plt.title('Cost vs (b, w)')
+    # Customize the view angle
+    ax.view_init(elev=20., azim=-65)
+    ax.plot(x, y, v,c=dlmagenta)
+
+    return
+
+# draw derivative line
+# y = m*(x - x1) + y1
+def add_line(dj_dx, x1, y1, d, ax):
+    x = np.linspace(x1-d, x1+d,50)
+    y = dj_dx*(x - x1) + y1
+    ax.scatter(x1, y1, color=dlblue, s=50)
+    ax.plot(x, y, '--', c=dldarkred,zorder=10, linewidth = 1)
+    xoff = 30 if x1 == 200 else 10
+    ax.annotate(r"$\frac{\partial J}{\partial w}$ =%d" % dj_dx, fontsize=14,
+                xy=(x1, y1), xycoords='data',
+            xytext=(xoff, 10), textcoords='offset points',
+            arrowprops=dict(arrowstyle="->"),
+            horizontalalignment='left', verticalalignment='top')
+
+def plt_gradients(x_train,y_train, f_compute_cost, f_compute_gradient):
+    #===============
+    #  First subplot
+    #===============
+    fig,ax = plt.subplots(1,2,figsize=(12,4))
+
+    # Print w vs cost to see minimum
+    fix_b = 100
+    w_array = np.linspace(-100, 500, 50)
+    w_array = np.linspace(0, 400, 50)
+    cost = np.zeros_like(w_array)
+
+    for i in range(len(w_array)):
+        tmp_w = w_array[i]
+        cost[i] = f_compute_cost(x_train, y_train, tmp_w, fix_b)
+    ax[0].plot(w_array, cost,linewidth=1)
+    ax[0].set_title("Cost vs w, with gradient; b set to 100")
+    ax[0].set_ylabel('Cost')
+    ax[0].set_xlabel('w')
+
+    # plot lines for fixed b=100
+    for tmp_w in [100,200,300]:
+        fix_b = 100
+        dj_dw,dj_db = f_compute_gradient(x_train, y_train, tmp_w, fix_b )
+        j = f_compute_cost(x_train, y_train, tmp_w, fix_b)
+        add_line(dj_dw, tmp_w, j, 30, ax[0])
+
+    #===============
+    # Second Subplot
+    #===============
+
+    tmp_b,tmp_w = np.meshgrid(np.linspace(-200, 200, 10), np.linspace(-100, 600, 10))
+    U = np.zeros_like(tmp_w)
+    V = np.zeros_like(tmp_b)
+    for i in range(tmp_w.shape[0]):
+        for j in range(tmp_w.shape[1]):
+            U[i][j], V[i][j] = f_compute_gradient(x_train, y_train, tmp_w[i][j], tmp_b[i][j] )
+    X = tmp_w
+    Y = tmp_b
+    n=-2
+    color_array = np.sqrt(((V-n)/2)**2 + ((U-n)/2)**2)
+
+    ax[1].set_title('Gradient shown in quiver plot')
+    Q = ax[1].quiver(X, Y, U, V, color_array, units='width', )
+    ax[1].quiverkey(Q, 0.9, 0.9, 2, r'$2 \frac{m}{s}$', labelpos='E',coordinates='figure')
+    ax[1].set_xlabel("w"); ax[1].set_ylabel("b")