[{"data":1,"prerenderedAt":4636},["ShallowReactive",2],{"lang-switch-post-\u002Fen\u002Fplaylists\u002Fmachine-learning-specialization\u002Flab04-gradient-descent":3,"post-en-machine-learning-specialization-lab04-gradient-descent":4},"\u002Fplaylists\u002Fmachine-learning-specialization\u002Flab04-gradient-descent",{"id":5,"title":6,"body":7,"cover":4621,"date":4622,"description":4623,"extension":4624,"meta":4625,"navigation":2887,"order":2878,"path":4626,"playlist":4627,"seo":4628,"status":4629,"stem":4630,"tags":4631,"__hash__":4635},"posts\u002Fen\u002Fplaylists\u002Fmachine-learning-specialization\u002Flab04-gradient-descent.md","Optional Lab: Gradient Descent",{"type":8,"value":9,"toc":4607},"minimark",[10,18,354,363,739,744,751,905,909,921,1286,1318,1322,1325,1933,2461,2684,2727,2849,2853,3033,3036,3176,3180,3183,3289,3296,3300,3360,3397,3429,3491,3494,3498,3505,3510,3513,3778,3834,3841,3844,3847,3853,3907,3911,4075,4078,4144,4176,4179,4183,4429,4432,4454,4469,4473,4487,4526,4537,4540,4543,4603],[11,12,13],"p",{},[14,15],"img",{"alt":16,"src":17},"An Olympic podium meme: in the first five panels, the athlete celebrates the gold medal next to a graph of a nicely-behaved, single-bottomed bowl. In the last panel, the real podium shows up with a graph full of bumpy valleys instead, and who takes gold, silver, or bronze depends on which valley each one fell into","\u002Fimages\u002Fposts\u002Fmachine-learning-specialization\u002Flab04-gradient-descent\u002Fmeme-gradient-descent.jpg",[11,19,20,21,26,27,233,234,238,239,300,301,353],{},"Quick recap of the last two posts: you ",[22,23,25],"a",{"href":24},"\u002Fen\u002Fplaylists\u002Fmachine-learning-specialization\u002Flab02-model-representation","built the model"," ",[28,29,32,93],"span",{"className":30},[31],"katex",[28,33,36],{"className":34},[35],"katex-mathml",[37,38,40],"math",{"xmlns":39},"http:\u002F\u002Fwww.w3.org\u002F1998\u002FMath\u002FMathML",[41,42,43,88],"semantics",{},[44,45,46,66,70,73,76,79,81,83,86],"mrow",{},[47,48,49,53],"msub",{},[50,51,52],"mi",{},"f",[44,54,55,58,63],{},[50,56,57],{},"w",[59,60,62],"mo",{"separator":61},"true",",",[50,64,65],{},"b",[59,67,69],{"stretchy":68},"false","(",[50,71,72],{},"x",[59,74,75],{"stretchy":68},")",[59,77,78],{},"=",[50,80,57],{},[50,82,72],{},[59,84,85],{},"+",[50,87,65],{},[89,90,92],"annotation",{"encoding":91},"application\u002Fx-tex","f_{w,b}(x) = wx + b",[28,94,97,199,223],{"className":95,"ariaHidden":61},[96],"katex-html",[28,98,101,106,176,180,183,187,192,196],{"className":99},[100],"base",[28,102],{"className":103,"style":105},[104],"strut","height:1.0361em;vertical-align:-0.2861em;",[28,107,110,115],{"className":108},[109],"mord",[28,111,52],{"className":112,"style":114},[109,113],"mathnormal","margin-right:0.1076em;",[28,116,119],{"className":117},[118],"msupsub",[28,120,124,167],{"className":121},[122,123],"vlist-t","vlist-t2",[28,125,128,162],{"className":126},[127],"vlist-r",[28,129,133],{"className":130,"style":132},[131],"vlist","height:0.3361em;",[28,134,136,141],{"style":135},"top:-2.55em;margin-left:-0.1076em;margin-right:0.05em;",[28,137],{"className":138,"style":140},[139],"pstrut","height:2.7em;",[28,142,148],{"className":143},[144,145,146,147],"sizing","reset-size6","size3","mtight",[28,149,151,155,159],{"className":150},[109,147],[28,152,57],{"className":153,"style":154},[109,113,147],"margin-right:0.0269em;",[28,156,62],{"className":157},[158,147],"mpunct",[28,160,65],{"className":161},[109,113,147],[28,163,166],{"className":164},[165],"vlist-s","​",[28,168,170],{"className":169},[127],[28,171,174],{"className":172,"style":173},[131],"height:0.2861em;",[28,175],{},[28,177,69],{"className":178},[179],"mopen",[28,181,72],{"className":182},[109,113],[28,184,75],{"className":185},[186],"mclose",[28,188],{"className":189,"style":191},[190],"mspace","margin-right:0.2778em;",[28,193,78],{"className":194},[195],"mrel",[28,197],{"className":198,"style":191},[190],[28,200,202,206,209,212,216,220],{"className":201},[100],[28,203],{"className":204,"style":205},[104],"height:0.6667em;vertical-align:-0.0833em;",[28,207,57],{"className":208,"style":154},[109,113],[28,210,72],{"className":211},[109,113],[28,213],{"className":214,"style":215},[190],"margin-right:0.2222em;",[28,217,85],{"className":218},[219],"mbin",[28,221],{"className":222,"style":215},[190],[28,224,226,230],{"className":225},[100],[28,227],{"className":228,"style":229},[104],"height:0.6944em;",[28,231,65],{"className":232},[109,113],", then ",[22,235,237],{"href":236},"\u002Fen\u002Fplaylists\u002Fmachine-learning-specialization\u002Flab03-cost-function","built a way to measure how wrong it is",", ",[28,240,242,267],{"className":241},[31],[28,243,245],{"className":244},[35],[37,246,247],{"xmlns":39},[41,248,249,264],{},[44,250,251,254,256,258,260,262],{},[50,252,253],{},"J",[59,255,69],{"stretchy":68},[50,257,57],{},[59,259,62],{"separator":61},[50,261,65],{},[59,263,75],{"stretchy":68},[89,265,266],{"encoding":91},"J(w,b)",[28,268,270],{"className":269,"ariaHidden":61},[96],[28,271,273,277,281,284,287,290,294,297],{"className":272},[100],[28,274],{"className":275,"style":276},[104],"height:1em;vertical-align:-0.25em;",[28,278,253],{"className":279,"style":280},[109,113],"margin-right:0.0962em;",[28,282,69],{"className":283},[179],[28,285,57],{"className":286,"style":154},[109,113],[28,288,62],{"className":289},[158],[28,291],{"className":292,"style":293},[190],"margin-right:0.1667em;",[28,295,65],{"className":296},[109,113],[28,298,75],{"className":299},[186],". But to find the best ",[28,302,304,326],{"className":303},[31],[28,305,307],{"className":306},[35],[37,308,309],{"xmlns":39},[41,310,311,323],{},[44,312,313,315,317,319,321],{},[59,314,69],{"stretchy":68},[50,316,57],{},[59,318,62],{"separator":61},[50,320,65],{},[59,322,75],{"stretchy":68},[89,324,325],{"encoding":91},"(w,b)",[28,327,329],{"className":328,"ariaHidden":61},[96],[28,330,332,335,338,341,344,347,350],{"className":331},[100],[28,333],{"className":334,"style":276},[104],[28,336,69],{"className":337},[179],[28,339,57],{"className":340,"style":154},[109,113],[28,342,62],{"className":343},[158],[28,345],{"className":346,"style":293},[190],[28,348,65],{"className":349},[109,113],[28,351,75],{"className":352},[186]," you were still doing the most primitive thing possible: dragging a slider and watching the number drop. That works for 2 points. For a real dataset, with thousands of examples and dozens of parameters, it's impossible.",[11,355,356,357,362],{},"This post closes the loop with the algorithm that does that search on its own: ",[358,359,361],"glossary-term",{"definition":360},"an algorithm that adjusts a model's parameters step by step, always in the direction that most reduces the cost, until it stops near the minimum","gradient descent",".",[11,364,365],{},[28,366,368,456],{"className":367},[31],[28,369,371],{"className":370},[35],[37,372,373],{"xmlns":39},[41,374,375,453],{},[44,376,377,379,381,383,386,389,416,419,421,423,425,427,429],{},[50,378,57],{},[59,380,78],{},[50,382,57],{},[59,384,385],{},"−",[50,387,388],{},"α",[390,391,392,410],"mfrac",{},[44,393,394,398,400,402,404,406,408],{},[50,395,397],{"mathvariant":396},"normal","∂",[50,399,253],{},[59,401,69],{"stretchy":68},[50,403,57],{},[59,405,62],{"separator":61},[50,407,65],{},[59,409,75],{"stretchy":68},[44,411,412,414],{},[50,413,397],{"mathvariant":396},[50,415,57],{},[190,417],{"width":418},"2em",[50,420,65],{},[59,422,78],{},[50,424,65],{},[59,426,385],{},[50,428,388],{},[390,430,431,447],{},[44,432,433,435,437,439,441,443,445],{},[50,434,397],{"mathvariant":396},[50,436,253],{},[59,438,69],{"stretchy":68},[50,440,57],{},[59,442,62],{"separator":61},[50,444,65],{},[59,446,75],{"stretchy":68},[44,448,449,451],{},[50,450,397],{"mathvariant":396},[50,452,65],{},[89,454,455],{"encoding":91},"w = w - \\alpha \\frac{\\partial J(w,b)}{\\partial w} \\qquad b = b - \\alpha \\frac{\\partial J(w,b)}{\\partial b}",[28,457,459,478,496,622,641],{"className":458,"ariaHidden":61},[96],[28,460,462,466,469,472,475],{"className":461},[100],[28,463],{"className":464,"style":465},[104],"height:0.4306em;",[28,467,57],{"className":468,"style":154},[109,113],[28,470],{"className":471,"style":191},[190],[28,473,78],{"className":474},[195],[28,476],{"className":477,"style":191},[190],[28,479,481,484,487,490,493],{"className":480},[100],[28,482],{"className":483,"style":205},[104],[28,485,57],{"className":486,"style":154},[109,113],[28,488],{"className":489,"style":215},[190],[28,491,385],{"className":492},[219],[28,494],{"className":495,"style":215},[190],[28,497,499,503,507,606,610,613,616,619],{"className":498},[100],[28,500],{"className":501,"style":502},[104],"height:1.355em;vertical-align:-0.345em;",[28,504,388],{"className":505,"style":506},[109,113],"margin-right:0.0037em;",[28,508,510,514,603],{"className":509},[109],[28,511],{"className":512},[179,513],"nulldelimiter",[28,515,517],{"className":516},[390],[28,518,520,594],{"className":519},[122,123],[28,521,523,591],{"className":522},[127],[28,524,527,547,558],{"className":525,"style":526},[131],"height:1.01em;",[28,528,530,534],{"style":529},"top:-2.655em;",[28,531],{"className":532,"style":533},[139],"height:3em;",[28,535,537],{"className":536},[144,145,146,147],[28,538,540,544],{"className":539},[109,147],[28,541,397],{"className":542,"style":543},[109,147],"margin-right:0.0556em;",[28,545,57],{"className":546,"style":154},[109,113,147],[28,548,550,553],{"style":549},"top:-3.23em;",[28,551],{"className":552,"style":533},[139],[28,554],{"className":555,"style":557},[556],"frac-line","border-bottom-width:0.04em;",[28,559,561,564],{"style":560},"top:-3.485em;",[28,562],{"className":563,"style":533},[139],[28,565,567],{"className":566},[144,145,146,147],[28,568,570,573,576,579,582,585,588],{"className":569},[109,147],[28,571,397],{"className":572,"style":543},[109,147],[28,574,253],{"className":575,"style":280},[109,113,147],[28,577,69],{"className":578},[179,147],[28,580,57],{"className":581,"style":154},[109,113,147],[28,583,62],{"className":584},[158,147],[28,586,65],{"className":587},[109,113,147],[28,589,75],{"className":590},[186,147],[28,592,166],{"className":593},[165],[28,595,597],{"className":596},[127],[28,598,601],{"className":599,"style":600},[131],"height:0.345em;",[28,602],{},[28,604],{"className":605},[186,513],[28,607],{"className":608,"style":609},[190],"margin-right:2em;",[28,611,65],{"className":612},[109,113],[28,614],{"className":615,"style":191},[190],[28,617,78],{"className":618},[195],[28,620],{"className":621,"style":191},[190],[28,623,625,629,632,635,638],{"className":624},[100],[28,626],{"className":627,"style":628},[104],"height:0.7778em;vertical-align:-0.0833em;",[28,630,65],{"className":631},[109,113],[28,633],{"className":634,"style":215},[190],[28,636,385],{"className":637},[219],[28,639],{"className":640,"style":215},[190],[28,642,644,647,650],{"className":643},[100],[28,645],{"className":646,"style":502},[104],[28,648,388],{"className":649,"style":506},[109,113],[28,651,653,656,736],{"className":652},[109],[28,654],{"className":655},[179,513],[28,657,659],{"className":658},[390],[28,660,662,728],{"className":661},[122,123],[28,663,665,725],{"className":664},[127],[28,666,668,685,693],{"className":667,"style":526},[131],[28,669,670,673],{"style":529},[28,671],{"className":672,"style":533},[139],[28,674,676],{"className":675},[144,145,146,147],[28,677,679,682],{"className":678},[109,147],[28,680,397],{"className":681,"style":543},[109,147],[28,683,65],{"className":684},[109,113,147],[28,686,687,690],{"style":549},[28,688],{"className":689,"style":533},[139],[28,691],{"className":692,"style":557},[556],[28,694,695,698],{"style":560},[28,696],{"className":697,"style":533},[139],[28,699,701],{"className":700},[144,145,146,147],[28,702,704,707,710,713,716,719,722],{"className":703},[109,147],[28,705,397],{"className":706,"style":543},[109,147],[28,708,253],{"className":709,"style":280},[109,113,147],[28,711,69],{"className":712},[179,147],[28,714,57],{"className":715,"style":154},[109,113,147],[28,717,62],{"className":718},[158,147],[28,720,65],{"className":721},[109,113,147],[28,723,75],{"className":724},[186,147],[28,726,166],{"className":727},[165],[28,729,731],{"className":730},[127],[28,732,734],{"className":733,"style":600},[131],[28,735],{},[28,737],{"className":738},[186,513],[740,741,743],"h2",{"id":742},"the-idea-in-one-sentence","The idea in one sentence",[11,745,746,747,750],{},"Remember the soup bowl from ",[22,748,749],{"href":236},"the last post","? Gradient descent is literally that: you start at some point on the bowl and take steps downhill, always in the direction that descends fastest, until you land near the bottom.",[11,752,753,754,869,870,899,900,904],{},"The \"feeling which direction descends fastest\" part is the derivative's job, that ",[28,755,757,783],{"className":756},[31],[28,758,760],{"className":759},[35],[37,761,762],{"xmlns":39},[41,763,764,780],{},[44,765,766],{},[390,767,768,774],{},[44,769,770,772],{},[50,771,397],{"mathvariant":396},[50,773,253],{},[44,775,776,778],{},[50,777,397],{"mathvariant":396},[50,779,57],{},[89,781,782],{"encoding":91},"\\frac{\\partial J}{\\partial w}",[28,784,786],{"className":785,"ariaHidden":61},[96],[28,787,789,793],{"className":788},[100],[28,790],{"className":791,"style":792},[104],"height:1.2251em;vertical-align:-0.345em;",[28,794,796,799,866],{"className":795},[109],[28,797],{"className":798},[179,513],[28,800,802],{"className":801},[390],[28,803,805,858],{"className":804},[122,123],[28,806,808,855],{"className":807},[127],[28,809,812,829,837],{"className":810,"style":811},[131],"height:0.8801em;",[28,813,814,817],{"style":529},[28,815],{"className":816,"style":533},[139],[28,818,820],{"className":819},[144,145,146,147],[28,821,823,826],{"className":822},[109,147],[28,824,397],{"className":825,"style":543},[109,147],[28,827,57],{"className":828,"style":154},[109,113,147],[28,830,831,834],{"style":549},[28,832],{"className":833,"style":533},[139],[28,835],{"className":836,"style":557},[556],[28,838,840,843],{"style":839},"top:-3.394em;",[28,841],{"className":842,"style":533},[139],[28,844,846],{"className":845},[144,145,146,147],[28,847,849,852],{"className":848},[109,147],[28,850,397],{"className":851,"style":543},[109,147],[28,853,253],{"className":854,"style":280},[109,113,147],[28,856,166],{"className":857},[165],[28,859,861],{"className":860},[127],[28,862,864],{"className":863,"style":600},[131],[28,865],{},[28,867],{"className":868},[186,513]," in the formula. It's the cost surface's slope at that specific point. And ",[28,871,873,887],{"className":872},[31],[28,874,876],{"className":875},[35],[37,877,878],{"xmlns":39},[41,879,880,884],{},[44,881,882],{},[50,883,388],{},[89,885,886],{"encoding":91},"\\alpha",[28,888,890],{"className":889,"ariaHidden":61},[96],[28,891,893,896],{"className":892},[100],[28,894],{"className":895,"style":465},[104],[28,897,388],{"className":898,"style":506},[109,113]," (",[358,901,903],{"definition":902},"the greek letter used for the learning rate, the size of the step the algorithm takes on each iteration","alpha",") is how big a step you take each time.",[740,906,908],{"id":907},"why-subtract-the-derivative","Why subtract the derivative",[11,910,911,912,916,917,920],{},"The derivative points toward where the cost ",[913,914,915],"strong",{},"increases",". Since you want a smaller cost, you walk in the ",[913,918,919],{},"opposite"," direction. Hence the minus sign in the formula.",[922,923,924,1081],"table",{},[925,926,927],"thead",{},[928,929,930,935,939],"tr",{},[931,932,934],"th",{"align":933},"left","Situation",[931,936,938],{"align":937},"center","Sign of the derivative",[931,940,941,942],{"align":933},"What happens to ",[28,943,945,977],{"className":944},[31],[28,946,948],{"className":947},[35],[37,949,950],{"xmlns":39},[41,951,952,974],{},[44,953,954,956,958,960],{},[50,955,57],{},[59,957,385],{},[50,959,388],{},[390,961,962,968],{},[44,963,964,966],{},[50,965,397],{"mathvariant":396},[50,967,253],{},[44,969,970,972],{},[50,971,397],{"mathvariant":396},[50,973,57],{},[89,975,976],{"encoding":91},"w - \\alpha \\frac{\\partial J}{\\partial w}",[28,978,980,998],{"className":979,"ariaHidden":61},[96],[28,981,983,986,989,992,995],{"className":982},[100],[28,984],{"className":985,"style":205},[104],[28,987,57],{"className":988,"style":154},[109,113],[28,990],{"className":991,"style":215},[190],[28,993,385],{"className":994},[219],[28,996],{"className":997,"style":215},[190],[28,999,1001,1004,1007],{"className":1000},[100],[28,1002],{"className":1003,"style":792},[104],[28,1005,388],{"className":1006,"style":506},[109,113],[28,1008,1010,1013,1078],{"className":1009},[109],[28,1011],{"className":1012},[179,513],[28,1014,1016],{"className":1015},[390],[28,1017,1019,1070],{"className":1018},[122,123],[28,1020,1022,1067],{"className":1021},[127],[28,1023,1025,1042,1050],{"className":1024,"style":811},[131],[28,1026,1027,1030],{"style":529},[28,1028],{"className":1029,"style":533},[139],[28,1031,1033],{"className":1032},[144,145,146,147],[28,1034,1036,1039],{"className":1035},[109,147],[28,1037,397],{"className":1038,"style":543},[109,147],[28,1040,57],{"className":1041,"style":154},[109,113,147],[28,1043,1044,1047],{"style":549},[28,1045],{"className":1046,"style":533},[139],[28,1048],{"className":1049,"style":557},[556],[28,1051,1052,1055],{"style":839},[28,1053],{"className":1054,"style":533},[139],[28,1056,1058],{"className":1057},[144,145,146,147],[28,1059,1061,1064],{"className":1060},[109,147],[28,1062,397],{"className":1063,"style":543},[109,147],[28,1065,253],{"className":1066,"style":280},[109,113,147],[28,1068,166],{"className":1069},[165],[28,1071,1073],{"className":1072},[127],[28,1074,1076],{"className":1075,"style":600},[131],[28,1077],{},[28,1079],{"className":1080},[186,513],[1082,1083,1084,1152,1219],"tbody",{},[928,1085,1086,1118,1121],{},[1087,1088,1089,1117],"td",{"align":933},[28,1090,1092,1105],{"className":1091},[31],[28,1093,1095],{"className":1094},[35],[37,1096,1097],{"xmlns":39},[41,1098,1099,1103],{},[44,1100,1101],{},[50,1102,57],{},[89,1104,57],{"encoding":91},[28,1106,1108],{"className":1107,"ariaHidden":61},[96],[28,1109,1111,1114],{"className":1110},[100],[28,1112],{"className":1113,"style":465},[104],[28,1115,57],{"className":1116,"style":154},[109,113]," is to the right of the minimum",[1087,1119,1120],{"align":937},"positive",[1087,1122,1123,1151],{"align":933},[28,1124,1126,1139],{"className":1125},[31],[28,1127,1129],{"className":1128},[35],[37,1130,1131],{"xmlns":39},[41,1132,1133,1137],{},[44,1134,1135],{},[50,1136,57],{},[89,1138,57],{"encoding":91},[28,1140,1142],{"className":1141,"ariaHidden":61},[96],[28,1143,1145,1148],{"className":1144},[100],[28,1146],{"className":1147,"style":465},[104],[28,1149,57],{"className":1150,"style":154},[109,113]," decreases, moves left",[928,1153,1154,1185,1188],{},[1087,1155,1156,1184],{"align":933},[28,1157,1159,1172],{"className":1158},[31],[28,1160,1162],{"className":1161},[35],[37,1163,1164],{"xmlns":39},[41,1165,1166,1170],{},[44,1167,1168],{},[50,1169,57],{},[89,1171,57],{"encoding":91},[28,1173,1175],{"className":1174,"ariaHidden":61},[96],[28,1176,1178,1181],{"className":1177},[100],[28,1179],{"className":1180,"style":465},[104],[28,1182,57],{"className":1183,"style":154},[109,113]," is to the left of the minimum",[1087,1186,1187],{"align":937},"negative",[1087,1189,1190,1218],{"align":933},[28,1191,1193,1206],{"className":1192},[31],[28,1194,1196],{"className":1195},[35],[37,1197,1198],{"xmlns":39},[41,1199,1200,1204],{},[44,1201,1202],{},[50,1203,57],{},[89,1205,57],{"encoding":91},[28,1207,1209],{"className":1208,"ariaHidden":61},[96],[28,1210,1212,1215],{"className":1211},[100],[28,1213],{"className":1214,"style":465},[104],[28,1216,57],{"className":1217,"style":154},[109,113]," increases, moves right",[928,1220,1221,1252,1255],{},[1087,1222,1223,1251],{"align":933},[28,1224,1226,1239],{"className":1225},[31],[28,1227,1229],{"className":1228},[35],[37,1230,1231],{"xmlns":39},[41,1232,1233,1237],{},[44,1234,1235],{},[50,1236,57],{},[89,1238,57],{"encoding":91},[28,1240,1242],{"className":1241,"ariaHidden":61},[96],[28,1243,1245,1248],{"className":1244},[100],[28,1246],{"className":1247,"style":465},[104],[28,1249,57],{"className":1250,"style":154},[109,113]," is exactly at the minimum",[1087,1253,1254],{"align":937},"zero",[1087,1256,1257,1285],{"align":933},[28,1258,1260,1273],{"className":1259},[31],[28,1261,1263],{"className":1262},[35],[37,1264,1265],{"xmlns":39},[41,1266,1267,1271],{},[44,1268,1269],{},[50,1270,57],{},[89,1272,57],{"encoding":91},[28,1274,1276],{"className":1275,"ariaHidden":61},[96],[28,1277,1279,1282],{"className":1278},[100],[28,1280],{"className":1281,"style":465},[104],[28,1283,57],{"className":1284,"style":154},[109,113]," stops changing, the algorithm halted on its own",[11,1287,1288,1289,1317],{},"Notice the last row: the algorithm doesn't need an \"if you've reached the minimum, stop\" check. It simply stops moving on its own, because the derivative hits zero. And since the derivative shrinks as you get closer to the bottom, the steps also get smaller on their own, even with a fixed ",[28,1290,1292,1305],{"className":1291},[31],[28,1293,1295],{"className":1294},[35],[37,1296,1297],{"xmlns":39},[41,1298,1299,1303],{},[44,1300,1301],{},[50,1302,388],{},[89,1304,886],{"encoding":91},[28,1306,1308],{"className":1307,"ariaHidden":61},[96],[28,1309,1311,1314],{"className":1310},[100],[28,1312],{"className":1313,"style":465},[104],[28,1315,388],{"className":1316,"style":506},[109,113],". That's a free property, not something you code separately.",[740,1319,1321],{"id":1320},"the-two-partial-derivatives","The two partial derivatives",[11,1323,1324],{},"For single-variable linear regression, the math works out to these two formulas (I didn't need to memorize the derivation, just get the pattern):",[11,1326,1327],{},[28,1328,1330,1463],{"className":1329},[31],[28,1331,1333],{"className":1332},[35],[37,1334,1335],{"xmlns":39},[41,1336,1337,1460],{},[44,1338,1339,1363,1365,1374,1398,1448],{},[390,1340,1341,1357],{},[44,1342,1343,1345,1347,1349,1351,1353,1355],{},[50,1344,397],{"mathvariant":396},[50,1346,253],{},[59,1348,69],{"stretchy":68},[50,1350,57],{},[59,1352,62],{"separator":61},[50,1354,65],{},[59,1356,75],{"stretchy":68},[44,1358,1359,1361],{},[50,1360,397],{"mathvariant":396},[50,1362,57],{},[59,1364,78],{},[390,1366,1367,1371],{},[1368,1369,1370],"mn",{},"1",[50,1372,1373],{},"m",[1375,1376,1377,1380,1390],"msubsup",{},[59,1378,1379],{},"∑",[44,1381,1382,1385,1387],{},[50,1383,1384],{},"i",[59,1386,78],{},[1368,1388,1389],{},"0",[44,1391,1392,1394,1396],{},[50,1393,1373],{},[59,1395,385],{},[1368,1397,1370],{},[44,1399,1400,1402,1414,1416,1429,1431,1433,1446],{},[59,1401,69],{"fence":61},[47,1403,1404,1406],{},[50,1405,52],{},[44,1407,1408,1410,1412],{},[50,1409,57],{},[59,1411,62],{"separator":61},[50,1413,65],{},[59,1415,69],{"stretchy":68},[1417,1418,1419,1421],"msup",{},[50,1420,72],{},[44,1422,1423,1425,1427],{},[59,1424,69],{"stretchy":68},[50,1426,1384],{},[59,1428,75],{"stretchy":68},[59,1430,75],{"stretchy":68},[59,1432,385],{},[1417,1434,1435,1438],{},[50,1436,1437],{},"y",[44,1439,1440,1442,1444],{},[59,1441,69],{"stretchy":68},[50,1443,1384],{},[59,1445,75],{"stretchy":68},[59,1447,75],{"fence":61},[1417,1449,1450,1452],{},[50,1451,72],{},[44,1453,1454,1456,1458],{},[59,1455,69],{"stretchy":68},[50,1457,1384],{},[59,1459,75],{"stretchy":68},[89,1461,1462],{"encoding":91},"\\frac{\\partial J(w,b)}{\\partial w} = \\frac{1}{m} \\sum_{i=0}^{m-1} \\left(f_{w,b}(x^{(i)}) - y^{(i)}\\right) x^{(i)}",[28,1464,1466,1570],{"className":1465,"ariaHidden":61},[96],[28,1467,1469,1472,1561,1564,1567],{"className":1468},[100],[28,1470],{"className":1471,"style":502},[104],[28,1473,1475,1478,1558],{"className":1474},[109],[28,1476],{"className":1477},[179,513],[28,1479,1481],{"className":1480},[390],[28,1482,1484,1550],{"className":1483},[122,123],[28,1485,1487,1547],{"className":1486},[127],[28,1488,1490,1507,1515],{"className":1489,"style":526},[131],[28,1491,1492,1495],{"style":529},[28,1493],{"className":1494,"style":533},[139],[28,1496,1498],{"className":1497},[144,145,146,147],[28,1499,1501,1504],{"className":1500},[109,147],[28,1502,397],{"className":1503,"style":543},[109,147],[28,1505,57],{"className":1506,"style":154},[109,113,147],[28,1508,1509,1512],{"style":549},[28,1510],{"className":1511,"style":533},[139],[28,1513],{"className":1514,"style":557},[556],[28,1516,1517,1520],{"style":560},[28,1518],{"className":1519,"style":533},[139],[28,1521,1523],{"className":1522},[144,145,146,147],[28,1524,1526,1529,1532,1535,1538,1541,1544],{"className":1525},[109,147],[28,1527,397],{"className":1528,"style":543},[109,147],[28,1530,253],{"className":1531,"style":280},[109,113,147],[28,1533,69],{"className":1534},[179,147],[28,1536,57],{"className":1537,"style":154},[109,113,147],[28,1539,62],{"className":1540},[158,147],[28,1542,65],{"className":1543},[109,113,147],[28,1545,75],{"className":1546},[186,147],[28,1548,166],{"className":1549},[165],[28,1551,1553],{"className":1552},[127],[28,1554,1556],{"className":1555,"style":600},[131],[28,1557],{},[28,1559],{"className":1560},[186,513],[28,1562],{"className":1563,"style":191},[190],[28,1565,78],{"className":1566},[195],[28,1568],{"className":1569,"style":191},[190],[28,1571,1573,1577,1646,1649,1726,1729,1892,1895],{"className":1572},[100],[28,1574],{"className":1575,"style":1576},[104],"height:1.304em;vertical-align:-0.35em;",[28,1578,1580,1583,1643],{"className":1579},[109],[28,1581],{"className":1582},[179,513],[28,1584,1586],{"className":1585},[390],[28,1587,1589,1635],{"className":1588},[122,123],[28,1590,1592,1632],{"className":1591},[127],[28,1593,1596,1610,1618],{"className":1594,"style":1595},[131],"height:0.8451em;",[28,1597,1598,1601],{"style":529},[28,1599],{"className":1600,"style":533},[139],[28,1602,1604],{"className":1603},[144,145,146,147],[28,1605,1607],{"className":1606},[109,147],[28,1608,1373],{"className":1609},[109,113,147],[28,1611,1612,1615],{"style":549},[28,1613],{"className":1614,"style":533},[139],[28,1616],{"className":1617,"style":557},[556],[28,1619,1620,1623],{"style":839},[28,1621],{"className":1622,"style":533},[139],[28,1624,1626],{"className":1625},[144,145,146,147],[28,1627,1629],{"className":1628},[109,147],[28,1630,1370],{"className":1631},[109,147],[28,1633,166],{"className":1634},[165],[28,1636,1638],{"className":1637},[127],[28,1639,1641],{"className":1640,"style":600},[131],[28,1642],{},[28,1644],{"className":1645},[186,513],[28,1647],{"className":1648,"style":293},[190],[28,1650,1653,1659],{"className":1651},[1652],"mop",[28,1654,1379],{"className":1655,"style":1658},[1652,1656,1657],"op-symbol","small-op","position:relative;top:0em;",[28,1660,1662],{"className":1661},[118],[28,1663,1665,1717],{"className":1664},[122,123],[28,1666,1668,1714],{"className":1667},[127],[28,1669,1672,1693],{"className":1670,"style":1671},[131],"height:0.954em;",[28,1673,1675,1678],{"style":1674},"top:-2.4003em;margin-left:0em;margin-right:0.05em;",[28,1676],{"className":1677,"style":140},[139],[28,1679,1681],{"className":1680},[144,145,146,147],[28,1682,1684,1687,1690],{"className":1683},[109,147],[28,1685,1384],{"className":1686},[109,113,147],[28,1688,78],{"className":1689},[195,147],[28,1691,1389],{"className":1692},[109,147],[28,1694,1696,1699],{"style":1695},"top:-3.2029em;margin-right:0.05em;",[28,1697],{"className":1698,"style":140},[139],[28,1700,1702],{"className":1701},[144,145,146,147],[28,1703,1705,1708,1711],{"className":1704},[109,147],[28,1706,1373],{"className":1707},[109,113,147],[28,1709,385],{"className":1710},[219,147],[28,1712,1370],{"className":1713},[109,147],[28,1715,166],{"className":1716},[165],[28,1718,1720],{"className":1719},[127],[28,1721,1724],{"className":1722,"style":1723},[131],"height:0.2997em;",[28,1725],{},[28,1727],{"className":1728,"style":293},[190],[28,1730,1733,1743,1792,1795,1835,1838,1841,1844,1847,1886],{"className":1731},[1732],"minner",[28,1734,1738],{"className":1735,"style":1737},[179,1736],"delimcenter","top:0em;",[28,1739,69],{"className":1740},[1741,1742],"delimsizing","size1",[28,1744,1746,1749],{"className":1745},[109],[28,1747,52],{"className":1748,"style":114},[109,113],[28,1750,1752],{"className":1751},[118],[28,1753,1755,1784],{"className":1754},[122,123],[28,1756,1758,1781],{"className":1757},[127],[28,1759,1761],{"className":1760,"style":132},[131],[28,1762,1763,1766],{"style":135},[28,1764],{"className":1765,"style":140},[139],[28,1767,1769],{"className":1768},[144,145,146,147],[28,1770,1772,1775,1778],{"className":1771},[109,147],[28,1773,57],{"className":1774,"style":154},[109,113,147],[28,1776,62],{"className":1777},[158,147],[28,1779,65],{"className":1780},[109,113,147],[28,1782,166],{"className":1783},[165],[28,1785,1787],{"className":1786},[127],[28,1788,1790],{"className":1789,"style":173},[131],[28,1791],{},[28,1793,69],{"className":1794},[179],[28,1796,1798,1801],{"className":1797},[109],[28,1799,72],{"className":1800},[109,113],[28,1802,1804],{"className":1803},[118],[28,1805,1807],{"className":1806},[122],[28,1808,1810],{"className":1809},[127],[28,1811,1814],{"className":1812,"style":1813},[131],"height:0.888em;",[28,1815,1817,1820],{"style":1816},"top:-3.063em;margin-right:0.05em;",[28,1818],{"className":1819,"style":140},[139],[28,1821,1823],{"className":1822},[144,145,146,147],[28,1824,1826,1829,1832],{"className":1825},[109,147],[28,1827,69],{"className":1828},[179,147],[28,1830,1384],{"className":1831},[109,113,147],[28,1833,75],{"className":1834},[186,147],[28,1836,75],{"className":1837},[186],[28,1839],{"className":1840,"style":215},[190],[28,1842,385],{"className":1843},[219],[28,1845],{"className":1846,"style":215},[190],[28,1848,1850,1854],{"className":1849},[109],[28,1851,1437],{"className":1852,"style":1853},[109,113],"margin-right:0.0359em;",[28,1855,1857],{"className":1856},[118],[28,1858,1860],{"className":1859},[122],[28,1861,1863],{"className":1862},[127],[28,1864,1866],{"className":1865,"style":1813},[131],[28,1867,1868,1871],{"style":1816},[28,1869],{"className":1870,"style":140},[139],[28,1872,1874],{"className":1873},[144,145,146,147],[28,1875,1877,1880,1883],{"className":1876},[109,147],[28,1878,69],{"className":1879},[179,147],[28,1881,1384],{"className":1882},[109,113,147],[28,1884,75],{"className":1885},[186,147],[28,1887,1889],{"className":1888,"style":1737},[186,1736],[28,1890,75],{"className":1891},[1741,1742],[28,1893],{"className":1894,"style":293},[190],[28,1896,1898,1901],{"className":1897},[109],[28,1899,72],{"className":1900},[109,113],[28,1902,1904],{"className":1903},[118],[28,1905,1907],{"className":1906},[122],[28,1908,1910],{"className":1909},[127],[28,1911,1913],{"className":1912,"style":1813},[131],[28,1914,1915,1918],{"style":1816},[28,1916],{"className":1917,"style":140},[139],[28,1919,1921],{"className":1920},[144,145,146,147],[28,1922,1924,1927,1930],{"className":1923},[109,147],[28,1925,69],{"className":1926},[179,147],[28,1928,1384],{"className":1929},[109,113,147],[28,1931,75],{"className":1932},[186,147],[11,1934,1935],{},[28,1936,1938,2050],{"className":1937},[31],[28,1939,1941],{"className":1940},[35],[37,1942,1943],{"xmlns":39},[41,1944,1945,2047],{},[44,1946,1947,1971,1973,1979,1999],{},[390,1948,1949,1965],{},[44,1950,1951,1953,1955,1957,1959,1961,1963],{},[50,1952,397],{"mathvariant":396},[50,1954,253],{},[59,1956,69],{"stretchy":68},[50,1958,57],{},[59,1960,62],{"separator":61},[50,1962,65],{},[59,1964,75],{"stretchy":68},[44,1966,1967,1969],{},[50,1968,397],{"mathvariant":396},[50,1970,65],{},[59,1972,78],{},[390,1974,1975,1977],{},[1368,1976,1370],{},[50,1978,1373],{},[1375,1980,1981,1983,1991],{},[59,1982,1379],{},[44,1984,1985,1987,1989],{},[50,1986,1384],{},[59,1988,78],{},[1368,1990,1389],{},[44,1992,1993,1995,1997],{},[50,1994,1373],{},[59,1996,385],{},[1368,1998,1370],{},[44,2000,2001,2003,2015,2017,2029,2031,2033,2045],{},[59,2002,69],{"fence":61},[47,2004,2005,2007],{},[50,2006,52],{},[44,2008,2009,2011,2013],{},[50,2010,57],{},[59,2012,62],{"separator":61},[50,2014,65],{},[59,2016,69],{"stretchy":68},[1417,2018,2019,2021],{},[50,2020,72],{},[44,2022,2023,2025,2027],{},[59,2024,69],{"stretchy":68},[50,2026,1384],{},[59,2028,75],{"stretchy":68},[59,2030,75],{"stretchy":68},[59,2032,385],{},[1417,2034,2035,2037],{},[50,2036,1437],{},[44,2038,2039,2041,2043],{},[59,2040,69],{"stretchy":68},[50,2042,1384],{},[59,2044,75],{"stretchy":68},[59,2046,75],{"fence":61},[89,2048,2049],{"encoding":91},"\\frac{\\partial J(w,b)}{\\partial b} = \\frac{1}{m} \\sum_{i=0}^{m-1} \\left(f_{w,b}(x^{(i)}) - y^{(i)}\\right)",[28,2051,2053,2157],{"className":2052,"ariaHidden":61},[96],[28,2054,2056,2059,2148,2151,2154],{"className":2055},[100],[28,2057],{"className":2058,"style":502},[104],[28,2060,2062,2065,2145],{"className":2061},[109],[28,2063],{"className":2064},[179,513],[28,2066,2068],{"className":2067},[390],[28,2069,2071,2137],{"className":2070},[122,123],[28,2072,2074,2134],{"className":2073},[127],[28,2075,2077,2094,2102],{"className":2076,"style":526},[131],[28,2078,2079,2082],{"style":529},[28,2080],{"className":2081,"style":533},[139],[28,2083,2085],{"className":2084},[144,145,146,147],[28,2086,2088,2091],{"className":2087},[109,147],[28,2089,397],{"className":2090,"style":543},[109,147],[28,2092,65],{"className":2093},[109,113,147],[28,2095,2096,2099],{"style":549},[28,2097],{"className":2098,"style":533},[139],[28,2100],{"className":2101,"style":557},[556],[28,2103,2104,2107],{"style":560},[28,2105],{"className":2106,"style":533},[139],[28,2108,2110],{"className":2109},[144,145,146,147],[28,2111,2113,2116,2119,2122,2125,2128,2131],{"className":2112},[109,147],[28,2114,397],{"className":2115,"style":543},[109,147],[28,2117,253],{"className":2118,"style":280},[109,113,147],[28,2120,69],{"className":2121},[179,147],[28,2123,57],{"className":2124,"style":154},[109,113,147],[28,2126,62],{"className":2127},[158,147],[28,2129,65],{"className":2130},[109,113,147],[28,2132,75],{"className":2133},[186,147],[28,2135,166],{"className":2136},[165],[28,2138,2140],{"className":2139},[127],[28,2141,2143],{"className":2142,"style":600},[131],[28,2144],{},[28,2146],{"className":2147},[186,513],[28,2149],{"className":2150,"style":191},[190],[28,2152,78],{"className":2153},[195],[28,2155],{"className":2156,"style":191},[190],[28,2158,2160,2163,2231,2234,2303,2306],{"className":2159},[100],[28,2161],{"className":2162,"style":1576},[104],[28,2164,2166,2169,2228],{"className":2165},[109],[28,2167],{"className":2168},[179,513],[28,2170,2172],{"className":2171},[390],[28,2173,2175,2220],{"className":2174},[122,123],[28,2176,2178,2217],{"className":2177},[127],[28,2179,2181,2195,2203],{"className":2180,"style":1595},[131],[28,2182,2183,2186],{"style":529},[28,2184],{"className":2185,"style":533},[139],[28,2187,2189],{"className":2188},[144,145,146,147],[28,2190,2192],{"className":2191},[109,147],[28,2193,1373],{"className":2194},[109,113,147],[28,2196,2197,2200],{"style":549},[28,2198],{"className":2199,"style":533},[139],[28,2201],{"className":2202,"style":557},[556],[28,2204,2205,2208],{"style":839},[28,2206],{"className":2207,"style":533},[139],[28,2209,2211],{"className":2210},[144,145,146,147],[28,2212,2214],{"className":2213},[109,147],[28,2215,1370],{"className":2216},[109,147],[28,2218,166],{"className":2219},[165],[28,2221,2223],{"className":2222},[127],[28,2224,2226],{"className":2225,"style":600},[131],[28,2227],{},[28,2229],{"className":2230},[186,513],[28,2232],{"className":2233,"style":293},[190],[28,2235,2237,2240],{"className":2236},[1652],[28,2238,1379],{"className":2239,"style":1658},[1652,1656,1657],[28,2241,2243],{"className":2242},[118],[28,2244,2246,2295],{"className":2245},[122,123],[28,2247,2249,2292],{"className":2248},[127],[28,2250,2252,2272],{"className":2251,"style":1671},[131],[28,2253,2254,2257],{"style":1674},[28,2255],{"className":2256,"style":140},[139],[28,2258,2260],{"className":2259},[144,145,146,147],[28,2261,2263,2266,2269],{"className":2262},[109,147],[28,2264,1384],{"className":2265},[109,113,147],[28,2267,78],{"className":2268},[195,147],[28,2270,1389],{"className":2271},[109,147],[28,2273,2274,2277],{"style":1695},[28,2275],{"className":2276,"style":140},[139],[28,2278,2280],{"className":2279},[144,145,146,147],[28,2281,2283,2286,2289],{"className":2282},[109,147],[28,2284,1373],{"className":2285},[109,113,147],[28,2287,385],{"className":2288},[219,147],[28,2290,1370],{"className":2291},[109,147],[28,2293,166],{"className":2294},[165],[28,2296,2298],{"className":2297},[127],[28,2299,2301],{"className":2300,"style":1723},[131],[28,2302],{},[28,2304],{"className":2305,"style":293},[190],[28,2307,2309,2315,2364,2367,2405,2408,2411,2414,2417,2455],{"className":2308},[1732],[28,2310,2312],{"className":2311,"style":1737},[179,1736],[28,2313,69],{"className":2314},[1741,1742],[28,2316,2318,2321],{"className":2317},[109],[28,2319,52],{"className":2320,"style":114},[109,113],[28,2322,2324],{"className":2323},[118],[28,2325,2327,2356],{"className":2326},[122,123],[28,2328,2330,2353],{"className":2329},[127],[28,2331,2333],{"className":2332,"style":132},[131],[28,2334,2335,2338],{"style":135},[28,2336],{"className":2337,"style":140},[139],[28,2339,2341],{"className":2340},[144,145,146,147],[28,2342,2344,2347,2350],{"className":2343},[109,147],[28,2345,57],{"className":2346,"style":154},[109,113,147],[28,2348,62],{"className":2349},[158,147],[28,2351,65],{"className":2352},[109,113,147],[28,2354,166],{"className":2355},[165],[28,2357,2359],{"className":2358},[127],[28,2360,2362],{"className":2361,"style":173},[131],[28,2363],{},[28,2365,69],{"className":2366},[179],[28,2368,2370,2373],{"className":2369},[109],[28,2371,72],{"className":2372},[109,113],[28,2374,2376],{"className":2375},[118],[28,2377,2379],{"className":2378},[122],[28,2380,2382],{"className":2381},[127],[28,2383,2385],{"className":2384,"style":1813},[131],[28,2386,2387,2390],{"style":1816},[28,2388],{"className":2389,"style":140},[139],[28,2391,2393],{"className":2392},[144,145,146,147],[28,2394,2396,2399,2402],{"className":2395},[109,147],[28,2397,69],{"className":2398},[179,147],[28,2400,1384],{"className":2401},[109,113,147],[28,2403,75],{"className":2404},[186,147],[28,2406,75],{"className":2407},[186],[28,2409],{"className":2410,"style":215},[190],[28,2412,385],{"className":2413},[219],[28,2415],{"className":2416,"style":215},[190],[28,2418,2420,2423],{"className":2419},[109],[28,2421,1437],{"className":2422,"style":1853},[109,113],[28,2424,2426],{"className":2425},[118],[28,2427,2429],{"className":2428},[122],[28,2430,2432],{"className":2431},[127],[28,2433,2435],{"className":2434,"style":1813},[131],[28,2436,2437,2440],{"style":1816},[28,2438],{"className":2439,"style":140},[139],[28,2441,2443],{"className":2442},[144,145,146,147],[28,2444,2446,2449,2452],{"className":2445},[109,147],[28,2447,69],{"className":2448},[179,147],[28,2450,1384],{"className":2451},[109,113,147],[28,2453,75],{"className":2454},[186,147],[28,2456,2458],{"className":2457,"style":1737},[186,1736],[28,2459,75],{"className":2460},[1741,1742],[11,2462,2463,2464,2492,2493,2567,2568,2596,2597,2625,2626,2654,2655,2683],{},"Notice the two are nearly twins: the one for ",[28,2465,2467,2480],{"className":2466},[31],[28,2468,2470],{"className":2469},[35],[37,2471,2472],{"xmlns":39},[41,2473,2474,2478],{},[44,2475,2476],{},[50,2477,57],{},[89,2479,57],{"encoding":91},[28,2481,2483],{"className":2482,"ariaHidden":61},[96],[28,2484,2486,2489],{"className":2485},[100],[28,2487],{"className":2488,"style":465},[104],[28,2490,57],{"className":2491,"style":154},[109,113]," has an ",[28,2494,2496,2520],{"className":2495},[31],[28,2497,2499],{"className":2498},[35],[37,2500,2501],{"xmlns":39},[41,2502,2503,2517],{},[44,2504,2505],{},[1417,2506,2507,2509],{},[50,2508,72],{},[44,2510,2511,2513,2515],{},[59,2512,69],{"stretchy":68},[50,2514,1384],{},[59,2516,75],{"stretchy":68},[89,2518,2519],{"encoding":91},"x^{(i)}",[28,2521,2523],{"className":2522,"ariaHidden":61},[96],[28,2524,2526,2529],{"className":2525},[100],[28,2527],{"className":2528,"style":1813},[104],[28,2530,2532,2535],{"className":2531},[109],[28,2533,72],{"className":2534},[109,113],[28,2536,2538],{"className":2537},[118],[28,2539,2541],{"className":2540},[122],[28,2542,2544],{"className":2543},[127],[28,2545,2547],{"className":2546,"style":1813},[131],[28,2548,2549,2552],{"style":1816},[28,2550],{"className":2551,"style":140},[139],[28,2553,2555],{"className":2554},[144,145,146,147],[28,2556,2558,2561,2564],{"className":2557},[109,147],[28,2559,69],{"className":2560},[179,147],[28,2562,1384],{"className":2563},[109,113,147],[28,2565,75],{"className":2566},[186,147]," multiplying the error, the one for ",[28,2569,2571,2584],{"className":2570},[31],[28,2572,2574],{"className":2573},[35],[37,2575,2576],{"xmlns":39},[41,2577,2578,2582],{},[44,2579,2580],{},[50,2581,65],{},[89,2583,65],{"encoding":91},[28,2585,2587],{"className":2586,"ariaHidden":61},[96],[28,2588,2590,2593],{"className":2589},[100],[28,2591],{"className":2592,"style":229},[104],[28,2594,65],{"className":2595},[109,113]," doesn't. That makes sense geometrically: changing ",[28,2598,2600,2613],{"className":2599},[31],[28,2601,2603],{"className":2602},[35],[37,2604,2605],{"xmlns":39},[41,2606,2607,2611],{},[44,2608,2609],{},[50,2610,57],{},[89,2612,57],{"encoding":91},[28,2614,2616],{"className":2615,"ariaHidden":61},[96],[28,2617,2619,2622],{"className":2618},[100],[28,2620],{"className":2621,"style":465},[104],[28,2623,57],{"className":2624,"style":154},[109,113]," affects examples with large ",[28,2627,2629,2642],{"className":2628},[31],[28,2630,2632],{"className":2631},[35],[37,2633,2634],{"xmlns":39},[41,2635,2636,2640],{},[44,2637,2638],{},[50,2639,72],{},[89,2641,72],{"encoding":91},[28,2643,2645],{"className":2644,"ariaHidden":61},[96],[28,2646,2648,2651],{"className":2647},[100],[28,2649],{"className":2650,"style":465},[104],[28,2652,72],{"className":2653},[109,113]," more strongly (the slope carries more weight farther from the origin), while ",[28,2656,2658,2671],{"className":2657},[31],[28,2659,2661],{"className":2660},[35],[37,2662,2663],{"xmlns":39},[41,2664,2665,2669],{},[44,2666,2667],{},[50,2668,65],{},[89,2670,65],{"encoding":91},[28,2672,2674],{"className":2673,"ariaHidden":61},[96],[28,2675,2677,2680],{"className":2676},[100],[28,2678],{"className":2679,"style":229},[104],[28,2681,65],{"className":2682},[109,113]," shifts the entire line the same amount for everyone.",[11,2685,2686,2687,2723,2724,2726],{},"And remember that \"2\" we put in ",[28,2688,2690,2707],{"className":2689},[31],[28,2691,2693],{"className":2692},[35],[37,2694,2695],{"xmlns":39},[41,2696,2697,2704],{},[44,2698,2699,2702],{},[1368,2700,2701],{},"2",[50,2703,1373],{},[89,2705,2706],{"encoding":91},"2m",[28,2708,2710],{"className":2709,"ariaHidden":61},[96],[28,2711,2713,2717,2720],{"className":2712},[100],[28,2714],{"className":2715,"style":2716},[104],"height:0.6444em;",[28,2718,2701],{"className":2719},[109],[28,2721,1373],{"className":2722},[109,113]," in ",[22,2725,749],{"href":236},", where I said it was just there to simplify the math later? Here's where it pays off: differentiating the squared term leaves behind a factor of 2 that cancels exactly with the 2 in the denominator. Without that 2 back there, these formulas here would carry a leftover 2.",[11,2728,2729,2732,2733,2761,2762,2790,2791,2819,2820,2848],{},[913,2730,2731],{},"Simultaneous updates matter."," You compute both derivatives first, using the current values of ",[28,2734,2736,2749],{"className":2735},[31],[28,2737,2739],{"className":2738},[35],[37,2740,2741],{"xmlns":39},[41,2742,2743,2747],{},[44,2744,2745],{},[50,2746,57],{},[89,2748,57],{"encoding":91},[28,2750,2752],{"className":2751,"ariaHidden":61},[96],[28,2753,2755,2758],{"className":2754},[100],[28,2756],{"className":2757,"style":465},[104],[28,2759,57],{"className":2760,"style":154},[109,113]," and ",[28,2763,2765,2778],{"className":2764},[31],[28,2766,2768],{"className":2767},[35],[37,2769,2770],{"xmlns":39},[41,2771,2772,2776],{},[44,2773,2774],{},[50,2775,65],{},[89,2777,65],{"encoding":91},[28,2779,2781],{"className":2780,"ariaHidden":61},[96],[28,2782,2784,2787],{"className":2783},[100],[28,2785],{"className":2786,"style":229},[104],[28,2788,65],{"className":2789},[109,113],", and only then swap both parameters at once. Using the new ",[28,2792,2794,2807],{"className":2793},[31],[28,2795,2797],{"className":2796},[35],[37,2798,2799],{"xmlns":39},[41,2800,2801,2805],{},[44,2802,2803],{},[50,2804,57],{},[89,2806,57],{"encoding":91},[28,2808,2810],{"className":2809,"ariaHidden":61},[96],[28,2811,2813,2816],{"className":2812},[100],[28,2814],{"className":2815,"style":465},[104],[28,2817,57],{"className":2818,"style":154},[109,113]," to compute ",[28,2821,2823,2836],{"className":2822},[31],[28,2824,2826],{"className":2825},[35],[37,2827,2828],{"xmlns":39},[41,2829,2830,2834],{},[44,2831,2832],{},[50,2833,65],{},[89,2835,65],{"encoding":91},[28,2837,2839],{"className":2838,"ariaHidden":61},[96],[28,2840,2842,2845],{"className":2841},[100],[28,2843],{"className":2844,"style":229},[104],[28,2846,65],{"className":2847},[109,113],"'s derivative is a classic mistake that changes the algorithm's behavior.",[740,2850,2852],{"id":2851},"putting-this-into-code","Putting this into code",[2854,2855,2860],"pre",{"className":2856,"code":2857,"language":2858,"meta":2859,"style":2859},"language-python shiki shiki-themes github-light github-dark","def compute_gradient(x, y, w, b):\n    \"\"\"\n    Computes the gradient of the cost function for linear regression.\n\n    Args:\n      x (ndarray (m,)) : input data, m examples\n      y (ndarray (m,)) : target values\n      w, b (scalar)    : model parameters\n\n    Returns:\n      dj_dw (scalar): partial derivative of the cost with respect to w\n      dj_db (scalar): partial derivative of the cost with respect to b\n    \"\"\"\n    m = x.shape[0]\n\n    dj_dw = 0\n    dj_db = 0\n\n    for i in range(m):\n        f_wb = w * x[i] + b\n        dj_dw_i = (f_wb - y[i]) * x[i]   # example i's contribution to dj_dw\n        dj_db_i = f_wb - y[i]            # example i's contribution to dj_db\n        dj_db += dj_db_i\n        dj_dw += dj_dw_i\n\n    dj_dw = dj_dw \u002F m\n    dj_db = dj_db \u002F m\n\n    return dj_dw, dj_db\n","python","",[2861,2862,2863,2870,2876,2882,2889,2895,2901,2907,2913,2918,2924,2930,2936,2941,2947,2952,2958,2964,2969,2975,2981,2987,2993,2999,3005,3010,3016,3022,3027],"code",{"__ignoreMap":2859},[28,2864,2867],{"class":2865,"line":2866},"line",1,[28,2868,2869],{},"def compute_gradient(x, y, w, b):\n",[28,2871,2873],{"class":2865,"line":2872},2,[28,2874,2875],{},"    \"\"\"\n",[28,2877,2879],{"class":2865,"line":2878},3,[28,2880,2881],{},"    Computes the gradient of the cost function for linear regression.\n",[28,2883,2885],{"class":2865,"line":2884},4,[28,2886,2888],{"emptyLinePlaceholder":2887},true,"\n",[28,2890,2892],{"class":2865,"line":2891},5,[28,2893,2894],{},"    Args:\n",[28,2896,2898],{"class":2865,"line":2897},6,[28,2899,2900],{},"      x (ndarray (m,)) : input data, m examples\n",[28,2902,2904],{"class":2865,"line":2903},7,[28,2905,2906],{},"      y (ndarray (m,)) : target values\n",[28,2908,2910],{"class":2865,"line":2909},8,[28,2911,2912],{},"      w, b (scalar)    : model parameters\n",[28,2914,2916],{"class":2865,"line":2915},9,[28,2917,2888],{"emptyLinePlaceholder":2887},[28,2919,2921],{"class":2865,"line":2920},10,[28,2922,2923],{},"    Returns:\n",[28,2925,2927],{"class":2865,"line":2926},11,[28,2928,2929],{},"      dj_dw (scalar): partial derivative of the cost with respect to w\n",[28,2931,2933],{"class":2865,"line":2932},12,[28,2934,2935],{},"      dj_db (scalar): partial derivative of the cost with respect to b\n",[28,2937,2939],{"class":2865,"line":2938},13,[28,2940,2875],{},[28,2942,2944],{"class":2865,"line":2943},14,[28,2945,2946],{},"    m = x.shape[0]\n",[28,2948,2950],{"class":2865,"line":2949},15,[28,2951,2888],{"emptyLinePlaceholder":2887},[28,2953,2955],{"class":2865,"line":2954},16,[28,2956,2957],{},"    dj_dw = 0\n",[28,2959,2961],{"class":2865,"line":2960},17,[28,2962,2963],{},"    dj_db = 0\n",[28,2965,2967],{"class":2865,"line":2966},18,[28,2968,2888],{"emptyLinePlaceholder":2887},[28,2970,2972],{"class":2865,"line":2971},19,[28,2973,2974],{},"    for i in range(m):\n",[28,2976,2978],{"class":2865,"line":2977},20,[28,2979,2980],{},"        f_wb = w * x[i] + b\n",[28,2982,2984],{"class":2865,"line":2983},21,[28,2985,2986],{},"        dj_dw_i = (f_wb - y[i]) * x[i]   # example i's contribution to dj_dw\n",[28,2988,2990],{"class":2865,"line":2989},22,[28,2991,2992],{},"        dj_db_i = f_wb - y[i]            # example i's contribution to dj_db\n",[28,2994,2996],{"class":2865,"line":2995},23,[28,2997,2998],{},"        dj_db += dj_db_i\n",[28,3000,3002],{"class":2865,"line":3001},24,[28,3003,3004],{},"        dj_dw += dj_dw_i\n",[28,3006,3008],{"class":2865,"line":3007},25,[28,3009,2888],{"emptyLinePlaceholder":2887},[28,3011,3013],{"class":2865,"line":3012},26,[28,3014,3015],{},"    dj_dw = dj_dw \u002F m\n",[28,3017,3019],{"class":2865,"line":3018},27,[28,3020,3021],{},"    dj_db = dj_db \u002F m\n",[28,3023,3025],{"class":2865,"line":3024},28,[28,3026,2888],{"emptyLinePlaceholder":2887},[28,3028,3030],{"class":2865,"line":3029},29,[28,3031,3032],{},"    return dj_dw, dj_db\n",[11,3034,3035],{},"And the main loop, which repeats the update until the iterations run out:",[2854,3037,3039],{"className":2856,"code":3038,"language":2858,"meta":2859,"style":2859},"def gradient_descent(x, y, w_in, b_in, alpha, num_iters, cost_function, gradient_function):\n    \"\"\"\n    Runs gradient descent to fit w and b.\n\n    Args:\n      x, y                : training data\n      w_in, b_in (scalar) : INITIAL parameter values\n      alpha (float)       : learning rate\n      num_iters (int)     : how many iterations to run\n      cost_function       : function to compute the cost\n      gradient_function   : function to compute the gradient\n\n    Returns:\n      w, b (scalar)    : parameters after training\n      J_history (list) : cost at every iteration\n    \"\"\"\n    J_history = []\n    w = w_in\n    b = b_in\n\n    for i in range(num_iters):\n        dj_dw, dj_db = gradient_function(x, y, w, b)\n\n        b = b - alpha * dj_db   # simultaneous update: both derivatives were\n        w = w - alpha * dj_dw   # already computed from the old values\n\n        J_history.append(cost_function(x, y, w, b))\n\n    return w, b, J_history\n",[2861,3040,3041,3046,3050,3055,3059,3063,3068,3073,3078,3083,3088,3093,3097,3101,3106,3111,3115,3120,3125,3130,3134,3139,3144,3148,3153,3158,3162,3167,3171],{"__ignoreMap":2859},[28,3042,3043],{"class":2865,"line":2866},[28,3044,3045],{},"def gradient_descent(x, y, w_in, b_in, alpha, num_iters, cost_function, gradient_function):\n",[28,3047,3048],{"class":2865,"line":2872},[28,3049,2875],{},[28,3051,3052],{"class":2865,"line":2878},[28,3053,3054],{},"    Runs gradient descent to fit w and b.\n",[28,3056,3057],{"class":2865,"line":2884},[28,3058,2888],{"emptyLinePlaceholder":2887},[28,3060,3061],{"class":2865,"line":2891},[28,3062,2894],{},[28,3064,3065],{"class":2865,"line":2897},[28,3066,3067],{},"      x, y                : training data\n",[28,3069,3070],{"class":2865,"line":2903},[28,3071,3072],{},"      w_in, b_in (scalar) : INITIAL parameter values\n",[28,3074,3075],{"class":2865,"line":2909},[28,3076,3077],{},"      alpha (float)       : learning rate\n",[28,3079,3080],{"class":2865,"line":2915},[28,3081,3082],{},"      num_iters (int)     : how many iterations to run\n",[28,3084,3085],{"class":2865,"line":2920},[28,3086,3087],{},"      cost_function       : function to compute the cost\n",[28,3089,3090],{"class":2865,"line":2926},[28,3091,3092],{},"      gradient_function   : function to compute the gradient\n",[28,3094,3095],{"class":2865,"line":2932},[28,3096,2888],{"emptyLinePlaceholder":2887},[28,3098,3099],{"class":2865,"line":2938},[28,3100,2923],{},[28,3102,3103],{"class":2865,"line":2943},[28,3104,3105],{},"      w, b (scalar)    : parameters after training\n",[28,3107,3108],{"class":2865,"line":2949},[28,3109,3110],{},"      J_history (list) : cost at every iteration\n",[28,3112,3113],{"class":2865,"line":2954},[28,3114,2875],{},[28,3116,3117],{"class":2865,"line":2960},[28,3118,3119],{},"    J_history = []\n",[28,3121,3122],{"class":2865,"line":2966},[28,3123,3124],{},"    w = w_in\n",[28,3126,3127],{"class":2865,"line":2971},[28,3128,3129],{},"    b = b_in\n",[28,3131,3132],{"class":2865,"line":2977},[28,3133,2888],{"emptyLinePlaceholder":2887},[28,3135,3136],{"class":2865,"line":2983},[28,3137,3138],{},"    for i in range(num_iters):\n",[28,3140,3141],{"class":2865,"line":2989},[28,3142,3143],{},"        dj_dw, dj_db = gradient_function(x, y, w, b)\n",[28,3145,3146],{"class":2865,"line":2995},[28,3147,2888],{"emptyLinePlaceholder":2887},[28,3149,3150],{"class":2865,"line":3001},[28,3151,3152],{},"        b = b - alpha * dj_db   # simultaneous update: both derivatives were\n",[28,3154,3155],{"class":2865,"line":3007},[28,3156,3157],{},"        w = w - alpha * dj_dw   # already computed from the old values\n",[28,3159,3160],{"class":2865,"line":3012},[28,3161,2888],{"emptyLinePlaceholder":2887},[28,3163,3164],{"class":2865,"line":3018},[28,3165,3166],{},"        J_history.append(cost_function(x, y, w, b))\n",[28,3168,3169],{"class":2865,"line":3024},[28,3170,2888],{"emptyLinePlaceholder":2887},[28,3172,3173],{"class":2865,"line":3029},[28,3174,3175],{},"    return w, b, J_history\n",[740,3177,3179],{"id":3178},"now-its-your-turn-but-with-the-algorithm-doing-the-work","Now it's your turn, but with the algorithm doing the work",[11,3181,3182],{},"No more dragging a slider until you land on the right value by hand. Below is the real algorithm running, with full control: pick a learning rate, take one step at a time or run a batch at once, and watch the little red dot walk downhill on its own, on the heatmap, on the 3D surface, or on the parabola.",[11,3184,3185,3186,238,3237,3288],{},"Start at ",[28,3187,3189,3207],{"className":3188},[31],[28,3190,3192],{"className":3191},[35],[37,3193,3194],{"xmlns":39},[41,3195,3196,3204],{},[44,3197,3198,3200,3202],{},[50,3199,57],{},[59,3201,78],{},[1368,3203,1389],{},[89,3205,3206],{"encoding":91},"w = 0",[28,3208,3210,3228],{"className":3209,"ariaHidden":61},[96],[28,3211,3213,3216,3219,3222,3225],{"className":3212},[100],[28,3214],{"className":3215,"style":465},[104],[28,3217,57],{"className":3218,"style":154},[109,113],[28,3220],{"className":3221,"style":191},[190],[28,3223,78],{"className":3224},[195],[28,3226],{"className":3227,"style":191},[190],[28,3229,3231,3234],{"className":3230},[100],[28,3232],{"className":3233,"style":2716},[104],[28,3235,1389],{"className":3236},[109],[28,3238,3240,3258],{"className":3239},[31],[28,3241,3243],{"className":3242},[35],[37,3244,3245],{"xmlns":39},[41,3246,3247,3255],{},[44,3248,3249,3251,3253],{},[50,3250,65],{},[59,3252,78],{},[1368,3254,1389],{},[89,3256,3257],{"encoding":91},"b = 0",[28,3259,3261,3279],{"className":3260,"ariaHidden":61},[96],[28,3262,3264,3267,3270,3273,3276],{"className":3263},[100],[28,3265],{"className":3266,"style":229},[104],[28,3268,65],{"className":3269},[109,113],[28,3271],{"className":3272,"style":191},[190],[28,3274,78],{"className":3275},[195],[28,3277],{"className":3278,"style":191},[190],[28,3280,3282,3285],{"className":3281},[100],[28,3283],{"className":3284,"style":2716},[104],[28,3286,1389],{"className":3287},[109]," (far from the answer) and click \"Rodar 2000\" a few times with the default alpha of 0.01. Notice how the cost drops fast at first and then slows down on its own, with no input from you.",[3290,3291],"gradient-descent-simulator",{":b-range":3292,":initial-b":1389,":initial-w":1389,":w-range":3293,"b-label":65,"w-label":57,":x-train":3294,":y-train":3295},"[-200, 400]","[-100, 500]","[1, 2]","[300, 500]",[740,3297,3299],{"id":3298},"when-the-learning-rate-is-too-big","When the learning rate is too big",[11,3301,3302,3303,2761,3331,3359],{},"Now click the alpha 0.8 preset (way bigger than the 0.01 that worked) and run a few steps. You'll watch ",[28,3304,3306,3319],{"className":3305},[31],[28,3307,3309],{"className":3308},[35],[37,3310,3311],{"xmlns":39},[41,3312,3313,3317],{},[44,3314,3315],{},[50,3316,57],{},[89,3318,57],{"encoding":91},[28,3320,3322],{"className":3321,"ariaHidden":61},[96],[28,3323,3325,3328],{"className":3324},[100],[28,3326],{"className":3327,"style":465},[104],[28,3329,57],{"className":3330,"style":154},[109,113],[28,3332,3334,3347],{"className":3333},[31],[28,3335,3337],{"className":3336},[35],[37,3338,3339],{"xmlns":39},[41,3340,3341,3345],{},[44,3342,3343],{},[50,3344,65],{},[89,3346,65],{"encoding":91},[28,3348,3350],{"className":3349,"ariaHidden":61},[96],[28,3351,3353,3356],{"className":3352},[100],[28,3354],{"className":3355,"style":229},[104],[28,3357,65],{"className":3358},[109,113]," become increasingly absurd, and instead of dropping, the cost climbs.",[11,3361,3362,3363,3367,3368,3396],{},"That's ",[358,3364,3366],{"definition":3365},"when gradient descent, instead of getting closer to the minimum, moves farther and farther away from it because the step is too big","divergence",", and the reason is easy to picture: the step size is proportional to the derivative. If ",[28,3369,3371,3384],{"className":3370},[31],[28,3372,3374],{"className":3373},[35],[37,3375,3376],{"xmlns":39},[41,3377,3378,3382],{},[44,3379,3380],{},[50,3381,388],{},[89,3383,886],{"encoding":91},[28,3385,3387],{"className":3386,"ariaHidden":61},[96],[28,3388,3390,3393],{"className":3389},[100],[28,3391],{"className":3392,"style":465},[104],[28,3394,388],{"className":3395,"style":506},[109,113]," is large, the step overshoots the bottom of the bowl and lands on the other side, higher up than it started. From that new spot, the derivative is even bigger (in magnitude) and flipped in sign, so the next step is even bigger in the opposite direction. It turns into a self-feeding loop that explodes, like pushing a swing harder and harder until it flips upside down.",[11,3398,3399,3400,3428],{},"In practice, if you're training a real model and see the cost climbing or bouncing back and forth without settling, the first thing I try is lowering ",[28,3401,3403,3416],{"className":3402},[31],[28,3404,3406],{"className":3405},[35],[37,3407,3408],{"xmlns":39},[41,3409,3410,3414],{},[44,3411,3412],{},[50,3413,388],{},[89,3415,886],{"encoding":91},[28,3417,3419],{"className":3418,"ariaHidden":61},[96],[28,3420,3422,3425],{"className":3421},[100],[28,3423],{"className":3424,"style":465},[104],[28,3426,388],{"className":3427,"style":506},[109,113]," (divide by 3 or by 10, for instance). The course suggests trying a sequence like 0.001, 0.003, 0.01, 0.03, 0.1 and comparing the cost curves until you find one that drops smoothly.",[922,3430,3431,3441],{},[925,3432,3433],{},[928,3434,3435,3438],{},[931,3436,3437],{"align":937},"Alpha",[931,3439,3440],{"align":933},"What happens (after 1000 steps, starting at w=0, b=0)",[1082,3442,3443,3451,3459,3467,3475,3483],{},[928,3444,3445,3448],{},[1087,3446,3447],{"align":937},"0.0001",[1087,3449,3450],{"align":933},"way too slow, barely left the starting point",[928,3452,3453,3456],{},[1087,3454,3455],{"align":937},"0.001",[1087,3457,3458],{"align":933},"still far from the target",[928,3460,3461,3464],{},[1087,3462,3463],{"align":937},"0.01",[1087,3465,3466],{"align":933},"good balance, this is what we used above",[928,3468,3469,3472],{},[1087,3470,3471],{"align":937},"0.1",[1087,3473,3474],{"align":933},"converges fast",[928,3476,3477,3480],{},[1087,3478,3479],{"align":937},"0.3",[1087,3481,3482],{"align":933},"still converges, but already close to the edge",[928,3484,3485,3488],{},[1087,3486,3487],{"align":937},"0.8",[1087,3489,3490],{"align":933},"diverges",[11,3492,3493],{},"Try these values yourself in the simulator above and compare against the table.",[740,3495,3497],{"id":3496},"bonus-not-every-bowl-is-this-well-behaved","Bonus: not every bowl is this well-behaved",[11,3499,3500,3501,3504],{},"Every cost surface we've drawn so far looks like the same soup bowl, because it comes from squared error, and that guarantees convexity (",[22,3502,3503],{"href":236},"last post","). But gradient descent doesn't only live on nicely-behaved bowls. Here are two classics that anyone studying optimization runs into sooner or later, just so you see that the trouble we hit with a big alpha is only the tip of the iceberg.",[3506,3507,3509],"h3",{"id":3508},"rosenbrocks-banana-valley","Rosenbrock's banana valley",[11,3511,3512],{},"This one's practically the standard stress test of the field, it even has its own name: the Rosenbrock function.",[11,3514,3515],{},[28,3516,3518,3581],{"className":3517},[31],[28,3519,3521],{"className":3520},[35],[37,3522,3523],{"xmlns":39},[41,3524,3525,3578],{},[44,3526,3527,3529,3531,3533,3535,3537,3539,3541,3543,3545,3547,3549,3555,3557,3560,3562,3564,3566,3572],{},[50,3528,52],{},[59,3530,69],{"stretchy":68},[50,3532,57],{},[59,3534,62],{"separator":61},[50,3536,65],{},[59,3538,75],{"stretchy":68},[59,3540,78],{},[59,3542,69],{"stretchy":68},[1368,3544,1370],{},[59,3546,385],{},[50,3548,57],{},[1417,3550,3551,3553],{},[59,3552,75],{"stretchy":68},[1368,3554,2701],{},[59,3556,85],{},[1368,3558,3559],{},"100",[59,3561,69],{"stretchy":68},[50,3563,65],{},[59,3565,385],{},[1417,3567,3568,3570],{},[50,3569,57],{},[1368,3571,2701],{},[1417,3573,3574,3576],{},[59,3575,75],{"stretchy":68},[1368,3577,2701],{},[89,3579,3580],{"encoding":91},"f(w,b) = (1-w)^2 + 100(b - w^2)^2",[28,3582,3584,3620,3641,3690,3714],{"className":3583,"ariaHidden":61},[96],[28,3585,3587,3590,3593,3596,3599,3602,3605,3608,3611,3614,3617],{"className":3586},[100],[28,3588],{"className":3589,"style":276},[104],[28,3591,52],{"className":3592,"style":114},[109,113],[28,3594,69],{"className":3595},[179],[28,3597,57],{"className":3598,"style":154},[109,113],[28,3600,62],{"className":3601},[158],[28,3603],{"className":3604,"style":293},[190],[28,3606,65],{"className":3607},[109,113],[28,3609,75],{"className":3610},[186],[28,3612],{"className":3613,"style":191},[190],[28,3615,78],{"className":3616},[195],[28,3618],{"className":3619,"style":191},[190],[28,3621,3623,3626,3629,3632,3635,3638],{"className":3622},[100],[28,3624],{"className":3625,"style":276},[104],[28,3627,69],{"className":3628},[179],[28,3630,1370],{"className":3631},[109],[28,3633],{"className":3634,"style":215},[190],[28,3636,385],{"className":3637},[219],[28,3639],{"className":3640,"style":215},[190],[28,3642,3644,3648,3651,3681,3684,3687],{"className":3643},[100],[28,3645],{"className":3646,"style":3647},[104],"height:1.0641em;vertical-align:-0.25em;",[28,3649,57],{"className":3650,"style":154},[109,113],[28,3652,3654,3657],{"className":3653},[186],[28,3655,75],{"className":3656},[186],[28,3658,3660],{"className":3659},[118],[28,3661,3663],{"className":3662},[122],[28,3664,3666],{"className":3665},[127],[28,3667,3670],{"className":3668,"style":3669},[131],"height:0.8141em;",[28,3671,3672,3675],{"style":1816},[28,3673],{"className":3674,"style":140},[139],[28,3676,3678],{"className":3677},[144,145,146,147],[28,3679,2701],{"className":3680},[109,147],[28,3682],{"className":3683,"style":215},[190],[28,3685,85],{"className":3686},[219],[28,3688],{"className":3689,"style":215},[190],[28,3691,3693,3696,3699,3702,3705,3708,3711],{"className":3692},[100],[28,3694],{"className":3695,"style":276},[104],[28,3697,3559],{"className":3698},[109],[28,3700,69],{"className":3701},[179],[28,3703,65],{"className":3704},[109,113],[28,3706],{"className":3707,"style":215},[190],[28,3709,385],{"className":3710},[219],[28,3712],{"className":3713,"style":215},[190],[28,3715,3717,3720,3749],{"className":3716},[100],[28,3718],{"className":3719,"style":3647},[104],[28,3721,3723,3726],{"className":3722},[109],[28,3724,57],{"className":3725,"style":154},[109,113],[28,3727,3729],{"className":3728},[118],[28,3730,3732],{"className":3731},[122],[28,3733,3735],{"className":3734},[127],[28,3736,3738],{"className":3737,"style":3669},[131],[28,3739,3740,3743],{"style":1816},[28,3741],{"className":3742,"style":140},[139],[28,3744,3746],{"className":3745},[144,145,146,147],[28,3747,2701],{"className":3748},[109,147],[28,3750,3752,3755],{"className":3751},[186],[28,3753,75],{"className":3754},[186],[28,3756,3758],{"className":3757},[118],[28,3759,3761],{"className":3760},[122],[28,3762,3764],{"className":3763},[127],[28,3765,3767],{"className":3766,"style":3669},[131],[28,3768,3769,3772],{"style":1816},[28,3770],{"className":3771,"style":140},[139],[28,3773,3775],{"className":3774},[144,145,146,147],[28,3776,2701],{"className":3777},[109,147],[11,3779,3780,3781,3833],{},"The global minimum sits at ",[28,3782,3784,3806],{"className":3783},[31],[28,3785,3787],{"className":3786},[35],[37,3788,3789],{"xmlns":39},[41,3790,3791,3803],{},[44,3792,3793,3795,3797,3799,3801],{},[59,3794,69],{"stretchy":68},[1368,3796,1370],{},[59,3798,62],{"separator":61},[1368,3800,1370],{},[59,3802,75],{"stretchy":68},[89,3804,3805],{"encoding":91},"(1,1)",[28,3807,3809],{"className":3808,"ariaHidden":61},[96],[28,3810,3812,3815,3818,3821,3824,3827,3830],{"className":3811},[100],[28,3813],{"className":3814,"style":276},[104],[28,3816,69],{"className":3817},[179],[28,3819,1370],{"className":3820},[109],[28,3822,62],{"className":3823},[158],[28,3825],{"className":3826,"style":293},[190],[28,3828,1370],{"className":3829},[109],[28,3831,75],{"className":3832},[186],", cost zero, but look at the shape:",[3835,3836],"cost-surface3d",{":b-max":3837,":b-min":3838,":w-max":2701,":w-min":3839,"fn":3840},"3","-1","-2","rosenbrock",[11,3842,3843],{},"It's not a round bowl, it's a curved valley, shaped like a banana. That's a real problem for gradient descent: the direction that descends fastest almost never points toward the bottom of the valley, it points toward the nearest wall. The algorithm ends up bouncing from one side of the banana to the other, barely making progress with each zig-zag, even near the bottom.",[11,3845,3846],{},"Try alpha 0.01 here (the same value that worked smoothly in our housing example) and watch what happens:",[3290,3848],{":b-range":3849,":initial-b":1370,":initial-w":3838,":w-range":3850,"b-label":65,"w-label":57,":alpha-presets":3851,":initial-alpha":3852,"fn":3840},"[-1, 3]","[-2, 2]","[0.0001, 0.0005, 0.001, 0.002, 0.005, 0.01]","0.002",[11,3854,3855,3856,362],{},"Alpha 0.01 diverges almost instantly here, the same value that was the \"good balance\" above. There's no universal alpha, it depends entirely on the shape of the surface you're descending. Drop it to 0.002 and click \"Rodar 2000\" a few times: now the red dot snakes slowly through the valley until it lands near ",[28,3857,3859,3880],{"className":3858},[31],[28,3860,3862],{"className":3861},[35],[37,3863,3864],{"xmlns":39},[41,3865,3866,3878],{},[44,3867,3868,3870,3872,3874,3876],{},[59,3869,69],{"stretchy":68},[1368,3871,1370],{},[59,3873,62],{"separator":61},[1368,3875,1370],{},[59,3877,75],{"stretchy":68},[89,3879,3805],{"encoding":91},[28,3881,3883],{"className":3882,"ariaHidden":61},[96],[28,3884,3886,3889,3892,3895,3898,3901,3904],{"className":3885},[100],[28,3887],{"className":3888,"style":276},[104],[28,3890,69],{"className":3891},[179],[28,3893,1370],{"className":3894},[109],[28,3896,62],{"className":3897},[158],[28,3899],{"className":3900,"style":293},[190],[28,3902,1370],{"className":3903},[109],[28,3905,75],{"className":3906},[186],[3506,3908,3910],{"id":3909},"the-saddle-point","The saddle point",[11,3912,3913],{},[28,3914,3916,3956],{"className":3915},[31],[28,3917,3919],{"className":3918},[35],[37,3920,3921],{"xmlns":39},[41,3922,3923,3953],{},[44,3924,3925,3927,3929,3931,3933,3935,3937,3939,3945,3947],{},[50,3926,52],{},[59,3928,69],{"stretchy":68},[50,3930,57],{},[59,3932,62],{"separator":61},[50,3934,65],{},[59,3936,75],{"stretchy":68},[59,3938,78],{},[1417,3940,3941,3943],{},[50,3942,57],{},[1368,3944,2701],{},[59,3946,385],{},[1417,3948,3949,3951],{},[50,3950,65],{},[1368,3952,2701],{},[89,3954,3955],{"encoding":91},"f(w,b) = w^2 - b^2",[28,3957,3959,3995,4040],{"className":3958,"ariaHidden":61},[96],[28,3960,3962,3965,3968,3971,3974,3977,3980,3983,3986,3989,3992],{"className":3961},[100],[28,3963],{"className":3964,"style":276},[104],[28,3966,52],{"className":3967,"style":114},[109,113],[28,3969,69],{"className":3970},[179],[28,3972,57],{"className":3973,"style":154},[109,113],[28,3975,62],{"className":3976},[158],[28,3978],{"className":3979,"style":293},[190],[28,3981,65],{"className":3982},[109,113],[28,3984,75],{"className":3985},[186],[28,3987],{"className":3988,"style":191},[190],[28,3990,78],{"className":3991},[195],[28,3993],{"className":3994,"style":191},[190],[28,3996,3998,4002,4031,4034,4037],{"className":3997},[100],[28,3999],{"className":4000,"style":4001},[104],"height:0.8974em;vertical-align:-0.0833em;",[28,4003,4005,4008],{"className":4004},[109],[28,4006,57],{"className":4007,"style":154},[109,113],[28,4009,4011],{"className":4010},[118],[28,4012,4014],{"className":4013},[122],[28,4015,4017],{"className":4016},[127],[28,4018,4020],{"className":4019,"style":3669},[131],[28,4021,4022,4025],{"style":1816},[28,4023],{"className":4024,"style":140},[139],[28,4026,4028],{"className":4027},[144,145,146,147],[28,4029,2701],{"className":4030},[109,147],[28,4032],{"className":4033,"style":215},[190],[28,4035,385],{"className":4036},[219],[28,4038],{"className":4039,"style":215},[190],[28,4041,4043,4046],{"className":4042},[100],[28,4044],{"className":4045,"style":3669},[104],[28,4047,4049,4052],{"className":4048},[109],[28,4050,65],{"className":4051},[109,113],[28,4053,4055],{"className":4054},[118],[28,4056,4058],{"className":4057},[122],[28,4059,4061],{"className":4060},[127],[28,4062,4064],{"className":4063,"style":3669},[131],[28,4065,4066,4069],{"style":1816},[28,4067],{"className":4068,"style":140},[139],[28,4070,4072],{"className":4071},[144,145,146,147],[28,4073,2701],{"className":4074},[109,147],[3835,4076],{":b-max":2701,":b-min":3839,":w-max":2701,":w-min":3839,"fn":4077,":marker-b":1389,":marker-w":1389},"saddle",[11,4079,4080,4081,4109,4110,4138,4139,4143],{},"Rotate this one slowly. It's a minimum if you only look along the ",[28,4082,4084,4097],{"className":4083},[31],[28,4085,4087],{"className":4086},[35],[37,4088,4089],{"xmlns":39},[41,4090,4091,4095],{},[44,4092,4093],{},[50,4094,57],{},[89,4096,57],{"encoding":91},[28,4098,4100],{"className":4099,"ariaHidden":61},[96],[28,4101,4103,4106],{"className":4102},[100],[28,4104],{"className":4105,"style":465},[104],[28,4107,57],{"className":4108,"style":154},[109,113]," axis, and a maximum if you only look along the ",[28,4111,4113,4126],{"className":4112},[31],[28,4114,4116],{"className":4115},[35],[37,4117,4118],{"xmlns":39},[41,4119,4120,4124],{},[44,4121,4122],{},[50,4123,65],{},[89,4125,65],{"encoding":91},[28,4127,4129],{"className":4128,"ariaHidden":61},[96],[28,4130,4132,4135],{"className":4131},[100],[28,4133],{"className":4134,"style":229},[104],[28,4136,65],{"className":4137},[109,113]," axis, at the same time, at the same point. That's called a ",[358,4140,4142],{"definition":4141},"a point where the gradient is zero, but that's neither a minimum nor a maximum, it's a minimum in one direction and a maximum in another at the same time, like the middle of a horse saddle","saddle point",", the red dot in the center marks exactly that.",[11,4145,4146,4147,4175],{},"Remember the rule \"when the derivative hits zero, the algorithm stops on its own\"? Well, right at this point it hits zero just the same. If gradient descent landed exactly there, it would think it was done, without being done at all, just balanced on top of a mountain pass. Any tiny nudge away from that exact center, and it slides downhill in whichever direction actually descends (the ",[28,4148,4150,4163],{"className":4149},[31],[28,4151,4153],{"className":4152},[35],[37,4154,4155],{"xmlns":39},[41,4156,4157,4161],{},[44,4158,4159],{},[50,4160,65],{},[89,4162,65],{"encoding":91},[28,4164,4166],{"className":4165,"ariaHidden":61},[96],[28,4167,4169,4172],{"className":4168},[100],[28,4170],{"className":4171,"style":229},[104],[28,4173,65],{"className":4174},[109,113]," axis), away from any real minimum.",[11,4177,4178],{},"This never happens on our own regression cost (it's always convex, no hidden saddle anywhere), but this exact kind of surface shows up constantly in more complex models, like neural networks. Keep that name in your back pocket, it comes back.",[740,4180,4182],{"id":4181},"wrapping-up-the-trilogy","Wrapping up the trilogy",[922,4184,4185,4195],{},[925,4186,4187],{},[928,4188,4189,4192],{},[931,4190,4191],{"align":933},"Post",[931,4193,4194],{"align":933},"What's done",[1082,4196,4197,4355,4421],{},[928,4198,4199,4204],{},[1087,4200,4201],{"align":933},[22,4202,4203],{"href":24},"Lab 02",[1087,4205,4206,4207],{"align":933},"the model, ",[28,4208,4210,4249],{"className":4209},[31],[28,4211,4213],{"className":4212},[35],[37,4214,4215],{"xmlns":39},[41,4216,4217,4247],{},[44,4218,4219,4231,4233,4235,4237,4239,4241,4243,4245],{},[47,4220,4221,4223],{},[50,4222,52],{},[44,4224,4225,4227,4229],{},[50,4226,57],{},[59,4228,62],{"separator":61},[50,4230,65],{},[59,4232,69],{"stretchy":68},[50,4234,72],{},[59,4236,75],{"stretchy":68},[59,4238,78],{},[50,4240,57],{},[50,4242,72],{},[59,4244,85],{},[50,4246,65],{},[89,4248,92],{"encoding":91},[28,4250,4252,4325,4346],{"className":4251,"ariaHidden":61},[96],[28,4253,4255,4258,4307,4310,4313,4316,4319,4322],{"className":4254},[100],[28,4256],{"className":4257,"style":105},[104],[28,4259,4261,4264],{"className":4260},[109],[28,4262,52],{"className":4263,"style":114},[109,113],[28,4265,4267],{"className":4266},[118],[28,4268,4270,4299],{"className":4269},[122,123],[28,4271,4273,4296],{"className":4272},[127],[28,4274,4276],{"className":4275,"style":132},[131],[28,4277,4278,4281],{"style":135},[28,4279],{"className":4280,"style":140},[139],[28,4282,4284],{"className":4283},[144,145,146,147],[28,4285,4287,4290,4293],{"className":4286},[109,147],[28,4288,57],{"className":4289,"style":154},[109,113,147],[28,4291,62],{"className":4292},[158,147],[28,4294,65],{"className":4295},[109,113,147],[28,4297,166],{"className":4298},[165],[28,4300,4302],{"className":4301},[127],[28,4303,4305],{"className":4304,"style":173},[131],[28,4306],{},[28,4308,69],{"className":4309},[179],[28,4311,72],{"className":4312},[109,113],[28,4314,75],{"className":4315},[186],[28,4317],{"className":4318,"style":191},[190],[28,4320,78],{"className":4321},[195],[28,4323],{"className":4324,"style":191},[190],[28,4326,4328,4331,4334,4337,4340,4343],{"className":4327},[100],[28,4329],{"className":4330,"style":205},[104],[28,4332,57],{"className":4333,"style":154},[109,113],[28,4335,72],{"className":4336},[109,113],[28,4338],{"className":4339,"style":215},[190],[28,4341,85],{"className":4342},[219],[28,4344],{"className":4345,"style":215},[190],[28,4347,4349,4352],{"className":4348},[100],[28,4350],{"className":4351,"style":229},[104],[28,4353,65],{"className":4354},[109,113],[928,4356,4357,4362],{},[1087,4358,4359],{"align":933},[22,4360,4361],{"href":236},"Lab 03",[1087,4363,4364,4365],{"align":933},"the error measurement, ",[28,4366,4368,4391],{"className":4367},[31],[28,4369,4371],{"className":4370},[35],[37,4372,4373],{"xmlns":39},[41,4374,4375,4389],{},[44,4376,4377,4379,4381,4383,4385,4387],{},[50,4378,253],{},[59,4380,69],{"stretchy":68},[50,4382,57],{},[59,4384,62],{"separator":61},[50,4386,65],{},[59,4388,75],{"stretchy":68},[89,4390,266],{"encoding":91},[28,4392,4394],{"className":4393,"ariaHidden":61},[96],[28,4395,4397,4400,4403,4406,4409,4412,4415,4418],{"className":4396},[100],[28,4398],{"className":4399,"style":276},[104],[28,4401,253],{"className":4402,"style":280},[109,113],[28,4404,69],{"className":4405},[179],[28,4407,57],{"className":4408,"style":154},[109,113],[28,4410,62],{"className":4411},[158],[28,4413],{"className":4414,"style":293},[190],[28,4416,65],{"className":4417},[109,113],[28,4419,75],{"className":4420},[186],[928,4422,4423,4426],{},[1087,4424,4425],{"align":933},"Lab 04 (this one)",[1087,4427,4428],{"align":933},"the algorithm that minimizes that error on its own, gradient descent",[11,4430,4431],{},"Three takeaways:",[4433,4434,4435,4442,4448],"ol",{},[4436,4437,4438,4441],"li",{},[913,4439,4440],{},"The gradient points toward where the cost increases",", so the algorithm always walks the opposite way.",[4436,4443,4444,4447],{},[913,4445,4446],{},"The update is simultaneous",", compute both derivatives first, swap the parameters after.",[4436,4449,4450,4453],{},[913,4451,4452],{},"The learning rate is the single most sensitive knob you'll touch",": too small is slow, too big diverges.",[11,4455,4456,4459,4460,4463,4464,4468],{},[913,4457,4458],{},"Coming up next:"," everything you've seen so far used one feature (the house's size). In Week 2 of the course, the model gains several features at once, and computing with a ",[2861,4461,4462],{},"for"," loop stops cutting it. Before touching a model with multiple features, ",[22,4465,4467],{"href":4466},"\u002Fen\u002Fplaylists\u002Fmachine-learning-specialization\u002Fw2-lab01-numpy-vectorization","the next post"," covers the tool that makes it viable: NumPy and vectorization.",[740,4470,4472],{"id":4471},"practical-application","Practical application",[11,4474,4475,4476,4482,4483,2761,4485,362],{},"Same real housing dataset from the last two posts (",[22,4477,4481],{"href":4478,"rel":4479},"https:\u002F\u002Fwww.kaggle.com\u002Fdatasets\u002Fdenkuznetz\u002Fhousing-prices-regression",[4480],"nofollow","Housing Prices Regression, Kaggle","), same scale (size in \"hundreds of sqft\", price in \"thousands of dollars\"). Let's watch the algorithm find, on its own, in real data, the fit you went hunting for by hand in posts ",[22,4484,1370],{"href":24},[22,4486,2701],{"href":236},[2854,4488,4490],{"className":2856,"code":4489,"language":2858,"meta":2859,"style":2859},"w, b, J_hist = gradient_descent(\n    x_sqft, y_price,          # the 50 real houses\n    w_in=0, b_in=0,           # starting from zero, same as the toy example\n    alpha=0.01, num_iters=4000,\n    cost_function=compute_cost, gradient_function=compute_gradient)\n\nprint(f\"(w, b) found: ({w:.1f}, {b:.1f})\")\n",[2861,4491,4492,4497,4502,4507,4512,4517,4521],{"__ignoreMap":2859},[28,4493,4494],{"class":2865,"line":2866},[28,4495,4496],{},"w, b, J_hist = gradient_descent(\n",[28,4498,4499],{"class":2865,"line":2872},[28,4500,4501],{},"    x_sqft, y_price,          # the 50 real houses\n",[28,4503,4504],{"class":2865,"line":2878},[28,4505,4506],{},"    w_in=0, b_in=0,           # starting from zero, same as the toy example\n",[28,4508,4509],{"class":2865,"line":2884},[28,4510,4511],{},"    alpha=0.01, num_iters=4000,\n",[28,4513,4514],{"class":2865,"line":2891},[28,4515,4516],{},"    cost_function=compute_cost, gradient_function=compute_gradient)\n",[28,4518,4519],{"class":2865,"line":2897},[28,4520,2888],{"emptyLinePlaceholder":2887},[28,4522,4523],{"class":2865,"line":2903},[28,4524,4525],{},"print(f\"(w, b) found: ({w:.1f}, {b:.1f})\")\n",[4527,4528,4529],"blockquote",{},[11,4530,4531,26,4534],{},[913,4532,4533],{},"Output:",[2861,4535,4536],{},"(w, b) found: (116.5, 398.3)",[11,4538,4539],{},"Matches almost exactly the fit cited in the last two posts. Click \"Rodar 2000\" twice on the simulator below and watch it live:",[4541,4542],"housing-gradient-descent-simulator",{},[11,4544,4545,4546,2761,4574,4602],{},"Notice convergence here is quite a bit slower than in the 2-point toy example, even already using the same small scale as before. That happens because ",[28,4547,4549,4562],{"className":4548},[31],[28,4550,4552],{"className":4551},[35],[37,4553,4554],{"xmlns":39},[41,4555,4556,4560],{},[44,4557,4558],{},[50,4559,57],{},[89,4561,57],{"encoding":91},[28,4563,4565],{"className":4564,"ariaHidden":61},[96],[28,4566,4568,4571],{"className":4567},[100],[28,4569],{"className":4570,"style":465},[104],[28,4572,57],{"className":4573,"style":154},[109,113],[28,4575,4577,4590],{"className":4576},[31],[28,4578,4580],{"className":4579},[35],[37,4581,4582],{"xmlns":39},[41,4583,4584,4588],{},[44,4585,4586],{},[50,4587,65],{},[89,4589,65],{"encoding":91},[28,4591,4593],{"className":4592,"ariaHidden":61},[96],[28,4594,4596,4599],{"className":4595},[100],[28,4597],{"className":4598,"style":229},[104],[28,4600,65],{"className":4601},[109,113]," still live on pretty different ranges from each other (one goes up to 300, the other up to 600), which stretches the cost valley. This is exactly the kind of situation feature scaling, later in the course, fixes for good.",[4604,4605,4606],"style",{},"html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":2859,"searchDepth":2872,"depth":2872,"links":4608},[4609,4610,4611,4612,4613,4614,4615,4619,4620],{"id":742,"depth":2872,"text":743},{"id":907,"depth":2872,"text":908},{"id":1320,"depth":2872,"text":1321},{"id":2851,"depth":2872,"text":2852},{"id":3178,"depth":2872,"text":3179},{"id":3298,"depth":2872,"text":3299},{"id":3496,"depth":2872,"text":3497,"children":4616},[4617,4618],{"id":3508,"depth":2878,"text":3509},{"id":3909,"depth":2878,"text":3910},{"id":4181,"depth":2872,"text":4182},{"id":4471,"depth":2872,"text":4472},null,"2026-08-18","The algorithm that walks itself down to the bottom of the cost bowl: the math behind gradient descent, what happens when the learning rate is too big, and why it stops on its own near the minimum.","md",{},"\u002Fen\u002Fplaylists\u002Fmachine-learning-specialization\u002Flab04-gradient-descent","machine-learning-specialization",{"title":6,"description":4623},"published","en\u002Fplaylists\u002Fmachine-learning-specialization\u002Flab04-gradient-descent",[4632,4633,4634],"gradient-descent","optimization","fundamentals","hxYODkFSVzZOMg12Zsd-Rt3eVggi00IEWJ5kpaCr-B4",1787338984721]