[{"data":1,"prerenderedAt":3681},["ShallowReactive",2],{"lang-switch-post-\u002Fen\u002Fplaylists\u002Fmachine-learning-specialization\u002Ffeature-engineering-pitfalls":3,"post-en-machine-learning-specialization-feature-engineering-pitfalls":4},"\u002Fplaylists\u002Fmachine-learning-specialization\u002Ffeature-engineering-pitfalls",{"id":5,"title":6,"body":7,"cover":3664,"date":3665,"description":3666,"extension":3667,"meta":3668,"navigation":3669,"order":3670,"path":3671,"playlist":3672,"seo":3673,"status":3674,"stem":3675,"tags":3676,"__hash__":3680},"posts\u002Fen\u002Fplaylists\u002Fmachine-learning-specialization\u002Ffeature-engineering-pitfalls.md","Just a Little Extra: Feature Engineering Pitfalls",{"type":8,"value":9,"toc":3645},"minimark",[10,20,25,347,892,896,1117,1177,1180,1263,1273,1337,1345,1349,1359,1363,1488,1839,1917,1921,1962,2297,2511,2643,2765,2858,2889,2898,2930,2934,2937,2942,2963,2981,2985,2996,3493,3497,3556,3565,3569,3584,3593,3597,3600],[11,12,13,14,19],"p",{},"This post doesn't come from any required section of the lab, it's what was left over after I split off the essentials of feature engineering and overfitting into the main post. Interestingly, this is the first time I actually get to show the fix for the problem ",[15,16,18],"a",{"href":17},"\u002Fen\u002Fplaylists\u002Fmachine-learning-specialization\u002Fw2-lab04-feature-engineering","the previous post"," only diagnosed.",[21,22,24],"h2",{"id":23},"i-normalize-after-creating-the-feature-not-before","I normalize after creating the feature, not before",[11,26,27,28,74,75,280,281,342,343,346],{},"Order matters. If I normalized ",[29,30,33,55],"span",{"className":31},[32],"katex",[29,34,37],{"className":35},[36],"katex-mathml",[38,39,41],"math",{"xmlns":40},"http:\u002F\u002Fwww.w3.org\u002F1998\u002FMath\u002FMathML",[42,43,44,51],"semantics",{},[45,46,47],"mrow",{},[48,49,50],"mi",{},"x",[52,53,50],"annotation",{"encoding":54},"application\u002Fx-tex",[29,56,60],{"className":57,"ariaHidden":59},[58],"katex-html","true",[29,61,64,69],{"className":62},[63],"base",[29,65],{"className":66,"style":68},[67],"strut","height:0.4306em;",[29,70,50],{"className":71},[72,73],"mord","mathnormal"," first and only then squared it, I'd get ",[29,76,78,122],{"className":77},[32],[29,79,81],{"className":80},[36],[38,82,83],{"xmlns":40},[42,84,85,119],{},[45,86,87],{},[88,89,90,115],"msup",{},[45,91,92,96,112],{},[93,94,95],"mo",{"fence":59},"(",[97,98,99,109],"mfrac",{},[45,100,101,103,106],{},[48,102,50],{},[93,104,105],{},"−",[48,107,108],{},"μ",[48,110,111],{},"σ",[93,113,114],{"fence":59},")",[116,117,118],"mn",{},"2",[52,120,121],{"encoding":54},"\\left(\\frac{x-\\mu}{\\sigma}\\right)^2",[29,123,125],{"className":124,"ariaHidden":59},[58],[29,126,128,132],{"className":127},[63],[29,129],{"className":130,"style":131},[67],"height:1.4085em;vertical-align:-0.35em;",[29,133,136,253],{"className":134},[135],"minner",[29,137,139,150,247],{"className":138},[135],[29,140,145],{"className":141,"style":144},[142,143],"mopen","delimcenter","top:0em;",[29,146,95],{"className":147},[148,149],"delimsizing","size1",[29,151,153,157,243],{"className":152},[72],[29,154],{"className":155},[142,156],"nulldelimiter",[29,158,160],{"className":159},[97],[29,161,165,234],{"className":162},[163,164],"vlist-t","vlist-t2",[29,166,169,229],{"className":167},[168],"vlist-r",[29,170,174,196,207],{"className":171,"style":173},[172],"vlist","height:0.8544em;",[29,175,177,182],{"style":176},"top:-2.655em;",[29,178],{"className":179,"style":181},[180],"pstrut","height:3em;",[29,183,189],{"className":184},[185,186,187,188],"sizing","reset-size6","size3","mtight",[29,190,192],{"className":191},[72,188],[29,193,111],{"className":194,"style":195},[72,73,188],"margin-right:0.0359em;",[29,197,199,202],{"style":198},"top:-3.23em;",[29,200],{"className":201,"style":181},[180],[29,203],{"className":204,"style":206},[205],"frac-line","border-bottom-width:0.04em;",[29,208,210,213],{"style":209},"top:-3.4461em;",[29,211],{"className":212,"style":181},[180],[29,214,216],{"className":215},[185,186,187,188],[29,217,219,222,226],{"className":218},[72,188],[29,220,50],{"className":221},[72,73,188],[29,223,105],{"className":224},[225,188],"mbin",[29,227,108],{"className":228},[72,73,188],[29,230,233],{"className":231},[232],"vlist-s","​",[29,235,237],{"className":236},[168],[29,238,241],{"className":239,"style":240},[172],"height:0.345em;",[29,242],{},[29,244],{"className":245},[246,156],"mclose",[29,248,250],{"className":249,"style":144},[246,143],[29,251,114],{"className":252},[148,149],[29,254,257],{"className":255},[256],"msupsub",[29,258,260],{"className":259},[163],[29,261,263],{"className":262},[168],[29,264,267],{"className":265,"style":266},[172],"height:1.0584em;",[29,268,270,274],{"style":269},"top:-3.3073em;margin-right:0.05em;",[29,271],{"className":272,"style":273},[180],"height:2.7em;",[29,275,277],{"className":276},[185,186,187,188],[29,278,118],{"className":279},[72,188],", which isn't the same feature as normalizing ",[29,282,284,302],{"className":283},[32],[29,285,287],{"className":286},[36],[38,288,289],{"xmlns":40},[42,290,291,299],{},[45,292,293],{},[88,294,295,297],{},[48,296,50],{},[116,298,118],{},[52,300,301],{"encoding":54},"x^2",[29,303,305],{"className":304,"ariaHidden":59},[58],[29,306,308,312],{"className":307},[63],[29,309],{"className":310,"style":311},[67],"height:0.8141em;",[29,313,315,318],{"className":314},[72],[29,316,50],{"className":317},[72,73],[29,319,321],{"className":320},[256],[29,322,324],{"className":323},[163],[29,325,327],{"className":326},[168],[29,328,330],{"className":329,"style":311},[172],[29,331,333,336],{"style":332},"top:-3.063em;margin-right:0.05em;",[29,334],{"className":335,"style":273},[180],[29,337,339],{"className":338},[185,186,187,188],[29,340,118],{"className":341},[72,188]," directly. Both work, but they produce different columns, and the second order (create the feature, normalize after) is the one I use since ",[15,344,345],{"href":17},"the main post",".",[11,348,349,350,510,511,738,739,797,798,826,827,855,856,886,887,891],{},"Here's the happy ending upfront: the RMSE from either order ends up practically the same, so relax, getting this backwards isn't a serious mistake. The reason is that ",[29,351,353,386],{"className":352},[32],[29,354,356],{"className":355},[36],[38,357,358],{"xmlns":40},[42,359,360,384],{},[45,361,362],{},[88,363,364,382],{},[45,365,366,368,380],{},[93,367,95],{"fence":59},[97,369,370,378],{},[45,371,372,374,376],{},[48,373,50],{},[93,375,105],{},[48,377,108],{},[48,379,111],{},[93,381,114],{"fence":59},[116,383,118],{},[52,385,121],{"encoding":54},[29,387,389],{"className":388,"ariaHidden":59},[58],[29,390,392,395],{"className":391},[63],[29,393],{"className":394,"style":131},[67],[29,396,398,487],{"className":397},[135],[29,399,401,407,481],{"className":400},[135],[29,402,404],{"className":403,"style":144},[142,143],[29,405,95],{"className":406},[148,149],[29,408,410,413,478],{"className":409},[72],[29,411],{"className":412},[142,156],[29,414,416],{"className":415},[97],[29,417,419,470],{"className":418},[163,164],[29,420,422,467],{"className":421},[168],[29,423,425,439,447],{"className":424,"style":173},[172],[29,426,427,430],{"style":176},[29,428],{"className":429,"style":181},[180],[29,431,433],{"className":432},[185,186,187,188],[29,434,436],{"className":435},[72,188],[29,437,111],{"className":438,"style":195},[72,73,188],[29,440,441,444],{"style":198},[29,442],{"className":443,"style":181},[180],[29,445],{"className":446,"style":206},[205],[29,448,449,452],{"style":209},[29,450],{"className":451,"style":181},[180],[29,453,455],{"className":454},[185,186,187,188],[29,456,458,461,464],{"className":457},[72,188],[29,459,50],{"className":460},[72,73,188],[29,462,105],{"className":463},[225,188],[29,465,108],{"className":466},[72,73,188],[29,468,233],{"className":469},[232],[29,471,473],{"className":472},[168],[29,474,476],{"className":475,"style":240},[172],[29,477],{},[29,479],{"className":480},[246,156],[29,482,484],{"className":483,"style":144},[246,143],[29,485,114],{"className":486},[148,149],[29,488,490],{"className":489},[256],[29,491,493],{"className":492},[163],[29,494,496],{"className":495},[168],[29,497,499],{"className":498,"style":266},[172],[29,500,501,504],{"style":269},[29,502],{"className":503,"style":273},[180],[29,505,507],{"className":506},[185,186,187,188],[29,508,118],{"className":509},[72,188]," expands into ",[29,512,514,559],{"className":513},[32],[29,515,517],{"className":516},[36],[38,518,519],{"xmlns":40},[42,520,521,556],{},[45,522,523],{},[97,524,525,550],{},[45,526,527,533,535,537,539,541,544],{},[88,528,529,531],{},[48,530,50],{},[116,532,118],{},[93,534,105],{},[116,536,118],{},[48,538,108],{},[48,540,50],{},[93,542,543],{},"+",[88,545,546,548],{},[48,547,108],{},[116,549,118],{},[88,551,552,554],{},[48,553,111],{},[116,555,118],{},[52,557,558],{"encoding":54},"\\frac{x^2 - 2\\mu x + \\mu^2}{\\sigma^2}",[29,560,562],{"className":561,"ariaHidden":59},[58],[29,563,565,569],{"className":564},[63],[29,566],{"className":567,"style":568},[67],"height:1.415em;vertical-align:-0.345em;",[29,570,572,575,735],{"className":571},[72],[29,573],{"className":574},[142,156],[29,576,578],{"className":577},[97],[29,579,581,727],{"className":580},[163,164],[29,582,584,724],{"className":583},[168],[29,585,588,632,640],{"className":586,"style":587},[172],"height:1.07em;",[29,589,590,593],{"style":176},[29,591],{"className":592,"style":181},[180],[29,594,596],{"className":595},[185,186,187,188],[29,597,599],{"className":598},[72,188],[29,600,602,605],{"className":601},[72,188],[29,603,111],{"className":604,"style":195},[72,73,188],[29,606,608],{"className":607},[256],[29,609,611],{"className":610},[163],[29,612,614],{"className":613},[168],[29,615,618],{"className":616,"style":617},[172],"height:0.7463em;",[29,619,621,625],{"style":620},"top:-2.786em;margin-right:0.0714em;",[29,622],{"className":623,"style":624},[180],"height:2.5em;",[29,626,629],{"className":627},[185,628,149,188],"reset-size3",[29,630,118],{"className":631},[72,188],[29,633,634,637],{"style":198},[29,635],{"className":636,"style":181},[180],[29,638],{"className":639,"style":206},[205],[29,641,642,645],{"style":209},[29,643],{"className":644,"style":181},[180],[29,646,648],{"className":647},[185,186,187,188],[29,649,651,682,685,688,692,695],{"className":650},[72,188],[29,652,654,657],{"className":653},[72,188],[29,655,50],{"className":656},[72,73,188],[29,658,660],{"className":659},[256],[29,661,663],{"className":662},[163],[29,664,666],{"className":665},[168],[29,667,670],{"className":668,"style":669},[172],"height:0.8913em;",[29,671,673,676],{"style":672},"top:-2.931em;margin-right:0.0714em;",[29,674],{"className":675,"style":624},[180],[29,677,679],{"className":678},[185,628,149,188],[29,680,118],{"className":681},[72,188],[29,683,105],{"className":684},[225,188],[29,686,118],{"className":687},[72,188],[29,689,691],{"className":690},[72,73,188],"μx",[29,693,543],{"className":694},[225,188],[29,696,698,701],{"className":697},[72,188],[29,699,108],{"className":700},[72,73,188],[29,702,704],{"className":703},[256],[29,705,707],{"className":706},[163],[29,708,710],{"className":709},[168],[29,711,713],{"className":712,"style":669},[172],[29,714,715,718],{"style":672},[29,716],{"className":717,"style":624},[180],[29,719,721],{"className":720},[185,628,149,188],[29,722,118],{"className":723},[72,188],[29,725,233],{"className":726},[232],[29,728,730],{"className":729},[168],[29,731,733],{"className":732,"style":240},[172],[29,734],{},[29,736],{"className":737},[246,156],", which is just a linear combination of ",[29,740,742,759],{"className":741},[32],[29,743,745],{"className":744},[36],[38,746,747],{"xmlns":40},[42,748,749,757],{},[45,750,751],{},[88,752,753,755],{},[48,754,50],{},[116,756,118],{},[52,758,301],{"encoding":54},[29,760,762],{"className":761,"ariaHidden":59},[58],[29,763,765,768],{"className":764},[63],[29,766],{"className":767,"style":311},[67],[29,769,771,774],{"className":770},[72],[29,772,50],{"className":773},[72,73],[29,775,777],{"className":776},[256],[29,778,780],{"className":779},[163],[29,781,783],{"className":782},[168],[29,784,786],{"className":785,"style":311},[172],[29,787,788,791],{"style":332},[29,789],{"className":790,"style":273},[180],[29,792,794],{"className":793},[185,186,187,188],[29,795,118],{"className":796},[72,188],", ",[29,799,801,814],{"className":800},[32],[29,802,804],{"className":803},[36],[38,805,806],{"xmlns":40},[42,807,808,812],{},[45,809,810],{},[48,811,50],{},[52,813,50],{"encoding":54},[29,815,817],{"className":816,"ariaHidden":59},[58],[29,818,820,823],{"className":819},[63],[29,821],{"className":822,"style":68},[67],[29,824,50],{"className":825},[72,73],", and a constant, three things the model already has on hand (",[29,828,830,843],{"className":829},[32],[29,831,833],{"className":832},[36],[38,834,835],{"xmlns":40},[42,836,837,841],{},[45,838,839],{},[48,840,50],{},[52,842,50],{"encoding":54},[29,844,846],{"className":845,"ariaHidden":59},[58],[29,847,849,852],{"className":848},[63],[29,850],{"className":851,"style":68},[67],[29,853,50],{"className":854},[72,73]," itself and the bias ",[29,857,859,873],{"className":858},[32],[29,860,862],{"className":861},[36],[38,863,864],{"xmlns":40},[42,865,866,871],{},[45,867,868],{},[48,869,870],{},"b",[52,872,870],{"encoding":54},[29,874,876],{"className":875,"ariaHidden":59},[58],[29,877,879,883],{"className":878},[63],[29,880],{"className":881,"style":882},[67],"height:0.6944em;",[29,884,870],{"className":885},[72,73],"). So normalizing before or after doesn't change what the model ",[888,889,890],"strong",{},"can"," represent, it just rewrites the same function in a different coordinate basis. What actually changes is the scale of the weights and how fast gradient descent converges to them, not the final result. (I walk through the full derivation in Exercise 2 below, if you want to see it on paper.)",[21,893,895],{"id":894},"polynomial-features-are-extremely-collinear","Polynomial features are extremely collinear",[11,897,898,797,926,984,985,1045,1046,1116],{},[29,899,901,914],{"className":900},[32],[29,902,904],{"className":903},[36],[38,905,906],{"xmlns":40},[42,907,908,912],{},[45,909,910],{},[48,911,50],{},[52,913,50],{"encoding":54},[29,915,917],{"className":916,"ariaHidden":59},[58],[29,918,920,923],{"className":919},[63],[29,921],{"className":922,"style":68},[67],[29,924,50],{"className":925},[72,73],[29,927,929,946],{"className":928},[32],[29,930,932],{"className":931},[36],[38,933,934],{"xmlns":40},[42,935,936,944],{},[45,937,938],{},[88,939,940,942],{},[48,941,50],{},[116,943,118],{},[52,945,301],{"encoding":54},[29,947,949],{"className":948,"ariaHidden":59},[58],[29,950,952,955],{"className":951},[63],[29,953],{"className":954,"style":311},[67],[29,956,958,961],{"className":957},[72],[29,959,50],{"className":960},[72,73],[29,962,964],{"className":963},[256],[29,965,967],{"className":966},[163],[29,968,970],{"className":969},[168],[29,971,973],{"className":972,"style":311},[172],[29,974,975,978],{"style":332},[29,976],{"className":977,"style":273},[180],[29,979,981],{"className":980},[185,186,187,188],[29,982,118],{"className":983},[72,188],", and ",[29,986,988,1007],{"className":987},[32],[29,989,991],{"className":990},[36],[38,992,993],{"xmlns":40},[42,994,995,1004],{},[45,996,997],{},[88,998,999,1001],{},[48,1000,50],{},[116,1002,1003],{},"3",[52,1005,1006],{"encoding":54},"x^3",[29,1008,1010],{"className":1009,"ariaHidden":59},[58],[29,1011,1013,1016],{"className":1012},[63],[29,1014],{"className":1015,"style":311},[67],[29,1017,1019,1022],{"className":1018},[72],[29,1020,50],{"className":1021},[72,73],[29,1023,1025],{"className":1024},[256],[29,1026,1028],{"className":1027},[163],[29,1029,1031],{"className":1030},[168],[29,1032,1034],{"className":1033,"style":311},[172],[29,1035,1036,1039],{"style":332},[29,1037],{"className":1038,"style":273},[180],[29,1040,1042],{"className":1041},[185,186,187,188],[29,1043,1003],{"className":1044},[72,188]," carry almost the same information over a narrow interval. That leaves the matrix ",[29,1047,1049,1073],{"className":1048},[32],[29,1050,1052],{"className":1051},[36],[38,1053,1054],{"xmlns":40},[42,1055,1056,1070],{},[45,1057,1058,1068],{},[88,1059,1060,1064],{},[48,1061,1063],{"mathvariant":1062},"bold","X",[48,1065,1067],{"mathvariant":1066},"normal","⊤",[48,1069,1063],{"mathvariant":1062},[52,1071,1072],{"encoding":54},"\\mathbf{X}^\\top\\mathbf{X}",[29,1074,1076],{"className":1075,"ariaHidden":59},[58],[29,1077,1079,1083,1113],{"className":1078},[63],[29,1080],{"className":1081,"style":1082},[67],"height:0.8491em;",[29,1084,1086,1090],{"className":1085},[72],[29,1087,1063],{"className":1088},[72,1089],"mathbf",[29,1091,1093],{"className":1092},[256],[29,1094,1096],{"className":1095},[163],[29,1097,1099],{"className":1098},[168],[29,1100,1102],{"className":1101,"style":1082},[172],[29,1103,1104,1107],{"style":332},[29,1105],{"className":1106,"style":273},[180],[29,1108,1110],{"className":1109},[185,186,187,188],[29,1111,1067],{"className":1112},[72,188],[29,1114,1063],{"className":1115},[72,1089]," poorly conditioned. I computed the ratio between the raw range of the widest and narrowest column, by degree:",[1118,1119,1120,1135],"table",{},[1121,1122,1123],"thead",{},[1124,1125,1126,1131],"tr",{},[1127,1128,1130],"th",{"align":1129},"center","Degree",[1127,1132,1134],{"align":1133},"right","Max\u002Fmin range (raw)",[1136,1137,1138,1146,1153,1161,1169],"tbody",{},[1124,1139,1140,1144],{},[1141,1142,1143],"td",{"align":1129},"1",[1141,1145,1143],{"align":1133},[1124,1147,1148,1150],{},[1141,1149,1003],{"align":1129},[1141,1151,1152],{"align":1133},"361",[1124,1154,1155,1158],{},[1141,1156,1157],{"align":1129},"5",[1141,1159,1160],{"align":1133},"1.3 × 10⁵",[1124,1162,1163,1166],{},[1141,1164,1165],{"align":1129},"9",[1141,1167,1168],{"align":1133},"1.7 × 10¹⁰",[1124,1170,1171,1174],{},[1141,1172,1173],{"align":1129},"13",[1141,1175,1176],{"align":1133},"2.2 × 10¹⁵",[11,1178,1179],{},"At degree 13 that ratio is already near the precision limit of 64-bit floating point (~2.2×10⁻¹⁶ relative precision). It's not an exaggeration to say the raw matrix at that degree sits right at the edge of numerical singularity.",[11,1181,1182,1183,1262],{},"And \"poorly conditioned\" isn't just a scary adjective, it has a very concrete consequence for the weight that comes out of the fit. Solving the normal equation involves, under the hood, inverting that matrix, and when it's nearly singular, that inversion behaves a lot like dividing by a number close to zero: a tiny wobble in the numerator (a bit of noise in the data, a floating-point rounding error) turns into a huge, unstable answer on the other side. In practice, that means two nearly identical datasets can produce completely different ",[29,1184,1186,1207],{"className":1185},[32],[29,1187,1189],{"className":1188},[36],[38,1190,1191],{"xmlns":40},[42,1192,1193,1204],{},[45,1194,1195],{},[1196,1197,1198,1201],"msub",{},[48,1199,1200],{},"w",[48,1202,1203],{},"j",[52,1205,1206],{"encoding":54},"w_j",[29,1208,1210],{"className":1209,"ariaHidden":59},[58],[29,1211,1213,1217],{"className":1212},[63],[29,1214],{"className":1215,"style":1216},[67],"height:0.7167em;vertical-align:-0.2861em;",[29,1218,1220,1224],{"className":1219},[72],[29,1221,1200],{"className":1222,"style":1223},[72,73],"margin-right:0.0269em;",[29,1225,1227],{"className":1226},[256],[29,1228,1230,1253],{"className":1229},[163,164],[29,1231,1233,1250],{"className":1232},[168],[29,1234,1237],{"className":1235,"style":1236},[172],"height:0.3117em;",[29,1238,1240,1243],{"style":1239},"top:-2.55em;margin-left:-0.0269em;margin-right:0.05em;",[29,1241],{"className":1242,"style":273},[180],[29,1244,1246],{"className":1245},[185,186,187,188],[29,1247,1203],{"className":1248,"style":1249},[72,73,188],"margin-right:0.0572em;",[29,1251,233],{"className":1252},[232],[29,1254,1256],{"className":1255},[168],[29,1257,1260],{"className":1258,"style":1259},[172],"height:0.2861em;",[29,1261],{}," weights, sometimes with opposite signs nearly canceling out, even if the predicted curve itself looks basically the same. The individual weights stop meaning anything reliable, only their combination still holds up.",[11,1264,1265,1266,1268,1269,1272],{},"Normalizing (which my component from ",[15,1267,345],{"href":17}," already does under the hood) improves this by several orders of magnitude, but doesn't fix the ",[888,1270,1271],{},"intrinsic"," collinearity between the powers, only the scale difference between them. For real collinearity, the remedy is regularization, which I show further down.",[21,1274,1276,1277,1307,1308],{"id":1275},"i-always-store-μmuμ-and-σsigmaσ","I always store ",[29,1278,1280,1294],{"className":1279},[32],[29,1281,1283],{"className":1282},[36],[38,1284,1285],{"xmlns":40},[42,1286,1287,1291],{},[45,1288,1289],{},[48,1290,108],{},[52,1292,1293],{"encoding":54},"\\mu",[29,1295,1297],{"className":1296,"ariaHidden":59},[58],[29,1298,1300,1304],{"className":1299},[63],[29,1301],{"className":1302,"style":1303},[67],"height:0.625em;vertical-align:-0.1944em;",[29,1305,108],{"className":1306},[72,73]," and ",[29,1309,1311,1325],{"className":1310},[32],[29,1312,1314],{"className":1313},[36],[38,1315,1316],{"xmlns":40},[42,1317,1318,1322],{},[45,1319,1320],{},[48,1321,111],{},[52,1323,1324],{"encoding":54},"\\sigma",[29,1326,1328],{"className":1327,"ariaHidden":59},[58],[29,1329,1331,1334],{"className":1330},[63],[29,1332],{"className":1333,"style":68},[67],[29,1335,111],{"className":1336,"style":195},[72,73],[11,1338,1339,1340,1344],{},"I hit this same note in ",[15,1341,1343],{"href":1342},"\u002Fen\u002Fplaylists\u002Fmachine-learning-specialization\u002Fw2-lab03-feature-scaling","the normalization post"," and I'm hitting it again here: without storing training's mean and standard deviation, there's no way to normalize any new data the same way, and the model becomes a paperweight. Every component I built for this post stores this internally before predicting anything.",[21,1346,1348],{"id":1347},"more-features-never-worsens-training-error","More features never worsens training error",[11,1350,1351,1352,1355,1356,1358],{},"This isn't a coincidence, it's nearly a theorem: adding a column only expands the space of possible models, so the minimum training cost can only drop or tie, never rise. That's why training error alone ",[888,1353,1354],{},"can't"," be used to pick model complexity, it always needs a separate set (which I already showed in ",[15,1357,345],{"href":17},").",[21,1360,1362],{"id":1361},"feature-engineering-isnt-just-powers","Feature engineering isn't just powers",[11,1364,1365,1366,1487],{},"Powers are the easiest textbook example. In the real world, the most useful features usually come from domain knowledge: square roots, logs, ratios, products between columns. I tested this with a ",[29,1367,1369,1392],{"className":1368},[32],[29,1370,1372],{"className":1371},[36],[38,1373,1374],{"xmlns":40},[42,1375,1376,1389],{},[45,1377,1378,1381,1384],{},[48,1379,1380],{},"y",[93,1382,1383],{},"=",[1385,1386,1387],"msqrt",{},[48,1388,50],{},[52,1390,1391],{"encoding":54},"y=\\sqrt{x}",[29,1393,1395,1416],{"className":1394,"ariaHidden":59},[58],[29,1396,1398,1401,1404,1409,1413],{"className":1397},[63],[29,1399],{"className":1400,"style":1303},[67],[29,1402,1380],{"className":1403,"style":195},[72,73],[29,1405],{"className":1406,"style":1408},[1407],"mspace","margin-right:0.2778em;",[29,1410,1383],{"className":1411},[1412],"mrel",[29,1414],{"className":1415,"style":1408},[1407],[29,1417,1419,1423],{"className":1418},[63],[29,1420],{"className":1421,"style":1422},[67],"height:1.04em;vertical-align:-0.2397em;",[29,1424,1427],{"className":1425},[72,1426],"sqrt",[29,1428,1430,1478],{"className":1429},[163,164],[29,1431,1433,1475],{"className":1432},[168],[29,1434,1437,1452],{"className":1435,"style":1436},[172],"height:0.8003em;",[29,1438,1442,1445],{"className":1439,"style":1441},[1440],"svg-align","top:-3em;",[29,1443],{"className":1444,"style":181},[180],[29,1446,1449],{"className":1447,"style":1448},[72],"padding-left:0.833em;",[29,1450,50],{"className":1451},[72,73],[29,1453,1455,1458],{"style":1454},"top:-2.7603em;",[29,1456],{"className":1457,"style":181},[180],[29,1459,1463],{"className":1460,"style":1462},[1461],"hide-tail","min-width:0.853em;height:1.08em;",[1464,1465,1471],"svg",{"xmlns":1466,"width":1467,"height":1468,"viewBox":1469,"preserveAspectRatio":1470},"http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","400em","1.08em","0 0 400000 1080","xMinYMin slice",[1472,1473],"path",{"d":1474},"M95,702\nc-2.7,0,-7.17,-2.7,-13.5,-8c-5.8,-5.3,-9.5,-10,-9.5,-14\nc0,-2,0.3,-3.3,1,-4c1.3,-2.7,23.83,-20.7,67.5,-54\nc44.2,-33.3,65.8,-50.3,66.5,-51c1.3,-1.3,3,-2,5,-2c4.7,0,8.7,3.3,12,10\ns173,378,173,378c0.7,0,35.3,-71,104,-213c68.7,-142,137.5,-285,206.5,-429\nc69,-144,104.5,-217.7,106.5,-221\nl0 -0\nc5.3,-9.3,12,-14,20,-14\nH400000v40H845.2724\ns-225.272,467,-225.272,467s-235,486,-235,486c-2.7,4.7,-9,7,-19,7\nc-6,0,-10,-1,-12,-3s-194,-422,-194,-422s-65,47,-65,47z\nM834 80h400000v40h-400000z",[29,1476,233],{"className":1477},[232],[29,1479,1481],{"className":1480},[168],[29,1482,1485],{"className":1483,"style":1484},[172],"height:0.2397em;",[29,1486],{}," target:",[1118,1489,1490,1504],{},[1121,1491,1492],{},[1124,1493,1494,1498,1501],{},[1127,1495,1497],{"align":1496},"left","Features",[1127,1499,1500],{"align":1133},"# parameters",[1127,1502,1503],{"align":1133},"RMSE",[1136,1505,1506,1543,1628,1753],{},[1124,1507,1508,1538,1540],{},[1141,1509,1510],{"align":1496},[29,1511,1513,1526],{"className":1512},[32],[29,1514,1516],{"className":1515},[36],[38,1517,1518],{"xmlns":40},[42,1519,1520,1524],{},[45,1521,1522],{},[48,1523,50],{},[52,1525,50],{"encoding":54},[29,1527,1529],{"className":1528,"ariaHidden":59},[58],[29,1530,1532,1535],{"className":1531},[63],[29,1533],{"className":1534,"style":68},[67],[29,1536,50],{"className":1537},[72,73],[1141,1539,118],{"align":1133},[1141,1541,1542],{"align":1133},"0.300",[1124,1544,1545,1623,1625],{},[1141,1546,1547],{"align":1496},[29,1548,1550,1573],{"className":1549},[32],[29,1551,1553],{"className":1552},[36],[38,1554,1555],{"xmlns":40},[42,1556,1557,1570],{},[45,1558,1559,1561,1564],{},[48,1560,50],{},[93,1562,1563],{"separator":59},",",[88,1565,1566,1568],{},[48,1567,50],{},[116,1569,118],{},[52,1571,1572],{"encoding":54},"x, x^2",[29,1574,1576],{"className":1575,"ariaHidden":59},[58],[29,1577,1579,1583,1586,1590,1594],{"className":1578},[63],[29,1580],{"className":1581,"style":1582},[67],"height:1.0085em;vertical-align:-0.1944em;",[29,1584,50],{"className":1585},[72,73],[29,1587,1563],{"className":1588},[1589],"mpunct",[29,1591],{"className":1592,"style":1593},[1407],"margin-right:0.1667em;",[29,1595,1597,1600],{"className":1596},[72],[29,1598,50],{"className":1599},[72,73],[29,1601,1603],{"className":1602},[256],[29,1604,1606],{"className":1605},[163],[29,1607,1609],{"className":1608},[168],[29,1610,1612],{"className":1611,"style":311},[172],[29,1613,1614,1617],{"style":332},[29,1615],{"className":1616,"style":273},[180],[29,1618,1620],{"className":1619},[185,186,187,188],[29,1621,118],{"className":1622},[72,188],[1141,1624,1003],{"align":1133},[1141,1626,1627],{"align":1133},"0.163",[1124,1629,1630,1747,1750],{},[1141,1631,1632],{"align":1496},[29,1633,1635,1665],{"className":1634},[32],[29,1636,1638],{"className":1637},[36],[38,1639,1640],{"xmlns":40},[42,1641,1642,1662],{},[45,1643,1644,1646,1648,1654,1656],{},[48,1645,50],{},[93,1647,1563],{"separator":59},[88,1649,1650,1652],{},[48,1651,50],{},[116,1653,118],{},[93,1655,1563],{"separator":59},[88,1657,1658,1660],{},[48,1659,50],{},[116,1661,1003],{},[52,1663,1664],{"encoding":54},"x, x^2, x^3",[29,1666,1668],{"className":1667,"ariaHidden":59},[58],[29,1669,1671,1674,1677,1680,1683,1712,1715,1718],{"className":1670},[63],[29,1672],{"className":1673,"style":1582},[67],[29,1675,50],{"className":1676},[72,73],[29,1678,1563],{"className":1679},[1589],[29,1681],{"className":1682,"style":1593},[1407],[29,1684,1686,1689],{"className":1685},[72],[29,1687,50],{"className":1688},[72,73],[29,1690,1692],{"className":1691},[256],[29,1693,1695],{"className":1694},[163],[29,1696,1698],{"className":1697},[168],[29,1699,1701],{"className":1700,"style":311},[172],[29,1702,1703,1706],{"style":332},[29,1704],{"className":1705,"style":273},[180],[29,1707,1709],{"className":1708},[185,186,187,188],[29,1710,118],{"className":1711},[72,188],[29,1713,1563],{"className":1714},[1589],[29,1716],{"className":1717,"style":1593},[1407],[29,1719,1721,1724],{"className":1720},[72],[29,1722,50],{"className":1723},[72,73],[29,1725,1727],{"className":1726},[256],[29,1728,1730],{"className":1729},[163],[29,1731,1733],{"className":1732},[168],[29,1734,1736],{"className":1735,"style":311},[172],[29,1737,1738,1741],{"style":332},[29,1739],{"className":1740,"style":273},[180],[29,1742,1744],{"className":1743},[185,186,187,188],[29,1745,1003],{"className":1746},[72,188],[1141,1748,1749],{"align":1133},"4",[1141,1751,1752],{"align":1133},"0.104",[1124,1754,1755,1832,1834],{},[1141,1756,1757],{"align":1496},[29,1758,1760,1776],{"className":1759},[32],[29,1761,1763],{"className":1762},[36],[38,1764,1765],{"xmlns":40},[42,1766,1767,1773],{},[45,1768,1769],{},[1385,1770,1771],{},[48,1772,50],{},[52,1774,1775],{"encoding":54},"\\sqrt{x}",[29,1777,1779],{"className":1778,"ariaHidden":59},[58],[29,1780,1782,1785],{"className":1781},[63],[29,1783],{"className":1784,"style":1422},[67],[29,1786,1788],{"className":1787},[72,1426],[29,1789,1791,1824],{"className":1790},[163,164],[29,1792,1794,1821],{"className":1793},[168],[29,1795,1797,1809],{"className":1796,"style":1436},[172],[29,1798,1800,1803],{"className":1799,"style":1441},[1440],[29,1801],{"className":1802,"style":181},[180],[29,1804,1806],{"className":1805,"style":1448},[72],[29,1807,50],{"className":1808},[72,73],[29,1810,1811,1814],{"style":1454},[29,1812],{"className":1813,"style":181},[180],[29,1815,1817],{"className":1816,"style":1462},[1461],[1464,1818,1819],{"xmlns":1466,"width":1467,"height":1468,"viewBox":1469,"preserveAspectRatio":1470},[1472,1820],{"d":1474},[29,1822,233],{"className":1823},[232],[29,1825,1827],{"className":1826},[168],[29,1828,1830],{"className":1829,"style":1484},[172],[29,1831],{},[1141,1833,118],{"align":1133},[1141,1835,1836],{"align":1133},[888,1837,1838],{},"0.000",[11,1840,1841,1842,1916],{},"A single ",[29,1843,1845,1860],{"className":1844},[32],[29,1846,1848],{"className":1847},[36],[38,1849,1850],{"xmlns":40},[42,1851,1852,1858],{},[45,1853,1854],{},[1385,1855,1856],{},[48,1857,50],{},[52,1859,1775],{"encoding":54},[29,1861,1863],{"className":1862,"ariaHidden":59},[58],[29,1864,1866,1869],{"className":1865},[63],[29,1867],{"className":1868,"style":1422},[67],[29,1870,1872],{"className":1871},[72,1426],[29,1873,1875,1908],{"className":1874},[163,164],[29,1876,1878,1905],{"className":1877},[168],[29,1879,1881,1893],{"className":1880,"style":1436},[172],[29,1882,1884,1887],{"className":1883,"style":1441},[1440],[29,1885],{"className":1886,"style":181},[180],[29,1888,1890],{"className":1889,"style":1448},[72],[29,1891,50],{"className":1892},[72,73],[29,1894,1895,1898],{"style":1454},[29,1896],{"className":1897,"style":181},[180],[29,1899,1901],{"className":1900,"style":1462},[1461],[1464,1902,1903],{"xmlns":1466,"width":1467,"height":1468,"viewBox":1469,"preserveAspectRatio":1470},[1472,1904],{"d":1474},[29,1906,233],{"className":1907},[232],[29,1909,1911],{"className":1910},[168],[29,1912,1914],{"className":1913,"style":1484},[172],[29,1915],{}," feature zeroes the error with 2 parameters, while three powers spend 4 parameters and still miss. Domain knowledge beats brute force.",[21,1918,1920],{"id":1919},"taming-overfitting-regularization","Taming overfitting: regularization",[11,1922,1923,1926,1927,1930,1931,1961],{},[15,1924,1925],{"href":17},"The main post"," ended with an open question: can I keep all of a high degree's flexibility without paying overfitting's price? The answer is L2 regularization, or ",[888,1928,1929],{},"Ridge",". Instead of choosing a discrete degree, I add a continuous penalty ",[29,1932,1934,1949],{"className":1933},[32],[29,1935,1937],{"className":1936},[36],[38,1938,1939],{"xmlns":40},[42,1940,1941,1946],{},[45,1942,1943],{},[48,1944,1945],{},"λ",[52,1947,1948],{"encoding":54},"\\lambda",[29,1950,1952],{"className":1951,"ariaHidden":59},[58],[29,1953,1955,1958],{"className":1954},[63],[29,1956],{"className":1957,"style":882},[67],[29,1959,1945],{"className":1960},[72,73]," that punishes big weights:",[11,1963,1964],{},[29,1965,1967,2046],{"className":1966},[32],[29,1968,1970],{"className":1969},[36],[38,1971,1972],{"xmlns":40},[42,1973,1974,2043],{},[45,1975,1976,1998,2001,2003,2005,2007,2009,2011,2013,2015,2017,2019,2021,2023,2025,2027,2034],{},[1196,1977,1978,1981],{},[48,1979,1980],{},"J",[45,1982,1983,1986,1989,1992,1995],{},[48,1984,1985],{},"r",[48,1987,1988],{},"i",[48,1990,1991],{},"d",[48,1993,1994],{},"g",[48,1996,1997],{},"e",[93,1999,95],{"stretchy":2000},"false",[48,2002,1200],{"mathvariant":1062},[93,2004,1563],{"separator":59},[48,2006,870],{},[93,2008,114],{"stretchy":2000},[93,2010,1383],{},[48,2012,1980],{},[93,2014,95],{"stretchy":2000},[48,2016,1200],{"mathvariant":1062},[93,2018,1563],{"separator":59},[48,2020,870],{},[93,2022,114],{"stretchy":2000},[93,2024,543],{},[48,2026,1945],{},[1196,2028,2029,2032],{},[93,2030,2031],{},"∑",[48,2033,1203],{},[2035,2036,2037,2039,2041],"msubsup",{},[48,2038,1200],{},[48,2040,1203],{},[116,2042,118],{},[52,2044,2045],{"encoding":54},"J_{ridge}(\\mathbf{w}, b) = J(\\mathbf{w}, b) + \\lambda \\sum_j w_j^2",[29,2047,2049,2143,2181],{"className":2048,"ariaHidden":59},[58],[29,2050,2052,2056,2115,2118,2122,2125,2128,2131,2134,2137,2140],{"className":2051},[63],[29,2053],{"className":2054,"style":2055},[67],"height:1.0361em;vertical-align:-0.2861em;",[29,2057,2059,2063],{"className":2058},[72],[29,2060,1980],{"className":2061,"style":2062},[72,73],"margin-right:0.0962em;",[29,2064,2066],{"className":2065},[256],[29,2067,2069,2107],{"className":2068},[163,164],[29,2070,2072,2104],{"className":2071},[168],[29,2073,2076],{"className":2074,"style":2075},[172],"height:0.3361em;",[29,2077,2079,2082],{"style":2078},"top:-2.55em;margin-left:-0.0962em;margin-right:0.05em;",[29,2080],{"className":2081,"style":273},[180],[29,2083,2085],{"className":2084},[185,186,187,188],[29,2086,2088,2092,2095,2098,2101],{"className":2087},[72,188],[29,2089,1985],{"className":2090,"style":2091},[72,73,188],"margin-right:0.0278em;",[29,2093,1988],{"className":2094},[72,73,188],[29,2096,1991],{"className":2097},[72,73,188],[29,2099,1994],{"className":2100,"style":195},[72,73,188],[29,2102,1997],{"className":2103},[72,73,188],[29,2105,233],{"className":2106},[232],[29,2108,2110],{"className":2109},[168],[29,2111,2113],{"className":2112,"style":1259},[172],[29,2114],{},[29,2116,95],{"className":2117},[142],[29,2119,1200],{"className":2120,"style":2121},[72,1089],"margin-right:0.016em;",[29,2123,1563],{"className":2124},[1589],[29,2126],{"className":2127,"style":1593},[1407],[29,2129,870],{"className":2130},[72,73],[29,2132,114],{"className":2133},[246],[29,2135],{"className":2136,"style":1408},[1407],[29,2138,1383],{"className":2139},[1412],[29,2141],{"className":2142,"style":1408},[1407],[29,2144,2146,2150,2153,2156,2159,2162,2165,2168,2171,2175,2178],{"className":2145},[63],[29,2147],{"className":2148,"style":2149},[67],"height:1em;vertical-align:-0.25em;",[29,2151,1980],{"className":2152,"style":2062},[72,73],[29,2154,95],{"className":2155},[142],[29,2157,1200],{"className":2158,"style":2121},[72,1089],[29,2160,1563],{"className":2161},[1589],[29,2163],{"className":2164,"style":1593},[1407],[29,2166,870],{"className":2167},[72,73],[29,2169,114],{"className":2170},[246],[29,2172],{"className":2173,"style":2174},[1407],"margin-right:0.2222em;",[29,2176,543],{"className":2177},[225],[29,2179],{"className":2180,"style":2174},[1407],[29,2182,2184,2188,2191,2194,2241,2244],{"className":2183},[63],[29,2185],{"className":2186,"style":2187},[67],"height:1.2499em;vertical-align:-0.4358em;",[29,2189,1945],{"className":2190},[72,73],[29,2192],{"className":2193,"style":1593},[1407],[29,2195,2198,2204],{"className":2196},[2197],"mop",[29,2199,2031],{"className":2200,"style":2203},[2197,2201,2202],"op-symbol","small-op","position:relative;top:0em;",[29,2205,2207],{"className":2206},[256],[29,2208,2210,2232],{"className":2209},[163,164],[29,2211,2213,2229],{"className":2212},[168],[29,2214,2217],{"className":2215,"style":2216},[172],"height:0.162em;",[29,2218,2220,2223],{"style":2219},"top:-2.4003em;margin-left:0em;margin-right:0.05em;",[29,2221],{"className":2222,"style":273},[180],[29,2224,2226],{"className":2225},[185,186,187,188],[29,2227,1203],{"className":2228,"style":1249},[72,73,188],[29,2230,233],{"className":2231},[232],[29,2233,2235],{"className":2234},[168],[29,2236,2239],{"className":2237,"style":2238},[172],"height:0.4358em;",[29,2240],{},[29,2242],{"className":2243,"style":1593},[1407],[29,2245,2247,2250],{"className":2246},[72],[29,2248,1200],{"className":2249,"style":1223},[72,73],[29,2251,2253],{"className":2252},[256],[29,2254,2256,2288],{"className":2255},[163,164],[29,2257,2259,2285],{"className":2258},[168],[29,2260,2262,2274],{"className":2261,"style":311},[172],[29,2263,2265,2268],{"style":2264},"top:-2.4413em;margin-left:-0.0269em;margin-right:0.05em;",[29,2266],{"className":2267,"style":273},[180],[29,2269,2271],{"className":2270},[185,186,187,188],[29,2272,1203],{"className":2273,"style":1249},[72,73,188],[29,2275,2276,2279],{"style":332},[29,2277],{"className":2278,"style":273},[180],[29,2280,2282],{"className":2281},[185,186,187,188],[29,2283,118],{"className":2284},[72,188],[29,2286,233],{"className":2287},[232],[29,2289,2291],{"className":2290},[168],[29,2292,2295],{"className":2293,"style":2294},[172],"height:0.3948em;",[29,2296],{},[11,2298,2299,2300,2369,2370,2510],{},"Why does this actually work? A high-degree polynomial memorizing 10 training points needs, somewhere along the curve, to shoot up fast and drop fast to pass exactly through each noisy point, and that's only possible if some weights ",[29,2301,2303,2320],{"className":2302},[32],[29,2304,2306],{"className":2305},[36],[38,2307,2308],{"xmlns":40},[42,2309,2310,2318],{},[45,2311,2312],{},[1196,2313,2314,2316],{},[48,2315,1200],{},[48,2317,1203],{},[52,2319,1206],{"encoding":54},[29,2321,2323],{"className":2322,"ariaHidden":59},[58],[29,2324,2326,2329],{"className":2325},[63],[29,2327],{"className":2328,"style":1216},[67],[29,2330,2332,2335],{"className":2331},[72],[29,2333,1200],{"className":2334,"style":1223},[72,73],[29,2336,2338],{"className":2337},[256],[29,2339,2341,2361],{"className":2340},[163,164],[29,2342,2344,2358],{"className":2343},[168],[29,2345,2347],{"className":2346,"style":1236},[172],[29,2348,2349,2352],{"style":1239},[29,2350],{"className":2351,"style":273},[180],[29,2353,2355],{"className":2354},[185,186,187,188],[29,2356,1203],{"className":2357,"style":1249},[72,73,188],[29,2359,233],{"className":2360},[232],[29,2362,2364],{"className":2363},[168],[29,2365,2367],{"className":2366,"style":1259},[172],[29,2368],{}," get enormous (often with opposite signs nearly canceling out over most of the domain, and \"un-canceling\" right where the curve needs to hook through one specific point). If I simply forbid the weights from getting too big, charging a price (",[29,2371,2373,2401],{"className":2372},[32],[29,2374,2376],{"className":2375},[36],[38,2377,2378],{"xmlns":40},[42,2379,2380,2398],{},[45,2381,2382,2384,2390],{},[48,2383,1945],{},[1196,2385,2386,2388],{},[93,2387,2031],{},[48,2389,1203],{},[2035,2391,2392,2394,2396],{},[48,2393,1200],{},[48,2395,1203],{},[116,2397,118],{},[52,2399,2400],{"encoding":54},"\\lambda \\sum_j w_j^2",[29,2402,2404],{"className":2403,"ariaHidden":59},[58],[29,2405,2407,2410,2413,2416,2456,2459],{"className":2406},[63],[29,2408],{"className":2409,"style":2187},[67],[29,2411,1945],{"className":2412},[72,73],[29,2414],{"className":2415,"style":1593},[1407],[29,2417,2419,2422],{"className":2418},[2197],[29,2420,2031],{"className":2421,"style":2203},[2197,2201,2202],[29,2423,2425],{"className":2424},[256],[29,2426,2428,2448],{"className":2427},[163,164],[29,2429,2431,2445],{"className":2430},[168],[29,2432,2434],{"className":2433,"style":2216},[172],[29,2435,2436,2439],{"style":2219},[29,2437],{"className":2438,"style":273},[180],[29,2440,2442],{"className":2441},[185,186,187,188],[29,2443,1203],{"className":2444,"style":1249},[72,73,188],[29,2446,233],{"className":2447},[232],[29,2449,2451],{"className":2450},[168],[29,2452,2454],{"className":2453,"style":2238},[172],[29,2455],{},[29,2457],{"className":2458,"style":1593},[1407],[29,2460,2462,2465],{"className":2461},[72],[29,2463,1200],{"className":2464,"style":1223},[72,73],[29,2466,2468],{"className":2467},[256],[29,2469,2471,2502],{"className":2470},[163,164],[29,2472,2474,2499],{"className":2473},[168],[29,2475,2477,2488],{"className":2476,"style":311},[172],[29,2478,2479,2482],{"style":2264},[29,2480],{"className":2481,"style":273},[180],[29,2483,2485],{"className":2484},[185,186,187,188],[29,2486,1203],{"className":2487,"style":1249},[72,73,188],[29,2489,2490,2493],{"style":332},[29,2491],{"className":2492,"style":273},[180],[29,2494,2496],{"className":2495},[185,186,187,188],[29,2497,118],{"className":2498},[72,188],[29,2500,233],{"className":2501},[232],[29,2503,2505],{"className":2504},[168],[29,2506,2508],{"className":2507,"style":2294},[172],[29,2509],{},") proportional to their size squared, the curve loses the physical ability to loop tightly around each individual point. It gets forced into a smoother trend that passes near most points instead of exactly through each one, which is another way of saying: it stops memorizing the noise and starts tracking the signal.",[11,2512,2513,2514,2542,2543,2606,2607,2610,2611,2613,2614,2642],{},"I implemented this in my own exact solution (adding ",[29,2515,2517,2530],{"className":2516},[32],[29,2518,2520],{"className":2519},[36],[38,2521,2522],{"xmlns":40},[42,2523,2524,2528],{},[45,2525,2526],{},[48,2527,1945],{},[52,2529,1948],{"encoding":54},[29,2531,2533],{"className":2532,"ariaHidden":59},[58],[29,2534,2536,2539],{"className":2535},[63],[29,2537],{"className":2538,"style":882},[67],[29,2540,1945],{"className":2541},[72,73]," to the diagonal of ",[29,2544,2546,2565],{"className":2545},[32],[29,2547,2549],{"className":2548},[36],[38,2550,2551],{"xmlns":40},[42,2552,2553,2563],{},[45,2554,2555,2561],{},[88,2556,2557,2559],{},[48,2558,1063],{"mathvariant":1062},[48,2560,1067],{"mathvariant":1066},[48,2562,1063],{"mathvariant":1062},[52,2564,1072],{"encoding":54},[29,2566,2568],{"className":2567,"ariaHidden":59},[58],[29,2569,2571,2574,2603],{"className":2570},[63],[29,2572],{"className":2573,"style":1082},[67],[29,2575,2577,2580],{"className":2576},[72],[29,2578,1063],{"className":2579},[72,1089],[29,2581,2583],{"className":2582},[256],[29,2584,2586],{"className":2585},[163],[29,2587,2589],{"className":2588},[168],[29,2590,2592],{"className":2591,"style":1082},[172],[29,2593,2594,2597],{"style":332},[29,2595],{"className":2596,"style":273},[180],[29,2598,2600],{"className":2599},[185,186,187,188],[29,2601,1067],{"className":2602},[72,188],[29,2604,1063],{"className":2605},[72,1089],", excluding the bias, which is exactly the closed form scikit-learn's ",[2608,2609,1929],"code",{}," solves under the hood). I don't have scikit-learn installed in this environment, but here's the actually-computed equivalent, not just the principle: same noisy 10-point, degree-9 case from ",[15,2612,345],{"href":17},", sweeping ",[29,2615,2617,2630],{"className":2616},[32],[29,2618,2620],{"className":2619},[36],[38,2621,2622],{"xmlns":40},[42,2623,2624,2628],{},[45,2625,2626],{},[48,2627,1945],{},[52,2629,1948],{"encoding":54},[29,2631,2633],{"className":2632,"ariaHidden":59},[58],[29,2634,2636,2639],{"className":2635},[63],[29,2637],{"className":2638,"style":882},[67],[29,2640,1945],{"className":2641},[72,73],":",[1118,2644,2645,2685],{},[1121,2646,2647],{},[1124,2648,2649,2679,2682],{},[1127,2650,2651],{"align":1129},[29,2652,2654,2667],{"className":2653},[32],[29,2655,2657],{"className":2656},[36],[38,2658,2659],{"xmlns":40},[42,2660,2661,2665],{},[45,2662,2663],{},[48,2664,1945],{},[52,2666,1948],{"encoding":54},[29,2668,2670],{"className":2669,"ariaHidden":59},[58],[29,2671,2673,2676],{"className":2672},[63],[29,2674],{"className":2675,"style":882},[67],[29,2677,1945],{"className":2678},[72,73],[1127,2680,2681],{"align":1133},"Train RMSE",[1127,2683,2684],{"align":1133},"Test RMSE",[1136,2686,2687,2697,2710,2721,2732,2743,2754],{},[1124,2688,2689,2692,2694],{},[1141,2690,2691],{"align":1129},"0 (no regularization)",[1141,2693,1838],{"align":1133},[1141,2695,2696],{"align":1133},"1.717",[1124,2698,2699,2702,2705],{},[1141,2700,2701],{"align":1129},"0.01",[1141,2703,2704],{"align":1133},"0.281",[1141,2706,2707],{"align":1133},[888,2708,2709],{},"0.296",[1124,2711,2712,2715,2718],{},[1141,2713,2714],{"align":1129},"0.1",[1141,2716,2717],{"align":1133},"0.478",[1141,2719,2720],{"align":1133},"0.646",[1124,2722,2723,2726,2729],{},[1141,2724,2725],{"align":1129},"0.5",[1141,2727,2728],{"align":1133},"0.587",[1141,2730,2731],{"align":1133},"0.652",[1124,2733,2734,2737,2740],{},[1141,2735,2736],{"align":1129},"1.0",[1141,2738,2739],{"align":1133},"0.631",[1141,2741,2742],{"align":1133},"0.650",[1124,2744,2745,2748,2751],{},[1141,2746,2747],{"align":1129},"2.0",[1141,2749,2750],{"align":1133},"0.666",[1141,2752,2753],{"align":1133},"0.658",[1124,2755,2756,2759,2762],{},[1141,2757,2758],{"align":1129},"5.0",[1141,2760,2761],{"align":1133},"0.696",[1141,2763,2764],{"align":1133},"0.674",[11,2766,2767,2768,2796,2797,2800,2801,2854,2855,2857],{},"Notice the pattern: with no regularization, train hits zero (10 parameters for 10 points, an exact fit) and test is the worst of all. As ",[29,2769,2771,2784],{"className":2770},[32],[29,2772,2774],{"className":2773},[36],[38,2775,2776],{"xmlns":40},[42,2777,2778,2782],{},[45,2779,2780],{},[48,2781,1945],{},[52,2783,1948],{"encoding":54},[29,2785,2787],{"className":2786,"ariaHidden":59},[58],[29,2788,2790,2793],{"className":2789},[63],[29,2791],{"className":2792,"style":882},[67],[29,2794,1945],{"className":2795},[72,73]," grows, train worsens and test ",[888,2798,2799],{},"improves",", up to a point (",[29,2802,2804,2823],{"className":2803},[32],[29,2805,2807],{"className":2806},[36],[38,2808,2809],{"xmlns":40},[42,2810,2811,2820],{},[45,2812,2813,2815,2818],{},[48,2814,1945],{},[93,2816,2817],{},"≈",[116,2819,2701],{},[52,2821,2822],{"encoding":54},"\\lambda \\approx 0.01",[29,2824,2826,2844],{"className":2825,"ariaHidden":59},[58],[29,2827,2829,2832,2835,2838,2841],{"className":2828},[63],[29,2830],{"className":2831,"style":882},[67],[29,2833,1945],{"className":2834},[72,73],[29,2836],{"className":2837,"style":1408},[1407],[29,2839,2817],{"className":2840},[1412],[29,2842],{"className":2843,"style":1408},[1407],[29,2845,2847,2851],{"className":2846},[63],[29,2848],{"className":2849,"style":2850},[67],"height:0.6444em;",[29,2852,2701],{"className":2853},[72]," here) where test starts worsening again because the model has gotten too rigid. It's the same bias-variance trade-off from ",[15,2856,345],{"href":17},", just controlled by a continuous dial instead of the discrete choice of degree.",[11,2859,2860,2861,2642],{},"Try it yourself, dragging ",[29,2862,2864,2877],{"className":2863},[32],[29,2865,2867],{"className":2866},[36],[38,2868,2869],{"xmlns":40},[42,2870,2871,2875],{},[45,2872,2873],{},[48,2874,1945],{},[52,2876,1948],{"encoding":54},[29,2878,2880],{"className":2879,"ariaHidden":59},[58],[29,2881,2883,2886],{"className":2882},[63],[29,2884],{"className":2885,"style":882},[67],[29,2887,1945],{"className":2888},[72,73],[2890,2891],"ridge-fit-explorer",{":degree":1165,":initial-lambda":2892,":lambda-max":1003,":x-test":2893,":x-train":2894,":y-test":2895,":y-train":2896,"x-label":50,"y-label":2897},"0","[1, 3, 5, 7, 9, 11, 13, 15, 17, 19]","[0, 2, 4, 6, 8, 10, 12, 14, 16, 18]","[1.094999, -0.043944, -0.796443, -1.151981, -0.19079, 0.571574, 0.966876, 0.427335, -0.243645, -1.018878]","[1.193228, 0.550253, -0.579973, -1.143308, -0.623747, 0.365632, 0.960921, 0.528028, -0.097393, -0.880685]","y (with noise)",[11,2899,2900,2901,2929],{},"Notice how the curve stops memorizing every training point and turns into something shaped like the true signal as soon as ",[29,2902,2904,2917],{"className":2903},[32],[29,2905,2907],{"className":2906},[36],[38,2908,2909],{"xmlns":40},[42,2910,2911,2915],{},[45,2912,2913],{},[48,2914,1945],{},[52,2916,1948],{"encoding":54},[29,2918,2920],{"className":2919,"ariaHidden":59},[58],[29,2921,2923,2926],{"className":2922},[63],[29,2924],{"className":2925,"style":882},[67],[29,2927,1945],{"className":2928},[72,73]," leaves zero, without me touching the degree at all.",[21,2931,2933],{"id":2932},"exercises","Exercises",[11,2935,2936],{},"Try them before opening the answer.",[2938,2939,2941],"h3",{"id":2940},"exercise-1-recover-the-transformation","Exercise 1: recover the transformation",[11,2943,2944,2945,2948,2949,2951,2952,2954,2955,2958,2959,2962],{},"Generate ",[2608,2946,2947],{},"y = 3 * log(x + 1) + 2"," for ",[2608,2950,50],{}," from 0 to 19. Fit (a) with just ",[2608,2953,50],{},", (b) with ",[2608,2956,2957],{},"x, x², x³",", and (c) with ",[2608,2960,2961],{},"log(x+1)",". Compare the RMSE.",[2964,2965,2966,2972],"details",{},[2967,2968,2969],"summary",{},[870,2970,2971],{},"Answer",[11,2973,2974,2975,2977,2978,2980],{},"The ",[2608,2976,2961],{}," feature nails it almost exactly with a single weight, because the target is linear in it by construction. The degree-3 polynomial gets close (any polynomial approximates a log over a bounded interval), but spends 3 parameters and misses outside the range. Just ",[2608,2979,50],{}," does poorly. The moral: when you know the shape of the relationship, the right feature beats any number of powers.",[2938,2982,2984],{"id":2983},"exercise-2-does-normalization-order-matter","Exercise 2: does normalization order matter?",[11,2986,2987,2988,2991,2992,2995],{},"Compare fitting ",[2608,2989,2990],{},"zscore(x)²"," against ",[2608,2993,2994],{},"zscore(x²)",". Are the final RMSE values equal? What about the weights?",[2964,2997,2998,3002],{},[2967,2999,3000],{},[870,3001,2971],{},[11,3003,3004,3005,3376,3377,797,3435,3463,3464,3492],{},"The final RMSE is nearly identical, but the weights and convergence speed aren't. The reason: ",[29,3006,3008,3076],{"className":3007},[32],[29,3009,3011],{"className":3010},[36],[38,3012,3013],{"xmlns":40},[42,3014,3015,3073],{},[45,3016,3017,3039,3041],{},[88,3018,3019,3037],{},[45,3020,3021,3023,3035],{},[93,3022,95],{"fence":59},[97,3024,3025,3033],{},[45,3026,3027,3029,3031],{},[48,3028,50],{},[93,3030,105],{},[48,3032,108],{},[48,3034,111],{},[93,3036,114],{"fence":59},[116,3038,118],{},[93,3040,1383],{},[97,3042,3043,3067],{},[45,3044,3045,3051,3053,3055,3057,3059,3061],{},[88,3046,3047,3049],{},[48,3048,50],{},[116,3050,118],{},[93,3052,105],{},[116,3054,118],{},[48,3056,108],{},[48,3058,50],{},[93,3060,543],{},[88,3062,3063,3065],{},[48,3064,108],{},[116,3066,118],{},[88,3068,3069,3071],{},[48,3070,111],{},[116,3072,118],{},[52,3074,3075],{"encoding":54},"\\left(\\frac{x-\\mu}{\\sigma}\\right)^2 = \\frac{x^2 - 2\\mu x + \\mu^2}{\\sigma^2}",[29,3077,3079,3209],{"className":3078,"ariaHidden":59},[58],[29,3080,3082,3085,3200,3203,3206],{"className":3081},[63],[29,3083],{"className":3084,"style":131},[67],[29,3086,3088,3177],{"className":3087},[135],[29,3089,3091,3097,3171],{"className":3090},[135],[29,3092,3094],{"className":3093,"style":144},[142,143],[29,3095,95],{"className":3096},[148,149],[29,3098,3100,3103,3168],{"className":3099},[72],[29,3101],{"className":3102},[142,156],[29,3104,3106],{"className":3105},[97],[29,3107,3109,3160],{"className":3108},[163,164],[29,3110,3112,3157],{"className":3111},[168],[29,3113,3115,3129,3137],{"className":3114,"style":173},[172],[29,3116,3117,3120],{"style":176},[29,3118],{"className":3119,"style":181},[180],[29,3121,3123],{"className":3122},[185,186,187,188],[29,3124,3126],{"className":3125},[72,188],[29,3127,111],{"className":3128,"style":195},[72,73,188],[29,3130,3131,3134],{"style":198},[29,3132],{"className":3133,"style":181},[180],[29,3135],{"className":3136,"style":206},[205],[29,3138,3139,3142],{"style":209},[29,3140],{"className":3141,"style":181},[180],[29,3143,3145],{"className":3144},[185,186,187,188],[29,3146,3148,3151,3154],{"className":3147},[72,188],[29,3149,50],{"className":3150},[72,73,188],[29,3152,105],{"className":3153},[225,188],[29,3155,108],{"className":3156},[72,73,188],[29,3158,233],{"className":3159},[232],[29,3161,3163],{"className":3162},[168],[29,3164,3166],{"className":3165,"style":240},[172],[29,3167],{},[29,3169],{"className":3170},[246,156],[29,3172,3174],{"className":3173,"style":144},[246,143],[29,3175,114],{"className":3176},[148,149],[29,3178,3180],{"className":3179},[256],[29,3181,3183],{"className":3182},[163],[29,3184,3186],{"className":3185},[168],[29,3187,3189],{"className":3188,"style":266},[172],[29,3190,3191,3194],{"style":269},[29,3192],{"className":3193,"style":273},[180],[29,3195,3197],{"className":3196},[185,186,187,188],[29,3198,118],{"className":3199},[72,188],[29,3201],{"className":3202,"style":1408},[1407],[29,3204,1383],{"className":3205},[1412],[29,3207],{"className":3208,"style":1408},[1407],[29,3210,3212,3215],{"className":3211},[63],[29,3213],{"className":3214,"style":568},[67],[29,3216,3218,3221,3373],{"className":3217},[72],[29,3219],{"className":3220},[142,156],[29,3222,3224],{"className":3223},[97],[29,3225,3227,3365],{"className":3226},[163,164],[29,3228,3230,3362],{"className":3229},[168],[29,3231,3233,3273,3281],{"className":3232,"style":587},[172],[29,3234,3235,3238],{"style":176},[29,3236],{"className":3237,"style":181},[180],[29,3239,3241],{"className":3240},[185,186,187,188],[29,3242,3244],{"className":3243},[72,188],[29,3245,3247,3250],{"className":3246},[72,188],[29,3248,111],{"className":3249,"style":195},[72,73,188],[29,3251,3253],{"className":3252},[256],[29,3254,3256],{"className":3255},[163],[29,3257,3259],{"className":3258},[168],[29,3260,3262],{"className":3261,"style":617},[172],[29,3263,3264,3267],{"style":620},[29,3265],{"className":3266,"style":624},[180],[29,3268,3270],{"className":3269},[185,628,149,188],[29,3271,118],{"className":3272},[72,188],[29,3274,3275,3278],{"style":198},[29,3276],{"className":3277,"style":181},[180],[29,3279],{"className":3280,"style":206},[205],[29,3282,3283,3286],{"style":209},[29,3284],{"className":3285,"style":181},[180],[29,3287,3289],{"className":3288},[185,186,187,188],[29,3290,3292,3321,3324,3327,3330,3333],{"className":3291},[72,188],[29,3293,3295,3298],{"className":3294},[72,188],[29,3296,50],{"className":3297},[72,73,188],[29,3299,3301],{"className":3300},[256],[29,3302,3304],{"className":3303},[163],[29,3305,3307],{"className":3306},[168],[29,3308,3310],{"className":3309,"style":669},[172],[29,3311,3312,3315],{"style":672},[29,3313],{"className":3314,"style":624},[180],[29,3316,3318],{"className":3317},[185,628,149,188],[29,3319,118],{"className":3320},[72,188],[29,3322,105],{"className":3323},[225,188],[29,3325,118],{"className":3326},[72,188],[29,3328,691],{"className":3329},[72,73,188],[29,3331,543],{"className":3332},[225,188],[29,3334,3336,3339],{"className":3335},[72,188],[29,3337,108],{"className":3338},[72,73,188],[29,3340,3342],{"className":3341},[256],[29,3343,3345],{"className":3344},[163],[29,3346,3348],{"className":3347},[168],[29,3349,3351],{"className":3350,"style":669},[172],[29,3352,3353,3356],{"style":672},[29,3354],{"className":3355,"style":624},[180],[29,3357,3359],{"className":3358},[185,628,149,188],[29,3360,118],{"className":3361},[72,188],[29,3363,233],{"className":3364},[232],[29,3366,3368],{"className":3367},[168],[29,3369,3371],{"className":3370,"style":240},[172],[29,3372],{},[29,3374],{"className":3375},[246,156],", meaning it's a linear combination of ",[29,3378,3380,3397],{"className":3379},[32],[29,3381,3383],{"className":3382},[36],[38,3384,3385],{"xmlns":40},[42,3386,3387,3395],{},[45,3388,3389],{},[88,3390,3391,3393],{},[48,3392,50],{},[116,3394,118],{},[52,3396,301],{"encoding":54},[29,3398,3400],{"className":3399,"ariaHidden":59},[58],[29,3401,3403,3406],{"className":3402},[63],[29,3404],{"className":3405,"style":311},[67],[29,3407,3409,3412],{"className":3408},[72],[29,3410,50],{"className":3411},[72,73],[29,3413,3415],{"className":3414},[256],[29,3416,3418],{"className":3417},[163],[29,3419,3421],{"className":3420},[168],[29,3422,3424],{"className":3423,"style":311},[172],[29,3425,3426,3429],{"style":332},[29,3427],{"className":3428,"style":273},[180],[29,3430,3432],{"className":3431},[185,186,187,188],[29,3433,118],{"className":3434},[72,188],[29,3436,3438,3451],{"className":3437},[32],[29,3439,3441],{"className":3440},[36],[38,3442,3443],{"xmlns":40},[42,3444,3445,3449],{},[45,3446,3447],{},[48,3448,50],{},[52,3450,50],{"encoding":54},[29,3452,3454],{"className":3453,"ariaHidden":59},[58],[29,3455,3457,3460],{"className":3456},[63],[29,3458],{"className":3459,"style":68},[67],[29,3461,50],{"className":3462},[72,73],", and a constant. Since the model already has ",[29,3465,3467,3480],{"className":3466},[32],[29,3468,3470],{"className":3469},[36],[38,3471,3472],{"xmlns":40},[42,3473,3474,3478],{},[45,3475,3476],{},[48,3477,50],{},[52,3479,50],{"encoding":54},[29,3481,3483],{"className":3482,"ariaHidden":59},[58],[29,3484,3486,3489],{"className":3485},[63],[29,3487],{"className":3488,"style":68},[67],[29,3490,50],{"className":3491},[72,73]," and the bias available, the reachable function space is the same, only the basis it's written in changes.",[2938,3494,3496],{"id":3495},"exercise-3-how-many-points-are-needed","Exercise 3: how many points are needed?",[11,3498,3499,3500,3555],{},"Keeping degree 13, repeat the train\u002Ftest analysis with 40, 100, and 500 points over ",[29,3501,3503,3528],{"className":3502},[32],[29,3504,3506],{"className":3505},[36],[38,3507,3508],{"xmlns":40},[42,3509,3510,3525],{},[45,3511,3512,3515,3517,3519,3522],{},[93,3513,3514],{"stretchy":2000},"[",[116,3516,2892],{},[93,3518,1563],{"separator":59},[116,3520,3521],{},"19",[93,3523,3524],{"stretchy":2000},"]",[52,3526,3527],{"encoding":54},"[0, 19]",[29,3529,3531],{"className":3530,"ariaHidden":59},[58],[29,3532,3534,3537,3540,3543,3546,3549,3552],{"className":3533},[63],[29,3535],{"className":3536,"style":2149},[67],[29,3538,3514],{"className":3539},[142],[29,3541,2892],{"className":3542},[72],[29,3544,1563],{"className":3545},[1589],[29,3547],{"className":3548,"style":1593},[1407],[29,3550,3521],{"className":3551},[72],[29,3553,3524],{"className":3554},[246],". Does the overfitting go away?",[2964,3557,3558,3562],{},[2967,3559,3560],{},[870,3561,2971],{},[11,3563,3564],{},"Largely, yes. Overfitting is a relationship between model capacity and amount of data, not a property of the model alone. With 500 points, 14 parameters stop being too many. That gives the three classic fixes for overfitting: more data, fewer features, or regularization.",[2938,3566,3568],{"id":3567},"exercise-4-interactions-between-variables","Exercise 4: interactions between variables",[11,3570,3571,3572,3575,3576,3579,3580,3583],{},"With two input variables, degree-2 polynomial features generate ",[2608,3573,3574],{},"x1, x2, x1², x1x2, x2²",". The ",[2608,3577,3578],{},"x1x2"," term is an interaction. Build a target where the interaction is essential (",[2608,3581,3582],{},"y = x1 * x2",") and show a model without it fails.",[2964,3585,3586,3590],{},[2967,3587,3588],{},[870,3589,2971],{},[11,3591,3592],{},"Without the cross term, the model is additive and can't represent \"the effect of x1 depends on x2.\" Interactions are the most common and most useful form of feature engineering in real tabular data, even more so than pure powers.",[2938,3594,3596],{"id":3595},"exercise-5-an-honest-stopping-criterion","Exercise 5: an honest stopping criterion",[11,3598,3599],{},"If I'd used gradient descent instead of an exact solution, how would I stop reliably?",[2964,3601,3602,3606],{},[2967,3603,3604],{},[870,3605,2971],{},[11,3607,3608,3609,3612,3613,3644],{},"By the gradient's norm (",[2608,3610,3611],{},"max(|dj_dw|, |dj_db|) \u003C tol","), not by cost improvement. With too small an ",[29,3614,3616,3631],{"className":3615},[32],[29,3617,3619],{"className":3618},[36],[38,3620,3621],{"xmlns":40},[42,3622,3623,3628],{},[45,3624,3625],{},[48,3626,3627],{},"α",[52,3629,3630],{"encoding":54},"\\alpha",[29,3632,3634],{"className":3633,"ariaHidden":59},[58],[29,3635,3637,3640],{"className":3636},[63],[29,3638],{"className":3639,"style":68},[67],[29,3641,3627],{"className":3642,"style":3643},[72,73],"margin-right:0.0037em;",", the cost also improves very little per iteration, and a cost-based criterion would make me stop thinking I'd converged without actually having converged.",{"title":3646,"searchDepth":3647,"depth":3647,"links":3648},"",2,[3649,3650,3651,3653,3654,3655,3656],{"id":23,"depth":3647,"text":24},{"id":894,"depth":3647,"text":895},{"id":1275,"depth":3647,"text":3652},"I always store μ\\muμ and σ\\sigmaσ",{"id":1347,"depth":3647,"text":1348},{"id":1361,"depth":3647,"text":1362},{"id":1919,"depth":3647,"text":1920},{"id":2932,"depth":3647,"text":2933,"children":3657},[3658,3660,3661,3662,3663],{"id":2940,"depth":3659,"text":2941},3,{"id":2983,"depth":3659,"text":2984},{"id":3495,"depth":3659,"text":3496},{"id":3567,"depth":3659,"text":3568},{"id":3595,"depth":3659,"text":3596},null,"2026-08-19","Normalization order, collinearity between powers, the theorem that more features never worsens training error, and how I tamed the previous post's overfitting with regularization, actually computed, no scikit-learn required.","md",{},true,9,"\u002Fen\u002Fplaylists\u002Fmachine-learning-specialization\u002Ffeature-engineering-pitfalls","machine-learning-specialization",{"title":6,"description":3666},"draft","en\u002Fplaylists\u002Fmachine-learning-specialization\u002Ffeature-engineering-pitfalls",[3677,3678,3679],"feature-engineering","ridge","regularization","fLf9vEEPDPOxexGGhWb2cnQ8LMeP5N5UzSvZP2hYLj0",1787338985327]