import numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nfrom sklearn.tree import DecisionTreeRegressor, DecisionTreeClassifier, plot_tree\nfrom sklearn.ensemble import GradientBoostingRegressor, GradientBoostingClassifier, RandomForestClassifier\nfrom sklearn.datasets import make_regression, make_classification, make_moons\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error, accuracy_score\nfrom sklearn.preprocessing import StandardScaler\nimport warnings\nwarnings.filterwarnings('ignore')\n\nrng = np.random.default_rng(42)\nplt.rcParams['figure.dpi'] = 100<\/code><\/pre>\n\n\n\n1. Decision Trees: The Weak Learner<\/h2>\n\n\n\n
GBDT builds an ensemble of decision trees<\/strong>. Each tree partitions the feature space into rectangular regions and predicts a constant value in each region.<\/p>\n\n\n\nA single deep tree overfits; a single shallow tree (a stump<\/em>) underfits. GBDT uses many shallow trees \u2014 deliberately weak \u2014 and combines them sequentially.<\/p>\n\n\n\nWhy shallow trees?<\/strong><\/p>\n\n\n\n\n- Low variance: they don’t chase noise<\/li>\n\n\n\n
- Fast to train<\/li>\n\n\n\n
- Their errors are systematic<\/em> (high bias), which subsequent trees can correct<\/li>\n<\/ul>\n\n\n\n
# Generate a simple 1D regression problem\nnp.random.seed(42)\nX_1d = np.sort(np.random.uniform(0, 10, 120))\ny_1d = np.sin(X_1d) + 0.3 * np.random.randn(120)\nX_1d = X_1d.reshape(-1, 1)\n\nx_plot = np.linspace(0, 10, 500).reshape(-1, 1)\n\nfig, axes = plt.subplots(1, 3, figsize=(15, 4))\ndepths = [1, 3, 10]\ntitles = ['Stump (depth=1)\\nHigh bias, low variance',\n 'Shallow tree (depth=3)\\nBetter balance',\n 'Deep tree (depth=10)\\nLow bias, high variance']\n\nfor ax, d, title in zip(axes, depths, titles):\n tree = DecisionTreeRegressor(max_depth=d, random_state=42)\n tree.fit(X_1d, y_1d)\n ax.scatter(X_1d, y_1d, s=15, alpha=0.5, color='gray', label='Data')\n ax.plot(x_plot, tree.predict(x_plot), color='crimson', lw=2, label=f'Tree (depth={d})')\n ax.plot(x_plot, np.sin(x_plot), 'k--', lw=1.5, alpha=0.5, label='True signal')\n train_mse = mean_squared_error(y_1d, tree.predict(X_1d))\n ax.set_title(f'{title}\\nTrain MSE={train_mse:.3f}')\n ax.legend(fontsize=8)\n ax.set_xlabel('x'); ax.set_ylabel('y')\n\nplt.suptitle('Single decision trees: bias-variance trade-off', fontsize=13)\nplt.tight_layout()\nplt.show()<\/code><\/pre>\n\n\n\n![\"\"](\"https:\/\/tensorzen.online\/wp-content\/uploads\/2026\/06\/image-5-1024x267.png\")
<\/figure>\n\n\n\n2. Ensemble Methods: Bagging vs. Boosting<\/h2>\n\n\n\n
Both methods combine many weak learners, but in fundamentally different ways:<\/p>\n\n\n\n<\/th> Bagging<\/th> Boosting<\/th><\/tr><\/thead> Training order<\/td> Parallel (independent)<\/td> Sequential (each depends on previous)<\/td><\/tr> How errors are addressed<\/td> Averaging reduces variance<\/td> Each model corrects prior errors<\/td><\/tr> Data sampling<\/td> Bootstrap resamples<\/td> Reweights or fits residuals<\/td><\/tr> Typical depth<\/td> Deep trees<\/td> Shallow trees (stumps)<\/td><\/tr> Risk<\/td> Low variance, can still have bias<\/td> Low bias, can overfit<\/td><\/tr> Example<\/td> Random Forest<\/td> GBDT, AdaBoost<\/td><\/tr><\/tbody><\/table><\/figure>\n\n\n\nThe key boosting idea:<\/strong> each new model is trained not on the original targets $y$, but on the errors<\/strong> of the current ensemble. The ensemble improves iteratively.<\/p>\n\n\n\nfig, axes = plt.subplots(1, 2, figsize=(13, 5))\n\n# \u2500\u2500 Bagging diagram \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\nax = axes[0]\nax.set_xlim(0, 10); ax.set_ylim(0, 8); ax.axis('off')\nax.set_title('Bagging (e.g. Random Forest)', fontsize=12, pad=10)\n\nax.text(5, 7.3, 'Original Data', ha='center', fontsize=10,\n bbox=dict(boxstyle='round', facecolor='lightblue'))\nfor i, x in enumerate([1.5, 3.5, 5.5, 7.5, 9.0]):\n if i < 4:\n ax.annotate('', xy=(x, 5.8), xytext=(5, 6.9),\n arrowprops=dict(arrowstyle='->', color='gray'))\n ax.text(x, 5.4, f'Bootstrap\\nSample {i+1}', ha='center', fontsize=8,\n bbox=dict(boxstyle='round', facecolor='lightyellow'))\n ax.annotate('', xy=(x, 3.8), xytext=(x, 5.0),\n arrowprops=dict(arrowstyle='->', color='gray'))\n ax.text(x, 3.3, f'Tree {i+1}', ha='center', fontsize=9,\n bbox=dict(boxstyle='round', facecolor='lightgreen'))\n ax.annotate('', xy=(5, 1.8), xytext=(x, 2.9),\n arrowprops=dict(arrowstyle='->', color='gray'))\nax.text(5, 1.3, 'Average\\nPredictions', ha='center', fontsize=10,\n bbox=dict(boxstyle='round', facecolor='salmon'))\n\n# \u2500\u2500 Boosting diagram \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\nax = axes[1]\nax.set_xlim(0, 10); ax.set_ylim(0, 8); ax.axis('off')\nax.set_title('Boosting (e.g. GBDT)', fontsize=12, pad=10)\n\nsteps = ['Train Tree 1\\non targets $y$',\n 'Train Tree 2\\non residuals $r_1$',\n 'Train Tree 3\\non residuals $r_2$',\n 'Final:\\n$F = T_1 + T_2 + T_3$']\ncolors = ['lightblue', 'lightyellow', 'lightgreen', 'salmon']\nys = [6.5, 4.8, 3.1, 1.3]\nfor i, (step, c, y) in enumerate(zip(steps, colors, ys)):\n ax.text(5, y, step, ha='center', fontsize=9,\n bbox=dict(boxstyle='round', facecolor=c))\n if i < len(steps) - 1:\n ax.annotate('', xy=(5, ys[i+1]+0.45), xytext=(5, y-0.1),\n arrowprops=dict(arrowstyle='->', color='gray', lw=1.5))\n\nplt.suptitle('Bagging vs. Boosting: structural difference', fontsize=13)\nplt.tight_layout()\nplt.show()\n<\/code><\/pre>\n\n\n\n![\"\"](\"https:\/\/tensorzen.online\/wp-content\/uploads\/2026\/06\/image-6-1024x428.png\")
<\/figure>\n\n\n\n3. The Gradient Boosting Algorithm<\/h2>\n\n\n\n
Gradient boosting frames boosting as gradient descent in function space<\/strong>.<\/p>\n\n\n\nSetup<\/h3>\n\n\n\n
We want to find a function $F(x)$ that minimises the expected loss:
$$\\mathcal{L} = \\sum_{i=1}^{n} L(y_i, F(x_i))$$<\/p>\n\n\n\n
Instead of optimising over parameters, we optimise over functions by taking gradient steps. At each step $m$, we ask:<\/p>\n\n\n\n
\nIn which direction should we nudge $F$ at each training point to most reduce the loss?<\/em><\/p>\n<\/blockquote>\n\n\n\nThe answer is the negative gradient<\/strong> of the loss with respect to $F(x_i)$:<\/p>\n\n\n\n$$r_{im} = -\\left[\\frac{\\partial L(y_i,\\, F(x_i))}{\\partial F(x_i)}\\right]{F = F<\/em>{m-1}}$$<\/p>\n\n\n\nThese $r_{im}$ are called pseudo-residuals<\/strong>.<\/p>\n\n\n\nAlgorithm<\/h3>\n\n\n\n
Initialise:<\/strong> $F_0(x) = \\arg\\min_\\gamma \\sum_{i} L(y_i, \\gamma)$ (a constant \u2014 e.g. the mean for MSE)<\/p>\n\n\n\nFor<\/strong> $m = 1, 2, \\ldots, M$:<\/p>\n\n\n\n\n- Compute pseudo-residuals: $r_{im} = -\\frac{\\partial L(y_i, F_{m-1}(x_i))}{\\partial F_{m-1}(x_i)}$<\/li>\n\n\n\n
- Fit a tree $h_m(x)$ to ${(x_i, r_{im})}$<\/li>\n\n\n\n
- Find the optimal step size: $\\gamma_m = \\arg\\min_\\gamma \\sum_i L(y_i,\\, F_{m-1}(x_i) + \\gamma\\, h_m(x_i))$<\/li>\n\n\n\n
- Update: $F_m(x) = F_{m-1}(x) + \\nu \\cdot \\gamma_m \\cdot h_m(x)$<\/li>\n<\/ol>\n\n\n\n
where $\\nu \\in (0,1]$ is the learning rate<\/strong> (shrinkage).<\/p>\n\n\n\nFor MSE loss<\/h3>\n\n\n\n
$$L(y, F) = \\tfrac{1}{2}(y – F)^2 \\implies r_{im} = y_i – F_{m-1}(x_i)$$<\/p>\n\n\n\n
The pseudo-residuals are exactly the ordinary residuals. Each tree fits the errors left by the current ensemble.<\/p>\n\n\n\n
4. GBDT from Scratch (Regression, MSE Loss)<\/h2>\n\n\n\n
We implement the algorithm step by step to see every moving part.<\/p>\n\n\n\n
class GBDTRegressor:\n """\n Gradient Boosted Decision Trees for regression with MSE loss.\n Pseudo-residuals = ordinary residuals (y - F).\n """\n def __init__(self, n_estimators=100, learning_rate=0.1, max_depth=3):\n self.n_estimators = n_estimators\n self.learning_rate = learning_rate\n self.max_depth = max_depth\n self.trees_ = []\n self.F0_ = None # initial constant prediction\n\n def fit(self, X, y):\n # Step 0: initialise with the mean\n self.F0_ = np.mean(y)\n F = np.full(len(y), self.F0_)\n\n for m in range(self.n_estimators):\n # Step 1: compute pseudo-residuals (negative gradient of MSE)\n residuals = y - F\n\n # Step 2: fit a tree to the pseudo-residuals\n tree = DecisionTreeRegressor(max_depth=self.max_depth, random_state=m)\n tree.fit(X, residuals)\n self.trees_.append(tree)\n\n # Step 3 & 4: update prediction\n F += self.learning_rate * tree.predict(X)\n\n return self\n\n def predict(self, X):\n F = np.full(len(X), self.F0_)\n for tree in self.trees_:\n F += self.learning_rate * tree.predict(X)\n return F\n\n def staged_predict(self, X):\n """Yield predictions after each boosting round."""\n F = np.full(len(X), self.F0_)\n yield F.copy()\n for tree in self.trees_:\n F += self.learning_rate * tree.predict(X)\n yield F.copy()<\/code><\/pre>\n\n\n\n# Train and visualise the sequential build-up\ngbdt_scratch = GBDTRegressor(n_estimators=50, learning_rate=0.3, max_depth=2)\ngbdt_scratch.fit(X_1d, y_1d)\n\nx_plot = np.linspace(0, 10, 500).reshape(-1, 1)\nstages = [1, 3, 8, 20, 50]\npredictions = list(gbdt_scratch.staged_predict(x_plot))\n\nfig, axes = plt.subplots(1, len(stages), figsize=(18, 4), sharey=True)\nfor ax, m in zip(axes, stages):\n ax.scatter(X_1d, y_1d, s=12, alpha=0.4, color='gray')\n ax.plot(x_plot, np.sin(x_plot), 'k--', lw=1.5, alpha=0.5, label='True')\n ax.plot(x_plot, predictions[m], color='crimson', lw=2, label=f'm={m}')\n train_mse = mean_squared_error(y_1d, list(gbdt_scratch.staged_predict(X_1d))[m])\n ax.set_title(f'{m} tree{"s" if m>1 else ""}\\nMSE={train_mse:.4f}')\n ax.legend(fontsize=8)\n ax.set_xlabel('x')\naxes[0].set_ylabel('y')\n\nplt.suptitle('GBDT from scratch: ensemble fit after $m$ trees (lr=0.3, depth=2)', fontsize=13)\nplt.tight_layout()\nplt.show()<\/code><\/pre>\n\n\n\n![\"\"](\"https:\/\/tensorzen.online\/wp-content\/uploads\/2026\/06\/image-7-1024x226.png\")
<\/figure>\n\n\n\n# Show the residuals being fitted at each step\nfig, axes = plt.subplots(1, 4, figsize=(16, 4))\nF = np.full(len(X_1d), gbdt_scratch.F0_)\n\nfor ax, m in zip(axes, [0, 1, 2, 4]):\n for i in range(m):\n F += gbdt_scratch.learning_rate * gbdt_scratch.trees_[i].predict(X_1d)\n residuals = y_1d - F\n ax.scatter(X_1d, residuals, s=12, alpha=0.5, color='steelblue')\n ax.axhline(0, color='crimson', lw=1.5, linestyle='--')\n ax.set_title(f'Residuals after {m} tree{"s" if m!=1 else ""}')\n ax.set_xlabel('x')\n rss = np.sum(residuals**2)\n ax.text(0.05, 0.95, f'RSS={rss:.2f}', transform=ax.transAxes, va='top', fontsize=9,\n bbox=dict(boxstyle='round', facecolor='wheat', alpha=0.7))\n F = np.full(len(X_1d), gbdt_scratch.F0_) # reset for next panel\n\naxes[0].set_ylabel('Residual')\nplt.suptitle('Pseudo-residuals shrink as more trees are added', fontsize=13)\nplt.tight_layout()\nplt.show()<\/code><\/pre>\n\n\n\n![\"\"](\"https:\/\/tensorzen.online\/wp-content\/uploads\/2026\/06\/image-8-1024x253.png\")
<\/figure>\n\n\n\n5. Loss Functions and Their Gradients<\/h2>\n\n\n\n
The choice of loss function determines what the pseudo-residuals represent.<\/p>\n\n\n\nTask<\/th> Loss $L(y, F)$<\/th> Pseudo-residual $r = -\\partial L \/ \\partial F$<\/th><\/tr><\/thead> Regression (MSE)<\/td> $\\frac{1}{2}(y-F)^2$<\/td> $y – F$<\/td><\/tr> Regression (MAE)<\/td> $<\/td> y – F<\/td><\/tr> Regression (Huber)<\/td> Quadratic near 0, linear in tails<\/td> Clipped residuals<\/td><\/tr> Classification<\/td> $\\log(1 + e^{-yF})$ (log-loss)<\/td> $y – \\sigma(F)$ where $\\sigma$ is sigmoid<\/td><\/tr><\/tbody><\/table><\/figure>\n\n\n\nThe gradient of MAE is just the sign \u2014 the tree fits the direction<\/em> of the error, not the magnitude. This makes MAE-based GBDT robust to outliers.<\/p>\n\n\n\nF_vals = np.linspace(-3, 3, 300)\ny_true = 1.0\n\nmse_loss = 0.5 * (y_true - F_vals)**2\nmse_grad = -(y_true - F_vals) # negative gradient = residual\nmae_loss = np.abs(y_true - F_vals)\nmae_grad = -np.sign(y_true - F_vals)\n\n# Huber\ndelta = 1.0\ndiff = y_true - F_vals\nhuber_loss = np.where(np.abs(diff) <= delta,\n 0.5 * diff**2,\n delta * (np.abs(diff) - 0.5 * delta))\nhuber_grad = np.where(np.abs(diff) <= delta, -diff, -delta * np.sign(diff))\n\nfig, axes = plt.subplots(1, 2, figsize=(13, 4))\n\nax = axes[0]\nax.plot(F_vals, mse_loss, label='MSE', color='steelblue', lw=2)\nax.plot(F_vals, mae_loss, label='MAE', color='crimson', lw=2)\nax.plot(F_vals, huber_loss, label='Huber', color='green', lw=2)\nax.axvline(y_true, color='gray', lw=1, linestyle='--', label=f'y={y_true}')\nax.set_xlabel('F(x)'); ax.set_ylabel('Loss'); ax.set_title('Loss functions')\nax.legend(); ax.grid(True, alpha=0.3)\n\nax = axes[1]\nax.plot(F_vals, mse_grad, label='MSE pseudo-residual', color='steelblue', lw=2)\nax.plot(F_vals, mae_grad, label='MAE pseudo-residual', color='crimson', lw=2)\nax.plot(F_vals, huber_grad, label='Huber pseudo-residual', color='green', lw=2)\nax.axvline(y_true, color='gray', lw=1, linestyle='--')\nax.axhline(0, color='gray', lw=0.7)\nax.set_xlabel('F(x)'); ax.set_ylabel('Pseudo-residual')\nax.set_title('Pseudo-residuals = negative gradient')\nax.legend(); ax.grid(True, alpha=0.3)\n\nplt.suptitle('Loss functions and their pseudo-residuals (y=1.0)', fontsize=13)\nplt.tight_layout()\nplt.show()<\/code><\/pre>\n\n\n\n![\"\"](\"https:\/\/tensorzen.online\/wp-content\/uploads\/2026\/06\/image-9-1024x310.png\")
<\/figure>\n\n\n\n6. Scikit-learn Implementation<\/h2>\n\n\n\n6a. Regression<\/h3>\n\n\n\nX_reg, y_reg = make_regression(n_samples=500, n_features=10, noise=20, random_state=42)\nX_tr, X_te, y_tr, y_te = train_test_split(X_reg, y_reg, test_size=0.2, random_state=42)\n\ngbr = GradientBoostingRegressor(\n n_estimators=200,\n learning_rate=0.1,\n max_depth=3,\n subsample=0.8, # stochastic gradient boosting\n random_state=42\n)\ngbr.fit(X_tr, y_tr)\n\n# Staged test error\ntrain_errors = [mean_squared_error(y_tr, pred) for pred in gbr.staged_predict(X_tr)]\ntest_errors = [mean_squared_error(y_te, pred) for pred in gbr.staged_predict(X_te)]\n\nfig, axes = plt.subplots(1, 2, figsize=(13, 4))\n\nax = axes[0]\nax.plot(train_errors, label='Train MSE', color='steelblue', lw=2)\nax.plot(test_errors, label='Test MSE', color='crimson', lw=2)\nax.set_xlabel('Number of trees'); ax.set_ylabel('MSE')\nax.set_title('Learning curve: MSE vs. number of trees')\nax.legend(); ax.grid(True, alpha=0.3)\n\nax = axes[1]\ny_pred = gbr.predict(X_te)\nax.scatter(y_te, y_pred, alpha=0.4, s=20, color='steelblue')\nmn, mx = min(y_te.min(), y_pred.min()), max(y_te.max(), y_pred.max())\nax.plot([mn, mx], [mn, mx], 'k--', lw=1.5, label='Perfect prediction')\nax.set_xlabel('True'); ax.set_ylabel('Predicted')\nax.set_title(f'Predicted vs. True (Test MSE={test_errors[-1]:.1f})')\nax.legend(); ax.grid(True, alpha=0.3)\n\nplt.suptitle('GradientBoostingRegressor \u2014 scikit-learn', fontsize=13)\nplt.tight_layout()\nplt.show()\n\nprint(f"Final Train MSE : {train_errors[-1]:.2f}")\nprint(f"Final Test MSE : {test_errors[-1]:.2f}")<\/code><\/pre>\n\n\n\n6b. Classification<\/h3>\n\n\n\n
For binary classification GBDT uses log-loss<\/strong> (cross-entropy). The model outputs a log-odds score $F(x)$, converted to a probability via the sigmoid:<\/p>\n\n\n\n$$P(y=1 \\mid x) = \\sigma(F(x)) = \\frac{1}{1 + e^{-F(x)}}$$<\/p>\n\n\n\n
The pseudo-residuals become: $r_i = y_i – \\sigma(F(x_i))$ \u2014 the difference between the true label and the predicted probability.<\/p>\n\n\n\n
X_moons, y_moons = make_moons(n_samples=600, noise=0.25, random_state=42)\nX_tr_c, X_te_c, y_tr_c, y_te_c = train_test_split(X_moons, y_moons, test_size=0.25, random_state=42)\n\ngbc = GradientBoostingClassifier(n_estimators=200, learning_rate=0.1,\n max_depth=3, random_state=42)\ngbc.fit(X_tr_c, y_tr_c)\n\n# Decision boundary\ndef plot_decision_boundary(ax, model, X, y, title):\n h = 0.03\n x_min, x_max = X[:,0].min()-0.5, X[:,0].max()+0.5\n y_min, y_max = X[:,1].min()-0.5, X[:,1].max()+0.5\n xx, yy = np.meshgrid(np.arange(x_min, x_max, h),\n np.arange(y_min, y_max, h))\n Z = model.predict_proba(np.c_[xx.ravel(), yy.ravel()])[:,1]\n Z = Z.reshape(xx.shape)\n ax.contourf(xx, yy, Z, alpha=0.3, cmap='RdBu_r', levels=20)\n ax.contour(xx, yy, Z, levels=[0.5], colors='black', linewidths=1.5)\n ax.scatter(X[:,0], X[:,1], c=y, cmap='RdBu_r', s=20, edgecolors='k', lw=0.3)\n acc = accuracy_score(y, model.predict(X))\n ax.set_title(f'{title}\\nAccuracy={acc:.3f}')\n\nfig, axes = plt.subplots(1, 3, figsize=(15, 4))\n\n# Show progression\nfor ax, n in zip(axes, [5, 30, 200]):\n m = GradientBoostingClassifier(n_estimators=n, learning_rate=0.1,\n max_depth=3, random_state=42)\n m.fit(X_tr_c, y_tr_c)\n plot_decision_boundary(ax, m, X_te_c, y_te_c, f'{n} trees')\n\nplt.suptitle('GBDT classification on moons dataset: decision boundary evolution', fontsize=13)\nplt.tight_layout()\nplt.show()<\/code><\/pre>\n\n\n\n7. Hyperparameter Effects<\/h2>\n\n\n\n
GBDT has several important hyperparameters:<\/p>\n\n\n\nParameter<\/th> Effect<\/th><\/tr><\/thead> n_estimators<\/code><\/td>More trees \u2192 lower bias, risk of overfitting<\/td><\/tr> learning_rate<\/code><\/td>Smaller \u2192 need more trees, but generalises better<\/td><\/tr> max_depth<\/code><\/td>Controls complexity of each tree<\/td><\/tr> subsample<\/code><\/td>Fraction of data per tree \u2014 reduces variance (stochastic GB)<\/td><\/tr> min_samples_leaf<\/code><\/td>Smooths leaf predictions, reduces overfitting<\/td><\/tr><\/tbody><\/table><\/figure>\n\n\n\nThe learning rate\u2013n_estimators trade-off:<\/strong> a small learning rate requires many trees to achieve the same training loss, but typically yields better generalisation. These two are always tuned together.<\/p>\n\n\n\nfig, axes = plt.subplots(2, 2, figsize=(13, 10))\n\n# \u2500\u2500 1. Learning rate \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\nax = axes[0][0]\nfor lr in [0.01, 0.05, 0.1, 0.5]:\n m = GradientBoostingRegressor(n_estimators=200, learning_rate=lr,\n max_depth=3, random_state=42)\n m.fit(X_tr, y_tr)\n te = [mean_squared_error(y_te, p) for p in m.staged_predict(X_te)]\n ax.plot(te, label=f'lr={lr}', lw=1.8)\nax.set_xlabel('n_estimators'); ax.set_ylabel('Test MSE')\nax.set_title('Effect of learning rate'); ax.legend(); ax.grid(True, alpha=0.3)\n\n# \u2500\u2500 2. Tree depth \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\nax = axes[0][1]\nfor d in [1, 2, 3, 5, 8]:\n m = GradientBoostingRegressor(n_estimators=200, learning_rate=0.1,\n max_depth=d, random_state=42)\n m.fit(X_tr, y_tr)\n te = [mean_squared_error(y_te, p) for p in m.staged_predict(X_te)]\n ax.plot(te, label=f'depth={d}', lw=1.8)\nax.set_xlabel('n_estimators'); ax.set_ylabel('Test MSE')\nax.set_title('Effect of tree depth'); ax.legend(); ax.grid(True, alpha=0.3)\n\n# \u2500\u2500 3. Subsample \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\nax = axes[1][0]\nfor ss in [0.3, 0.5, 0.8, 1.0]:\n m = GradientBoostingRegressor(n_estimators=200, learning_rate=0.1,\n max_depth=3, subsample=ss, random_state=42)\n m.fit(X_tr, y_tr)\n te = [mean_squared_error(y_te, p) for p in m.staged_predict(X_te)]\n ax.plot(te, label=f'subsample={ss}', lw=1.8)\nax.set_xlabel('n_estimators'); ax.set_ylabel('Test MSE')\nax.set_title('Effect of subsample (stochastic GB)'); ax.legend(); ax.grid(True, alpha=0.3)\n\n# \u2500\u2500 4. lr \u00d7 n_estimators trade-off \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\nax = axes[1][1]\nconfigs = [(0.5, 40), (0.1, 200), (0.05, 400), (0.01, 2000)]\nresults = []\nfor lr, n in configs:\n m = GradientBoostingRegressor(n_estimators=n, learning_rate=lr,\n max_depth=3, random_state=42)\n m.fit(X_tr, y_tr)\n results.append(mean_squared_error(y_te, m.predict(X_te)))\nlabels = [f'lr={lr}, n={n}' for lr, n in configs]\nbars = ax.bar(labels, results, color=['steelblue','darkorange','green','crimson'], alpha=0.8)\nax.set_ylabel('Test MSE')\nax.set_title('lr \u00d7 n_estimators trade-off\\n(same approximate total "work")')\nax.tick_params(axis='x', rotation=15)\nfor bar, val in zip(bars, results):\n ax.text(bar.get_x() + bar.get_width()\/2, bar.get_height() + 5,\n f'{val:.0f}', ha='center', fontsize=9)\nax.grid(True, alpha=0.3, axis='y')\n\nplt.suptitle('Hyperparameter effects on test MSE', fontsize=13)\nplt.tight_layout()\nplt.show()<\/code><\/pre>\n\n\n\n8. Feature Importance<\/h2>\n\n\n\n
GBDT computes feature importance as the total reduction in loss<\/strong> (impurity) due to splits on each feature, summed across all trees:<\/p>\n\n\n\n$$\\text{importance}(f) = \\sum_{m=1}^{M} \\sum_{\\text{nodes } t \\in h_m \\text{ that split on } f} \\Delta L_t$$<\/p>\n\n\n\n
where $\\Delta L_t$ is the reduction in the split criterion at node $t$.<\/p>\n\n\n\n
Limitations:<\/strong> this measure favours high-cardinality features and can be misleading when features are correlated. Permutation importance is a more reliable alternative.<\/p>\n\n\n\nfrom sklearn.inspection import permutation_importance\n\nfeature_names = [f'Feature {i+1}' for i in range(X_reg.shape[1])]\n\n# Built-in impurity importance\nimp_builtin = gbr.feature_importances_\n\n# Permutation importance (on test set)\nperm = permutation_importance(gbr, X_te, y_te, n_repeats=20, random_state=42)\nimp_perm = perm.importances_mean\n\nidx_b = np.argsort(imp_builtin)[::-1]\nidx_p = np.argsort(imp_perm)[::-1]\n\nfig, axes = plt.subplots(1, 2, figsize=(13, 4))\n\nax = axes[0]\nax.bar(range(len(idx_b)), imp_builtin[idx_b], color='steelblue', alpha=0.8)\nax.set_xticks(range(len(idx_b)))\nax.set_xticklabels([feature_names[i] for i in idx_b], rotation=45, ha='right')\nax.set_title('Built-in impurity-based importance'); ax.set_ylabel('Importance')\nax.grid(True, alpha=0.3, axis='y')\n\nax = axes[1]\nax.bar(range(len(idx_p)), imp_perm[idx_p], color='crimson', alpha=0.8,\n yerr=perm.importances_std[idx_p], capsize=4)\nax.set_xticks(range(len(idx_p)))\nax.set_xticklabels([feature_names[i] for i in idx_p], rotation=45, ha='right')\nax.set_title('Permutation importance (test set)'); ax.set_ylabel('Mean MSE increase')\nax.grid(True, alpha=0.3, axis='y')\n\nplt.suptitle('Feature importance: impurity-based vs. permutation', fontsize=13)\nplt.tight_layout()\nplt.show()<\/code><\/pre>\n\n\n\n9. Overfitting and Regularisation<\/h2>\n\n\n\n
GBDT can overfit, especially with:<\/p>\n\n\n\n
\n- Too many trees (
n_estimators<\/code> too large without shrinkage)<\/li>\n\n\n\n- Too large a learning rate<\/li>\n\n\n\n
- Too deep trees (
max_depth<\/code>)<\/li>\n<\/ul>\n\n\n\nRegularisation strategies:<\/strong><\/p>\n\n\n\nStrategy<\/th> Parameter<\/th> Mechanism<\/th><\/tr><\/thead> Shrinkage<\/td> learning_rate<\/code><\/td>Scales each tree’s contribution down<\/td><\/tr> Stochastic GB<\/td> subsample < 1<\/code><\/td>Trains each tree on a random subsample<\/td><\/tr> Tree constraints<\/td> max_depth<\/code>, min_samples_leaf<\/code><\/td>Limits individual tree complexity<\/td><\/tr> Early stopping<\/td> n_iter_no_change<\/code><\/td>Halts when validation loss stops improving<\/td><\/tr><\/tbody><\/table><\/figure>\n\n\n\n# Demonstrate overfitting then early stopping\nfig, axes = plt.subplots(1, 2, figsize=(13, 4))\n\n# \u2500\u2500 Overfitting: high lr, deep trees \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\nax = axes[0]\nm_overfit = GradientBoostingRegressor(n_estimators=500, learning_rate=0.5,\n max_depth=6, random_state=42)\nm_overfit.fit(X_tr, y_tr)\ntr_e = [mean_squared_error(y_tr, p) for p in m_overfit.staged_predict(X_tr)]\nte_e = [mean_squared_error(y_te, p) for p in m_overfit.staged_predict(X_te)]\nbest_n = np.argmin(te_e)\nax.plot(tr_e, label='Train', color='steelblue', lw=2)\nax.plot(te_e, label='Test', color='crimson', lw=2)\nax.axvline(best_n, color='green', lw=2, linestyle='--',\n label=f'Best test @ n={best_n}')\nax.set_xlabel('n_estimators'); ax.set_ylabel('MSE')\nax.set_title('Overfitting: high lr (0.5), deep trees (depth=6)')\nax.legend(); ax.grid(True, alpha=0.3)\n\n# \u2500\u2500 Early stopping \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\nax = axes[1]\nm_es = GradientBoostingRegressor(\n n_estimators=500,\n learning_rate=0.5,\n max_depth=6,\n n_iter_no_change=15, # stop if no improvement for 15 rounds\n validation_fraction=0.15,\n random_state=42\n)\nm_es.fit(X_tr, y_tr)\nte_e2 = [mean_squared_error(y_te, p) for p in m_es.staged_predict(X_te)]\nax.plot(te_e, label='Without early stopping', color='crimson', lw=2, alpha=0.7)\nax.plot(te_e2, label='With early stopping', color='steelblue', lw=2)\nax.axvline(m_es.n_estimators_ - 1, color='green', lw=2, linestyle='--',\n label=f'Stopped @ n={m_es.n_estimators_}')\nax.set_xlabel('n_estimators'); ax.set_ylabel('Test MSE')\nax.set_title('Early stopping prevents overfitting')\nax.legend(); ax.grid(True, alpha=0.3)\n\nplt.suptitle('Overfitting and early stopping in GBDT', fontsize=13)\nplt.tight_layout()\nplt.show()\n\nprint(f"Without early stopping \u2014 final test MSE : {te_e[-1]:.2f} ({len(te_e)} trees)")\nprint(f"With early stopping \u2014 final test MSE : {te_e2[-1]:.2f} ({m_es.n_estimators_} trees)")<\/code><\/pre>\n\n\n\n10. Random Forest vs. GBDT<\/h2>\n\n\n\n
Both are tree ensembles but operate on different principles. Neither is universally better \u2014 the choice depends on the problem.<\/p>\n\n\n\n
from sklearn.ensemble import RandomForestClassifier\n\nX_cl, y_cl = make_classification(n_samples=1000, n_features=20, n_informative=10,\n n_redundant=5, random_state=42)\nX_tr2, X_te2, y_tr2, y_te2 = train_test_split(X_cl, y_cl, test_size=0.25, random_state=42)\n\nn_range = [1, 5, 10, 20, 50, 100, 200]\nrf_scores, gbdt_scores = [], []\n\nfor n in n_range:\n rf = RandomForestClassifier(n_estimators=n, random_state=42)\n rf.fit(X_tr2, y_tr2)\n rf_scores.append(accuracy_score(y_te2, rf.predict(X_te2)))\n\n gb = GradientBoostingClassifier(n_estimators=n, learning_rate=0.1,\n max_depth=3, random_state=42)\n gb.fit(X_tr2, y_tr2)\n gbdt_scores.append(accuracy_score(y_te2, gb.predict(X_te2)))\n\nfig, axes = plt.subplots(1, 2, figsize=(13, 4))\n\nax = axes[0]\nax.plot(n_range, rf_scores, 'o-', label='Random Forest', color='steelblue', lw=2)\nax.plot(n_range, gbdt_scores, 's-', label='GBDT', color='crimson', lw=2)\nax.set_xlabel('n_estimators'); ax.set_ylabel('Test Accuracy')\nax.set_title('Accuracy vs. number of trees')\nax.legend(); ax.grid(True, alpha=0.3)\n\n# Side-by-side comparison on moons\nax = axes[1]\nmodels = [\n ('Random Forest', RandomForestClassifier(n_estimators=200, random_state=42)),\n ('GBDT', GradientBoostingClassifier(n_estimators=200, learning_rate=0.1,\n max_depth=3, random_state=42)),\n]\nfor name, mdl in models:\n mdl.fit(X_tr_c, y_tr_c)\naccs = [accuracy_score(y_te_c, m.predict(X_te_c)) for _, m in models]\nnames = [n for n, _ in models]\nbars = ax.bar(names, accs, color=['steelblue', 'crimson'], alpha=0.8)\nfor bar, acc in zip(bars, accs):\n ax.text(bar.get_x() + bar.get_width()\/2, bar.get_height() - 0.02,\n f'{acc:.3f}', ha='center', va='top', color='white', fontweight='bold')\nax.set_ylim(0.8, 1.0); ax.set_ylabel('Test Accuracy')\nax.set_title('Accuracy on moons dataset (n=200 trees)')\nax.grid(True, alpha=0.3, axis='y')\n\nplt.suptitle('Random Forest vs. GBDT', fontsize=13)\nplt.tight_layout()\nplt.show()\n\nprint("\\nRandom Forest vs. GBDT \u2014 summary:")\nprint(f"{'Property':<30} {'Random Forest':<22} {'GBDT'}")\nprint("-" * 70)\nrows = [\n ('Training strategy', 'Parallel (independent)', 'Sequential (corrective)'),\n ('Bias-variance', 'Reduces variance', 'Reduces bias'),\n ('Tree depth', 'Deep trees', 'Shallow trees'),\n ('Overfitting risk', 'Lower', 'Higher (needs tuning)'),\n ('Training speed', 'Faster (parallelisable)','Slower (sequential)'),\n ('Hyperparameter sens.', 'Lower', 'Higher'),\n ('Performance (typical)','Strong baseline', 'Often best in class'),\n]\nfor prop, rf, gbdt in rows:\n print(f"{prop:<30} {rf:<22} {gbdt}")<\/code><\/pre>\n\n\n\nSummary<\/h2>\n\n\n\nConcept<\/th> Key point<\/th><\/tr><\/thead> Weak learner<\/td> Shallow decision tree \u2014 high bias, low variance<\/td><\/tr> Boosting idea<\/td> Each new tree corrects the errors of the current ensemble<\/td><\/tr> Gradient descent analogy<\/td> Pseudo-residuals are the negative gradient of the loss w.r.t. $F$<\/td><\/tr> MSE pseudo-residuals<\/td> Ordinary residuals $y – F$<\/td><\/tr> Log-loss pseudo-residuals<\/td> $y – \\sigma(F)$ \u2014 label minus predicted probability<\/td><\/tr> Learning rate $\\nu$<\/td> Shrinks each tree’s contribution \u2014 trades speed for generalisation<\/td><\/tr> Subsample<\/td> Stochastic gradient boosting \u2014 reduces variance, speeds training<\/td><\/tr> Feature importance<\/td> Total impurity reduction across all splits on that feature<\/td><\/tr> Overfitting<\/td> Controlled via learning rate, depth, subsample, and early stopping<\/td><\/tr> vs. Random Forest<\/td> GBDT reduces bias (sequential); RF reduces variance (parallel)<\/td><\/tr><\/tbody><\/table><\/figure>\n\n\n\n<\/p>\n","protected":false},"excerpt":{"rendered":"
This notebook builds a complete understanding of GBDT from the ground up: 1. Decision Trees: The Weak Learner GBDT builds an ensemble of decision trees. Each tree partitions […]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[13,25,4],"tags":[],"class_list":["post-1000","post","type-post","status-publish","format-standard","hentry","category-classical-algorithm","category-in-english","category-machine-learning"],"_links":{"self":[{"href":"https:\/\/tensorzen.online\/index.php?rest_route=\/wp\/v2\/posts\/1000","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/tensorzen.online\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/tensorzen.online\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/tensorzen.online\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/tensorzen.online\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=1000"}],"version-history":[{"count":1,"href":"https:\/\/tensorzen.online\/index.php?rest_route=\/wp\/v2\/posts\/1000\/revisions"}],"predecessor-version":[{"id":1006,"href":"https:\/\/tensorzen.online\/index.php?rest_route=\/wp\/v2\/posts\/1000\/revisions\/1006"}],"wp:attachment":[{"href":"https:\/\/tensorzen.online\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=1000"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/tensorzen.online\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=1000"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/tensorzen.online\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=1000"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}