L1 regularization adds the sum of absolute model weights to the cost function. Increasing λ strengthens the penalty, shrinking unimportant weights to exactly zero and selecting a smaller set of features. This is useful for sparse molecular fingerprints, where correlated features can make coefficients unstable and encourage overfitting. section 3.4.1
Use L1 regularization when:
- You need feature selection because the dataset contains many descriptors relative to the number of ligands. section 6.3.1
- You want a simpler, more interpretable model while controlling overfitting. section 3.4.1
- You can tune λ with cross-validation rather than choosing penalty strength arbitrarily. section 6.3.1
Selecting descriptors before QSAR regression
For a small RNA-binding dataset, standardize the descriptors, search 1,000 Lasso models across L1 values from 10² to 10⁻⁶, and select the model with the best five-fold cross-validated R². Retain descriptors whose coefficients are nonzero, then fit multiple linear regression models using descriptor combinations and compare training R² with test-set Q². section 6.3.1
Listing 6.7 Lasso Feature Selection and Exhaustive MLR Search
def mask_train_test_split(X, y, test_idxs):
mask = y.index.isin(test_idxs)
sc = StandardScaler()
X_train = sc.fit_transform(X[~mask])
X_test = sc.transform(X[mask])
return X_train, X_test, y[~mask], y[mask]
def lasso_feature_selection(X_train, X_test, y_train, y_test):
param_grid = {'alpha': np.logspace(2, -6, num=1000)}
lasso = Lasso()
grid_search = GridSearchCV(
lasso, param_grid, cv=5, scoring='r2', n_jobs=-1
)
grid_search.fit(X_train, y_train)
best_alpha = grid_search.best_params_['alpha']
best_model = grid_search.best_estimator_
y_pred = best_model.predict(X_test)
r2 = r2_score(y_test, y_pred)
print(f"Best alpha: {best_alpha}")
print(f"Best R2: {r2}")
selected_features = np.where(best_model.coef_ != 0)[0]
return best_alpha, selected_features
def compute_feature_pvals(model, X, y):
# Calculate the standard errors of the coefficients
n = len(y)
p = X.shape[1]
y_pred = model.predict(X)
residuals = y - y_pred
mse = np.sum(residuals**2) / (n - p - 1)
var_b = mse * (np.linalg.inv(np.dot(X.T, X)).diagonal())
std_err = np.sqrt(var_b)
# Calculate the t-statistics and p-values
t_stat = model.coef_ / std_err
p_values = 2 * (1 - stats.t.cdf(np.abs(t_stat), n - p - 1))
return np.round(p_values, 3)
def MLR_exhaustive_search(
X_train, X_test, y_train, y_test,
selected_features, threshold=0.7, max_descs=7
):
best_score = -np.inf
best_model = None
best_p_values = None
best_features = None
X_train = X_train[:, selected_features]
X_test = X_test[:, selected_features]
for r in range(2, max_descs+1):
for feature_combination in itertools.combinations(
range(X_train.shape[1]), r
):
model = LinearRegression()
model.fit(X_train[:, feature_combination], y_train)
y_train_pred = model.predict(X_train[:, feature_combination])
y_test_pred = model.predict(X_test[:, feature_combination])
r2_train = r2_score(y_train, y_train_pred)
q2_test = r2_score(y_test, y_test_pred)
if r2_train <= threshold or q2_test <= threshold:
continue
p_values = compute_feature_pvals(
model, X_train[:, feature_combination], y_train
)
if q2_test > best_score and np.all(p_values < 0.05):
best_score = q2_test
best_model = model
best_features = feature_combination
best_p_values = p_values
best_selected_features = [
selected_features[feature] for feature in best_features
]
print(f"Best Q2: {best_score}")
print(f"Best Features: {best_features}")
return best_model, best_selected_features, best_score, best_p_values
def paper_MLR_workflow(X, y, test_idxs, threshold=0.7, max_descs=7):
X_train, X_test, y_train, y_test = mask_train_test_split(
X, y, test_idxs
)
print("Begin Lasso Feature Selection")
best_alpha, selected_features = lasso_feature_selection(
X_train, X_test, y_train, y_test
)
print("Screen Performative MLR Models")
return MLR_exhaustive_search(
X_train, X_test, y_train, y_test,
selected_features, threshold, max_descs
)
best_model_kon, best_features_kon, best_score_kon, best_p_values_kon = \
paper_MLR_workflow(
hiv_tar1_lnkon.drop(columns=["lnkon"]),
hiv_tar1_lnkon["lnkon"], test_idxs_paper
)
best_model_koff, best_features_koff, best_score_koff, best_p_values_koff = paper_MLR_workflow(hiv_tar1_lnkoff.drop(columns=["lnkoff"]), hiv_tar1_lnkoff["lnkoff"], test_idxs_paper, threshold=0.6)
best_model_kd, best_features_kd, best_score_kd, best_p_values_kd = \
paper_MLR_workflow(
hiv_tar1_lnkd.drop(columns=["lnKD"]),
hiv_tar1_lnkd["lnKD"], test_idxs_paper, max_descs=4
)Inspect the runnable workflow for scaling descriptors, tuning Lasso with GridSearchCV, predicting held-out data, and extracting nonzero coefficients. section 6.3.1







