``````enum class Optimization {
SGD_Momentum, // SGD with Momentum
RMSProp, // Root Mean Square Propagation
};``````

``````void LogisticRegression2::calculate_gradient_descent(int start, int end)
{
switch (optim_) {
int len = end - start;
std::vector m(feature_length_, 0.), u(feature_length_, 1e-8), mhat(feature_length_, 0.);
std::vector z(len, 0.), dz(len, 0.);
float beta1t = 1.;
for (int i = start, x = 0; i < end; ++i, ++x) {
z[x] = calculate_z(data_->samples[random_shuffle_[i]]);
dz[x] = calculate_loss_function_derivative(calculate_activation_function(z[x]), data_->labels[random_shuffle_[i]]);

beta1t *= beta1_;

for (int j = 0; j < feature_length_; ++j) {
float dw = data_->samples[random_shuffle_[i]][j] * dz[x];
m[j] = beta1_ * m[j] + (1. - beta1_) * dw; // formula 19
u[j] = std::max(beta2_ * u[j], std::fabs(dw)); // formula 24

mhat[j] = m[j] / (1. - beta1t); // formula 20

// Note: need to ensure than u[j] cannot be 0.
// (1). u[j] is initialized to 1e-8, or
// (2). if u[j] is initialized to 0., then u[j] adjusts to (u[j] + 1e-8)
w_[j] = w_[j] - alpha_ * mhat[j] / u[j]; // formula 25
}

b_ -= (alpha_ * dz[x]);
}
}
break;
int len = end - start;
std::vector m(feature_length_, 0.), v(feature_length_, 0.), mhat(feature_length_, 0.), vhat(feature_length_, 0.);
std::vector z(len, 0.), dz(len, 0.);
float beta1t = 1., beta2t = 1.;
for (int i = start, x = 0; i < end; ++i, ++x) {
z[x] = calculate_z(data_->samples[random_shuffle_[i]]);
dz[x] = calculate_loss_function_derivative(calculate_activation_function(z[x]), data_->labels[random_shuffle_[i]]);

beta1t *= beta1_;
beta2t *= beta2_;

for (int j = 0; j < feature_length_; ++j) {
float dw = data_->samples[random_shuffle_[i]][j] * dz[x];
m[j] = beta1_ * m[j] + (1. - beta1_) * dw; // formula 19
v[j] = beta2_ * v[j] + (1. - beta2_) * (dw * dw); // formula 19

mhat[j] = m[j] / (1. - beta1t); // formula 20
vhat[j] = v[j] / (1. - beta2t); // formula 20

w_[j] = w_[j] - alpha_ * mhat[j] / (std::sqrt(vhat[j]) + eps_); // formula 21
}

b_ -= (alpha_ * dz[x]);
}
}
break;
int len = end - start;
std::vector g(feature_length_, 0.), p(feature_length_, 0.);
std::vector z(len, 0.), dz(len, 0.);
for (int i = start, x = 0; i < end; ++i, ++x) {
z[x] = calculate_z(data_->samples[random_shuffle_[i]]);
dz[x] = calculate_loss_function_derivative(calculate_activation_function(z[x]), data_->labels[random_shuffle_[i]]);

for (int j = 0; j < feature_length_; ++j) {
float dw = data_->samples[random_shuffle_[i]][j] * dz[x];
g[j] = mu_ * g[j] + (1. - mu_) * (dw * dw); // formula 10

float alpha = (eps_ + std::sqrt(p[j])) / (eps_ + std::sqrt(g[j]));
float change = alpha * dw;
p[j] = mu_ * p[j] +  (1. - mu_) * (change * change); // formula 15

w_[j] = w_[j] - change;
}

b_ -= (eps_ * dz[x]);
}
}
break;
case Optimization::RMSProp: {
int len = end - start;
std::vector g(feature_length_, 0.);
std::vector z(len, 0), dz(len, 0);
for (int i = start, x = 0; i < end; ++i, ++x) {
z[x] = calculate_z(data_->samples[random_shuffle_[i]]);
dz[x] = calculate_loss_function_derivative(calculate_activation_function(z[x]), data_->labels[random_shuffle_[i]]);

for (int j = 0; j < feature_length_; ++j) {
float dw = data_->samples[random_shuffle_[i]][j] * dz[x];
g[j] = mu_ * g[j] + (1. - mu_) * (dw * dw); // formula 18
w_[j] = w_[j] - alpha_ * dw / (std::sqrt(g[j]) + eps_);
}

b_ -= (alpha_ * dz[x]);
}
}
break;
int len = end - start;
std::vector g(feature_length_, 0.);
std::vector z(len, 0), dz(len, 0);
for (int i = start, x = 0; i < end; ++i, ++x) {
z[x] = calculate_z(data_->samples[random_shuffle_[i]]);
dz[x] = calculate_loss_function_derivative(calculate_activation_function(z[x]), data_->labels[random_shuffle_[i]]);

for (int j = 0; j < feature_length_; ++j) {
float dw = data_->samples[random_shuffle_[i]][j] * dz[x];
g[j] += dw * dw;
w_[j] = w_[j] - alpha_ * dw / (std::sqrt(g[j]) + eps_);
}

b_ -= (alpha_ * dz[x]);
}
}
break;
case Optimization::SGD_Momentum: {
int len = end - start;
std::vector change(feature_length_, 0.);
std::vector z(len, 0), dz(len, 0);
for (int i = start, x = 0; i < end; ++i, ++x) {
z[x] = calculate_z(data_->samples[random_shuffle_[i]]);
dz[x] = calculate_loss_function_derivative(calculate_activation_function(z[x]), data_->labels[random_shuffle_[i]]);

for (int j = 0; j < feature_length_; ++j) {
float new_change = mu_ * change[j] - alpha_ * (data_->samples[random_shuffle_[i]][j] * dz[x]);
w_[j] += new_change;
change[j] = new_change;
}

b_ -= (alpha_ * dz[x]);
}
}
break;
case Optimization::SGD:
case Optimization::MBGD: {
int len = end - start;
std::vector z(len, 0), dz(len, 0);
for (int i = start, x = 0; i < end; ++i, ++x) {
z[x] = calculate_z(data_->samples[random_shuffle_[i]]);
dz[x] = calculate_loss_function_derivative(calculate_activation_function(z[x]), data_->labels[random_shuffle_[i]]);

for (int j = 0; j < feature_length_; ++j) {
w_[j] = w_[j] - alpha_ * (data_->samples[random_shuffle_[i]][j] * dz[x]);
}

b_ -= (alpha_ * dz[x]);
}
}
break;
case Optimization::BGD:
default: // BGD
std::vector z(m_, 0), dz(m_, 0);
float db = 0.;
std::vector dw(feature_length_, 0.);
for (int i = 0; i < m_; ++i) {
z[i] = calculate_z(data_->samples[i]);
o_[i] = calculate_activation_function(z[i]);
dz[i] = calculate_loss_function_derivative(o_[i], data_->labels[i]);

for (int j = 0; j < feature_length_; ++j) {
dw[j] += data_->samples[i][j] * dz[i]; // dw(i)+=x(i)(j)*dz(i)
}
db += dz[i]; // db+=dz(i)
}

for (int j = 0; j < feature_length_; ++j) {
dw[j] /= m_;
w_[j] -= alpha_ * dw[j];
}

b_ -= alpha_*(db/m_);
}
}``````