41 using namespace shogun;
75 REQUIRE(
"i<j",
"i(%i) must be less that j(%i)\n", i, j);
102 int32_t num_inputs = 0;
113 input_indices[k] = i;
142 get_section(m_param_regularizable, i);
145 layer_param_regularizable, sigma);
165 if (output_activations[i]>0.5) labels_vec[i] = 1;
166 else labels_vec[i] = -1;
170 if (output_activations[2*i]>output_activations[2*i+1])
172 else labels_vec[i] = -1;
185 labels_vec[i] = output_activations[i];
250 "Gradient descent momentum (%f) must be > 0\n",
gd_momentum);
252 int32_t training_set_size = inputs.
num_cols;
261 param_updates.
zero();
263 float64_t error_last_time = -1.0, error = 0;
269 bool continue_training =
true;
272 for (int32_t i=0; continue_training; i++)
288 m_num_inputs, gd_mini_batch_size,
false);
290 for (int32_t k=0; k<n_param; k++)
294 error = (1.0-c) * error +
297 for (int32_t k=0; k<n_param; k++)
305 if (error_last_time!=-1.0)
307 float64_t error_change = (error_last_time-error)/error;
308 if (error_change< epsilon && error_change>=0)
310 SG_INFO(
"Gradient Descent Optimization Converged\n");
311 continue_training =
false;
315 SG_INFO(
"Epoch %i: Error = %f\n",i, error);
317 error_last_time = error;
327 int32_t training_set_size = inputs.
num_cols;
334 lbfgs_param.
past = 1;
337 m_lbfgs_temp_inputs = &inputs;
338 m_lbfgs_temp_targets = &targets;
343 &CNeuralNetwork::lbfgs_evaluate,
344 &CNeuralNetwork::lbfgs_progress,
348 m_lbfgs_temp_inputs = NULL;
349 m_lbfgs_temp_targets = NULL;
353 SG_INFO(
"L-BFGS Optimization Converged\n");
357 SG_INFO(
"L-BFGS Max Number of Epochs reached\n");
361 SG_INFO(
"L-BFGS optimization ended with return code %i\n",result);
366 float64_t CNeuralNetwork::lbfgs_evaluate(
void* userdata,
377 *network->m_lbfgs_temp_targets, grad_vector);
380 int CNeuralNetwork::lbfgs_progress(
void* instance,
387 int n,
int k,
int ls)
389 SG_SINFO(
"Epoch %i: Error = %f\n",k, fx);
406 for (int32_t i=0; i<=j; i++)
429 for (int32_t i=m_num_layers-1; i>=0; i--)
431 if (i==m_num_layers-1)
433 m_layers, get_section(gradients,i));
514 for (int32_t i=0; i<x.
num_rows; i++)
519 for (int32_t i=0; i<y.
num_rows; i++)
523 for (int32_t i=0; i<y.
num_rows; i++)
542 gradients_numerical[i] = (error_plus-error_minus)/(2*c);
552 sum +=
CMath::abs(gradients_backprop[i]-gradients_numerical[i]);
570 REQUIRE(features != NULL,
"Invalid (NULL) feature pointer\n");
572 "Feature type must be F_DREAL\n");
574 "Feature class must be C_DENSE\n");
578 "Number of features (%i) must match the network's number of inputs "
581 return inputs->get_feature_matrix();
586 REQUIRE(labs != NULL,
"Invalid (NULL) labels pointer\n");
595 "Number of classes (%i) must match the network's number of "
608 targets[i] = (labels_bin->
get_label(i)==1);
614 targets[i*2] = (labels_bin->
get_label(i)==1);
615 targets[i*2+1] = (labels_bin->
get_label(i)==-1);
655 "with more that 2 output neurons\n", lab->
get_name());
660 "with more that 1 output neuron\n", lab->
get_name());
668 REQUIRE(i<m_num_layers && i >= 0,
"Layer index (%i) out of range\n", i);
704 void CNeuralNetwork::init()
724 m_lbfgs_temp_inputs = NULL;
725 m_lbfgs_temp_targets = NULL;
771 "DynamicObjectArray of NeuralNetwork objects",
virtual const char * get_name() const =0
virtual float64_t compute_error(SGMatrix< float64_t > targets)
SGVector< int32_t > m_index_offsets
virtual void initialize(CDynamicObjectArray *layers, SGVector< int32_t > input_indices)
virtual CBinaryLabels * apply_binary(CFeatures *data)
virtual ELabelType get_label_type() const =0
Real Labels are real-valued labels.
virtual int32_t get_num_parameters()
int32_t lbfgs(int32_t n, float64_t *x, float64_t *ptr_fx, lbfgs_evaluate_t proc_evaluate, lbfgs_progress_t proc_progress, void *instance, lbfgs_parameter_t *_param, lbfgs_adjust_step_t proc_adjust_step)
int32_t get_num_parameters()
virtual int32_t get_num_labels() const
int32_t gd_mini_batch_size
The class Labels models labels, i.e. class assignments of objects.
virtual int32_t get_num_labels() const =0
real valued labels (e.g. for regression, classifier outputs)
multi-class labels 0,1,...
static T sum(T *vec, int32_t len)
return sum(vec)
virtual bool train_machine(CFeatures *data=NULL)
SGVector< float64_t > m_params
virtual void compute_gradients(SGVector< float64_t > parameters, SGMatrix< float64_t > targets, CDynamicObjectArray *layers, SGVector< float64_t > parameter_gradients)
static int32_t arg_max(T *vec, int32_t inc, int32_t len, T *maxv_ptr=NULL)
return arg_max(vec)
virtual int32_t get_num_vectors() const =0
virtual SGMatrix< float64_t > get_activation_gradients()
A generic multi-layer neural network.
virtual int32_t get_num_neurons()
SGMatrix< bool > m_adj_matrix
int32_t get_num_elements() const
float64_t get_label(int32_t idx)
SGMatrix< float64_t > features_to_matrix(CFeatures *features)
virtual void disconnect(int32_t i, int32_t j)
Base class for neural network layers.
virtual bool train_gradient_descent(SGMatrix< float64_t > inputs, SGMatrix< float64_t > targets)
virtual void enforce_max_norm(SGVector< float64_t > parameters, float64_t max_norm)
virtual void quick_connect()
virtual SGMatrix< float64_t > get_activations()
virtual float64_t compute_error(SGMatrix< float64_t > inputs, SGMatrix< float64_t > targets)
A generic learning machine interface.
int32_t get_num_classes()
float64_t gd_learning_rate_decay
SGVector< bool > m_param_regularizable
virtual CMulticlassLabels * apply_multiclass(CFeatures *data)
Multiclass Labels for multi-class classification.
ENNOptimizationMethod optimization_method
virtual void initialize_parameters(SGVector< float64_t > parameters, SGVector< bool > parameter_regularizable, float64_t sigma)
Class SGObject is the base class of all shogun objects.
CDynamicObjectArray * m_layers
CSGObject * element(int32_t idx1, int32_t idx2=0, int32_t idx3=0)
float64_t gd_error_damping_coeff
virtual void connect(int32_t i, int32_t j)
virtual void set_batch_size(int32_t batch_size)
virtual void disconnect_all()
virtual ~CNeuralNetwork()
int32_t m_total_num_parameters
virtual CRegressionLabels * apply_regression(CFeatures *data)
virtual EFeatureClass get_feature_class() const =0
Dynamic array class for CSGObject pointers that creates an array that can be used like a list or an a...
virtual void dropout_activations()
void set_const(T const_elem)
CDynamicObjectArray * get_layers()
virtual float64_t check_gradients(float64_t approx_epsilon=1.0e-3, float64_t s=1.0e-9)
CNeuralLayer * get_layer(int32_t i)
virtual bool is_label_valid(CLabels *lab) const
virtual CDenseFeatures< float64_t > * transform(CDenseFeatures< float64_t > *data)
virtual void set_labels(CLabels *lab)
virtual void set_batch_size(int32_t batch_size)
virtual bool train_lbfgs(SGMatrix< float64_t > inputs, SGMatrix< float64_t > targets)
The class Features is the base class of all feature objects.
virtual void compute_activations(SGMatrix< float64_t > inputs)
SGMatrix< float64_t > labels_to_matrix(CLabels *labs)
virtual SGMatrix< float64_t > forward_propagate(CFeatures *data, int32_t j=-1)
int32_t get_num_outputs()
virtual EProblemType get_machine_problem_type() const
Binary Labels for binary classification.
virtual void initialize(float64_t sigma=0.01f)
virtual void set_layers(CDynamicObjectArray *layers)
SGVector< float64_t > * get_layer_parameters(int32_t i)
virtual void set_labels(CLabels *lab)
float64_t gd_learning_rate
virtual EFeatureType get_feature_type() const =0
virtual float64_t compute_gradients(SGMatrix< float64_t > inputs, SGMatrix< float64_t > targets, SGVector< float64_t > gradients)
static T abs(T a)
return the absolute value of a number
void lbfgs_parameter_init(lbfgs_parameter_t *param)