SHOGUN  3.2.1
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
NeuralConvolutionalLayer.cpp
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2014, Shogun Toolbox Foundation
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions are met:
7 
8  * 1. Redistributions of source code must retain the above copyright notice,
9  * this list of conditions and the following disclaimer.
10  *
11  * 2. Redistributions in binary form must reproduce the above copyright notice,
12  * this list of conditions and the following disclaimer in the documentation
13  * and/or other materials provided with the distribution.
14  *
15  * 3. Neither the name of the copyright holder nor the names of its
16  * contributors may be used to endorse or promote products derived from this
17  * software without specific prior written permission.
18 
19  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
23  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29  * POSSIBILITY OF SUCH DAMAGE.
30  *
31  * Written (W) 2014 Khaled Nasr
32  */
33 
36 #include <shogun/lib/SGVector.h>
37 
38 using namespace shogun;
39 
41 {
42  init();
43 }
44 
47  int32_t num_maps,
48  int32_t input_width, int32_t input_height,
49  int32_t radius_x, int32_t radius_y,
50  int32_t pooling_width, int32_t pooling_height,
51  int32_t stride_x, int32_t stride_y) : CNeuralLayer()
52 {
53  init();
54  m_num_maps = num_maps;
55  m_input_width = input_width;
56  m_input_height = input_height;
57  m_radius_x = radius_x;
58  m_radius_y = radius_y;
59  m_pooling_width = pooling_width;
60  m_pooling_height = pooling_height;
61  m_stride_x = stride_x;
62  m_stride_y = stride_y;
63 
66 }
67 
69 {
70  CNeuralLayer::set_batch_size(batch_size);
71 
74 
76 
78 
81 }
82 
83 
85  SGVector< int32_t > input_indices)
86 {
87  CNeuralLayer::initialize(layers, input_indices);
88 
90  m_num_maps*(2*m_radius_x+1)*(2*m_radius_y+1);
91 }
92 
94  SGVector<bool> parameter_regularizable,
95  float64_t sigma)
96 {
97  int32_t num_parameters_per_map =
99 
100  for (int32_t m=0; m<m_num_maps; m++)
101  {
102  float64_t* map_params = parameters.vector+m*num_parameters_per_map;
103  bool* map_param_regularizable =
104  parameter_regularizable.vector+m*num_parameters_per_map;
105 
106  for (int32_t i=0; i<num_parameters_per_map; i++)
107  {
108  map_params[i] = CMath::normal_random(0.0, sigma);
109 
110  // turn off regularization for the bias parameters, on for the rest
111  map_param_regularizable[i] = (i>=m_num_neurons/m_num_maps);
112  }
113  }
114 }
115 
117  SGVector<float64_t> parameters,
118  CDynamicObjectArray* layers)
119 {
120  int32_t num_parameters_per_map =
122 
123  for (int32_t m=0; m<m_num_maps; m++)
124  {
125  SGVector<float64_t> map_params(
126  parameters.vector+m*num_parameters_per_map,
127  num_parameters_per_map, false);
128 
131 
132  map.compute_activations(map_params, layers, m_input_indices,
134 
137  }
138 }
139 
141  SGVector<float64_t> parameters,
142  SGMatrix<float64_t> targets,
143  CDynamicObjectArray* layers,
144  SGVector<float64_t> parameter_gradients)
145 {
146  if (dropout_prop>0.0)
147  {
148  int32_t len = m_num_neurons*m_batch_size;
149  for (int32_t i=0; i<len; i++)
151  }
152 
153  // compute the pre-pooling activation gradients
155  for (int32_t i=0; i<m_num_neurons; i++)
156  for (int32_t j=0; j<m_batch_size; j++)
159 
160  int32_t num_parameters_per_map =
161  m_num_neurons/m_num_maps + (2*m_radius_x+1)*(2*m_radius_y+1);
162 
163  for (int32_t m=0; m<m_num_maps; m++)
164  {
165  SGVector<float64_t> map_params(
166  parameters.vector+m*num_parameters_per_map,
167  num_parameters_per_map, false);
168 
169  SGVector<float64_t> map_gradients(
170  parameter_gradients.vector+m*num_parameters_per_map,
171  num_parameters_per_map, false);
172 
175 
176  map.compute_gradients(map_params, m_convolution_output,
178  m_input_indices, map_gradients);
179  }
180 }
181 
183  float64_t max_norm)
184 {
185  int32_t num_weights = (2*m_radius_x+1)*(2*m_radius_y+1);
186 
187  int32_t num_biases = m_num_neurons/m_num_maps;
188 
189  int32_t num_parameters_per_map = num_biases + num_weights;
190 
191  for (int32_t m=0; m<m_num_maps; m++)
192  {
193  float64_t* weights = parameters.vector+m*num_parameters_per_map+num_biases;
194 
195  float64_t norm =
196  SGVector<float64_t>::twonorm(weights, num_weights);
197 
198  if (norm > max_norm)
199  {
200  float64_t multiplier = max_norm/norm;
201  for (int32_t i=0; i<num_weights; i++)
202  weights[i] *= multiplier;
203  }
204  }
205 }
206 
207 void CNeuralConvolutionalLayer::init()
208 {
209  m_num_maps = 1;
210  m_input_width = 0;
211  m_input_height = 0;
212  m_radius_x = 0;
213  m_radius_y = 0;
214  m_pooling_width = 1;
215  m_pooling_height = 1;
216  m_stride_x = 1;
217  m_stride_y = 1;
219 
220  SG_ADD(&m_num_maps, "num_maps", "Number of maps", MS_NOT_AVAILABLE);
221  SG_ADD(&m_input_width, "input_width", "Input Width", MS_NOT_AVAILABLE);
222  SG_ADD(&m_input_height, "nput_height", "Input Height", MS_NOT_AVAILABLE);
223  SG_ADD(&m_radius_x, "radius_x", "X Radius", MS_NOT_AVAILABLE);
224  SG_ADD(&m_radius_y, "radius_y", "Y Radius", MS_NOT_AVAILABLE);
225  SG_ADD(&m_pooling_width, "pooling_width", "Pooling Width", MS_NOT_AVAILABLE);
226  SG_ADD(&m_pooling_height, "pooling_height", "Pooling Height", MS_NOT_AVAILABLE);
227  SG_ADD(&m_stride_x, "stride_x", "X Stride", MS_NOT_AVAILABLE);
228  SG_ADD(&m_stride_y, "stride_y", "Y Stride", MS_NOT_AVAILABLE);
229  SG_ADD((machine_int_t*) &m_activation_function, "activation_function",
230  "Activation Function", MS_NOT_AVAILABLE);
231 
232  SG_ADD(&m_convolution_output, "convolution_output",
233  "Convolution Output", MS_NOT_AVAILABLE);
234 
235  SG_ADD(&m_convolution_output_gradients, "convolution_output_gradients",
236  "Convolution Output Gradients", MS_NOT_AVAILABLE);
237 
238  SG_ADD(&m_buffer, "buffer",
239  "Buffer", MS_NOT_AVAILABLE);
240 }
double norm(double *v, double p, int n)
Definition: epph.cpp:452
virtual void initialize(CDynamicObjectArray *layers, SGVector< int32_t > input_indices)
Definition: NeuralLayer.cpp:59
virtual void initialize(CDynamicObjectArray *layers, SGVector< int32_t > input_indices)
EConvMapActivationFunction m_activation_function
virtual void compute_activations(SGVector< float64_t > parameters, CDynamicObjectArray *layers)
static float32_t normal_random(float32_t mean, float32_t std_dev)
Definition: Math.h:655
virtual void set_batch_size(int32_t batch_size)
SGMatrix< float64_t > m_activations
Definition: NeuralLayer.h:317
virtual void enforce_max_norm(SGVector< float64_t > parameters, float64_t max_norm)
SGMatrix< float64_t > m_convolution_output_gradients
virtual void compute_gradients(SGVector< float64_t > parameters, SGMatrix< float64_t > targets, CDynamicObjectArray *layers, SGVector< float64_t > parameter_gradients)
SGVector< int32_t > m_input_indices
Definition: NeuralLayer.h:304
Base class for neural network layers.
Definition: NeuralLayer.h:73
SGMatrix< float64_t > m_activation_gradients
Definition: NeuralLayer.h:322
void compute_activations(SGVector< float64_t > parameters, CDynamicObjectArray *layers, SGVector< int32_t > input_indices, SGMatrix< float64_t > activations, SGMatrix< float64_t > buffer)
Handles convolution and gradient calculation for a single feature map in a convolutional neural netwo...
double float64_t
Definition: common.h:50
index_t num_rows
Definition: SGMatrix.h:298
virtual void initialize_parameters(SGVector< float64_t > parameters, SGVector< bool > parameter_regularizable, float64_t sigma)
Dynamic array class for CSGObject pointers that creates an array that can be used like a list or an a...
index_t num_cols
Definition: SGMatrix.h:300
float64_t dropout_prop
Definition: NeuralLayer.h:283
virtual void set_batch_size(int32_t batch_size)
Definition: NeuralLayer.cpp:73
int machine_int_t
Definition: common.h:59
EConvMapActivationFunction
Determines the activation function for neurons in a convolutional feature map.
#define SG_ADD(...)
Definition: SGObject.h:67
void compute_gradients(SGVector< float64_t > parameters, SGMatrix< float64_t > activations, SGMatrix< float64_t > activation_gradients, CDynamicObjectArray *layers, SGVector< int32_t > input_indices, SGVector< float64_t > parameter_gradients)
SGMatrix< bool > m_dropout_mask
Definition: NeuralLayer.h:334
void pool_activations(SGMatrix< float64_t > activations, int32_t pooling_width, int32_t pooling_height, SGMatrix< float64_t > pooled_activations, SGMatrix< float64_t > max_indices)
static T twonorm(const T *x, int32_t len)
|| x ||_2

SHOGUN Machine Learning Toolbox - Documentation