dopt.nnet.layers.dense source code

1 /**
2     Contains an implementation of dense (i.e., fully connected) layers.
3     Authors: Henry Gouk
4 */
5 module dopt.nnet.layers.dense;
6 
7 import dopt.core;
8 import dopt.nnet;
9 import dopt.nnet.layers.util;
10 import dopt.online;
11 
12 /**
13     Encapsulates additional options for dense layers.
14 */
15 class DenseOptions
16 {
17     this()
18     {
19         _weightInit = heGaussianInit();
20         _biasInit = constantInit(0.0f);
21         _useBias = true;
22         _weightDecay = 0;
23     }
24 
25     mixin(dynamicProperties(
26         "ParamInitializer", "weightInit",
27         "ParamInitializer", "biasInit",
28         "Projection", "weightProj",
29         "Projection", "biasProj",
30         "float", "weightDecay",
31         "bool", "useBias"
32     ));
33 }
34 
35 ///
36 unittest
37 {
38     //Create a DenseOptions object with the default parameters
39     auto opts = new DenseOptions()
40                .weightInit(heGaussianInit())
41                .biasInit(constantInit(0.0f))
42                .weightProj(null)
43                .biasProj(null)
44                .weightDecay(0.0f)
45                .useBias(true);
46     
47     //Options can also be read back again later
48     assert(opts.weightDecay == 0.0f);
49     assert(opts.useBias == true);
50 }
51 
52 /**
53     Creates a fully connected (AKA, dense) layer.
54 
55     Params:
56         input = The previous layer in the network.
57         numOutputs = The number of units in this layer.
58         opts = Additional options with sensible default values.
59     
60     Returns:
61         The new layer.
62 */
63 Layer dense(Layer input, size_t numOutputs, DenseOptions opts = new DenseOptions())
64 {
65     auto x = input.output;
66     auto xTr = input.trainOutput;
67 
68     x = x.reshape([x.shape[0], x.volume / x.shape[0]]);
69     xTr = xTr.reshape([xTr.shape[0], xTr.volume / xTr.shape[0]]);
70 
71     auto weights = float32([numOutputs, x.shape[1]]);
72     opts._weightInit(weights);
73 
74     auto weightLoss = (opts.weightDecay == 0.0f) ? null : (opts.weightDecay * sum(weights * weights));
75 
76     Parameter[] params = [
77         Parameter(weights, weightLoss, opts.weightProj)
78     ];
79 
80     auto y = matmul(x, weights.transpose([1, 0]));
81     auto yTr = matmul(xTr, weights.transpose([1, 0]));
82 
83     if(opts.useBias)
84     {
85         auto bias = float32([numOutputs]);
86         opts._biasInit(bias);
87 
88         y = y + bias.repeat(y.shape[0]);
89         yTr = yTr + bias.repeat(yTr.shape[0]);
90 
91         params ~= Parameter(bias, null, opts.biasProj);
92     }
93 
94     return new Layer([input], y, yTr, params);
95 }