Only in /home/dumitru/src/TorchReleases/0.1: Linux.cfg diff -ur /home/dumitru/src/TorchReleases/0.1/core/ClassMeasurer.cc ./core/ClassMeasurer.cc --- /home/dumitru/src/TorchReleases/0.1/core/ClassMeasurer.cc 2008-10-17 18:10:48.830047000 -0700 +++ ./core/ClassMeasurer.cc 2008-10-21 14:57:50.566587000 -0700 @@ -1,3 +1,31 @@ +// Copyright (C) 2003--2004 Ronan Collobert (collober@idiap.ch) +// +// This file is part of Torch 3.1. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// 3. The name of the author may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + #include "ClassMeasurer.h" namespace Torch { @@ -18,7 +46,6 @@ for(int i = 0; i < n_classes; i++) confusion[i] = (int *)allocator->alloc(sizeof(int)*n_classes); } - current_error = 0; reset_(); } @@ -41,7 +68,6 @@ void ClassMeasurer::measureIteration() { internal_error /= n_examples; - current_error = internal_error; if(binary_mode) file->write(&internal_error, sizeof(real), 1); diff -ur /home/dumitru/src/TorchReleases/0.1/core/ClassMeasurer.h ./core/ClassMeasurer.h --- /home/dumitru/src/TorchReleases/0.1/core/ClassMeasurer.h 2008-10-17 18:10:48.837047000 -0700 +++ ./core/ClassMeasurer.h 2008-10-21 14:57:50.569584000 -0700 @@ -1,3 +1,31 @@ +// Copyright (C) 2003--2004 Ronan Collobert (collober@idiap.ch) +// +// This file is part of Torch 3.1. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// 3. The name of the author may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + #ifndef CLASS_MEASURER_INC #define CLASS_MEASURER_INC diff -ur /home/dumitru/src/TorchReleases/0.1/core/ClassNLLMeasurer.cc ./core/ClassNLLMeasurer.cc --- /home/dumitru/src/TorchReleases/0.1/core/ClassNLLMeasurer.cc 2008-10-17 18:10:48.845051000 -0700 +++ ./core/ClassNLLMeasurer.cc 2008-10-21 14:57:50.571586000 -0700 @@ -1,3 +1,31 @@ +// Copyright (C) 2003--2004 Ronan Collobert (collober@idiap.ch) +// +// This file is part of Torch 3.1. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// 3. The name of the author may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + #include "ClassNLLMeasurer.h" namespace Torch { @@ -7,7 +35,6 @@ class_format = class_format_; inputs = inputs_; internal_error = 0; - current_error = 0; addBOption("average examples", &average_examples, true, "divided by the number of examples"); addBOption("average frames", &average_frames, true, "divided by the number of frames"); @@ -32,8 +59,6 @@ if(average_examples) internal_error /= data->n_examples; - current_error = internal_error; - if(binary_mode) file->write(&internal_error, sizeof(real), 1); else diff -ur /home/dumitru/src/TorchReleases/0.1/core/ClassNLLMeasurer.h ./core/ClassNLLMeasurer.h --- /home/dumitru/src/TorchReleases/0.1/core/ClassNLLMeasurer.h 2008-10-17 18:10:48.854047000 -0700 +++ ./core/ClassNLLMeasurer.h 2008-10-21 14:57:50.573588000 -0700 @@ -1,3 +1,31 @@ +// Copyright (C) 2003--2004 Ronan Collobert (collober@idiap.ch) +// +// This file is part of Torch 3.1. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// 3. The name of the author may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + #ifndef CLASS_NLL_MEASURER_INC #define CLASS_NLL_MEASURER_INC diff -ur /home/dumitru/src/TorchReleases/0.1/core/MSEMeasurer.cc ./core/MSEMeasurer.cc --- /home/dumitru/src/TorchReleases/0.1/core/MSEMeasurer.cc 2008-10-17 18:10:49.091963000 -0700 +++ ./core/MSEMeasurer.cc 2008-10-21 14:57:50.686585000 -0700 @@ -1,3 +1,31 @@ +// Copyright (C) 2003--2004 Ronan Collobert (collober@idiap.ch) +// +// This file is part of Torch 3.1. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// 3. The name of the author may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + #include "MSEMeasurer.h" namespace Torch { @@ -6,7 +34,6 @@ { inputs = inputs_; internal_error = 0; - current_error = 0; addBOption("average examples", &average_examples, true, "divided by the number of examples"); addBOption("average frame size", &average_frame_size, true, "divided by the frame size"); addBOption("average frames", &average_frames, true, "divided by the number of frames"); @@ -39,8 +66,6 @@ if(average_examples) internal_error /= data->n_examples; - current_error = internal_error; - if(binary_mode) file->write(&internal_error, sizeof(real), 1); else diff -ur /home/dumitru/src/TorchReleases/0.1/core/MSEMeasurer.h ./core/MSEMeasurer.h --- /home/dumitru/src/TorchReleases/0.1/core/MSEMeasurer.h 2008-10-17 18:10:49.099966000 -0700 +++ ./core/MSEMeasurer.h 2008-10-21 14:57:50.688589000 -0700 @@ -1,3 +1,31 @@ +// Copyright (C) 2003--2004 Ronan Collobert (collober@idiap.ch) +// +// This file is part of Torch 3.1. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// 3. The name of the author may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + #ifndef MSE_MEASURER_INC #define MSE_MEASURER_INC diff -ur /home/dumitru/src/TorchReleases/0.1/core/Measurer.cc ./core/Measurer.cc --- /home/dumitru/src/TorchReleases/0.1/core/Measurer.cc 2008-10-17 18:10:49.137963000 -0700 +++ ./core/Measurer.cc 2008-10-21 14:57:50.717586000 -0700 @@ -1,3 +1,31 @@ +// Copyright (C) 2003--2004 Ronan Collobert (collober@idiap.ch) +// +// This file is part of Torch 3.1. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// 3. The name of the author may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + #include "Measurer.h" #include "DataSet.h" #include "DiskXFile.h" diff -ur /home/dumitru/src/TorchReleases/0.1/core/Measurer.h ./core/Measurer.h --- /home/dumitru/src/TorchReleases/0.1/core/Measurer.h 2008-10-17 18:10:49.145963000 -0700 +++ ./core/Measurer.h 2008-10-21 14:57:50.720585000 -0700 @@ -1,3 +1,31 @@ +// Copyright (C) 2003--2004 Ronan Collobert (collober@idiap.ch) +// +// This file is part of Torch 3.1. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// 3. The name of the author may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + #ifndef MEASURER_INC #define MEASURER_INC @@ -57,17 +85,15 @@ /// Measure something at the end of the training/testing phase. virtual void measureEnd(); - /// Holds the error after the last iteration, if it's a real - real current_error; - /// Reset the measurer. (By default, do nothing). virtual void reset(); //----- virtual ~Measurer(); - }; + + } #endif Only in /home/dumitru/src/TorchReleases/0.1: deepTorch diff -ur /home/dumitru/src/TorchReleases/0.1/gradients/Linear.cc ./gradients/Linear.cc --- /home/dumitru/src/TorchReleases/0.1/gradients/Linear.cc 2008-10-17 18:10:50.636891000 -0700 +++ ./gradients/Linear.cc 2008-10-21 14:57:51.657501000 -0700 @@ -1,3 +1,31 @@ +// Copyright (C) 2003--2004 Ronan Collobert (collober@idiap.ch) +// +// This file is part of Torch 3.1. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// 3. The name of the author may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + #include "Linear.h" #include "Random.h" @@ -6,9 +34,6 @@ Linear::Linear(int n_inputs_, int n_outputs_) : GradientMachine(n_inputs_, n_outputs_, (n_inputs_+1)*n_outputs_) { addROption("weight decay", &weight_decay, 0, "weight decay"); - addROption("l1 weight decay", &l1_weight_decay, 0, "l1 weight decay"); - addROption("l1 fast weight decay", &l1_fast_weight_decay, 1.0, "l1 fast weight decay"); - addROption("bias decay", &bias_decay, 0.0, "bias decay"); weights = params->data[0]; bias = params->data[0]+n_inputs*n_outputs; der_weights = der_params->data[0]; @@ -79,42 +104,14 @@ der_bias[i] += z; } - // Decrements the bias by a constant term. Very simplistic way of encouraging - // sparse representations, if used in a sigmoid-like hidden unit. - if (bias_decay !=0) - for (int i = 0; i < n_outputs; i++) - der_bias[i] += bias_decay; - - if (weight_decay != 0) + if(weight_decay != 0) { real *src_ = params->data[0]; real *dest_ = der_params->data[0]; - // Note: no penalty applied on the biases. + // Note: pas de weight decay sur les biais. for(int i = 0; i < n_inputs*n_outputs; i++) dest_[i] += weight_decay * src_[i]; } - - // Note that this is not exactly an L1 penalty. - // If the abs(weight) > l1_weight_decay we get an L1 update. - // If abs(weight) < l1_weight_decay we apply a (sort of) L2 update: - // - when 0 < l1_fast_weight_decay < 1 then it's basically an L2 update. - // - when l1_fast_weight_decay = 1.0, then we move the weight faster towards - // zero and can potentially make the weight equal to zero (which is what we - // would like, ideally, since we're performing L1). - if(l1_weight_decay != 0) { - real *src_ = params->data[0]; - real *dest_ = der_params->data[0]; - // Note: no penalty applied on the biases. - for(int i=0; i l1_weight_decay) - dest_[i] += l1_weight_decay; - else if(src_[i] < -l1_weight_decay) - dest_[i] -= l1_weight_decay; - else - dest_[i] += l1_fast_weight_decay * src_[i]; - } - } - } Linear::~Linear() diff -ur /home/dumitru/src/TorchReleases/0.1/gradients/Linear.h ./gradients/Linear.h --- /home/dumitru/src/TorchReleases/0.1/gradients/Linear.h 2008-10-17 18:10:50.641879000 -0700 +++ ./gradients/Linear.h 2008-10-21 14:57:51.659503000 -0700 @@ -1,3 +1,31 @@ +// Copyright (C) 2003--2004 Ronan Collobert (collober@idiap.ch) +// +// This file is part of Torch 3.1. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// 3. The name of the author may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + #ifndef LINEAR_INC #define LINEAR_INC @@ -13,19 +41,9 @@ If you want, you can add a weight decay which looks like $\sum_i,j w_{ij}^2 + sum_i b_i^2$. - Alternatively, you can add an L1 penalty, which looks like - $\sum_i,j |w_{ij}|$ (but see comments for details). - - You can also add a non-zero ``bias decay'' term, which pushes the \\ - biases towards $-\infty$. This results in (simplistically) sparser \\ - hidden representations. - Options: \begin{tabular}{lcll} "weight decay" & real & the weight decay & [0] - "l1 weight decay" & real & the l1 weight decay & [1] - "l1 fast weight decay" & real & speed of l1 decay near 0 & [2] - "bias decay" & real & "simple way of achieving sparse representations" [3] \end{tabular} @author Ronan Collobert (collober@idiap.ch) @@ -34,9 +52,6 @@ { public: real weight_decay; - real l1_weight_decay; - real l1_fast_weight_decay; - real bias_decay; real *weights; real *bias; real *der_weights; Only in .: patchfile