From 52100814fb126a4c9dc25b08c6913d582f8f51c4 Mon Sep 17 00:00:00 2001 From: Hessel Tuinhof Date: Fri, 23 Jun 2017 10:54:29 +0200 Subject: [PATCH 1/4] [R] switch order of LRN and pooling layer Original paper (section 3.5) performs local response normalization of relu. --- example/image-classification/symbol_alexnet.R | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/example/image-classification/symbol_alexnet.R b/example/image-classification/symbol_alexnet.R index 097444335451..b6698288cd42 100644 --- a/example/image-classification/symbol_alexnet.R +++ b/example/image-classification/symbol_alexnet.R @@ -5,13 +5,13 @@ get_symbol <- function(num_classes = 1000) { # stage 1 conv1 <- mx.symbol.Convolution(data = input_data, kernel = c(11, 11), stride = c(4, 4), num_filter = 96) relu1 <- mx.symbol.Activation(data = conv1, act_type = "relu") - pool1 <- mx.symbol.Pooling(data = relu1, pool_type = "max", kernel = c(3, 3), stride = c(2, 2)) - lrn1 <- mx.symbol.LRN(data = pool1, alpha = 0.0001, beta = 0.75, knorm = 2, nsize = 5) + lrn1 <- mx.symbol.LRN(data = relu1, alpha = 0.0001, beta = 0.75, knorm = 2, nsize = 5) + pool1 <- mx.symbol.Pooling(data = lrn1, kernel = c(3, 3), stride = c(2, 2), pool_type = "max") # stage 2 conv2 <- mx.symbol.Convolution(data = lrn1, kernel = c(5, 5), pad = c(2, 2), num_filter = 256) relu2 <- mx.symbol.Activation(data = conv2, act_type = "relu") - pool2 <- mx.symbol.Pooling(data = relu2, kernel = c(3, 3), stride = c(2, 2), pool_type = "max") - lrn2 <- mx.symbol.LRN(data = pool2, alpha = 0.0001, beta = 0.75, knorm = 2, nsize = 5) + lrn2 <- mx.symbol.LRN(data = relu2, alpha = 0.0001, beta = 0.75, knorm = 2, nsize = 5) + pool2 <- mx.symbol.Pooling(data = lrn2, kernel = c(3, 3), stride = c(2, 2), pool_type = "max") # stage 3 conv3 <- mx.symbol.Convolution(data = lrn2, kernel = c(3, 3), pad = c(1, 1), num_filter = 384) relu3 <- mx.symbol.Activation(data = conv3, act_type = "relu") From c5888f41360876e69f719c228da96e9aae59fade Mon Sep 17 00:00:00 2001 From: Hessel Tuinhof Date: Tue, 1 Aug 2017 17:19:09 +0200 Subject: [PATCH 2/4] clearify definition of cross entropy --- python/mxnet/metric.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/python/mxnet/metric.py b/python/mxnet/metric.py index 2fe38ab751e0..6fcf8ef377aa 100644 --- a/python/mxnet/metric.py +++ b/python/mxnet/metric.py @@ -837,10 +837,14 @@ def update(self, labels, preds): class CrossEntropy(EvalMetric): """Computes Cross Entropy loss. - The cross entropy is given by + The cross entropy over a batch of sample size :math:`N` is given by .. math:: - -y\\log \\hat{y} + (1-y)\\log (1-\\hat{y}) + -\\sum_{n=1}^{N}\\sum_{c=1}^{C}t_{nk}\\log (y_{nk}), + + where :math:`t_{nk}=1` iff sample :math:`n` belongs to class :math:`k`. + :math:`y_{nk}` denotes the probability of sample :math:`n` belonging to + class :math:`k`. Parameters ---------- From efe386139a0e058a6c9fa5374c8e5799728aa8e5 Mon Sep 17 00:00:00 2001 From: Hessel Tuinhof Date: Tue, 1 Aug 2017 17:23:27 +0200 Subject: [PATCH 3/4] fix small type --- python/mxnet/metric.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/mxnet/metric.py b/python/mxnet/metric.py index 6fcf8ef377aa..c7a2f9719bef 100644 --- a/python/mxnet/metric.py +++ b/python/mxnet/metric.py @@ -840,7 +840,7 @@ class CrossEntropy(EvalMetric): The cross entropy over a batch of sample size :math:`N` is given by .. math:: - -\\sum_{n=1}^{N}\\sum_{c=1}^{C}t_{nk}\\log (y_{nk}), + -\\sum_{n=1}^{N}\\sum_{k=1}^{K}t_{nk}\\log (y_{nk}), where :math:`t_{nk}=1` iff sample :math:`n` belongs to class :math:`k`. :math:`y_{nk}` denotes the probability of sample :math:`n` belonging to From 3374b15906b2ba59d75c2889abb8cc290501a4b0 Mon Sep 17 00:00:00 2001 From: Hessel Tuinhof Date: Mon, 7 Aug 2017 16:25:56 +0200 Subject: [PATCH 4/4] fixed lint, trailing wspace --- python/mxnet/metric.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/mxnet/metric.py b/python/mxnet/metric.py index c7a2f9719bef..806a522b8729 100644 --- a/python/mxnet/metric.py +++ b/python/mxnet/metric.py @@ -842,8 +842,8 @@ class CrossEntropy(EvalMetric): .. math:: -\\sum_{n=1}^{N}\\sum_{k=1}^{K}t_{nk}\\log (y_{nk}), - where :math:`t_{nk}=1` iff sample :math:`n` belongs to class :math:`k`. - :math:`y_{nk}` denotes the probability of sample :math:`n` belonging to + where :math:`t_{nk}=1` if and only if sample :math:`n` belongs to class :math:`k`. + :math:`y_{nk}` denotes the probability of sample :math:`n` belonging to class :math:`k`. Parameters