From 4441baf99488a753c1483ac298ac8fc1c77e407a Mon Sep 17 00:00:00 2001 From: Christiane Hofer Date: Fri, 9 Jan 2015 13:27:11 +0100 Subject: [PATCH 001/248] Test for Cochran --- .../java/ec/satoolkit/x11/CochranTest.java | 221 ++++++++++++++++++ 1 file changed, 221 insertions(+) create mode 100644 jtstoolkit/src/test/java/ec/satoolkit/x11/CochranTest.java diff --git a/jtstoolkit/src/test/java/ec/satoolkit/x11/CochranTest.java b/jtstoolkit/src/test/java/ec/satoolkit/x11/CochranTest.java new file mode 100644 index 000000000..b02bce1b6 --- /dev/null +++ b/jtstoolkit/src/test/java/ec/satoolkit/x11/CochranTest.java @@ -0,0 +1,221 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package ec.satoolkit.x11; + +import org.junit.Test; +import ec.tstoolkit.timeseries.simplets.TsData; +import data.*; +import ec.satoolkit.DecompositionMode; +import static org.junit.Assert.assertEquals; + +/** + * Das ist eine reine Entwiclungsklasse für den Cochrantest, diese wird Später + * nur TEsts für den CochranTest usw enthatlen + * + * @author s4504ch + */ +public class CochranTest { + + public CochranTest() { + } + + private static final Integer round = 5;//CH: Not sure weather this is enough or 10 is possible + + @Test + public void TestCochranDefault() { + TsData TsInput, TsOutD10Default; + X11Kernel kernel; + + TsInput = DataCochran.C; //Die Variable bräuchte man eigentlich nicht zu belegen + kernel = getX11Kernel(); + X11Results rslt = kernel.process(TsInput); + + System.out.println("Test Cochran Default:"); + + //Solution Calculated with WinX13 Build 9 + TsOutD10Default = DataCochran.C_D10_Default.round(round); + + assertEquals(rslt.getData("d-tables.d10", TsData.class).round(round), TsOutD10Default); + System.out.println("Results D10 from Win X13"); + System.out.println(TsOutD10Default); + System.out.println("Results D10 calculated"); + System.out.println(rslt.getData("d-tables.d10", TsData.class).round(round)); + + } + + private X11Kernel getX11Kernel() { + X11Specification spec = new X11Specification(); + + SeasonalFilterOption[] filters = new SeasonalFilterOption[12]; + + filters[0] = SeasonalFilterOption.S3X9; + filters[1] = SeasonalFilterOption.S3X9; + filters[2] = SeasonalFilterOption.S3X9; + filters[3] = SeasonalFilterOption.S3X9; + filters[4] = SeasonalFilterOption.S3X9; + filters[5] = SeasonalFilterOption.S3X9; + filters[6] = SeasonalFilterOption.S3X9; + filters[7] = SeasonalFilterOption.S3X9; + filters[8] = SeasonalFilterOption.S3X9; + filters[9] = SeasonalFilterOption.S3X9; + filters[10] = SeasonalFilterOption.S3X9; + filters[11] = SeasonalFilterOption.S3X9; + + spec.setSigma(1.5, 2.5); + spec.setHendersonFilterLength(17); + spec.setMode(DecompositionMode.Additive); + spec.setForecastHorizon(0); + spec.setSeasonal(true); + spec.setSeasonalFilters(filters); + + X11Toolkit toolkit = X11Toolkit.create(spec); + X11Kernel kernel = new X11Kernel(); + kernel.setToolkit(toolkit); + return kernel; + } + + @Test + public void TestCochranTest() { + System.out.println("Cochran Test"); + assertEquals(true, calcCochranTest()); + System.out.println("Test tw"); + assertEquals(0.116226, tw, 0.000001); + //assertEquals(0.11622,tw); + } + private double[] s; + private double tw; + // Critical values for monthly data + private static final double[] t = {0.5410, 0.3934, 0.3264, 0.2880, 0.2624, 0.2439, 0.2299, 0.2187, + 0.2098, 0.2020, 0.1980, 0.194, 0.186, 0.182, 0.178, 0.174, 0.17, + 0.166, 0.162, 0.158, 0.15, 0.15, 0.15, 0.15, 0.15, 0.15, 0.15, + 0.15, 0.15, 0.15, 0.15, 0.15, 0.15, 0.15, 0.15, 0.1403, 0.14, + 0.14, 0.14, 0.14}; + +//critical values for quarterly data + private static final double[] t4 = { + 0.9065, 0.7679, 0.6841, 0.6287, 0.5895, 0.5598, 0.5365, 0.5175, + .5017, 0.4884, 0.480, 0.471, 0.463, 0.454, 0.445, 0.4366, + .433, 0.430, 0.427, 0.424, 0.421, 0.417, 0.414, 0.411, 0.408, + .404, 0.401, 0.398, 0.395, 0.391, 0.388, 0.385, 0.382, 0.379, + .375, 0.3720, 0.369, 0.366, 0.362, 0.359}; +//calculate the Corantest for a given Timseries, and gives the false if the Nullhypothesis + // of equal variances of the periodes has to be rejected and different standarddeviations should be used + //for outlier detection + + private boolean calcCochranTest() { +// X,I1,Ib,Ie Was steht in diesen variablen, daher folgende Testbelegung + int Ib = 0; //X conatins the values from the beginning to the end of the, Ib index of the first period + int Ie;//index of the last period + int PSP = 5;// warum benötigt man den Wert und nimmt nicht Ny + double[] X;//array of the values of the time series, cant be usesd +//Damit ich mit den Daten arbeiten kann + TsData TsInput; + TsInput = DataCochran.C; + System.out.println("Periode " + TsInput.getStart() + " Erster Wert: " + TsInput.getValues().get(0)); + System.out.println("Periode" + TsInput.getLastPeriod() + "Letzter Wert: " + TsInput.getValues().get(TsInput.getLength() - 1)); + X11Kernel kernel; + kernel = getX11Kernel(); + X11Results rslt = kernel.process(TsInput); +// + + boolean I1 = true; //is the boolen that is true if the test has not be rejected + + int j; //counter for the periodes from 1,...4 or 12 + int n1; //number of values in a periode eg. in January + int nmin; // minimal number of observations of a period + boolean blngoto = true; + // double[] s;//array for the standarddeviation for each period dimension 1,...4 or 12 + + double smax; //max standarddeviation of periods + double st;// theoretical mean 0 for multi and 1 for add + // double tw;// teststatistik + double tt;// critical vaule + int Ny = TsInput.getFrequency().intValue(); //Beobachtungen pro Jahr + System.out.println("Anzahl an Perioden pro Jahr: " + TsInput.getFrequency().intValue()); + s = new double[Ny]; //original PSP first remains empty 0,...,Ny-1 + // Double X[],t[40],t4[40],s[PSP]; +//C----------------------------------------------------------------------- + // LOGICAL dpeq + // EXTERNAL dpeq +//C----------------------------------------------------------------------- + +// +// This routine performs Cochran's test to determine if the months +// are heteroskedastic. +//C + tw = 0; //wird schon größer werden + smax = -10.0; + nmin = 100; + + st = 1; //Additve + if (rslt.getSeriesDecomposition().getMode().isMultiplicative())//(Muladd.eq.1)// das muss hinterher im X11 anders getesetet werden + { + st = 0; + } +// + System.out.println("st= " + st); + for (int i = 0; i <= Ny - 1; i++) { //each period is taken into accoutn + n1 = 1; + j = Ib + i; //hier später ggf. mit start und endperiode arbeiten + s[i] = 0; + blngoto = true; + do { + s[i] = s[i] + ((TsInput.getValues().get(j) - st)*(TsInput.getValues().get(j) - st));// + + j = j + Ny; // for each year + n1 = n1 + 1;//count values + + if (j > TsInput.getValues().getLength() - 1) { //IE keine Ahnung ob +1 oder nicht + if (nmin > n1 - 3) {// fortran initilizes with zero + nmin = n1 - 3;// + } + s[i] = s[i] / (n1 - 1); + if (smax < s[i]) { + smax = s[i]; + } + tw = tw + s[i]; + System.out.println("s(" + i + "):" + s[i] + " n1= " + n1); + blngoto = false; + }; + + } while (blngoto); + + } + if (!(tw == 0)) { + tw = smax / tw; + } + + if (nmin > 40) { + nmin = 40; + } + + tt = t[nmin]; + if (Ny == 4) { + tt = t4[nmin]; + } + System.out.println("tw: " + tw); + System.out.println("tt: " + tt); + if (tw >= tt) { + I1 = false; + } + System.out.println("I1: " + I1); + + return I1; + } + + //calculates the Cochran Test for a ts + private boolean Cochran() { + //Daten einlesen + TsData TsInput; + TsInput = DataCochran.C; + int nmin; // minimal number of observations of a period + double smax; //max standarddeviation of periods + double st;// theoretical mean 0 for multi and 1 for add + + return true; + } + +} From 8125a29deb46312c413ab3816f0cce8f47172895 Mon Sep 17 00:00:00 2001 From: s4504ch Date: Fri, 9 Jan 2015 13:27:41 +0100 Subject: [PATCH 002/248] Data to Test Cochran --- .../src/test/java/data/DataCochran.java | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 jtstoolkit/src/test/java/data/DataCochran.java diff --git a/jtstoolkit/src/test/java/data/DataCochran.java b/jtstoolkit/src/test/java/data/DataCochran.java new file mode 100644 index 000000000..a63e1973c --- /dev/null +++ b/jtstoolkit/src/test/java/data/DataCochran.java @@ -0,0 +1,70 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ + +package data; + +import ec.tstoolkit.timeseries.simplets.TsData; +import ec.tstoolkit.timeseries.simplets.TsFrequency; + +/** + *Data to Test the Cochran Test and the use of it, the X11 Specification calendarsigma + * @author Christiane Hofer + */ +public class DataCochran { + private static final double[] g_inputser = {73.4 , 101.1 , 126.0 , 141.2 , 144.7 , 147.4 , 147.9 , 143.3 , + 153.6 , 155.0 , 149.1 , 92.0 , 60.3 , 50.7 , 95.8 , 136.1 , 139.4 , 148.8 , 143.7 , 137.4 , 153.1 , + 147.5 , 137.8 , 103.1 , 50.5 , 72.2 , 117.4 , 128.2 , 126.6 , 135.3 , 136.7 , 125.4 , 142.3 , + 141.2 , 133.4 , 91.6 , 72.9 , 74.1 , 109.6 , 119.6 , 122.3 , 129.9 , 130.9 , 121.3 , 136.0 , + 132.7 , 122.0 , 79.1 , 78.2 , 67.1 , 112.1 , 125.9 , 126.6 , 133.2 , 135.7 , 126.6 , 140.8 , + 141.9 , 129.4 , 90.9 , 73.4 , 86.1 , 107.7 , 121.8 , 125.2 , 125.5 , 128.3 , 121.3 , 132.4 , + 129.3 , 126.7 , 90.3 , 64.0 , 76.9 , 97.6 , 111.6 , 120.3 , 126.3 , 127.5 , 121.9 , 133.5 , + 134.3 , 123.7 , 88.4 , 60.9 , 75.9 , 105.7 , 116.8 , 115.1 , 122.0 , 122.6 , 116.3 , 129.6 , + 125.8 , 121.2 , 75.3 , 55.7 , 56.3 , 99.4 , 112.3 , 109.5 , 118.4 , 121.5 , 110.6 , 122.5 , + 122.0 , 115.7 , 80.8 , 50.5 , 62.4 , 87.6 , 101.3 , 102.9 , 110.3 , 114.3 , 105.3 , 116.0 , + 117.1 , 109.3 , 76.1 , 53.3 , 48.2 , 70.3 , 99.6 , 102.4 , 109.0 , 113.2 , 105.3 , 115.5 , + 118.0 , 108.7 , 74.0 , 43.7 , 47.9 , 69.9 , 105.7 , 109.8 , 113.3 , 119.6 , 111.5 , 122.4 , + 121.4 , 120.0 , 84.9 , 60.5 , 66.7 , 91.8 , 104.6 , 106.2 , 112.5 , 116.5 , 109.9 , 119.7 , + 120.0 , 111.7 , 81.2 , 65.1 , 74.3 , 90.1 , 103.9 , 107.3 , 112.8 , 115.3 , 111.7 , 120.3 , + 118.6 , 113.7 , 79.7 , 46.7 , 55.8 , 94.3 , 110.8 , 111.6 , 117.0 , 119.5 , 118.7 , 124.5 , + 122.4 , 115.8 , 77.8 , 40.1 , 43.9 , 91.5 , 115.3 , 113.3 , 119.3 , 123.5 , 116.4 , 125.5 , + 129.0 , 120.2 , 52.5 , 59.7 , 73.3 , 111.1 , 123.5 , 123.9 , 122.6 , 131.8 , 124.3 , 130.9 , + 131.6 , 130.7 , 80.4 , 62.0 , 51.5 , 108.5 , 117.4 , 120.8 , 123.4 , 130.1 , 121.7 , 132.7 , + 130.3 , 125.7 , 75.8 , 51.5 , 60.1 , 87.5 , 121.3 , 121.5 , 130.4 , 135.9 , 127.6 , 136.3 , + 133.6 , 131.1 , 90.7 , 68.8 , 80.8 , 112.6 , 123.8 , 121.6 , 130.2 , 133.5 , 125.8 , 136.3 , + 135.8 }; + + private static final double[] g_C_D10_Default = {-54.02225219,-45.11294503,-5.811536789,8.445469807,11.41557937, + 17.95915193,18.10631883,10.72141533,25.36894982,24.35050617,13.10647373,-25.15263412,-53.09538028,-44.91617116, + -5.929736555,8.332789698,11.37168865,17.69337583,18.11276734,10.65348414,25.03905485,24.01303444,13.18404679, + -24.88800739,-52.14535297,-44.43984383,-6.246659684,8.023756572,11.19829662,17.36021549,18.02265723,10.69446862, + 24.77212969,23.64601872,13.42633813,-24.56725022,-51.38749406,-44.09489207,-6.463939272,7.882133448,10.81981622, + 17.11573318,17.98079543,10.63581093,24.50114087,23.34906572,13.67155753,-24.2770387,-50.59229479,-43.73055072, + -6.644328487,7.778041138,10.30757522,16.9132389,18.03823408,10.40975819,24.12662205,22.83259529,14.00559008, + -23.64305121,-49.53167345,-43.47231682,-6.939711023,7.591255806,9.712143662,16.55414664,18.28103073,10.24296267, + 23.59592267,22.53657669,14.2720691,-22.93653417,-48.63700836,-43.10579655,-7.431434128,7.335807417,9.425781495, + 16.13964517,18.47071298,10.24598414,23.07001633,22.17271394,14.47145324,-22.21535851,-47.95241869,-42.72201474, + -7.92927243,7.199242737,9.261093867,15.62516449,18.5561281,10.42887564,22.56113294,22.03864613,14.6514487, + -21.69939207,-47.55724449,-42.16169078,-8.623534614,6.986510283,9.307305328,15.33954504,18.50839281,10.76171404, + 22.19536667,21.62471357,14.84576715,-20.95821458,-47.57652493,-41.7998579,-8.961188336,6.830871356,9.090133255, + 15.19951285,18.44061457,11.20996719,22.02458058,21.48671675,14.95368151,-20.58994873,-47.95284425,-41.73727206, + -9.076444049,7.061658334,8.926346221,15.14026869,18.45717825,11.59944911,21.91638719,21.39137551,15.01507119, + -20.56332517,-48.49706765,-41.86903485,-8.99860075,7.588536484,8.958553795,14.97729214,18.66075752,11.88716793, + 21.64277391,21.4040805,15.03497859,-21.00014368,-48.82874987,-41.81659658,-8.830174309,7.925507935,9.209217821, + 14.66204427,18.89910832,12.11475964,21.49176033,21.42330105,15.26932031,-21.72579482,-49.14030643,-42.01689709, + -8.648391073,8.209027606,9.638401487,14.51820691,19.19577658,12.42289019,21.38218353,21.22062322,15.43068966, + -22.67782024,-49.46796432,-41.96772501,-8.198558401,8.406705007,9.792334069,14.54290887,19.35374988,12.6811525, + 21.51733811,20.99644148,15.5418463,-23.76131853,-49.77380512,-41.83448146,-7.735971242,8.737164805,9.721226049, + 14.68392796,19.49173799,12.9041423,21.6017827,20.81076994,15.52932028,-24.70423365,-50.11061563,-41.54050472, + -7.195971501,8.980036639,9.580015706,14.70709384,19.5515664,13.04065061,21.75695344,20.88188141,15.70517834, + -25.61138285,-50.60253952,-41.62923669,-6.724660983,9.397600581,9.59530237,14.7354309,19.69383459,13.0719856, + 21.82123806,21.00909198,15.94261326,-26.59405103,-50.99258388,-41.74937864,-6.382691148,9.849199619,9.703434491, + 14.78510205,19.9824697,12.85259929,21.75190821,21.01795779,16.15547327,-27.41811641,-50.99034489,-41.56989576, + -6.09973296,9.845962955,9.671617204,14.8025957,20.21607764,12.65407241,21.77651433,20.94625442 + }; + + public static final TsData C = new TsData(TsFrequency.Monthly, 1995, 0, g_inputser, false); + public static final TsData C_D10_Default = new TsData(TsFrequency.Monthly, 1995, 0, g_C_D10_Default, false); +} From 8af36fc1eadd61d3d751879ec150e9d90174c14f Mon Sep 17 00:00:00 2001 From: Christiane Hofer Date: Thu, 15 Jan 2015 09:49:44 +0100 Subject: [PATCH 003/248] set signif as comment, because not in UI jet --- .../java/ec/satoolkit/x11/CalendarSigma.java | 31 ++++++++++--------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/jtstoolkit/src/main/java/ec/satoolkit/x11/CalendarSigma.java b/jtstoolkit/src/main/java/ec/satoolkit/x11/CalendarSigma.java index 874bcc970..d5bcd2bb2 100644 --- a/jtstoolkit/src/main/java/ec/satoolkit/x11/CalendarSigma.java +++ b/jtstoolkit/src/main/java/ec/satoolkit/x11/CalendarSigma.java @@ -14,17 +14,20 @@ * See the Licence for the specific language governing permissions and * limitations under the Licence. */ - - -package ec.satoolkit.x11; - -/** - * - * @author Kristof Bayens - */ -public enum CalendarSigma { - None, - Signif, - All, - Select -} + + +package ec.satoolkit.x11; + +import ec.tstoolkit.design.Development; + +/** + * Possible selections of calendarsigma + * @author Kristof Bayens, Christiane Hofer + */ +@Development(status = Development.Status.Exploratory) +public enum CalendarSigma { + None, + // Signif, //TODO CH: not supported jet, sigmavec is missing + All, + Select +} From 8863cb0e1ada689c3f9e0e370c14abc1b682e99d Mon Sep 17 00:00:00 2001 From: Christiane Hofer Date: Thu, 15 Jan 2015 09:51:29 +0100 Subject: [PATCH 004/248] add specification calendarsigma --- .../ec/satoolkit/x11/X11Specification.java | 96 +++++++++++++------ 1 file changed, 68 insertions(+), 28 deletions(-) diff --git a/jtstoolkit/src/main/java/ec/satoolkit/x11/X11Specification.java b/jtstoolkit/src/main/java/ec/satoolkit/x11/X11Specification.java index dfb6b5c97..1c7ce8a9e 100644 --- a/jtstoolkit/src/main/java/ec/satoolkit/x11/X11Specification.java +++ b/jtstoolkit/src/main/java/ec/satoolkit/x11/X11Specification.java @@ -1,20 +1,19 @@ /* -* Copyright 2013 National Bank of Belgium -* -* Licensed under the EUPL, Version 1.1 or – as soon they will be approved -* by the European Commission - subsequent versions of the EUPL (the "Licence"); -* You may not use this work except in compliance with the Licence. -* You may obtain a copy of the Licence at: -* -* http://ec.europa.eu/idabc/eupl -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the Licence is distributed on an "AS IS" basis, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the Licence for the specific language governing permissions and -* limitations under the Licence. -*/ - + * Copyright 2013 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or – as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ package ec.satoolkit.x11; import ec.satoolkit.DecompositionMode; @@ -35,24 +34,26 @@ public class X11Specification implements IProcSpecification, Cloneable { public static final double DEF_LSIGMA = 1.5, DEF_USIGMA = 2.5; - public static final int DEF_FCASTS=-1; + public static final int DEF_FCASTS = -1; public static final String MODE = "mode", LSIGMA = "lsigma", USIGMA = "usigma", TRENDMA = "trendma", SEASONALMA = "seasonalma", - FCASTS = "fcasts"; + FCASTS = "fcasts", + CALENDARSIGMA = "calendarsigma"; public static void fillDictionary(String prefix, Map dic) { - dic.put(InformationSet.item(prefix, MODE), String.class); - dic.put(InformationSet.item(prefix, LSIGMA), Double.class); - dic.put(InformationSet.item(prefix, USIGMA), Double.class); - dic.put(InformationSet.item(prefix, TRENDMA), Integer.class); - dic.put(InformationSet.item(prefix, SEASONALMA), String[].class); - dic.put(InformationSet.item(prefix, FCASTS), Integer.class); - } - + dic.put(InformationSet.item(prefix, MODE), String.class); + dic.put(InformationSet.item(prefix, LSIGMA), Double.class); + dic.put(InformationSet.item(prefix, USIGMA), Double.class); + dic.put(InformationSet.item(prefix, TRENDMA), Integer.class); + dic.put(InformationSet.item(prefix, SEASONALMA), String[].class); + dic.put(InformationSet.item(prefix, FCASTS), Integer.class); + dic.put(InformationSet.item(prefix, CALENDARSIGMA), String.class); + } + private DecompositionMode mode_ = DecompositionMode.Undefined; private boolean seasonal_ = true; private SeasonalFilterOption[] filters_; @@ -60,6 +61,8 @@ public static void fillDictionary(String prefix, Map dic) { private int henderson_ = 0; private int fcasts_ = DEF_FCASTS; + private CalendarSigma calendarsigma_ = CalendarSigma.None; + /** * Number of forecasts used in X11. By default, 0. When pre-processing is * used, the number of forecasts corresponds usually to 1 year. @@ -81,6 +84,16 @@ public int getHendersonFilterLength() { return henderson_; } + /** + * Option of Calendarsigma[X12], specifies the calculation of the standard + * error calculation used for outlier detection in the X11 part + * + * @return + */ + public CalendarSigma getCalendarSigma() { + return calendarsigma_; + } + public double getLowerSigma() { return lsigma_; } @@ -113,8 +126,14 @@ public boolean isDefault() { if (!seasonal_ || mode_ != DecompositionMode.Multiplicative) { return false; } - if (fcasts_ != DEF_FCASTS) + + if (calendarsigma_ != CalendarSigma.None) { + return false; + } + + if (fcasts_ != DEF_FCASTS) { return false; + } if (filters_ != null) { for (int i = 0; i < filters_.length; ++i) { if (filters_[i] != SeasonalFilterOption.Msr) { @@ -149,6 +168,16 @@ public void setForecastHorizon(int forecastsHorizon) { this.fcasts_ = forecastsHorizon; } + /** + * Option of Calendarsigma[X12], specifies the calculation of the standard + * error calculation used for outlier detection in the X11 part + * + * @param calendarsigma + */ + public void setCalendarSigma(CalendarSigma calendarsigma) { + calendarsigma_ = calendarsigma; + } + /** * Set the decomposition mode of X11 * @@ -237,7 +266,8 @@ private boolean equals(X11Specification spec) { && spec.henderson_ == henderson_ && spec.lsigma_ == lsigma_ && spec.usigma_ == usigma_ - && spec.mode_ == mode_; + && spec.mode_ == mode_ + && spec.calendarsigma_ == calendarsigma_; } @Override @@ -280,6 +310,11 @@ public InformationSet write(boolean verbose) { if (verbose || fcasts_ != -1) { info.add(FCASTS, fcasts_); } + + if (verbose || calendarsigma_ != CalendarSigma.None) { + info.add(CALENDARSIGMA, calendarsigma_.name()); + } + return info; } @@ -314,6 +349,11 @@ public boolean read(InformationSet info) { } } + String calendarsigma = info.get(CALENDARSIGMA, String.class); + if (calendarsigma != null) { + calendarsigma_ = CalendarSigma.valueOf(calendarsigma); + } + return true; } catch (Exception err) { return false; From 68dd613b09245e8e3f8dce43bf440c17df24f111 Mon Sep 17 00:00:00 2001 From: Christiane Hofer Date: Fri, 23 Jan 2015 17:41:56 +0100 Subject: [PATCH 005/248] enum for sigmavec settings --- .../main/java/ec/satoolkit/x11/Sigmavec.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 jtstoolkit/src/main/java/ec/satoolkit/x11/Sigmavec.java diff --git a/jtstoolkit/src/main/java/ec/satoolkit/x11/Sigmavec.java b/jtstoolkit/src/main/java/ec/satoolkit/x11/Sigmavec.java new file mode 100644 index 000000000..5b698a38a --- /dev/null +++ b/jtstoolkit/src/main/java/ec/satoolkit/x11/Sigmavec.java @@ -0,0 +1,22 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ + +package ec.satoolkit.x11; + +import ec.tstoolkit.design.Development; + +/** + * + * @author s4504ch + */ +@Development(status = Development.Status.Exploratory) +public enum Sigmavec { + group1, + + group2; + + +} From 7909f612ed788c2c9af6ee3c1e515a4b16f8e2eb Mon Sep 17 00:00:00 2001 From: Christiane Hofer Date: Fri, 23 Jan 2015 17:45:00 +0100 Subject: [PATCH 006/248] add signif for sigmavec --- .../java/ec/satoolkit/x11/CalendarSigma.java | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/jtstoolkit/src/main/java/ec/satoolkit/x11/CalendarSigma.java b/jtstoolkit/src/main/java/ec/satoolkit/x11/CalendarSigma.java index d5bcd2bb2..184c8bb5a 100644 --- a/jtstoolkit/src/main/java/ec/satoolkit/x11/CalendarSigma.java +++ b/jtstoolkit/src/main/java/ec/satoolkit/x11/CalendarSigma.java @@ -1,33 +1,33 @@ /* -* Copyright 2013 National Bank of Belgium -* -* Licensed under the EUPL, Version 1.1 or – as soon they will be approved -* by the European Commission - subsequent versions of the EUPL (the "Licence"); -* You may not use this work except in compliance with the Licence. -* You may obtain a copy of the Licence at: -* -* http://ec.europa.eu/idabc/eupl -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the Licence is distributed on an "AS IS" basis, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the Licence for the specific language governing permissions and -* limitations under the Licence. -*/ - - + * Copyright 2013 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or – as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ package ec.satoolkit.x11; import ec.tstoolkit.design.Development; /** * Possible selections of calendarsigma + * * @author Kristof Bayens, Christiane Hofer */ @Development(status = Development.Status.Exploratory) public enum CalendarSigma { + None, - // Signif, //TODO CH: not supported jet, sigmavec is missing + Signif, All, Select } From 4672475f22518761de0bcd514e56794400d220a9 Mon Sep 17 00:00:00 2001 From: Christiane Hofer Date: Fri, 23 Jan 2015 17:48:20 +0100 Subject: [PATCH 007/248] spec Sigmavec mit aufgenommen --- .../ec/satoolkit/x11/X11Specification.java | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/jtstoolkit/src/main/java/ec/satoolkit/x11/X11Specification.java b/jtstoolkit/src/main/java/ec/satoolkit/x11/X11Specification.java index 1c7ce8a9e..ff7fc21eb 100644 --- a/jtstoolkit/src/main/java/ec/satoolkit/x11/X11Specification.java +++ b/jtstoolkit/src/main/java/ec/satoolkit/x11/X11Specification.java @@ -42,7 +42,8 @@ public class X11Specification implements IProcSpecification, Cloneable { TRENDMA = "trendma", SEASONALMA = "seasonalma", FCASTS = "fcasts", - CALENDARSIGMA = "calendarsigma"; + CALENDARSIGMA = "calendarsigma", + SIGMAVEC="sigmavec"; public static void fillDictionary(String prefix, Map dic) { dic.put(InformationSet.item(prefix, MODE), String.class); @@ -52,6 +53,7 @@ public static void fillDictionary(String prefix, Map dic) { dic.put(InformationSet.item(prefix, SEASONALMA), String[].class); dic.put(InformationSet.item(prefix, FCASTS), Integer.class); dic.put(InformationSet.item(prefix, CALENDARSIGMA), String.class); + dic.put(InformationSet.item(prefix, MODE), String.class); } private DecompositionMode mode_ = DecompositionMode.Undefined; @@ -62,6 +64,7 @@ public static void fillDictionary(String prefix, Map dic) { private int fcasts_ = DEF_FCASTS; private CalendarSigma calendarsigma_ = CalendarSigma.None; + private Sigmavec[] sigmavec_; /** * Number of forecasts used in X11. By default, 0. When pre-processing is @@ -93,7 +96,11 @@ public int getHendersonFilterLength() { public CalendarSigma getCalendarSigma() { return calendarsigma_; } - + + public Sigmavec[] getSigmavec(){ + return sigmavec_ ; + } + public double getLowerSigma() { return lsigma_; } @@ -141,6 +148,15 @@ public boolean isDefault() { } } } + + if (sigmavec_ != null){ + for (int i = 0; i < sigmavec_.length; ++i) { + if (sigmavec_[i] != Sigmavec.group1){ + return false; + } + } + + } if (lsigma_ != DEF_LSIGMA) { return false; } @@ -178,6 +194,10 @@ public void setCalendarSigma(CalendarSigma calendarsigma) { calendarsigma_ = calendarsigma; } + public void setSigmavec(Sigmavec[] sigmavec){ + sigmavec_= sigmavec.clone(); + } + /** * Set the decomposition mode of X11 * @@ -267,7 +287,8 @@ private boolean equals(X11Specification spec) { && spec.lsigma_ == lsigma_ && spec.usigma_ == usigma_ && spec.mode_ == mode_ - && spec.calendarsigma_ == calendarsigma_; + && spec.calendarsigma_ == calendarsigma_ + && Arrays.deepEquals(spec.sigmavec_,sigmavec_); } @Override From 703fc420dd68838f825cf55329e29bee026428e1 Mon Sep 17 00:00:00 2001 From: Christiane Hofer Date: Tue, 3 Feb 2015 13:39:57 +0100 Subject: [PATCH 008/248] Enum for sigmavec --- .../java/ec/satoolkit/x11/SigmavecOption.java | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 jtstoolkit/src/main/java/ec/satoolkit/x11/SigmavecOption.java diff --git a/jtstoolkit/src/main/java/ec/satoolkit/x11/SigmavecOption.java b/jtstoolkit/src/main/java/ec/satoolkit/x11/SigmavecOption.java new file mode 100644 index 000000000..eee3426dc --- /dev/null +++ b/jtstoolkit/src/main/java/ec/satoolkit/x11/SigmavecOption.java @@ -0,0 +1,24 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ + +package ec.satoolkit.x11; + +import ec.tstoolkit.design.Development; + +/** + * + * @author Christiane Hofer + */ +@Development(status = Development.Status.Alpha) +public enum SigmavecOption { + + Group1, + /** + * + */ + Group2; + +} From 3c2d2ba61e3faf96fe7bc4261ed38ac4645a07ea Mon Sep 17 00:00:00 2001 From: Christiane Hofer Date: Tue, 3 Feb 2015 13:43:36 +0100 Subject: [PATCH 009/248] sigmavec added --- .../ec/satoolkit/x11/X11Specification.java | 53 +++++++++++++------ 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/jtstoolkit/src/main/java/ec/satoolkit/x11/X11Specification.java b/jtstoolkit/src/main/java/ec/satoolkit/x11/X11Specification.java index ff7fc21eb..4b570bce1 100644 --- a/jtstoolkit/src/main/java/ec/satoolkit/x11/X11Specification.java +++ b/jtstoolkit/src/main/java/ec/satoolkit/x11/X11Specification.java @@ -1,7 +1,7 @@ /* * Copyright 2013 National Bank of Belgium * - * Licensed under the EUPL, Version 1.1 or – as soon they will be approved + * Licensed under the EUPL, Version 1.1 or – as soon they will be approved * by the European Commission - subsequent versions of the EUPL (the "Licence"); * You may not use this work except in compliance with the Licence. * You may obtain a copy of the Licence at: @@ -53,7 +53,9 @@ public static void fillDictionary(String prefix, Map dic) { dic.put(InformationSet.item(prefix, SEASONALMA), String[].class); dic.put(InformationSet.item(prefix, FCASTS), Integer.class); dic.put(InformationSet.item(prefix, CALENDARSIGMA), String.class); - dic.put(InformationSet.item(prefix, MODE), String.class); + // dic.put(InformationSet.item(prefix, MODE), String.class); + dic.put(InformationSet.item(prefix, SIGMAVEC), String[].class); + } private DecompositionMode mode_ = DecompositionMode.Undefined; @@ -62,9 +64,8 @@ public static void fillDictionary(String prefix, Map dic) { private double lsigma_ = DEF_LSIGMA, usigma_ = DEF_USIGMA; private int henderson_ = 0; private int fcasts_ = DEF_FCASTS; - private CalendarSigma calendarsigma_ = CalendarSigma.None; - private Sigmavec[] sigmavec_; + private SigmavecOption[] sigmavec_; /** * Number of forecasts used in X11. By default, 0. When pre-processing is @@ -97,7 +98,7 @@ public CalendarSigma getCalendarSigma() { return calendarsigma_; } - public Sigmavec[] getSigmavec(){ + public SigmavecOption[] getSigmavec(){ return sigmavec_ ; } @@ -149,14 +150,13 @@ public boolean isDefault() { } } - if (sigmavec_ != null){ - for (int i = 0; i < sigmavec_.length; ++i) { - if (sigmavec_[i] != Sigmavec.group1){ + if (sigmavec_ != null) { + for (int i = 0; i < sigmavec_.length; ++i) { + if (sigmavec_[i] != SigmavecOption.Group1) { return false; } } - - } + } if (lsigma_ != DEF_LSIGMA) { return false; } @@ -194,10 +194,15 @@ public void setCalendarSigma(CalendarSigma calendarsigma) { calendarsigma_ = calendarsigma; } - public void setSigmavec(Sigmavec[] sigmavec){ + public void setSigmavec(SigmavecOption[] sigmavec){ sigmavec_= sigmavec.clone(); } - + + // public void setSigmavec(Sigmavec sigmavec) { CH: Das war Quatsch + // sigmavec_ = new Sigmavec[] {sigmavec}; + // } + + /** * Set the decomposition mode of X11 * @@ -268,6 +273,9 @@ public X11Specification clone() { if (filters_ != null) { cspec.filters_ = filters_.clone(); } + if (sigmavec_ != null) { + cspec.sigmavec_ = sigmavec_.clone(); + } return cspec; } catch (CloneNotSupportedException err) { throw new AssertionError(); @@ -282,13 +290,14 @@ public boolean equals(Object obj) { private boolean equals(X11Specification spec) { return spec.fcasts_ == fcasts_ && Arrays.deepEquals(spec.filters_, filters_) + && Arrays.deepEquals(spec.sigmavec_, sigmavec_) && spec.seasonal_ == seasonal_ && spec.henderson_ == henderson_ && spec.lsigma_ == lsigma_ && spec.usigma_ == usigma_ && spec.mode_ == mode_ - && spec.calendarsigma_ == calendarsigma_ - && Arrays.deepEquals(spec.sigmavec_,sigmavec_); + && spec.calendarsigma_ == calendarsigma_; + } @Override @@ -335,7 +344,13 @@ public InformationSet write(boolean verbose) { if (verbose || calendarsigma_ != CalendarSigma.None) { info.add(CALENDARSIGMA, calendarsigma_.name()); } - + if (sigmavec_ != null) { + String[] sigmavec = new String[sigmavec_.length]; + for (int i = 0; i < sigmavec.length; ++i) { + sigmavec[i] = filters_[i].name(); + } + info.add(SIGMAVEC, sigmavec); + } return info; } @@ -375,6 +390,14 @@ public boolean read(InformationSet info) { calendarsigma_ = CalendarSigma.valueOf(calendarsigma); } + String[] sigmavec = info.get(SIGMAVEC, String[].class); + if (sigmavec != null) { + sigmavec_ = new SigmavecOption[sigmavec.length]; + for (int i = 0; i < sigmavec.length; ++i) { + sigmavec_[i] = SigmavecOption.valueOf(sigmavec[i]); + } + } + return true; } catch (Exception err) { return false; From 9cbc669db4fe9db636687e8861431bc6583382f6 Mon Sep 17 00:00:00 2001 From: s4504ch Date: Tue, 3 Feb 2015 13:46:26 +0100 Subject: [PATCH 010/248] new class SigmavecOption --- .../main/java/ec/satoolkit/x11/Sigmavec.java | 22 ------------------- 1 file changed, 22 deletions(-) delete mode 100644 jtstoolkit/src/main/java/ec/satoolkit/x11/Sigmavec.java diff --git a/jtstoolkit/src/main/java/ec/satoolkit/x11/Sigmavec.java b/jtstoolkit/src/main/java/ec/satoolkit/x11/Sigmavec.java deleted file mode 100644 index 5b698a38a..000000000 --- a/jtstoolkit/src/main/java/ec/satoolkit/x11/Sigmavec.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ - -package ec.satoolkit.x11; - -import ec.tstoolkit.design.Development; - -/** - * - * @author s4504ch - */ -@Development(status = Development.Status.Exploratory) -public enum Sigmavec { - group1, - - group2; - - -} From 746e9a25ca726edf4070c7442817d329dc372aa8 Mon Sep 17 00:00:00 2001 From: Christiane Hofer Date: Fri, 6 Feb 2015 11:23:29 +0100 Subject: [PATCH 011/248] restrict the creat of the X11Toolkit to the spec calendarsigma.none other cases will be done later --- .../src/main/java/ec/satoolkit/x11/X11Toolkit.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/jtstoolkit/src/main/java/ec/satoolkit/x11/X11Toolkit.java b/jtstoolkit/src/main/java/ec/satoolkit/x11/X11Toolkit.java index feaa708d0..599a9e12a 100644 --- a/jtstoolkit/src/main/java/ec/satoolkit/x11/X11Toolkit.java +++ b/jtstoolkit/src/main/java/ec/satoolkit/x11/X11Toolkit.java @@ -73,10 +73,15 @@ public static X11Toolkit create(X11Specification spec) { spec.getHendersonFilterLength())); } - DefaultExtremeValuesCorrector xcorrector = new DefaultExtremeValuesCorrector(); - xcorrector.setSigma(spec.getLowerSigma(), spec.getUpperSigma()); - toolkit.setExtremeValuescorrector(xcorrector); + /* Define which ExtremeExtremeValuesCorrector hat to be used */ + if (spec.getCalendarSigma().equals(CalendarSigma.None)) { + DefaultExtremeValuesCorrector xcorrector = new DefaultExtremeValuesCorrector(); + xcorrector.setSigma(spec.getLowerSigma(), spec.getUpperSigma()); + toolkit.setExtremeValuescorrector(xcorrector); + } + /*die anderen Settings für Calendarsigma fehlen nun auch noch ggf. müssen auch noch die Gruppen mit übergeben werden/ + /*In Case that one or more and not all of the filters are stable the normalizer needs this information*/ if (spec.isSeasonal()) { DefaultSeasonalNormalizer nprovider = new DefaultSeasonalNormalizer(); From 84c2bc5a2c0a2bf1f2bfe0505800198ebcf3cab4 Mon Sep 17 00:00:00 2001 From: Christiane Hofer Date: Fri, 6 Feb 2015 11:35:32 +0100 Subject: [PATCH 012/248] added diagnostic for CochranTestResult to the Decompostion/Quality measures/Details and new TestClass for the CochranTest new TestClass for the spec Calendarsigma with SigmavecOption add in Mstatistics btables move class CochranTest to package package ec.tstoolkit.stats; --- .../implementation/HtmlX11Diagnostics.java | 31 ++- .../java/ec/satoolkit/x11/Mstatistics.java | 48 +++- .../java/ec/tstoolkit/stats/CochranTest.java | 176 ++++++++++++++ .../src/test/java/data/DataCochran.java | 3 +- .../ec/satoolkit/x11/CalendarsigmaTest.java | 88 +++++++ .../java/ec/satoolkit/x11/CochranTest.java | 221 ------------------ .../ec/tstoolkit/stats/CochranTestTest.java | 89 +++++++ 7 files changed, 432 insertions(+), 224 deletions(-) create mode 100644 jtstoolkit/src/main/java/ec/tstoolkit/stats/CochranTest.java create mode 100644 jtstoolkit/src/test/java/ec/satoolkit/x11/CalendarsigmaTest.java delete mode 100644 jtstoolkit/src/test/java/ec/satoolkit/x11/CochranTest.java create mode 100644 jtstoolkit/src/test/java/ec/tstoolkit/stats/CochranTestTest.java diff --git a/jtss/src/main/java/ec/tss/html/implementation/HtmlX11Diagnostics.java b/jtss/src/main/java/ec/tss/html/implementation/HtmlX11Diagnostics.java index 95fe048e5..f5f4a8460 100644 --- a/jtss/src/main/java/ec/tss/html/implementation/HtmlX11Diagnostics.java +++ b/jtss/src/main/java/ec/tss/html/implementation/HtmlX11Diagnostics.java @@ -18,6 +18,7 @@ import ec.satoolkit.x11.Mstatistics; import ec.satoolkit.x11.SeriesEvolution; +import ec.satoolkit.x11.SigmavecOption; import ec.tss.html.AbstractHtmlElement; import ec.tss.html.HtmlStream; import ec.tss.html.HtmlStyle; @@ -47,6 +48,8 @@ public void write(HtmlStream stream) throws IOException { writeF2E(stream); writeF2F(stream); writeF2G(stream); + writeF2H(stream); + } private void writeF2A(HtmlStream stream) throws IOException { @@ -267,6 +270,30 @@ private void writeF2G(HtmlStream stream) throws IOException { stream.close(HtmlTag.TABLE); stream.newLines(2); } + + private void writeF2H(HtmlStream stream) throws IOException { + stream.write(HtmlTag.HEADER2, h2, F2H_TITLE); + stream.write("Cochran Result:"); + boolean testResultCochran = stats_.getCochranResult(); + stream.write("CriticalValue: "); + stream.write(stats_.getCriticalValue()); + + if (testResultCochran) { + stream.write(" eaquals or greater than TestValue: "); + } else { + stream.write(" less than TestValue: "); + } + stream.write(Math.round(stats_.getTestValue() * 10000d) / 10000d); + + if (testResultCochran) { + stream.write(" Nullhypothesis for identical variances of each period has to be rejected, and different variances should be used. "); + } else { + stream.write(" Nullhypothesis for identical variances of each period has to be rejected, and different variances can't be rejected. ");} + + + stream.newLines(2); + } + private static final String F2A_TITLE_MUL = "Average percent change without regard to sign over the indicated span", F2B_TITLE_MUL = "Relative contributions to the variance of the percent change in the components of the original series", F2C_TITLE_MUL = "Average percent change with regard to sign and standard deviation over indicated span", @@ -276,7 +303,9 @@ private void writeF2G(HtmlStream stream) throws IOException { F2D_TITLE = "Average duration of run", F2E_TITLE = "I/C Ratio for indicated span", F2F_TITLE = "Relative contribution of the components to the stationary portion of the variance in the original series", - F2G_TITLE = "Autocorrelation of the irregular"; + F2G_TITLE = "Autocorrelation of the irregular", + F2H_TITLE = "Heteroscedasticity"; + private static final String[] F2A_HEADERS = new String[]{ "Span", "O", "CI", "I", "C", "S", "P", "TD&H", "Mod.O", "Mod.CI", "Mod.I" }; diff --git a/jtstoolkit/src/main/java/ec/satoolkit/x11/Mstatistics.java b/jtstoolkit/src/main/java/ec/satoolkit/x11/Mstatistics.java index 6d1bf3538..8d9a44365 100644 --- a/jtstoolkit/src/main/java/ec/satoolkit/x11/Mstatistics.java +++ b/jtstoolkit/src/main/java/ec/satoolkit/x11/Mstatistics.java @@ -27,6 +27,7 @@ import ec.tstoolkit.information.InformationMapper; import ec.tstoolkit.information.InformationSet; import ec.tstoolkit.stats.AutoCorrelations; +import ec.tstoolkit.stats.CochranTest; import ec.tstoolkit.timeseries.simplets.PeriodIterator; import ec.tstoolkit.timeseries.simplets.TsData; import java.util.Collections; @@ -54,6 +55,7 @@ public static Mstatistics computeFromX11(DecompositionMode mode, InformationSet dtables = info.getSubSet(X11Kernel.D); InformationSet atables = info.getSubSet(X11Kernel.A); InformationSet etables = info.getSubSet(X11Kernel.E); + InformationSet btables=info.getSubSet(X11Kernel.B); if (dtables == null) { return null; } @@ -123,7 +125,12 @@ public static Mstatistics computeFromX11(DecompositionMode mode, mstats.calcSNorm(); mstats.calcEvolutions(); mstats.calcM(); - return mstats; + // TODO: CH: Welches Table muss hier rein? + mstats.calcCochran(btables.get(X11Kernel.B3, TsData.class),mode, mstats); + + + + return mstats; } catch (RuntimeException err) { return null; } @@ -233,6 +240,28 @@ public double getVarTD() { return varTD; } + /** + * + * @return CriticalValue for Cochran Test + */ + public double getCriticalValue(){ + return criticalvalue; + } + + /** + * + * @return TestValue from Cochran Test + */ + public double getTestValue(){ + return testvalue; + + } + + public boolean getCochranResult() + { + return cochranTestResult; + } + /** * Gets the average duration of run of CI * @@ -300,7 +329,13 @@ public double[] getAutoCorrelationsOfIrregular() { private DecompositionMode mode; private static double[] wtFull = {10, 11, 10, 8, 11, 10, 18, 7, 7, 4, 4}; private static double[] wtShort = {14, 15, 10, 8, 11, 10, 32}; + + //Variables for Calendarsigma sigmavec testvalue criticalvalue + private double testvalue = 0; + private double criticalvalue = 0; + private boolean cochranTestResult = true; //Default Value of Cochran Test + private Mstatistics(DecompositionMode mode) { this.mode = mode; for (int i = 0; i < m.length; ++i) { @@ -608,6 +643,17 @@ private void calcStationaryVariances() { varI /= varO; } + private void calcCochran(TsData ts, DecompositionMode mode, Mstatistics mstats){ + // die folgenden müssen mit dem chochran Test in Abhängigkeit von den Sigmavec Einstellungen berechnet werden + + CochranTest cochranTest = new CochranTest(ts, mode); + cochranTest.calcCochranTest(); + mstats.criticalvalue = cochranTest.getCriticalValue(); + mstats.testvalue = cochranTest.getTestValue(); + mstats.cochranTestResult = cochranTest.getTestResult(); + // mstats.testvalue=inf. ; + + }; public DecompositionMode getMode() { return this.mode; } diff --git a/jtstoolkit/src/main/java/ec/tstoolkit/stats/CochranTest.java b/jtstoolkit/src/main/java/ec/tstoolkit/stats/CochranTest.java new file mode 100644 index 000000000..998aca44c --- /dev/null +++ b/jtstoolkit/src/main/java/ec/tstoolkit/stats/CochranTest.java @@ -0,0 +1,176 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ + +package ec.tstoolkit.stats; + +import ec.satoolkit.DecompositionMode; +import ec.tstoolkit.design.Development; +import ec.tstoolkit.timeseries.simplets.TsData; + +/** + * + * @author Christiane Hofer + */ +@Development(status = Development.Status.Exploratory) +public class CochranTest { + + public CochranTest(TsData tsData, DecompositionMode mode){ + ts_=tsData; + mode_= mode; + + }; + + private final TsData ts_; + private final DecompositionMode mode_; + private double[] s_; + private double tw_; + private double tt_;// critical vaule + boolean I1_ = true; //is the boolen that is true if the test has not be rejected +// Critical values for monthly data + private static final double[] t = {0.5410, 0.3934, 0.3264, 0.2880, 0.2624, 0.2439, 0.2299, 0.2187, + 0.2098, 0.2020, 0.1980, 0.194, 0.186, 0.182, 0.178, 0.174, 0.17, + 0.166, 0.162, 0.158, 0.15, 0.15, 0.15, 0.15, 0.15, 0.15, 0.15, + 0.15, 0.15, 0.15, 0.15, 0.15, 0.15, 0.15, 0.15, 0.1403, 0.14, + 0.14, 0.14, 0.14}; + +//critical values for quarterly data + private static final double[] t4 = { + 0.9065, 0.7679, 0.6841, 0.6287, 0.5895, 0.5598, 0.5365, 0.5175, + .5017, 0.4884, 0.480, 0.471, 0.463, 0.454, 0.445, 0.4366, + .433, 0.430, 0.427, 0.424, 0.421, 0.417, 0.414, 0.411, 0.408, + .404, 0.401, 0.398, 0.395, 0.391, 0.388, 0.385, 0.382, 0.379, + .375, 0.3720, 0.369, 0.366, 0.362, 0.359}; +//calculate the Corantest for a given Timseries, and gives the false if the Nullhypothesis + // of equal variances of the periodes has to be rejected and different standarddeviations should be used + //for outlier detection + + + + + public void calcCochranTest() { +// X,I1,Ib,Ie Was steht in diesen variablen, daher folgende Testbelegung + int Ib = ts_.getStart().getPosition(); //Ib index of the first period //X conatins the values from the beginning to the end of the, + int Ie;//index of the last period + int PSP = 5;// warum benötigt man den Wert und nimmt nicht Ny + double[] X;//array of the values of the time series, cant be usesd + + int iPeriode; // is the concidered periode, with corresponds to the relevant periode + int j; //counter for the periodes from 1,...4 or 12 + int n1; //number of values in a periode eg. in January + int nmin; // minimal number of observations of a period + boolean blngoto = true; + // double[] s;//array for the standarddeviation for each period dimension 1,...4 or 12 + + double smax; //max standarddeviation of periods + double st;// theoretical mean 0 for multi and 1 for add + // double tw;// teststatistik + + int Ny = ts_.getFrequency().intValue(); //Observations per year 12 or 4 + s_ = new double[Ny]; //original PSP first remains empty 0,...,Ny-1 + // Double X[],t[40],t4[40],s[PSP]; +//C----------------------------------------------------------------------- + // LOGICAL dpeq + // EXTERNAL dpeq +//C----------------------------------------------------------------------- + +// +// This routine performs Cochran's test to determine if the months or quaters +// are heteroskedastic. +//C + tw_ = 0; //wird schon größer werden + smax = -10.0; + nmin = 100; + + st = 1; //Additve + if (mode_.isMultiplicative())//(Muladd.eq.1)// das muss hinterher im X11 anders getesetet werden + { + st = 0; + } + + for (int i = 0; i <= Ny - 1; i++) { //each period is taken into accoutn + + n1 = 1; + j = i; //+IB hier später ggf. mit start und endperiode arbeiten + iPeriode=i+Ib; + if (iPeriode > Ny - 1) { + iPeriode = iPeriode - Ny; + } + s_[iPeriode] = 0;// s_[i] = 0; + blngoto = true; + do { + if (!Double.isNaN(ts_.getValues().get(j))){ + // s_[i] = s_[i] + ((ts_.getValues().get(j) - st)*(ts_.getValues().get(j) - st));// + s_[iPeriode] = s_[iPeriode] + ((ts_.getValues().get(j) - st)*(ts_.getValues().get(j) - st));// + n1 = n1 + 1;//count values + } + j = j + Ny; // for each year + if (j > ts_.getValues().getLength() - 1) { //IE keine Ahnung ob +1 oder nicht + if (nmin > n1 - 3) {// fortran initilizes with zero + nmin = n1 - 3;// + } + // s_[i] = s_[i] / (n1 - 1); + s_[iPeriode] = s_[iPeriode] / (n1 - 1); + if (smax < s_[iPeriode]) { //if (smax < s_[i]) + smax = s_[iPeriode]; // smax = s_[i]; + } + tw_ = tw_ + s_[iPeriode]; // tw_ = tw_ + s_[i]; + blngoto = false; + }; + + } while (blngoto); + + } + if (!(tw_ == 0)) { + tw_ = smax / tw_; + } + + if (nmin > 40) { + nmin = 40; + } + + tt_ = t[nmin]; + if (Ny == 4) { + tt_ = t4[nmin]; + } + + if (tw_ >= tt_) { + I1_ = false; + } + + + } + + public double getCriticalValue() { + return tt_; + }; + + public double getTestValue() { + return tw_; + } + + /** + * + * @return standardeviation for each period + */ + public double[] getS(){ + return s_; + } + + + /** + * + * @return true if CriticalValue > TestValue; Nullhypothesis for identical variances + * each period has to be rejected, and different variances should be used + */ + public boolean getTestResult(){ + return I1_; + } + } + + + + + diff --git a/jtstoolkit/src/test/java/data/DataCochran.java b/jtstoolkit/src/test/java/data/DataCochran.java index a63e1973c..c08575ba6 100644 --- a/jtstoolkit/src/test/java/data/DataCochran.java +++ b/jtstoolkit/src/test/java/data/DataCochran.java @@ -65,6 +65,7 @@ public class DataCochran { -6.09973296,9.845962955,9.671617204,14.8025957,20.21607764,12.65407241,21.77651433,20.94625442 }; - public static final TsData C = new TsData(TsFrequency.Monthly, 1995, 0, g_inputser, false); + public static final TsData CStartJan = new TsData(TsFrequency.Monthly, 1995, 0, g_inputser, false); + public static final TsData CStartAprl = new TsData(TsFrequency.Monthly, 1995, 3, g_inputser, false); public static final TsData C_D10_Default = new TsData(TsFrequency.Monthly, 1995, 0, g_C_D10_Default, false); } diff --git a/jtstoolkit/src/test/java/ec/satoolkit/x11/CalendarsigmaTest.java b/jtstoolkit/src/test/java/ec/satoolkit/x11/CalendarsigmaTest.java new file mode 100644 index 000000000..710d7da84 --- /dev/null +++ b/jtstoolkit/src/test/java/ec/satoolkit/x11/CalendarsigmaTest.java @@ -0,0 +1,88 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package ec.satoolkit.x11; + +import ec.tstoolkit.stats.*; +import org.junit.Test; +import ec.tstoolkit.timeseries.simplets.TsData; +import data.*; +import ec.satoolkit.DecompositionMode; +import static org.junit.Assert.assertEquals; + +/** + * + * @author Christiane Hofer + */ +public class CalendarsigmaTest { + + public CalendarsigmaTest() { + } + + private static final Integer round = 5;//CH: Not sure weather this is enough or 10 is possible + + @Test + public void TestCochranDefault() { + TsData TsInput, TsOutD10Default; + X11Kernel kernel; + + TsInput = DataCochran.CStartJan; + kernel = getX11Kernel(); + X11Results rslt = kernel.process(TsInput); + + System.out.println("Test Cochran Default:"); + + //Solution Calculated with WinX13 Build 9 + TsOutD10Default = DataCochran.C_D10_Default.round(round); + + assertEquals(rslt.getData("d-tables.d10", TsData.class).round(round), TsOutD10Default); + System.out.println("Results D10 from Win X13"); + System.out.println(TsOutD10Default); + System.out.println("Results D10 calculated"); + System.out.println(rslt.getData("d-tables.d10", TsData.class).round(round)); + + } + + private X11Kernel getX11Kernel() { + X11Specification spec = new X11Specification(); + + SeasonalFilterOption[] filters = new SeasonalFilterOption[12]; + + filters[0] = SeasonalFilterOption.S3X9; + filters[1] = SeasonalFilterOption.S3X9; + filters[2] = SeasonalFilterOption.S3X9; + filters[3] = SeasonalFilterOption.S3X9; + filters[4] = SeasonalFilterOption.S3X9; + filters[5] = SeasonalFilterOption.S3X9; + filters[6] = SeasonalFilterOption.S3X9; + filters[7] = SeasonalFilterOption.S3X9; + filters[8] = SeasonalFilterOption.S3X9; + filters[9] = SeasonalFilterOption.S3X9; + filters[10] = SeasonalFilterOption.S3X9; + filters[11] = SeasonalFilterOption.S3X9; + + spec.setSigma(1.5, 2.5); + spec.setHendersonFilterLength(17); + spec.setMode(DecompositionMode.Additive); + spec.setForecastHorizon(0); + spec.setSeasonal(true); + spec.setSeasonalFilters(filters); + + X11Toolkit toolkit = X11Toolkit.create(spec); + X11Kernel kernel = new X11Kernel(); + kernel.setToolkit(toolkit); + return kernel; + } + + + +} + + + + + + + diff --git a/jtstoolkit/src/test/java/ec/satoolkit/x11/CochranTest.java b/jtstoolkit/src/test/java/ec/satoolkit/x11/CochranTest.java deleted file mode 100644 index b02bce1b6..000000000 --- a/jtstoolkit/src/test/java/ec/satoolkit/x11/CochranTest.java +++ /dev/null @@ -1,221 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ -package ec.satoolkit.x11; - -import org.junit.Test; -import ec.tstoolkit.timeseries.simplets.TsData; -import data.*; -import ec.satoolkit.DecompositionMode; -import static org.junit.Assert.assertEquals; - -/** - * Das ist eine reine Entwiclungsklasse für den Cochrantest, diese wird Später - * nur TEsts für den CochranTest usw enthatlen - * - * @author s4504ch - */ -public class CochranTest { - - public CochranTest() { - } - - private static final Integer round = 5;//CH: Not sure weather this is enough or 10 is possible - - @Test - public void TestCochranDefault() { - TsData TsInput, TsOutD10Default; - X11Kernel kernel; - - TsInput = DataCochran.C; //Die Variable bräuchte man eigentlich nicht zu belegen - kernel = getX11Kernel(); - X11Results rslt = kernel.process(TsInput); - - System.out.println("Test Cochran Default:"); - - //Solution Calculated with WinX13 Build 9 - TsOutD10Default = DataCochran.C_D10_Default.round(round); - - assertEquals(rslt.getData("d-tables.d10", TsData.class).round(round), TsOutD10Default); - System.out.println("Results D10 from Win X13"); - System.out.println(TsOutD10Default); - System.out.println("Results D10 calculated"); - System.out.println(rslt.getData("d-tables.d10", TsData.class).round(round)); - - } - - private X11Kernel getX11Kernel() { - X11Specification spec = new X11Specification(); - - SeasonalFilterOption[] filters = new SeasonalFilterOption[12]; - - filters[0] = SeasonalFilterOption.S3X9; - filters[1] = SeasonalFilterOption.S3X9; - filters[2] = SeasonalFilterOption.S3X9; - filters[3] = SeasonalFilterOption.S3X9; - filters[4] = SeasonalFilterOption.S3X9; - filters[5] = SeasonalFilterOption.S3X9; - filters[6] = SeasonalFilterOption.S3X9; - filters[7] = SeasonalFilterOption.S3X9; - filters[8] = SeasonalFilterOption.S3X9; - filters[9] = SeasonalFilterOption.S3X9; - filters[10] = SeasonalFilterOption.S3X9; - filters[11] = SeasonalFilterOption.S3X9; - - spec.setSigma(1.5, 2.5); - spec.setHendersonFilterLength(17); - spec.setMode(DecompositionMode.Additive); - spec.setForecastHorizon(0); - spec.setSeasonal(true); - spec.setSeasonalFilters(filters); - - X11Toolkit toolkit = X11Toolkit.create(spec); - X11Kernel kernel = new X11Kernel(); - kernel.setToolkit(toolkit); - return kernel; - } - - @Test - public void TestCochranTest() { - System.out.println("Cochran Test"); - assertEquals(true, calcCochranTest()); - System.out.println("Test tw"); - assertEquals(0.116226, tw, 0.000001); - //assertEquals(0.11622,tw); - } - private double[] s; - private double tw; - // Critical values for monthly data - private static final double[] t = {0.5410, 0.3934, 0.3264, 0.2880, 0.2624, 0.2439, 0.2299, 0.2187, - 0.2098, 0.2020, 0.1980, 0.194, 0.186, 0.182, 0.178, 0.174, 0.17, - 0.166, 0.162, 0.158, 0.15, 0.15, 0.15, 0.15, 0.15, 0.15, 0.15, - 0.15, 0.15, 0.15, 0.15, 0.15, 0.15, 0.15, 0.15, 0.1403, 0.14, - 0.14, 0.14, 0.14}; - -//critical values for quarterly data - private static final double[] t4 = { - 0.9065, 0.7679, 0.6841, 0.6287, 0.5895, 0.5598, 0.5365, 0.5175, - .5017, 0.4884, 0.480, 0.471, 0.463, 0.454, 0.445, 0.4366, - .433, 0.430, 0.427, 0.424, 0.421, 0.417, 0.414, 0.411, 0.408, - .404, 0.401, 0.398, 0.395, 0.391, 0.388, 0.385, 0.382, 0.379, - .375, 0.3720, 0.369, 0.366, 0.362, 0.359}; -//calculate the Corantest for a given Timseries, and gives the false if the Nullhypothesis - // of equal variances of the periodes has to be rejected and different standarddeviations should be used - //for outlier detection - - private boolean calcCochranTest() { -// X,I1,Ib,Ie Was steht in diesen variablen, daher folgende Testbelegung - int Ib = 0; //X conatins the values from the beginning to the end of the, Ib index of the first period - int Ie;//index of the last period - int PSP = 5;// warum benötigt man den Wert und nimmt nicht Ny - double[] X;//array of the values of the time series, cant be usesd -//Damit ich mit den Daten arbeiten kann - TsData TsInput; - TsInput = DataCochran.C; - System.out.println("Periode " + TsInput.getStart() + " Erster Wert: " + TsInput.getValues().get(0)); - System.out.println("Periode" + TsInput.getLastPeriod() + "Letzter Wert: " + TsInput.getValues().get(TsInput.getLength() - 1)); - X11Kernel kernel; - kernel = getX11Kernel(); - X11Results rslt = kernel.process(TsInput); -// - - boolean I1 = true; //is the boolen that is true if the test has not be rejected - - int j; //counter for the periodes from 1,...4 or 12 - int n1; //number of values in a periode eg. in January - int nmin; // minimal number of observations of a period - boolean blngoto = true; - // double[] s;//array for the standarddeviation for each period dimension 1,...4 or 12 - - double smax; //max standarddeviation of periods - double st;// theoretical mean 0 for multi and 1 for add - // double tw;// teststatistik - double tt;// critical vaule - int Ny = TsInput.getFrequency().intValue(); //Beobachtungen pro Jahr - System.out.println("Anzahl an Perioden pro Jahr: " + TsInput.getFrequency().intValue()); - s = new double[Ny]; //original PSP first remains empty 0,...,Ny-1 - // Double X[],t[40],t4[40],s[PSP]; -//C----------------------------------------------------------------------- - // LOGICAL dpeq - // EXTERNAL dpeq -//C----------------------------------------------------------------------- - -// -// This routine performs Cochran's test to determine if the months -// are heteroskedastic. -//C - tw = 0; //wird schon größer werden - smax = -10.0; - nmin = 100; - - st = 1; //Additve - if (rslt.getSeriesDecomposition().getMode().isMultiplicative())//(Muladd.eq.1)// das muss hinterher im X11 anders getesetet werden - { - st = 0; - } -// - System.out.println("st= " + st); - for (int i = 0; i <= Ny - 1; i++) { //each period is taken into accoutn - n1 = 1; - j = Ib + i; //hier später ggf. mit start und endperiode arbeiten - s[i] = 0; - blngoto = true; - do { - s[i] = s[i] + ((TsInput.getValues().get(j) - st)*(TsInput.getValues().get(j) - st));// - - j = j + Ny; // for each year - n1 = n1 + 1;//count values - - if (j > TsInput.getValues().getLength() - 1) { //IE keine Ahnung ob +1 oder nicht - if (nmin > n1 - 3) {// fortran initilizes with zero - nmin = n1 - 3;// - } - s[i] = s[i] / (n1 - 1); - if (smax < s[i]) { - smax = s[i]; - } - tw = tw + s[i]; - System.out.println("s(" + i + "):" + s[i] + " n1= " + n1); - blngoto = false; - }; - - } while (blngoto); - - } - if (!(tw == 0)) { - tw = smax / tw; - } - - if (nmin > 40) { - nmin = 40; - } - - tt = t[nmin]; - if (Ny == 4) { - tt = t4[nmin]; - } - System.out.println("tw: " + tw); - System.out.println("tt: " + tt); - if (tw >= tt) { - I1 = false; - } - System.out.println("I1: " + I1); - - return I1; - } - - //calculates the Cochran Test for a ts - private boolean Cochran() { - //Daten einlesen - TsData TsInput; - TsInput = DataCochran.C; - int nmin; // minimal number of observations of a period - double smax; //max standarddeviation of periods - double st;// theoretical mean 0 for multi and 1 for add - - return true; - } - -} diff --git a/jtstoolkit/src/test/java/ec/tstoolkit/stats/CochranTestTest.java b/jtstoolkit/src/test/java/ec/tstoolkit/stats/CochranTestTest.java new file mode 100644 index 000000000..ce1979f0f --- /dev/null +++ b/jtstoolkit/src/test/java/ec/tstoolkit/stats/CochranTestTest.java @@ -0,0 +1,89 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ + +package ec.tstoolkit.stats; + +import data.DataCochran; +import ec.satoolkit.DecompositionMode; +import static org.junit.Assert.assertEquals; +import org.junit.Test; + +/** + * + * @author Christiane Hofer + */ +public class CochranTestTest { + + @Test + public void TestCochranTestStartJan() { + CochranTest cochranTestTestStartJan; + System.out.println("Cochran Test"); + + DecompositionMode mode = DecompositionMode.Additive; + + cochranTestTestStartJan = new CochranTest(DataCochran.CStartJan, mode); + cochranTestTestStartJan.calcCochranTest(); + assertEquals(true, cochranTestTestStartJan.getTestResult()); + System.out.println("Testvalue: " + cochranTestTestStartJan.getTestValue()); + System.out.println("Criticalvalue: " + cochranTestTestStartJan.getCriticalValue()); + + for (int i = 0; i <= cochranTestTestStartJan.getS().length - 1; i++) { + System.out.println("Standarddeviation of period(" + i + ") = " + cochranTestTestStartJan.getS()[i]); + } + assertEquals(3535.889, cochranTestTestStartJan.getS()[0], 0.0001); + assertEquals(4465.5955, cochranTestTestStartJan.getS()[1], 0.0001); + assertEquals(9870.1095, cochranTestTestStartJan.getS()[2], 0.0001); + assertEquals(13583.4685, cochranTestTestStartJan.getS()[3], 0.0001); + assertEquals(13937.615, cochranTestTestStartJan.getS()[4], 0.0001); + assertEquals(15341.996, cochranTestTestStartJan.getS()[5], 0.0001); + assertEquals(16068.224, cochranTestTestStartJan.getS()[6], 0.0001); + assertEquals(14279.8785, cochranTestTestStartJan.getS()[7], 0.0001); + assertEquals(17062.8645, cochranTestTestStartJan.getS()[8], 0.0001); + assertEquals(16839.1775, cochranTestTestStartJan.getS()[9], 0.0001); + assertEquals(15099.7847, cochranTestTestStartJan.getS()[10], 0.0001); + assertEquals(6722.4789, cochranTestTestStartJan.getS()[11], 0.0001); + + assertEquals(0.116226, cochranTestTestStartJan.getTestValue(), 0.000001); + assertEquals(0.166, cochranTestTestStartJan.getCriticalValue(), 0.000001); + assertEquals(true, cochranTestTestStartJan.getTestResult()); + } + + @Test + public void TestCochranTestStartAprl() { + System.out.println("Cochran Test"); + CochranTest cochranTestTestStartApril; + DecompositionMode mode = DecompositionMode.Additive; + + cochranTestTestStartApril = new CochranTest(DataCochran.CStartAprl, mode); + cochranTestTestStartApril.calcCochranTest(); + assertEquals(true, cochranTestTestStartApril.getTestResult()); + System.out.println("Testvalue: " + cochranTestTestStartApril.getTestValue()); + System.out.println("Criticalvalue: " + cochranTestTestStartApril.getCriticalValue()); + + for (int i = 0; i <= cochranTestTestStartApril.getS().length - 1; i++) { + System.out.println("Standarddeviation of period(" + i + ") = " + cochranTestTestStartApril.getS()[i]); + } + assertEquals(16839.1775, cochranTestTestStartApril.getS()[0], 0.0001); + assertEquals(15099.7847, cochranTestTestStartApril.getS()[1], 0.0001); + assertEquals(6722.4789, cochranTestTestStartApril.getS()[2], 0.0001); + assertEquals(3535.889, cochranTestTestStartApril.getS()[3], 0.0001); + assertEquals(4465.5955, cochranTestTestStartApril.getS()[4], 0.0001); + assertEquals(9870.1095, cochranTestTestStartApril.getS()[5], 0.0001); + assertEquals(13583.4685, cochranTestTestStartApril.getS()[6], 0.0001); + assertEquals(13937.615, cochranTestTestStartApril.getS()[7], 0.0001); + assertEquals(15341.996, cochranTestTestStartApril.getS()[8], 0.0001); + assertEquals(16068.224, cochranTestTestStartApril.getS()[9], 0.0001); + assertEquals(14279.8785, cochranTestTestStartApril.getS()[10], 0.0001); + assertEquals(17062.8645, cochranTestTestStartApril.getS()[11], 0.0001); + + + assertEquals(0.116226, cochranTestTestStartApril.getTestValue(), 0.000001); + assertEquals(0.166, cochranTestTestStartApril.getCriticalValue(), 0.000001); + assertEquals(true, cochranTestTestStartApril.getTestResult()); + + } + +} From 2616b776e920079114fcb95b0852953a2b102605 Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Mon, 9 Feb 2015 10:56:31 +0100 Subject: [PATCH 013/248] Fixed parsing of invalid timeformats. --- .../tsproviders/sdmx/engine/TimeFormat.java | 36 ++++++++++++------- .../sdmx/engine/TimeFormatTest.java | 6 ++-- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/sdmxprovider/src/main/java/ec/tss/tsproviders/sdmx/engine/TimeFormat.java b/sdmxprovider/src/main/java/ec/tss/tsproviders/sdmx/engine/TimeFormat.java index 6022e5eda..9d855273d 100644 --- a/sdmxprovider/src/main/java/ec/tss/tsproviders/sdmx/engine/TimeFormat.java +++ b/sdmxprovider/src/main/java/ec/tss/tsproviders/sdmx/engine/TimeFormat.java @@ -16,12 +16,13 @@ */ package ec.tss.tsproviders.sdmx.engine; -import ec.tss.tsproviders.utils.DataFormat; import ec.tss.tsproviders.utils.Parsers; import ec.tss.tsproviders.utils.Parsers.Parser; import static ec.tss.tsproviders.utils.StrangeParsers.yearFreqPosParser; import ec.tstoolkit.timeseries.TsAggregationType; import ec.tstoolkit.timeseries.simplets.TsFrequency; +import java.text.DateFormat; +import java.text.SimpleDateFormat; import java.util.Date; import java.util.Locale; @@ -39,7 +40,9 @@ public enum TimeFormat { P1Y(TsFrequency.Yearly, TsAggregationType.None) { @Override public Parser getParser() { - return onDatePattern("yyyy"); + return onStrictDatePattern("yyyy") + .or(onStrictDatePattern("yyyy'-01'")) + .or(onStrictDatePattern("yyyy'-A1'")); } }, /** @@ -48,7 +51,7 @@ public Parser getParser() { P6M(TsFrequency.HalfYearly, TsAggregationType.None) { @Override public Parser getParser() { - return yearFreqPosParser().or(onDatePattern("yyyy-MM")); + return yearFreqPosParser().or(onStrictDatePattern("yyyy-MM")); } }, /** @@ -57,7 +60,7 @@ public Parser getParser() { P4M(TsFrequency.QuadriMonthly, TsAggregationType.None) { @Override public Parser getParser() { - return yearFreqPosParser().or(onDatePattern("yyyy-MM")); + return yearFreqPosParser().or(onStrictDatePattern("yyyy-MM")); } }, /** @@ -66,7 +69,7 @@ public Parser getParser() { P3M(TsFrequency.Quarterly, TsAggregationType.None) { @Override public Parser getParser() { - return yearFreqPosParser().or(onDatePattern("yyyy-MM")); + return yearFreqPosParser().or(onStrictDatePattern("yyyy-MM")); } }, /** @@ -75,7 +78,7 @@ public Parser getParser() { P1M(TsFrequency.Monthly, TsAggregationType.None) { @Override public Parser getParser() { - return yearFreqPosParser().or(onDatePattern("yyyy-MM")); + return yearFreqPosParser().or(onStrictDatePattern("yyyy-MM")); } }, /** @@ -84,7 +87,7 @@ public Parser getParser() { P7D(TsFrequency.Monthly, TsAggregationType.Last) { @Override public Parser getParser() { - return onDatePattern("yyyy-MM-dd"); + return onStrictDatePattern("yyyy-MM-dd"); } }, /** @@ -93,7 +96,7 @@ public Parser getParser() { P1D(TsFrequency.Monthly, TsAggregationType.Last) { @Override public Parser getParser() { - return onDatePattern("yyyy-MM-dd"); + return onStrictDatePattern("yyyy-MM-dd"); } }, /** @@ -102,7 +105,7 @@ public Parser getParser() { PT1M(TsFrequency.Monthly, TsAggregationType.Last) { @Override public Parser getParser() { - return onDatePattern("yyyy-MM-dd"); + return onStrictDatePattern("yyyy-MM-dd"); } }, /** @@ -111,7 +114,7 @@ public Parser getParser() { UNDEFINED(TsFrequency.Undefined, TsAggregationType.None) { @Override public Parser getParser() { - return onDatePattern("yyyy-MM"); + return onStrictDatePattern("yyyy-MM"); } }; @@ -133,7 +136,16 @@ public TsAggregationType getAggregationType() { abstract public Parser getParser(); - private static Parsers.Parser onDatePattern(String datePattern) { - return new DataFormat(Locale.ROOT, datePattern, null).dateParser(); + private static Parsers.Parser onStrictDatePattern(String datePattern) { + final DateFormat dateFormat = new SimpleDateFormat(datePattern, Locale.ROOT); + dateFormat.setLenient(false); + return new Parsers.FailSafeParser() { + @Override + protected Date doParse(CharSequence input) throws Exception { + String inputAsString = input.toString(); + Date result = dateFormat.parse(inputAsString); + return result != null && inputAsString.equals(dateFormat.format(result)) ? result : null; + } + }; } } diff --git a/sdmxprovider/src/test/java/ec/tss/tsproviders/sdmx/engine/TimeFormatTest.java b/sdmxprovider/src/test/java/ec/tss/tsproviders/sdmx/engine/TimeFormatTest.java index cd38b963f..cff00cd36 100644 --- a/sdmxprovider/src/test/java/ec/tss/tsproviders/sdmx/engine/TimeFormatTest.java +++ b/sdmxprovider/src/test/java/ec/tss/tsproviders/sdmx/engine/TimeFormatTest.java @@ -40,9 +40,10 @@ public class TimeFormatTest { public void testP1Y() { Parser parser = TimeFormat.P1Y.getParser(); assertEquals(D_2000_01_01, parser.parse("2000")); + assertEquals(D_2000_01_01, parser.parse("2000-01")); assertEquals(D_2000_01_01, parser.parse("2000-A1")); -// assertNull(parser.parse("2000-A0")); -// assertNull(parser.parse("2000-A2")); + assertNull(parser.parse("2000-A0")); + assertNull(parser.parse("2000-A2")); assertNull(parser.parse("hello")); } @@ -84,6 +85,7 @@ public void testP1M() { assertEquals(D_2000_02_01, parser.parse("2000-M2")); assertEquals(D_2000_01_01, parser.parse("2000M1")); assertEquals(D_2000_02_01, parser.parse("2000M2")); + assertNull(parser.parse("2000-01-01")); assertNull(parser.parse("2000-M0")); assertNull(parser.parse("2000-M13")); assertNull(parser.parse("hello")); From a72774d3d58f6b12d8d08b695168da7a8f179693 Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Mon, 9 Feb 2015 11:38:03 +0100 Subject: [PATCH 014/248] Fixed threading issue when calling LinearIdBuilder#toString(). --- .../tsproviders/legacy/LinearIdBuilder.java | 40 +++++++++---------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/jtss/src/main/java/ec/tss/tsproviders/legacy/LinearIdBuilder.java b/jtss/src/main/java/ec/tss/tsproviders/legacy/LinearIdBuilder.java index d8f3e048e..888471b5b 100644 --- a/jtss/src/main/java/ec/tss/tsproviders/legacy/LinearIdBuilder.java +++ b/jtss/src/main/java/ec/tss/tsproviders/legacy/LinearIdBuilder.java @@ -1,19 +1,19 @@ /* -* Copyright 2013 National Bank of Belgium -* -* Licensed under the EUPL, Version 1.1 or – as soon they will be approved -* by the European Commission - subsequent versions of the EUPL (the "Licence"); -* You may not use this work except in compliance with the Licence. -* You may obtain a copy of the Licence at: -* -* http://ec.europa.eu/idabc/eupl -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the Licence is distributed on an "AS IS" basis, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the Licence for the specific language governing permissions and -* limitations under the Licence. -*/ + * Copyright 2013 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or – as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ package ec.tss.tsproviders.legacy; import ec.tstoolkit.design.Immutable; @@ -44,7 +44,6 @@ public static LinearIdBuilder from(IStringHandler stringHandler, String arg0, // <- private final IStringHandler stringHandler; private final String[] data; - private String cache; private LinearIdBuilder(IStringHandler stringHandler, String[] data) { this.stringHandler = stringHandler; @@ -93,21 +92,18 @@ public int getCount() { @Override public String toString() { - if (cache == null) { - cache = stringHandler.aggregate(data); - } - return cache; + return stringHandler.aggregate(data); } @Override public boolean equals(Object obj) { return this == obj || (obj instanceof LinearIdBuilder && equals((LinearIdBuilder) obj)); } - + private boolean equals(LinearIdBuilder other) { return this.toString().equals(other.toString()); } - + @Override public int hashCode() { return toString().hashCode(); From 530cdf5d929cc2088c03047d5c3b412c3fad0617 Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Mon, 9 Feb 2015 12:05:56 +0100 Subject: [PATCH 015/248] Fixed threading issue when dealing with SdmxDataSourceId. Modified dimension separator to use '.' instead of ':'. --- .../tsproviders/sdmx/SdmxDataSourceId.java | 232 +++++++++--------- .../ec/tss/tsproviders/sdmx/SdmxProvider.java | 26 +- 2 files changed, 133 insertions(+), 125 deletions(-) diff --git a/sdmxprovider/src/main/java/ec/tss/tsproviders/sdmx/SdmxDataSourceId.java b/sdmxprovider/src/main/java/ec/tss/tsproviders/sdmx/SdmxDataSourceId.java index 15e6a0c20..8575a0bad 100644 --- a/sdmxprovider/src/main/java/ec/tss/tsproviders/sdmx/SdmxDataSourceId.java +++ b/sdmxprovider/src/main/java/ec/tss/tsproviders/sdmx/SdmxDataSourceId.java @@ -1,116 +1,118 @@ /* -* Copyright 2013 National Bank of Belgium -* -* Licensed under the EUPL, Version 1.1 or – as soon they will be approved -* by the European Commission - subsequent versions of the EUPL (the "Licence"); -* You may not use this work except in compliance with the Licence. -* You may obtain a copy of the Licence at: -* -* http://ec.europa.eu/idabc/eupl -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the Licence is distributed on an "AS IS" basis, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the Licence for the specific language governing permissions and -* limitations under the Licence. -*/ - -package ec.tss.tsproviders.sdmx; - -import ec.tss.tsproviders.DataSource; -import ec.tss.tsproviders.legacy.IStringHandler; -import ec.tss.tsproviders.legacy.LinearIdBuilder; -import ec.tss.tsproviders.legacy.StringHandlers; -import ec.tss.tsproviders.utils.Parsers; -import ec.tss.tsproviders.utils.Parsers.Parser; -import java.io.File; - -/** - * - * @author Kristof Bayens - */ -@Deprecated -public final class SdmxDataSourceId implements CharSequence { - - private static final IStringHandler SH = StringHandlers.BASE64; - - public static SdmxDataSourceId parse(CharSequence input) { - return input instanceof SdmxDataSourceId ? (SdmxDataSourceId) input : parse(input.toString()); - } - - public static SdmxDataSourceId parse(String input) { - LinearIdBuilder idBuilder = LinearIdBuilder.parse(SH, input); - if (idBuilder == null || idBuilder.getCount() != 2) { - return null; - } - return new SdmxDataSourceId(idBuilder); - } - - public static SdmxDataSourceId from(File file) { - return parse(file.getAbsolutePath()); - } - private final LinearIdBuilder id_; - - private SdmxDataSourceId(LinearIdBuilder id) { - id_ = id; - } - - public SdmxDataSourceId(String factory, String url) { - id_ = LinearIdBuilder.from(SH, factory, url); - } - - @Override - public String toString() { - return id_.toString(); - } - - @Override - public int hashCode() { - return id_.hashCode(); - } - - @Override - public boolean equals(Object obj) { - return this == obj || (obj instanceof SdmxDataSourceId && equals((SdmxDataSourceId) obj)); - } - - private boolean equals(SdmxDataSourceId other) { - return this.id_.equals(other.id_); - } - - public String getFactory() { - return id_.get(0); - } - - public String getUrl() { - return id_.get(1); - } - - @Override - public int length() { - return id_.toString().length(); - } - - @Override - public char charAt(int index) { - return id_.toString().charAt(index); - } - - @Override - public CharSequence subSequence(int start, int end) { - return id_.toString().subSequence(start, end); - } - static final String X_FACTORY = "factory"; - static final String X_URL = "url"; - - public static Parsers.Parser legacyParser(final String providerName, final String version) { - return new Parser() { - @Override - public DataSource parse(CharSequence input) throws NullPointerException { - SdmxDataSourceId id = SdmxDataSourceId.parse(input); - return id != null - ? DataSource.builder(providerName, version).put(X_URL, id.getUrl()).put(X_FACTORY, id.getFactory()).build() : null; - } - }; - } -} + * Copyright 2013 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or – as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package ec.tss.tsproviders.sdmx; + +import ec.tss.tsproviders.DataSource; +import ec.tss.tsproviders.legacy.IStringHandler; +import ec.tss.tsproviders.legacy.LinearIdBuilder; +import ec.tss.tsproviders.legacy.StringHandlers; +import ec.tss.tsproviders.utils.Parsers; +import ec.tss.tsproviders.utils.Parsers.Parser; +import ec.tstoolkit.design.Immutable; +import java.io.File; + +/** + * + * @author Kristof Bayens + */ +@Deprecated +@Immutable +public final class SdmxDataSourceId implements CharSequence { + + private static final IStringHandler SH = StringHandlers.BASE64; + + public static SdmxDataSourceId parse(CharSequence input) { + return input instanceof SdmxDataSourceId ? (SdmxDataSourceId) input : parse(input.toString()); + } + + public static SdmxDataSourceId parse(String input) { + LinearIdBuilder idBuilder = LinearIdBuilder.parse(SH, input); + if (idBuilder == null || idBuilder.getCount() != 2) { + return null; + } + return new SdmxDataSourceId(idBuilder); + } + + public static SdmxDataSourceId from(File file) { + return parse(file.getAbsolutePath()); + } + private final LinearIdBuilder id_; + + private SdmxDataSourceId(LinearIdBuilder id) { + id_ = id; + } + + public SdmxDataSourceId(String factory, String url) { + id_ = LinearIdBuilder.from(SH, factory, url); + } + + @Override + public String toString() { + return id_.toString(); + } + + @Override + public int hashCode() { + return id_.hashCode(); + } + + @Override + public boolean equals(Object obj) { + return this == obj || (obj instanceof SdmxDataSourceId && equals((SdmxDataSourceId) obj)); + } + + private boolean equals(SdmxDataSourceId other) { + return this.id_.equals(other.id_); + } + + public String getFactory() { + return id_.get(0); + } + + public String getUrl() { + return id_.get(1); + } + + @Override + public int length() { + return id_.toString().length(); + } + + @Override + public char charAt(int index) { + return id_.toString().charAt(index); + } + + @Override + public CharSequence subSequence(int start, int end) { + return id_.toString().subSequence(start, end); + } + + static final String X_FACTORY = "factory"; + static final String X_URL = "url"; + + public static Parsers.Parser legacyParser(final String providerName, final String version) { + return new Parser() { + @Override + public DataSource parse(CharSequence input) throws NullPointerException { + SdmxDataSourceId id = SdmxDataSourceId.parse(input); + return id != null + ? DataSource.builder(providerName, version).put(X_URL, id.getUrl()).put(X_FACTORY, id.getFactory()).build() : null; + } + }; + } +} diff --git a/sdmxprovider/src/main/java/ec/tss/tsproviders/sdmx/SdmxProvider.java b/sdmxprovider/src/main/java/ec/tss/tsproviders/sdmx/SdmxProvider.java index 522ab188a..e61bcd2fa 100644 --- a/sdmxprovider/src/main/java/ec/tss/tsproviders/sdmx/SdmxProvider.java +++ b/sdmxprovider/src/main/java/ec/tss/tsproviders/sdmx/SdmxProvider.java @@ -52,19 +52,27 @@ public class SdmxProvider extends AbstractFileLoader { public static final String SOURCE = "TSProviders.Sdmx.SdmxProvider"; public static final String VERSION = "20120106"; + static final IParam Y_GROUP = Params.onString("", "group"); static final IParam Z_SERIES = Params.onString("", "series"); + private static final Logger LOGGER = LoggerFactory.getLogger(SdmxProvider.class); - private final ISdmxSourceFactory[] factories = {new CunningPlanFactory()}; - protected final Parsers.Parser legacyDataSourceParser; - protected final Parsers.Parser legacyDataSetParser; + + private final ISdmxSourceFactory[] factories; + private final Parsers.Parser legacyDataSourceParser; + private final Parsers.Parser legacyDataSetParser; + private final Splitter.MapSplitter keyValueSplitter; + private final Joiner compactNamingJoiner; private boolean compactNaming; private boolean keysInMetaData; public SdmxProvider() { super(LOGGER, SOURCE, TsAsyncMode.None); + this.factories = new ISdmxSourceFactory[]{new CunningPlanFactory()}; this.legacyDataSourceParser = SdmxLegacy.dataSourceParser(); this.legacyDataSetParser = SdmxLegacy.dataSetParser(); + this.keyValueSplitter = Splitter.on(',').trimResults().withKeyValueSeparator('='); + this.compactNamingJoiner = Joiner.on('.'); this.compactNaming = false; this.keysInMetaData = false; } @@ -75,9 +83,8 @@ public DataSet toDataSet(TsMoniker moniker) throws IllegalArgumentException { if (result != null) { return result; } - synchronized (legacyDataSetParser) { - return legacyDataSetParser.parse(moniker.getId()); - } + String id = moniker.getId(); + return id != null ? legacyDataSetParser.parse(id) : null; } @Override @@ -86,9 +93,8 @@ public DataSource toDataSource(TsMoniker moniker) throws IllegalArgumentExceptio if (result != null) { return result; } - synchronized (legacyDataSourceParser) { - return legacyDataSourceParser.parse(moniker.getId()); - } + String id = moniker.getId(); + return id != null ? legacyDataSourceParser.parse(id) : null; } @Override @@ -299,7 +305,7 @@ public void setKeysInMetaData(boolean keysInMetaData) { private String applyNaming(String input) { if (compactNaming) { try { - return Joiner.on(':').join(Splitter.on(',').trimResults().withKeyValueSeparator('=').split(input).values()); + return compactNamingJoiner.join(keyValueSplitter.split(input).values()); } catch (Exception ex) { } } From 424e426d0e21f787ecec354d2ce6982f9fabfda6 Mon Sep 17 00:00:00 2001 From: Philippe Charles Date: Mon, 9 Feb 2015 15:08:45 +0100 Subject: [PATCH 016/248] Fixed previous erroneous merge. --- .../ec/util/spreadsheet/poi/ByteSource.java | 104 ++++---- .../ec/util/spreadsheet/poi/DateUtil2.java | 48 ---- .../spreadsheet/poi/ExcelBookFactory.java | 24 +- .../ec/util/spreadsheet/poi/FastPoiBook.java | 94 +++---- .../util/spreadsheet/poi/FastPoiContext.java | 94 +++---- .../spreadsheet/poi/FastPoiSheetBuilder.java | 118 ++------- .../spreadsheet/poi/FastPoiValueFactory.java | 240 ++++++++--------- .../spreadsheet/poi/ExcelBookFactoryTest.java | 82 +++++- .../poi/ExcelClassicBookFactoryTest.java | 1 - .../util/spreadsheet/poi/FastPoiBookTest.java | 94 +++---- .../util/spreadsheet/poi/FastPoiCellTest.java | 25 -- .../poi/FastPoiValueFactoryTest.java | 246 +++++++++--------- .../spreadsheet/poi/SpreadsheetAssert.java | 6 +- 13 files changed, 542 insertions(+), 634 deletions(-) delete mode 100644 spreadsheet/spreadsheet-poi/src/main/java/ec/util/spreadsheet/poi/DateUtil2.java delete mode 100644 spreadsheet/spreadsheet-poi/src/test/java/ec/util/spreadsheet/poi/FastPoiCellTest.java diff --git a/spreadsheet/spreadsheet-poi/src/main/java/ec/util/spreadsheet/poi/ByteSource.java b/spreadsheet/spreadsheet-poi/src/main/java/ec/util/spreadsheet/poi/ByteSource.java index 02dabcd6a..295071800 100644 --- a/spreadsheet/spreadsheet-poi/src/main/java/ec/util/spreadsheet/poi/ByteSource.java +++ b/spreadsheet/spreadsheet-poi/src/main/java/ec/util/spreadsheet/poi/ByteSource.java @@ -1,52 +1,52 @@ -/* - * Copyright 2013 National Bank of Belgium - * - * Licensed under the EUPL, Version 1.1 or – as soon they will be approved - * by the European Commission - subsequent versions of the EUPL (the "Licence"); - * You may not use this work except in compliance with the Licence. - * You may obtain a copy of the Licence at: - * - * http://ec.europa.eu/idabc/eupl - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the Licence is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the Licence for the specific language governing permissions and - * limitations under the Licence. - */ -package ec.util.spreadsheet.poi; - -import java.io.IOException; -import java.io.InputStream; -import java.net.URL; -import java.util.Objects; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; - -// @VisibleForTesting -abstract class ByteSource { - - @Nullable - abstract public InputStream openStream() throws IOException; - - @Nonnull - public static ByteSource fromURL(@Nonnull final URL url) { - Objects.requireNonNull(url); - return new ByteSource() { - @Override - public InputStream openStream() throws IOException { - return url.openStream(); - } - }; - } - - @Nonnull - public static ByteSource noStream() { - return new ByteSource() { - @Override - public InputStream openStream() throws IOException { - return null; - } - }; - } -} +/* + * Copyright 2013 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or – as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package ec.util.spreadsheet.poi; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.util.Objects; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +// @VisibleForTesting +abstract class ByteSource { + + @Nullable + abstract public InputStream openStream() throws IOException; + + @Nonnull + public static ByteSource fromURL(@Nonnull final URL url) { + Objects.requireNonNull(url); + return new ByteSource() { + @Override + public InputStream openStream() throws IOException { + return url.openStream(); + } + }; + } + + @Nonnull + public static ByteSource noStream() { + return new ByteSource() { + @Override + public InputStream openStream() throws IOException { + return null; + } + }; + } +} diff --git a/spreadsheet/spreadsheet-poi/src/main/java/ec/util/spreadsheet/poi/DateUtil2.java b/spreadsheet/spreadsheet-poi/src/main/java/ec/util/spreadsheet/poi/DateUtil2.java deleted file mode 100644 index 258c4aafd..000000000 --- a/spreadsheet/spreadsheet-poi/src/main/java/ec/util/spreadsheet/poi/DateUtil2.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2013 National Bank of Belgium - * - * Licensed under the EUPL, Version 1.1 or – as soon they will be approved - * by the European Commission - subsequent versions of the EUPL (the "Licence"); - * You may not use this work except in compliance with the Licence. - * You may obtain a copy of the Licence at: - * - * http://ec.europa.eu/idabc/eupl - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the Licence is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the Licence for the specific language governing permissions and - * limitations under the Licence. - */ -package ec.util.spreadsheet.poi; - -import java.util.Calendar; -import java.util.Date; -import org.apache.poi.ss.usermodel.DateUtil; - -/** - * - * @author Philippe Charles - */ -final class DateUtil2 { - - private DateUtil2() { - // static class - } - - /** - * Same as {@link DateUtil#getJavaDate(double, boolean)} but with calendar - * as first parameter to reduce memory usage. - * - * @param calendar - * @param date - * @param use1904windowing - * @return - */ - public static Date getJavaDate(Calendar calendar, double date, boolean use1904windowing) { - int wholeDays = (int) Math.floor(date); - int millisecondsInDay = (int) ((date - wholeDays) * DateUtil.DAY_MILLISECONDS + 0.5); - DateUtil.setCalendar(calendar, wholeDays, millisecondsInDay, use1904windowing); - return calendar.getTime(); - } -} diff --git a/spreadsheet/spreadsheet-poi/src/main/java/ec/util/spreadsheet/poi/ExcelBookFactory.java b/spreadsheet/spreadsheet-poi/src/main/java/ec/util/spreadsheet/poi/ExcelBookFactory.java index ba6e95fd9..049b1c6ed 100644 --- a/spreadsheet/spreadsheet-poi/src/main/java/ec/util/spreadsheet/poi/ExcelBookFactory.java +++ b/spreadsheet/spreadsheet-poi/src/main/java/ec/util/spreadsheet/poi/ExcelBookFactory.java @@ -21,8 +21,12 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.file.AccessDeniedException; +import java.nio.file.FileSystemException; +import java.nio.file.NoSuchFileException; import java.util.Locale; -import org.apache.poi.openxml4j.exceptions.InvalidFormatException; +import javax.annotation.Nonnull; +import org.apache.poi.openxml4j.exceptions.InvalidOperationException; import org.apache.poi.openxml4j.exceptions.OpenXML4JException; import org.apache.poi.xssf.usermodel.XSSFWorkbook; @@ -61,11 +65,10 @@ public boolean accept(File pathname) { @Override public Book load(File file) throws IOException { + checkFile(file); try { return fast ? FastPoiBook.create(file) : PoiBook.create(file); - } catch (InvalidFormatException ex) { - throw new IOException(ex); - } catch (OpenXML4JException ex) { + } catch (OpenXML4JException | InvalidOperationException ex) { throw new IOException(ex); } } @@ -74,8 +77,6 @@ public Book load(File file) throws IOException { public Book load(InputStream stream) throws IOException { try { return fast ? FastPoiBook.create(stream) : PoiBook.create(stream); - } catch (InvalidFormatException ex) { - throw new IOException(ex); } catch (OpenXML4JException ex) { throw new IOException(ex); } @@ -87,4 +88,15 @@ public void store(OutputStream stream, Book book) throws IOException { PoiBookWriter.copy(book, target); target.write(stream); } + + @Nonnull + private static File checkFile(@Nonnull File file) throws FileSystemException { + if (!file.exists() || file.isDirectory()) { + throw new NoSuchFileException(file.getPath()); + } + if (!file.canRead()) { + throw new AccessDeniedException(file.getPath()); + } + return file; + } } diff --git a/spreadsheet/spreadsheet-poi/src/main/java/ec/util/spreadsheet/poi/FastPoiBook.java b/spreadsheet/spreadsheet-poi/src/main/java/ec/util/spreadsheet/poi/FastPoiBook.java index 87bc37158..fbb1c5ad1 100644 --- a/spreadsheet/spreadsheet-poi/src/main/java/ec/util/spreadsheet/poi/FastPoiBook.java +++ b/spreadsheet/spreadsheet-poi/src/main/java/ec/util/spreadsheet/poi/FastPoiBook.java @@ -18,6 +18,7 @@ import ec.util.spreadsheet.Book; import ec.util.spreadsheet.Sheet; +import ec.util.spreadsheet.helpers.ArraySheet; import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -26,12 +27,12 @@ import java.util.List; import java.util.Map; import javax.annotation.Nonnull; -import javax.annotation.Nullable; import javax.annotation.concurrent.Immutable; import org.apache.poi.openxml4j.exceptions.InvalidFormatException; import org.apache.poi.openxml4j.exceptions.OpenXML4JException; import org.apache.poi.openxml4j.opc.OPCPackage; import org.apache.poi.openxml4j.opc.PackageAccess; +import org.apache.poi.ss.usermodel.DateUtil; import org.apache.poi.xssf.eventusermodel.XSSFReader; import org.xml.sax.Attributes; import org.xml.sax.InputSource; @@ -46,11 +47,6 @@ */ final class FastPoiBook extends Book { - private final OPCPackage pkg; - private final XSSFReader reader; - private final List sheets; - private final SheetContext sheetContext; - /** * Opens a book from a file. * @@ -61,7 +57,7 @@ final class FastPoiBook extends Book { */ @Nonnull public static FastPoiBook create(@Nonnull File file) throws IOException, OpenXML4JException { - return new FastPoiBook(OPCPackage.open(file.getPath(), PackageAccess.READ)); + return create(OPCPackage.open(file.getPath(), PackageAccess.READ)); } /** @@ -78,19 +74,31 @@ public static FastPoiBook create(@Nonnull File file) throws IOException, OpenXML */ @Nonnull public static FastPoiBook create(@Nonnull InputStream stream) throws IOException, OpenXML4JException { - return new FastPoiBook(OPCPackage.open(stream)); + return create(OPCPackage.open(stream)); } - private FastPoiBook(OPCPackage pkg) throws IOException, OpenXML4JException { - this.pkg = pkg; - this.reader = new XSSFReader(pkg); + @Nonnull + private static FastPoiBook create(@Nonnull OPCPackage pkg) throws IOException, OpenXML4JException { + XSSFReader reader = new XSSFReader(pkg); WorkbookData workbookData = new WorkbookDataSax2EventHandler().parse(newWorkBookDataSupplier(reader)); - this.sheets = workbookData.sheets; - this.sheetContext = new SheetContext( + FastPoiContext sheetContext = new FastPoiContext( new SharedStringsDataSax2EventHandler().parse(newSharedStringsDataSupplier(reader)), new StylesDataSax2EventHandler().parse(newStylesDataSupplier(reader)), workbookData.date1904 ); + return new FastPoiBook(pkg, reader, workbookData.sheets, sheetContext); + } + + private final OPCPackage pkg; + private final XSSFReader reader; + private final List sheets; + private final FastPoiContext sheetContext; + + private FastPoiBook(OPCPackage pkg, XSSFReader reader, List sheets, FastPoiContext sheetContext) { + this.pkg = pkg; + this.reader = reader; + this.sheets = sheets; + this.sheetContext = sheetContext; } @Override @@ -135,46 +143,9 @@ public SheetMeta(String relationId, String name) { this.name = name; } } - -// @VisibleForTesting - @Immutable - static final class Style { - - public final int formatId; - @Nullable - public final String formatString; - - public Style(int formatId, @Nullable String formatString) { - this.formatId = formatId; - this.formatString = formatString; - } - } - -// @VisibleForTesting - @Immutable - static final class SheetContext { - - @Nonnull - public final List sharedStrings; - @Nonnull - public final List