Issue #692 obv dupe changes for ftd and matcher

common/client/src/main/java/zingg/common/client/Arguments.java

@@ -111,7 +111,7 @@ public class Arguments implements Serializable {
 	float stopWordsCutoff = 0.1f;
 	long blockSize = 100L;
 	String column;
-	String obviousDupeCondition;
+	ObviousDupes[] obviousDupes;
 
 
 	public void setThreshold(double threshold) {
@@ -478,12 +478,12 @@ public void setColumn(String column) {
 		this.column = column;
 	}
 
-	public String getObviousDupeCondition() {
-		return obviousDupeCondition;
+	public ObviousDupes[] getObviousDupes() {
+		return obviousDupes;
 	}
 
-	public void setObviousDupeCondition(String obviousDupeCondition) {
-		this.obviousDupeCondition = obviousDupeCondition;
+	public void setObviousDupes(ObviousDupes[] obviousDupes) {
+		this.obviousDupes = obviousDupes;
 	}
 
 	public long getBlockSize() {

common/client/src/main/java/zingg/common/client/ObviousDupes.java

@@ -0,0 +1,40 @@
+package zingg.common.client;
+
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.HashMap;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class ObviousDupes implements Serializable {
+
+	private static final long serialVersionUID = 1L;
+	public static final Log LOG = LogFactory.getLog(ObviousDupes.class);
+
+	public static final String fieldName = "fieldName";
+
+	public ObviousDupes() {
+
+	}
+
+	public ObviousDupes(HashMap<String, String>[] matchCondition) {
+		this.matchCondition = matchCondition;
+	}
+
+	HashMap<String,String>[]  matchCondition;
+
+	public HashMap<String, String>[] getMatchCondition() {
+		return matchCondition;
+	}
+
+	public void setMatchCondition(HashMap<String, String>[] matchCondition) {
+		this.matchCondition = matchCondition;
+	}
+
+	@Override
+	public String toString() {
+		return Arrays.toString(matchCondition);
+	}
+
+}

common/client/src/main/java/zingg/common/client/ZFrame.java

@@ -2,7 +2,6 @@
 
 import java.util.List;
 
-
 //Dataset, Row, column
 public interface ZFrame<D, R, C> {
 
@@ -11,11 +10,7 @@ public interface ZFrame<D, R, C> {
 
 	public static final String COL_COUNT = "count";
 	public static final String COL_VALUE = "VALUE";
-
-	public static final String orSeperator = "\\|";	
-	public static final String andSeperator = "\\&";	
-
-
+
     public ZFrame<D, R, C> cache();
     public ZFrame<D, R, C> as(String s);
     public String[] columns();
@@ -99,7 +94,9 @@ public interface ZFrame<D, R, C> {
     public C gt(String c, double val);
 
 	public C equalTo(String c, String e);
-
+
+	public C equalTo(C column1, C column2);
+
 	public C notEqual(String c, String e);
 
     public C notEqual(String e);
@@ -110,6 +107,13 @@ public interface ZFrame<D, R, C> {
 
 	public C notEqual(String c, int e);
 
+	public C not(C col);
+
+	public C isNotNull(C col);
+
+	public C and(C col1, C col2);
+
+	public C or(C col1, C col2);
 
     public void show(int num);
     public void show();
@@ -160,13 +164,5 @@ public interface ZFrame<D, R, C> {
 	public ZFrame<D, R, C> filterNotNullCond(String colName);
 
 	public ZFrame<D, R, C> filterNullCond(String colName);
-
-	public C getObviousDupesFilter(String obviousDupeString, C extraAndCond);
-
-	public C getObviousDupesFilter(ZFrame<D, R, C> dfToJoin, String obviousDupeString, C extraAndCond);
-
-	public C getReverseObviousDupesFilter(String obviousDupeString, C extraAndCond);
-
-	public C getReverseObviousDupesFilter(ZFrame<D, R, C> dfToJoin, String obviousDupeString, C extraAndCond);
-
+
 }

common/client/src/test/java/zingg/common/client/TestArguments.java

@@ -7,6 +7,7 @@
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Paths;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -241,4 +242,25 @@ public void testMatchTypeWrong() {
 
 	}
 
+
+	@Test
+	public void testObvDupe() {
+			Arguments args;
+            try {
+                args = argsUtil.createArgumentsFromJSON(getClass().getResource("../../../testArguments/configObvDupe.json").getFile(), "test");
+
+                ObviousDupes[] obviousDupes = args.getObviousDupes();
+                HashMap<String,String>[]  matchCondition = obviousDupes[0].getMatchCondition();
+
+                assertEquals("fname", matchCondition[0].get(ObviousDupes.fieldName));
+
+            } catch (Exception | ZinggClientException e) {
+                // TODO Auto-generated catch block
+                e.printStackTrace();
+				fail("Could not read config");
+            }
+
+	}
+
+
 }

common/client/src/test/resources/testArguments/configObvDupe.json

@@ -0,0 +1,117 @@
+{	
+	"fieldDefinition":[
+		{
+			"fieldName" : "recId",
+			"matchType" : "dont_use",
+			"fields" : "recId",
+			"dataType": "string" 
+		},
+		{
+			"fieldName" : "fname",
+			"matchType" : "fuzzy",
+			"fields" : "fname",
+			"dataType": "string" 
+		},
+		{
+			"fieldName" : "lname",
+			"matchType" : "fuzzy",
+			"fields" : "lname",
+			"dataType": "string" 
+		},
+		{
+			"fieldName" : "stNo",
+			"matchType": "fuzzy",
+			"fields" : "stNo",
+			"dataType": "string" 
+		},
+		{
+			"fieldName" : "add1",
+			"matchType": "fuzzy",
+			"fields" : "add1",
+			"dataType": "string" 
+		},
+		{
+			"fieldName" : "add2",
+			"matchType": "fuzzy",
+			"fields" : "add2",
+			"dataType": "string" 
+		},
+		{
+			"fieldName" : "city",
+			"matchType": "fuzzy",
+			"fields" : "city",
+			"dataType": "string" 
+		},
+		{
+			"fieldName" : "areacode",
+			"matchType": "fuzzy",
+			"fields" : "areacode",
+			"dataType": "string" 
+		},
+		{
+			"fieldName" : "state",
+			"matchType": "fuzzy",
+			"fields" : "state",
+			"dataType": "string" 
+		},
+		{
+			"fieldName" : "dob",
+			"matchType": "fuzzy",
+			"fields" : "dob",
+			"dataType": "string" 
+		},
+		{
+			"fieldName" : "ssn",
+			"matchType": "fuzzy",
+			"fields" : "ssn",
+			"dataType": "string" 
+		}
+		],
+		"output" : [{
+			"name":"output", 
+			"format":"csv", 
+			"props": {
+				"location": "/tmp/zinggOutput",
+				"delimiter": ",",
+				"header":true
+			}
+		}],
+		"data" : [{
+			"name":"test", 
+			"format":"csv", 
+			"props": {
+				"location": "examples/febrl/test.csv",
+				"delimiter": ",",
+				"header":false					
+			},
+			"schema": "recId string, fname string, lname string, stNo string, add1 string, add2 string, city string, state string, areacode string, dob string, ssn  string" 
+			}
+		],
+		"obviousDupes":[
+			{
+			   "matchCondition":[
+				  {
+					 "fieldName":"fname"
+				  },
+				  {
+					 "fieldName":"stNo"
+				  },
+				  {
+					 "fieldName":"add1"
+				  }
+			   ]
+			},
+			{
+			   "matchCondition":[
+				  {
+					"fieldName":"recId"
+				  }
+			   ]
+			}
+		 ],
+		"labelDataSampleSize" : 0.5,
+		"numPartitions":4,
+		"modelId": 100,
+		"zinggDir": "models"
+
+}