Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Session wrapper and license interface #619

Merged
merged 6 commits into from
Jun 29, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions common/client/src/main/java/zingg/common/client/Client.java
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import zingg.common.client.license.IZinggLicense;
import zingg.common.client.util.Email;
import zingg.common.client.util.EmailBody;

Expand Down Expand Up @@ -223,11 +224,12 @@ else if (options.get(ClientOptions.CONF).value.endsWith("env")) {
}

public void init() throws ZinggClientException {
zingg.init(getArguments(), options.get(ClientOptions.LICENSE).value.trim());
zingg.init(getArguments(), getLicense(options.get(ClientOptions.LICENSE).value.trim()));
if (session != null) zingg.setSession(session);
zingg.setClientOptions(options);
}


protected abstract IZinggLicense getLicense(String license) throws ZinggClientException ;

/**
* Stop the Spark job running context
Expand Down
4 changes: 3 additions & 1 deletion common/client/src/main/java/zingg/common/client/IZingg.java
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
package zingg.common.client;

import zingg.common.client.license.IZinggLicense;

public interface IZingg<S,D,R,C> {

public void init(Arguments args, String license)
public void init(Arguments args, IZinggLicense license)
throws ZinggClientException;

public void execute() throws ZinggClientException;
Expand Down
16 changes: 16 additions & 0 deletions common/client/src/main/java/zingg/common/client/ZSession.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
package zingg.common.client;

import zingg.common.client.license.IZinggLicense;

public interface ZSession<S> {

public S getSession();

public void setSession(S session);

public IZinggLicense getLicense();

public void setLicense(IZinggLicense license);


}
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
package zingg.common.client.license;

public interface ILicenseValidator {

public String getName();

public boolean validate();
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

will probably need a name by which it will be looked up in the IZinggLicense

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

added getName method in commit e9a5741


}
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
package zingg.common.client.license;

public interface IZinggLicense {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

its not clear to me how this will evolve to reading, loading and passing the license constraints to the validators. If that's clear to you, please ignore and we can discuss during the implementation of the concrete classes.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

let's discuss during the implementation of the concrete classes


public ILicenseValidator getValidator(String name);


}
3 changes: 2 additions & 1 deletion common/core/src/main/java/zingg/common/core/Context.java
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import java.io.Serializable;

import zingg.common.client.ZinggClientException;
import zingg.common.client.license.IZinggLicense;
import zingg.common.core.util.BlockingTreeUtil;
import zingg.common.core.util.DSUtil;
import zingg.common.core.util.GraphUtil;
Expand All @@ -29,7 +30,7 @@ public interface Context <S,D, R, C,T> extends Serializable {
public PipeUtilBase<S,D,R,C> getPipeUtil();
public BlockingTreeUtil<S, D,R,C,T> getBlockingTreeUtil() ;

public void init(String license)
public void init(IZinggLicense license)
throws ZinggClientException;

public void cleanup();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
import freemarker.template.Configuration;
import freemarker.template.Template;
import freemarker.template.TemplateExceptionHandler;
// import org.apache.spark.sql.SparkSession;
import freemarker.template.Version;
import zingg.common.client.Arguments;
import zingg.common.client.ZinggClientException;
Expand All @@ -17,6 +16,7 @@
import zingg.common.core.executor.ZinggBase;

public abstract class DocumenterBase<S,D,R,C,T> extends ZinggBase<S,D,R,C,T>{
private static final long serialVersionUID = 1L;
protected static Configuration config;

public DocumenterBase(Context<S,D,R,C,T> context, Arguments args) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import zingg.common.client.Arguments;
import zingg.common.client.ZinggClientException;
import zingg.common.client.ZinggOptions;
import zingg.common.client.license.IZinggLicense;

public abstract class FindAndLabeller<S, D, R, C, T> extends Labeller<S, D, R, C, T> {
private static final long serialVersionUID = 1L;
Expand All @@ -19,7 +20,7 @@ public FindAndLabeller() {
}

@Override
public void init(Arguments args, String license) throws ZinggClientException {
public void init(Arguments args, IZinggLicense license) throws ZinggClientException {
finder.init(args, license);
super.init(args, license);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import zingg.common.client.Arguments;
import zingg.common.client.ZinggClientException;
import zingg.common.client.ZinggOptions;
import zingg.common.client.license.IZinggLicense;

public abstract class TrainMatcher<S,D,R,C,T> extends Matcher<S,D,R,C,T>{

Expand All @@ -18,7 +19,7 @@ public TrainMatcher() {
}

@Override
public void init(Arguments args, String license)
public void init(Arguments args, IZinggLicense license)
throws ZinggClientException {
trainer.init(args, license);
super.init(args, license);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import zingg.common.client.ZFrame;
import zingg.common.client.ZinggClientException;
import zingg.common.client.ZinggOptions;
import zingg.common.client.license.IZinggLicense;
import zingg.common.client.util.ColName;
import zingg.common.client.util.ColValues;
import zingg.common.core.Context;
Expand Down Expand Up @@ -62,7 +63,7 @@ public ZinggBase() {



public void init(Arguments args, String license)
public void init(Arguments args, IZinggLicense license)
throws ZinggClientException {
startTime = System.currentTimeMillis();
this.args = args;
Expand Down
21 changes: 15 additions & 6 deletions spark/client/src/main/java/zingg/spark/client/SparkClient.java
Original file line number Diff line number Diff line change
@@ -1,22 +1,26 @@
package zingg.spark.client;

import java.io.Serializable;
import org.apache.spark.api.java.JavaSparkContext;

import org.apache.spark.sql.Column;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.types.DataType;

import zingg.common.client.*;
import zingg.common.client.Arguments;
import zingg.common.client.Client;
import zingg.common.client.ClientOptions;
import zingg.common.client.IZinggFactory;
import zingg.common.client.ZinggClientException;
import zingg.common.client.license.IZinggLicense;

/**
* This is the main point of interface with the Zingg matching product.
*
* @author sgoyal
*
*/
public class SparkClient extends Client<SparkSession, Dataset<Row>, Row, Column, DataType> implements Serializable {
public class SparkClient extends Client<ZSparkSession, Dataset<Row>, Row, Column, DataType> implements Serializable {

private static final long serialVersionUID = 1L;

Expand All @@ -25,7 +29,7 @@ public SparkClient(Arguments args, ClientOptions options) throws ZinggClientExce

}

public SparkClient(Arguments args, ClientOptions options, SparkSession s) throws ZinggClientException {
public SparkClient(Arguments args, ClientOptions options, ZSparkSession s) throws ZinggClientException {
super(args, options, s);
}

Expand All @@ -47,7 +51,7 @@ public IZinggFactory getZinggFactory() throws InstantiationException, IllegalAcc


@Override
public Client<SparkSession, Dataset<Row>, Row, Column, DataType> getClient(Arguments args,
public Client<ZSparkSession, Dataset<Row>, Row, Column, DataType> getClient(Arguments args,
ClientOptions options) throws ZinggClientException {
// TODO Auto-generated method stub
SparkClient client = null;
Expand All @@ -67,5 +71,10 @@ public static void main(String... args) {
client.mainMethod(args);
}

@Override
protected IZinggLicense getLicense(String license) throws ZinggClientException {
return null;
}


}
39 changes: 39 additions & 0 deletions spark/client/src/main/java/zingg/spark/client/ZSparkSession.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
package zingg.spark.client;
import org.apache.spark.sql.SparkSession;

import zingg.common.client.ZSession;
import zingg.common.client.license.IZinggLicense;

public class ZSparkSession implements ZSession<SparkSession> {

private SparkSession session;

private IZinggLicense license;

public ZSparkSession(SparkSession session, IZinggLicense license) {
super();
this.session = session;
this.license = license;
}

@Override
public SparkSession getSession() {
return session;
}

@Override
public void setSession(SparkSession session) {
this.session = session;
}

@Override
public IZinggLicense getLicense() {
return license;
}

@Override
public void setLicense(IZinggLicense license) {
this.license = license;
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -3,24 +3,24 @@
import org.apache.spark.sql.Column;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.types.DataType;

import freemarker.template.Version;
import zingg.common.client.Arguments;
import zingg.common.core.Context;
import zingg.common.core.documenter.DataColDocumenter;
import zingg.common.core.documenter.RowWrapper;
import zingg.spark.client.ZSparkSession;

/**
* Spark specific implementation of DataColDocumenter
*
*/
public class SparkDataColDocumenter extends DataColDocumenter<SparkSession, Dataset<Row>, Row, Column,DataType> {
public class SparkDataColDocumenter extends DataColDocumenter<ZSparkSession, Dataset<Row>, Row, Column,DataType> {

private static final long serialVersionUID = 1L;

public SparkDataColDocumenter(Context<SparkSession, Dataset<Row>, Row, Column,DataType> context, Arguments args) {
public SparkDataColDocumenter(Context<ZSparkSession, Dataset<Row>, Row, Column,DataType> context, Arguments args) {
super(context, args);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,24 +3,24 @@
import org.apache.spark.sql.Column;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.types.DataType;

import freemarker.template.Version;
import zingg.common.client.Arguments;
import zingg.common.core.Context;
import zingg.common.core.documenter.DataDocumenter;
import zingg.common.core.documenter.RowWrapper;
import zingg.spark.client.ZSparkSession;

/**
* Spark specific implementation of DataDocumenter
*
*/
public class SparkDataDocumenter extends DataDocumenter<SparkSession, Dataset<Row>, Row, Column,DataType> {
public class SparkDataDocumenter extends DataDocumenter<ZSparkSession, Dataset<Row>, Row, Column,DataType> {

private static final long serialVersionUID = 1L;

public SparkDataDocumenter(Context<SparkSession, Dataset<Row>, Row, Column,DataType> context, Arguments args) {
public SparkDataDocumenter(Context<ZSparkSession, Dataset<Row>, Row, Column,DataType> context, Arguments args) {
super(context, args);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,25 +3,25 @@
import org.apache.spark.sql.Column;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.types.DataType;

import freemarker.template.Version;
import zingg.common.client.Arguments;
import zingg.common.core.Context;
import zingg.common.core.documenter.ModelColDocumenter;
import zingg.common.core.documenter.RowWrapper;
import zingg.spark.client.ZSparkSession;

/**
* Spark specific implementation of ModelColDocumenter
*
*
*/
public class SparkModelColDocumenter extends ModelColDocumenter<SparkSession, Dataset<Row>, Row, Column,DataType> {
public class SparkModelColDocumenter extends ModelColDocumenter<ZSparkSession, Dataset<Row>, Row, Column,DataType> {

private static final long serialVersionUID = 1L;

public SparkModelColDocumenter(Context<SparkSession, Dataset<Row>, Row, Column,DataType> context, Arguments args) {
public SparkModelColDocumenter(Context<ZSparkSession, Dataset<Row>, Row, Column,DataType> context, Arguments args) {
super(context, args);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,24 +3,24 @@
import org.apache.spark.sql.Column;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.types.DataType;

import freemarker.template.Version;
import zingg.common.client.Arguments;
import zingg.common.core.Context;
import zingg.common.core.documenter.ModelDocumenter;
import zingg.common.core.documenter.RowWrapper;
import zingg.spark.client.ZSparkSession;

/**
* Spark specific implementation of ModelDocumenter
*
*/
public class SparkModelDocumenter extends ModelDocumenter<SparkSession, Dataset<Row>, Row, Column,DataType> {
public class SparkModelDocumenter extends ModelDocumenter<ZSparkSession, Dataset<Row>, Row, Column,DataType> {

private static final long serialVersionUID = 1L;

public SparkModelDocumenter(Context<SparkSession, Dataset<Row>, Row, Column,DataType> context, Arguments args) {
public SparkModelDocumenter(Context<ZSparkSession, Dataset<Row>, Row, Column,DataType> context, Arguments args) {
super(context, args);
super.modelColDoc = new SparkModelColDocumenter(context,args);
}
Expand Down
Loading