Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DANS Optimizations of UpdateDatasetVersionCommand #10856

Draft
wants to merge 25 commits into
base: develop
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
be7f8ba
handle OptimisticLockException for edit/publish
qqmyers Aug 22, 2024
709ecf7
Updates to speed difference calcs and cleanup for terms differencing
qqmyers Aug 29, 2024
4bb3943
tests for metadata, varmetadata, replace,add, remove cases
qqmyers Aug 29, 2024
8437c63
tests for terms cleanup
qqmyers Aug 29, 2024
4f4736e
Cleanup logging
qqmyers Aug 29, 2024
1fc6f51
Show loop execution time in log
qqmyers Aug 29, 2024
b27739f
release note
qqmyers Aug 29, 2024
c83a260
Merge branch 'IQSS/10814-Improve_dataset_version_differencing' into
qqmyers Sep 4, 2024
6852070
Merge remote-tracking branch 'IQSS/develop' into DANS-performance
qqmyers Sep 4, 2024
d12a559
separate modified date table
qqmyers Sep 10, 2024
3d7a62e
Merge remote-tracking branch 'IQSS/develop' into DANS-performance
qqmyers Sep 16, 2024
74229f8
add mod date object when creating ds version
qqmyers Sep 17, 2024
ca2627d
get/set modified date object
qqmyers Sep 17, 2024
8503412
make more methods public
qqmyers Sep 17, 2024
f3a8362
add version, serialize, primary key info
qqmyers Sep 17, 2024
f6c51b4
use common Difference object to get changed mdbs
qqmyers Sep 17, 2024
b3e79da
create a difference up front and use it, kludge for non md changes
qqmyers Sep 17, 2024
11721e5
drop unused ref to unuseful version field
qqmyers Sep 17, 2024
25fdff0
handle other uses of checkSystemMetadataKeyIfNeeded
qqmyers Sep 17, 2024
a57f052
add contructor to initialize modifiedDate
qqmyers Sep 17, 2024
926c672
fix/support add file
qqmyers Sep 18, 2024
f9f039f
use merged files in dataset (which is passed to indexing)
qqmyers Sep 19, 2024
573215b
Merge remote-tracking branch 'IQSS/develop' into DANS-performance
qqmyers Sep 19, 2024
7c2c68a
fix differencing, try persist in fmds
qqmyers Sep 19, 2024
c312622
fix merge of modified date
qqmyers Sep 20, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions doc/release-notes/10814-Differencing improvement.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
### More Scalable Dataset Version Differencing

Differencing between dataset versions, which is done during dataset edit operations and to populate the dataset page versions table has been made signficantly more scalable.
2 changes: 2 additions & 0 deletions src/main/java/edu/harvard/iq/dataverse/Dataset.java
Original file line number Diff line number Diff line change
Expand Up @@ -353,6 +353,8 @@ public void setVersions(List<DatasetVersion> versions) {
private DatasetVersion createNewDatasetVersion(Template template, FileMetadata fmVarMet) {

DatasetVersion dsv = new DatasetVersion();
DatasetVersionModifiedDate date = new DatasetVersionModifiedDate();
dsv.setModifiedDate(date);
dsv.setVersionState(DatasetVersion.VersionState.DRAFT);
dsv.setFileMetadatas(new ArrayList<>());
DatasetVersion latestVersion;
Expand Down
17 changes: 17 additions & 0 deletions src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@
import jakarta.faces.view.ViewScoped;
import jakarta.inject.Inject;
import jakarta.inject.Named;
import jakarta.persistence.OptimisticLockException;

import org.apache.commons.lang3.StringUtils;
import org.primefaces.event.FileUploadEvent;
Expand Down Expand Up @@ -2888,6 +2889,9 @@ private String releaseDataset(boolean minor) {
// the lock info system.
JsfHelper.addErrorMessage(ex.getLocalizedMessage());
}
if(ex.getCause()!=null && ex.getCause() instanceof OptimisticLockException) {
JsfHelper.addErrorMessage(BundleUtil.getStringFromBundle("dataset.message.parallelPublishError"));
}
logger.severe(ex.getMessage());
}

Expand Down Expand Up @@ -4002,6 +4006,10 @@ public String save() {
Throwable cause = ex;
while (cause.getCause()!= null) {
cause = cause.getCause();
if (cause != null && cause instanceof OptimisticLockException) {
JsfHelper.addErrorMessage(BundleUtil.getStringFromBundle("dataset.message.parallelUpdateError"));
return null;
}
error.append(cause).append(" ");
error.append(cause.getMessage()).append(" ");
}
Expand All @@ -4011,6 +4019,15 @@ public String save() {
} catch (CommandException ex) {
//FacesContext.getCurrentInstance().addMessage(null, new FacesMessage(FacesMessage.SEVERITY_ERROR, "Dataset Save Failed", " - " + ex.toString()));
logger.log(Level.SEVERE, "CommandException, when attempting to update the dataset: " + ex.getMessage(), ex);
Throwable cause = ex;
while (cause.getCause()!= null) {
cause = cause.getCause();
logger.info("Cause is: " + cause.getClass().getName() + ", Message: " + cause.getMessage());
if (cause != null && cause instanceof OptimisticLockException) {
JsfHelper.addErrorMessage(BundleUtil.getStringFromBundle("dataset.message.parallelUpdateError"));
return null;
}
}
populateDatasetUpdateFailureMessage();
return returnToDraftVersion();
}
Expand Down
46 changes: 23 additions & 23 deletions src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,12 @@
@ValidateVersionNote(versionNote = "versionNote", versionState = "versionState")
public class DatasetVersion implements Serializable {

public DatasetVersion() {
super();
this.modifiedDate = new DatasetVersionModifiedDate();

}

private static final Logger logger = Logger.getLogger(DatasetVersion.class.getCanonicalName());
private static final Validator validator = Validation.buildDefaultValidatorFactory().getValidator();

Expand Down Expand Up @@ -127,8 +133,17 @@ public enum VersionState {

private String UNF;

@Version
private Long version;

@OneToOne(cascade = {CascadeType.REMOVE, CascadeType.MERGE, CascadeType.PERSIST})
private DatasetVersionModifiedDate modifiedDate;

public void setModifiedDate(DatasetVersionModifiedDate modifiedDate) {
this.modifiedDate = modifiedDate;
}

public DatasetVersionModifiedDate getModifiedDate() {
return modifiedDate;
}

private Long versionNumber;
private Long minorVersionNumber;
Expand Down Expand Up @@ -163,10 +178,6 @@ public enum VersionState {
@Column( nullable=false )
private Date createTime;

@Temporal(value = TemporalType.TIMESTAMP)
@Column( nullable=false )
private Date lastUpdateTime;

@Temporal(value = TemporalType.TIMESTAMP)
private Date releaseTime;

Expand Down Expand Up @@ -231,17 +242,6 @@ public void setUNF(String UNF) {
this.UNF = UNF;
}

/**
* This is JPA's optimistic locking mechanism, and has no semantic meaning in the DV object model.
* @return the object db version
*/
public Long getVersion() {
return this.version;
}

public void setVersion(Long version) {
}

public String getDataverseSiteUrl() {
return dataverseSiteUrl;
}
Expand Down Expand Up @@ -429,25 +429,25 @@ public void setCreateTime(Date createTime) {
}

public Date getLastUpdateTime() {
return lastUpdateTime;
return modifiedDate.getLastUpdateTime();
}

public void setLastUpdateTime(Date lastUpdateTime) {
if (createTime == null) {
createTime = lastUpdateTime;
}
this.lastUpdateTime = lastUpdateTime;
modifiedDate.setLastUpdateTime(lastUpdateTime);
}

public String getVersionDate() {
if (this.lastUpdateTime == null){
if (modifiedDate.getLastUpdateTime() == null){
return null;
}
return DateUtil.formatDate(lastUpdateTime);
return DateUtil.formatDate(modifiedDate.getLastUpdateTime());
}

public String getVersionYear() {
return new SimpleDateFormat("yyyy").format(lastUpdateTime);
return new SimpleDateFormat("yyyy").format(modifiedDate.getLastUpdateTime());
}

public Date getReleaseTime() {
Expand Down Expand Up @@ -2122,7 +2122,7 @@ public String getJsonLd() {
}

public String getLocaleLastUpdateTime() {
return DateUtil.formatDate(new Timestamp(lastUpdateTime.getTime()));
return DateUtil.formatDate(new Timestamp(modifiedDate.getLastUpdateTime().getTime()));
}

public String getExternalStatusLabel() {
Expand Down
Loading
Loading