CVOC : allow flexible params in retrievalUri (Ontoportal integration)

doc/release-notes/9276-allow-flexible-params-in-retrievaluri-cvoc.md

@@ -0,0 +1,14 @@
+## Release Highlights
+
+### Updates on Support for External Vocabulary Services
+
+#### HTTP Headers
+
+You are now able to add HTTP request headers required by the service you are implementing (#10331)
+
+#### Flexible params in retrievalUri
+
+You can now use `managed-fields` field names as well as the `term-uri-field` field name as parameters in the `retrieval-uri` when configuring an external vocabulary service. `{0}` as an alternative to using the `term-uri-field` name is still supported for backward compatibility.
+Also you can specify if the value must be url encoded with `encodeUrl:`. (#10404)
+
+For example : `"retrieval-uri": "https://data.agroportal.lirmm.fr/ontologies/{keywordVocabulary}/classes/{encodeUrl:keywordTermURL}"`

src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java

@@ -4,6 +4,7 @@
 import java.io.StringReader;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.net.URLEncoder;
 import java.nio.charset.StandardCharsets;
 import java.sql.Timestamp;
 import java.text.MessageFormat;
@@ -349,8 +350,12 @@ public void registerExternalVocabValues(DatasetField df) {
         logger.fine("Registering for field: " + dft.getName());
         JsonObject cvocEntry = getCVocConf(true).get(dft.getId());
         if (dft.isPrimitive()) {
+            List<DatasetField> siblingsDatasetFields = new ArrayList<>();
+            if(dft.getParentDatasetFieldType()!=null) {
+                siblingsDatasetFields = df.getParentDatasetFieldCompoundValue().getChildDatasetFields();
+            }
             for (DatasetFieldValue dfv : df.getDatasetFieldValues()) {
-                registerExternalTerm(cvocEntry, dfv.getValue());
+                registerExternalTerm(cvocEntry, dfv.getValue(), siblingsDatasetFields);
             }
         } else {
             if (df.getDatasetFieldType().isCompound()) {
@@ -359,7 +364,7 @@ public void registerExternalVocabValues(DatasetField df) {
                     for (DatasetField cdf : cv.getChildDatasetFields()) {
                         logger.fine("Found term uri field type id: " + cdf.getDatasetFieldType().getId());
                         if (cdf.getDatasetFieldType().equals(termdft)) {
-                            registerExternalTerm(cvocEntry, cdf.getValue());
+                            registerExternalTerm(cvocEntry, cdf.getValue(), cv.getChildDatasetFields());
                         }
                     }
                 }
@@ -447,15 +452,17 @@ public JsonObject getExternalVocabularyValue(String termUri) {
 
     /**
      * Perform a call to the external service to retrieve information about the term URI
-     * @param cvocEntry - the configuration for the DatasetFieldType associated with this term 
-     * @param term - the term uri as a string
+     *
+     * @param cvocEntry             - the configuration for the DatasetFieldType associated with this term
+     * @param term                  - the term uri as a string
+     * @param relatedDatasetFields  - siblings or childs of the term
      */
-    @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
-    public void registerExternalTerm(JsonObject cvocEntry, String term) {
+    public void registerExternalTerm(JsonObject cvocEntry, String term, List<DatasetField> relatedDatasetFields) {
         String retrievalUri = cvocEntry.getString("retrieval-uri");
+        String termUriFieldName = cvocEntry.getString("term-uri-field");
         String prefix = cvocEntry.getString("prefix", null);
         if(term.isBlank()) {
-            logger.fine("Ingoring blank term");
+            logger.fine("Ignoring blank term");
             return;
         }
         boolean isExternal = false;
@@ -486,7 +493,13 @@ public void registerExternalTerm(JsonObject cvocEntry, String term) {
             }
             if (evv.getValue() == null) {
                 String adjustedTerm = (prefix==null)? term: term.replace(prefix, "");
-                retrievalUri = retrievalUri.replace("{0}", adjustedTerm);
+
+                retrievalUri = replaceRetrievalUriParam(retrievalUri, "0", adjustedTerm);
+                retrievalUri = replaceRetrievalUriParam(retrievalUri, termUriFieldName, adjustedTerm);
+                for (DatasetField f : relatedDatasetFields) {
+                    retrievalUri = replaceRetrievalUriParam(retrievalUri, f.getDatasetFieldType().getName(), f.getValue());
+                }
+
                 logger.fine("Didn't find " + term + ", calling " + retrievalUri);
                 try (CloseableHttpClient httpClient = HttpClients.custom()
                         .addInterceptorLast(new HttpResponseInterceptor() {
@@ -546,6 +559,17 @@ public void process(HttpResponse response, HttpContext context) throws HttpExcep
 
     }
 
+    private String replaceRetrievalUriParam(String retrievalUri, String paramName, String value) {
+
+        if(retrievalUri.contains("encodeUrl:" + paramName)) {
+            retrievalUri = retrievalUri.replace("{encodeUrl:"+paramName+"}", URLEncoder.encode(value, StandardCharsets.UTF_8));
+        } else {
+            retrievalUri = retrievalUri.replace("{"+paramName+"}", value);
+        }
+
+        return retrievalUri;
+    }
+
     /**
      * Parse the raw value returned by an external service for a give term uri and
      * filter it according to the 'retrieval-filtering' configuration for this

src/main/java/edu/harvard/iq/dataverse/DatasetPage.java

@@ -3932,12 +3932,6 @@ public String save() {
                 ((UpdateDatasetVersionCommand) cmd).setValidateLenient(true);
             }
             dataset = commandEngine.submit(cmd);
-            for (DatasetField df : dataset.getLatestVersion().getFlatDatasetFields()) {
-                logger.fine("Found id: " + df.getDatasetFieldType().getId());
-                if (fieldService.getCVocConf(true).containsKey(df.getDatasetFieldType().getId())) {
-                    fieldService.registerExternalVocabValues(df);
-                }
-            }
             if (editMode == EditMode.CREATE) {
                 if (session.getUser() instanceof AuthenticatedUser) {
                     userNotificationService.sendNotification((AuthenticatedUser) session.getUser(), dataset.getCreateDate(), UserNotification.Type.CREATEDS, dataset.getLatestVersion().getId());

src/main/java/edu/harvard/iq/dataverse/EjbDataverseEngine.java

@@ -134,6 +134,9 @@ public class EjbDataverseEngine {
     @EJB
     DatasetLinkingServiceBean dsLinking;
 
+    @EJB
+    DatasetFieldServiceBean dsField;
+
     @EJB
     ExplicitGroupServiceBean explicitGroups;
 
@@ -509,7 +512,12 @@ public DataverseLinkingServiceBean dvLinking() {
                 public DatasetLinkingServiceBean dsLinking() {
                     return dsLinking;
                 }
-
+
+                @Override
+                public DatasetFieldServiceBean dsField() {
+                    return dsField;
+                }
+
                 @Override
                 public StorageUseServiceBean storageUse() {
                     return storageUseService;

src/main/java/edu/harvard/iq/dataverse/engine/command/CommandContext.java

@@ -1,6 +1,7 @@
 package edu.harvard.iq.dataverse.engine.command;
 
 import edu.harvard.iq.dataverse.DataFileServiceBean;
+import edu.harvard.iq.dataverse.DatasetFieldServiceBean;
 import edu.harvard.iq.dataverse.DatasetLinkingServiceBean;
 import edu.harvard.iq.dataverse.DatasetServiceBean;
 import edu.harvard.iq.dataverse.DatasetVersionServiceBean;
@@ -146,4 +147,6 @@ public interface CommandContext {
     public Stack<Command> getCommandsCalled();
 
     public void addCommand(Command command);
+
+    public DatasetFieldServiceBean dsField();
 }

src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractCreateDatasetCommand.java

@@ -1,26 +1,19 @@
 package edu.harvard.iq.dataverse.engine.command.impl;
 
-import edu.harvard.iq.dataverse.*;
+import edu.harvard.iq.dataverse.DataFile;
+import edu.harvard.iq.dataverse.Dataset;
+import edu.harvard.iq.dataverse.DatasetVersion;
 import edu.harvard.iq.dataverse.authorization.Permission;
 import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
-import edu.harvard.iq.dataverse.batch.util.LoggingUtil;
 import edu.harvard.iq.dataverse.dataaccess.DataAccess;
-import edu.harvard.iq.dataverse.datacapturemodule.DataCaptureModuleUtil;
-import edu.harvard.iq.dataverse.datacapturemodule.ScriptRequestResponse;
 import edu.harvard.iq.dataverse.engine.command.CommandContext;
 import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
 import edu.harvard.iq.dataverse.engine.command.RequiredPermissions;
 import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
-import edu.harvard.iq.dataverse.engine.command.exception.CommandExecutionException;
 import edu.harvard.iq.dataverse.pidproviders.PidProvider;
-import edu.harvard.iq.dataverse.pidproviders.PidUtil;
-import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 import static edu.harvard.iq.dataverse.util.StringUtil.isEmpty;
-import java.io.IOException;
 import java.util.Objects;
-import java.util.logging.Level;
 import java.util.logging.Logger;
-import org.apache.solr.client.solrj.SolrServerException;
 
 /**;
  * An abstract base class for commands that creates {@link Dataset}s.
@@ -97,6 +90,8 @@ public Dataset execute(CommandContext ctxt) throws CommandException {
         if(!harvested) {
             checkSystemMetadataKeyIfNeeded(dsv, null);
         }
+
+        registerExternalVocabValuesIfAny(ctxt, dsv);
 
         theDataset.setCreator((AuthenticatedUser) getRequest().getUser());
 

src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractDatasetCommand.java

@@ -2,10 +2,13 @@
 
 import edu.harvard.iq.dataverse.Dataset;
 import edu.harvard.iq.dataverse.DatasetField;
+import edu.harvard.iq.dataverse.DatasetFieldServiceBean;
 import edu.harvard.iq.dataverse.DatasetVersion;
 import edu.harvard.iq.dataverse.DatasetVersionDifference;
 import edu.harvard.iq.dataverse.DatasetVersionUser;
 import edu.harvard.iq.dataverse.Dataverse;
+import edu.harvard.iq.dataverse.MetadataBlock;
+import edu.harvard.iq.dataverse.TermsOfUseAndAccess;
 import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
 import edu.harvard.iq.dataverse.engine.command.AbstractCommand;
 import edu.harvard.iq.dataverse.engine.command.CommandContext;
@@ -24,9 +27,8 @@
 import java.util.logging.Logger;
 import static java.util.stream.Collectors.joining;
 
+import jakarta.ejb.EJB;
 import jakarta.validation.ConstraintViolation;
-import edu.harvard.iq.dataverse.MetadataBlock;
-import edu.harvard.iq.dataverse.TermsOfUseAndAccess;
 import edu.harvard.iq.dataverse.settings.JvmSettings;
 
 /**
@@ -231,4 +233,13 @@ protected void checkSystemMetadataKeyIfNeeded(DatasetVersion newVersion, Dataset
             }
         }
     }
+
+    protected void registerExternalVocabValuesIfAny(CommandContext ctxt, DatasetVersion newVersion) {
+        for (DatasetField df : newVersion.getFlatDatasetFields()) {
+            logger.fine("Found id: " + df.getDatasetFieldType().getId());
+            if (ctxt.dsField().getCVocConf(true).containsKey(df.getDatasetFieldType().getId())) {
+                ctxt.dsField().registerExternalVocabValues(df);
+            }
+        }
+    }
 }

src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDatasetVersionCommand.java

@@ -59,7 +59,8 @@ public DatasetVersion execute(CommandContext ctxt) throws CommandException {
         //Will throw an IllegalCommandException if a system metadatablock is changed and the appropriate key is not supplied.
         checkSystemMetadataKeyIfNeeded(newVersion, latest);
 
-
+        registerExternalVocabValuesIfAny(ctxt, newVersion);
+
         List<FileMetadata> newVersionMetadatum = new ArrayList<>(latest.getFileMetadatas().size());
         for ( FileMetadata fmd : latest.getFileMetadatas() ) {
             FileMetadata fmdCopy = fmd.createCopy();

src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java

@@ -1,14 +1,19 @@
 package edu.harvard.iq.dataverse.engine.command.impl;
 
-import edu.harvard.iq.dataverse.*;
+import edu.harvard.iq.dataverse.DataFile;
+import edu.harvard.iq.dataverse.DataFileCategory;
+import edu.harvard.iq.dataverse.Dataset;
+import edu.harvard.iq.dataverse.DatasetLock;
+import edu.harvard.iq.dataverse.DatasetVersion;
+import edu.harvard.iq.dataverse.DatasetVersionDifference;
+import edu.harvard.iq.dataverse.FileMetadata;
 import edu.harvard.iq.dataverse.authorization.Permission;
 import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
 import edu.harvard.iq.dataverse.engine.command.CommandContext;
 import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
 import edu.harvard.iq.dataverse.engine.command.RequiredPermissions;
 import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
 import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException;
-import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 import edu.harvard.iq.dataverse.util.DatasetFieldUtil;
 import edu.harvard.iq.dataverse.util.FileMetadataUtil;
 
@@ -115,8 +120,11 @@ public Dataset execute(CommandContext ctxt) throws CommandException {
 
         //Will throw an IllegalCommandException if a system metadatablock is changed and the appropriate key is not supplied.
         checkSystemMetadataKeyIfNeeded(getDataset().getOrCreateEditVersion(fmVarMet), persistedVersion);
-
-
+
+        getDataset().getOrCreateEditVersion().setLastUpdateTime(getTimestamp());
+
+        registerExternalVocabValuesIfAny(ctxt, getDataset().getOrCreateEditVersion(fmVarMet));
+
         try {
             // Invariant: Dataset has no locks preventing the update
             String lockInfoMessage = "saving current edits";
@@ -256,7 +264,6 @@ public Dataset execute(CommandContext ctxt) throws CommandException {
                 ctxt.ingest().recalculateDatasetVersionUNF(theDataset.getOrCreateEditVersion());
             }
 
-            theDataset.getOrCreateEditVersion().setLastUpdateTime(getTimestamp());
             theDataset.setModificationTime(getTimestamp());
 
             savedDataset = ctxt.em().merge(theDataset);

src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java

@@ -466,7 +466,6 @@ private static void addField(DatasetField dsf, JsonArray valArray, DatasetFieldT
                     if(!datasetFieldSvc.isValidCVocValue(dsft, strValue)) {
                         throw new BadRequestException("Invalid values submitted for " + dsft.getName() + " which is limited to specific vocabularies.");
                     }
-                    datasetFieldSvc.registerExternalTerm(cvocMap.get(dsft.getId()), strValue);
                 }
                 DatasetFieldValue datasetFieldValue = new DatasetFieldValue();
 

src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java

@@ -847,7 +847,6 @@ public void parsePrimitiveValue(DatasetField dsf, DatasetFieldType dft , JsonObj
                     if(!datasetFieldSvc.isValidCVocValue(dft, datasetFieldValue.getValue())) {
                         throw new JsonParseException("Invalid values submitted for " + dft.getName() + " which is limited to specific vocabularies.");
                     }
-                    datasetFieldSvc.registerExternalTerm(cvocMap.get(dft.getId()), datasetFieldValue.getValue());
                 }
                 vals.add(datasetFieldValue);
             }
@@ -864,7 +863,6 @@ public void parsePrimitiveValue(DatasetField dsf, DatasetFieldType dft , JsonObj
                 if(!datasetFieldSvc.isValidCVocValue(dft, datasetFieldValue.getValue())) {
                     throw new JsonParseException("Invalid values submitted for " + dft.getName() + " which is limited to specific vocabularies.");
                 }
-                datasetFieldSvc.registerExternalTerm(cvocMap.get(dft.getId()), datasetFieldValue.getValue());
             }
             vals.add(datasetFieldValue);
         }

src/test/java/edu/harvard/iq/dataverse/engine/TestCommandContext.java

@@ -150,6 +150,11 @@ public DatasetLinkingServiceBean dsLinking() {
         return null;
     }
 
+    @Override
+    public DatasetFieldServiceBean dsField() {
+        return null;
+    }
+
     @Override
     public AuthenticationServiceBean authentication() {
         return null;