Merge pull request #6188 from titellus/4.1.x

Version 4 improvements

.gitignore

@@ -40,9 +40,6 @@ schematrons/.build
 target/
 transifex/transifex-format/
 web-itest/jcs_caching
-web-ui-docs/node
-web-ui-docs/node_modules
-web-ui-docs/package-lock.json
 web-ui/src/main/resources/catalog/lib/gn*.min.js
 web-ui/src/main/resources/catalog/style/gn*.css
 web-ui/src/main/resources/catalog/style/gn.css

common/pom.xml

@@ -170,6 +170,10 @@
       <version>20140107</version>
       <scope>compile</scope>
     </dependency>
+    <dependency>
+      <groupId>diff_match_patch</groupId>
+      <artifactId>diff_match_patch</artifactId>
+    </dependency>
   </dependencies>
 
   <build>

common/src/main/java/org/fao/geonet/utils/Diff.java

@@ -0,0 +1,24 @@
+package org.fao.geonet.utils;
+
+import name.fraser.neil.plaintext.diff_match_patch;
+import org.jdom.Element;
+
+import java.util.LinkedList;
+
+public class Diff {
+
+    public static String diff(String oldVersion, String newVersion, DiffType diffType) {
+        diff_match_patch dmp = new diff_match_patch();
+        if (DiffType.patch.equals(diffType)) {
+            LinkedList<diff_match_patch.Patch> diffs =
+                dmp.patch_make(oldVersion, newVersion);
+            return diffs.toString();
+        } else if (DiffType.diff.equals(diffType)) {
+            LinkedList<diff_match_patch.Diff> diffs = dmp.diff_main(oldVersion, newVersion);
+            return diffs.toString();
+        } else {
+            LinkedList<diff_match_patch.Diff> diffs = dmp.diff_main(oldVersion, newVersion);
+            return dmp.diff_prettyHtml(diffs);
+        }
+    }
+}

common/src/main/java/org/fao/geonet/utils/DiffType.java

@@ -0,0 +1,7 @@
+package org.fao.geonet.utils;
+
+public enum DiffType {
+    patch,
+    diff,
+    diffhtml
+}

common/src/main/java/org/fao/geonet/utils/Xml.java

@@ -136,12 +136,14 @@ public final class Xml {
         + "\ud800\udc00-\udbff\udfff"
         + "]";
 
-    /**
-     *
-     * @param validate
-     * @return
-     */
-    private static SAXBuilder getSAXBuilder(boolean validate, Path base) {
+    public static SAXBuilder getSAXBuilder(boolean validate) {
+        SAXBuilder builder = getSAXBuilderWithPathXMLResolver(validate, null);
+        Resolver resolver = ResolverWrapper.getInstance();
+        builder.setEntityResolver(resolver.getXmlResolver());
+        return builder;
+    }
+
+    public static SAXBuilder getSAXBuilder(boolean validate, Path base) {
         SAXBuilder builder = getSAXBuilderWithPathXMLResolver(validate, base);
         Resolver resolver = ResolverWrapper.getInstance();
         builder.setEntityResolver(resolver.getXmlResolver());
@@ -151,6 +153,10 @@ private static SAXBuilder getSAXBuilder(boolean validate, Path base) {
     private static SAXBuilder getSAXBuilderWithPathXMLResolver(boolean validate, Path base) {
         SAXBuilder builder = new SAXBuilder(validate);
         builder.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
+        builder.setFeature("http://apache.org/xml/features/allow-java-encodings", true);
+
+        // To avoid CVE-2021-33813
+        builder.setExpandEntities(false);
 
         if (base != null) {
             NioPathHolder.setBase(base);

core/pom.xml

@@ -126,7 +126,7 @@
     <dependency>
       <groupId>org.keycloak</groupId>
       <artifactId>keycloak-spring-security-adapter</artifactId>
-      <version>4.8.3.Final</version>
+      <version>15.0.2</version>
     </dependency>
     <dependency>
       <groupId>org.springframework.security</groupId>
@@ -499,6 +499,10 @@
       <artifactId>opendap</artifactId>
       <version>2.1</version>
     </dependency>
+    <dependency>
+      <groupId>org.jsoup</groupId>
+      <artifactId>jsoup</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.owasp.esapi</groupId>
       <artifactId>esapi</artifactId>

core/src/main/java/jeeves/server/sources/http/ServletPathFinder.java

@@ -59,6 +59,8 @@ public ServletPathFinder(ServletContext servletContext) {
                 baseUrl = resource.substring(resource.indexOf('/', 1), resource.length() - 1);
             } catch (java.net.MalformedURLException e) { // unlikely
                 baseUrl = servletContext.getServletContextName();
+            } catch (NullPointerException e) { // in test using GeonetMockServletContext
+                baseUrl = "/";
             }
         }
 

core/src/main/java/org/fao/geonet/MetadataResourceDatabaseMigration.java

@@ -170,15 +170,17 @@ public static boolean updateMetadataResourcesLink(@Nonnull Element xml,
     public void update(Connection connection) throws SQLException {
         Log.debug(Geonet.DB, "MetadataResourceDatabaseMigration");
 
+        final SettingManager settingManager = applicationContext.getBean(SettingManager.class);
+
         try (PreparedStatement update = connection.prepareStatement(
             "UPDATE metadata SET data=? WHERE id=?")
         ) {
             try (Statement statement = connection.createStatement();
-                 ResultSet resultSet = statement.executeQuery(
-                     "SELECT data,id,uuid FROM metadata WHERE isharvested = 'n'")
+                 ResultSet resultSet = statement.executeQuery(String.format(
+                     "SELECT data, id, uuid FROM metadata WHERE data LIKE '%%%s%%'",
+                     settingManager.getServerURL()))
             ) {
                 int numInBatch = 0;
-                final SettingManager settingManager = applicationContext.getBean(SettingManager.class);
 
                 while (resultSet.next()) {
                     final Element xml = Xml.loadString(resultSet.getString(1), false);

core/src/main/java/org/fao/geonet/api/records/attachments/FilesystemStore.java

@@ -192,6 +192,7 @@ public MetadataResource putResource(final ServiceContext context, final String m
                                         final InputStream is, @Nullable final Date changeDate, final MetadataResourceVisibility visibility,
                                         Boolean approved) throws Exception {
         int metadataId = canEdit(context, metadataUuid, approved);
+        checkResourceId(filename);
         Path filePath = getPath(context, metadataId, visibility, filename, approved);
         Files.copy(is, filePath, StandardCopyOption.REPLACE_EXISTING);
         if (changeDate != null) {

core/src/main/java/org/fao/geonet/constants/Geonet.java

@@ -648,7 +648,7 @@ public static class IndexFieldNames {
         public static final String GROUP_WEBSITE = "_groupWebsite";
         public static final String LOGO = "_logo";
         public static final String OP_PREFIX = "op";
-        public static final String GROUP_PUBLISHED = "_groupPublished";
+        public static final String GROUP_PUBLISHED = "groupPublished";
         public static final String CAT = "_cat";
         public static final String STATUS = "mdStatus";
         public static final String STATUS_CHANGE_DATE = "mdStatusChangeDate";
@@ -657,7 +657,7 @@ public static class IndexFieldNames {
         public static final String VALID_INSPIRE = "_valid_inspire";
         public static final String ANY = "any";
         public static final String LOCALE = "locale";
-        public static final String IS_PUBLISHED_TO_ALL = "_isPublishedToAll";
+        public static final String IS_PUBLISHED_TO_ALL = "isPublishedToAll";
         public static final String FEEDBACKCOUNT = "feedbackCount";
         public static final String DRAFT = "draft";
         public static final String RESOURCETITLE = "resourceTitle";

core/src/main/java/org/fao/geonet/kernel/AllThesaurus.java

@@ -276,7 +276,7 @@ public synchronized Thesaurus updateCodeByURI(String olduri, String newuri) thro
     }
 
     @Override
-    public void addTitleElement(String thesaurusTitle) throws IOException, AccessDeniedException, GraphException {
+    public void addTitleElement(String thesaurusTitle, String namespace) throws IOException, AccessDeniedException, GraphException {
         throw new UnsupportedOperationException();
     }
 

core/src/main/java/org/fao/geonet/kernel/BatchEditParameter.java

@@ -29,9 +29,6 @@
 import javax.xml.bind.annotation.XmlElement;
 import javax.xml.bind.annotation.XmlRootElement;
 
-/**
- * Created by francois on 22/10/15.
- */
 @XmlRootElement(name = "edit")
 public class BatchEditParameter implements Serializable {
     private String xpath;

core/src/main/java/org/fao/geonet/kernel/MetadataIndexerProcessor.java

@@ -53,7 +53,7 @@ public MetadataIndexerProcessor(DataManager dm) {
         this.dm = dm;
     }
 
-    public abstract void process() throws Exception;
+    public abstract void process(String catalogueId) throws Exception;
 
     protected DataManager getDataManager() {
         return dm;

core/src/main/java/org/fao/geonet/kernel/Thesaurus.java

@@ -681,7 +681,7 @@ private Thesaurus updateElementCode(Graph myGraph, URI oldobj, URI newobj) {
     /**
      * Set the title of the thesaurus and save the graph to the repository.
      */
-    public void addTitleElement(String thesaurusTitle) throws IOException, AccessDeniedException, GraphException {
+    public void addTitleElement(String thesaurusTitle, String namespace) throws IOException, AccessDeniedException, GraphException {
 
         Graph myGraph = new org.openrdf.model.impl.GraphImpl();
 
@@ -690,7 +690,7 @@ public void addTitleElement(String thesaurusTitle) throws IOException, AccessDen
         String namespaceSkos = "http://www.w3.org/2004/02/skos/core#";
         String namespaceDC = "http://purl.org/dc/elements/1.1/";
 
-        URI mySubject = myFactory.createURI("http://geonetwork-opensource.org/", thesaurusTitle);
+        URI mySubject = myFactory.createURI(namespace);
         URI skosClass = myFactory.createURI(namespaceSkos, "ConceptScheme");
         URI titleURI = myFactory.createURI(namespaceDC, "title");
 

core/src/main/java/org/fao/geonet/kernel/ThesaurusManager.java

@@ -289,9 +289,21 @@ public void addThesaurus(Thesaurus gst, boolean writeTitle) throws Exception {
         thesauriMap.put(thesaurusName, gst);
 
         if (writeTitle) {
-            gst.addTitleElement(gst.getTitle());
+            gst.addTitleElement(gst.getTitle(), gst.getDefaultNamespace());
         }
+    }
+
+    public void addOrReloadThesaurus(Thesaurus gst) throws Exception {
+        if (thesauriMap.replace(gst.getKey(), gst) != null) {
+            service.removeRepository(gst.getKey());
+        }
+
+        createThesaurusRepository(gst);
+        thesauriMap.put(gst.getKey(), gst);
 
+        if (Log.isDebugEnabled(Geonet.THESAURUS_MAN)) {
+            Log.debug(Geonet.THESAURUS_MAN, "Thesaurus " + gst.getKey() + " loaded.");
+        }
     }
 
     /**
@@ -403,17 +415,7 @@ public String createUpdateThesaurusFromRegister(String uuid, String type, Servic
         String theKey = gst.getKey();
         gst.retrieveThesaurusTitle();
 
-        if (thesauriMap.replace(theKey, gst) == null) {
-            createThesaurusRepository(gst);
-            thesauriMap.put(theKey, gst);
-            if (Log.isDebugEnabled(Geonet.THESAURUS_MAN))
-                Log.debug(Geonet.THESAURUS_MAN, "Created thesaurus " + theKey + " from register " + uuid);
-        } else {
-            service.removeRepository(theKey);
-            createThesaurusRepository(gst);
-            if (Log.isDebugEnabled(Geonet.THESAURUS_MAN))
-                Log.debug(Geonet.THESAURUS_MAN, "Rebuilt thesaurus " + theKey + " from register " + uuid);
-        }
+        addOrReloadThesaurus(gst);
 
         return theKey;
     }

core/src/main/java/org/fao/geonet/kernel/ThesaurusType.java

@@ -0,0 +1,28 @@
+//===	Copyright (C) 2001-2021 Food and Agriculture Organization of the
+//===	United Nations (FAO-UN), United Nations World Food Programme (WFP)
+//===	and United Nations Environment Programme (UNEP)
+//===
+//===	This program is free software; you can redistribute it and/or modify
+//===	it under the terms of the GNU General Public License as published by
+//===	the Free Software Foundation; either version 2 of the License, or (at
+//===	your option) any later version.
+//===
+//===	This program is distributed in the hope that it will be useful, but
+//===	WITHOUT ANY WARRANTY; without even the implied warranty of
+//===	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//===	General Public License for more details.
+//===
+//===	You should have received a copy of the GNU General Public License
+//===	along with this program; if not, write to the Free Software
+//===	Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+//===
+//===	Contact: Jeroen Ticheler - FAO - Viale delle Terme di Caracalla 2,
+//===	Rome - Italy. email: geonetwork@osgeo.org
+//==============================================================================
+
+package org.fao.geonet.kernel;
+
+public enum ThesaurusType {
+    external,
+    local
+}

core/src/main/java/org/fao/geonet/kernel/datamanager/IMetadataStatus.java

@@ -47,6 +47,8 @@ public interface IMetadataStatus {
      */
     boolean isUserMetadataStatus(int userId) throws Exception;
 
+    void transferMetadataStatusOwnership(int oldUserId, int newUserId) throws Exception;
+
     /**
      * If groupOwner match regular expression defined in setting metadata/workflow/draftWhenInGroup, then set status to draft to enable
      * workflow.

core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataIndexer.java

@@ -568,6 +568,7 @@ private Multimap<String, Object> buildFieldsForPrivileges(int recordId) {
                 Optional<Group> g = groupRepository.findById(groupId);
                 if (g.isPresent()) {
                     privilegesFields.put(Geonet.IndexFieldNames.GROUP_PUBLISHED, g.get().getName());
+                    privilegesFields.put(Geonet.IndexFieldNames.GROUP_PUBLISHED + "Id", g.get().getId());
 
 
                     if (g.get().getId() == ReservedGroup.all.getId()) {

core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java

@@ -238,7 +238,7 @@ public void synchronizeDbWithIndex(ServiceContext context, Boolean force, Boolea
                 Set<Integer> integerList = toIndex.stream().map(Integer::parseInt).collect(Collectors.toSet());
                 new BatchOpsMetadataReindexer(
                     context.getBean(DataManager.class),
-                    integerList).process(false);
+                    integerList).process(settingManager.getSiteId(), false);
             } else {
                 metadataIndexer.batchIndexInThreadPool(context, toIndex);
             }
@@ -726,15 +726,17 @@ public synchronized AbstractMetadata updateMetadata(final ServiceContext context
             session.removeProperty(Geonet.Session.VALIDATION_REPORT + metadataId);
         }
         String schema = metadataSchemaUtils.getMetadataSchema(metadataId);
+
+        String uuidBeforeUfo = null;
         if (ufo) {
             String parentUuid = null;
             Integer intId = Integer.valueOf(metadataId);
 
             final AbstractMetadata metadata = metadataUtils.findOne(metadataId);
 
-            String uuid = findUuid(metadataXml, schema, metadata);
+            uuidBeforeUfo = findUuid(metadataXml, schema, metadata);
 
-            metadataXml = updateFixedInfo(schema, Optional.of(intId), uuid, metadataXml, parentUuid,
+            metadataXml = updateFixedInfo(schema, Optional.of(intId), uuidBeforeUfo, metadataXml, parentUuid,
                 (updateDateStamp ? UpdateDatestamp.YES : UpdateDatestamp.NO), context);
         }
 
@@ -758,7 +760,10 @@ public synchronized AbstractMetadata updateMetadata(final ServiceContext context
             }
         } finally {
             if (index) {
-                // --- update search criteria
+                // Delete old record if UUID changed
+                if (uuidBeforeUfo != null && !uuidBeforeUfo.equals(uuid)) {
+                    getSearchManager().delete(String.format("+uuid:\"%s\"", uuidBeforeUfo));
+                }
                 metadataIndexer.indexMetadata(metadataId, true);
             }
         }

core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataStatus.java

@@ -71,6 +71,15 @@ public boolean isUserMetadataStatus(int userId) throws Exception {
         return metadataStatusRepository.count(MetadataStatusSpecs.hasUserId(userId)) > 0;
     }
 
+    @Override
+    public void transferMetadataStatusOwnership(int oldUserId, int newUserId) throws Exception {
+        List<MetadataStatus> oldUserStatus = metadataStatusRepository.findAll(MetadataStatusSpecs.hasUserId(oldUserId));
+        oldUserStatus.stream().forEach(s -> {
+            s.setUserId(newUserId);
+        });
+        metadataStatusRepository.saveAll(oldUserStatus);
+    }
+
     /**
      * Return last workflow status for the metadata id
      */

core/src/main/java/org/fao/geonet/kernel/mef/MEF2Exporter.java

@@ -230,9 +230,6 @@ private static void createMetadataFolder(ServiceContext context,
         String id = "" + record.getId();
         String isTemp = record.getDataInfo().getType().codeString;
 
-        if (!"y".equals(isTemp) && !"n".equals(isTemp))
-            throw new Exception("Cannot export sub template");
-
         final Path metadataXmlDir = metadataRootDir.resolve(MD_DIR);
         Files.createDirectories(metadataXmlDir);