Merge pull request #615 from fxprunayre/improvement/schema-plugin-bean

Schema plugin / Add schema bean to make it more plugable

.gitignore

@@ -5,6 +5,10 @@ rebel.xml
 harvesters/harvester_*.log
 web/src/main/webapp/WEB-INF/data/config/schema_plugins/*/schematron/schematron*.xsl
 web/src/main/webapp/WEB-INF/data/config/schemaplugin-uri-catalog.xml
+web/src/main/webapp/WEB-INF/data/config/schema_plugins/csw-record
+web/src/main/webapp/WEB-INF/data/config/schema_plugins/dublin-core
+web/src/main/webapp/WEB-INF/data/config/schema_plugins/fgdc-std
+web/src/main/webapp/WEB-INF/data/config/schema_plugins/iso19*
 web/src/main/webapp/scripts/lib/*
 web/src/main/webapp/images/logos/*
 web/src/main/webapp/images/statTmp
@@ -16,6 +20,7 @@ e2e-tests/*.db
 changes-*
 web/coverage.ec
 */target/
+*/*/target/
 */.externalToolBuilders
 web/src/main/webapp/data/
 web/src/main/webapp/WEB-INF/lucene/

common/src/main/java/org/fao/geonet/utils/Xml.java

@@ -41,6 +41,7 @@
 import org.jdom.JDOMException;
 import org.jdom.Namespace;
 import org.jdom.Text;
+import org.jdom.filter.ElementFilter;
 import org.jdom.input.SAXBuilder;
 import org.jdom.output.Format;
 import org.jdom.output.SAXOutputter;
@@ -75,10 +76,7 @@
 import java.nio.charset.Charset;
 import java.nio.charset.CharsetDecoder;
 import java.nio.charset.CharsetEncoder;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
+import java.util.*;
 import javax.xml.XMLConstants;
 import javax.xml.transform.Result;
 import javax.xml.transform.Source;
@@ -976,6 +974,42 @@ public static Number selectNumber(Element xml, String xpath, List<Namespace> the
 		return xp.numberValueOf(xml);
 	}
 
+    /**
+     * Search in metadata all matching element for the filter
+     * and return a list of uuid separated by or to be used in a
+     * search on uuid. Extract uuid from matched element if
+     * elementName is null or from the elementName child.
+     *
+     * @param element
+     * @param elementFilter Filter to get element descendants
+     * @param elementName   Child element to get value from. If null, filtered element value is returned
+     * @param elementNamespace
+     * @param attributeName Attribute name to get value from. If null, TODO: improve
+     * @return
+     */
+    public static Set<String> filterElementValues(Element element,
+                                                     ElementFilter elementFilter,
+                                                     String elementName,
+                                                     Namespace elementNamespace,
+                                                     String attributeName) {
+        @SuppressWarnings("unchecked")
+        Iterator<Element> i = element.getDescendants(elementFilter);
+        Set<String> values = new HashSet<String>();
+        boolean first = true;
+        while (i.hasNext()) {
+            Element e = i.next();
+            String uuid = elementName == null && attributeName == null?
+                    e.getText() :
+                    (attributeName == null ?
+                            e.getChildText(elementName, elementNamespace) :
+                            e.getAttributeValue(attributeName)
+                    );
+            if (!"".equals(uuid)) {
+                values.add(uuid);
+            }
+        }
+        return values;
+    }
 	//---------------------------------------------------------------------------
 
     /**

core/pom.xml

@@ -274,6 +274,11 @@
             <artifactId>cachingxslt</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+          <groupId>${project.groupId}</groupId>
+          <artifactId>schema-iso19139</artifactId>
+          <version>${project.version}</version>
+        </dependency>
         <dependency> <!-- dummy API for ARC SDE stuff -->
             <groupId>${project.groupId}</groupId>
             <artifactId>dummy-api</artifactId>

core/src/main/java/org/fao/geonet/kernel/SchemaManager.java

@@ -27,15 +27,14 @@
 
 package org.fao.geonet.kernel;
 
-import jeeves.server.ServiceConfig;
-
 import org.fao.geonet.Constants;
 import org.fao.geonet.exceptions.OperationAbortedEx;
 
 import jeeves.server.context.ServiceContext;
 import jeeves.server.dispatchers.guiservices.XmlFile;
 import jeeves.server.overrides.ConfigurationOverrides;
 
+import org.fao.geonet.kernel.schema.SchemaPlugin;
 import org.fao.geonet.repository.SchematronCriteriaGroupRepository;
 import org.fao.geonet.repository.SchematronRepository;
 import org.fao.geonet.utils.BinaryFile;
@@ -59,7 +58,6 @@
 import org.jdom.Namespace;
 import org.jdom.filter.ElementFilter;
 import org.springframework.context.ApplicationContext;
-import org.springframework.stereotype.Component;
 
 import java.io.BufferedOutputStream;
 import java.io.File;
@@ -179,6 +177,7 @@ public void configure(ApplicationContext applicationContext, String basePath, St
                 if (!saSchemas[i].equals("CVS") && !saSchemas[i].startsWith(".")) {
                     File schemaDir = new File(this.schemaPluginsDir + FS + saSchemas[i]);
                     if (schemaDir.isDirectory()) {
+                        Log.info(Geonet.SCHEMA_MANAGER, "Loading schema " + saSchemas[i] + "...");
                         processSchema(applicationContext, schemaPluginsDir + FS, saSchemas[i], schemaPluginCatRoot);
                     }
                 }
@@ -229,6 +228,33 @@ public MetadataSchema getSchema(String name) {
 		}
 	}
 
+    public static SchemaPlugin getSchemaPlugin(ServiceContext context, String schemaIdentifier) {
+        String schemaBeanIdentifier = schemaIdentifier + "SchemaPlugin";
+        SchemaPlugin schemaPlugin = null;
+        try {
+            schemaPlugin = (SchemaPlugin) context.getApplicationContext().getBean(schemaBeanIdentifier);
+
+            String iso19139SchemaIdentifier = "iso19139";
+            if (schemaPlugin == null && schemaIdentifier.startsWith(iso19139SchemaIdentifier)){
+                // For ISO19139 profiles, get the ISO19139 bean if no custom one defined
+                // Can't depend here on ISO19139SchemaPlugin to avoid to introduce
+                // circular ref.
+                schemaBeanIdentifier = iso19139SchemaIdentifier + "SchemaPlugin";
+                schemaPlugin = (SchemaPlugin) context.getApplicationContext().getBean(schemaBeanIdentifier);
+            }
+
+
+        } catch (Exception e) {
+            // No bean for this schema
+            if (Log.isDebugEnabled(Geonet.SCHEMA_MANAGER)) {
+                Log.debug(Geonet.SCHEMA_MANAGER, "No bean defined for the schema plugin '" +
+                        schemaIdentifier + "'. " +
+                        e.getMessage());
+            }
+        }
+        return schemaPlugin;
+    }
+
 	/**
      * Return the Id and Version of the schema
      *

core/src/main/java/org/fao/geonet/kernel/schema/SchemaLoader.java

@@ -649,6 +649,7 @@ private List<ElementInfo> loadFile(String xmlSchemaFile, HashSet<String> loadedF
 		String path = new File(xmlSchemaFile).getParent() + "/";
 
 		//--- load xml-schema
+        Log.debug(Geonet.SCHEMA_MANAGER, "Loading schema " + xmlSchemaFile);
 
         Element elRoot = Xml.loadFile(xmlSchemaFile);
 		if (elFirst == null) elFirst = elRoot;
@@ -810,19 +811,29 @@ private void buildGlobalElement(ElementInfo ei)
 			}
 			al.add(ee);
 
-			if (hmSubsLink.get(ee.name) != null) {
-				throw new IllegalArgumentException("Substitution link collision for : "+ee.name+" link to "+ee.substGroup);
+            String existingSubstitionGroup = hmSubsLink.get(ee.name);
+			if (existingSubstitionGroup != null
+                    && !ee.substGroup.equals(existingSubstitionGroup)) {
+				throw new IllegalArgumentException("Substitution link collision" +
+                        " for " + ee.name +
+                        " link to " + existingSubstitionGroup +
+                        ". Already bound to " + ee.substGroup);
 			} else {
-				hmSubsLink.put(ee.name,ee.substGroup);
+				hmSubsLink.put(ee.name, ee.substGroup);
 			}
 		}
 		if (ee.abstrElem)
 		{
-			if (hmAbsElems.containsKey(ee.name))
-				throw new IllegalArgumentException("Namespace collision for : " + ee.name);
-			hmAbsElems.put(ee.name, ee.type);
-
 
+            String existingType = hmAbsElems.get(ee.name);
+			if (existingType != null && !ee.type.equals(existingType)) {
+				throw new IllegalArgumentException("Namespace collision" +
+                        " for " + ee.name +
+                        " type " + existingType +
+                        ". Already bound to " + ee.type);
+            } else {
+			    hmAbsElems.put(ee.name, ee.type);
+            }
 			return;
 		}
 		if (ee.complexType != null)
@@ -865,11 +876,15 @@ else if (ee.simpleType != null)
 	private void buildComplexType(ElementInfo ei)
 	{
 		ComplexTypeEntry ct = new ComplexTypeEntry(ei);
-		if (hmTypes.containsKey(ct.name))
-			throw new IllegalArgumentException("Namespace collision for : " + ct.name);
 
-		hmTypes.put(ct.name, ct);
-	}
+        ComplexTypeEntry existingType = hmTypes.get(ct.name);
+        if (existingType != null && !ct.name.equals(existingType.name)) {
+            throw new IllegalArgumentException("Namespace collision" +
+                    " for complex type " + ct.name +
+                    " type " + existingType.name + "already defined.");
+        }
+        hmTypes.put(ct.name, ct);
+    }
 
 	//---------------------------------------------------------------------------
 

core/src/test/java/org/fao/geonet/AbstractCoreIntegrationTest.java

@@ -54,10 +54,7 @@
 import java.lang.reflect.Constructor;
 import java.net.URL;
 import java.sql.Connection;
-import java.util.ArrayList;
-import java.util.Calendar;
-import java.util.HashMap;
-import java.util.List;
+import java.util.*;
 import java.util.concurrent.TimeUnit;
 
 import static org.junit.Assert.assertTrue;
@@ -119,7 +116,8 @@ public void configureAppContext() throws Exception {
 
         final GeonetworkDataDirectory geonetworkDataDirectory = _applicationContext.getBean(GeonetworkDataDirectory.class);
 
-        final SyncReport syncReport = synchronizeDataDirectory(new File(webappDir, "WEB-INF/data"));
+        final SyncReport syncReport = synchronizeDataDirectory(
+                new File(webappDir, "WEB-INF/data"));
 
         final ArrayList<Element> params = getServiceConfigParameterElements();
 
@@ -152,12 +150,15 @@ public void configureAppContext() throws Exception {
         _directoryFactory.resetIndex();
 
         final String schemaPluginsDir = geonetworkDataDirectory.getSchemaPluginsDir().getPath();
+
         final String resourcePath = geonetworkDataDirectory.getResourcesDir().getPath();
 
         final SchemaManager schemaManager = _applicationContext.getBean(SchemaManager.class);
         if (syncReport.updateSchemaManager || !schemaManager.existsSchema("iso19139")) {
+
             new File(_dataDirectory, "config/schemaplugin-uri-catalog.xml").delete();
             final String schemaPluginsCatalogFile = new File(schemaPluginsDir, "/schemaplugin-uri-catalog.xml").getPath();
+            deploySchema(webappDir, schemaPluginsDir);
 
             _applicationContext.getBean(LuceneConfig.class).configure("WEB-INF/config-lucene.xml");
             SchemaManager.registerXmlCatalogFiles(webappDir, schemaPluginsCatalogFile);
@@ -221,7 +222,6 @@ private void setUpDataDirectory() {
 
             _dataDirContainer = new File(dir.getPath()+i);
 
-
             _dataDirectory = new File(_dataDirContainer, "defaultDataDir");
             _dataDirLockFile = new File(_dataDirContainer, DATA_DIR_LOCK_NAME);
         }
@@ -248,14 +248,20 @@ private SyncReport synchronizeDataDirectory(File srcDataDir) throws IOException
                 String relativePath = dataDirFile.getPath().substring(prefixPathLength2);
                 final File srcFile = new File(srcDataDir, relativePath);
                 if (!srcFile.exists()) {
-                    if (srcFile.getParent().endsWith("schematron") && relativePath.contains("schema_plugins") && relativePath.endsWith(".xsl")) {
+                    if (srcFile.getParent().endsWith("schematron") &&
+                            relativePath.contains("schema_plugins") &&
+                            relativePath.endsWith(".xsl")) {
                         // don't copy because the schematron xsl files are generated.
                         // normally they shouldn't be here because they don't need to be in the
-                        // repository but some tests can generate them into the schemtrons folder
+                        // repository but some tests can generate them into the schematrons folder
                         // so ignore them here.
                         continue;
                     }
 
+                    if (relativePath.contains("/removed/")) {
+                        // Ignore removed files which may contains MEF files
+                        continue;
+                    }
                     if (relativePath.endsWith("schemaplugin-uri-catalog.xml")) {
                         // we will handle this special case later.
                         continue;
@@ -307,10 +313,27 @@ private SyncReport synchronizeDataDirectory(File srcDataDir) throws IOException
                 report.updateSchemaManager |= relativePath.contains("schema_plugins");
             }
         }
-
         return report;
     }
 
+    private void deploySchema(String srcDataDir, String schemaPluginPath) {
+        // Copy schema plugin
+        final String schemaModulePath = "schemas";
+        File schemaModuleDir = new File(srcDataDir + "/../../../../" + schemaModulePath);
+        if (schemaModuleDir.exists()) {
+            String[] listOfSchemaToLoad = {"iso19139", "dublin-core", "iso19115", "fgdc-std"};
+            for (String schema : listOfSchemaToLoad) {
+                String srcPath = schemaModuleDir + "/" + schema + "/src/main/plugin/" + schema;
+                String destPath = schemaPluginPath + "/" + schema;
+                try {
+                    BinaryFile.copyDirectory(new File(srcPath), new File(destPath));
+                } catch (IOException e) {
+                    e.printStackTrace();
+                }
+            }
+        }
+    }
+
     @After
     public void deleteNonDefaultNodeDataDirectories() throws IOException {
         synchronized (AbstractCoreIntegrationTest.class) {

jeeves/src/main/java/jeeves/server/JeevesEngine.java

@@ -460,7 +460,8 @@ private void initAppHandler(Element handler, JeevesServlet servlet) throws Excep
 				info("--- Handler started ---------------------------------------");
 			} catch (Exception e) {
                 Map<String, String> errors = new HashMap<String, String>();
-                String eS = "Raised exception while starting appl handler. Skipped.";
+                String eS = "Raised exception while starting the application. " +
+                        "Fix the error and restart.";
                 error(eS);
                 errors.put("Error", eS);
                 error("   Handler   : " + className);

pom.xml

@@ -1112,6 +1112,7 @@
     <module>domain</module>
     <module>oaipmh</module>
     <module>core</module>
+    <module>schemas</module>
     <module>csw-server</module>
     <module>harvesters</module>
     <module>healthmonitor</module>

schemas/README.md

@@ -0,0 +1,17 @@
+# Maven module containing schema plugins
+
+## Plugin structure
+
+A schema plugin is composed off:
+
+* A configuration folder containing XML file for configuration,
+the schema as XSDs and a set of transformation (See http://geonetwork-opensource.org/manuals/trunk/eng/developer/schemaPlugins/index.html).
+
+* (optional) A SchemaPlugin bean
+
+
+## Add a plugin to the build
+
+To include a new schema plugin in a build, copy the schema folder
+here and add it to the copy-schemas execution in web/pom.xml.
+

schemas/csw-record/pom.xml

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <parent>
+    <artifactId>schemas</artifactId>
+    <groupId>org.geonetwork-opensource</groupId>
+    <version>2.11.0-SNAPSHOT</version>
+  </parent>
+  <modelVersion>4.0.0</modelVersion>
+  <artifactId>schema-csw-record</artifactId>
+  <name>GeoNetwork schema plugin for Dublin Core records retrieved by CSW</name>
+</project>