Admin / Languages and translations

domain/src/main/java/org/fao/geonet/domain/Language.java

@@ -114,7 +114,7 @@ protected void setInspire_JPAWorkaround(char isinspire) {
     }
 
     /**
-     * Return true if this is a language required byt the inspire standards.
+     * Return true if this is a language required by the inspire standards.
      *
      * @return return true if required by inspire.
      */

es/README.md

@@ -81,24 +81,23 @@ curl -X DELETE http://localhost:9200/$IDX_PREFIX-features
 curl -X DELETE http://localhost:9200/$IDX_PREFIX-searchlogs
 ```
 
-## Multilingual configuration (beta)
+## Multilingual configuration
 
-* Stop Elasticsearch
-* Define which analyzer to use https://www.elastic.co/guide/en/elasticsearch/reference/current/analysis-analyzers.html depending on the language(s) used in your catalogue
-* Build the application with the default analyzer to use (or configure it in `data/index/records.json`)
+Default index is configured with analyzer for the following languages:
+* English
+* French
+* German
+* Italian
+
+To add a new language, update the index schema in `datadir/config/index/records.json` and update the containing fields starting with `lang`.
 
-  ```shell script
-  mvn clean install -Des.default.analyzer=french_heavy
-  ```
+First create a full text search field for the new language in the `any` object field eg. `any.langfre` and define the proper analyzer.
 
-* Install ICU plugin (if needed)
+Then add the new language like the others.
 
-  ```shell script
-  elasticsearch-plugin install analysis-icu
-  ```
+From the admin console > tools, Delete index and reindex.
 
-* Start Elasticsearch
-* Drop and rebuild your index
+Don't hesitate to propose a Pull Request with the new language.
 
 
 # Production use

schemas/iso19115-3.2018/src/main/plugin/iso19115-3.2018/index-fields/index.xsl

@@ -385,6 +385,8 @@
 
 
         <xsl:copy-of select="gn-fn-index:add-multilingual-field('resourceCredit', mri:credit[* != ''], $allLanguages)"/>
+        <xsl:copy-of select="gn-fn-index:add-multilingual-field('supplementalInformation', mri:supplementalInformation[* != ''], $allLanguages)"/>
+        <xsl:copy-of select="gn-fn-index:add-multilingual-field('purpose', mri:purpose[* != ''], $allLanguages)"/>
 
         <xsl:variable name="overviews"
                       select="mri:graphicOverview/mcc:MD_BrowseGraphic/

schemas/iso19139/src/main/plugin/iso19139/index-fields/index-subtemplate.xsl

@@ -99,10 +99,9 @@
 
     <xsl:copy-of select="gn-fn-index:add-field('Org', $org)"/>
 
-    <any><xsl:value-of select="$title"/></any>
-    <any><xsl:value-of select="$org"/></any>
-    <any><xsl:value-of select="$name"/></any>
-    <any><xsl:value-of select="$mail"/></any>
+    <any type="object">{"common": "<xsl:value-of
+      select="gn-fn-index:json-escape(normalize-space(.))"/>"}</any>
+
     <xsl:for-each
       select="gmd:contactInfo/*/gmd:address/*/gmd:electronicMailAddress/gco:CharacterString">
       <xsl:copy-of select="gn-fn-index:add-field('email', .)"/>

schemas/iso19139/src/main/plugin/iso19139/index-fields/index.xsl

@@ -362,7 +362,8 @@
         </xsl:apply-templates>
 
         <xsl:copy-of select="gn-fn-index:add-multilingual-field('resourceCredit', gmd:credit, $allLanguages)"/>
-
+        <xsl:copy-of select="gn-fn-index:add-multilingual-field('supplementalInformation', gmd:supplementalInformation, $allLanguages)"/>
+        <xsl:copy-of select="gn-fn-index:add-multilingual-field('purpose', gmd:purpose, $allLanguages)"/>
 
         <xsl:variable name="overviews"
                       select="gmd:graphicOverview/gmd:MD_BrowseGraphic/

services/src/main/java/org/fao/geonet/api/languages/LanguagesApi.java

@@ -28,8 +28,10 @@
 import io.swagger.v3.oas.annotations.responses.ApiResponses;
 import io.swagger.v3.oas.annotations.tags.Tag;
 import jeeves.server.context.ServiceContext;
+import org.fao.geonet.ApplicationContextHolder;
 import org.fao.geonet.api.ApiParams;
 import org.fao.geonet.api.ApiUtils;
+import org.fao.geonet.api.exception.NotAllowedException;
 import org.fao.geonet.api.exception.ResourceNotFoundException;
 import org.fao.geonet.domain.Language;
 import org.fao.geonet.kernel.GeonetworkDataDirectory;
@@ -41,14 +43,15 @@
 import org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.web.bind.annotation.*;
 
+import javax.annotation.Resource;
 import javax.servlet.http.HttpServletRequest;
 import java.io.BufferedReader;
 import java.io.FileReader;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.List;
+import java.util.*;
+import java.util.stream.Collectors;
 
 @RequestMapping(value = {
     "/{portal}/api/languages"
@@ -64,6 +67,37 @@ public class LanguagesApi {
     @Autowired
     private GeonetworkDataDirectory dataDirectory;
 
+    private String defaultLanguage;
+
+    @Resource(name="defaultLanguage")
+    public void setDefaultLanguage(String defaultLanguage) {
+        this.defaultLanguage = defaultLanguage;
+    }
+
+    @io.swagger.v3.oas.annotations.Operation(
+        summary = "Get languages available in the application",
+        description = "Languages available in this version of the application. Those that you can add using PUT operation and which have SQL script to initialize the language.")
+    @RequestMapping(
+        value = "/application",
+        produces = MediaType.APPLICATION_JSON_VALUE,
+        method = RequestMethod.GET)
+    @ResponseStatus(value = HttpStatus.OK)
+    @PreAuthorize("hasAuthority('Administrator')")
+    @ResponseBody
+    public List<Language> getApplicationLanguages() throws Exception {
+        Set<String> applicationLanguages =
+            (Set<String>) ApplicationContextHolder.get().getBean("languages");
+        List<Language> list = applicationLanguages.stream().map(l -> {
+            Language language = new Language();
+            language.setId(l);
+            if (l.equals(defaultLanguage)) {
+                language.setDefaultLanguage(true);
+            }
+            return language;
+        }).collect(Collectors.toList());
+        return list;
+    }
+
 
     @io.swagger.v3.oas.annotations.Operation(
         summary = "Get languages",
@@ -109,8 +143,8 @@ public void addLanguages(
             HttpServletRequest request
     ) throws IOException, ResourceNotFoundException {
 
-        Language lang = languageRepository.findById(langCode).get();
-        if (lang == null) {
+        Optional<Language> lang = languageRepository.findById(langCode);
+        if (!lang.isPresent()) {
             String languageDataFile = "loc-" + langCode + "-default.sql";
             Path templateFile = dataDirectory.getWebappDir().resolve("WEB-INF")
                 .resolve("classes").resolve("setup").resolve("sql").resolve("data")
@@ -134,7 +168,7 @@ public void addLanguages(
             ));
         } else {
             throw new RuntimeException(String.format(
-                "Language '%s' already available.", lang.getId()
+                "Language '%s' already available.", lang.get().getId()
             ));
         }
     }
@@ -166,12 +200,19 @@ public void deleteLanguage(
             String langCode,
         HttpServletRequest request
     ) throws IOException, ResourceNotFoundException {
-        Language lang = languageRepository.findById(langCode).get();
-        if (lang == null) {
+        Optional<Language> lang = languageRepository.findById(langCode);
+        if (!lang.isPresent()) {
             throw new ResourceNotFoundException(String.format(
                 "Language '%s' not found.", langCode
             ));
         } else {
+            long count = languageRepository.count();
+            if (count == 1) {
+                throw new NotAllowedException(String.format(
+                    "You can't delete the last language. Add another one before removing %s.",
+                    langCode
+                ));
+            }
             final String LANGUAGE_DELETE_SQL = "language-delete.sql";
 
             Path templateFile = dataDirectory.getWebappDir().resolve("WEB-INF")
@@ -182,7 +223,7 @@ public void deleteLanguage(
                 try (BufferedReader br = new BufferedReader(new FileReader(templateFile.toFile()))) {
                     String line;
                     while ((line = br.readLine()) != null) {
-                        data.add(String.format(line, lang.getId()));
+                        data.add(String.format(line, lang.get().getId()));
                     }
                 }
                 if (data.size() > 0) {

services/src/main/java/org/fao/geonet/api/tools/i18n/TranslationApi.java

@@ -26,8 +26,6 @@
 import io.swagger.v3.oas.annotations.Parameter;
 import io.swagger.v3.oas.annotations.tags.Tag;
 import jeeves.server.context.ServiceContext;
-import org.apache.commons.lang.StringUtils;
-import org.fao.geonet.ApplicationContextHolder;
 import org.fao.geonet.api.ApiUtils;
 import org.fao.geonet.api.exception.ResourceNotFoundException;
 import org.fao.geonet.domain.*;
@@ -36,7 +34,6 @@
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.ApplicationContext;
 import org.springframework.context.ApplicationContextAware;
-import org.springframework.context.ConfigurableApplicationContext;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.MediaType;
 import org.springframework.http.ResponseEntity;
@@ -46,7 +43,9 @@
 import javax.servlet.ServletRequest;
 import javax.servlet.http.HttpServletRequest;
 import java.util.*;
+import java.util.stream.Collectors;
 
+import static java.util.stream.Collectors.groupingBy;
 import static org.springframework.http.HttpStatus.CREATED;
 import static org.springframework.http.HttpStatus.OK;
 
@@ -75,15 +74,96 @@ public synchronized void setApplicationContext(ApplicationContext context) {
         this.context = context;
     }
 
+    /**
+     * @param type The type of object to return.
+     * @return A map of translations in JSON format.
+     */
+    @io.swagger.v3.oas.annotations.Operation(
+        summary = "List translations for database description table")
+    @RequestMapping(value = "/db",
+        method = RequestMethod.GET,
+        produces = {
+            MediaType.APPLICATION_JSON_VALUE
+        })
+    @ResponseBody
+    public Map<String, String> getTranslations(
+        @RequestParam(required = false) final List<String> type,
+        ServletRequest request
+    ) throws Exception {
+        Locale locale = languageUtils.parseAcceptLanguage(request.getLocales());
+        String language = languageUtils.locale2gnCode(locale.getISO3Language());
+        return translationPackBuilder.getDbTranslation(language, type);
+    }
+
+    @io.swagger.v3.oas.annotations.Operation(summary = "List custom user translations")
+    @RequestMapping(value = "/db/custom",
+        method = RequestMethod.GET,
+        produces = {
+            MediaType.APPLICATION_JSON_VALUE
+        })
+    @ResponseBody
+    public List<Translations> getCustomTranslations(
+        @RequestParam(required = false) final List<String> type,
+        ServletRequest request
+    ) throws Exception {
+        return translationsRepository.findAll();
+    }
+
     @io.swagger.v3.oas.annotations.Operation(
-        summary = "Add or update database translations.")
-    @PutMapping(value = "/db/translations/{key}",
+        summary = "Add or update all database translations.")
+    @PutMapping(value = "/db/translations",
         produces = {
             MediaType.APPLICATION_JSON_VALUE
         })
     @PreAuthorize("hasAuthority('Administrator')")
     @ResponseStatus(CREATED)
     public ResponseEntity addTranslations(
+        @Parameter(
+            name = "values"
+        )
+        @RequestBody(required = true)
+        final List<Translations> values,
+        @RequestParam(required = false)
+        final boolean replace
+    ) throws Exception {
+        if (replace) {
+            translationsRepository.deleteAll();
+        }
+
+        Map<String, Map<String, String>> translations = values.stream()
+            .filter(Objects::nonNull)
+            .collect(groupingBy(Translations::getFieldName,
+                Collectors.toMap(Translations::getLangId, Translations::getValue)));
+
+        translations.forEach((key, t) -> {
+            updateTranslation(key, t);
+        });
+        translationPackBuilder.clearCache();
+        return new ResponseEntity(HttpStatus.CREATED);
+    }
+
+    @io.swagger.v3.oas.annotations.Operation(
+        summary = "Remove all database translations.")
+    @DeleteMapping(value = "/db/translations",
+        produces = {
+            MediaType.APPLICATION_JSON_VALUE
+        })
+    @PreAuthorize("hasAuthority('Administrator')")
+    @ResponseStatus(CREATED)
+    public void removeAllTranslations() throws Exception {
+        translationsRepository.deleteAll();
+        translationPackBuilder.clearCache();
+    }
+
+    @io.swagger.v3.oas.annotations.Operation(
+        summary = "Add or update database translations for a key.")
+    @PutMapping(value = "/db/translations/{key:.+}",
+        produces = {
+            MediaType.APPLICATION_JSON_VALUE
+        })
+    @PreAuthorize("hasAuthority('Administrator')")
+    @ResponseStatus(CREATED)
+    public ResponseEntity addTranslationsFor(
         @PathVariable
         final String key,
         @Parameter(
@@ -100,6 +180,13 @@ public ResponseEntity addTranslations(
                 translationsRepository.findAllByFieldName(key)
             );
         }
+        updateTranslation(key, values);
+        translationPackBuilder.clearCache();
+        return new ResponseEntity(HttpStatus.CREATED);
+    }
+
+
+    private void updateTranslation(String key, Map<String, String> values) {
         List<Translations> translations = translationsRepository.findAllByFieldName(key);
         if(translations.size() == 0) {
             values.forEach((l, v) -> {
@@ -117,12 +204,13 @@ public ResponseEntity addTranslations(
             });
             translationsRepository.saveAll(translations);
         }
-        return new ResponseEntity(HttpStatus.CREATED);
+        translationPackBuilder.clearCache();
     }
 
+
     @io.swagger.v3.oas.annotations.Operation(
         summary = "Delete database translations.")
-    @DeleteMapping(value = "/db/translations/{key}",
+    @DeleteMapping(value = "/db/translations/{key:.+}",
         produces = {
             MediaType.APPLICATION_JSON_VALUE
         })
@@ -142,6 +230,7 @@ public void deleteTranslations(
                 key, language));
         } else {
             translationsRepository.deleteInBatch(translations);
+            translationPackBuilder.clearCache();
         }
     }
 
@@ -163,27 +252,6 @@ public Map<String, String> getDbTranslations(
     }
 
 
-    /**
-     * @param type The type of object to return.
-     * @return A map of translations in JSON format.
-     */
-    @io.swagger.v3.oas.annotations.Operation(summary = "List translations for database description table")
-    @RequestMapping(value = "/db",
-        method = RequestMethod.GET,
-        produces = {
-            MediaType.APPLICATION_JSON_VALUE
-        })
-    @ResponseBody
-    public Map<String, String> getTranslations(
-        @RequestParam(required = false) final List<String> type,
-        ServletRequest request
-    ) throws Exception {
-        Locale locale = languageUtils.parseAcceptLanguage(request.getLocales());
-        String language = languageUtils.locale2gnCode(locale.getISO3Language());
-        return translationPackBuilder.getDbTranslation(language, type);
-    }
-
-
     /**
      * Get list of packages.
      */