Fix robots.txt and sitemap 500 errors if no right content type is pro…

…vided (#7327) * Return 200 OK for robots.txt and sitemap. Return 200 OK for /robots.xt and /srv/api/sitemap instead of 500 "service not found". Previously if the request didn't contain "Accept: text/plain" or "Accept: application/xml" the server returned a 500 error. Now the server accepts any "Accept" header without complaining, returning a 200 response with "Content-Type: text/plain" or "Content-Type: application/xml" and the right content. * Disallow /catalog and /static in robots.txt. Some search engines are indexing the Angular JS HTML template files. Exclude the ${context_path}/catalog and ${context_path}/static from being crawled by robots. Fix #3585.
geonetwork · Sep 11, 2023 · 260719a · 260719a
1 parent 4e2e182
commit 260719a
Show file tree

Hide file tree

Showing 2 changed files with 32 additions and 30 deletions.
diff --git a/services/src/main/java/org/fao/geonet/api/site/SitemapApi.java b/services/src/main/java/org/fao/geonet/api/site/SitemapApi.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2001-2016 Food and Agriculture Organization of the
+ * Copyright (C) 2001-2023 Food and Agriculture Organization of the
  * United Nations (FAO-UN), United Nations World Food Programme (WFP)
  * and United Nations Environment Programme (UNEP)
  *
@@ -28,7 +28,6 @@
 import io.swagger.v3.oas.annotations.responses.ApiResponses;
 import io.swagger.v3.oas.annotations.tags.Tag;
 import org.fao.geonet.NodeInfo;
-import org.fao.geonet.api.API;
 import org.fao.geonet.api.ApiParams;
 import org.fao.geonet.domain.*;
 import org.fao.geonet.exceptions.SitemapDocumentNotFoundEx;
@@ -44,22 +43,23 @@
 import org.springframework.data.domain.PageRequest;
 import org.springframework.data.domain.Sort;
 import org.springframework.data.jpa.domain.Specification;
+import org.springframework.http.HttpHeaders;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.MediaType;
-import org.springframework.http.ResponseEntity;
 import org.springframework.stereotype.Controller;
-import org.springframework.web.bind.annotation.*;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.ResponseStatus;
 
 import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
 import java.nio.file.Path;
 import java.util.List;
 
 import static org.fao.geonet.api.ApiParams.API_CLASS_CATALOG_TAG;
 
-/**
- *
- */
-
 @RequestMapping(value = {
     "/{portal}/api"
 })
@@ -94,33 +94,34 @@ public class SitemapApi {
         description = "")
     @RequestMapping(
         path = "/robots.txt",
-        produces = MediaType.TEXT_PLAIN_VALUE,
         method = RequestMethod.GET)
     @ResponseStatus(HttpStatus.OK)
     @ApiResponses(value = {
         @ApiResponse(responseCode = "200", description = "robots.txt file for SEO.")
     })
-    @ResponseBody
-    public String getRobotsText() throws Exception {
-        StringBuffer response = new StringBuffer();
-        response.append("sitemap: ").append(settingManager.getNodeURL()).append("api/sitemap").append("\n");
-        response.append("sitemap: ").append(settingManager.getNodeURL()).append("api/sitemap?format=rdf");
-        return response.toString();
+    public void getRobotsText(HttpServletRequest request, HttpServletResponse response) throws IOException {
+        StringBuilder content = new StringBuilder(256);
+        String contextPath = request.getContextPath();
+        content.append("User-agent: *\n");
+        content.append("Disallow: ").append(contextPath).append("/catalog/\n");
+        content.append("Disallow: ").append(contextPath).append("/static/\n");
+        content.append("Sitemap: ").append(settingManager.getNodeURL()).append("api/sitemap\n");
+        content.append("Sitemap: ").append(settingManager.getNodeURL()).append("api/sitemap?format=rdf");
+        response.setHeader(HttpHeaders.CONTENT_TYPE, MediaType.TEXT_PLAIN_VALUE);
+        response.getWriter().append(content);
     }
 
     @io.swagger.v3.oas.annotations.Operation(
         summary = "Get sitemap",
         description = "")
     @RequestMapping(
         path = "/sitemap",
-        produces = MediaType.APPLICATION_XML_VALUE,
         method = RequestMethod.GET)
     @ResponseStatus(HttpStatus.OK)
     @ApiResponses(value = {
         @ApiResponse(responseCode = "200", description = "Site map.")
     })
-    public @ResponseBody
-    ResponseEntity<Element> getSitemap(
+    public void getSitemap(
         @Parameter(
             description = "Format (xml or html).",
             required = false
@@ -129,7 +130,7 @@ ResponseEntity<Element> getSitemap(
             required = false,
             defaultValue = FORMAT_HTML
         )
-            String format,
+        String format,
         @Parameter(
             description = "page.",
             required = false
@@ -138,20 +139,20 @@ ResponseEntity<Element> getSitemap(
             required = false,
             defaultValue = "0"
         )
-            Integer doc,
+        Integer doc,
         @Parameter(hidden = true)
-            HttpServletRequest request
+        HttpServletResponse response
     ) throws Exception {
         if (!(format.equalsIgnoreCase(FORMAT_HTML) ||
             format.equalsIgnoreCase(FORMAT_XML))) {
             format = FORMAT_HTML;
         }
 
-        Integer allgroup = 1;
+        int allGroup = 1;
 
         Specification<OperationAllowed> spec = Specification.where(
             OperationAllowedSpecs.hasOperation(ReservedOperation.view));
-        spec = spec.and(OperationAllowedSpecs.hasGroupId(allgroup));
+        spec = spec.and(OperationAllowedSpecs.hasGroupId(allGroup));
 
         final List<Integer> list = operationAllowedRepository.findAllIds(
             spec,
@@ -161,10 +162,10 @@ ResponseEntity<Element> getSitemap(
             Sort.Direction.DESC,
             Metadata_.dataInfo.getName() + "." + MetadataDataInfo_.changeDate.getName());
 
-        long metadatataCount = metadataRepository.count((Specification<Metadata>) MetadataSpecs.hasMetadataIdIn(list));
-        long pages = (long) Math.ceil((double) metadatataCount / MAX_ITEMS_PER_PAGE);
+        long metadataCount = metadataRepository.count((Specification<Metadata>) MetadataSpecs.hasMetadataIdIn(list));
+        long pages = (long) Math.ceil((double) metadataCount / MAX_ITEMS_PER_PAGE);
 
-        Element result = null;
+        Element result;
 
         if (doc > 0) {
             // Requesting a sitemap specific document
@@ -180,7 +181,7 @@ ResponseEntity<Element> getSitemap(
             }
         } else {
             // Request the sitemap (no specific document)
-            if (metadatataCount <= MAX_ITEMS_PER_PAGE) {
+            if (metadataCount <= MAX_ITEMS_PER_PAGE) {
                 // Request the full sitemap
                 result = metadataRepository.findUuidsAndChangeDatesAndSchemaId(list);
 
@@ -208,7 +209,7 @@ ResponseEntity<Element> getSitemap(
         root.addContent(requestElt);
         root.addContent(result);
 
-        Element sitemap = Xml.transform(root, xslt);
-        return new ResponseEntity<>(sitemap, HttpStatus.OK);
+        response.setHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_XML_VALUE);
+        Xml.transform(root, xslt, response.getOutputStream());
     }
 }
diff --git a/web-ui/src/main/resources/catalog/components/edit/onlinesrc/partials/addOnlinesrc.html b/web-ui/src/main/resources/catalog/components/edit/onlinesrc/partials/addOnlinesrc.html
@@ -395,7 +395,8 @@
                   />
                 </div>
               </div>
-              <div class="col-sm-9 col-xs-9 "
+              <div
+                class="col-sm-9 col-xs-9"
                 data-ng-if="params.linkType.fields.desc.directive === 'gn-multientry-combiner-online-resources-description'"
               >
                 <div