Convert DOAJ fetcher to new interface (#4139)

tobiasdiez · web-flow · commit fffb57bf2a29 · 2018-07-02T16:02:14.000+02:00
* Convert DOAJ fetcher to new interface

* Add doc url

* Move to logic

* Increase limit to 30
diff --git a/src/main/java/org/jabref/gui/importer/fetcher/DOAJFetcher.java b/src/main/java/org/jabref/gui/importer/fetcher/DOAJFetcher.java
diff --git a/src/main/java/org/jabref/gui/importer/fetcher/EntryFetchers.java b/src/main/java/org/jabref/gui/importer/fetcher/EntryFetchers.java
@@ -18,7 +18,6 @@ public EntryFetchers(JournalAbbreviationLoader abbreviationLoader) {
         entryFetchers.add(new INSPIREFetcher());
         // entryFetchers.add(new OAI2Fetcher()); - new arXiv fetcher in place, see below
         entryFetchers.add(new ACMPortalFetcher());
-        entryFetchers.add(new DOAJFetcher());
 
         WebFetchers.getSearchBasedFetchers(Globals.prefs.getImportFormatPreferences()).stream()
                 .map(SearchBasedEntryFetcher::new)
diff --git a/src/main/java/org/jabref/gui/importer/fetcher/SearchBasedEntryFetcher.java b/src/main/java/org/jabref/gui/importer/fetcher/SearchBasedEntryFetcher.java
@@ -31,7 +31,6 @@ public SearchBasedEntryFetcher(SearchBasedFetcher fetcher) {
 
     @Override
     public boolean processQuery(String query, ImportInspector inspector, OutputPrinter status) {
-
         status.setStatus(Localization.lang("Processing %0", query));
         try {
             List<BibEntry> matches = fetcher.performSearch(query);
diff --git a/src/main/java/org/jabref/logic/importer/WebFetchers.java b/src/main/java/org/jabref/logic/importer/WebFetchers.java
@@ -10,6 +10,7 @@
 import org.jabref.logic.importer.fetcher.AstrophysicsDataSystem;
 import org.jabref.logic.importer.fetcher.CrossRef;
 import org.jabref.logic.importer.fetcher.DBLPFetcher;
+import org.jabref.logic.importer.fetcher.DOAJFetcher;
 import org.jabref.logic.importer.fetcher.DiVA;
 import org.jabref.logic.importer.fetcher.DoiFetcher;
 import org.jabref.logic.importer.fetcher.DoiResolution;
@@ -84,6 +85,7 @@ public static List<SearchBasedFetcher> getSearchBasedFetchers(ImportFormatPrefer
         list.add(new DBLPFetcher(importFormatPreferences));
         list.add(new SpringerFetcher());
         list.add(new CrossRef());
+        list.add(new DOAJFetcher(importFormatPreferences));
         list.sort(Comparator.comparing(WebFetcher::getName));
         return list;
     }
diff --git a/src/main/java/org/jabref/logic/importer/fetcher/DOAJFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/DOAJFetcher.java
@@ -0,0 +1,79 @@
+package org.jabref.logic.importer.fetcher;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.net.MalformedURLException;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import org.jabref.logic.help.HelpFile;
+import org.jabref.logic.importer.FetcherException;
+import org.jabref.logic.importer.ImportFormatPreferences;
+import org.jabref.logic.importer.Parser;
+import org.jabref.logic.importer.SearchBasedParserFetcher;
+import org.jabref.logic.importer.util.JSONEntryParser;
+import org.jabref.logic.net.URLUtil;
+import org.jabref.logic.util.OS;
+import org.jabref.model.entry.BibEntry;
+
+import org.apache.http.client.utils.URIBuilder;
+import org.json.JSONArray;
+import org.json.JSONObject;
+
+/**
+ * Fetches data from the Directory of Open Access Journals (DOAJ)
+ *
+ * @implNote <a href="https://doaj.org/api/v1/docs">API documentation</a>
+ */
+public class DOAJFetcher implements SearchBasedParserFetcher {
+
+    private static final String SEARCH_URL = "https://doaj.org/api/v1/search/articles/";
+    private final ImportFormatPreferences preferences;
+
+    public DOAJFetcher(ImportFormatPreferences preferences) {
+        this.preferences = Objects.requireNonNull(preferences);
+    }
+
+    @Override
+    public String getName() {
+        return "DOAJ";
+    }
+
+    @Override
+    public HelpFile getHelpPage() {
+        return HelpFile.FETCHER_DOAJ;
+    }
+
+    @Override
+    public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException {
+        URIBuilder uriBuilder = new URIBuilder(SEARCH_URL);
+        URLUtil.addPath(uriBuilder, query);
+        uriBuilder.addParameter("pageSize", "30"); // Number of results
+        //uriBuilder.addParameter("page", "1"); // Page (not needed so far)
+        return uriBuilder.build().toURL();
+    }
+
+    @Override
+    public Parser getParser() {
+        return inputStream -> {
+            JSONEntryParser jsonConverter = new JSONEntryParser();
+            String response = new BufferedReader(new InputStreamReader(inputStream)).lines().collect(Collectors.joining(OS.NEWLINE));
+            JSONObject jsonObject = new JSONObject(response);
+
+            List<BibEntry> entries = new ArrayList<>();
+            if (jsonObject.has("results")) {
+                JSONArray results = jsonObject.getJSONArray("results");
+                for (int i = 0; i < results.length(); i++) {
+                    JSONObject bibJsonEntry = results.getJSONObject(i).getJSONObject("bibjson");
+                    BibEntry entry = jsonConverter.parseBibJSONtoBibtex(bibJsonEntry, preferences.getKeywordSeparator());
+                    entries.add(entry);
+                }
+            }
+            return entries;
+        };
+    }
+}
diff --git a/src/main/java/org/jabref/logic/importer/fetcher/MathSciNet.java b/src/main/java/org/jabref/logic/importer/fetcher/MathSciNet.java
@@ -85,12 +85,10 @@ public URL getURLForID(String identifier) throws URISyntaxException, MalformedUR
 
     @Override
     public Parser getParser() {
-
         // MathSciNet returns the BibTeX result embedded in HTML
         // So we extract the BibTeX string from the <pre>bibtex</pre> tags and pass the content to the BibTeX parser
         return inputStream -> {
-            String response = new BufferedReader(new InputStreamReader(inputStream)).lines().collect(
-                    Collectors.joining(OS.NEWLINE));
+            String response = new BufferedReader(new InputStreamReader(inputStream)).lines().collect(Collectors.joining(OS.NEWLINE));
 
             List<BibEntry> entries = new ArrayList<>();
             BibtexParser bibtexParser = new BibtexParser(preferences, new DummyFileUpdateMonitor());
diff --git a/src/main/java/org/jabref/logic/importer/util/JSONEntryParser.java b/src/main/java/org/jabref/logic/importer/util/JSONEntryParser.java
@@ -72,7 +72,7 @@ public BibEntry parseBibJSONtoBibtex(JSONObject bibJsonEntry, Character keywordS
             JSONObject journal = bibJsonEntry.getJSONObject("journal");
             // Journal title
             if (journal.has("title")) {
-                entry.setField(FieldName.JOURNAL, journal.getString("title"));
+                entry.setField(FieldName.JOURNAL, journal.getString("title").trim());
             } else {
                 LOGGER.info("No journal title found.");
             }
@@ -91,7 +91,7 @@ public BibEntry parseBibJSONtoBibtex(JSONObject bibJsonEntry, Character keywordS
             JSONArray keywords = bibJsonEntry.getJSONArray("keywords");
             for (int i = 0; i < keywords.length(); i++) {
                 if (!keywords.isNull(i)) {
-                    entry.addKeyword(keywords.getString(i), keywordSeparator);
+                    entry.addKeyword(keywords.getString(i).trim(), keywordSeparator);
                 }
             }
         }
diff --git a/src/main/java/org/jabref/logic/net/URLUtil.java b/src/main/java/org/jabref/logic/net/URLUtil.java
@@ -7,6 +7,10 @@
 import java.nio.charset.StandardCharsets;
 import java.util.Objects;
 
+import org.jabref.model.strings.StringUtil;
+
+import org.apache.http.client.utils.URIBuilder;
+
 public class URLUtil {
     private static final String URL_EXP = "^(https?|ftp)://.+";
 
@@ -74,4 +78,28 @@ public static boolean isURL(String url) {
         return url.contains("://");
     }
 
+    /**
+     * @implNote slightly altered version based on https://gist.github.com/enginer/230e2dc2f1d213a825d5
+     */
+    public static URIBuilder addPath(URIBuilder base, String subPath) {
+        if (StringUtil.isBlank(subPath) || "/".equals(subPath)) {
+            return base;
+        } else {
+            base.setPath(appendSegmentToPath(base.getPath(), subPath));
+            return base;
+        }
+    }
+
+    private static String appendSegmentToPath(String path, String segment) {
+        if (StringUtil.isBlank(path)) {
+            path = "/";
+        }
+
+        if (path.charAt(path.length() - 1) == '/' || segment.startsWith("/")) {
+            return path + segment;
+        }
+
+        return path + "/" + segment;
+    }
+
 }
diff --git a/src/test/java/org/jabref/logic/importer/WebFetchersTest.java b/src/test/java/org/jabref/logic/importer/WebFetchersTest.java
@@ -3,20 +3,11 @@
 import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
-import org.jabref.logic.importer.fetcher.ACS;
 import org.jabref.logic.importer.fetcher.AbstractIsbnFetcher;
-import org.jabref.logic.importer.fetcher.ArXiv;
-import org.jabref.logic.importer.fetcher.DoiResolution;
-import org.jabref.logic.importer.fetcher.GoogleScholar;
-import org.jabref.logic.importer.fetcher.IEEE;
 import org.jabref.logic.importer.fetcher.IsbnViaChimboriFetcher;
 import org.jabref.logic.importer.fetcher.IsbnViaEbookDeFetcher;
 import org.jabref.logic.importer.fetcher.MrDLibFetcher;
-import org.jabref.logic.importer.fetcher.OpenAccessDoi;
-import org.jabref.logic.importer.fetcher.ScienceDirect;
-import org.jabref.logic.importer.fetcher.SpringerLink;
 
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -25,18 +16,18 @@
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.mockito.Mockito.mock;
 
-public class WebFetchersTest {
+class WebFetchersTest {
 
     Reflections reflections = new Reflections("org.jabref");
     ImportFormatPreferences importFormatPreferences;
 
     @BeforeEach
-    public void setUp() throws Exception {
+    void setUp() throws Exception {
         importFormatPreferences = mock(ImportFormatPreferences.class);
     }
 
     @Test
-    public void getIdBasedFetchersReturnsAllFetcherDerivingFromIdBasedFetcher() throws Exception {
+    void getIdBasedFetchersReturnsAllFetcherDerivingFromIdBasedFetcher() throws Exception {
         List<IdBasedFetcher> idFetchers = WebFetchers.getIdBasedFetchers(importFormatPreferences);
 
         Set<Class<? extends IdBasedFetcher>> expected = reflections.getSubTypesOf(IdBasedFetcher.class);
@@ -49,7 +40,7 @@ public void getIdBasedFetchersReturnsAllFetcherDerivingFromIdBasedFetcher() thro
     }
 
     @Test
-    public void getEntryBasedFetchersReturnsAllFetcherDerivingFromEntryBasedFetcher() throws Exception {
+    void getEntryBasedFetchersReturnsAllFetcherDerivingFromEntryBasedFetcher() throws Exception {
         List<EntryBasedFetcher> idFetchers = WebFetchers.getEntryBasedFetchers(importFormatPreferences);
 
         Set<Class<? extends EntryBasedFetcher>> expected = reflections.getSubTypesOf(EntryBasedFetcher.class);
@@ -59,7 +50,7 @@ public void getEntryBasedFetchersReturnsAllFetcherDerivingFromEntryBasedFetcher(
     }
 
     @Test
-    public void getSearchBasedFetchersReturnsAllFetcherDerivingFromSearchBasedFetcher() throws Exception {
+    void getSearchBasedFetchersReturnsAllFetcherDerivingFromSearchBasedFetcher() throws Exception {
         List<SearchBasedFetcher> searchBasedFetchers = WebFetchers.getSearchBasedFetchers(importFormatPreferences);
 
         Set<Class<? extends SearchBasedFetcher>> expected = reflections.getSubTypesOf(SearchBasedFetcher.class);
@@ -68,15 +59,15 @@ public void getSearchBasedFetchersReturnsAllFetcherDerivingFromSearchBasedFetche
     }
 
     @Test
-    public void getFullTextFetchersReturnsAllFetcherDerivingFromFullTextFetcher() throws Exception {
+    void getFullTextFetchersReturnsAllFetcherDerivingFromFullTextFetcher() throws Exception {
         List<FulltextFetcher> fullTextFetchers = WebFetchers.getFullTextFetchers(importFormatPreferences);
 
         Set<Class<? extends FulltextFetcher>> expected = reflections.getSubTypesOf(FulltextFetcher.class);
         assertEquals(expected, getClasses(fullTextFetchers));
     }
 
     @Test
-    public void getIdFetchersReturnsAllFetcherDerivingFromIdFetcher() throws Exception {
+    void getIdFetchersReturnsAllFetcherDerivingFromIdFetcher() throws Exception {
         List<IdFetcher> idFetchers = WebFetchers.getIdFetchers(importFormatPreferences);
 
         Set<Class<? extends IdFetcher>> expected = reflections.getSubTypesOf(IdFetcher.class);
diff --git a/src/test/java/org/jabref/logic/importer/fetcher/DOAJFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/DOAJFetcherTest.java
diff --git a/src/test/java/org/jabref/logic/net/URLUtilTest.java b/src/test/java/org/jabref/logic/net/URLUtilTest.java

Original file line number	Diff line number	Diff line change
`@@ -72,7 +72,7 @@ public BibEntry parseBibJSONtoBibtex(JSONObject bibJsonEntry, Character keywordS`
`72`	`72`	`JSONObject journal = bibJsonEntry.getJSONObject("journal");`
`73`	`73`	`// Journal title`
`74`	`74`	`if (journal.has("title")) {`
`75`		`- entry.setField(FieldName.JOURNAL, journal.getString("title"));`
	`75`	`+ entry.setField(FieldName.JOURNAL, journal.getString("title").trim());`
`76`	`76`	`} else {`
`77`	`77`	`LOGGER.info("No journal title found.");`
`78`	`78`	`}`
`@@ -91,7 +91,7 @@ public BibEntry parseBibJSONtoBibtex(JSONObject bibJsonEntry, Character keywordS`
`91`	`91`	`JSONArray keywords = bibJsonEntry.getJSONArray("keywords");`
`92`	`92`	`for (int i = 0; i < keywords.length(); i++) {`
`93`	`93`	`if (!keywords.isNull(i)) {`
`94`		`- entry.addKeyword(keywords.getString(i), keywordSeparator);`
	`94`	`+ entry.addKeyword(keywords.getString(i).trim(), keywordSeparator);`
`95`	`95`	`}`
`96`	`96`	`}`
`97`	`97`	`}`