diff --git a/docs/adr/0022-remove-stop-words-during-query-transformation.md b/docs/adr/0022-remove-stop-words-during-query-transformation.md new file mode 100644 index 00000000000..cd66b098ca4 --- /dev/null +++ b/docs/adr/0022-remove-stop-words-during-query-transformation.md @@ -0,0 +1,33 @@ +# Remove stop words during query transformation + +## Context and Problem Statement + +When quering for a title of a paper, the title might contain stop words such as "a", "for", "and". Some data providers return 0 results when querying for a stop word. When transforming a query to the lucene syntax, the default Boolean operator `and` is used. When using IEEE, this often leads to zero search results. + +## Decision Drivers + +* Consistent to the Google search engine +* Allow reproducible searches +* Avoid WTFs on the user's side + +## Considered Options + +* Remove stop words from the query +* Automatically enclose in quotes if no Boolean operator is contained + +## Decision Outcome + +Chosen option: "Remove stop words from the query", because comes out best. + +## Pros and Cons of the Options + +### Remove stop words from the query + +* Good, because Good search results if no Boolean operators are used +* Bad, because When using complex queries and stop words are used alone, they are silently removed + +### Automatically enclose in quotes if no Boolean operator is contained + +* Good, because Good search results if no Boolean operators are used +* Bad, because Silently leads to different results +* Bad, because Inconsistent to Google behavior diff --git a/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java b/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java index 18d2254e013..77b3c626cd0 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java @@ -58,6 +58,8 @@ public class IEEE implements FulltextFetcher, PagedSearchBasedParserFetcher { private final ImportFormatPreferences preferences; + private IEEEQueryTransformer transformer; + public IEEE(ImportFormatPreferences preferences) { this.preferences = Objects.requireNonNull(preferences); } @@ -65,7 +67,7 @@ public IEEE(ImportFormatPreferences preferences) { /** * @implNote documentation */ - private static BibEntry parseJsonRespone(JSONObject jsonEntry, Character keywordSeparator) { + private static BibEntry parseJsonResponse(JSONObject jsonEntry, Character keywordSeparator) { BibEntry entry = new BibEntry(); switch (jsonEntry.optString("content_type")) { @@ -205,8 +207,24 @@ public Parser getParser() { JSONArray results = jsonObject.getJSONArray("articles"); for (int i = 0; i < results.length(); i++) { JSONObject jsonEntry = results.getJSONObject(i); - BibEntry entry = parseJsonRespone(jsonEntry, preferences.getKeywordSeparator()); - entries.add(entry); + BibEntry entry = parseJsonResponse(jsonEntry, preferences.getKeywordSeparator()); + boolean addEntry; + // In case entry has no year, add it + // In case an entry has a year, check if its in the year range + // The implementation uses some Java 8 Optional magic to implement that + if (entry.hasField(StandardField.YEAR)) { + addEntry = entry.getField(StandardField.YEAR).filter(year -> { + Integer yearAsInteger = Integer.valueOf(year); + return + transformer.getStartYear().map(startYear -> yearAsInteger >= startYear).orElse(true) && + transformer.getEndYear().map(endYear -> yearAsInteger <= endYear).orElse(true); + }).map(x -> true).orElse(false); + } else { + addEntry = true; + } + if (addEntry) { + entries.add(entry); + } } } @@ -226,7 +244,9 @@ public Optional getHelpPage() { @Override public URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException, FetcherException { - IEEEQueryTransformer transformer = new IEEEQueryTransformer(); + // transformer is stored globally, because we need to filter out the bib entries by the year manually + // the transformer stores the min and max year + transformer = new IEEEQueryTransformer(); String transformedQuery = transformer.transformLuceneQuery(luceneQuery).orElse(""); URIBuilder uriBuilder = new URIBuilder("https://ieeexploreapi.ieee.org/api/v1/search/articles"); uriBuilder.addParameter("apikey", API_KEY); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformers/AbstractQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformers/AbstractQueryTransformer.java index 22e8c642d80..21d45a13952 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/transformers/AbstractQueryTransformer.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformers/AbstractQueryTransformer.java @@ -4,6 +4,8 @@ import java.util.StringJoiner; import java.util.stream.Collectors; +import org.jabref.model.strings.StringUtil; + import org.apache.lucene.queryparser.flexible.core.nodes.BooleanQueryNode; import org.apache.lucene.queryparser.flexible.core.nodes.FieldQueryNode; import org.apache.lucene.queryparser.flexible.core.nodes.GroupQueryNode; @@ -96,7 +98,7 @@ private Optional transform(FieldQueryNode query) { return s.isEmpty() ? Optional.empty() : Optional.of(s); } case NO_EXPLICIT_FIELD -> { - return Optional.of(handleUnFieldedTerm(term)); + return handleUnFieldedTerm(term); } default -> { // Just add unknown fields as default @@ -184,21 +186,8 @@ protected String handleYearRange(String yearRange) { * * Default implementation: just return the term (in quotes if a space is contained) */ - protected String handleUnFieldedTerm(String term) { - return quoteStringIfSpaceIsContained(term); - } - - /** - * Encloses the given string with " if there is a space contained - * - * @return Returns a string - */ - protected String quoteStringIfSpaceIsContained(String string) { - if (string.contains(" ")) { - return "\"" + string + "\""; - } else { - return string; - } + protected Optional handleUnFieldedTerm(String term) { + return Optional.of(StringUtil.quoteStringIfSpaceIsContained(term)); } protected String createKeyValuePair(String fieldAsString, String term) { @@ -206,7 +195,7 @@ protected String createKeyValuePair(String fieldAsString, String term) { } protected String createKeyValuePair(String fieldAsString, String term, String separator) { - return String.format("%s%s%s", fieldAsString, separator, quoteStringIfSpaceIsContained(term)); + return String.format("%s%s%s", fieldAsString, separator, StringUtil.quoteStringIfSpaceIsContained(term)); } /** diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformers/ArXivQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformers/ArXivQueryTransformer.java index 51efafba3dd..031333adda0 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/transformers/ArXivQueryTransformer.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformers/ArXivQueryTransformer.java @@ -1,5 +1,7 @@ package org.jabref.logic.importer.fetcher.transformers; +import java.util.Optional; + public class ArXivQueryTransformer extends YearRangeByFilteringQueryTransformer { @Override protected String getLogicalAndOperator() { @@ -42,8 +44,8 @@ protected String handleYear(String year) { } @Override - protected String handleUnFieldedTerm(String term) { - return createKeyValuePair("all", term); + protected Optional handleUnFieldedTerm(String term) { + return Optional.of(createKeyValuePair("all", term)); } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformers/CollectionOfComputerScienceBibliographiesQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformers/CollectionOfComputerScienceBibliographiesQueryTransformer.java index 18e43a1f114..e58f2a01254 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/transformers/CollectionOfComputerScienceBibliographiesQueryTransformer.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformers/CollectionOfComputerScienceBibliographiesQueryTransformer.java @@ -1,5 +1,7 @@ package org.jabref.logic.importer.fetcher.transformers; +import org.jabref.model.strings.StringUtil; + public class CollectionOfComputerScienceBibliographiesQueryTransformer extends AbstractQueryTransformer { @Override @@ -29,7 +31,7 @@ protected String handleTitle(String title) { @Override protected String handleJournal(String journalTitle) { - return quoteStringIfSpaceIsContained(journalTitle); + return StringUtil.quoteStringIfSpaceIsContained(journalTitle); } @Override diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformers/DBLPQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformers/DBLPQueryTransformer.java index 97fd186a27e..0c7878c5935 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/transformers/DBLPQueryTransformer.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformers/DBLPQueryTransformer.java @@ -1,5 +1,7 @@ package org.jabref.logic.importer.fetcher.transformers; +import org.jabref.model.strings.StringUtil; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,18 +30,18 @@ protected String getLogicalNotOperator() { @Override protected String handleAuthor(String author) { // DBLP does not support explicit author field search - return quoteStringIfSpaceIsContained(author); + return StringUtil.quoteStringIfSpaceIsContained(author); } @Override protected String handleTitle(String title) { // DBLP does not support explicit title field search - return quoteStringIfSpaceIsContained(title); + return StringUtil.quoteStringIfSpaceIsContained(title); } @Override protected String handleJournal(String journalTitle) { // DBLP does not support explicit journal field search - return quoteStringIfSpaceIsContained(journalTitle); + return StringUtil.quoteStringIfSpaceIsContained(journalTitle); } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformers/DefaultQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformers/DefaultQueryTransformer.java index f77c42b99b9..40f2064e62d 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/transformers/DefaultQueryTransformer.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformers/DefaultQueryTransformer.java @@ -1,5 +1,7 @@ package org.jabref.logic.importer.fetcher.transformers; +import org.jabref.model.strings.StringUtil; + /** * Default query transformer without any boolean operators */ @@ -22,16 +24,16 @@ protected String getLogicalNotOperator() { @Override protected String handleAuthor(String author) { - return quoteStringIfSpaceIsContained(author); + return StringUtil.quoteStringIfSpaceIsContained(author); } @Override protected String handleTitle(String title) { - return quoteStringIfSpaceIsContained(title); + return StringUtil.quoteStringIfSpaceIsContained(title); } @Override protected String handleJournal(String journalTitle) { - return quoteStringIfSpaceIsContained(journalTitle); + return StringUtil.quoteStringIfSpaceIsContained(journalTitle); } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformers/GVKQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformers/GVKQueryTransformer.java index e2da33c57ed..7b649a89078 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/transformers/GVKQueryTransformer.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformers/GVKQueryTransformer.java @@ -48,10 +48,10 @@ protected String handleYear(String year) { } @Override - protected String handleUnFieldedTerm(String term) { + protected Optional handleUnFieldedTerm(String term) { // all does not search in full-text // Other option is txt: but this does not search in meta data - return createKeyValuePair("pica.all", term, "="); + return Optional.of(createKeyValuePair("pica.all", term, "=")); } @Override diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformers/IEEEQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformers/IEEEQueryTransformer.java index ad18946be53..a5015a82fab 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/transformers/IEEEQueryTransformer.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformers/IEEEQueryTransformer.java @@ -1,18 +1,25 @@ package org.jabref.logic.importer.fetcher.transformers; +import java.util.List; import java.util.Objects; import java.util.Optional; +import org.jabref.model.strings.StringUtil; + /** * Needs to be instantiated for each new query */ public class IEEEQueryTransformer extends YearRangeByFilteringQueryTransformer { + /** + * Returns words ignored by the engine. Need to be removed when querying for them. + * See ADR-0022 + */ + private static final List STOP_WORDS = List.of("a", "and", "for", "or", "with"); + // These have to be integrated into the IEEE query URL as these are just supported as query parameters // Journal is wrapped in quotes by the transformer private String journal; private String articleNumber; - private int startYear = Integer.MAX_VALUE; - private int endYear = Integer.MIN_VALUE; @Override protected String getLogicalAndOperator() { @@ -40,8 +47,9 @@ protected String handleTitle(String title) { } @Override - protected String handleJournal(String journalTitle) { - return handleUnFieldedTerm(journalTitle); + protected String handleJournal(String journal) { + this.journal = journal; + return StringUtil.quoteStringIfSpaceIsContained(journal); } @Override @@ -59,6 +67,14 @@ protected Optional handleOtherField(String fieldAsString, String term) { }; } + @Override + protected Optional handleUnFieldedTerm(String term) { + if (STOP_WORDS.contains(term)) { + return Optional.empty(); + } + return super.handleUnFieldedTerm(term); + } + private Optional handleArticleNumber(String term) { articleNumber = term; return Optional.empty(); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformers/ZbMathQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformers/ZbMathQueryTransformer.java index b1e0c42d300..64816c7e14e 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/transformers/ZbMathQueryTransformer.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformers/ZbMathQueryTransformer.java @@ -1,5 +1,7 @@ package org.jabref.logic.importer.fetcher.transformers; +import java.util.Optional; + public class ZbMathQueryTransformer extends AbstractQueryTransformer { @Override @@ -43,7 +45,7 @@ protected String handleYearRange(String yearRange) { } @Override - protected String handleUnFieldedTerm(String term) { - return createKeyValuePair("any", term); + protected Optional handleUnFieldedTerm(String term) { + return Optional.of(createKeyValuePair("any", term)); } } diff --git a/src/main/java/org/jabref/model/strings/StringUtil.java b/src/main/java/org/jabref/model/strings/StringUtil.java index 6709e1a633f..cb31437d442 100644 --- a/src/main/java/org/jabref/model/strings/StringUtil.java +++ b/src/main/java/org/jabref/model/strings/StringUtil.java @@ -738,4 +738,17 @@ public static String substringBetween(String str, String open, String close) { public static String ignoreCurlyBracket(String title) { return isNotBlank(title) ? title.replace("{", "").replace("}", "") : title; } + + /** + * Encloses the given string with " if there is a space contained + * + * @return Returns a string + */ + public static String quoteStringIfSpaceIsContained(String string) { + if (string.contains(" ")) { + return "\"" + string + "\""; + } else { + return string; + } + } } diff --git a/src/test/java/org/jabref/logic/importer/fetcher/IEEETest.java b/src/test/java/org/jabref/logic/importer/fetcher/IEEETest.java index 93224f5e8b0..7eab1cf0fb1 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/IEEETest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/IEEETest.java @@ -23,6 +23,23 @@ @FetcherTest class IEEETest implements SearchBasedFetcherCapabilityTest, PagedSearchFetcherTest { + private final BibEntry IGOR_NEWCOMERS = new BibEntry(StandardEntryType.InProceedings) + .withField(StandardField.AUTHOR, "Igor Steinmacher and Tayana Uchoa Conte and Christoph Treude and Marco Aurélio Gerosa") + .withField(StandardField.DATE, "14-22 May 2016") + .withField(StandardField.YEAR, "2016") + .withField(StandardField.EVENTDATE, "14-22 May 2016") + .withField(StandardField.EVENTTITLEADDON, "Austin, TX, USA") + .withField(StandardField.LOCATION, "Austin, TX, USA") + .withField(StandardField.DOI, "10.1145/2884781.2884806") + .withField(StandardField.JOURNALTITLE, "2016 IEEE/ACM 38th International Conference on Software Engineering (ICSE)") + .withField(StandardField.PAGES, "273--284") + .withField(StandardField.ISBN, "978-1-5090-2071-3") + .withField(StandardField.ISSN, "1558-1225") + .withField(StandardField.PUBLISHER, "IEEE") + .withField(StandardField.KEYWORDS, "Portals, Documentation, Computer bugs, Joining processes, Industries, Open source software, Newcomers, Newbies, Novices, Beginners, Open Source Software, Barriers, Obstacles, Onboarding, Joining Process") + .withField(StandardField.TITLE, "Overcoming Open Source Project Entry Barriers with a Portal for Newcomers") + .withField(StandardField.FILE, ":https\\://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=7886910:PDF"); + private IEEE fetcher; private BibEntry entry; @@ -105,26 +122,19 @@ void searchResultHasNoKeywordTerms() throws Exception { } @Test - void searchByQueryFindsEntry() throws Exception { - BibEntry expected = new BibEntry(StandardEntryType.InProceedings) - .withField(StandardField.AUTHOR, "Igor Steinmacher and Tayana Uchoa Conte and Christoph Treude and Marco Aurélio Gerosa") - .withField(StandardField.DATE, "14-22 May 2016") - .withField(StandardField.YEAR, "2016") - .withField(StandardField.EVENTDATE, "14-22 May 2016") - .withField(StandardField.EVENTTITLEADDON, "Austin, TX") - .withField(StandardField.LOCATION, "Austin, TX") - .withField(StandardField.DOI, "10.1145/2884781.2884806") - .withField(StandardField.JOURNALTITLE, "2016 IEEE/ACM 38th International Conference on Software Engineering (ICSE)") - .withField(StandardField.PAGES, "273--284") - .withField(StandardField.ISBN, "978-1-5090-2071-3") - .withField(StandardField.ISSN, "1558-1225") - .withField(StandardField.PUBLISHER, "IEEE") - .withField(StandardField.KEYWORDS, "Portals, Documentation, Computer bugs, Joining processes, Industries, Open source software, Newcomers, Newbies, Novices, Beginners, Open Source Software, Barriers, Obstacles, Onboarding, Joining Process") - .withField(StandardField.TITLE, "Overcoming Open Source Project Entry Barriers with a Portal for Newcomers") - .withField(StandardField.FILE, ":https\\://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=7886910:PDF") - .withField(StandardField.ABSTRACT, "Community-based Open Source Software (OSS) projects are usually self-organized and dynamic, receiving contributions from distributed volunteers. Newcomer are important to the survival, long-term success, and continuity of these communities. However, newcomers face many barriers when making their first contribution to an OSS project, leading in many cases to dropouts. Therefore, a major challenge for OSS projects is to provide ways to support newcomers during their first contribution. In this paper, we propose and evaluate FLOSScoach, a portal created to support newcomers to OSS projects. FLOSScoach was designed based on a conceptual model of barriers created in our previous work. To evaluate the portal, we conducted a study with 65 students, relying on qualitative data from diaries, self-efficacy questionnaires, and the Technology Acceptance Model. The results indicate that FLOSScoach played an important role in guiding newcomers and in lowering barriers related to the orientation and contribution process, whereas it was not effective in lowering technical barriers. We also found that FLOSScoach is useful, easy to use, and increased newcomers' confidence to contribute. Our results can help project maintainers on deciding the points that need more attention in order to help OSS project newcomers overcome entry barriers."); + void searchByPlainQueryFindsEntry() throws Exception { List fetchedEntries = fetcher.performSearch("Overcoming Open Source Project Entry Barriers with a Portal for Newcomers"); - assertEquals(Collections.singletonList(expected), fetchedEntries); + // Abstract should not be included in JabRef tests + fetchedEntries.forEach(entry -> entry.clearField(StandardField.ABSTRACT)); + assertEquals(Collections.singletonList(IGOR_NEWCOMERS), fetchedEntries); + } + + @Test + void searchByQuotedQueryFindsEntry() throws Exception { + List fetchedEntries = fetcher.performSearch("\"Overcoming Open Source Project Entry Barriers with a Portal for Newcomers\""); + // Abstract should not be included in JabRef tests + fetchedEntries.forEach(entry -> entry.clearField(StandardField.ABSTRACT)); + assertEquals(Collections.singletonList(IGOR_NEWCOMERS), fetchedEntries); } @Override diff --git a/src/test/java/org/jabref/logic/importer/fetcher/SearchBasedFetcherCapabilityTest.java b/src/test/java/org/jabref/logic/importer/fetcher/SearchBasedFetcherCapabilityTest.java index a5bd223cdf2..53c0838db04 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/SearchBasedFetcherCapabilityTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/SearchBasedFetcherCapabilityTest.java @@ -93,6 +93,7 @@ default void supportsJournalSearch() throws Exception { assertFalse(result.isEmpty()); result.forEach(bibEntry -> { + assertTrue(bibEntry.hasField(StandardField.JOURNAL)); String journal = bibEntry.getField(StandardField.JOURNAL).orElse(""); assertTrue(journal.contains(getTestJournal().replace("\"", ""))); }); diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformers/IEEEQueryTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformers/IEEEQueryTransformerTest.java index e1b252e1f12..3a9e6cd275e 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/transformers/IEEEQueryTransformerTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformers/IEEEQueryTransformerTest.java @@ -1,9 +1,13 @@ package org.jabref.logic.importer.fetcher.transformers; import java.util.Optional; +import java.util.stream.Stream; import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -36,7 +40,7 @@ public String getTitlePrefix() { @Override public void convertJournalField() throws Exception { - IEEEQueryTransformer transformer = ((IEEEQueryTransformer) getTransformer()); + IEEEQueryTransformer transformer = getTransformer(); String queryString = "journal:Nature"; QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); @@ -50,11 +54,11 @@ public void convertYearField() throws Exception { // IEEE does not support year range // Thus, a generic test does not work - IEEEQueryTransformer transformer = ((IEEEQueryTransformer) getTransformer()); + IEEEQueryTransformer transformer = getTransformer(); String queryString = "year:2021"; QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); - Optional result = transformer.transformLuceneQuery(luceneQuery); + transformer.transformLuceneQuery(luceneQuery); assertEquals(2021, transformer.getStartYear()); assertEquals(2021, transformer.getEndYear()); @@ -62,7 +66,7 @@ public void convertYearField() throws Exception { @Override public void convertYearRangeField() throws Exception { - IEEEQueryTransformer transformer = ((IEEEQueryTransformer) getTransformer()); + IEEEQueryTransformer transformer = getTransformer(); String queryString = "year-range:2018-2021"; QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); @@ -71,4 +75,20 @@ public void convertYearRangeField() throws Exception { assertEquals(2018, transformer.getStartYear()); assertEquals(2021, transformer.getEndYear()); } + + private static Stream getTitleTestData() { + return Stream.of( + Arguments.of("Overcoming AND Open AND Source AND Project AND Entry AND Barriers AND Portal AND Newcomers", "Overcoming Open Source Project Entry Barriers with a Portal for Newcomers"), + Arguments.of("Overcoming AND Open AND Source AND Project AND Entry AND Barriers", "Overcoming Open Source Project Entry Barriers"), + Arguments.of(null, "and") + ); + } + + @ParameterizedTest + @MethodSource("getTitleTestData") + public void testStopWordRemoval(String expected, String queryString) throws Exception { + QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + Optional result = getTransformer().transformLuceneQuery(luceneQuery); + assertEquals(Optional.ofNullable(expected), result); + } } diff --git a/src/test/java/org/jabref/model/strings/StringUtilTest.java b/src/test/java/org/jabref/model/strings/StringUtilTest.java index fb078a3a506..393f94ad283 100644 --- a/src/test/java/org/jabref/model/strings/StringUtilTest.java +++ b/src/test/java/org/jabref/model/strings/StringUtilTest.java @@ -4,8 +4,12 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.Optional; +import java.util.stream.Stream; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -347,4 +351,19 @@ void testCapitalizeFirst() { assertEquals("A", StringUtil.capitalizeFirst("a")); assertEquals("Aa", StringUtil.capitalizeFirst("AA")); } + + private static Stream getQuoteStringIfSpaceIsContainedData() { + return Stream.of( + Arguments.of("", ""), + Arguments.of("\" \"", " "), + Arguments.of("world", "world"), + Arguments.of("\"hello world\"", "hello world") + ); + } + + @ParameterizedTest + @MethodSource("getQuoteStringIfSpaceIsContainedData") + void testGuoteStringIfSpaceIsContained(String expected, String source) { + assertEquals(expected, StringUtil.quoteStringIfSpaceIsContained(source)); + } }