AI - MVP - v1

build.gradle

@@ -44,6 +44,7 @@ version = project.findProperty('projVersion') ?: '100.0.0'
 java {
     sourceCompatibility = JavaVersion.VERSION_21
     targetCompatibility = JavaVersion.VERSION_21
+
     // Workaround needed for Eclipse, probably because of https://github.com/gradle/gradle/issues/16922
     // Should be removed as soon as Gradle 7.0.1 is released ( https://github.com/gradle/gradle/issues/16922#issuecomment-828217060 )
     modularity.inferModulePath.set(false)
@@ -94,6 +95,13 @@ application {
 // See also https://github.com/java9-modularity/gradle-modules-plugin/issues/165
 modularity.disableEffectiveArgumentsAdjustment()
 
+// Required as workaround for  https://github.com/langchain4j/langchain4j/issues/1066
+// modularity.patchModule("langchain4j", "langchain4j-0.28.0.jar")
+modularity.patchModule("langchain4j", "langchain4j-core-0.28.0.jar")
+modularity.patchModule("langchain4j", "langchain4j-embeddings-0.28.0.jar")
+modularity.patchModule("langchain4j", "langchain4j-embeddings-all-minilm-l6-v2-0.28.0.jar")
+modularity.patchModule("langchain4j", "langchain4j-open-ai-0.28.0.jar")
+
 sourceSets {
     main {
         java {
@@ -219,14 +227,19 @@ dependencies {
     implementation 'org.fxmisc.flowless:flowless:0.7.2'
     implementation 'org.fxmisc.richtext:richtextfx:0.11.2'
     implementation (group: 'com.dlsc.gemsfx', name: 'gemsfx', version: '2.12.0') {
-        exclude module: 'javax.inject' // Split package, use only jakarta.inject
         exclude module: 'commons-lang3'
+        exclude module: 'javax.inject' // Split package, use only jakarta.inject
+        exclude module: 'kotlin-stdlib-jdk8'
+        exclude group: 'com.squareup.retrofit2'
         exclude group: 'org.openjfx'
         exclude group: 'org.apache.logging.log4j'
         exclude group: 'tech.units'
     }
     // Required by gemsfx
     implementation 'tech.units:indriya:2.2'
+    implementation ('com.squareup.retrofit2:retrofit:2.11.0') {
+        exclude group: 'com.squareup.okhttp3'
+    }
 
     implementation 'org.controlsfx:controlsfx:11.2.1'
 
@@ -296,6 +309,19 @@ dependencies {
     // YAML formatting
     implementation 'org.yaml:snakeyaml:2.2'
 
+    // AI
+    implementation 'dev.langchain4j:langchain4j-embeddings-all-minilm-l6-v2:0.28.0'
+    implementation('dev.langchain4j:langchain4j-open-ai:0.28.0') {
+        exclude group: 'org.jetbrains.kotlin', module: 'kotlin-stdlib-jdk8'
+    }
+    // openai depends on okhttp, which needs kotlin - see https://github.com/square/okhttp/issues/5299 for details
+    // GemxFX also (transitively) depends on kotlin
+    implementation 'org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.24'
+
+    implementation ('com.squareup.okhttp3:okhttp:4.12.0') {
+        exclude group: 'org.jetbrains.kotlin', module: 'kotlin-stdlib-jdk8'
+    }
+
     implementation 'commons-io:commons-io:2.16.1'
 
     testImplementation 'io.github.classgraph:classgraph:4.8.172'
@@ -436,7 +462,19 @@ compileJava {
         // TODO: Remove access to internal api
         addExports = [
                 'javafx.controls/com.sun.javafx.scene.control' : 'org.jabref',
-                'org.controlsfx.controls/impl.org.controlsfx.skin' : 'org.jabref'
+                'org.controlsfx.controls/impl.org.controlsfx.skin' : 'org.jabref',
+
+                'langchain4j/dev.langchain4j.data.document' : 'org.jabref',
+                'langchain4j/dev.langchain4j.data.segment' : 'org.jabref',
+                'langchain4j/dev.langchain4j.model.chat' : 'org.jabref',
+                'langchain4j/dev.langchain4j.model.embedding' : 'org.jabref',
+                'langchain4j/dev.langchain4j.model.openai' : 'org.jabref',
+                'langchain4j/dev.langchain4j.rag.content.retriever' : 'org.jabref',
+                'langchain4j/dev.langchain4j.store.embedding' : 'org.jabref',
+                'langchain4j/dev.langchain4j.memory' : 'org.jabref',
+                'langchain4j/dev.langchain4j.store.memory.chat' : 'org.jabref',
+                'langchain4j/dev.langchain4j.data.message' : 'org.jabref',
+                'langchain4j/dev.langchain4j.store.embedding.filter' : 'org.jabref',
         ]
     }
 }
@@ -455,6 +493,9 @@ run {
                 'javafx.base/com.sun.javafx.event' : 'org.jabref.merged.module',
                 'javafx.controls/com.sun.javafx.scene.control' : 'org.jabref',
 
+                'langchain4j/dev.langchain4j.model.chat' : 'org.jabref',
+                'langchain4j/dev.langchain4j.model.chat' : 'org.jabref.merged.module',
+
                 // We need to restate the ControlsFX exports, because we get following error otherwise:
                 //   java.lang.IllegalAccessError:
                 //     class org.controlsfx.control.textfield.AutoCompletionBinding (in module org.controlsfx.controls)

src/main/java/module-info.java

@@ -124,6 +124,10 @@
 
     requires org.jooq.jool;
 
+    // AI
+    requires langchain4j;
+    requires kotlin.stdlib;
+
     // fulltext search
     requires org.apache.lucene.core;
     // In case the version is updated, please also adapt SearchFieldConstants#VERSION to the newly used version

src/main/java/org/jabref/gui/Base.css

@@ -252,6 +252,10 @@
 
     /* Consistent size for headers of tab-pane and side-panels*/
     -jr-header-height: 3em;
+
+    /* AI chat style */
+    -jr-ai-message-user: #7ee3fb;
+    -jr-ai-message-ai: #bac8cb;
 }
 
 .unchanged {

src/main/java/org/jabref/gui/entryeditor/EntryEditor.java

@@ -31,6 +31,7 @@
 import org.jabref.gui.StateManager;
 import org.jabref.gui.citationkeypattern.GenerateCitationKeySingleAction;
 import org.jabref.gui.cleanup.CleanupSingleAction;
+import org.jabref.gui.entryeditor.aichattab.AiChatTab;
 import org.jabref.gui.entryeditor.citationrelationtab.CitationRelationsTab;
 import org.jabref.gui.entryeditor.fileannotationtab.FileAnnotationTab;
 import org.jabref.gui.entryeditor.fileannotationtab.FulltextSearchResultsTab;
@@ -45,6 +46,7 @@
 import org.jabref.gui.undo.CountingUndoManager;
 import org.jabref.gui.util.DefaultTaskExecutor;
 import org.jabref.gui.util.TaskExecutor;
+import org.jabref.logic.ai.AiService;
 import org.jabref.logic.bibtex.TypedBibEntry;
 import org.jabref.logic.help.HelpFile;
 import org.jabref.logic.importer.EntryBasedFetcher;
@@ -87,6 +89,9 @@ public class EntryEditor extends BorderPane {
     private final ExternalFilesEntryLinker fileLinker;
     private final DirectoryMonitorManager directoryMonitorManager;
 
+    // TODO: Move this out.
+    private final AiService aiService;
+
     private Subscription typeSubscription;
 
     /*
@@ -131,6 +136,8 @@ public EntryEditor(LibraryTab libraryTab) {
                   .root(this)
                   .load();
 
+        this.aiService = new AiService(preferencesService.getAiPreferences());
+
         this.entryEditorPreferences = preferencesService.getEntryEditorPreferences();
         this.fileLinker = new ExternalFilesEntryLinker(preferencesService.getFilePreferences(), databaseContext, dialogService);
 
@@ -314,6 +321,7 @@ private List<EntryEditorTab> createTabs() {
         entryEditorTabs.add(sourceTab);
         entryEditorTabs.add(new LatexCitationsTab(databaseContext, preferencesService, dialogService, directoryMonitorManager));
         entryEditorTabs.add(new FulltextSearchResultsTab(stateManager, preferencesService, dialogService, taskExecutor));
+        entryEditorTabs.add(new AiChatTab(preferencesService, aiService, databaseContext, taskExecutor));
 
         return entryEditorTabs;
     }

src/main/java/org/jabref/gui/entryeditor/EntryEditorPreferences.java

@@ -40,6 +40,7 @@ public static JournalPopupEnabled fromString(String status) {
     private final MapProperty<String, Set<Field>> defaultEntryEditorTabList;
     private final BooleanProperty shouldOpenOnNewEntry;
     private final BooleanProperty shouldShowRecommendationsTab;
+    private final BooleanProperty shouldShowAiChatTab;
     private final BooleanProperty shouldShowLatexCitationsTab;
     private final BooleanProperty showSourceTabByDefault;
     private final BooleanProperty enableValidation;
@@ -54,6 +55,7 @@ public EntryEditorPreferences(Map<String, Set<Field>> entryEditorTabList,
                                   Map<String, Set<Field>> defaultEntryEditorTabList,
                                   boolean shouldOpenOnNewEntry,
                                   boolean shouldShowRecommendationsTab,
+                                  boolean shouldShowAiChatTab,
                                   boolean shouldShowLatexCitationsTab,
                                   boolean showSourceTabByDefault,
                                   boolean enableValidation,
@@ -68,6 +70,7 @@ public EntryEditorPreferences(Map<String, Set<Field>> entryEditorTabList,
         this.defaultEntryEditorTabList = new SimpleMapProperty<>(FXCollections.observableMap(defaultEntryEditorTabList));
         this.shouldOpenOnNewEntry = new SimpleBooleanProperty(shouldOpenOnNewEntry);
         this.shouldShowRecommendationsTab = new SimpleBooleanProperty(shouldShowRecommendationsTab);
+        this.shouldShowAiChatTab = new SimpleBooleanProperty(shouldShowAiChatTab);
         this.shouldShowLatexCitationsTab = new SimpleBooleanProperty(shouldShowLatexCitationsTab);
         this.showSourceTabByDefault = new SimpleBooleanProperty(showSourceTabByDefault);
         this.enableValidation = new SimpleBooleanProperty(enableValidation);
@@ -119,6 +122,18 @@ public void setShouldShowRecommendationsTab(boolean shouldShowRecommendationsTab
         this.shouldShowRecommendationsTab.set(shouldShowRecommendationsTab);
     }
 
+    public boolean shouldShowAiChatTab() {
+        return shouldShowAiChatTab.get();
+    }
+
+    public BooleanProperty shouldShowAiChatTabProperty() {
+        return shouldShowAiChatTab;
+    }
+
+    public void setShouldShowAiChatTab(boolean shouldShowAiChatTab) {
+        this.shouldShowAiChatTab.set(shouldShowAiChatTab);
+    }
+
     public boolean shouldShowLatexCitationsTab() {
         return shouldShowLatexCitationsTab.get();
     }

src/main/java/org/jabref/gui/entryeditor/aichattab/AiChatTab.java

@@ -0,0 +1,156 @@
+package org.jabref.gui.entryeditor.aichattab;
+
+import java.nio.file.Path;
+import java.util.Optional;
+
+import javafx.scene.control.*;
+
+import org.jabref.gui.entryeditor.EntryEditorPreferences;
+import org.jabref.gui.entryeditor.EntryEditorTab;
+import org.jabref.gui.entryeditor.aichattab.components.AiChatComponentOld;
+import org.jabref.gui.entryeditor.aichattab.components.aichat.AiChatComponent;
+import org.jabref.gui.util.BackgroundTask;
+import org.jabref.gui.util.TaskExecutor;
+import org.jabref.logic.ai.AiChat;
+import org.jabref.logic.ai.AiService;
+import org.jabref.logic.ai.AiIngestor;
+import org.jabref.logic.ai.ChatMessage;
+import org.jabref.logic.l10n.Localization;
+import org.jabref.logic.util.io.FileUtil;
+import org.jabref.model.database.BibDatabaseContext;
+import org.jabref.model.entry.BibEntry;
+import org.jabref.model.entry.LinkedFile;
+import org.jabref.preferences.AiPreferences;
+import org.jabref.preferences.FilePreferences;
+import org.jabref.preferences.PreferencesService;
+
+import com.tobiasdiez.easybind.EasyBind;
+import dev.langchain4j.data.segment.TextSegment;
+import dev.langchain4j.store.embedding.EmbeddingStore;
+import dev.langchain4j.store.embedding.filter.Filter;
+import dev.langchain4j.store.embedding.filter.MetadataFilterBuilder;
+import dev.langchain4j.store.embedding.inmemory.InMemoryEmbeddingStore;
+import org.slf4j.LoggerFactory;
+
+public class AiChatTab extends EntryEditorTab {
+    public static final String NAME = "AI chat";
+
+    private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(AiChatTab.class.getName());
+
+    private static final String QA_SYSTEM_MESSAGE = """
+            You are an AI research assistant. You read and analyze scientific articles.
+            The user will send you a question regarding a paper. You will be supplied also with the relevant information found in the article.
+            Answer the question only by using the relevant information. Don't make up the answer.
+            If you can't answer the user question using the provided information, then reply that you couldn't do it.""";
+
+    private final FilePreferences filePreferences;
+    private final AiPreferences aiPreferences;
+    private final EntryEditorPreferences entryEditorPreferences;
+    private final BibDatabaseContext bibDatabaseContext;
+    private final TaskExecutor taskExecutor;
+
+    private AiChatComponent aiChatComponent = null;
+
+    private final AiService aiService;
+
+    private AiChat aiChat = null;
+
+    private BibEntry currentBibEntry = null;
+
+    public AiChatTab(PreferencesService preferencesService, AiService aiService,
+                     BibDatabaseContext bibDatabaseContext, TaskExecutor taskExecutor) {
+        this.filePreferences = preferencesService.getFilePreferences();
+        this.aiPreferences = preferencesService.getAiPreferences();
+        this.entryEditorPreferences = preferencesService.getEntryEditorPreferences();
+
+        this.aiService = aiService;
+
+        this.bibDatabaseContext = bibDatabaseContext;
+
+        this.taskExecutor = taskExecutor;
+
+        setText(Localization.lang(NAME));
+        setTooltip(new Tooltip(Localization.lang("AI chat with full-text article")));
+    }
+
+    @Override
+    public boolean shouldShow(BibEntry entry) {
+        return entryEditorPreferences.shouldShowAiChatTab();
+    }
+
+    @Override
+    protected void bindToEntry(BibEntry entry) {
+        if (!aiPreferences.getEnableChatWithFiles()) {
+            setContent(new Label(Localization.lang("JabRef uses OpenAI to enable \"chatting\" with PDF files. OpenAI is an external service. To enable JabRef chatgting with PDF files, the content of the PDF files need to be shared with OpenAI. As soon as you ask a question, the text content of all PDFs attached to the entry are send to OpenAI. The privacy policy of OpenAI applies. You find it at <https://openai.com/policies/privacy-policy/>.")));
+        } else if (entry.getCitationKey().isEmpty()) {
+            setContent(new Label(Localization.lang("Please provide a citation key for the entry in order to enable chatting with PDF files.")));
+        } else if (!checkIfCitationKeyIsUnique(bibDatabaseContext, entry.getCitationKey().get())) {
+            setContent(new Label(Localization.lang("Please provide a unique citation key for the entry in order to enable chatting with PDF files.")));
+        } else if (entry.getFiles().isEmpty()) {
+            setContent(new Label(Localization.lang("No files attached")));
+        } else if (!entry.getFiles().stream().map(LinkedFile::getLink).map(Path::of).allMatch(FileUtil::isPDFFile)) {
+            setContent(new Label(Localization.lang("Only PDF files are supported")));
+        } else {
+            bindToCorrectEntry(entry);
+        }
+    }
+
+    private static boolean checkIfCitationKeyIsUnique(BibDatabaseContext bibDatabaseContext, String citationKey) {
+        return bibDatabaseContext.getDatabase().getEntries().stream()
+              .map(BibEntry::getCitationKey)
+              .filter(Optional::isPresent)
+              .map(Optional::get)
+              .filter(key -> key.equals(citationKey))
+              .count() == 1;
+    }
+
+    private void bindToCorrectEntry(BibEntry entry) {
+        currentBibEntry = entry;
+
+        createAiChat();
+        aiChat.restoreMessages(entry.getAiChatMessages());
+        ingestFiles(entry);
+        buildChatUI(entry);
+    }
+
+    private void createAiChat() {
+        aiChat = new AiChat(aiService, MetadataFilterBuilder.metadataKey("linkedFile").isIn(currentBibEntry.getFiles().stream().map(LinkedFile::getLink).toList()));
+        aiChat.setSystemMessage(QA_SYSTEM_MESSAGE);
+    }
+
+    private void ingestFiles(BibEntry entry) {
+        AiIngestor aiIngestor = new AiIngestor(aiService.getEmbeddingStore(), aiService.getEmbeddingModel());
+        entry.getFiles().forEach(file -> {
+            aiIngestor.ingestLinkedFile(file, bibDatabaseContext, filePreferences);
+        });
+    }
+
+    private void buildChatUI(BibEntry entry) {
+        aiChatComponent = new AiChatComponent((userPrompt) -> {
+            ChatMessage userMessage = ChatMessage.user(userPrompt);
+            aiChatComponent.addMessage(userMessage);
+            entry.getAiChatMessages().add(userMessage);
+            aiChatComponent.setLoading(true);
+
+            BackgroundTask.wrap(() -> aiChat.execute(userPrompt))
+                    .onSuccess(aiMessageText -> {
+                        aiChatComponent.setLoading(false);
+
+                        ChatMessage aiMessage = ChatMessage.assistant(aiMessageText);
+                        aiChatComponent.addMessage(aiMessage);
+                        entry.getAiChatMessages().add(aiMessage);
+                    })
+                    .onFailure(e -> {
+                        // TODO: User-friendly error message.
+                        LOGGER.error("Got an error while sending a message to AI", e);
+                        aiChatComponent.setLoading(false);
+                        aiChatComponent.addError(e.getMessage());
+                    })
+                    .executeWith(taskExecutor);
+        });
+
+        entry.getAiChatMessages().forEach(aiChatComponent::addMessage);
+
+        setContent(aiChatComponent);
+    }
+}