Add prefix harvesting CLI

.gitignore

@@ -4,6 +4,9 @@
 # !/setup/build/env.sh
 ##############
 
+# csv files may be generated using the Typed PID Maker CLI (write-file)
+*.csv
+
 ###  SPRING BOOT ###
 # Ignore local settings for spring boot
 # If you want to provide default settings use 'application.properties.default' and rename it while building service.

README.md

@@ -21,6 +21,8 @@ The Typed PID Maker enables the creation, maintenance, and validation of PIDs. I
   - ✅ Search for information stored within PIDs. This includes PIDs you created, updated or resolved at some point.
   - ✅ Supports the [full elastic DSL](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html) (and requires an Elasticsearch 8 instance).
 - ✅ Authentication via [JWT](https://jwt.io/introduction) or [KeyCloak](https://www.keycloak.org/)
+- ✅ Bootstrap with existing PIDs in your PID Prefix (see command line options).
+- ✅ Extract all your PIDs to CSV files (see command line options).
 
 ### Search example
 
@@ -118,6 +120,21 @@ Furthermore, you can use this Web interface to test single API calls in order to
 
 Details on the version being used and other build information can be found on http://localhost:8090/actuator/info.
 
+### Command line options
+
+- `--spring.config.location=config/application.properties` set the configuration files location to be used. Not required if the file is in the same directory as the jar file.
+- `bootstrap all-pids-from-prefix` starts the service and bootstraps all PIDs. This means:
+  - store the PIDs as "known PIDs" in the local database (as configured)
+  - send one message per PID to the message broker (if configured)
+  - (WIP, #128) store the PID records in the search index (if configured)
+  - after the bootstrap, the application will continue to run
+- `bootstrap known-pids` same as above, but:
+  - not using all PIDs from prefix, but only the ones stored in the local database ("known PIDs")
+  - useful to, for example, re-send PIDs via messaging to notify new services
+- `write-file all-pids-from-prefix` writes all PIDs of the configured PID prefix to a CSV file (one PID per line).
+- `write-file known-pids` same as above but:
+  - only with the PIDs stored in the local database ("known PIDs").
+
 ## License
 
 The KIT Data Manager is licensed under the Apache License, Version 2.0.

config/application.properties

@@ -44,7 +44,8 @@ logging.level.edu.kit: WARN
 #logging.level.org.springframework.transaction: TRACE
 logging.level.org.springframework: WARN 
 logging.level.org.springframework.amqp: WARN
-logging.level.com.zaxxer.hikari: DEBUG
+#logging.level.com.zaxxer.hikari: ERROR
+logging.level.edu.kit.datamanager.pit.cli: INFO
 
 ######################
 ### Authentication ###

src/main/java/edu/kit/datamanager/pit/Application.java

@@ -24,6 +24,11 @@
 import com.google.common.cache.LoadingCache;
 import com.google.common.cache.RemovalNotification;
 
+import edu.kit.datamanager.pit.cli.CliTaskBootstrap;
+import edu.kit.datamanager.pit.cli.CliTaskWriteFile;
+import edu.kit.datamanager.pit.cli.ICliTask;
+import edu.kit.datamanager.pit.cli.PidSource;
+import edu.kit.datamanager.pit.common.InvalidConfigException;
 import edu.kit.datamanager.pit.configuration.ApplicationProperties;
 import edu.kit.datamanager.pit.domain.PIDRecord;
 import edu.kit.datamanager.pit.domain.TypeDefinition;
@@ -37,7 +42,9 @@
 
 import java.io.IOException;
 import java.net.URISyntaxException;
+import java.util.Objects;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Stream;
 
 import org.apache.http.client.HttpClient;
 import org.apache.http.impl.client.cache.CacheConfig;
@@ -50,6 +57,7 @@
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.autoconfigure.domain.EntityScan;
 import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.ConfigurableApplicationContext;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.ComponentScan;
 import org.springframework.context.annotation.Scope;
@@ -79,6 +87,16 @@ public class Application {
 
     private static final Logger LOG = LoggerFactory.getLogger(Application.class);
 
+    protected static final String CMD_BOOTSTRAP = "bootstrap";
+    protected static final String CMD_WRITE_FILE = "write-file";
+
+    protected static final String SOURCE_FROM_PREFIX = "all-pids-from-prefix";
+    protected static final String SOURCE_KNOWN_PIDS = "known-pids";
+
+    protected static final String ERROR_COMMUNICATION = "Communication error: {}";
+    protected static final String ERROR_CONFIGURATION = "Configuration error: {}";
+
+
     @Bean
     @Scope("prototype")
     public Logger logger(InjectionPoint injectionPoint) {
@@ -155,7 +173,6 @@ public TypeDefinition load(String typeIdentifier) throws IOException, URISyntaxE
                 });
     }
 
-    @Bean
     @ConfigurationProperties("pit")
     public ApplicationProperties applicationProperties() {
         return new ApplicationProperties();
@@ -173,8 +190,80 @@ public HttpMessageConverter<PIDRecord> simplePidRecordConverter() {
     }
 
     public static void main(String[] args) {
-        SpringApplication.run(Application.class, args);
+        ConfigurableApplicationContext context = SpringApplication.run(Application.class, args);
         System.out.println("Spring is running!");
+
+        final boolean cliArgsAmountValid = args != null && args.length != 0 && args.length >= 2;
+
+        if (cliArgsAmountValid) {
+            ICliTask task = null;
+            Stream<String> pidSource = null;
+
+            if (Objects.equals(args[1], SOURCE_FROM_PREFIX)) {
+                try {
+                    pidSource = PidSource.fromPrefix(context);
+                } catch (IOException e) {
+                    e.printStackTrace();
+                    LOG.error(ERROR_COMMUNICATION, e.getMessage());
+                    exitApp(context, 1);
+                }
+            } else if (Objects.equals(args[1], SOURCE_KNOWN_PIDS)) {
+                pidSource = PidSource.fromKnown(context);
+            }
+
+            if (Objects.equals(args[0], CMD_BOOTSTRAP)) {
+                task = new CliTaskBootstrap(context, pidSource);
+            } else if (Objects.equals(args[0], CMD_WRITE_FILE)) {
+                task = new CliTaskWriteFile(pidSource);
+            }
+
+            try {
+                if (task != null && pidSource != null) {
+                    // ---process task---
+                    if (task.process()) {
+                        exitApp(context, 0);
+                    }
+                } else {
+                    printUsage(args);
+                    exitApp(context, 1);
+                }
+            } catch (InvalidConfigException e) {
+                e.printStackTrace();
+                LOG.error(ERROR_CONFIGURATION, e.getMessage());
+                exitApp(context, 1);
+            } catch (IOException e) {
+                e.printStackTrace();
+                LOG.error(ERROR_COMMUNICATION, e.getMessage());
+                exitApp(context, 1);
+            }
+        }
+    }
+
+    private static void printUsage(String[] args) {
+        String firstArg = args[0].replaceAll("[\r\n]","");
+        String secondArg = args[1].replaceAll("[\r\n]","");
+        LOG.error("Got commands: {} and {}", firstArg, secondArg);
+        LOG.error("CLI usage incorrect. Usage:");
+        LOG.error("java -jar TypedPIDMaker.jar [ACTION] [SOURCE]");
+        LOG.error("java -jar TypedPIDMaker.jar bootstrap all-pids-from-prefix");
+        LOG.error("java -jar TypedPIDMaker.jar bootstrap known-pids");
+        LOG.error("java -jar TypedPIDMaker.jar write-file all-pids-from-prefix");
+        LOG.error("java -jar TypedPIDMaker.jar write-file known-pids");
+    }
+
+    private static void exitApp(ConfigurableApplicationContext context, int errCode) {
+        context.close();
+        try {
+            Thread.sleep(2 * 1000L);
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+        }
+        if (errCode != 0) {
+            LOG.error("Exited with error.");
+        } else {
+            LOG.info("Success");
+        }
+        System.exit(errCode);
     }
 
 }

src/main/java/edu/kit/datamanager/pit/cli/CliTaskBootstrap.java

@@ -0,0 +1,91 @@
+package edu.kit.datamanager.pit.cli;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.Optional;
+import java.util.stream.Stream;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.NoSuchBeanDefinitionException;
+import org.springframework.context.ConfigurableApplicationContext;
+
+import edu.kit.datamanager.entities.messaging.PidRecordMessage;
+import edu.kit.datamanager.pit.common.InvalidConfigException;
+import edu.kit.datamanager.pit.configuration.ApplicationProperties;
+import edu.kit.datamanager.pit.domain.PIDRecord;
+import edu.kit.datamanager.pit.elasticsearch.PidRecordElasticRepository;
+import edu.kit.datamanager.pit.elasticsearch.PidRecordElasticWrapper;
+import edu.kit.datamanager.pit.pidlog.KnownPid;
+import edu.kit.datamanager.pit.pidlog.KnownPidsDao;
+import edu.kit.datamanager.pit.pitservice.ITypingService;
+import edu.kit.datamanager.service.IMessagingService;
+import edu.kit.datamanager.util.AuthenticationHelper;
+import edu.kit.datamanager.util.ControllerUtils;
+
+public class CliTaskBootstrap implements ICliTask {
+
+    private static final Logger LOG = LoggerFactory.getLogger(CliTaskBootstrap.class);
+
+    protected Stream<String> pids;
+    protected ApplicationProperties appProps;
+    protected KnownPidsDao knownPids;
+    protected IMessagingService messagingService;
+    protected Optional<PidRecordElasticRepository> elastic;
+    protected ITypingService typingService;
+
+    public CliTaskBootstrap(ConfigurableApplicationContext context, Stream<String> pids) {
+        this.pids = pids;
+        this.appProps = context.getBean(ApplicationProperties.class);
+        this.knownPids = context.getBean(KnownPidsDao.class);
+        this.messagingService = context.getBean(IMessagingService.class);
+        this.elastic = this.getBeanOptional(context, PidRecordElasticRepository.class);
+        this.typingService = context.getBean(ITypingService.class);
+    }
+
+    private <T> Optional<T> getBeanOptional(ConfigurableApplicationContext context, Class<T> clazz) {
+        try {
+            return Optional.of(context.getBean(clazz));
+        } catch (NoSuchBeanDefinitionException e) {
+            return Optional.empty();
+        }
+    }
+
+    @Override
+    public boolean process() throws IOException, InvalidConfigException {
+        Instant unknownTime = Instant.ofEpochMilli(0);
+        pids
+            .map(pid -> new KnownPid(pid, unknownTime, unknownTime))
+            .forEach(known -> {
+                // store PIDs in the local database of known PIDs
+                LOG.info("Store PID {} in the local database of known PIDs.", known.getPid());
+                boolean exists = knownPids.findByPid(known.getPid()).isPresent();
+                if (!exists) {
+                    knownPids.save(known);
+                }
+                // send to message broker
+                PidRecordMessage message = PidRecordMessage.creation(
+                    known.getPid(),
+                    "",
+                    AuthenticationHelper.getPrincipal(),
+                    ControllerUtils.getLocalHostname()
+                );
+                LOG.info("Send PID {} to message broker.", known.getPid());
+                messagingService.send(message);
+                // store in Elasticsearch
+                elastic.ifPresent(elastic -> {
+                    try {
+                        PIDRecord rec = typingService.queryAllProperties(known.getPid());
+                        LOG.info("Store PID {} in Elasticsearch.", known.getPid());
+                        PidRecordElasticWrapper wrapper = new PidRecordElasticWrapper(rec, typingService.getOperations());
+                        elastic.save(wrapper);
+                    } catch (IOException e) {
+                        LOG.error("Failed to query PID {}.", known.getPid(), e);
+                    }
+                });
+            });
+        knownPids.flush();
+        return false;
+    }
+
+}

src/main/java/edu/kit/datamanager/pit/cli/CliTaskWriteFile.java

@@ -0,0 +1,62 @@
+package edu.kit.datamanager.pit.cli;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.util.Iterator;
+import java.util.stream.Stream;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class CliTaskWriteFile implements ICliTask {
+
+    private static final Logger LOG = LoggerFactory.getLogger(CliTaskWriteFile.class);
+
+    protected Stream<String> pids;
+
+    public CliTaskWriteFile(Stream<String> pids) {
+        this.pids = pids;
+    }
+
+    @Override
+    public boolean process() throws IOException {
+        String filename = createFilename();
+
+        Path path = Paths.get(filename);
+        ensureFileExists(path);
+
+        writeToFile(path);
+        return true;
+    }
+
+    protected void writeToFile(Path path) throws IOException {
+        for (Iterator<String> iter = pids.iterator(); iter.hasNext(); ) {
+            String pid = iter.next();
+            LOG.info("Storing into CSV: {}", pid);
+            Files.writeString(path, pid + "\n", StandardOpenOption.APPEND);
+        }
+    }
+
+    protected void ensureFileExists(Path path) throws IOException {
+        File f = path.toFile();
+        f.createNewFile();
+    }
+
+    protected String createFilename() {
+        String date = ZonedDateTime
+                .now(ZoneId.systemDefault())
+                .toString()
+                .replace(":", ".")
+                .replace("[", "(")
+                .replace("]", ")")
+                .replace("/", "-")
+                .replace("+", "-");
+        return String.format("%s.%s", date, "csv");
+    }
+}

src/main/java/edu/kit/datamanager/pit/cli/ICliTask.java

@@ -0,0 +1,15 @@
+package edu.kit.datamanager.pit.cli;
+
+import java.io.IOException;
+import edu.kit.datamanager.pit.common.InvalidConfigException;
+
+public interface ICliTask {
+    /**
+     * Processes this CLI task.
+     * 
+     * @return true, if application should shut down after this task.
+     * @throws IOException            on IO issues
+     * @throws InvalidConfigException on configuration issues
+     */
+    public boolean process() throws IOException, InvalidConfigException;
+}