Add prefix harvesting CLI

.gitignore

@@ -4,6 +4,9 @@
 # !/setup/build/env.sh
 ##############
 
+# csv files may be generated using the Typed PID Maker CLI (write-file)
+*.csv
+
 ###  SPRING BOOT ###
 # Ignore local settings for spring boot
 # If you want to provide default settings use 'application.properties.default' and rename it while building service.

config/application.properties

@@ -44,7 +44,8 @@ logging.level.edu.kit: WARN
 #logging.level.org.springframework.transaction: TRACE
 logging.level.org.springframework: WARN 
 logging.level.org.springframework.amqp: WARN
-logging.level.com.zaxxer.hikari: DEBUG
+#logging.level.com.zaxxer.hikari: ERROR
+logging.level.edu.kit.datamanager.pit.cli: INFO
 
 ######################
 ### Authentication ###

src/main/java/edu/kit/datamanager/pit/Application.java

@@ -24,6 +24,11 @@
 import com.google.common.cache.LoadingCache;
 import com.google.common.cache.RemovalNotification;
 
+import edu.kit.datamanager.pit.cli.CliTaskBootstrap;
+import edu.kit.datamanager.pit.cli.CliTaskWriteFile;
+import edu.kit.datamanager.pit.cli.ICliTask;
+import edu.kit.datamanager.pit.cli.PidSource;
+import edu.kit.datamanager.pit.common.InvalidConfigException;
 import edu.kit.datamanager.pit.configuration.ApplicationProperties;
 import edu.kit.datamanager.pit.domain.PIDRecord;
 import edu.kit.datamanager.pit.domain.TypeDefinition;
@@ -37,7 +42,10 @@
 
 import java.io.IOException;
 import java.net.URISyntaxException;
+import java.util.Arrays;
+import java.util.Objects;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Stream;
 
 import org.apache.http.client.HttpClient;
 import org.apache.http.impl.client.cache.CacheConfig;
@@ -50,6 +58,7 @@
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.autoconfigure.domain.EntityScan;
 import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.ConfigurableApplicationContext;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.ComponentScan;
 import org.springframework.context.annotation.Scope;
@@ -155,7 +164,6 @@ public TypeDefinition load(String typeIdentifier) throws IOException, URISyntaxE
                 });
     }
 
-    @Bean
     @ConfigurationProperties("pit")
     public ApplicationProperties applicationProperties() {
         return new ApplicationProperties();
@@ -173,8 +181,87 @@ public HttpMessageConverter<PIDRecord> simplePidRecordConverter() {
     }
 
     public static void main(String[] args) {
-        SpringApplication.run(Application.class, args);
+        ConfigurableApplicationContext context = SpringApplication.run(Application.class, args);
         System.out.println("Spring is running!");
+
+        final boolean cliArgsGiven = args != null && args.length != 0;
+        final boolean cliArgsAmountValid = cliArgsGiven && args.length >= 2;
+
+        final String writeFileCmd = "write-file";
+        final String bootstrapCmd = "bootstrap";
+
+        final String sourceFromPrefix = "all-pids-from-prefix";
+        final String sourceKnownPids = "known-pids";
+
+        final String errorCommunication = "Communication error: {}";
+        final String errorConfiguration = "Configuration error: {}";
+
+        if (cliArgsAmountValid) {
+            ICliTask task = null;
+            Stream<String> pidSource = null;
+
+            if (Objects.equals(args[1], sourceFromPrefix)) {
+                try {
+                    pidSource = PidSource.fromPrefix(context);
+                } catch (IOException e) {
+                    e.printStackTrace();
+                    LOG.error(errorCommunication, e.getMessage());
+                    exitApp(context, 1);
+                }
+            } else if (Objects.equals(args[1], sourceKnownPids)) {
+                pidSource = PidSource.fromKnown(context);
+            }
+
+            if (Objects.equals(args[0], bootstrapCmd)) {
+                task = new CliTaskBootstrap(context, pidSource);
+            } else if (Objects.equals(args[0], writeFileCmd)) {
+                task = new CliTaskWriteFile(context, pidSource);
+            }
+
+            try {
+                if (task != null && pidSource != null) {
+                    // ---process task---
+                    if (task.process()) {
+                        exitApp(context, 0);
+                    }
+                } else {
+                    printUsage(args);
+                    exitApp(context, 1);
+                }
+            } catch (InvalidConfigException e) {
+                e.printStackTrace();
+                LOG.error(errorConfiguration, e.getMessage());
+                exitApp(context, 1);
+            } catch (IOException e) {
+                e.printStackTrace();
+                LOG.error(errorCommunication, e.getMessage());
+                exitApp(context, 1);
+            }
+        }
+    }
+
+    private static void printUsage(String[] args) {
+        LOG.error("Got commands: {} and {}", args[0], args[1]);
+        LOG.error("CLI usage incorrect. Usage:");
+        LOG.error("java -jar TypedPIDMaker.jar [ACTION] [SOURCE]");
+        LOG.error("java -jar TypedPIDMaker.jar bootstrap all-pids-from-prefix");
+        LOG.error("java -jar TypedPIDMaker.jar write-file all-pids-from-prefix");
+        LOG.error("java -jar TypedPIDMaker.jar write-file known-pids");
+    }
+
+    private static void exitApp(ConfigurableApplicationContext context, int errCode) {
+        context.close();
+        try {
+            Thread.sleep(2 * 1000L);
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+        }
+        if (errCode != 0) {
+            LOG.error("Exited with error.");
+        } else {
+            LOG.info("Success");
+        }
+        System.exit(errCode);
     }
 
 }

src/main/java/edu/kit/datamanager/pit/cli/CliTaskBootstrap.java

@@ -0,0 +1,59 @@
+package edu.kit.datamanager.pit.cli;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.stream.Stream;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.ConfigurableApplicationContext;
+
+import edu.kit.datamanager.entities.messaging.PidRecordMessage;
+import edu.kit.datamanager.pit.common.InvalidConfigException;
+import edu.kit.datamanager.pit.configuration.ApplicationProperties;
+import edu.kit.datamanager.pit.pidlog.KnownPid;
+import edu.kit.datamanager.pit.pidlog.KnownPidsDao;
+import edu.kit.datamanager.service.IMessagingService;
+import edu.kit.datamanager.util.AuthenticationHelper;
+import edu.kit.datamanager.util.ControllerUtils;
+
+public class CliTaskBootstrap implements ICliTask {
+
+    private static final Logger LOG = LoggerFactory.getLogger(CliTaskBootstrap.class);
+
+    Stream<String> pids;
+    ApplicationProperties appProps;
+    KnownPidsDao knownPids;
+    IMessagingService messagingService;
+
+    public CliTaskBootstrap(ConfigurableApplicationContext context, Stream<String> pids) {
+        this.pids = pids;
+        this.appProps = context.getBean(ApplicationProperties.class);
+        this.knownPids = context.getBean(KnownPidsDao.class);
+        this.messagingService = context.getBean(IMessagingService.class);
+    }
+
+    @Override
+    public boolean process() throws IOException, InvalidConfigException {
+        Instant unknownTime = Instant.ofEpochMilli(0);
+        pids
+            .map(pid -> new KnownPid(pid, unknownTime, unknownTime))
+            .forEach(known -> {
+                // store PIDs in the local database of known PIDs
+                LOG.info("Store PID {} in the local database of known PIDs.", known.getPid());
+                knownPids.save(known);
+                // send to message broker
+                PidRecordMessage message = PidRecordMessage.creation(
+                    known.getPid(),
+                    "",
+                    AuthenticationHelper.getPrincipal(),
+                    ControllerUtils.getLocalHostname()
+                );
+                LOG.info("Send PID {} to message broker.", known.getPid());
+                messagingService.send(message);
+            });
+        knownPids.flush();
+        return false;
+    }
+
+}

src/main/java/edu/kit/datamanager/pit/cli/CliTaskWriteFile.java

@@ -0,0 +1,58 @@
+package edu.kit.datamanager.pit.cli;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.util.Iterator;
+import java.util.stream.Stream;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.ConfigurableApplicationContext;
+
+import edu.kit.datamanager.pit.configuration.ApplicationProperties;
+
+public class CliTaskWriteFile implements ICliTask {
+
+    private static final Logger LOG = LoggerFactory.getLogger(CliTaskWriteFile.class);
+
+    Stream<String> pids;
+    ConfigurableApplicationContext context;
+    ApplicationProperties appProps;
+
+    public CliTaskWriteFile(ConfigurableApplicationContext context, Stream<String> pids) {
+        this.pids = pids;
+        this.context = context;
+        this.appProps = context.getBean(ApplicationProperties.class);
+    }
+
+    @Override
+    public boolean process() throws IOException {
+        String date = ZonedDateTime
+                .now(ZoneId.systemDefault())
+                .toString()
+                //.replace(":", ".")
+                .replace("[", "(")
+                .replace("]", ")")
+                .replace("/", "-")
+                //.replace("+", "_")
+                ;
+        String filename = String.format("%s.%s", date, "csv");
+        Path path = Paths.get(filename);
+        {
+            File f = path.toFile();
+            f.createNewFile();
+        }
+        for (Iterator<String> iter = pids.iterator(); iter.hasNext(); ) {
+            String pid = iter.next();
+            LOG.info("Storing into CSV: {}", pid);
+            Files.writeString(path, pid + "\n", StandardOpenOption.APPEND);
+        }
+        return true;
+    }
+}

src/main/java/edu/kit/datamanager/pit/cli/ICliTask.java

@@ -0,0 +1,15 @@
+package edu.kit.datamanager.pit.cli;
+
+import java.io.IOException;
+import edu.kit.datamanager.pit.common.InvalidConfigException;
+
+public interface ICliTask {
+    /**
+     * Processes this CLI task.
+     * 
+     * @return true, if application should shut down after this task.
+     * @throws IOException            on IO issues
+     * @throws InvalidConfigException on configuration issues
+     */
+    public boolean process() throws IOException, InvalidConfigException;
+}

src/main/java/edu/kit/datamanager/pit/cli/PidSource.java

@@ -0,0 +1,42 @@
+package edu.kit.datamanager.pit.cli;
+
+import java.io.IOException;
+import java.util.Map.Entry;
+import java.util.stream.Stream;
+
+import org.springframework.context.ConfigurableApplicationContext;
+
+import edu.kit.datamanager.pit.pidlog.KnownPidsDao;
+import edu.kit.datamanager.pit.pidsystem.IIdentifierSystem;
+
+/**
+ * All static methods are sources for PIDs. Used in Command Line Tasks.
+ */
+public class PidSource {
+
+    private PidSource() {
+        // This class contains only static methods and shall not be instanciated.
+    }
+
+    public static Stream<String> fromPrefix(ConfigurableApplicationContext context) throws IOException {
+        return context
+                .getBeansOfType(IIdentifierSystem.class)
+                .entrySet()
+                .stream()
+                .filter(e -> !e.getKey().contains("typing"))
+                .map(Entry<String, IIdentifierSystem>::getValue)
+                .findFirst()
+                .orElseThrow()
+                .resolveAllPidsOfPrefix()
+                .stream();
+    }
+
+    public static Stream<String> fromKnown(ConfigurableApplicationContext context) {
+        return context
+                .getBean(KnownPidsDao.class)
+                .findAll()
+                .stream()
+                .map(e -> e.getPid());
+    }
+
+}