-
Notifications
You must be signed in to change notification settings - Fork 4
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #127 from kit-data-manager/add-prefix-harvesting-cli
Add prefix harvesting & bootstrap CLI
- Loading branch information
Showing
21 changed files
with
875 additions
and
188 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
91 changes: 91 additions & 0 deletions
91
src/main/java/edu/kit/datamanager/pit/cli/CliTaskBootstrap.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,91 @@ | ||
| package edu.kit.datamanager.pit.cli; | ||
|
|
||
| import java.io.IOException; | ||
| import java.time.Instant; | ||
| import java.util.Optional; | ||
| import java.util.stream.Stream; | ||
|
|
||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
| import org.springframework.beans.factory.NoSuchBeanDefinitionException; | ||
| import org.springframework.context.ConfigurableApplicationContext; | ||
|
|
||
| import edu.kit.datamanager.entities.messaging.PidRecordMessage; | ||
| import edu.kit.datamanager.pit.common.InvalidConfigException; | ||
| import edu.kit.datamanager.pit.configuration.ApplicationProperties; | ||
| import edu.kit.datamanager.pit.domain.PIDRecord; | ||
| import edu.kit.datamanager.pit.elasticsearch.PidRecordElasticRepository; | ||
| import edu.kit.datamanager.pit.elasticsearch.PidRecordElasticWrapper; | ||
| import edu.kit.datamanager.pit.pidlog.KnownPid; | ||
| import edu.kit.datamanager.pit.pidlog.KnownPidsDao; | ||
| import edu.kit.datamanager.pit.pitservice.ITypingService; | ||
| import edu.kit.datamanager.service.IMessagingService; | ||
| import edu.kit.datamanager.util.AuthenticationHelper; | ||
| import edu.kit.datamanager.util.ControllerUtils; | ||
|
|
||
| public class CliTaskBootstrap implements ICliTask { | ||
|
|
||
| private static final Logger LOG = LoggerFactory.getLogger(CliTaskBootstrap.class); | ||
|
|
||
| protected Stream<String> pids; | ||
| protected ApplicationProperties appProps; | ||
| protected KnownPidsDao knownPids; | ||
| protected IMessagingService messagingService; | ||
| protected Optional<PidRecordElasticRepository> elastic; | ||
| protected ITypingService typingService; | ||
|
|
||
| public CliTaskBootstrap(ConfigurableApplicationContext context, Stream<String> pids) { | ||
| this.pids = pids; | ||
| this.appProps = context.getBean(ApplicationProperties.class); | ||
| this.knownPids = context.getBean(KnownPidsDao.class); | ||
| this.messagingService = context.getBean(IMessagingService.class); | ||
| this.elastic = this.getBeanOptional(context, PidRecordElasticRepository.class); | ||
| this.typingService = context.getBean(ITypingService.class); | ||
| } | ||
|
|
||
| private <T> Optional<T> getBeanOptional(ConfigurableApplicationContext context, Class<T> clazz) { | ||
| try { | ||
| return Optional.of(context.getBean(clazz)); | ||
| } catch (NoSuchBeanDefinitionException e) { | ||
| return Optional.empty(); | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public boolean process() throws IOException, InvalidConfigException { | ||
| Instant unknownTime = Instant.ofEpochMilli(0); | ||
| pids | ||
| .map(pid -> new KnownPid(pid, unknownTime, unknownTime)) | ||
| .forEach(known -> { | ||
| // store PIDs in the local database of known PIDs | ||
| LOG.info("Store PID {} in the local database of known PIDs.", known.getPid()); | ||
| boolean exists = knownPids.findByPid(known.getPid()).isPresent(); | ||
| if (!exists) { | ||
| knownPids.save(known); | ||
| } | ||
| // send to message broker | ||
| PidRecordMessage message = PidRecordMessage.creation( | ||
| known.getPid(), | ||
| "", | ||
| AuthenticationHelper.getPrincipal(), | ||
| ControllerUtils.getLocalHostname() | ||
| ); | ||
| LOG.info("Send PID {} to message broker.", known.getPid()); | ||
| messagingService.send(message); | ||
| // store in Elasticsearch | ||
| elastic.ifPresent(elastic -> { | ||
| try { | ||
| PIDRecord rec = typingService.queryAllProperties(known.getPid()); | ||
| LOG.info("Store PID {} in Elasticsearch.", known.getPid()); | ||
| PidRecordElasticWrapper wrapper = new PidRecordElasticWrapper(rec, typingService.getOperations()); | ||
| elastic.save(wrapper); | ||
| } catch (IOException e) { | ||
| LOG.error("Failed to query PID {}.", known.getPid(), e); | ||
| } | ||
| }); | ||
| }); | ||
| knownPids.flush(); | ||
| return false; | ||
| } | ||
|
|
||
| } |
62 changes: 62 additions & 0 deletions
62
src/main/java/edu/kit/datamanager/pit/cli/CliTaskWriteFile.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,62 @@ | ||
| package edu.kit.datamanager.pit.cli; | ||
|
|
||
| import java.io.File; | ||
| import java.io.IOException; | ||
| import java.nio.file.Files; | ||
| import java.nio.file.Path; | ||
| import java.nio.file.Paths; | ||
| import java.nio.file.StandardOpenOption; | ||
| import java.time.ZoneId; | ||
| import java.time.ZonedDateTime; | ||
| import java.util.Iterator; | ||
| import java.util.stream.Stream; | ||
|
|
||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
|
|
||
| public class CliTaskWriteFile implements ICliTask { | ||
|
|
||
| private static final Logger LOG = LoggerFactory.getLogger(CliTaskWriteFile.class); | ||
|
|
||
| protected Stream<String> pids; | ||
|
|
||
| public CliTaskWriteFile(Stream<String> pids) { | ||
| this.pids = pids; | ||
| } | ||
|
|
||
| @Override | ||
| public boolean process() throws IOException { | ||
| String filename = createFilename(); | ||
|
|
||
| Path path = Paths.get(filename); | ||
| ensureFileExists(path); | ||
|
|
||
| writeToFile(path); | ||
| return true; | ||
| } | ||
|
|
||
| protected void writeToFile(Path path) throws IOException { | ||
| for (Iterator<String> iter = pids.iterator(); iter.hasNext(); ) { | ||
| String pid = iter.next(); | ||
| LOG.info("Storing into CSV: {}", pid); | ||
| Files.writeString(path, pid + "\n", StandardOpenOption.APPEND); | ||
| } | ||
| } | ||
|
|
||
| protected void ensureFileExists(Path path) throws IOException { | ||
| File f = path.toFile(); | ||
| f.createNewFile(); | ||
| } | ||
|
|
||
| protected String createFilename() { | ||
| String date = ZonedDateTime | ||
| .now(ZoneId.systemDefault()) | ||
| .toString() | ||
| .replace(":", ".") | ||
| .replace("[", "(") | ||
| .replace("]", ")") | ||
| .replace("/", "-") | ||
| .replace("+", "-"); | ||
| return String.format("%s.%s", date, "csv"); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| package edu.kit.datamanager.pit.cli; | ||
|
|
||
| import java.io.IOException; | ||
| import edu.kit.datamanager.pit.common.InvalidConfigException; | ||
|
|
||
| public interface ICliTask { | ||
| /** | ||
| * Processes this CLI task. | ||
| * | ||
| * @return true, if application should shut down after this task. | ||
| * @throws IOException on IO issues | ||
| * @throws InvalidConfigException on configuration issues | ||
| */ | ||
| public boolean process() throws IOException, InvalidConfigException; | ||
| } |
Oops, something went wrong.