Skip to content

Commit

Permalink
feat(cmd-api-server): fully dynamic plugin imports via config file/en…
Browse files Browse the repository at this point in the history
…v/CLI

It leverages the PluginRegistry to its full potential. Had to slightly refactor how the plugin
factories are exported from the plugin packages but it is a non-breaking change.

Also: decoupled
the consortium web service plugins tests from the plugin itself (e.g. moved it to a separate
package) so that circular package dependencies are avoided where with cmd-api-server

Signed-off-by: Peter Somogyvari <peter.somogyvari@accenture.com>
  • Loading branch information
petermetz committed May 19, 2020
1 parent 2e19ba8 commit fe396c9
Show file tree
Hide file tree
Showing 23 changed files with 2,822 additions and 47 deletions.
51 changes: 23 additions & 28 deletions packages/cactus-cmd-api-server/package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 4 additions & 2 deletions packages/cactus-cmd-api-server/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -58,9 +58,11 @@
"@hyperledger/cactus-core-api": "0.2.0",
"@hyperledger/cactus-plugin-keychain-memory": "0.2.0",
"@hyperledger/cactus-plugin-kv-storage-memory": "0.2.0",
"@hyperledger/cactus-plugin-web-service-consortium": "0.2.0",
"body-parser": "1.19.0",
"compression": "1.7.4",
"convict": "5.2.0",
"convict": "6.0.0",
"convict-format-with-validator": "6.0.0",
"cors": "2.8.5",
"express": "4.17.1",
"express-openapi-validator": "3.10.0",
Expand All @@ -74,7 +76,7 @@
},
"devDependencies": {
"@types/compression": "1.7.0",
"@types/convict": "5.2.0",
"@types/convict": "5.2.1",
"@types/cors": "2.8.6",
"@types/express": "4.17.6",
"@types/joi": "14.3.4",
Expand Down
27 changes: 14 additions & 13 deletions packages/cactus-cmd-api-server/src/main/typescript/api-server.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
import path from "path";
import { Server } from "http";
import { Config } from "convict";
import express, {
Express,
Request,
Expand All @@ -13,10 +12,8 @@ import compression from "compression";
import bodyParser from "body-parser";
import cors, { CorsOptions } from "cors";
import {
IPluginKVStorage,
PluginFactory,
ICactusPlugin,
PluginAspect,
isIPluginWebService,
IPluginWebService,
PluginRegistry,
Expand Down Expand Up @@ -73,8 +70,10 @@ export class ApiServer {
public async getOrInitPluginRegistry(): Promise<PluginRegistry> {
if (!this.pluginRegistry) {
if (!this.options.pluginRegistry) {
this.log.info(`getOrInitPluginRegistry() initializing a new one...`);
this.pluginRegistry = await this.initPluginRegistry();
} else {
this.log.info(`getOrInitPluginRegistry() re-using injected one...`);
this.pluginRegistry = this.options.pluginRegistry;
}
}
Expand All @@ -83,16 +82,18 @@ export class ApiServer {

public async initPluginRegistry(): Promise<PluginRegistry> {
const registry = new PluginRegistry({ plugins: [] });
// FIXME load the plugins here:

{
const storagePluginPackage = this.options.config.storagePluginPackage;
const { PluginFactoryKVStorage } = await import(storagePluginPackage);
const storagePluginOptionsJson = this.options.config
.storagePluginOptionsJson;
const storagePluginOptions = JSON.parse(storagePluginOptionsJson);
const pluginFactory = new PluginFactoryKVStorage();
const plugin = await pluginFactory.create(storagePluginOptions);

this.log.info(`Instantiated empty registry, invoking plugin factories...`);
for (const pluginImport of this.options.config.plugins) {
const { packageName, options } = pluginImport;
this.log.info(`Creating plugin from package: ${packageName}`, options);
const pluginOptions = { ...options, pluginRegistry: registry };
const { createPluginFactory } = await import(packageName);
const pluginFactory: PluginFactory<
ICactusPlugin,
any
> = await createPluginFactory();
const plugin = await pluginFactory.create(pluginOptions);
registry.add(plugin);
}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,12 +1,23 @@
import { randomBytes } from "crypto";
import convict, { Schema, Config, SchemaObj } from "convict";
import { ipaddress } from "convict-format-with-validator";
import secp256k1 from "secp256k1";
import { v4 as uuidV4 } from "uuid";
import {
LoggerProvider,
Logger,
LogLevelDesc,
} from "@hyperledger/cactus-common";
import { PluginAspect } from "@hyperledger/cactus-core-api";
import { FORMAT_PLUGIN_ARRAY } from "./convict-plugin-array-format";

convict.addFormat(FORMAT_PLUGIN_ARRAY);
convict.addFormat(ipaddress);

export interface IPluginImport {
packageName: string;
options?: any;
}

export interface ICactusApiServerOptions {
configFile: string;
Expand All @@ -18,6 +29,7 @@ export interface ICactusApiServerOptions {
apiHost: string;
apiPort: number;
apiCorsDomainCsv: string;
plugins: IPluginImport[];
storagePluginPackage: string;
storagePluginOptionsJson: string;
keychainPluginPackage: string;
Expand Down Expand Up @@ -60,6 +72,18 @@ export class ConfigService {

private static getConfigSchema(): Schema<ICactusApiServerOptions> {
return {
plugins: {
doc: "A collection of plugins to load at runtime.",
format: "plugin-array",
default: [],
env: "PLUGINS",
arg: "plugins",
pluginSchema: {
aspect: "*",
packageName: "*",
options: {},
},
} as any,
configFile: {
doc:
"The path to a config file that holds the configuration itself which will be parsed and validated.",
Expand Down Expand Up @@ -272,6 +296,22 @@ export class ConfigService {
const publicKey = Buffer.from(publicKeyBytes).toString("hex");

return {
plugins: [
{
packageName: "@hyperledger/cactus-plugin-kv-storage-memory",
options: {},
},
{
packageName: "@hyperledger/cactus-plugin-keychain-memory",
options: {},
},
{
packageName: "@hyperledger/cactus-plugin-web-service-consortium",
options: {
privateKey: "some-fake-key",
},
},
],
configFile: ".config.json",
cactusNodeId: uuidV4(),
logLevel: "debug",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
import convict from "convict";

export const validate = (sources: any[], schema?: any) => {
if (!Array.isArray(sources)) {
throw new Error("must be of type Array");
}

for (const source of sources) {
convict(schema.pluginSchema).load(source).validate();
}
};

export const coerce = (value: string) => {
// CLI sends comman separated objects as a JSON string without the array square brackets
// ENV sends the proper array that is valid JSON so we have to detect and handle both cases
const isJsonArray = value.startsWith("[") && value.endsWith("]");
return isJsonArray ? JSON.parse(value) : JSON.parse(`[${value}]`);
};

export const FORMAT_PLUGIN_ARRAY = {
name: "plugin-array",
validate,
coerce,
};
Original file line number Diff line number Diff line change
@@ -1,2 +1,10 @@
export { PluginKeychainMemory } from "./plugin-keychain-memory";
export { PluginFactoryKeychain } from "./plugin-factory-keychain";

import { PluginFactoryKeychain } from "./plugin-factory-keychain";

export async function createPluginFactory(
options?: any
): Promise<PluginFactoryKeychain> {
return new PluginFactoryKeychain();
}
Original file line number Diff line number Diff line change
@@ -1,2 +1,10 @@
export { PluginKVStorageMemory } from "./plugin-kv-storage-memory";
export { PluginFactoryKVStorage } from "./plugin-factory-kv-storage";

import { PluginFactoryKVStorage } from "./plugin-factory-kv-storage";

export async function createPluginFactory(
options?: any
): Promise<PluginFactoryKVStorage> {
return new PluginFactoryKVStorage();
}
Original file line number Diff line number Diff line change
@@ -1,2 +1,10 @@
export { PluginLedgerConnectorBesu } from "./plugin-ledger-connector-besu";
export { PluginFactoryLedgerConnector } from "./plugin-factory-ledger-connector";

import { PluginFactoryLedgerConnector } from "./plugin-factory-ledger-connector";

export async function createPluginFactory(
options?: any
): Promise<PluginFactoryLedgerConnector> {
return new PluginFactoryLedgerConnector();
}
Original file line number Diff line number Diff line change
Expand Up @@ -6,3 +6,10 @@ export {
} from "./plugin-ledger-connector-quorum";
export { PluginFactoryLedgerConnector } from "./plugin-factory-ledger-connector";
export { Contract as Web3EthContract } from "web3-eth-contract";

import { PluginFactoryLedgerConnector } from "./plugin-factory-ledger-connector";
export async function createPluginFactory(
options?: any
): Promise<PluginFactoryLedgerConnector> {
return new PluginFactoryLedgerConnector();
}
1 change: 0 additions & 1 deletion packages/cactus-plugin-web-service-consortium/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,6 @@
"typescript-optional": "2.0.1"
},
"devDependencies": {
"@hyperledger/cactus-cmd-api-server": "0.2.0",
"@hyperledger/cactus-plugin-kv-storage-memory": "0.2.0",
"@hyperledger/cactus-sdk": "0.2.0",
"@types/express": "4.17.6",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,10 @@ export {
} from "./plugin-web-service-consortium";
export { PluginFactoryWebService } from "./plugin-factory-web-service-consortium";
export * from "./generated/openapi/typescript-axios/index";

import { PluginFactoryWebService } from "./plugin-factory-web-service-consortium";
export async function createPluginFactory(
options?: any
): Promise<PluginFactoryWebService> {
return new PluginFactoryWebService();
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
// tslint:disable-next-line: no-var-requires
const tap = require("tap");
import axios, { AxiosPromise, AxiosInstance, AxiosResponse } from "axios";
import axios, { AxiosResponse } from "axios";
import {
QuorumTestLedger,
IQuorumGenesisOptions,
Expand Down
14 changes: 14 additions & 0 deletions packages/cactus-test-plugin-web-service-consortium/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
# `@hyperledger/cactus-test-plugin-web-service-consortium`


## Usage

```
// TODO: DEMONSTRATE API
```

## FAQ

### **What is a dedicatd test package for?**

This is a dedicated test package meaning that it verifies the integration between two packages that are somehow dependent on each other and therefore these tests cannot be added properly in the child package due to circular dependency issues and it would not be fitting to add it in the parent because the child package's tests should not be held by the parent as a matter of principle.
Loading

0 comments on commit fe396c9

Please sign in to comment.