From 76f73dd0f8a35573449da52bc0528b2f6cf30f9c Mon Sep 17 00:00:00 2001 From: Juan Cubeddu Date: Tue, 12 Sep 2023 14:15:38 -0500 Subject: [PATCH 01/16] Added server timeout config + fetch timeout --- .../node-server.js | 76 +++++++++++-------- 1 file changed, 43 insertions(+), 33 deletions(-) diff --git a/packages/graph-explorer-proxy-server/node-server.js b/packages/graph-explorer-proxy-server/node-server.js index e3904e64c..cef2e828c 100644 --- a/packages/graph-explorer-proxy-server/node-server.js +++ b/packages/graph-explorer-proxy-server/node-server.js @@ -12,6 +12,12 @@ const { fromNodeProviderChain } = require("@aws-sdk/credential-providers"); const aws4 = require("aws4"); dotenv.config({ path: "../graph-explorer/.env" }); +const milis = 5 * 60 * 1000; +// convert milis to miliseconds +const + +const proxyTimeout = process.env.PROXY_REQUEST_TIMEOUT || 5 * 60 * 1000; // 5 minutes in milliseconds +const refetchMaxRetries = process.env.PROXY_MAX_RETRIES || 1; const proxyLogger = pino({ level: process.env.LOG_LEVEL, @@ -79,7 +85,7 @@ const errorHandler = (error, request, response, next) => { ); } - const retryFetch = async (url, headers, retryDelay = 10000, refetchMaxRetries = 1) => { + const retryFetch = async (url, headers, retryDelay = 10000) => { // remove the existing host headers, we want ensure that we are passing the DB endpoint hostname. delete headers["host"]; if (headers["aws-neptune-region"]) { @@ -95,7 +101,10 @@ const errorHandler = (error, request, response, next) => { for (let i = 0; i < refetchMaxRetries; i++) { try { proxyLogger.debug("Fetching: " + url.href); - const res = await fetch(url.href, {headers}); + const res = await fetch(url.href, { + headers: headers, + signal: AbortSignal.timeout(proxyTimeout - 1000), // timing out the request before the proxy timeout + }); if (!res.ok) { const result = await res.json(); proxyLogger.error("!!Request failure!!"); @@ -196,44 +205,45 @@ const errorHandler = (error, request, response, next) => { } }); - app.get("/logger", (req, res, next) => { - let message; - let level; + app.get("/logger", (req, res) => { + let message; + let level; - try { - if (req.headers["level"] === undefined) { - throw new Error("No log level passed."); - } else { - level = req.headers["level"]; - } + try { + if (req.headers["level"] === undefined) { + throw new Error("No log level passed."); + } else { + level = req.headers["level"]; + } - if (req.headers["message"] === undefined) { - throw new Error("No log message passed."); - } else { - message = req.headers["message"].replaceAll("\\", ""); - } + if (req.headers["message"] === undefined) { + throw new Error("No log message passed."); + } else { + message = req.headers["message"].replaceAll("\\", ""); + } - if (level.toLowerCase() === "error") { - proxyLogger.error(message); - } else if (level.toLowerCase() === "warn") { - proxyLogger.warn(message); - } else if (level.toLowerCase() === "info") { - proxyLogger.info(message); - } else if (level.toLowerCase() === "debug") { - proxyLogger.debug(message); - } else if (level.toLowerCase() === "trace") { - proxyLogger.trace(message); - } else { - throw new Error("Tried to log to an unknown level."); - } + if (level.toLowerCase() === "error") { + proxyLogger.error(message); + } else if (level.toLowerCase() === "warn") { + proxyLogger.warn(message); + } else if (level.toLowerCase() === "info") { + proxyLogger.info(message); + } else if (level.toLowerCase() === "debug") { + proxyLogger.debug(message); + } else if (level.toLowerCase() === "trace") { + proxyLogger.trace(message); + } else { + throw new Error("Tried to log to an unknown level."); + } - res.send("Log received."); - } catch (error) { - next(error); - } + res.send("Log received."); + } catch (error) { + next(error); + } }); app.use(errorHandler); + app.timeout = proxyTimeout; // Express default is 120000 (2 minutes) // Start the server on port 80 or 443 (if HTTPS is enabled) if (process.env.NEPTUNE_NOTEBOOK === "true") { From bffd8378a14adc2696f022d9f3098324ae8fc57d Mon Sep 17 00:00:00 2001 From: Juan Cubeddu Date: Tue, 12 Sep 2023 14:16:00 -0500 Subject: [PATCH 02/16] updated README --- README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 41d98a862..469d494a7 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,8 @@ To provide a default connection such that initial loads of the graph explorer al - `IAM` - `False` - See [Add a New Connection](#connections-ui) - `GRAPH_EXP_HTTPS_CONNECTION` - `True` - Controls whether the Graph Explorer uses SSL or not - `PROXY_SERVER_HTTPS_CONNECTION` - `True` - Controls whether the server uses SSL or not - - `GRAPH_EXP_FETCH_REQUEST_TIMEOUT` - `9000` - Controls the timeout for the fetch request + - `PROXY_SERVER_REQUEST_TIMEOUT` - `300000` - Controls the timeout for proxy requests + - `PROXY_SERVER_MAX_RETRIES` - `1` - Controls the number of retries for proxy requests - Conditionally Required: - Required if `USING_PROXY_SERVER=True` - `GRAPH_CONNECTION_URL` - `None` - See [Add a New Connection](#connections-ui) @@ -137,7 +138,8 @@ First, create a `config.json` file containing values for the connection attribut "GRAPH_TYPE": "gremlin" (Possible Values: "gremlin", "sparql", "opencypher"), "GRAPH_EXP_HTTPS_CONNECTION": true (Can be string or boolean), "PROXY_SERVER_HTTPS_CONNECTION": true, (Can be string or boolean), - "GRAPH_EXP_FETCH_REQUEST_TIMEOUT": 9000 (Can be number) + "PROXY_SERVER_REQUEST_TIMEOUT": 300000, + "PROXY_SERVER_MAX_RETRIES": 1 } ``` @@ -161,7 +163,8 @@ docker run -p 80:80 -p 443:443 \ --env GRAPH_CONNECTION_URL=https://cluster-cqmizgqgrsbf.us-west-2.neptune.amazonaws.com:8182 \ --env AWS_REGION=us-west-2 \ --env PROXY_SERVER_HTTPS_CONNECTION=true \ - --env GRAPH_EXP_FETCH_REQUEST_TIMEOUT=9000 \ + --env PROXY_SERVER_REQUEST_TIMEOUT=300000 \ + --env PROXY_SERVER_MAX_RETRIES=1 \ graph-explorer ``` From b463e330771ffedc59b3cfa90dbd9ceda8b0d1a4 Mon Sep 17 00:00:00 2001 From: Juan Cubeddu Date: Tue, 12 Sep 2023 14:16:46 -0500 Subject: [PATCH 03/16] env to configure proxy server timeout and retry --- packages/graph-explorer-proxy-server/node-server.js | 5 +---- packages/graph-explorer/.env | 4 ++-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/packages/graph-explorer-proxy-server/node-server.js b/packages/graph-explorer-proxy-server/node-server.js index cef2e828c..dd4366975 100644 --- a/packages/graph-explorer-proxy-server/node-server.js +++ b/packages/graph-explorer-proxy-server/node-server.js @@ -12,9 +12,6 @@ const { fromNodeProviderChain } = require("@aws-sdk/credential-providers"); const aws4 = require("aws4"); dotenv.config({ path: "../graph-explorer/.env" }); -const milis = 5 * 60 * 1000; -// convert milis to miliseconds -const const proxyTimeout = process.env.PROXY_REQUEST_TIMEOUT || 5 * 60 * 1000; // 5 minutes in milliseconds const refetchMaxRetries = process.env.PROXY_MAX_RETRIES || 1; @@ -103,7 +100,7 @@ const errorHandler = (error, request, response, next) => { proxyLogger.debug("Fetching: " + url.href); const res = await fetch(url.href, { headers: headers, - signal: AbortSignal.timeout(proxyTimeout - 1000), // timing out the request before the proxy timeout + timeout: proxyTimeout - 1000, }); if (!res.ok) { const result = await res.json(); diff --git a/packages/graph-explorer/.env b/packages/graph-explorer/.env index 0c9e59631..fe87ea757 100644 --- a/packages/graph-explorer/.env +++ b/packages/graph-explorer/.env @@ -1,4 +1,4 @@ ## Server and client side logging level LOG_LEVEL=info -## Client side fetch request timeout -GRAPH_EXP_FETCH_REQUEST_TIMEOUT=9000 +PROXY_REQUEST_TIMEOUT=300000 +PROXY_MAX_RETRIES=1 From 58efcf257dc398afa8bd3ef374363b788f9e78c0 Mon Sep 17 00:00:00 2001 From: Juan Cubeddu Date: Tue, 12 Sep 2023 14:22:30 -0500 Subject: [PATCH 04/16] use the new variable names --- packages/graph-explorer/.env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/graph-explorer/.env b/packages/graph-explorer/.env index fe87ea757..3ceb65b88 100644 --- a/packages/graph-explorer/.env +++ b/packages/graph-explorer/.env @@ -1,4 +1,4 @@ ## Server and client side logging level LOG_LEVEL=info -PROXY_REQUEST_TIMEOUT=300000 -PROXY_MAX_RETRIES=1 +PROXY_SERVER_REQUEST_TIMEOUT=300000 +PROXY_SERVER_MAX_RETRIES=1 From 4863c90562cc2977234cc49866f0621f8bd58f22 Mon Sep 17 00:00:00 2001 From: Juan Cubeddu Date: Fri, 10 Nov 2023 10:59:46 -0600 Subject: [PATCH 05/16] removed RequestSig class and use IAMService environment variable in node-server.js. --- .../graph-explorer-proxy-server/RequestSig.js | 27 ------------------- .../node-server.js | 5 ++-- 2 files changed, 3 insertions(+), 29 deletions(-) delete mode 100644 packages/graph-explorer-proxy-server/RequestSig.js diff --git a/packages/graph-explorer-proxy-server/RequestSig.js b/packages/graph-explorer-proxy-server/RequestSig.js deleted file mode 100644 index 38c2ddddc..000000000 --- a/packages/graph-explorer-proxy-server/RequestSig.js +++ /dev/null @@ -1,27 +0,0 @@ -const aws4 = require("aws4"); - -class RequestSig { - _algorithm = "AWS4-HMAC-SHA256"; - _region; - _service = "neptune-db"; - _host; - _ac; - _sac; - - constructor(host, region, accessKey, secretAccessKey, sessionToken) { - this._region = region; - this._host = host; - this._ac = accessKey; - this._sac = secretAccessKey; - this._st = sessionToken; - } - - requestAuthHeaders(inputPort, requestedPath) { - var opts = { host: this._host.split(":")[0], path: requestedPath, service: this._service, region: this._region, port: inputPort }; - return aws4.sign(opts, {accessKeyId: this._ac, secretAccessKey: this._sac, sessionToken: this._st}); - } - } - - module.exports = { - RequestSig - }; \ No newline at end of file diff --git a/packages/graph-explorer-proxy-server/node-server.js b/packages/graph-explorer-proxy-server/node-server.js index dd4366975..d2df1451e 100644 --- a/packages/graph-explorer-proxy-server/node-server.js +++ b/packages/graph-explorer-proxy-server/node-server.js @@ -15,6 +15,7 @@ dotenv.config({ path: "../graph-explorer/.env" }); const proxyTimeout = process.env.PROXY_REQUEST_TIMEOUT || 5 * 60 * 1000; // 5 minutes in milliseconds const refetchMaxRetries = process.env.PROXY_MAX_RETRIES || 1; +const IAMService = process.env.IAM_SERVICE || "neptune-db"; const proxyLogger = pino({ level: process.env.LOG_LEVEL, @@ -90,7 +91,7 @@ const errorHandler = (error, request, response, next) => { host: url.hostname, port: url.port, path: url.pathname + url.search, - service: "neptune-db", + service: IAMService, region: headers["aws-neptune-region"], }); headers = { ...headers, ...data }; @@ -237,7 +238,7 @@ const errorHandler = (error, request, response, next) => { } catch (error) { next(error); } -}); + }); app.use(errorHandler); app.timeout = proxyTimeout; // Express default is 120000 (2 minutes) From 3bacb01c1f422f30f9bc664d44f02a32eabb2cec Mon Sep 17 00:00:00 2001 From: Juan Cubeddu Date: Fri, 10 Nov 2023 11:19:46 -0600 Subject: [PATCH 06/16] Add Neptune graph permissions and update IAM service type --- .../sagemaker/graph-explorer-policy.json | 7 +++ .../node-server.js | 4 +- .../src/core/ConfigurationProvider/types.ts | 4 ++ .../CreateConnection/CreateConnection.tsx | 49 +++++++++++++------ 4 files changed, 46 insertions(+), 18 deletions(-) diff --git a/additionaldocs/sagemaker/graph-explorer-policy.json b/additionaldocs/sagemaker/graph-explorer-policy.json index d5f21b2d8..a95d7b080 100644 --- a/additionaldocs/sagemaker/graph-explorer-policy.json +++ b/additionaldocs/sagemaker/graph-explorer-policy.json @@ -8,6 +8,13 @@ "arn:aws:neptune-db:[AWS_REGION]:[AWS_ACCOUNT_ID]:[NEPTUNE_CLUSTER_RESOURCE_ID]/*" ] }, + { + "Effect": "Allow", + "Action": "neptune-graph:*", + "Resource": [ + "arn:aws:neptune-graph:[AWS_REGION]:[AWS_ACCOUNT_ID]:[NEPTUNE_CLUSTER_RESOURCE_ID]/*" + ] + }, { "Effect": "Allow", "Action": "sagemaker:DescribeNotebookInstance", diff --git a/packages/graph-explorer-proxy-server/node-server.js b/packages/graph-explorer-proxy-server/node-server.js index d2df1451e..5c2d8ec1c 100644 --- a/packages/graph-explorer-proxy-server/node-server.js +++ b/packages/graph-explorer-proxy-server/node-server.js @@ -15,7 +15,7 @@ dotenv.config({ path: "../graph-explorer/.env" }); const proxyTimeout = process.env.PROXY_REQUEST_TIMEOUT || 5 * 60 * 1000; // 5 minutes in milliseconds const refetchMaxRetries = process.env.PROXY_MAX_RETRIES || 1; -const IAMService = process.env.IAM_SERVICE || "neptune-db"; +const serviceType = process.env.SERVICE_TYPE || "neptune-db"; const proxyLogger = pino({ level: process.env.LOG_LEVEL, @@ -91,7 +91,7 @@ const errorHandler = (error, request, response, next) => { host: url.hostname, port: url.port, path: url.pathname + url.search, - service: IAMService, + service: serviceType, region: headers["aws-neptune-region"], }); headers = { ...headers, ...data }; diff --git a/packages/graph-explorer/src/core/ConfigurationProvider/types.ts b/packages/graph-explorer/src/core/ConfigurationProvider/types.ts index ecfa22cf8..dcc1541c2 100644 --- a/packages/graph-explorer/src/core/ConfigurationProvider/types.ts +++ b/packages/graph-explorer/src/core/ConfigurationProvider/types.ts @@ -131,6 +131,10 @@ export type ConnectionConfig = { * If it is Neptune, it could need authentication. */ awsAuthEnabled?: boolean; + /** + * If it is Neptune, it could need authentication. + */ + serviceType: 'neptune-db' | 'neptune-graph', /** * AWS Region where the Neptune cluster is deployed. * It is needed to sign requests. diff --git a/packages/graph-explorer/src/modules/CreateConnection/CreateConnection.tsx b/packages/graph-explorer/src/modules/CreateConnection/CreateConnection.tsx index d3ef6cef6..7ccffcbdf 100644 --- a/packages/graph-explorer/src/modules/CreateConnection/CreateConnection.tsx +++ b/packages/graph-explorer/src/modules/CreateConnection/CreateConnection.tsx @@ -29,6 +29,7 @@ type ConnectionForm = { proxyConnection?: boolean; graphDbUrl?: string; awsAuthEnabled?: boolean; + serviceType?: "neptune-db" | "neptune-graph"; awsRegion?: string; enableCache?: boolean; cacheTimeMs?: number; @@ -39,9 +40,9 @@ export const CONNECTIONS_OP: { label: string; value: NonNullable; }[] = [ - { label: "PG (Property Graph) - Gremlin", value: "gremlin" }, - { label: "PG (Property Graph) - OpenCypher", value: "openCypher" }, - { label: "RDF (Resource Description Framework) - SPARQL", value: "sparql" }, + { label: "Gremlin - PG (Property Graph)", value: "gremlin" }, + { label: "OpenCypher - PG (Property Graph)", value: "openCypher" }, + { label: "SPARQL - RDF (Resource Description Framework)", value: "sparql" }, ]; export type CreateConnectionProps = { @@ -74,6 +75,7 @@ const CreateConnection = ({ proxyConnection: data.proxyConnection, graphDbUrl: data.graphDbUrl, awsAuthEnabled: data.awsAuthEnabled, + serviceType: data.serviceType, awsRegion: data.awsRegion, enableCache: data.enableCache, cacheTimeMs: data.cacheTimeMs * 60 * 1000, @@ -103,6 +105,7 @@ const CreateConnection = ({ proxyConnection: data.proxyConnection, graphDbUrl: data.graphDbUrl, awsAuthEnabled: data.awsAuthEnabled, + serviceType: data.serviceType, awsRegion: data.awsRegion, cacheTimeMs: data.cacheTimeMs * 60 * 1000, fetchTimeoutMs: data.fetchTimeMs, @@ -144,6 +147,7 @@ const CreateConnection = ({ proxyConnection: initialData?.proxyConnection || false, graphDbUrl: initialData?.graphDbUrl || "", awsAuthEnabled: initialData?.awsAuthEnabled || false, + serviceType: initialData?.serviceType || "neptune-db", awsRegion: initialData?.awsRegion || "", enableCache: true, cacheTimeMs: (initialData?.cacheTimeMs ?? 10 * 60 * 1000) / 60000, @@ -271,19 +275,32 @@ const CreateConnection = ({ )} {form.proxyConnection && form.awsAuthEnabled && ( -
- -
+ <> +
+ +
+
+
From 200a1a992f19aeac940fc3805e27a0f8b9e700af Mon Sep 17 00:00:00 2001 From: Valentyn Kahamlyk Date: Thu, 29 Feb 2024 21:41:30 -0800 Subject: [PATCH 16/16] ... --- additionaldocs/sagemaker/install-graph-explorer-lc.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/additionaldocs/sagemaker/install-graph-explorer-lc.sh b/additionaldocs/sagemaker/install-graph-explorer-lc.sh index 2687a6b55..14d368818 100644 --- a/additionaldocs/sagemaker/install-graph-explorer-lc.sh +++ b/additionaldocs/sagemaker/install-graph-explorer-lc.sh @@ -4,6 +4,7 @@ sudo -u ec2-user -i <<'EOF' echo "export GRAPH_NOTEBOOK_AUTH_MODE=DEFAULT" >> ~/.bashrc # set to IAM instead of DEFAULT if cluster is IAM enabled echo "export GRAPH_NOTEBOOK_HOST=CHANGE-ME" >> ~/.bashrc +echo "export GRAPH_NOTEBOOK_SERVICE=neptune-db" >> ~/.bashrc # set to `neptune-db` for Neptune database or `neptune-graph` for Neptune Analytics echo "export GRAPH_NOTEBOOK_PORT=8182" >> ~/.bashrc echo "export AWS_REGION=us-west-2" >> ~/.bashrc # modify region if needed