From 1bf7ce4d76563f7b26382f6377d111ef5f5f186f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Nagygy=C3=B6rgy?= Date: Mon, 16 Oct 2023 14:15:35 +0200 Subject: [PATCH] feat: simplified node installation & docs --- README.md | 5 + web/backend/.env.example | 11 +- .../install-script/install-docker.ps1.hbr | 19 --- .../install-script/install-docker.sh.hbr | 133 +----------------- .../migration.sql | 10 +- .../prisma/migrations/migration_lock.toml | 3 + web/backend/prisma/schema.prisma | 9 +- web/backend/src/app/agent/agent.service.ts | 21 ++- .../src/app/agent/guards/agent.auth.guard.ts | 24 +++- web/backend/src/app/node/node.dto.ts | 3 - web/backend/src/app/node/node.mapper.ts | 6 +- web/backend/src/app/node/node.service.ts | 20 ++- web/backend/src/domain/agent-installer.ts | 24 ---- web/backend/src/domain/agent-token.ts | 6 + web/backend/src/domain/node.ts | 6 +- .../interceptors/prisma-error-interceptor.ts | 1 - .../src/shared/grpc-node-connection.ts | 15 +- .../src/components/nodes/dyo-node-setup.tsx | 32 ++--- web/frontend/src/models/node.ts | 1 - 19 files changed, 99 insertions(+), 250 deletions(-) rename web/backend/prisma/migrations/{20231002091837_init => 20231016114638_init}/migration.sql (75%) create mode 100644 web/backend/prisma/migrations/migration_lock.toml diff --git a/README.md b/README.md index 816fa40..6aeeaaa 100644 --- a/README.md +++ b/README.md @@ -56,9 +56,14 @@ Ensure data persistence by using Docker volumes to store Darklens data. 2. Enter a name for your node and click Save - This'll be the name of the agent running as a container on your node 3. On the right side of the screen, select whether you'd like to install the agent with a Shell (UNIX) or a PowerShell (Windows) script + +> Note: Shell scripts might not work on Windows using Git Bash. + 4. Generate the script and copy & paste it into Shell or PowerShell. Press enter to install the agent 5. When node status turns green, you're ready to use darklens +> Note: Agents connect to the host defined in their JWT token (`GRPC_TOKEN` environment), by default the `AGENT_ADDRESS` backend environment variable is used, but users can override this value on the UI. + ## Development 1. Setup backend diff --git a/web/backend/.env.example b/web/backend/.env.example index 12bf4c6..00c258e 100644 --- a/web/backend/.env.example +++ b/web/backend/.env.example @@ -1,16 +1,25 @@ NODE_ENV=development +# Database access path DATABASE=file:dev.db +# Domain used to sign and verify auth tokens PUBLIC_URL=http://localhost:8000 +# Disable authorization for the UI +DISABLE_AUTH=true + +# Port where the HTTP server is hosted HTTP_API_PORT=3100 +# Port where the gRPC server is hosted for agents GRPC_AGENT_PORT=5000 + +# The address used for the issuer of agent JWT tokens, this is the address where agents will connect to AGENT_ADDRESS=http://host.docker.internal:5000 +# Secret string used to sign JWT tokens JWT_SECRET=jwt-secret-token -DISABLE_AUTH=true # Possible values: trace, debug, info, warn, error, and fatal # The settings above come in a hierarchic order diff --git a/web/backend/assets/install-script/install-docker.ps1.hbr b/web/backend/assets/install-script/install-docker.ps1.hbr index bafb74b..8939e7d 100644 --- a/web/backend/assets/install-script/install-docker.ps1.hbr +++ b/web/backend/assets/install-script/install-docker.ps1.hbr @@ -1,25 +1,6 @@ -Set-PSDebug -strict -Set-StrictMode -version latest -$ErrorActionPreference = 'Stop' - -try { docker stop '{{name}}' } -catch { Write-Output "{{name}} couldn't be found or stopped" } -finally { - try { docker rm '{{name}}' } - catch { Write-Output "{{name}} couldn't be found or removed" } -} - -If ( -not ${{disableForcePull}} ) { - docker pull ghcr.io/dyrector-io/darklens/agent:{{agentImageTag}} -} - docker run ` --restart on-failure ` - {{#if network}} - --network {{networkName}} ` - {{/if}} -e GRPC_TOKEN='{{token}}' ` - -e HOST_DOCKER_SOCK_PATH=//var/run/docker.sock ` --add-host=host.docker.internal:host-gateway ` --name '{{name}}' ` -v //var/run/docker.sock:/var/run/docker.sock ` diff --git a/web/backend/assets/install-script/install-docker.sh.hbr b/web/backend/assets/install-script/install-docker.sh.hbr index 5e55831..3650bf7 100644 --- a/web/backend/assets/install-script/install-docker.sh.hbr +++ b/web/backend/assets/install-script/install-docker.sh.hbr @@ -1,126 +1,7 @@ -#!/bin/sh - -set -eu - -case $(uname) in - Linux*) - ROOTLESS="${ROOTLESS:-false}" - ;; - *) - ROOTLESS="${ROOTLESS:-true}" - ;; -esac - -if [ "$ROOTLESS" = "false" ]; then - if [ "$(id -u)" -ne 0 ]; then - echo "Installation process needs root privileges" 1>&2 - # This will quit the non-root script & restart the script as root - exec sudo -s "$0" - fi -fi - -case $(uname) in - Darwin*) - PLATFORM="OSX" - ;; - Linux*) - PLATFORM="LINUX" - ;; - MINGW*) - PLATFORM="WINDOWS" - if [ -n "${MSYS_NO_PATHCONV+x}" ]; then - ORIGINAL_PATHCONV_CONFIG=$MSYS_NO_PATHCONV - fi - - export MSYS_NO_PATHCONV=1 - HOST_DOCKER_SOCK_PATH="${HOST_DOCKER_SOCK_PATH:-//var/run/docker.sock}" - ;; - *) - echo "Not Supported OS" - exit 1 - ;; -esac - -set_environment() { - if [ -z "${CRI_EXECUTABLE:-}" ]; then - if ! command -v docker >/dev/null 2>&1; then - if ! command -v podman >/dev/null 2>&1; then - echo "Docker is required, make sure it is installed and available in PATH!" - exit 1 - else - CRI_EXECUTABLE="podman" - fi - else - CRI_EXECUTABLE="docker" - fi - fi - - if [ -z "${HOST_DOCKER_SOCK_PATH:-}" ]; then - if [ -z "${DOCKER_HOST:-}" ]; then - HOST_DOCKER_SOCK_PATH="/var/run/docker.sock" - else - if [ $(echo "$DOCKER_HOST" | cut -b -7) = "unix://" ]; then - HOST_DOCKER_SOCK_PATH=$(echo "$DOCKER_HOST" | cut -b 7-) - else - echo "Invalid DOCKER_HOST variable please set HOST_DOCKER_SOCK_PATH if your socket is in a custom location otherwise unset DOCKER_HOST!" - exit 1 - fi - fi - fi -} - -agent_clean() { - if [ -n "$($CRI_EXECUTABLE container list --filter name=^{{name}}$ --filter=status=running --filter=status=restarting --filter=status=paused --format '\{{ .Names }}' 2>/dev/null)" ]; then - set +e - echo "Stopping existing dyrector.io agent ({{name}})..." - $CRI_EXECUTABLE stop '{{name}}' - if ! $CRI_EXECUTABLE stop '{{name}}'; then - set -e - $CRI_EXECUTABLE kill '{{name}}' - fi - set -e - fi - - if [ -n "$($CRI_EXECUTABLE container list --filter name=^{{name}}$ --filter=status=exited --filter=status=created --filter=status=dead --format '\{{ .Names }}' 2>/dev/null)" ]; then - set +e - echo "Removing existing dyrector.io agent ({{name}})..." - $CRI_EXECUTABLE rm '{{name}}' - if $CRI_EXECUTABLE rm '{{name}}'; then - set -e - $CRI_EXECUTABLE rm -f '{{name}}' - fi - set -e - fi -} - -agent_install() { - echo "Installing Darklens Agent ({{name}})..." - - if ! {{disableForcePull}}; then - $CRI_EXECUTABLE pull ghcr.io/dyrector-io/darklens/agent:{{agentImageTag}} - fi - - $CRI_EXECUTABLE run \ - --restart on-failure \ - {{#if network}} - --network {{networkName}} \ - {{/if}} - -e GRPC_TOKEN='{{token}}' \ - -e HOST_DOCKER_SOCK_PATH="$HOST_DOCKER_SOCK_PATH" \ - --add-host=host.docker.internal:host-gateway \ - --name '{{name}}' \ - -v "$HOST_DOCKER_SOCK_PATH":/var/run/docker.sock \ - -d ghcr.io/dyrector-io/darklens/agent:{{agentImageTag}} - - if [ -z "${ORIGINAL_PATHCONV_CONFIG+x}" ]; then - unset MSYS_NO_PATHCONV - else - export MSYS_NO_PATHCONV="$ORIGINAL_PATHCONV_CONFIG" - fi -} - -set_environment - -agent_clean - -agent_install +docker run \ + --restart on-failure \ + -e GRPC_TOKEN='{{token}}' \ + --add-host=host.docker.internal:host-gateway \ + --name '{{name}}' \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -d ghcr.io/dyrector-io/darklens/agent:{{agentImageTag}} diff --git a/web/backend/prisma/migrations/20231002091837_init/migration.sql b/web/backend/prisma/migrations/20231016114638_init/migration.sql similarity index 75% rename from web/backend/prisma/migrations/20231002091837_init/migration.sql rename to web/backend/prisma/migrations/20231016114638_init/migration.sql index c5c13e9..8c9cab8 100644 --- a/web/backend/prisma/migrations/20231002091837_init/migration.sql +++ b/web/backend/prisma/migrations/20231016114638_init/migration.sql @@ -6,14 +6,8 @@ CREATE TABLE "Node" ( "icon" TEXT, "address" TEXT, "connectedAt" DATETIME, - "disconnectedAt" DATETIME -); - --- CreateTable -CREATE TABLE "NodeToken" ( - "nodeId" TEXT NOT NULL PRIMARY KEY, - "nonce" TEXT NOT NULL, - CONSTRAINT "NodeToken_nodeId_fkey" FOREIGN KEY ("nodeId") REFERENCES "Node" ("id") ON DELETE CASCADE ON UPDATE CASCADE + "disconnectedAt" DATETIME, + "tokenNonce" TEXT ); -- CreateTable diff --git a/web/backend/prisma/migrations/migration_lock.toml b/web/backend/prisma/migrations/migration_lock.toml new file mode 100644 index 0000000..e5e5c47 --- /dev/null +++ b/web/backend/prisma/migrations/migration_lock.toml @@ -0,0 +1,3 @@ +# Please do not edit this file manually +# It should be added in your version-control system (i.e. Git) +provider = "sqlite" \ No newline at end of file diff --git a/web/backend/prisma/schema.prisma b/web/backend/prisma/schema.prisma index 10b94ff..74c6b90 100644 --- a/web/backend/prisma/schema.prisma +++ b/web/backend/prisma/schema.prisma @@ -17,20 +17,13 @@ model Node { address String? connectedAt DateTime? disconnectedAt DateTime? + tokenNonce String? events NodeEvent[] - token NodeToken? @@unique([name]) } -model NodeToken { - nodeId String @id @unique - nonce String - - node Node @relation(fields: [nodeId], references: [id], onDelete: Cascade) -} - model NodeEvent { id String @id @default(uuid()) nodeId String diff --git a/web/backend/src/app/agent/agent.service.ts b/web/backend/src/app/agent/agent.service.ts index b79fd57..0d7b89a 100644 --- a/web/backend/src/app/agent/agent.service.ts +++ b/web/backend/src/app/agent/agent.service.ts @@ -99,23 +99,22 @@ export default class AgentService { scriptType, }) + await this.prisma.node.update({ + where: { + id: node.id, + }, + data: { + tokenNonce: token.nonce, + }, + }) + this.installers.set(node.id, installer) return installer } - async discardInstaller(nodeId: string): Promise { - if (!this.installers.has(nodeId)) { - throw new CruxNotFoundException({ - message: 'Installer not found', - property: 'installer', - value: nodeId, - }) - } - + discardInstaller(nodeId: string) { this.installers.delete(nodeId) - - return Empty } async completeInstaller(installer: AgentInstaller) { diff --git a/web/backend/src/app/agent/guards/agent.auth.guard.ts b/web/backend/src/app/agent/guards/agent.auth.guard.ts index 0de81d3..1731b7d 100644 --- a/web/backend/src/app/agent/guards/agent.auth.guard.ts +++ b/web/backend/src/app/agent/guards/agent.auth.guard.ts @@ -1,17 +1,35 @@ import { Metadata } from '@grpc/grpc-js' import { CanActivate, ExecutionContext, Injectable } from '@nestjs/common' import { JwtService } from '@nestjs/jwt' +import PrismaService from 'src/services/prisma.service' import GrpcNodeConnection, { NodeGrpcCall } from 'src/shared/grpc-node-connection' @Injectable() export default class AgentAuthGuard implements CanActivate { - constructor(private jwt: JwtService) {} + constructor( + private jwt: JwtService, + private prisma: PrismaService, + ) {} - canActivate(context: ExecutionContext): boolean { + async canActivate(context: ExecutionContext): Promise { const metadata = context.getArgByIndex(1) const call = context.getArgByIndex(2) const connection = new GrpcNodeConnection(metadata, call) - return connection.verify(this.jwt) + if (!connection.verify(this.jwt)) { + return false + } + + const node = await this.prisma.node.findFirst({ + where: { + id: connection.nodeId, + tokenNonce: connection.tokenNonce, + }, + }) + if (!node) { + return false + } + + return true } } diff --git a/web/backend/src/app/node/node.dto.ts b/web/backend/src/app/node/node.dto.ts index 2a21a68..180576e 100644 --- a/web/backend/src/app/node/node.dto.ts +++ b/web/backend/src/app/node/node.dto.ts @@ -65,9 +65,6 @@ export class NodeDto extends BasicNodeDto { } export class NodeInstallDto { - @IsString() - command: string - @IsString() script: string diff --git a/web/backend/src/app/node/node.mapper.ts b/web/backend/src/app/node/node.mapper.ts index 0f5bec3..1be0d50 100644 --- a/web/backend/src/app/node/node.mapper.ts +++ b/web/backend/src/app/node/node.mapper.ts @@ -2,7 +2,6 @@ import { Inject, Injectable, forwardRef } from '@nestjs/common' import { Node } from '@prisma/client' import { AgentConnectionMessage } from 'src/domain/agent' import AgentInstaller from 'src/domain/agent-installer' -import { NodeWithToken } from 'src/domain/node' import { fromTimestamp } from 'src/domain/utils' import { ContainerInspectMessage, @@ -69,14 +68,14 @@ export default class NodeMapper { } } - detailsToDto(node: NodeWithToken): NodeDetailsDto { + detailsToDto(node: Node): NodeDetailsDto { const installer = this.agentService.getInstallerByNodeId(node.id) const agent = this.agentService.getById(node.id) return { ...this.toDto(node), - hasToken: !!node.token, + hasToken: !!node.tokenNonce, install: installer ? this.installerToDto(installer) : null, updatable: agent && (agent.outdated || !this.agentService.agentVersionIsUpToDate(agent.version)), } @@ -84,7 +83,6 @@ export default class NodeMapper { installerToDto(installer: AgentInstaller): NodeInstallDto { return { - command: installer.getCommand(), script: installer.getScript(), expireAt: installer.expireAt, } diff --git a/web/backend/src/app/node/node.service.ts b/web/backend/src/app/node/node.service.ts index 54b684a..c18a5d3 100644 --- a/web/backend/src/app/node/node.service.ts +++ b/web/backend/src/app/node/node.service.ts @@ -68,9 +68,6 @@ export default class NodeService { where: { id, }, - include: { - token: true, - }, }) return this.mapper.detailsToDto(node) @@ -134,7 +131,16 @@ export default class NodeService { } async discardScript(id: string): Promise { - await this.agentService.discardInstaller(id) + this.agentService.discardInstaller(id) + + await this.prisma.node.update({ + where: { + id, + }, + data: { + tokenNonce: null, + }, + }) } async revokeToken(id: string): Promise { @@ -143,12 +149,12 @@ export default class NodeService { id, }, data: { - token: { - delete: true, - }, + tokenNonce: null, }, }) + this.agentService.discardInstaller(id) + await this.agentService.kick(id, 'revoke-token') } diff --git a/web/backend/src/domain/agent-installer.ts b/web/backend/src/domain/agent-installer.ts index 9189da2..af74018 100644 --- a/web/backend/src/domain/agent-installer.ts +++ b/web/backend/src/domain/agent-installer.ts @@ -5,7 +5,6 @@ import { join } from 'path' import { cwd } from 'process' import { CruxBadRequestException, - CruxInternalServerErrorException, CruxPreconditionFailedException, CruxUnauthorizedException, } from 'src/exception/crux-exception' @@ -45,34 +44,13 @@ export default class AgentInstaller { return now.getTime() - this.expirationDate.getTime() > JWT_EXPIRATION_MILLIS } - getCommand(): string { - const scriptUrl = `${this.configService.get('PUBLIC_URL')}/api/nodes/${this.node.id}/script` - - switch (this.options.scriptType) { - case 'shell': - return `curl -sL ${scriptUrl} | sh -` - case 'powershell': - return `Invoke-WebRequest -Uri ${scriptUrl} -Method GET | Select-Object -Expand Content | Invoke-Expression` - default: - throw new CruxInternalServerErrorException({ - message: 'Unknown script type', - property: 'scriptType', - value: this.options.scriptType, - }) - } - } - getScript(): string { - const configLocalDeployment = this.configService.get('LOCAL_DEPLOYMENT') - const configLocalDeploymentNetwork = this.configService.get('LOCAL_DEPLOYMENT_NETWORK') const disableForcePull = this.configService.get('AGENT_INSTALL_SCRIPT_DISABLE_PULL', false) const agentImageTag = this.configService.get('AGENT_IMAGE', getAgentVersionFromPackage(this.configService)) const installScriptParams: InstallScriptConfig = { name: this.node.name.toLowerCase().replace(/\s/g, ''), token: this.options.signedToken, - network: configLocalDeployment, - networkName: configLocalDeploymentNetwork, disableForcePull, agentImageTag, } @@ -138,8 +116,6 @@ export default class AgentInstaller { export type InstallScriptConfig = { name: string token: string - network: string - networkName: string agentImageTag: string disableForcePull?: boolean } diff --git a/web/backend/src/domain/agent-token.ts b/web/backend/src/domain/agent-token.ts index b6326f7..397dca3 100644 --- a/web/backend/src/domain/agent-token.ts +++ b/web/backend/src/domain/agent-token.ts @@ -1,3 +1,5 @@ +import { generateNonce } from './utils' + export type AgentLegacyToken = { sub: string iss: string @@ -8,16 +10,20 @@ export type AgentToken = { sub: string iss: string iat: number + nonce: string host?: string } export const generateAgentToken = (nodeId: string, host?: string): AgentToken => { const now = new Date().getTime() + const nonce = generateNonce() + const token: AgentToken = { iat: Math.floor(now / 1000), iss: undefined, // this gets filled by JwtService by the sign() call sub: nodeId, + nonce, host: host ?? undefined, } diff --git a/web/backend/src/domain/node.ts b/web/backend/src/domain/node.ts index 376131a..1c43734 100644 --- a/web/backend/src/domain/node.ts +++ b/web/backend/src/domain/node.ts @@ -1,9 +1,5 @@ -import { Node, NodeToken } from 'prisma/prisma-client' +import { Node } from 'prisma/prisma-client' export type BasicNode = Pick export type NodeScriptType = 'shell' | 'powershell' - -export type NodeWithToken = Node & { - token?: NodeToken -} diff --git a/web/backend/src/interceptors/prisma-error-interceptor.ts b/web/backend/src/interceptors/prisma-error-interceptor.ts index d6f019e..7401703 100644 --- a/web/backend/src/interceptors/prisma-error-interceptor.ts +++ b/web/backend/src/interceptors/prisma-error-interceptor.ts @@ -85,7 +85,6 @@ export default class PrismaErrorInterceptor implements NestInterceptor { private static readonly NOT_FOUND_ERRORS: NotFoundErrorMappings = { Node: 'node', NodeEvent: 'nodeEvent', - NodeToken: 'token', // its thrown on agent connections, so in that context this is a gRPC token User: 'user', } } diff --git a/web/backend/src/shared/grpc-node-connection.ts b/web/backend/src/shared/grpc-node-connection.ts index 1f00d09..512e9b8 100644 --- a/web/backend/src/shared/grpc-node-connection.ts +++ b/web/backend/src/shared/grpc-node-connection.ts @@ -16,10 +16,10 @@ export default class GrpcNodeConnection { private token: AgentToken - private signedToken: string + private jwtToken: string get jwt(): string { - return this.signedToken + return this.jwtToken } readonly address: string @@ -30,6 +30,10 @@ export default class GrpcNodeConnection { return this.token.sub } + get tokenNonce() { + return this.token.nonce + } + constructor( public readonly metadata: Metadata, private call: NodeGrpcCall, @@ -42,7 +46,7 @@ export default class GrpcNodeConnection { call.end = nestjsClientStreamEndCallWorkaround } - this.signedToken = this.getStringMetadataOrThrow(GrpcNodeConnection.META_NODE_TOKEN) + this.jwtToken = this.getStringMetadataOrThrow(GrpcNodeConnection.META_NODE_TOKEN) const xRealIp = this.getFirstItemOfStringArrayMetadata('x-real-ip') const xForwarderFor = this.getFirstItemOfStringArrayMetadata('x-forwarded-for') @@ -95,11 +99,6 @@ export default class GrpcNodeConnection { return value } - onTokenReplaced(token: AgentToken, signedToken: string) { - this.token = token - this.signedToken = signedToken - } - private onClose() { this.statusChannel.next('unreachable') diff --git a/web/frontend/src/components/nodes/dyo-node-setup.tsx b/web/frontend/src/components/nodes/dyo-node-setup.tsx index 85b2fc8..b8b121f 100644 --- a/web/frontend/src/components/nodes/dyo-node-setup.tsx +++ b/web/frontend/src/components/nodes/dyo-node-setup.tsx @@ -58,7 +58,7 @@ const DyoNodeSetup = (props: DyoNodeSetupProps) => { onNodeInstallChanged(null) } - const onCopyScript = () => writeToClipboard(t, node.install.command) + const onCopyScript = () => writeToClipboard(t, node.install.script) const formik = useDyoFormik({ initialValues: { @@ -140,24 +140,20 @@ const DyoNodeSetup = (props: DyoNodeSetupProps) => { ) : ( <>
- {t('command')} - -
- ev.target.select()} - /> - -
+ {t('script')} + +
+ + +
-
+
{t('scriptExpiresIn')} @@ -169,12 +165,6 @@ const DyoNodeSetup = (props: DyoNodeSetupProps) => { {t('common:discard')}
- -
- {t('script')} - - -
)} diff --git a/web/frontend/src/models/node.ts b/web/frontend/src/models/node.ts index 4004664..c140e02 100644 --- a/web/frontend/src/models/node.ts +++ b/web/frontend/src/models/node.ts @@ -34,7 +34,6 @@ export type DyoNode = NodeConnection & { } export type NodeInstall = { - command: string script: string expireAt: string }