From ad670cd4714e0d356590d9587898bfc556ce35c8 Mon Sep 17 00:00:00 2001 From: Vlad Frangu Date: Sun, 6 Jul 2025 19:25:52 +0300 Subject: [PATCH 1/2] feat: input schema defaults in getInput --- package.json | 2 +- packages/apify/src/actor.ts | 48 +++++++++++++++- packages/apify/src/input-schemas.ts | 70 +++++++++++++++++++++++ test/e2e/runSdkTests.mjs | 2 +- test/e2e/sdk/actorInput/.actor/Dockerfile | 26 +++++++++ test/e2e/sdk/actorInput/.actor/actor.json | 19 ++++++ test/e2e/sdk/actorInput/package.json | 17 ++++++ test/e2e/sdk/actorInput/src/main.mjs | 11 ++++ test/e2e/sdk/actorInput/test.mjs | 44 ++++++++++++++ 9 files changed, 234 insertions(+), 5 deletions(-) create mode 100644 packages/apify/src/input-schemas.ts create mode 100644 test/e2e/sdk/actorInput/.actor/Dockerfile create mode 100644 test/e2e/sdk/actorInput/.actor/actor.json create mode 100644 test/e2e/sdk/actorInput/package.json create mode 100644 test/e2e/sdk/actorInput/src/main.mjs create mode 100644 test/e2e/sdk/actorInput/test.mjs diff --git a/package.json b/package.json index 681480fe20..72990006ef 100644 --- a/package.json +++ b/package.json @@ -61,7 +61,7 @@ "test/**/*": [ "eslint --fix" ], - "*": "prettier --write" + "*": "prettier --write --ignore-unknown" }, "devDependencies": { "@apify/consts": "^2.29.0", diff --git a/packages/apify/src/actor.ts b/packages/apify/src/actor.ts index 627cacaefb..644c4b4c59 100644 --- a/packages/apify/src/actor.ts +++ b/packages/apify/src/actor.ts @@ -47,6 +47,10 @@ import { addTimeoutToPromise } from '@apify/timeout'; import type { ChargeOptions, ChargeResult } from './charging.js'; import { ChargingManager } from './charging.js'; import { Configuration } from './configuration.js'; +import { + getDefaultsFromInputSchema, + readInputSchema, +} from './input-schemas.js'; import { KeyValueStore } from './key_value_store.js'; import { PlatformEventManager } from './platform_event_manager.js'; import type { ProxyConfigurationOptions } from './proxy_configuration.js'; @@ -1219,9 +1223,12 @@ export class Actor { const inputSecretsPrivateKeyPassphrase = this.config.get( 'inputSecretsPrivateKeyPassphrase', ); - const input = await this.getValue(this.config.get('inputKey')); + const rawInput = await this.getValue(this.config.get('inputKey')); + + let input = rawInput as T; + if ( - ow.isValid(input, ow.object.nonEmpty) && + ow.isValid(rawInput, ow.object.nonEmpty) && inputSecretsPrivateKeyFile && inputSecretsPrivateKeyPassphrase ) { @@ -1229,8 +1236,14 @@ export class Actor { key: Buffer.from(inputSecretsPrivateKeyFile, 'base64'), passphrase: inputSecretsPrivateKeyPassphrase, }); - return decryptInputSecrets({ input, privateKey }); + + input = decryptInputSecrets({ input: rawInput, privateKey }); + } + + if (ow.isValid(input, ow.object.nonEmpty) && !Buffer.isBuffer(input)) { + input = await this.insertDefaultsFromInputSchema(input); } + return input; } @@ -2273,4 +2286,33 @@ export class Actor { ].join('\n'), ); } + + private async insertDefaultsFromInputSchema( + input: T, + ): Promise { + // TODO: v0, move all this logic from here and apify-cli to input_schema module + + const env = this.getEnv(); + let inputSchema: Dictionary | undefined | null; + + // On platform, we can get the input schema from the build data + if (this.isAtHome() && env.actorBuildId) { + const buildData = await this.apifyClient + .build(env.actorBuildId) + .get(); + + inputSchema = buildData?.actorDefinition?.input; + } else { + // On local, we can get the input schema from the local config + inputSchema = readInputSchema(); + } + + if (!inputSchema) { + return input; + } + + const defaults = getDefaultsFromInputSchema(inputSchema); + + return { ...defaults, ...input }; + } } diff --git a/packages/apify/src/input-schemas.ts b/packages/apify/src/input-schemas.ts new file mode 100644 index 0000000000..b0874e6a52 --- /dev/null +++ b/packages/apify/src/input-schemas.ts @@ -0,0 +1,70 @@ +// TODO: v0, move all this logic from here and apify-cli to input_schema module + +import { existsSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import process from 'node:process'; + +import type { Dictionary } from '@crawlee/utils'; + +const DEFAULT_INPUT_SCHEMA_PATHS = [ + ['.actor', 'INPUT_SCHEMA.json'], + ['INPUT_SCHEMA.json'], + ['.actor', 'input_schema.json'], + ['input_schema.json'], +]; + +const ACTOR_SPECIFICATION_FOLDER = '.actor'; + +const LOCAL_CONFIG_NAME = 'actor.json'; + +function readJSONIfExists(path: string): Dictionary | null { + if (existsSync(path)) { + const content = readFileSync(path, 'utf8'); + return JSON.parse(content); + } + + return null; +} + +export const readInputSchema = async (): Promise => { + const localConfig = readJSONIfExists( + join(process.cwd(), ACTOR_SPECIFICATION_FOLDER, LOCAL_CONFIG_NAME), + ); + + if (typeof localConfig?.input === 'object') { + return localConfig.input; + } + + if (typeof localConfig?.input === 'string') { + const fullPath = join( + process.cwd(), + ACTOR_SPECIFICATION_FOLDER, + localConfig.input, + ); + + return readJSONIfExists(fullPath); + } + + for (const path of DEFAULT_INPUT_SCHEMA_PATHS) { + const fullPath = join(process.cwd(), ...path); + if (existsSync(fullPath)) { + return readJSONIfExists(fullPath); + } + } + + return null; +}; + +export const getDefaultsFromInputSchema = (inputSchema: any) => { + const defaults: Record = {}; + + for (const [key, fieldSchema] of Object.entries( + inputSchema.properties, + )) { + if (fieldSchema.default !== undefined) { + defaults[key] = fieldSchema.default; + } + } + + return defaults; +}; diff --git a/test/e2e/runSdkTests.mjs b/test/e2e/runSdkTests.mjs index 1c6fc2b965..7aec843091 100644 --- a/test/e2e/runSdkTests.mjs +++ b/test/e2e/runSdkTests.mjs @@ -8,7 +8,7 @@ import { isMainThread, Worker, workerData } from 'node:worker_threads'; import { ApifyClient } from 'apify-client'; import { ACTOR_SOURCE_TYPES } from '@apify/consts'; -import { log } from '@apify/log'; +import log from '@apify/log'; import { cryptoRandomObjectId } from '@apify/utilities'; const rootPath = dirname(fileURLToPath(import.meta.url)); diff --git a/test/e2e/sdk/actorInput/.actor/Dockerfile b/test/e2e/sdk/actorInput/.actor/Dockerfile new file mode 100644 index 0000000000..6418c61e77 --- /dev/null +++ b/test/e2e/sdk/actorInput/.actor/Dockerfile @@ -0,0 +1,26 @@ +FROM node:22 AS builder + +COPY /package*.json ./ +RUN npm --quiet set progress=false \ + && npm install --only=prod --no-optional --no-audit \ + && npm update + +COPY /apify.tgz /apify.tgz +RUN npm --quiet install /apify.tgz + +FROM apify/actor-node:22 + +RUN rm -r node_modules +COPY --from=builder /node_modules ./node_modules +COPY --from=builder /package*.json ./ +COPY /.actor ./.actor +COPY /src ./src + +RUN echo "Installed NPM packages:" \ + && (npm list --only=prod --no-optional --all || true) \ + && echo "Node.js version:" \ + && node --version \ + && echo "NPM version:" \ + && npm --version + +CMD npm start --silent diff --git a/test/e2e/sdk/actorInput/.actor/actor.json b/test/e2e/sdk/actorInput/.actor/actor.json new file mode 100644 index 0000000000..4db86f9b86 --- /dev/null +++ b/test/e2e/sdk/actorInput/.actor/actor.json @@ -0,0 +1,19 @@ +{ + "actorSpecification": 1, + "name": "apify-sdk-js-test-input", + "version": "0.0", + "input": { + "title": "Actor Input", + "description": "Test input", + "type": "object", + "schemaVersion": 1, + "properties": { + "foo": { + "title": "Foo", + "type": "string", + "description": "Foo", + "default": "bar" + } + } + } +} diff --git a/test/e2e/sdk/actorInput/package.json b/test/e2e/sdk/actorInput/package.json new file mode 100644 index 0000000000..945dc75da6 --- /dev/null +++ b/test/e2e/sdk/actorInput/package.json @@ -0,0 +1,17 @@ +{ + "name": "apify-sdk-js-test-harness", + "version": "0.0.1", + "type": "module", + "description": "This is an example of an Apify actor.", + "engines": { + "node": ">=22.0.0" + }, + "dependencies": { + "apify": "*" + }, + "scripts": { + "start": "node ./src/main.mjs" + }, + "author": "It's not you it's me", + "license": "ISC" +} diff --git a/test/e2e/sdk/actorInput/src/main.mjs b/test/e2e/sdk/actorInput/src/main.mjs new file mode 100644 index 0000000000..80c28f7d1d --- /dev/null +++ b/test/e2e/sdk/actorInput/src/main.mjs @@ -0,0 +1,11 @@ +import { Actor, log } from 'apify'; + +await Actor.init(); + +const input = await Actor.getInput(); + +log.info(`Input: ${JSON.stringify(input)}`); + +await Actor.setValue('RECEIVED_INPUT', input); + +await Actor.exit(); diff --git a/test/e2e/sdk/actorInput/test.mjs b/test/e2e/sdk/actorInput/test.mjs new file mode 100644 index 0000000000..cdee0fb44e --- /dev/null +++ b/test/e2e/sdk/actorInput/test.mjs @@ -0,0 +1,44 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { ApifyClient, KeyValueStore } from 'apify'; +import { sleep } from 'crawlee'; + +const client = new ApifyClient({ + token: process.env.APIFY_TOKEN, +}); + +const actor = client.actor(process.argv[2]); + +const runActor = async (input, options) => { + const { id: runId } = await actor.call(input, options); + await client.run(runId).waitForFinish(); + await sleep(6000); // wait for updates to propagate to MongoDB + return await client.run(runId).get(); +}; + +test('defaults work', async () => { + const run = await runActor({}, {}); + + assert.strictEqual(run.status, 'SUCCEEDED'); + + const store = await KeyValueStore.open(run.defaultKeyValueStoreId, { + storageClient: client, + }); + + const receivedInput = await store.getValue('RECEIVED_INPUT'); + assert.deepEqual(receivedInput, { foo: 'bar' }); +}); + +test('input is passed through', async () => { + const run = await runActor({ foo: 'baz' }, {}); + + assert.strictEqual(run.status, 'SUCCEEDED'); + + const store = await KeyValueStore.open(run.defaultKeyValueStoreId, { + storageClient: client, + }); + + const receivedInput = await store.getValue('RECEIVED_INPUT'); + assert.deepEqual(receivedInput, { foo: 'baz' }); +}); From ad45ea90f7810b5bc283f24982c776b0d3739e4e Mon Sep 17 00:00:00 2001 From: Vlad Frangu Date: Sun, 6 Jul 2025 19:33:50 +0300 Subject: [PATCH 2/2] chore: oops --- packages/apify/src/input-schemas.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apify/src/input-schemas.ts b/packages/apify/src/input-schemas.ts index b0874e6a52..26bee37db6 100644 --- a/packages/apify/src/input-schemas.ts +++ b/packages/apify/src/input-schemas.ts @@ -26,7 +26,7 @@ function readJSONIfExists(path: string): Dictionary | null { return null; } -export const readInputSchema = async (): Promise => { +export const readInputSchema = (): Dictionary | null => { const localConfig = readJSONIfExists( join(process.cwd(), ACTOR_SPECIFICATION_FOLDER, LOCAL_CONFIG_NAME), );