From 096ea5bd179762b38c9ee4f9e1918fff7bfdc43e Mon Sep 17 00:00:00 2001 From: MQ37 Date: Tue, 23 Jun 2026 16:06:29 +0200 Subject: [PATCH 01/38] feat: per-run workerd sandbox that runs MCP Code Mode programs --- .actor/actor.json | 38 ++++++ .dockerignore | 4 + .gitignore | 4 + Dockerfile | 31 +++++ README.md | 75 ++++++++++++ package.json | 13 ++ pnpm-lock.yaml | 75 ++++++++++++ worker/config.capnp | 39 ++++++ worker/entrypoint.sh | 54 ++++++++ worker/guard.js | 45 +++++++ worker/runner.js | 286 +++++++++++++++++++++++++++++++++++++++++++ 11 files changed, 664 insertions(+) create mode 100644 .actor/actor.json create mode 100644 .dockerignore create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 package.json create mode 100644 pnpm-lock.yaml create mode 100644 worker/config.capnp create mode 100755 worker/entrypoint.sh create mode 100644 worker/guard.js create mode 100644 worker/runner.js diff --git a/.actor/actor.json b/.actor/actor.json new file mode 100644 index 0000000..3b0434e --- /dev/null +++ b/.actor/actor.json @@ -0,0 +1,38 @@ +{ + "actorSpecification": 1, + "name": "code-runtime", + "title": "Code Runtime", + "description": "Runs an LLM-submitted TypeScript/JavaScript program in a sandboxed workerd V8 isolate with Apify bindings. One program per run; the captured { stdout, stderr } is pushed as a single item to the default dataset.", + "version": "0.1", + "buildTag": "latest", + "usesStandbyMode": false, + "input": { + "title": "Code Runtime Input", + "description": "The program to run inside the sandbox.", + "type": "object", + "schemaVersion": 1, + "properties": { + "code": { + "title": "Code", + "type": "string", + "description": "TypeScript/JavaScript program executed inside the sandbox. It receives an `apify` binding and `console`; stdout and stderr are captured separately and pushed to the default dataset as { stdout, stderr }.", + "editor": "javascript" + } + }, + "required": ["code"] + }, + "output": { + "actorOutputSchemaVersion": 1, + "title": "Code Runtime Output", + "description": "One dataset item { stdout, stderr } with the program's captured output.", + "type": "object", + "properties": { + "output": { + "type": "string", + "title": "Execution output", + "template": "{{links.apiDefaultDatasetUrl}}/items" + } + } + }, + "dockerfile": "../Dockerfile" +} diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..ec15a3b --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +node_modules +.git +*.log +.DS_Store diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1e13533 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +node_modules/ +worker/usercode.js +*.log +.DS_Store diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..b675bf7 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,31 @@ +# Two-stage build: drop the Node runtime entirely. workerd is a standalone +# glibc binary; only libc + libm are needed at runtime (verified via `ldd`). +# +# Stage 1: pull the workerd binary via a Node base. pnpm keeps it in the virtual +# store (not hoisted), so resolve the path through `require('workerd')`. +FROM node:24-bookworm-slim AS builder +WORKDIR /build +COPY package.json pnpm-lock.yaml ./ +# --ignore-scripts skips workerd's postinstall (a binary-download fallback we +# don't need — the binary ships in the @cloudflare/workerd-linux-64 optional dep) +# and avoids pnpm's hard error on unapproved dependency build scripts. +RUN corepack enable \ + && pnpm install --prod --frozen-lockfile --ignore-scripts \ + && BIN="$(node -e "process.stdout.write(require('workerd').default)")" \ + && cp "$BIN" /workerd \ + && chmod +x /workerd + +# Stage 2: minimal runtime — debian + ca-certificates + the workerd binary. +FROM debian:bookworm-slim + +# curl: loopback HTTP client + Actor-input fetch; jq: extract `code` from the input JSON. +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates curl jq \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /workerd /usr/local/bin/workerd + +WORKDIR /app +COPY worker/ ./worker/ + +ENTRYPOINT ["sh", "/app/worker/entrypoint.sh"] diff --git a/README.md b/README.md index e1d04ba..ca5c9e3 100644 --- a/README.md +++ b/README.md @@ -1 +1,76 @@ # Code Runtime (experimental) + +> ⚠️ **Experimental infrastructure Actor.** It powers **Code Mode** on +> [mcp.apify.com](https://mcp.apify.com) and is normally invoked by the Apify +> MCP Server, not run by hand. Its behaviour and API may change without notice. + +## What it does + +This Actor executes a single TypeScript/JavaScript program that an AI agent +submits through the Apify MCP Server's **Code Mode**, then returns whatever the +program printed. + +Code Mode exists so an agent can do many Apify operations in **one** program — +search the Store, run an Actor, read its dataset, filter and aggregate the +results — instead of sending every intermediate result back through the model. +This Actor is the sandbox that runs that program. + +## Enabling Code Mode on the MCP Server + +Code Mode is opt-in. Add the Code Mode tools to the `tools` query parameter of +your mcp.apify.com connection URL: + +``` +https://mcp.apify.com/?tools=run-code,get-code-docs +``` + +For full configuration options, use the configurator at +[mcp.apify.com](https://mcp.apify.com). + +## How it works + +- **One program per run.** The Actor reads your `code`, runs it once, writes the + result, and exits. +- The code runs inside a [`workerd`](https://github.com/cloudflare/workerd) V8 + isolate: **no filesystem, no package imports**, and outbound network is + restricted to `*.apify.com`. +- Inside the program a global **`apify`** object exposes a small, typed subset of + the Apify API — run Actors, read/write datasets and key-value stores — using + the current run's token. +- `console.log` / `console.info` go to **stdout**; `console.error` / + `console.warn` go to **stderr**. The two streams are captured separately. + +## Input + +```json +{ + "code": "const { items } = await apify.actor.runAndGetItems({ actorId: 'apify/rag-web-browser', input: { query: 'apify' }, limit: 3 });\nconsole.log(items.map((i) => i.metadata?.title).join('\\n'));" +} +``` + +| Field | Type | Description | +|---|---|---| +| `code` | string | The TypeScript/JavaScript program to run. It receives the `apify` binding and `console`. | + +## Output + +A single **dataset item** with the captured streams: + +```json +{ "stdout": "Apify: Full-stack web scraping ...\n...", "stderr": "" } +``` + +If the program throws, the error lands in `stderr`; `stdout` keeps whatever was +printed before the failure. + +## Permissions & safety + +- Runs with **limited permissions**: the sandbox has no filesystem and can reach + only the Apify API (`*.apify.com`). +- It uses the **run's own token**, so the program can access only what you can. +- Each run is an isolated, single-use container — nothing persists between runs. + +## Learn more + +- Apify MCP Server: +- Code Mode design: [apify/apify-mcp-server#794](https://github.com/apify/apify-mcp-server/pull/794) diff --git a/package.json b/package.json new file mode 100644 index 0000000..33190c6 --- /dev/null +++ b/package.json @@ -0,0 +1,13 @@ +{ + "name": "code-runtime", + "version": "0.1.0", + "description": "workerd as a normal (per-run) Apify Actor; one V8 isolate per run via the Worker Loader API.", + "private": true, + "packageManager": "pnpm@11.1.3", + "dependencies": { + "workerd": "1.20260402.1" + }, + "engines": { + "node": ">=24" + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml new file mode 100644 index 0000000..95ed99b --- /dev/null +++ b/pnpm-lock.yaml @@ -0,0 +1,75 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + workerd: + specifier: 1.20260402.1 + version: 1.20260402.1 + +packages: + + '@cloudflare/workerd-darwin-64@1.20260402.1': + resolution: {integrity: sha512-/kyyZ5HjOPT202Vsw3P+vICsVulldC9ym+W5UyKl9dufxfHgcdeT4EYtT+tkk+3SlkB1RmoVxgr3VRlqP2ojjw==} + engines: {node: '>=16'} + cpu: [x64] + os: [darwin] + + '@cloudflare/workerd-darwin-arm64@1.20260402.1': + resolution: {integrity: sha512-nr+AgUonmepiuD4utn4KQl2fIn/aDuWEO+/B2fzjMnjLSgLpN1IQADUW1uD4FVJdd0ss9ugK1dztFJJWpImCaQ==} + engines: {node: '>=16'} + cpu: [arm64] + os: [darwin] + + '@cloudflare/workerd-linux-64@1.20260402.1': + resolution: {integrity: sha512-0vJj0pO6ARpCmvyLgbQk204dogL72geEyC+rOnkFgwcDJI4e8oxKrTQZWjPHra3BHUknEomUd37i3K4L4JfU4w==} + engines: {node: '>=16'} + cpu: [x64] + os: [linux] + + '@cloudflare/workerd-linux-arm64@1.20260402.1': + resolution: {integrity: sha512-aRsxuv1bmwkkX4sG8igutkwVHgsJ1I6mH4/u/1Jfhpb7Bj3xr9p9N2fjDwax+Dhy7xt2tC8TS/ZWLeFFjK2NMA==} + engines: {node: '>=16'} + cpu: [arm64] + os: [linux] + + '@cloudflare/workerd-windows-64@1.20260402.1': + resolution: {integrity: sha512-aFKYAuIYTPsuWyHxv39yOFi9bmIvAsIuNmHGYkYltHZUoN4by6tJVCOQUp6PDlZO7TjHSD2XjjM8JBwK/D6tVA==} + engines: {node: '>=16'} + cpu: [x64] + os: [win32] + + workerd@1.20260402.1: + resolution: {integrity: sha512-Cg+OUlukdcCHrTTg0MBCIMFRE6XO3yGVGiWCnJPvfffy2Ga2girrEq3qF/YlHSTmbIyEE5ebCFxBYYYZueQ/Mg==} + engines: {node: '>=16'} + hasBin: true + +snapshots: + + '@cloudflare/workerd-darwin-64@1.20260402.1': + optional: true + + '@cloudflare/workerd-darwin-arm64@1.20260402.1': + optional: true + + '@cloudflare/workerd-linux-64@1.20260402.1': + optional: true + + '@cloudflare/workerd-linux-arm64@1.20260402.1': + optional: true + + '@cloudflare/workerd-windows-64@1.20260402.1': + optional: true + + workerd@1.20260402.1: + optionalDependencies: + '@cloudflare/workerd-darwin-64': 1.20260402.1 + '@cloudflare/workerd-darwin-arm64': 1.20260402.1 + '@cloudflare/workerd-linux-64': 1.20260402.1 + '@cloudflare/workerd-linux-arm64': 1.20260402.1 + '@cloudflare/workerd-windows-64': 1.20260402.1 diff --git a/worker/config.capnp b/worker/config.capnp new file mode 100644 index 0000000..e7288de --- /dev/null +++ b/worker/config.capnp @@ -0,0 +1,39 @@ +using Workerd = import "/workerd/workerd.capnp"; + +const config :Workerd.Config = ( + services = [ + (name = "main", worker = .codeRuntime), + # Outbound for fetch(). Apify's api.apify.com may resolve to a private + # address inside the platform network, so allow private/local too. + # tlsOptions enables HTTPS egress (trust workerd's built-in CA set); + # the hostname allowlist is enforced by guard.js, not here. + (name = "internet", network = (allow = ["public", "private", "local"], tlsOptions = (trustBrowserCas = true))), + ], + sockets = [ + ( + name = "http", + address = "127.0.0.1:8787", + http = (), + service = "main", + ), + ], +); + +# runner.js is the entrypoint module; usercode.js is generated at container +# start by entrypoint.sh (the Actor input wrapped as `export async function run`). +const codeRuntime :Workerd.Worker = ( + modules = [ + (name = "runner.js", esModule = embed "runner.js"), + (name = "guard.js", esModule = embed "guard.js"), + (name = "usercode.js", esModule = embed "usercode.js"), + ], + bindings = [ + (name = "APIFY_TOKEN", fromEnvironment = "APIFY_TOKEN"), + (name = "DEFAULT_DATASET_ID", fromEnvironment = "ACTOR_DEFAULT_DATASET_ID"), + (name = "DEFAULT_DATASET_ID_LEGACY", fromEnvironment = "APIFY_DEFAULT_DATASET_ID"), + (name = "API_BASE_URL", fromEnvironment = "APIFY_API_BASE_URL"), + ], + globalOutbound = "internet", + compatibilityDate = "2026-01-15", + compatibilityFlags = ["nodejs_compat"], +); diff --git a/worker/entrypoint.sh b/worker/entrypoint.sh new file mode 100755 index 0000000..27892bd --- /dev/null +++ b/worker/entrypoint.sh @@ -0,0 +1,54 @@ +#!/bin/sh +# Normal-mode (non-standby) Apify Actor entrypoint. Single-tenant per run: +# 1. read the Actor input and wrap its `code` into the runnable usercode.js module +# 2. boot workerd on loopback (it embeds runner.js + usercode.js) +# 3. trigger /run once, then exit +# workerd hosts the sandboxed worker and is reached only over loopback. +set -eu + +PORT=8787 +READINESS_ATTEMPTS=100 # 100 * 0.1s = 10s budget for workerd to bind the socket + +API_BASE="${APIFY_API_BASE_URL:-https://api.apify.com}" +API_BASE="${API_BASE%/}" # APIFY_API_BASE_URL ships with a trailing slash +STORE_ID="${ACTOR_DEFAULT_KEY_VALUE_STORE_ID:-${APIFY_DEFAULT_KEY_VALUE_STORE_ID:-}}" +INPUT_KEY="${APIFY_INPUT_KEY:-INPUT}" + +if [ -z "${APIFY_TOKEN:-}" ] || [ -z "$STORE_ID" ]; then + echo "[code-runtime] missing APIFY_TOKEN or default key-value store ID" >&2 + exit 1 +fi + +# Fetch the Actor input and wrap its `code` into the runnable module. The `code` +# is inserted as code between the wrapper lines (not as a string) — no escaping. +INPUT_URL="${API_BASE}/v2/key-value-stores/${STORE_ID}/records/${INPUT_KEY}" +input_status=$(curl -sS -o /tmp/input.json -w '%{http_code}' \ + -H "Authorization: Bearer ${APIFY_TOKEN}" "$INPUT_URL") +if [ "$input_status" != "200" ]; then + echo "[code-runtime] failed to read Actor input from ${INPUT_URL} (HTTP ${input_status})" >&2 + exit 1 +fi + +{ + printf 'export async function run(apify, console) {\n' + jq -r '.code // ""' < /tmp/input.json + printf '\n}\n' +} > /app/worker/usercode.js + +/usr/local/bin/workerd serve --experimental /app/worker/config.capnp & +workerd_pid=$! +trap 'kill "$workerd_pid" 2>/dev/null || true' EXIT + +attempt=0 +until curl -sf "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; do + attempt=$((attempt + 1)) + if [ "$attempt" -ge "$READINESS_ATTEMPTS" ]; then + echo "[code-runtime] workerd did not become ready in time" >&2 + exit 1 + fi + sleep 0.1 +done + +# Trigger the single run. The worker runs the program and pushes { stdout, stderr } +# to the default dataset. A non-2xx response fails the Actor run (curl -f). +curl -fsS -X POST "http://127.0.0.1:${PORT}/run" diff --git a/worker/guard.js b/worker/guard.js new file mode 100644 index 0000000..426a170 --- /dev/null +++ b/worker/guard.js @@ -0,0 +1,45 @@ +// Restrict the user program's global fetch() to apify.com and its subdomains. +// Imported before usercode.js so the override is in place even for code that +// runs at module-evaluation time. Our own Apify API calls use the exported +// realFetch (the internal API is a private IP, not *.apify.com), so they are +// unaffected by this guard. +// +// NOTE: this guards the fetch() API only. It is not a complete egress boundary — +// the airtight control is workerd's globalOutbound. See SECURITY notes in the repo. + +const realFetch = globalThis.fetch.bind(globalThis); + +// Match apify.com exactly or any subdomain. The leading dot in the suffix is +// what rejects look-alikes: `evilapify.com` (no dot) and `apify.com.evil.com` +// (ends with `.evil.com`) both fail. +function isAllowedHost(hostname) { + const host = hostname.toLowerCase().replace(/\.$/, ''); // strip FQDN trailing dot + return host === 'apify.com' || host.endsWith('.apify.com'); +} + +function requestUrl(input) { + if (typeof input === 'string') return input; + if (input instanceof URL) return input.href; + if (input && typeof input.url === 'string') return input.url; // Request + return String(input); +} + +globalThis.fetch = (input, init) => { + let url; + try { + // Parse to the real host — defeats userinfo (`apify.com@evil.com`), + // path/query/fragment (`evil.com/apify.com`) and similar tricks. + url = new URL(requestUrl(input)); + } catch { + throw new Error('Blocked fetch: only absolute http(s) URLs to apify.com are allowed'); + } + if (url.protocol !== 'https:' && url.protocol !== 'http:') { + throw new Error(`Blocked fetch: protocol "${url.protocol}" is not allowed`); + } + if (!isAllowedHost(url.hostname)) { + throw new Error(`Blocked fetch to "${url.hostname}": only apify.com and its subdomains are allowed`); + } + return realFetch(input, init); +}; + +export { realFetch }; diff --git a/worker/runner.js b/worker/runner.js new file mode 100644 index 0000000..18b57ac --- /dev/null +++ b/worker/runner.js @@ -0,0 +1,286 @@ +// Single worker for the per-run code-runtime Actor. It runs the user's program +// (imported from the generated `usercode.js` module) with the `apify` REST +// binding and a captured `console`, then pushes `{ stdout, stderr }` to the +// run's default dataset. The container entrypoint generates `usercode.js`, +// boots workerd, and triggers `/run` once. +// +// Single-tenant: one run = one container = one program = one token. No Worker +// Loader / per-request isolate is needed — the program runs in this worker, +// which is itself the sandbox (no filesystem, restricted outbound network). +// guard.js must be imported before usercode.js: it overrides globalThis.fetch +// to allow only apify.com, and exports realFetch for our own (internal) API calls. +import { realFetch } from './guard.js'; +import { run } from './usercode.js'; + +const DEFAULT_ITERATE_BATCH = 1000; +const DEFAULT_GET_SCHEMA_SAMPLE = 5; + +function stringify(x) { + if (typeof x === 'string') return x; + try { return JSON.stringify(x); } catch { return String(x); } +} + +function makeApifyBinding(token, apiV2) { + const baseHeaders = { Authorization: `Bearer ${token}` }; + + // Build a URL with optional query params; null/undefined values are dropped. + const buildUrl = (path, searchParams) => { + const url = new URL(`${apiV2}${path}`); + if (searchParams) { + for (const [k, v] of Object.entries(searchParams)) { + if (v !== undefined && v !== null) url.searchParams.set(k, String(v)); + } + } + return url; + }; + + // Single-source HTTP wrapper. Throws on non-2xx with the response body in the message. + // `body`: string / Uint8Array passed through; objects are JSON.stringify'd. + const apiCall = async (method, path, { searchParams, body, contentType } = {}) => { + const init = { method, headers: { ...baseHeaders } }; + if (body !== undefined) { + const isRaw = typeof body === 'string' || body instanceof Uint8Array || body instanceof ArrayBuffer; + init.body = isRaw ? body : JSON.stringify(body); + init.headers['content-type'] = contentType ?? (isRaw ? 'application/octet-stream' : 'application/json'); + } + const r = await realFetch(buildUrl(path, searchParams), init); + if (!r.ok) throw new Error(`${method} ${path} failed: ${r.status} ${await r.text()}`); + return r; + }; + + const apiJson = async (...args) => (await apiCall(...args)).json(); + const apiData = async (...args) => (await apiJson(...args)).data; + + const actor = { + // GET /v2/store — Apify Store search. Returns the items array directly. + search: ({ query, limit, category }) => + apiData('GET', '/store', { searchParams: { search: query, limit, category } }) + .then((d) => d.items), + + getDetails: ({ actorId }) => + apiData('GET', `/acts/${encodeURIComponent(actorId)}`), + + // POST /runs with waitForFinish blocks until the run completes (max 60s per the + // Apify API; for longer runs the caller should use start() + apify.run.wait()). + // Returns the run record so the caller can read defaultDatasetId / defaultKeyValueStoreId. + // Intentionally does NOT use /run-sync, which returns the OUTPUT KVS record (a pattern + // only some Actors follow) rather than the structured run record. + run: ({ actorId, input, memoryMbytes, timeoutSecs, waitForFinishSecs = 60, maxTotalChargeUsd, maxItems }) => + apiData('POST', `/acts/${encodeURIComponent(actorId)}/runs`, { + searchParams: { + waitForFinish: waitForFinishSecs, + memory: memoryMbytes, + timeout: timeoutSecs, + maxTotalChargeUsd, + maxItems, + }, + body: input ?? {}, + }), + + // Async kickoff. Returns immediately with a run record in READY/RUNNING state. + start: ({ actorId, input, memoryMbytes, timeoutSecs, maxTotalChargeUsd, maxItems }) => + apiData('POST', `/acts/${encodeURIComponent(actorId)}/runs`, { + searchParams: { + memory: memoryMbytes, + timeout: timeoutSecs, + maxTotalChargeUsd, + maxItems, + }, + body: input ?? {}, + }), + // runAndGetItems is added below once `dataset.listItems` is defined. + }; + + const run = { + get: ({ runId }) => + apiData('GET', `/actor-runs/${encodeURIComponent(runId)}`), + + // Block until the run terminates or `waitForFinishSecs` elapses (whichever comes first). + // The Apify API caps this at 60s per request; longer waits require a polling loop. + wait: ({ runId, waitForFinishSecs = 60 }) => + apiData('GET', `/actor-runs/${encodeURIComponent(runId)}`, { + searchParams: { waitForFinish: waitForFinishSecs }, + }), + + abort: ({ runId }) => + apiData('POST', `/actor-runs/${encodeURIComponent(runId)}/abort`), + + // Returns the full run log as text. `limit` tails the last N characters; the Apify API + // does not paginate logs, so this is a client-side slice (the full body is fetched). + getLog: async ({ runId, limit }) => { + const r = await apiCall('GET', `/logs/${encodeURIComponent(runId)}`); + const text = await r.text(); + return limit && text.length > limit ? text.slice(-limit) : text; + }, + }; + + const dataset = { + // Returns the items array directly (no wrapper). The Apify API's + // `x-apify-pagination-total` header is unreliable for freshly-created datasets + // (eventually consistent), so we don't surface a `total`. Use `getSchema` if you + // need an item count, or iterate to consume the whole dataset. + listItems: async ({ datasetId, fields, omit, limit, offset, clean, desc }) => { + const r = await apiCall('GET', `/datasets/${encodeURIComponent(datasetId)}/items`, { + searchParams: { + fields: fields?.join(','), + omit: omit?.join(','), + limit, + offset, + clean: clean ? '1' : undefined, + desc: desc ? '1' : undefined, + }, + }); + return r.json(); + }, + + // Async generator over the entire dataset. Pages internally in `batchSize` chunks + // so the user can `for await (const item of apify.dataset.iterate({...}))` without + // worrying about offsets. Stops when a page returns fewer items than `batchSize` + // (the natural end-of-data signal — pagination total is not used, see listItems). + iterate: async function* ({ datasetId, fields, omit, clean, desc, batchSize = DEFAULT_ITERATE_BATCH }) { + let offset = 0; + while (true) { + const items = await dataset.listItems({ + datasetId, fields, omit, clean, desc, + limit: batchSize, offset, + }); + if (items.length === 0) break; + for (const item of items) yield item; + if (items.length < batchSize) break; + offset += items.length; + } + }, + + // Apify has no dedicated schema endpoint; we infer one from a small sample of items. + // Returns { itemCount, sampleSize, fields: [{ name, types, nullable }] }. + getSchema: async ({ datasetId, sample = DEFAULT_GET_SCHEMA_SAMPLE }) => { + const meta = await apiData('GET', `/datasets/${encodeURIComponent(datasetId)}`); + const items = await dataset.listItems({ datasetId, limit: sample }); + const fields = new Map(); + for (const item of items) { + for (const [name, value] of Object.entries(item ?? {})) { + const type = value === null ? 'null' : Array.isArray(value) ? 'array' : typeof value; + if (!fields.has(name)) fields.set(name, new Set()); + fields.get(name).add(type); + } + } + return { + itemCount: meta?.itemCount, + sampleSize: items.length, + fields: [...fields.entries()].map(([name, types]) => ({ + name, + types: [...types], + nullable: types.has('null'), + })), + }; + }, + + create: ({ name } = {}) => + apiData('POST', '/datasets', { searchParams: { name } }), + + pushItems: async ({ datasetId, items }) => { + await apiCall('POST', `/datasets/${encodeURIComponent(datasetId)}/items`, { body: items }); + }, + }; + + actor.runAndGetItems = async ({ actorId, input, fields, limit, ...runOpts }) => { + const runRecord = await actor.run({ actorId, input, ...runOpts }); + const items = await dataset.listItems({ + datasetId: runRecord.defaultDatasetId, fields, limit, + }); + return { run: runRecord, items }; + }; + + const kvs = { + // Returns the value directly (parsed when JSON, string when text/*, Uint8Array otherwise). + // Returns null when the key does not exist (404), not an error — this matches the common + // "lookup or default" pattern in code. + get: async ({ storeId, key }) => { + const r = await realFetch(buildUrl(`/key-value-stores/${encodeURIComponent(storeId)}/records/${encodeURIComponent(key)}`), { + headers: baseHeaders, + }); + if (r.status === 404) return null; + if (!r.ok) throw new Error(`GET kvs.get failed: ${r.status} ${await r.text()}`); + const ct = r.headers.get('content-type') ?? ''; + if (ct.includes('application/json')) return r.json(); + if (ct.startsWith('text/')) return r.text(); + return new Uint8Array(await r.arrayBuffer()); + }, + + // `value`: object → application/json; string → text/plain; Uint8Array/ArrayBuffer → + // application/octet-stream (or whatever the caller passed via `contentType`). + set: async ({ storeId, key, value, contentType }) => { + let body; + let ct = contentType; + if (value instanceof Uint8Array || value instanceof ArrayBuffer) { + body = value; + ct = ct ?? 'application/octet-stream'; + } else if (typeof value === 'string') { + body = value; + ct = ct ?? 'text/plain; charset=utf-8'; + } else { + body = JSON.stringify(value); + ct = ct ?? 'application/json; charset=utf-8'; + } + await apiCall('PUT', `/key-value-stores/${encodeURIComponent(storeId)}/records/${encodeURIComponent(key)}`, { + body, contentType: ct, + }); + }, + + list: ({ storeId, limit, exclusiveStartKey }) => + apiData('GET', `/key-value-stores/${encodeURIComponent(storeId)}/keys`, { + searchParams: { limit, exclusiveStartKey }, + }), + + create: ({ name } = {}) => + apiData('POST', '/key-value-stores', { searchParams: { name } }), + }; + + return { actor, run, dataset, kvs }; +} + +// Push the captured streams as a single item to the run's default dataset. +async function pushOutput(apiV2, token, env, item) { + const datasetId = env.DEFAULT_DATASET_ID || env.DEFAULT_DATASET_ID_LEGACY; + if (!datasetId) throw new Error('Default dataset ID missing from Actor run environment.'); + const r = await realFetch(`${apiV2}/datasets/${encodeURIComponent(datasetId)}/items`, { + method: 'POST', + headers: { Authorization: `Bearer ${token}`, 'content-type': 'application/json; charset=utf-8' }, + body: JSON.stringify(item), + }); + if (!r.ok) throw new Error(`Failed to push dataset item: ${r.status} ${await r.text()}`); +} + +export default { + async fetch(request, env) { + const url = new URL(request.url); + if (url.pathname === '/health') return new Response('ok'); + if (url.pathname !== '/run') return new Response('Not found', { status: 404 }); + + const token = env.APIFY_TOKEN; + if (!token) throw new Error('APIFY_TOKEN missing from Actor run environment.'); + // APIFY_API_BASE_URL is the platform-internal API (may have a trailing slash). + const apiV2 = `${(env.API_BASE_URL || 'https://api.apify.com').replace(/\/+$/, '')}/v2`; + + const stdout = []; + const stderr = []; + const captureConsole = { + log: (...args) => stdout.push(args.map(stringify).join(' ')), + error: (...args) => stderr.push(args.map(stringify).join(' ')), + warn: (...args) => stderr.push(args.map(stringify).join(' ')), + info: (...args) => stdout.push(args.map(stringify).join(' ')), + }; + + // A thrown program is a user-level failure: capture it in stderr and still + // push the output, so the run SUCCEEDS with diagnostics. Infra failures + // (missing env, dataset push) throw and fail the run. + try { + await run(makeApifyBinding(token, apiV2), captureConsole); + } catch (err) { + stderr.push(err?.stack ?? err?.message ?? String(err)); + } + + await pushOutput(apiV2, token, env, { stdout: stdout.join('\n'), stderr: stderr.join('\n') }); + return Response.json({ ok: true }); + }, +}; From 6abfe193e0dd0a31203514658c297d89ee6d594a Mon Sep 17 00:00:00 2001 From: MQ37 Date: Wed, 24 Jun 2026 17:45:27 +0200 Subject: [PATCH 02/38] test: add apify binding smoke test (tests/binding-smoke.js + test.sh) --- test.sh | 33 ++++++++++++ tests/binding-smoke.js | 111 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+) create mode 100755 test.sh create mode 100644 tests/binding-smoke.js diff --git a/test.sh b/test.sh new file mode 100755 index 0000000..6d72c0d --- /dev/null +++ b/test.sh @@ -0,0 +1,33 @@ +#!/bin/sh +# Deploy the Actor (apify push) and run the binding smoke test on the freshly +# built version via `apify call`. Exits non-zero if any binding check fails. +# +# Usage: ./test.sh +set -eu + +cd "$(dirname "$0")" + +TEST_JS="tests/binding-smoke.js" + +command -v apify >/dev/null 2>&1 || { echo "apify CLI not found" >&2; exit 1; } +command -v jq >/dev/null 2>&1 || { echo "jq not found" >&2; exit 1; } + +input_json="$(mktemp)" +trap 'rm -f "$input_json"' EXIT + +echo "==> apify push" +apify push + +echo "==> building input from ${TEST_JS}" +jq -n --arg code "$(cat "$TEST_JS")" '{ code: $code }' > "$input_json" + +echo "==> apify call (running the test on the built Actor)" +output="$(apify call -f "$input_json" -o 2>&1)" +echo "$output" + +if printf '%s' "$output" | grep -q 'ALL_TESTS_PASSED'; then + echo "==> all binding tests passed" +else + echo "==> binding tests FAILED" >&2 + exit 1 +fi diff --git a/tests/binding-smoke.js b/tests/binding-smoke.js new file mode 100644 index 0000000..98b5b55 --- /dev/null +++ b/tests/binding-smoke.js @@ -0,0 +1,111 @@ +// Smoke test for the `apify` binding exposed to Code Mode programs. Submitted as +// the Actor's `code` input by test.sh and executed on the built Actor via +// `apify call`. Exercises every binding method and prints a sentinel line +// (ALL_TESTS_PASSED) that test.sh greps for. +const results = []; +async function check(name, fn) { + try { + const out = await fn(); + console.log(`PASS ${name}: ${out ?? ''}`); + results.push(true); + } catch (e) { + console.error(`FAIL ${name}: ${e.message}`); + results.push(false); + } +} + +const ACTOR = 'apify/hello-world'; + +// ---- actor (read) ---- +await check('actor.search', async () => { + const items = await apify.actor.search({ query: 'hello world', limit: 3 }); + if (!Array.isArray(items)) throw new Error('expected array'); + return `${items.length} actors`; +}); +await check('actor.getDetails', async () => { + const d = await apify.actor.getDetails({ actorId: ACTOR }); + return `${d.username}/${d.name}`; +}); + +// ---- dataset ---- +let datasetId; +await check('dataset.create', async () => { + datasetId = (await apify.dataset.create()).id; + return datasetId; +}); +await check('dataset.pushItems', async () => { + await apify.dataset.pushItems({ datasetId, items: [{ a: 1, b: 'x' }, { a: 2, b: 'y' }] }); + return '2 pushed'; +}); +await check('dataset.listItems', async () => { + const items = await apify.dataset.listItems({ datasetId }); + return `${items.length} items`; +}); +await check('dataset.getSchema', async () => { + const s = await apify.dataset.getSchema({ datasetId }); + return `itemCount=${s.itemCount} fields=${s.fields.map((f) => f.name).join(',')}`; +}); +await check('dataset.iterate', async () => { + let n = 0; + for await (const _ of apify.dataset.iterate({ datasetId, batchSize: 1 })) n++; + return `${n} iterated`; +}); + +// ---- key-value store ---- +let storeId; +await check('kvs.create', async () => { + storeId = (await apify.kvs.create()).id; + return storeId; +}); +await check('kvs.set', async () => { + await apify.kvs.set({ storeId, key: 'obj', value: { hello: 'world' } }); + await apify.kvs.set({ storeId, key: 'txt', value: 'plain' }); + return 'set obj + txt'; +}); +await check('kvs.get', async () => { + const obj = await apify.kvs.get({ storeId, key: 'obj' }); + const txt = await apify.kvs.get({ storeId, key: 'txt' }); + const missing = await apify.kvs.get({ storeId, key: 'nope' }); + return `obj.hello=${obj.hello} txt=${txt} missing=${missing}`; +}); +await check('kvs.list', async () => { + const l = await apify.kvs.list({ storeId }); + return `${l.items.length} keys`; +}); + +// ---- run lifecycle ---- +let runId; +await check('actor.start', async () => { + const run = await apify.actor.start({ actorId: ACTOR }); + runId = run.id; + return `runId=${runId} status=${run.status}`; +}); +await check('run.get', async () => { + return `status=${(await apify.run.get({ runId })).status}`; +}); +await check('run.wait', async () => { + return `status=${(await apify.run.wait({ runId, waitForFinishSecs: 60 })).status}`; +}); +await check('run.getLog', async () => { + return `${(await apify.run.getLog({ runId, limit: 200 })).length} chars`; +}); + +// ---- run + get items (sync) ---- +await check('actor.run', async () => { + return `status=${(await apify.actor.run({ actorId: ACTOR, waitForFinishSecs: 60 })).status}`; +}); +await check('actor.runAndGetItems', async () => { + const { run, items } = await apify.actor.runAndGetItems({ actorId: ACTOR, limit: 5, waitForFinishSecs: 60 }); + return `status=${run.status} items=${items.length}`; +}); + +// ---- abort ---- +await check('run.abort', async () => { + const run = await apify.actor.start({ actorId: ACTOR }); + return `status=${(await apify.run.abort({ runId: run.id })).status}`; +}); + +const passed = results.filter(Boolean).length; +console.log(`\n=== SUMMARY: ${passed}/${results.length} passed ===`); +if (passed === results.length) console.log('ALL_TESTS_PASSED'); +else console.error(`SOME_TESTS_FAILED (${results.length - passed} failed)`); From b669c05d1eed22b18b1fa547e5d5fa993af9ced0 Mon Sep 17 00:00:00 2001 From: MQ37 Date: Wed, 24 Jun 2026 17:51:46 +0200 Subject: [PATCH 03/38] docs: tighten README copy and add compact apify binding reference --- README.md | 51 ++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 42 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index ca5c9e3..fac65a6 100644 --- a/README.md +++ b/README.md @@ -6,14 +6,14 @@ ## What it does -This Actor executes a single TypeScript/JavaScript program that an AI agent +This Actor executes a single TypeScript/JavaScript script that an AI agent submits through the Apify MCP Server's **Code Mode**, then returns whatever the -program printed. +script printed. -Code Mode exists so an agent can do many Apify operations in **one** program — +Code Mode exists so an agent can do many Apify operations in **one go** — search the Store, run an Actor, read its dataset, filter and aggregate the results — instead of sending every intermediate result back through the model. -This Actor is the sandbox that runs that program. +This Actor is the sandbox that runs that script. ## Enabling Code Mode on the MCP Server @@ -29,17 +29,50 @@ For full configuration options, use the configurator at ## How it works -- **One program per run.** The Actor reads your `code`, runs it once, writes the +- **One script per run.** The Actor reads your `code`, runs it once, writes the result, and exits. - The code runs inside a [`workerd`](https://github.com/cloudflare/workerd) V8 isolate: **no filesystem, no package imports**, and outbound network is restricted to `*.apify.com`. -- Inside the program a global **`apify`** object exposes a small, typed subset of +- Inside the script a global **`apify`** object exposes a small, typed subset of the Apify API — run Actors, read/write datasets and key-value stores — using - the current run's token. + the current run's token (see below). - `console.log` / `console.info` go to **stdout**; `console.error` / `console.warn` go to **stderr**. The two streams are captured separately. +## The `apify` binding + +Every method takes one options object and returns parsed JSON +(`?` = optional, `= x` = default): + +```js +// Actors +apify.actor.search({ query, limit?, category? }) // → actors[] +apify.actor.getDetails({ actorId }) // → actor +apify.actor.start({ actorId, input?, memoryMbytes?, timeoutSecs?, maxTotalChargeUsd?, maxItems? }) // → run +apify.actor.run({ actorId, ...startOpts, waitForFinishSecs = 60 }) // → run (waits) +apify.actor.runAndGetItems({ actorId, input?, fields?, limit?, ...runOpts }) // → { run, items } + +// Runs +apify.run.get({ runId }) // → run +apify.run.wait({ runId, waitForFinishSecs = 60 }) // → run +apify.run.abort({ runId }) // → run +apify.run.getLog({ runId, limit? }) // → string + +// Datasets +apify.dataset.create({ name? }) // → dataset +apify.dataset.pushItems({ datasetId, items }) // → void +apify.dataset.listItems({ datasetId, fields?, omit?, limit?, offset?, clean?, desc? }) // → items[] +apify.dataset.iterate({ datasetId, batchSize = 1000, ...filters }) // → async iterable +apify.dataset.getSchema({ datasetId, sample = 5 }) // → { itemCount, fields[] } + +// Key-value stores +apify.kvs.create({ name? }) // → store +apify.kvs.set({ storeId, key, value, contentType? }) // → void +apify.kvs.get({ storeId, key }) // → value | null +apify.kvs.list({ storeId, limit?, exclusiveStartKey? }) // → { items } +``` + ## Input ```json @@ -50,7 +83,7 @@ For full configuration options, use the configurator at | Field | Type | Description | |---|---|---| -| `code` | string | The TypeScript/JavaScript program to run. It receives the `apify` binding and `console`. | +| `code` | string | The TypeScript/JavaScript script to run. It receives the `apify` binding and `console`. | ## Output @@ -60,7 +93,7 @@ A single **dataset item** with the captured streams: { "stdout": "Apify: Full-stack web scraping ...\n...", "stderr": "" } ``` -If the program throws, the error lands in `stderr`; `stdout` keeps whatever was +If the script throws, the error lands in `stderr`; `stdout` keeps whatever was printed before the failure. ## Permissions & safety From f20b4896cdcae673ae617bd16bdd02ee3f300ffe Mon Sep 17 00:00:00 2001 From: MQ37 Date: Wed, 24 Jun 2026 17:55:08 +0200 Subject: [PATCH 04/38] docs: move apify binding section above Learn more; trim intro copy --- README.md | 69 +++++++++++++++++++++++++++---------------------------- 1 file changed, 34 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index fac65a6..99f71bb 100644 --- a/README.md +++ b/README.md @@ -6,14 +6,13 @@ ## What it does -This Actor executes a single TypeScript/JavaScript script that an AI agent -submits through the Apify MCP Server's **Code Mode**, then returns whatever the -script printed. +This Actor executes TypeScript/JavaScript that an AI agent submits through the +Apify MCP Server's **Code Mode**, then returns whatever the script printed. Code Mode exists so an agent can do many Apify operations in **one go** — search the Store, run an Actor, read its dataset, filter and aggregate the -results — instead of sending every intermediate result back through the model. -This Actor is the sandbox that runs that script. +results — instead of sending every intermediate result back through the model +and wasting tokens. This Actor is the sandbox that runs that script. ## Enabling Code Mode on the MCP Server @@ -40,6 +39,36 @@ For full configuration options, use the configurator at - `console.log` / `console.info` go to **stdout**; `console.error` / `console.warn` go to **stderr**. The two streams are captured separately. +## Input + +```json +{ + "code": "const { items } = await apify.actor.runAndGetItems({ actorId: 'apify/rag-web-browser', input: { query: 'apify' }, limit: 3 });\nconsole.log(items.map((i) => i.metadata?.title).join('\\n'));" +} +``` + +| Field | Type | Description | +|---|---|---| +| `code` | string | The TypeScript/JavaScript script to run. It receives the `apify` binding and `console`. | + +## Output + +A single **dataset item** with the captured streams: + +```json +{ "stdout": "Apify: Full-stack web scraping ...\n...", "stderr": "" } +``` + +If the script throws, the error lands in `stderr`; `stdout` keeps whatever was +printed before the failure. + +## Permissions & safety + +- Runs with **limited permissions**: the sandbox has no filesystem and can reach + only the Apify API (`*.apify.com`). +- It uses the **run's own token**, so the program can access only what you can. +- Each run is an isolated, single-use container — nothing persists between runs. + ## The `apify` binding Every method takes one options object and returns parsed JSON @@ -73,36 +102,6 @@ apify.kvs.get({ storeId, key }) // → value | null apify.kvs.list({ storeId, limit?, exclusiveStartKey? }) // → { items } ``` -## Input - -```json -{ - "code": "const { items } = await apify.actor.runAndGetItems({ actorId: 'apify/rag-web-browser', input: { query: 'apify' }, limit: 3 });\nconsole.log(items.map((i) => i.metadata?.title).join('\\n'));" -} -``` - -| Field | Type | Description | -|---|---|---| -| `code` | string | The TypeScript/JavaScript script to run. It receives the `apify` binding and `console`. | - -## Output - -A single **dataset item** with the captured streams: - -```json -{ "stdout": "Apify: Full-stack web scraping ...\n...", "stderr": "" } -``` - -If the script throws, the error lands in `stderr`; `stdout` keeps whatever was -printed before the failure. - -## Permissions & safety - -- Runs with **limited permissions**: the sandbox has no filesystem and can reach - only the Apify API (`*.apify.com`). -- It uses the **run's own token**, so the program can access only what you can. -- Each run is an isolated, single-use container — nothing persists between runs. - ## Learn more - Apify MCP Server: From 63590fdc4ce2f51b64eba39767e2e3d9a7108b39 Mon Sep 17 00:00:00 2001 From: MQ37 Date: Wed, 24 Jun 2026 18:13:57 +0200 Subject: [PATCH 05/38] docs: drop run-token sentence from Permissions & safety --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 99f71bb..b64b77b 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,6 @@ printed before the failure. - Runs with **limited permissions**: the sandbox has no filesystem and can reach only the Apify API (`*.apify.com`). -- It uses the **run's own token**, so the program can access only what you can. - Each run is an isolated, single-use container — nothing persists between runs. ## The `apify` binding From 8d0c7ffa23b7fdf40c6324050684834f478d9d5f Mon Sep 17 00:00:00 2001 From: MQ37 Date: Wed, 24 Jun 2026 18:21:03 +0200 Subject: [PATCH 06/38] docs: add docs/API.md detailed reference and link it from README --- README.md | 3 +- docs/API.md | 307 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 309 insertions(+), 1 deletion(-) create mode 100644 docs/API.md diff --git a/README.md b/README.md index b64b77b..085eb7e 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,8 @@ printed before the failure. ## The `apify` binding Every method takes one options object and returns parsed JSON -(`?` = optional, `= x` = default): +(`?` = optional, `= x` = default). Full API documentation is available +[here](https://github.com/apify/actor-code-runtime/blob/master/docs/API.md). ```js // Actors diff --git a/docs/API.md b/docs/API.md new file mode 100644 index 0000000..718be3f --- /dev/null +++ b/docs/API.md @@ -0,0 +1,307 @@ +# The `apify` binding — API reference + +Inside a Code Mode script a global **`apify`** object exposes a small, typed +subset of the Apify API, authenticated with the current run's token. This +document describes every method in detail. + +## Conventions + +- **Every method is `async`** — `await` the result (or `for await` for + `dataset.iterate`). +- **One options object.** Each method takes a single object argument; there are + no positional parameters. +- **`actorId`** accepts either `username/name` (e.g. `apify/rag-web-browser`) or + the Actor's ID. +- **Errors.** A non-2xx API response throws an `Error` whose message is + ` failed: `. The one exception is + [`kvs.get`](#kvsget--value--null), which returns `null` for a missing key + instead of throwing. +- **Network.** Outbound `fetch` from your script is restricted to `apify.com` + and its subdomains. + +--- + +## `apify.actor` + +### `actor.search({ query, limit?, category? })` → `Actor[]` + +Search the Apify Store. + +| Param | Type | Required | Description | +|---|---|---|---| +| `query` | `string` | yes | Full-text search query. | +| `limit` | `number` | no | Maximum number of results. | +| `category` | `string` | no | Restrict to a Store category. | + +Returns the array of matching Store Actor records. + +```js +const actors = await apify.actor.search({ query: 'web scraper', limit: 5 }); +console.log(actors.map((a) => `${a.username}/${a.name}`).join('\n')); +``` + +### `actor.getDetails({ actorId })` → `Actor` + +Fetch the full record for one Actor. + +| Param | Type | Required | Description | +|---|---|---|---| +| `actorId` | `string` | yes | `username/name` or Actor ID. | + +### `actor.start({ actorId, input?, memoryMbytes?, timeoutSecs?, maxTotalChargeUsd?, maxItems? })` → `Run` + +Start an Actor **asynchronously** and return immediately with a run record in +`READY`/`RUNNING` state. Use [`run.wait`](#runwait--run) to block for the result. + +| Param | Type | Required | Description | +|---|---|---|---| +| `actorId` | `string` | yes | `username/name` or Actor ID. | +| `input` | `object` | no | Actor input (defaults to `{}`). | +| `memoryMbytes` | `number` | no | Memory limit for the run. | +| `timeoutSecs` | `number` | no | Run timeout in seconds. | +| `maxTotalChargeUsd` | `number` | no | Hard cap on the run's cost. | +| `maxItems` | `number` | no | Max dataset items for pay-per-result Actors. | + +### `actor.run({ actorId, input?, waitForFinishSecs?, memoryMbytes?, timeoutSecs?, maxTotalChargeUsd?, maxItems? })` → `Run` + +Start an Actor and **wait** for it to finish (or until `waitForFinishSecs` +elapses), then return the run record. + +| Param | Type | Required | Default | Description | +|---|---|---|---|---| +| `actorId` | `string` | yes | | `username/name` or Actor ID. | +| `input` | `object` | no | `{}` | Actor input. | +| `waitForFinishSecs` | `number` | no | `60` | Seconds to wait. **The Apify API caps a single wait at 60s** — for longer runs use `start()` + a `run.wait()` loop. | +| `memoryMbytes` | `number` | no | | Memory limit. | +| `timeoutSecs` | `number` | no | | Run timeout. | +| `maxTotalChargeUsd` | `number` | no | | Cost cap. | +| `maxItems` | `number` | no | | Max items (pay-per-result). | + +The run record exposes `defaultDatasetId` and `defaultKeyValueStoreId` for +reading results. + +### `actor.runAndGetItems({ actorId, input?, fields?, limit?, ...runOpts })` → `{ run, items }` + +Convenience wrapper: `actor.run(...)` followed by reading the run's default +dataset. + +| Param | Type | Required | Description | +|---|---|---|---| +| `actorId` | `string` | yes | `username/name` or Actor ID. | +| `input` | `object` | no | Actor input. | +| `fields` | `string[]` | no | Restrict returned item fields. | +| `limit` | `number` | no | Max items to fetch. | +| `...runOpts` | | no | Any `actor.run` option (`waitForFinishSecs`, `memoryMbytes`, `timeoutSecs`, `maxTotalChargeUsd`, `maxItems`). | + +Returns `{ run: Run, items: object[] }`. + +```js +const { run, items } = await apify.actor.runAndGetItems({ + actorId: 'apify/rag-web-browser', + input: { query: 'apify' }, + limit: 3, +}); +console.log(run.status, items.length); +``` + +--- + +## `apify.run` + +### `run.get({ runId })` → `Run` + +Fetch the current run record (status, stats, default storage IDs). + +| Param | Type | Required | Description | +|---|---|---|---| +| `runId` | `string` | yes | The run ID. | + +### `run.wait({ runId, waitForFinishSecs? })` → `Run` + +Block until the run terminates or `waitForFinishSecs` elapses, whichever comes +first, then return the run record. + +| Param | Type | Required | Default | Description | +|---|---|---|---|---| +| `runId` | `string` | yes | | The run ID. | +| `waitForFinishSecs` | `number` | no | `60` | Seconds to wait. **Capped at 60s by the API**; poll in a loop for longer runs. | + +### `run.abort({ runId })` → `Run` + +Abort a running run. Returns the run record (status transitions to `ABORTING`). + +| Param | Type | Required | Description | +|---|---|---|---| +| `runId` | `string` | yes | The run ID. | + +### `run.getLog({ runId, limit? })` → `string` + +Return the run's log as text. + +| Param | Type | Required | Description | +|---|---|---|---| +| `runId` | `string` | yes | The run ID. | +| `limit` | `number` | no | If set, return only the **last** `limit` characters (client-side tail; the full log is fetched). | + +--- + +## `apify.dataset` + +### `dataset.create({ name? })` → `Dataset` + +Create a dataset and return its record (use `.id` for subsequent calls). + +| Param | Type | Required | Description | +|---|---|---|---| +| `name` | `string` | no | Named (persistent) dataset; omit for an unnamed (temporary) one. | + +### `dataset.pushItems({ datasetId, items })` → `void` + +Append one or more items to a dataset. + +| Param | Type | Required | Description | +|---|---|---|---| +| `datasetId` | `string` | yes | Target dataset ID. | +| `items` | `object \| object[]` | yes | A single item or an array of items. | + +### `dataset.listItems({ datasetId, fields?, omit?, limit?, offset?, clean?, desc? })` → `object[]` + +Read a page of items. Returns the items array directly (no pagination wrapper). + +| Param | Type | Required | Description | +|---|---|---|---| +| `datasetId` | `string` | yes | Dataset ID. | +| `fields` | `string[]` | no | Only include these fields. | +| `omit` | `string[]` | no | Exclude these fields. | +| `limit` | `number` | no | Page size. | +| `offset` | `number` | no | Starting offset. | +| `clean` | `boolean` | no | Skip empty items / hidden fields. | +| `desc` | `boolean` | no | Reverse (newest first). | + +> A dataset's pagination total is eventually consistent right after creation, so +> no `total` is surfaced. Use [`getSchema`](#datasetgetschema--schema) for an +> item count, or [`iterate`](#datasetiterate--asyncgeneratorobject) to consume +> everything. + +### `dataset.iterate({ datasetId, fields?, omit?, clean?, desc?, batchSize? })` → `AsyncGenerator` + +Async-iterate the **entire** dataset, paging internally so you don't manage +offsets. + +| Param | Type | Required | Default | Description | +|---|---|---|---|---| +| `datasetId` | `string` | yes | | Dataset ID. | +| `fields` | `string[]` | no | | Only include these fields. | +| `omit` | `string[]` | no | | Exclude these fields. | +| `clean` | `boolean` | no | | Skip empty items / hidden fields. | +| `desc` | `boolean` | no | | Reverse order. | +| `batchSize` | `number` | no | `1000` | Items fetched per page. | + +```js +let count = 0; +for await (const item of apify.dataset.iterate({ datasetId })) count++; +console.log('total items:', count); +``` + +### `dataset.getSchema({ datasetId, sample? })` → `Schema` + +Infer a lightweight schema from a sample of items (Apify has no schema endpoint). + +| Param | Type | Required | Default | Description | +|---|---|---|---|---| +| `datasetId` | `string` | yes | | Dataset ID. | +| `sample` | `number` | no | `5` | Number of items to inspect. | + +Returns: + +```js +{ + itemCount, // number | undefined (dataset metadata; eventually consistent) + sampleSize, // number of items actually inspected + fields: [ // one entry per field seen in the sample + { name, types, nullable } // types: string[] (e.g. ['string','null']); nullable: boolean + ] +} +``` + +--- + +## `apify.kvs` + +### `kvs.create({ name? })` → `Store` + +Create a key-value store and return its record. + +| Param | Type | Required | Description | +|---|---|---|---| +| `name` | `string` | no | Named (persistent) store; omit for an unnamed (temporary) one. | + +### `kvs.set({ storeId, key, value, contentType? })` → `void` + +Write a record. The content type is inferred from `value`: + +| `value` type | Stored as | +|---|---| +| `object` | `application/json` | +| `string` | `text/plain; charset=utf-8` | +| `Uint8Array` / `ArrayBuffer` | `application/octet-stream` | + +| Param | Type | Required | Description | +|---|---|---|---| +| `storeId` | `string` | yes | Store ID. | +| `key` | `string` | yes | Record key. | +| `value` | `object \| string \| Uint8Array \| ArrayBuffer` | yes | Value to store. | +| `contentType` | `string` | no | Override the inferred content type. | + +### `kvs.get({ storeId, key })` → `value` \| `null` + +Read a record. The return type follows the stored content type: + +- `application/json` → parsed **object** +- `text/*` → **string** +- anything else → **`Uint8Array`** + +Returns **`null`** when the key does not exist (404), so you can do +lookup-or-default without a `try/catch`. + +| Param | Type | Required | Description | +|---|---|---|---| +| `storeId` | `string` | yes | Store ID. | +| `key` | `string` | yes | Record key. | + +### `kvs.list({ storeId, limit?, exclusiveStartKey? })` → `{ items, ... }` + +List keys in a store. + +| Param | Type | Required | Description | +|---|---|---|---| +| `storeId` | `string` | yes | Store ID. | +| `limit` | `number` | no | Max keys to return. | +| `exclusiveStartKey` | `string` | no | Continue listing after this key (pagination). | + +Returns the store-keys listing, whose `items` is an array of `{ key, size }`. + +--- + +## `console` + +`console` is captured, not printed live: + +| Method | Stream | +|---|---| +| `console.log`, `console.info` | **stdout** | +| `console.error`, `console.warn` | **stderr** | + +Non-string arguments are `JSON.stringify`'d. When the script finishes, both +streams are written to the run's default dataset as a single item: + +```json +{ "stdout": "...", "stderr": "..." } +``` + +## Error handling + +- A non-2xx API response throws `Error: failed: `. +- If your script throws, the error (stack/message) is appended to **stderr** and + the run still **succeeds** with whatever was printed beforehand — so failures + are observable in the output rather than crashing the run. From 4c3a430b3da5cbc63dc964b6e1d0846b78817c24 Mon Sep 17 00:00:00 2001 From: MQ37 Date: Wed, 24 Jun 2026 19:49:34 +0200 Subject: [PATCH 07/38] docs: document exact output shapes and link each method to its Apify API v2 endpoint --- docs/API.md | 140 +++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 105 insertions(+), 35 deletions(-) diff --git a/docs/API.md b/docs/API.md index 718be3f..3347393 100644 --- a/docs/API.md +++ b/docs/API.md @@ -12,10 +12,17 @@ document describes every method in detail. no positional parameters. - **`actorId`** accepts either `username/name` (e.g. `apify/rag-web-browser`) or the Actor's ID. +- **Return values.** The Apify API wraps most responses in a `{ "data": … }` + envelope. These methods **unwrap it for you** and return the inner value; the + linked API page describes that inner `data` shape. Where a method transforms + the response further (extracts an array, parses by content type, infers a + schema, returns plain text), the exact output is spelled out below. +- **`Apify API:`** each method links to its underlying endpoint on + [docs.apify.com/api/v2](https://docs.apify.com/api/v2) so you can inspect the + live request/response schema. - **Errors.** A non-2xx API response throws an `Error` whose message is ` failed: `. The one exception is - [`kvs.get`](#kvsget--value--null), which returns `null` for a missing key - instead of throwing. + [`kvs.get`](#kvsget--value--null), which returns `null` for a missing key. - **Network.** Outbound `fetch` from your script is restricted to `apify.com` and its subdomains. @@ -33,7 +40,9 @@ Search the Apify Store. | `limit` | `number` | no | Maximum number of results. | | `category` | `string` | no | Restrict to a Store category. | -Returns the array of matching Store Actor records. +**Output:** the `data.items` array of the Store listing (the pagination wrapper +is dropped) — i.e. an `Actor[]`. +**Apify API:** [`GET /v2/store`](https://docs.apify.com/api/v2/store-get) ```js const actors = await apify.actor.search({ query: 'web scraper', limit: 5 }); @@ -48,6 +57,9 @@ Fetch the full record for one Actor. |---|---|---|---| | `actorId` | `string` | yes | `username/name` or Actor ID. | +**Output:** the Actor object (unwrapped `data`). +**Apify API:** [`GET /v2/acts/{actorId}`](https://docs.apify.com/api/v2/act-get) + ### `actor.start({ actorId, input?, memoryMbytes?, timeoutSecs?, maxTotalChargeUsd?, maxItems? })` → `Run` Start an Actor **asynchronously** and return immediately with a run record in @@ -57,11 +69,14 @@ Start an Actor **asynchronously** and return immediately with a run record in |---|---|---|---| | `actorId` | `string` | yes | `username/name` or Actor ID. | | `input` | `object` | no | Actor input (defaults to `{}`). | -| `memoryMbytes` | `number` | no | Memory limit for the run. | -| `timeoutSecs` | `number` | no | Run timeout in seconds. | +| `memoryMbytes` | `number` | no | Memory limit for the run (`memory` query param). | +| `timeoutSecs` | `number` | no | Run timeout in seconds (`timeout`). | | `maxTotalChargeUsd` | `number` | no | Hard cap on the run's cost. | | `maxItems` | `number` | no | Max dataset items for pay-per-result Actors. | +**Output:** the Run object (unwrapped `data`). +**Apify API:** [`POST /v2/acts/{actorId}/runs`](https://docs.apify.com/api/v2/act-runs-post) + ### `actor.run({ actorId, input?, waitForFinishSecs?, memoryMbytes?, timeoutSecs?, maxTotalChargeUsd?, maxItems? })` → `Run` Start an Actor and **wait** for it to finish (or until `waitForFinishSecs` @@ -71,19 +86,21 @@ elapses), then return the run record. |---|---|---|---|---| | `actorId` | `string` | yes | | `username/name` or Actor ID. | | `input` | `object` | no | `{}` | Actor input. | -| `waitForFinishSecs` | `number` | no | `60` | Seconds to wait. **The Apify API caps a single wait at 60s** — for longer runs use `start()` + a `run.wait()` loop. | +| `waitForFinishSecs` | `number` | no | `60` | Seconds to wait (`waitForFinish`). **The Apify API caps a single wait at 60s** — for longer runs use `start()` + a `run.wait()` loop. | | `memoryMbytes` | `number` | no | | Memory limit. | | `timeoutSecs` | `number` | no | | Run timeout. | | `maxTotalChargeUsd` | `number` | no | | Cost cap. | | `maxItems` | `number` | no | | Max items (pay-per-result). | -The run record exposes `defaultDatasetId` and `defaultKeyValueStoreId` for -reading results. +**Output:** the Run object (unwrapped `data`), exposing `defaultDatasetId` and +`defaultKeyValueStoreId` for reading results. Uses the standard run endpoint +(not `/run-sync`, which returns the output record instead of the run object). +**Apify API:** [`POST /v2/acts/{actorId}/runs`](https://docs.apify.com/api/v2/act-runs-post) ### `actor.runAndGetItems({ actorId, input?, fields?, limit?, ...runOpts })` → `{ run, items }` Convenience wrapper: `actor.run(...)` followed by reading the run's default -dataset. +dataset via `dataset.listItems`. | Param | Type | Required | Description | |---|---|---|---| @@ -93,7 +110,17 @@ dataset. | `limit` | `number` | no | Max items to fetch. | | `...runOpts` | | no | Any `actor.run` option (`waitForFinishSecs`, `memoryMbytes`, `timeoutSecs`, `maxTotalChargeUsd`, `maxItems`). | -Returns `{ run: Run, items: object[] }`. +**Output (custom):** + +```js +{ + run: Run, // the run object, as actor.run returns + items: object[] // items from run.defaultDatasetId +} +``` + +**Apify API:** [`POST /v2/acts/{actorId}/runs`](https://docs.apify.com/api/v2/act-runs-post) +then [`GET /v2/datasets/{datasetId}/items`](https://docs.apify.com/api/v2/dataset-items-get) ```js const { run, items } = await apify.actor.runAndGetItems({ @@ -116,6 +143,9 @@ Fetch the current run record (status, stats, default storage IDs). |---|---|---|---| | `runId` | `string` | yes | The run ID. | +**Output:** the Run object (unwrapped `data`). +**Apify API:** [`GET /v2/actor-runs/{runId}`](https://docs.apify.com/api/v2/actor-run-get) + ### `run.wait({ runId, waitForFinishSecs? })` → `Run` Block until the run terminates or `waitForFinishSecs` elapses, whichever comes @@ -124,16 +154,23 @@ first, then return the run record. | Param | Type | Required | Default | Description | |---|---|---|---|---| | `runId` | `string` | yes | | The run ID. | -| `waitForFinishSecs` | `number` | no | `60` | Seconds to wait. **Capped at 60s by the API**; poll in a loop for longer runs. | +| `waitForFinishSecs` | `number` | no | `60` | Seconds to wait (`waitForFinish`). **Capped at 60s by the API**; poll in a loop for longer runs. | + +**Output:** the Run object (unwrapped `data`). +**Apify API:** [`GET /v2/actor-runs/{runId}`](https://docs.apify.com/api/v2/actor-run-get) ### `run.abort({ runId })` → `Run` Abort a running run. Returns the run record (status transitions to `ABORTING`). +Aborting a finished run is an API error. | Param | Type | Required | Description | |---|---|---|---| | `runId` | `string` | yes | The run ID. | +**Output:** the Run object (unwrapped `data`). +**Apify API:** [`POST /v2/actor-runs/{runId}/abort`](https://docs.apify.com/api/v2/act-run-abort-post) + ### `run.getLog({ runId, limit? })` → `string` Return the run's log as text. @@ -143,6 +180,10 @@ Return the run's log as text. | `runId` | `string` | yes | The run ID. | | `limit` | `number` | no | If set, return only the **last** `limit` characters (client-side tail; the full log is fetched). | +**Output (custom):** the raw log **text** (not JSON). With `limit`, the last +`limit` characters. +**Apify API:** [`GET /v2/logs/{runId}`](https://docs.apify.com/api/v2/log-get) + --- ## `apify.dataset` @@ -155,6 +196,9 @@ Create a dataset and return its record (use `.id` for subsequent calls). |---|---|---|---| | `name` | `string` | no | Named (persistent) dataset; omit for an unnamed (temporary) one. | +**Output:** the Dataset object (unwrapped `data`). +**Apify API:** [`POST /v2/datasets`](https://docs.apify.com/api/v2/datasets-post) + ### `dataset.pushItems({ datasetId, items })` → `void` Append one or more items to a dataset. @@ -164,29 +208,34 @@ Append one or more items to a dataset. | `datasetId` | `string` | yes | Target dataset ID. | | `items` | `object \| object[]` | yes | A single item or an array of items. | +**Output:** none (resolves once the items are stored). +**Apify API:** [`POST /v2/datasets/{datasetId}/items`](https://docs.apify.com/api/v2/dataset-items-post) + ### `dataset.listItems({ datasetId, fields?, omit?, limit?, offset?, clean?, desc? })` → `object[]` -Read a page of items. Returns the items array directly (no pagination wrapper). +Read a page of items. | Param | Type | Required | Description | |---|---|---|---| | `datasetId` | `string` | yes | Dataset ID. | -| `fields` | `string[]` | no | Only include these fields. | +| `fields` | `string[]` | no | Only include these fields (joined into `fields`). | | `omit` | `string[]` | no | Exclude these fields. | | `limit` | `number` | no | Page size. | | `offset` | `number` | no | Starting offset. | -| `clean` | `boolean` | no | Skip empty items / hidden fields. | -| `desc` | `boolean` | no | Reverse (newest first). | +| `clean` | `boolean` | no | Skip empty items / hidden fields (`clean=1`). | +| `desc` | `boolean` | no | Reverse (newest first, `desc=1`). | -> A dataset's pagination total is eventually consistent right after creation, so -> no `total` is surfaced. Use [`getSchema`](#datasetgetschema--schema) for an -> item count, or [`iterate`](#datasetiterate--asyncgeneratorobject) to consume -> everything. +**Output (custom):** the **items array directly** — this endpoint already +returns a bare array (no `data`/pagination wrapper). A dataset's pagination +total is eventually consistent right after creation, so no `total` is surfaced; +use [`getSchema`](#datasetgetschema--schema) for a count or +[`iterate`](#datasetiterate--asyncgeneratorobject) to consume everything. +**Apify API:** [`GET /v2/datasets/{datasetId}/items`](https://docs.apify.com/api/v2/dataset-items-get) ### `dataset.iterate({ datasetId, fields?, omit?, clean?, desc?, batchSize? })` → `AsyncGenerator` Async-iterate the **entire** dataset, paging internally so you don't manage -offsets. +offsets. Stops when a page returns fewer than `batchSize` items. | Param | Type | Required | Default | Description | |---|---|---|---|---| @@ -197,6 +246,9 @@ offsets. | `desc` | `boolean` | no | | Reverse order. | | `batchSize` | `number` | no | `1000` | Items fetched per page. | +**Output (custom):** an async generator yielding one item (`object`) at a time. +**Apify API:** [`GET /v2/datasets/{datasetId}/items`](https://docs.apify.com/api/v2/dataset-items-get) (paged internally) + ```js let count = 0; for await (const item of apify.dataset.iterate({ datasetId })) count++; @@ -212,18 +264,25 @@ Infer a lightweight schema from a sample of items (Apify has no schema endpoint) | `datasetId` | `string` | yes | | Dataset ID. | | `sample` | `number` | no | `5` | Number of items to inspect. | -Returns: +**Output (custom):** ```js { - itemCount, // number | undefined (dataset metadata; eventually consistent) + itemCount, // number | undefined — from the dataset metadata (eventually consistent) sampleSize, // number of items actually inspected - fields: [ // one entry per field seen in the sample - { name, types, nullable } // types: string[] (e.g. ['string','null']); nullable: boolean + fields: [ // one entry per field seen across the sample + { + name, // field name + types, // string[], e.g. ['string'] or ['number','null'] + nullable // boolean — true if any sampled value was null + } ] } ``` +**Apify API:** [`GET /v2/datasets/{datasetId}`](https://docs.apify.com/api/v2/dataset-get) +(for `itemCount`) + [`GET /v2/datasets/{datasetId}/items`](https://docs.apify.com/api/v2/dataset-items-get) (the sample) + --- ## `apify.kvs` @@ -236,13 +295,16 @@ Create a key-value store and return its record. |---|---|---|---| | `name` | `string` | no | Named (persistent) store; omit for an unnamed (temporary) one. | +**Output:** the Store object (unwrapped `data`). +**Apify API:** [`POST /v2/key-value-stores`](https://docs.apify.com/api/v2/key-value-stores-post) + ### `kvs.set({ storeId, key, value, contentType? })` → `void` Write a record. The content type is inferred from `value`: | `value` type | Stored as | |---|---| -| `object` | `application/json` | +| `object` | `application/json; charset=utf-8` | | `string` | `text/plain; charset=utf-8` | | `Uint8Array` / `ArrayBuffer` | `application/octet-stream` | @@ -253,23 +315,29 @@ Write a record. The content type is inferred from `value`: | `value` | `object \| string \| Uint8Array \| ArrayBuffer` | yes | Value to store. | | `contentType` | `string` | no | Override the inferred content type. | -### `kvs.get({ storeId, key })` → `value` \| `null` - -Read a record. The return type follows the stored content type: +**Output:** none (resolves once the record is stored). +**Apify API:** [`PUT /v2/key-value-stores/{storeId}/records/{key}`](https://docs.apify.com/api/v2/key-value-store-record-put) -- `application/json` → parsed **object** -- `text/*` → **string** -- anything else → **`Uint8Array`** +### `kvs.get({ storeId, key })` → `value` \| `null` -Returns **`null`** when the key does not exist (404), so you can do -lookup-or-default without a `try/catch`. +Read a record. | Param | Type | Required | Description | |---|---|---|---| | `storeId` | `string` | yes | Store ID. | | `key` | `string` | yes | Record key. | -### `kvs.list({ storeId, limit?, exclusiveStartKey? })` → `{ items, ... }` +**Output (custom):** the value, typed by the stored content type — + +- `application/json` → parsed **object** +- `text/*` → **string** +- anything else → **`Uint8Array`** + +Returns **`null`** when the key does not exist (404) instead of throwing, so you +can do lookup-or-default without a `try/catch`. +**Apify API:** [`GET /v2/key-value-stores/{storeId}/records/{key}`](https://docs.apify.com/api/v2/key-value-store-record-get) + +### `kvs.list({ storeId, limit?, exclusiveStartKey? })` → `{ items, … }` List keys in a store. @@ -279,7 +347,9 @@ List keys in a store. | `limit` | `number` | no | Max keys to return. | | `exclusiveStartKey` | `string` | no | Continue listing after this key (pagination). | -Returns the store-keys listing, whose `items` is an array of `{ key, size }`. +**Output:** the unwrapped `data`: `{ items: [{ key, size }], count, limit, +isTruncated, exclusiveStartKey, nextExclusiveStartKey }`. +**Apify API:** [`GET /v2/key-value-stores/{storeId}/keys`](https://docs.apify.com/api/v2/key-value-store-keys-get) --- From db006c37f8417f7367f9671f28f163c73c6fb23c Mon Sep 17 00:00:00 2001 From: MQ37 Date: Tue, 7 Jul 2026 13:11:59 +0200 Subject: [PATCH 08/38] feat: add exitCode to run output Write { stdout, stderr, exitCode } instead of { stdout, stderr }. exitCode is the user script's effective status: 0 when it returns normally, 1 when it throws. The Actor run itself still SUCCEEDS on a throw, so callers detect a failed script via exitCode rather than heuristics on stderr (console.error / console.warn are legitimate log channels). Update README and API docs. --- README.md | 11 +++++++---- docs/API.md | 17 ++++++++++++----- worker/runner.js | 9 ++++++++- 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 085eb7e..c3343e1 100644 --- a/README.md +++ b/README.md @@ -53,14 +53,17 @@ For full configuration options, use the configurator at ## Output -A single **dataset item** with the captured streams: +A single **dataset item** with the captured streams and the script's exit status: ```json -{ "stdout": "Apify: Full-stack web scraping ...\n...", "stderr": "" } +{ "stdout": "Apify: Full-stack web scraping ...\n...", "stderr": "", "exitCode": 0 } ``` -If the script throws, the error lands in `stderr`; `stdout` keeps whatever was -printed before the failure. +If the script throws, the error lands in `stderr`, `stdout` keeps whatever was +printed before the failure, and `exitCode` is `1`. The Actor run itself still +**succeeds** — `exitCode` is the reliable signal for a failed script (`0` = the +script returned normally, `1` = it threw), since `stderr` is also a legitimate +log channel (`console.error` / `console.warn`). ## Permissions & safety diff --git a/docs/API.md b/docs/API.md index 3347393..c746fb1 100644 --- a/docs/API.md +++ b/docs/API.md @@ -363,15 +363,22 @@ isTruncated, exclusiveStartKey, nextExclusiveStartKey }`. | `console.error`, `console.warn` | **stderr** | Non-string arguments are `JSON.stringify`'d. When the script finishes, both -streams are written to the run's default dataset as a single item: +streams and the script's exit status are written to the run's default dataset as +a single item: ```json -{ "stdout": "...", "stderr": "..." } +{ "stdout": "...", "stderr": "...", "exitCode": 0 } ``` +`exitCode` is the script's effective exit status: `0` when it returns normally, +`1` when it throws. It is distinct from the Actor run's status — see below. + ## Error handling - A non-2xx API response throws `Error: failed: `. -- If your script throws, the error (stack/message) is appended to **stderr** and - the run still **succeeds** with whatever was printed beforehand — so failures - are observable in the output rather than crashing the run. +- If your script throws, the error (stack/message) is appended to **stderr**, + `exitCode` is set to `1`, and the run still **succeeds** with whatever was + printed beforehand — so failures are observable in the output rather than + crashing the run. Check `exitCode` (not `stderr`) to detect a failed script: + `stderr` may be non-empty from ordinary `console.error` / `console.warn` + logging even on a successful run. diff --git a/worker/runner.js b/worker/runner.js index 18b57ac..c7b192e 100644 --- a/worker/runner.js +++ b/worker/runner.js @@ -274,13 +274,20 @@ export default { // A thrown program is a user-level failure: capture it in stderr and still // push the output, so the run SUCCEEDS with diagnostics. Infra failures // (missing env, dataset push) throw and fail the run. + // + // exitCode is the user script's effective status, distinct from the Actor run's + // status: 0 when the script returns normally, 1 when it throws. The run itself + // still SUCCEEDS on a throw, so callers detect a failed script via this field + // rather than heuristics on stderr (console.error is a legitimate log channel). + let exitCode = 0; try { await run(makeApifyBinding(token, apiV2), captureConsole); } catch (err) { stderr.push(err?.stack ?? err?.message ?? String(err)); + exitCode = 1; } - await pushOutput(apiV2, token, env, { stdout: stdout.join('\n'), stderr: stderr.join('\n') }); + await pushOutput(apiV2, token, env, { stdout: stdout.join('\n'), stderr: stderr.join('\n'), exitCode }); return Response.json({ ok: true }); }, }; From 6486cd0e79ed16067cc3b55bc752ac210a4925e4 Mon Sep 17 00:00:00 2001 From: MQ37 Date: Tue, 7 Jul 2026 14:27:45 +0200 Subject: [PATCH 09/38] fix: remove nodejs_compat to close sandbox egress and token leaks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The workerd nodejs_compat flag exposed Node built-ins to user code, which broke the sandbox's egress boundary two ways (ai-team#216): - node:net gave a raw-socket egress path that bypassed guard.js's *.apify.com fetch allowlist — arbitrary public hosts were reachable (finding A). - process.env exposed the run's APIFY_TOKEN to user code (finding B). It also made the docs' 'no imports' claim false (finding C). runner.js and guard.js use only web-standard APIs (fetch/URL/Response/ Uint8Array), so dropping the flag needs no code changes. Without it, node:* imports fail and process/require are undefined, while fetch and the apify binding keep working. Add tests/sandbox-isolation.js (run via test.sh, same deploy+call pattern as binding-smoke.js) asserting node builtins are blocked, process/require are undefined, fetch works, guard.js still blocks non-apify hosts, and the apify binding still works. Verified on the deployed Actor: isolation 9/9, binding smoke 18/18. Update docs to state no imports are available. --- README.md | 11 ++++++-- test.sh | 37 ++++++++++++++----------- tests/sandbox-isolation.js | 55 ++++++++++++++++++++++++++++++++++++++ worker/config.capnp | 6 ++++- 4 files changed, 90 insertions(+), 19 deletions(-) create mode 100644 tests/sandbox-isolation.js diff --git a/README.md b/README.md index c3343e1..9cd014e 100644 --- a/README.md +++ b/README.md @@ -31,8 +31,10 @@ For full configuration options, use the configurator at - **One script per run.** The Actor reads your `code`, runs it once, writes the result, and exits. - The code runs inside a [`workerd`](https://github.com/cloudflare/workerd) V8 - isolate: **no filesystem, no package imports**, and outbound network is - restricted to `*.apify.com`. + isolate: **no imports** — neither npm packages nor Node built-in `node:*` + modules are available (`import`/`require` of any module fails); web-standard + globals such as `fetch` are present. Outbound network is restricted to + `*.apify.com`. - Inside the script a global **`apify`** object exposes a small, typed subset of the Apify API — run Actors, read/write datasets and key-value stores — using the current run's token (see below). @@ -69,6 +71,11 @@ log channel (`console.error` / `console.warn`). - Runs with **limited permissions**: the sandbox has no filesystem and can reach only the Apify API (`*.apify.com`). +- **No imports.** The isolate runs without workerd's `nodejs_compat`, so user + code cannot import Node built-ins (`node:net`, `node:fs`, …) or npm packages. + This removes `node:net` — a raw-socket egress path that would otherwise bypass + the `fetch` allowlist — and keeps the run token out of `process.env` (which is + not defined). - Each run is an isolated, single-use container — nothing persists between runs. ## The `apify` binding diff --git a/test.sh b/test.sh index 6d72c0d..ed09ad5 100755 --- a/test.sh +++ b/test.sh @@ -1,13 +1,15 @@ #!/bin/sh -# Deploy the Actor (apify push) and run the binding smoke test on the freshly -# built version via `apify call`. Exits non-zero if any binding check fails. +# Deploy the Actor (apify push) once, then run each test probe on the freshly +# built version via `apify call`. Each probe is a script submitted as the `code` +# input; it prints ALL_TESTS_PASSED on success. Exits non-zero if any probe fails. # # Usage: ./test.sh set -eu cd "$(dirname "$0")" -TEST_JS="tests/binding-smoke.js" +# Probes run against the built Actor. Add a file here to register a new probe. +PROBES="tests/binding-smoke.js tests/sandbox-isolation.js" command -v apify >/dev/null 2>&1 || { echo "apify CLI not found" >&2; exit 1; } command -v jq >/dev/null 2>&1 || { echo "jq not found" >&2; exit 1; } @@ -18,16 +20,19 @@ trap 'rm -f "$input_json"' EXIT echo "==> apify push" apify push -echo "==> building input from ${TEST_JS}" -jq -n --arg code "$(cat "$TEST_JS")" '{ code: $code }' > "$input_json" - -echo "==> apify call (running the test on the built Actor)" -output="$(apify call -f "$input_json" -o 2>&1)" -echo "$output" - -if printf '%s' "$output" | grep -q 'ALL_TESTS_PASSED'; then - echo "==> all binding tests passed" -else - echo "==> binding tests FAILED" >&2 - exit 1 -fi +failed=0 +for probe in $PROBES; do + echo "==> apify call: ${probe}" + jq -n --arg code "$(cat "$probe")" '{ code: $code }' > "$input_json" + output="$(apify call -f "$input_json" -o 2>&1)" + echo "$output" + if printf '%s' "$output" | grep -q 'ALL_TESTS_PASSED'; then + echo "==> ${probe} passed" + else + echo "==> ${probe} FAILED" >&2 + failed=1 + fi +done + +[ "$failed" -eq 0 ] || { echo "==> some probes FAILED" >&2; exit 1; } +echo "==> all probes passed" diff --git a/tests/sandbox-isolation.js b/tests/sandbox-isolation.js new file mode 100644 index 0000000..c091059 --- /dev/null +++ b/tests/sandbox-isolation.js @@ -0,0 +1,55 @@ +// Sandbox isolation test. Submitted as the Actor's `code` input by test.sh and +// executed on the built Actor via `apify call`. Asserts the sandbox boundary +// holds and prints a sentinel line (ALL_TESTS_PASSED) that test.sh greps for. +// +// Guards github.com/apify/ai-team#216 (findings A, B, C): the isolate runs +// WITHOUT workerd's nodejs_compat, so user code cannot reach Node built-ins. +// That removes node:net (a raw-socket egress path that bypassed guard.js's +// *.apify.com fetch allowlist — finding A) and process.env (which held the +// run's APIFY_TOKEN — finding B), and makes the "no imports" docs accurate +// (finding C). fetch() and the apify binding must still work. +const results = []; +function check(name, cond, detail = '') { + if (cond) { + console.log(`PASS ${name}: ${detail}`); + results.push(true); + } else { + console.error(`FAIL ${name}: ${detail}`); + results.push(false); + } +} + +// Node built-ins must NOT be importable (no nodejs_compat). +for (const mod of ['node:net', 'node:fs', 'node:dns', 'node:child_process']) { + let imported = false; + try { await import(mod); imported = true; } catch { /* expected */ } + check(`import ${mod} blocked`, !imported, imported ? 'IMPORTED (leak!)' : 'blocked'); +} + +// The run token lives in process.env under nodejs_compat; without it, process +// and require must be undefined so user code can't read the credential. +check('process undefined', typeof process === 'undefined', `typeof process = ${typeof process}`); +check('require undefined', typeof require === 'undefined', `typeof require = ${typeof require}`); + +// fetch must remain — the apify binding depends on it. +check('fetch available', typeof fetch === 'function', `typeof fetch = ${typeof fetch}`); + +// guard.js must still block a non-apify host over fetch. +let nonApifyBlocked = false; +try { await fetch('https://example.com'); } catch { nonApifyBlocked = true; } +check('non-apify fetch blocked', nonApifyBlocked, nonApifyBlocked ? 'blocked' : 'REACHED example.com (leak!)'); + +// The apify binding must still work (fetch to *.apify.com). +let bindingWorks = false; +try { + const found = await apify.actor.search({ query: 'hello world', limit: 1 }); + bindingWorks = Array.isArray(found); +} catch (e) { + console.error(`apify.actor.search threw: ${e.message}`); +} +check('apify binding works', bindingWorks, bindingWorks ? 'actor.search ok' : 'binding broken'); + +const passed = results.filter(Boolean).length; +console.log(`\n=== SUMMARY: ${passed}/${results.length} passed ===`); +if (passed === results.length) console.log('ALL_TESTS_PASSED'); +else console.error(`SOME_TESTS_FAILED (${results.length - passed} failed)`); diff --git a/worker/config.capnp b/worker/config.capnp index e7288de..cad84c8 100644 --- a/worker/config.capnp +++ b/worker/config.capnp @@ -35,5 +35,9 @@ const codeRuntime :Workerd.Worker = ( ], globalOutbound = "internet", compatibilityDate = "2026-01-15", - compatibilityFlags = ["nodejs_compat"], + # No nodejs_compat: user code runs with web-standard APIs only. This is a + # security boundary, not a convenience toggle — the flag would expose node:net + # (a raw-socket egress path that bypasses guard.js's *.apify.com fetch allowlist) + # and process.env (which holds the run's APIFY_TOKEN). runner.js and guard.js use + # only web-standard APIs (fetch/URL/Response/Uint8Array), so they need nothing here. ); From 7a091e247d74130891946201cb77072b297d4866 Mon Sep 17 00:00:00 2001 From: MQ37 Date: Tue, 7 Jul 2026 14:33:33 +0200 Subject: [PATCH 10/38] test: assert fetch allowlist covers apify.com + subdomains only Expand the sandbox isolation probe: positively verify apify.com and an *.apify.com subdomain are allowed, and that guard.js blocks unrelated hosts, subdomain/userinfo look-alikes (evilapify.com, apify.com.evil.com, apify.com@evil.com), and the cloud metadata IP. Classify by the guard's 'Blocked fetch' rejection, not by request success. Verified on the deployed Actor: 15/15. --- tests/sandbox-isolation.js | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/tests/sandbox-isolation.js b/tests/sandbox-isolation.js index c091059..6f8cfe0 100644 --- a/tests/sandbox-isolation.js +++ b/tests/sandbox-isolation.js @@ -34,10 +34,36 @@ check('require undefined', typeof require === 'undefined', `typeof require = ${t // fetch must remain — the apify binding depends on it. check('fetch available', typeof fetch === 'function', `typeof fetch = ${typeof fetch}`); -// guard.js must still block a non-apify host over fetch. -let nonApifyBlocked = false; -try { await fetch('https://example.com'); } catch { nonApifyBlocked = true; } -check('non-apify fetch blocked', nonApifyBlocked, nonApifyBlocked ? 'blocked' : 'REACHED example.com (leak!)'); +// guard.js allowlist: apify.com and *.apify.com only. A guard rejection throws +// synchronously with a "Blocked fetch" message BEFORE any network I/O; anything +// else (a real network/HTTP error) means guard let the request through. So we +// classify by the error message, not by whether the request ultimately succeeds. +async function guardBlocks(url) { + try { + await fetch(url); + return false; // request went out — guard allowed it + } catch (e) { + return /Blocked fetch/.test(e.message); // guard rejection vs. network error + } +} + +// Allowed: the main domain and any subdomain must NOT be guard-blocked. +for (const url of ['https://apify.com/', 'https://api.apify.com/v2/browser-info']) { + check(`allow ${url}`, !(await guardBlocks(url)), 'not blocked by guard'); +} + +// Blocked: other public hosts, subdomain look-alikes, userinfo/host tricks, and +// the cloud metadata IP must all be guard-blocked. +const blockedTargets = [ + 'https://example.com/', // unrelated public host + 'https://evilapify.com/', // suffix without the dot — must not match .apify.com + 'https://apify.com.evil.com/', // real host is evil.com + 'https://apify.com@evil.com/', // userinfo trick — real host is evil.com + 'http://169.254.169.254/', // cloud link-local metadata +]; +for (const url of blockedTargets) { + check(`block ${url}`, await guardBlocks(url), 'blocked by guard'); +} // The apify binding must still work (fetch to *.apify.com). let bindingWorks = false; From 2493f54b0f25fa427f79f42c7e63d2348c7d8985 Mon Sep 17 00:00:00 2001 From: MQ37 Date: Tue, 7 Jul 2026 16:55:28 +0200 Subject: [PATCH 11/38] fix: block WebSocket and EventSource egress in guard.js Removing nodejs_compat closed the node:net egress path, but WebSocket and EventSource are web-standard globals present without that flag and connect directly, not through the fetch guard. A script could open a wss:// or SSE channel to any public host and exfiltrate data around the *.apify.com allowlist (apify/ai-team#216 finding A, second primitive). runner.js and the apify binding use only fetch, so neutralize both globals (non-configurable throwing stubs). This closes the JS egress surface: fetch is allowlisted, WebSocket/EventSource are removed, and raw sockets need module imports that are already blocked. Extend the isolation probe with wss:// and SSE egress cases. Verified on the deployed Actor: isolation 17/17, binding 18/18. --- tests/sandbox-isolation.js | 17 +++++++++++++++++ worker/guard.js | 32 ++++++++++++++++++++++++++++---- 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/tests/sandbox-isolation.js b/tests/sandbox-isolation.js index 6f8cfe0..fda8b3d 100644 --- a/tests/sandbox-isolation.js +++ b/tests/sandbox-isolation.js @@ -65,6 +65,23 @@ for (const url of blockedTargets) { check(`block ${url}`, await guardBlocks(url), 'blocked by guard'); } +// Non-fetch egress primitives must be neutralized: WebSocket and EventSource +// are web-standard globals that connect directly (not through the fetch guard), +// so a script could otherwise open a wss:// / SSE channel to any public host and +// exfiltrate around the *.apify.com allowlist (apify/ai-team#216 finding A). +function blocksConstruct(name, url) { + const Ctor = globalThis[name]; + if (typeof Ctor !== 'function') return true; // absent → not an egress path + try { + new Ctor(url); + return false; // constructed → egress opened + } catch (e) { + return /Blocked/.test(e.message); // our guard rejection vs. any other error + } +} +check('WebSocket blocked', blocksConstruct('WebSocket', 'wss://echo.websocket.org'), 'no wss egress'); +check('EventSource blocked', blocksConstruct('EventSource', 'https://example.com/sse'), 'no SSE egress'); + // The apify binding must still work (fetch to *.apify.com). let bindingWorks = false; try { diff --git a/worker/guard.js b/worker/guard.js index 426a170..2d2d6e8 100644 --- a/worker/guard.js +++ b/worker/guard.js @@ -1,11 +1,16 @@ -// Restrict the user program's global fetch() to apify.com and its subdomains. -// Imported before usercode.js so the override is in place even for code that +// Restrict the user program's outbound network to apify.com and its subdomains. +// Imported before usercode.js so the overrides are in place even for code that // runs at module-evaluation time. Our own Apify API calls use the exported // realFetch (the internal API is a private IP, not *.apify.com), so they are // unaffected by this guard. // -// NOTE: this guards the fetch() API only. It is not a complete egress boundary — -// the airtight control is workerd's globalOutbound. See SECURITY notes in the repo. +// Egress surface (workerd, no nodejs_compat): the only JS-reachable outbound +// primitives are fetch, WebSocket, and EventSource. Raw sockets (node:net, +// cloudflare:sockets connect()) need module imports, which are already blocked. +// fetch is allowlisted below; WebSocket and EventSource are removed outright +// because runner.js and the apify binding never use them — leaving them would +// be a non-fetch egress path around the allowlist (apify/ai-team#216 finding A, +// via WebSocket). If a future need arises, wrap them like fetch instead. const realFetch = globalThis.fetch.bind(globalThis); @@ -42,4 +47,23 @@ globalThis.fetch = (input, init) => { return realFetch(input, init); }; +// Remove the non-fetch egress primitives. These are web-standard globals present +// even without nodejs_compat, and they connect directly (not through the fetch +// guard), so a script could otherwise open a wss:// or SSE connection to any +// public host and exfiltrate data around the *.apify.com allowlist. +function blockGlobal(name) { + const blocked = function () { + throw new Error(`Blocked ${name}: only fetch() to apify.com and its subdomains is allowed`); + }; + Object.defineProperty(globalThis, name, { + value: blocked, + writable: false, + configurable: false, + enumerable: false, + }); +} +for (const name of ['WebSocket', 'EventSource']) { + if (name in globalThis) blockGlobal(name); +} + export { realFetch }; From 5461aeec650cf3c3af225f7356f37337c1c65953 Mon Sep 17 00:00:00 2001 From: MQ37 Date: Mon, 13 Jul 2026 14:12:56 +0200 Subject: [PATCH 12/38] fix: close realFetch allowlist bypass via one-shot claim in guard.js guard.js exported realFetch as a standing module binding. Since ES modules are singleton-cached, the sandboxed user script could recover it at runtime via `(await import('./guard.js')).realFetch(...)`, fully bypassing the *.apify.com fetch allowlist (and reaching Apify's private network, since the outbound service allows public/private/local). Reopened exactly the egress class apify/ai-team#216 was meant to close. guard.js now exposes claimRealFetch(), a one-shot accessor that hands out the real fetch once and nulls its internal reference. runner.js claims it during its own module load, strictly before usercode.js can run. A later import('./guard.js') from inside the script reaches the same cached module instance, but the value is already gone. Verified against real workerd, before/after, same PoC: - before: guard.js exports ["realFetch"] -> BYPASS SUCCEEDED, exfil request logged on a disallowed listener. - after: guard.js exports ["claimRealFetch"] -> BYPASS FAILED (null), zero hits on the disallowed listener. --- worker/guard.js | 24 +++++++++++++++++++----- worker/runner.js | 11 +++++++++-- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/worker/guard.js b/worker/guard.js index 2d2d6e8..b14ae12 100644 --- a/worker/guard.js +++ b/worker/guard.js @@ -1,8 +1,9 @@ // Restrict the user program's outbound network to apify.com and its subdomains. // Imported before usercode.js so the overrides are in place even for code that -// runs at module-evaluation time. Our own Apify API calls use the exported -// realFetch (the internal API is a private IP, not *.apify.com), so they are -// unaffected by this guard. +// runs at module-evaluation time. Our own Apify API calls use the real, +// unrestricted fetch (the internal API is a private IP, not *.apify.com) — +// see claimRealFetch() below for how runner.js gets it without leaving it +// reachable from user code. // // Egress surface (workerd, no nodejs_compat): the only JS-reachable outbound // primitives are fetch, WebSocket, and EventSource. Raw sockets (node:net, @@ -14,6 +15,21 @@ const realFetch = globalThis.fetch.bind(globalThis); +// One-shot handoff of the unrestricted fetch to runner.js. ES modules are +// evaluated once and cached, so `guard.js` is the same module instance no +// matter who imports it. runner.js imports this module (and calls +// claimRealFetch()) before usercode.js is ever imported, so it always claims +// first. If the sandboxed script later does `await import('./guard.js')` to +// try to recover the unrestricted fetch, it gets this same cached instance — +// but the value is already gone. A standing `export { realFetch }` would hand +// it to that later import too; don't reintroduce one. +let unclaimedRealFetch = realFetch; +export function claimRealFetch() { + const fetchFn = unclaimedRealFetch; + unclaimedRealFetch = null; + return fetchFn; +} + // Match apify.com exactly or any subdomain. The leading dot in the suffix is // what rejects look-alikes: `evilapify.com` (no dot) and `apify.com.evil.com` // (ends with `.evil.com`) both fail. @@ -65,5 +81,3 @@ function blockGlobal(name) { for (const name of ['WebSocket', 'EventSource']) { if (name in globalThis) blockGlobal(name); } - -export { realFetch }; diff --git a/worker/runner.js b/worker/runner.js index c7b192e..2e6b1b8 100644 --- a/worker/runner.js +++ b/worker/runner.js @@ -8,10 +8,17 @@ // Loader / per-request isolate is needed — the program runs in this worker, // which is itself the sandbox (no filesystem, restricted outbound network). // guard.js must be imported before usercode.js: it overrides globalThis.fetch -// to allow only apify.com, and exports realFetch for our own (internal) API calls. -import { realFetch } from './guard.js'; +// to allow only apify.com, and hands us the real, unrestricted fetch via a +// one-shot claimRealFetch() for our own (internal) API calls — see guard.js +// for why this is a claim, not a standing export. +import { claimRealFetch } from './guard.js'; import { run } from './usercode.js'; +// Must run before usercode.js's `run()` is ever invoked (it does, here — module +// evaluation order puts this ahead of any dynamic import from inside `run()`). +const realFetch = claimRealFetch(); +if (!realFetch) throw new Error('realFetch already claimed — guard.js imported out of order.'); + const DEFAULT_ITERATE_BATCH = 1000; const DEFAULT_GET_SCHEMA_SAMPLE = 5; From c36f402eb3d276438ece76b963d6e94b0b91b97a Mon Sep 17 00:00:00 2001 From: MQ37 Date: Mon, 13 Jul 2026 14:16:10 +0200 Subject: [PATCH 13/38] refactor: rename single-letter locals and de-duplicate the loopback port runner.js: r -> response, d -> page, [k, v] -> [key, value], ct -> contentType / resolvedContentType. Matches code-quality and apify-coding-standards naming rules (no single-letter/abbreviated locals). config.capnp + entrypoint.sh both hardcoded the loopback port 8787 independently. config.capnp now takes a __PORT__ placeholder that entrypoint.sh substitutes from its own $PORT before starting workerd \u2014 one source of truth. --- worker/config.capnp | 4 +++- worker/entrypoint.sh | 3 +++ worker/runner.js | 48 ++++++++++++++++++++++---------------------- 3 files changed, 30 insertions(+), 25 deletions(-) diff --git a/worker/config.capnp b/worker/config.capnp index cad84c8..c5d380b 100644 --- a/worker/config.capnp +++ b/worker/config.capnp @@ -9,10 +9,12 @@ const config :Workerd.Config = ( # the hostname allowlist is enforced by guard.js, not here. (name = "internet", network = (allow = ["public", "private", "local"], tlsOptions = (trustBrowserCas = true))), ], + # __PORT__ is substituted by entrypoint.sh from its own $PORT before workerd + # starts — single source of truth, see entrypoint.sh. sockets = [ ( name = "http", - address = "127.0.0.1:8787", + address = "127.0.0.1:__PORT__", http = (), service = "main", ), diff --git a/worker/entrypoint.sh b/worker/entrypoint.sh index 27892bd..7f9b473 100755 --- a/worker/entrypoint.sh +++ b/worker/entrypoint.sh @@ -35,6 +35,9 @@ fi printf '\n}\n' } > /app/worker/usercode.js +# config.capnp hardcodes __PORT__ as a placeholder so the port has one source ($PORT above). +sed -i "s/__PORT__/${PORT}/" /app/worker/config.capnp + /usr/local/bin/workerd serve --experimental /app/worker/config.capnp & workerd_pid=$! trap 'kill "$workerd_pid" 2>/dev/null || true' EXIT diff --git a/worker/runner.js b/worker/runner.js index 2e6b1b8..7ca82b4 100644 --- a/worker/runner.js +++ b/worker/runner.js @@ -34,8 +34,8 @@ function makeApifyBinding(token, apiV2) { const buildUrl = (path, searchParams) => { const url = new URL(`${apiV2}${path}`); if (searchParams) { - for (const [k, v] of Object.entries(searchParams)) { - if (v !== undefined && v !== null) url.searchParams.set(k, String(v)); + for (const [key, value] of Object.entries(searchParams)) { + if (value !== undefined && value !== null) url.searchParams.set(key, String(value)); } } return url; @@ -50,9 +50,9 @@ function makeApifyBinding(token, apiV2) { init.body = isRaw ? body : JSON.stringify(body); init.headers['content-type'] = contentType ?? (isRaw ? 'application/octet-stream' : 'application/json'); } - const r = await realFetch(buildUrl(path, searchParams), init); - if (!r.ok) throw new Error(`${method} ${path} failed: ${r.status} ${await r.text()}`); - return r; + const response = await realFetch(buildUrl(path, searchParams), init); + if (!response.ok) throw new Error(`${method} ${path} failed: ${response.status} ${await response.text()}`); + return response; }; const apiJson = async (...args) => (await apiCall(...args)).json(); @@ -62,7 +62,7 @@ function makeApifyBinding(token, apiV2) { // GET /v2/store — Apify Store search. Returns the items array directly. search: ({ query, limit, category }) => apiData('GET', '/store', { searchParams: { search: query, limit, category } }) - .then((d) => d.items), + .then((page) => page.items), getDetails: ({ actorId }) => apiData('GET', `/acts/${encodeURIComponent(actorId)}`), @@ -115,8 +115,8 @@ function makeApifyBinding(token, apiV2) { // Returns the full run log as text. `limit` tails the last N characters; the Apify API // does not paginate logs, so this is a client-side slice (the full body is fetched). getLog: async ({ runId, limit }) => { - const r = await apiCall('GET', `/logs/${encodeURIComponent(runId)}`); - const text = await r.text(); + const response = await apiCall('GET', `/logs/${encodeURIComponent(runId)}`); + const text = await response.text(); return limit && text.length > limit ? text.slice(-limit) : text; }, }; @@ -127,7 +127,7 @@ function makeApifyBinding(token, apiV2) { // (eventually consistent), so we don't surface a `total`. Use `getSchema` if you // need an item count, or iterate to consume the whole dataset. listItems: async ({ datasetId, fields, omit, limit, offset, clean, desc }) => { - const r = await apiCall('GET', `/datasets/${encodeURIComponent(datasetId)}/items`, { + const response = await apiCall('GET', `/datasets/${encodeURIComponent(datasetId)}/items`, { searchParams: { fields: fields?.join(','), omit: omit?.join(','), @@ -137,7 +137,7 @@ function makeApifyBinding(token, apiV2) { desc: desc ? '1' : undefined, }, }); - return r.json(); + return response.json(); }, // Async generator over the entire dataset. Pages internally in `batchSize` chunks @@ -203,34 +203,34 @@ function makeApifyBinding(token, apiV2) { // Returns null when the key does not exist (404), not an error — this matches the common // "lookup or default" pattern in code. get: async ({ storeId, key }) => { - const r = await realFetch(buildUrl(`/key-value-stores/${encodeURIComponent(storeId)}/records/${encodeURIComponent(key)}`), { + const response = await realFetch(buildUrl(`/key-value-stores/${encodeURIComponent(storeId)}/records/${encodeURIComponent(key)}`), { headers: baseHeaders, }); - if (r.status === 404) return null; - if (!r.ok) throw new Error(`GET kvs.get failed: ${r.status} ${await r.text()}`); - const ct = r.headers.get('content-type') ?? ''; - if (ct.includes('application/json')) return r.json(); - if (ct.startsWith('text/')) return r.text(); - return new Uint8Array(await r.arrayBuffer()); + if (response.status === 404) return null; + if (!response.ok) throw new Error(`GET kvs.get failed: ${response.status} ${await response.text()}`); + const contentType = response.headers.get('content-type') ?? ''; + if (contentType.includes('application/json')) return response.json(); + if (contentType.startsWith('text/')) return response.text(); + return new Uint8Array(await response.arrayBuffer()); }, // `value`: object → application/json; string → text/plain; Uint8Array/ArrayBuffer → // application/octet-stream (or whatever the caller passed via `contentType`). set: async ({ storeId, key, value, contentType }) => { let body; - let ct = contentType; + let resolvedContentType = contentType; if (value instanceof Uint8Array || value instanceof ArrayBuffer) { body = value; - ct = ct ?? 'application/octet-stream'; + resolvedContentType = resolvedContentType ?? 'application/octet-stream'; } else if (typeof value === 'string') { body = value; - ct = ct ?? 'text/plain; charset=utf-8'; + resolvedContentType = resolvedContentType ?? 'text/plain; charset=utf-8'; } else { body = JSON.stringify(value); - ct = ct ?? 'application/json; charset=utf-8'; + resolvedContentType = resolvedContentType ?? 'application/json; charset=utf-8'; } await apiCall('PUT', `/key-value-stores/${encodeURIComponent(storeId)}/records/${encodeURIComponent(key)}`, { - body, contentType: ct, + body, contentType: resolvedContentType, }); }, @@ -250,12 +250,12 @@ function makeApifyBinding(token, apiV2) { async function pushOutput(apiV2, token, env, item) { const datasetId = env.DEFAULT_DATASET_ID || env.DEFAULT_DATASET_ID_LEGACY; if (!datasetId) throw new Error('Default dataset ID missing from Actor run environment.'); - const r = await realFetch(`${apiV2}/datasets/${encodeURIComponent(datasetId)}/items`, { + const response = await realFetch(`${apiV2}/datasets/${encodeURIComponent(datasetId)}/items`, { method: 'POST', headers: { Authorization: `Bearer ${token}`, 'content-type': 'application/json; charset=utf-8' }, body: JSON.stringify(item), }); - if (!r.ok) throw new Error(`Failed to push dataset item: ${r.status} ${await r.text()}`); + if (!response.ok) throw new Error(`Failed to push dataset item: ${response.status} ${await response.text()}`); } export default { From 4607d26c8aede1891d8c70e39ec5ab94b8535f2d Mon Sep 17 00:00:00 2001 From: MQ37 Date: Mon, 13 Jul 2026 14:52:58 +0200 Subject: [PATCH 14/38] fix: close redirect-following allowlist bypass, lock globalThis.fetch guard.js validated only the initial URL; fetch defaults to redirect:'follow', so an allowlisted *.apify.com host issuing a 3xx to any other host was followed out silently. Verified live against real workerd: an allowed host redirecting to a disallowed one reached it (before) / was blocked (after). Now fetches with redirect:'manual' and re-validates each Location against the allowlist before following, one hop at a time (capped at 5), with WHATWG-spec method/body downgrade rules (303 always -> GET; 301/302 -> GET only if the original method was POST; 307/308 preserve method + body). Also lock globalThis.fetch via Object.defineProperty(writable:false, configurable:false), matching the existing WebSocket/EventSource treatment (a plain assignment could be reassigned/deleted by the sandboxed script). Verified live: reassigning globalThis.fetch from a script now throws TypeError instead of succeeding. --- worker/guard.js | 48 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/worker/guard.js b/worker/guard.js index b14ae12..662fddb 100644 --- a/worker/guard.js +++ b/worker/guard.js @@ -45,7 +45,9 @@ function requestUrl(input) { return String(input); } -globalThis.fetch = (input, init) => { +// Parses and validates one URL against the allowlist. Returns the parsed URL +// (callers use it to resolve a relative redirect Location) or throws. +function validateUrl(input) { let url; try { // Parse to the real host — defeats userinfo (`apify.com@evil.com`), @@ -60,8 +62,48 @@ globalThis.fetch = (input, init) => { if (!isAllowedHost(url.hostname)) { throw new Error(`Blocked fetch to "${url.hostname}": only apify.com and its subdomains are allowed`); } - return realFetch(input, init); -}; + return url; +} + +// fetch() follows redirects internally by default, invisibly to a wrapper that +// only checks the initial URL — an allowlisted host could 302 to anywhere. +// Follow redirects ourselves, one hop at a time, and re-validate each Location +// against the allowlist before following it. Status-to-method mapping matches +// the WHATWG fetch spec: 303 always downgrades to GET; 301/302 downgrade to +// GET only when the original method was POST; 307/308 preserve method + body. +const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]); +const MAX_REDIRECT_HOPS = 5; + +function nextRedirectInit(init, status) { + const method = (init?.method ?? 'GET').toUpperCase(); + const downgradeToGet = status === 303 || ((status === 301 || status === 302) && method === 'POST'); + if (!downgradeToGet) return init; + return { ...init, method: 'GET', body: undefined }; +} + +async function guardedFetch(input, init, hop = 0) { + if (hop > MAX_REDIRECT_HOPS) { + throw new Error(`Blocked fetch: exceeded ${MAX_REDIRECT_HOPS} redirects`); + } + const url = validateUrl(input); + const response = await realFetch(input, { ...init, redirect: 'manual' }); + if (!REDIRECT_STATUSES.has(response.status)) return response; + const location = response.headers.get('location'); + if (!location) return response; // redirect status with no Location: nothing to follow + const nextUrl = new URL(location, url); // resolves a relative Location against the current URL + return guardedFetch(nextUrl.href, nextRedirectInit(init, response.status), hop + 1); +} + +// writable:false + configurable:false, matching blockGlobal() below — a plain +// assignment could be overwritten or deleted by the sandboxed script to +// recover the ambient (real, unrestricted) fetch reference some engines +// expose under a different name; locking it closes that off. +Object.defineProperty(globalThis, 'fetch', { + value: (input, init) => guardedFetch(input, init), + writable: false, + configurable: false, + enumerable: true, +}); // Remove the non-fetch egress primitives. These are web-standard globals present // even without nodejs_compat, and they connect directly (not through the fetch From 11e7c968af0e686b9b1d9090bd6e48092d593979 Mon Sep 17 00:00:00 2001 From: MQ37 Date: Mon, 13 Jul 2026 14:55:13 +0200 Subject: [PATCH 15/38] fix: catch usercode.js compile failures, scope run.abort, freeze bindings usercode.js wraps the user's code inside 'export async function run(...) { ... }' with nothing else at module scope, so a syntax error in it fails module evaluation before any request reaches runner.js's try/catch \u2014 workerd exits immediately (verified: raw workerd binary, exit code 1, no /health response, ever). The reviewer's proposed fix (dynamic import inside the try/catch) does NOT help: verified live that workerd eagerly evaluates every module declared in config.capnp regardless of static vs dynamic import, so both crash identically. The actual fix has to live in entrypoint.sh, which is what this commit does: - entrypoint.sh now captures workerd's stderr and, if workerd exits before becoming ready, checks whether the crash names usercode.js. If so (this is structurally exact, not a heuristic, given usercode.js's shape above) it pushes a { stdout: '', stderr, exitCode: 1, statusMessage: 'Failed to compile: ...' } item directly and exits 0, so the run SUCCEEDS with diagnostics instead of failing as an opaque infra error. Any other crash (our own runner.js/guard.js, config issue) still hard-fails the run. Verified live for both the positive (usercode.js) and negative (unrelated file) cases. - runner.js adds the same statusMessage field ('Script completed' / 'Script threw: ...') to the normal exitCode 0/1 path, so callers get a prose signal without branching on exitCode. - run.abort({ runId }) is now scoped to run IDs this script itself started. actor.run()/actor.start() share one createRun() helper that records each created run's ID in a Set; abort() throws on an unrecognized runId instead of silently no-op'ing (a script shouldn't think an abort succeeded when it didn't). Previously any account-wide runId could be aborted. Verified live. - console and the apify binding's namespaces are now Object.freeze'd so a script can't reassign e.g. console.log to corrupt its own output capture. Verified live: reassignment now throws TypeError. - entrypoint.sh also validates the default dataset ID up front (needed by the new compile-failure push path), matching the existing token/KV-store check. --- worker/entrypoint.sh | 45 ++++++++++++++++--- worker/runner.js | 105 ++++++++++++++++++++++++++++--------------- 2 files changed, 109 insertions(+), 41 deletions(-) diff --git a/worker/entrypoint.sh b/worker/entrypoint.sh index 7f9b473..bcdb6ce 100755 --- a/worker/entrypoint.sh +++ b/worker/entrypoint.sh @@ -12,10 +12,12 @@ READINESS_ATTEMPTS=100 # 100 * 0.1s = 10s budget for workerd to bind the socke API_BASE="${APIFY_API_BASE_URL:-https://api.apify.com}" API_BASE="${API_BASE%/}" # APIFY_API_BASE_URL ships with a trailing slash STORE_ID="${ACTOR_DEFAULT_KEY_VALUE_STORE_ID:-${APIFY_DEFAULT_KEY_VALUE_STORE_ID:-}}" +DATASET_ID="${ACTOR_DEFAULT_DATASET_ID:-${APIFY_DEFAULT_DATASET_ID:-}}" INPUT_KEY="${APIFY_INPUT_KEY:-INPUT}" +WORKERD_STDERR=/tmp/workerd.err -if [ -z "${APIFY_TOKEN:-}" ] || [ -z "$STORE_ID" ]; then - echo "[code-runtime] missing APIFY_TOKEN or default key-value store ID" >&2 +if [ -z "${APIFY_TOKEN:-}" ] || [ -z "$STORE_ID" ] || [ -z "$DATASET_ID" ]; then + echo "[code-runtime] missing APIFY_TOKEN or default key-value store / dataset ID" >&2 exit 1 fi @@ -38,12 +40,44 @@ fi # config.capnp hardcodes __PORT__ as a placeholder so the port has one source ($PORT above). sed -i "s/__PORT__/${PORT}/" /app/worker/config.capnp -/usr/local/bin/workerd serve --experimental /app/worker/config.capnp & +/usr/local/bin/workerd serve --experimental /app/worker/config.capnp 2>"$WORKERD_STDERR" & workerd_pid=$! trap 'kill "$workerd_pid" 2>/dev/null || true' EXIT +# push_compile_failure reports a usercode.js syntax error as a normal, SUCCEEDED script +# result (same contract as a script that throws at runtime) instead of failing the whole +# Actor run. usercode.js wraps the user's `code` inside `export async function run(...) { +# ... }` with nothing else at module scope, so nothing in it can fail to *parse* except +# that inserted code — a startup crash naming usercode.js is therefore always a syntax +# error in the user's script, never our own code. See detection below. +push_compile_failure() { + crash_log=$(cat "$WORKERD_STDERR" 2>/dev/null || true) + echo "[code-runtime] usercode.js failed to compile: $crash_log" >&2 + item=$(jq -n --arg stderr "$crash_log" \ + '{stdout: "", stderr: $stderr, exitCode: 1, statusMessage: ("Failed to compile: " + $stderr)}') + push_status=$(curl -sS -o /dev/null -w '%{http_code}' -X POST \ + "${API_BASE}/v2/datasets/${DATASET_ID}/items" \ + -H "Authorization: Bearer ${APIFY_TOKEN}" \ + -H 'content-type: application/json; charset=utf-8' \ + --data-binary "$item") + case "$push_status" in + 2??) exit 0 ;; + *) + echo "[code-runtime] failed to push compile-failure diagnostic (HTTP $push_status)" >&2 + exit 1 + ;; + esac +} + attempt=0 until curl -sf "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; do + if ! kill -0 "$workerd_pid" 2>/dev/null; then + if grep -q 'usercode\.js' "$WORKERD_STDERR" 2>/dev/null; then + push_compile_failure + fi + echo "[code-runtime] workerd exited before startup: $(cat "$WORKERD_STDERR" 2>/dev/null)" >&2 + exit 1 + fi attempt=$((attempt + 1)) if [ "$attempt" -ge "$READINESS_ATTEMPTS" ]; then echo "[code-runtime] workerd did not become ready in time" >&2 @@ -52,6 +86,7 @@ until curl -sf "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; do sleep 0.1 done -# Trigger the single run. The worker runs the program and pushes { stdout, stderr } -# to the default dataset. A non-2xx response fails the Actor run (curl -f). +# Trigger the single run. The worker runs the program and pushes { stdout, stderr, +# exitCode, statusMessage } to the default dataset. A non-2xx response fails the Actor +# run (curl -f). curl -fsS -X POST "http://127.0.0.1:${PORT}/run" diff --git a/worker/runner.js b/worker/runner.js index 7ca82b4..9a0dd79 100644 --- a/worker/runner.js +++ b/worker/runner.js @@ -1,8 +1,8 @@ // Single worker for the per-run code-runtime Actor. It runs the user's program // (imported from the generated `usercode.js` module) with the `apify` REST -// binding and a captured `console`, then pushes `{ stdout, stderr }` to the -// run's default dataset. The container entrypoint generates `usercode.js`, -// boots workerd, and triggers `/run` once. +// binding and a captured `console`, then pushes `{ stdout, stderr, exitCode, +// statusMessage }` to the run's default dataset. The container entrypoint +// generates `usercode.js`, boots workerd, and triggers `/run` once. // // Single-tenant: one run = one container = one program = one token. No Worker // Loader / per-request isolate is needed — the program runs in this worker, @@ -58,6 +58,33 @@ function makeApifyBinding(token, apiV2) { const apiJson = async (...args) => (await apiCall(...args)).json(); const apiData = async (...args) => (await apiJson(...args)).data; + // Run IDs this script itself started, via actor.run() / actor.start() (and transitively + // actor.runAndGetItems(), which calls actor.run()). run.abort() below is scoped to this + // set — a script can only abort runs it started, not any account-wide runId it's handed + // or guesses. + const startedRunIds = new Set(); + + // POST /acts/:id/runs, shared by actor.run() (start+wait, waitForFinishSecs defaults to 60, + // capped at 60s per the Apify API — for longer runs use start() + apify.run.wait()) and + // actor.start() (async kickoff, no wait). Returns the run record so the caller can read + // defaultDatasetId / defaultKeyValueStoreId. Intentionally does NOT use /run-sync, which + // returns the OUTPUT KVS record (a pattern only some Actors follow) rather than the + // structured run record. + const createRun = ({ actorId, input, memoryMbytes, timeoutSecs, waitForFinishSecs, maxTotalChargeUsd, maxItems }) => + apiData('POST', `/acts/${encodeURIComponent(actorId)}/runs`, { + searchParams: { + waitForFinish: waitForFinishSecs, + memory: memoryMbytes, + timeout: timeoutSecs, + maxTotalChargeUsd, + maxItems, + }, + body: input ?? {}, + }).then((runRecord) => { + startedRunIds.add(runRecord.id); + return runRecord; + }); + const actor = { // GET /v2/store — Apify Store search. Returns the items array directly. search: ({ query, limit, category }) => @@ -67,34 +94,13 @@ function makeApifyBinding(token, apiV2) { getDetails: ({ actorId }) => apiData('GET', `/acts/${encodeURIComponent(actorId)}`), - // POST /runs with waitForFinish blocks until the run completes (max 60s per the - // Apify API; for longer runs the caller should use start() + apify.run.wait()). - // Returns the run record so the caller can read defaultDatasetId / defaultKeyValueStoreId. - // Intentionally does NOT use /run-sync, which returns the OUTPUT KVS record (a pattern - // only some Actors follow) rather than the structured run record. - run: ({ actorId, input, memoryMbytes, timeoutSecs, waitForFinishSecs = 60, maxTotalChargeUsd, maxItems }) => - apiData('POST', `/acts/${encodeURIComponent(actorId)}/runs`, { - searchParams: { - waitForFinish: waitForFinishSecs, - memory: memoryMbytes, - timeout: timeoutSecs, - maxTotalChargeUsd, - maxItems, - }, - body: input ?? {}, - }), + // Shared by run() and start(): both POST /acts/:id/runs, differing only in whether + // waitForFinish is set. Records the created run's ID in startedRunIds so run.abort() + // can be scoped to runs this script itself started (see the run.abort definition below). + run: (opts) => createRun({ waitForFinishSecs: 60, ...opts }), // Async kickoff. Returns immediately with a run record in READY/RUNNING state. - start: ({ actorId, input, memoryMbytes, timeoutSecs, maxTotalChargeUsd, maxItems }) => - apiData('POST', `/acts/${encodeURIComponent(actorId)}/runs`, { - searchParams: { - memory: memoryMbytes, - timeout: timeoutSecs, - maxTotalChargeUsd, - maxItems, - }, - body: input ?? {}, - }), + start: (opts) => createRun(opts), // runAndGetItems is added below once `dataset.listItems` is defined. }; @@ -109,8 +115,15 @@ function makeApifyBinding(token, apiV2) { searchParams: { waitForFinish: waitForFinishSecs }, }), - abort: ({ runId }) => - apiData('POST', `/actor-runs/${encodeURIComponent(runId)}/abort`), + // Scoped to runs this script itself started (see startedRunIds above) — without this, + // any runId a script is handed (e.g. read from a dataset item, or guessed) could abort + // an unrelated, account-wide run. + abort: ({ runId }) => { + if (!startedRunIds.has(runId)) { + throw new Error(`Blocked run.abort: "${runId}" was not started by this script`); + } + return apiData('POST', `/actor-runs/${encodeURIComponent(runId)}/abort`); + }, // Returns the full run log as text. `limit` tails the last N characters; the Apify API // does not paginate logs, so this is a client-side slice (the full body is fetched). @@ -243,7 +256,14 @@ function makeApifyBinding(token, apiV2) { apiData('POST', '/key-value-stores', { searchParams: { name } }), }; - return { actor, run, dataset, kvs }; + // Freeze every namespace (and the wrapper) so the script can't reassign a method to + // corrupt its own behavior or, for `console` below, its own output capture. + return Object.freeze({ + actor: Object.freeze(actor), + run: Object.freeze(run), + dataset: Object.freeze(dataset), + kvs: Object.freeze(kvs), + }); } // Push the captured streams as a single item to the run's default dataset. @@ -271,12 +291,13 @@ export default { const stdout = []; const stderr = []; - const captureConsole = { + // Frozen so the script can't reassign e.g. console.log to corrupt its own capture. + const captureConsole = Object.freeze({ log: (...args) => stdout.push(args.map(stringify).join(' ')), error: (...args) => stderr.push(args.map(stringify).join(' ')), warn: (...args) => stderr.push(args.map(stringify).join(' ')), info: (...args) => stdout.push(args.map(stringify).join(' ')), - }; + }); // A thrown program is a user-level failure: capture it in stderr and still // push the output, so the run SUCCEEDS with diagnostics. Infra failures @@ -286,15 +307,27 @@ export default { // status: 0 when the script returns normally, 1 when it throws. The run itself // still SUCCEEDS on a throw, so callers detect a failed script via this field // rather than heuristics on stderr (console.error is a legitimate log channel). + // statusMessage carries the same signal in prose, for callers that don't want to + // branch on exitCode. A script that fails to *compile* never reaches this handler at + // all (workerd fails the whole run before any request arrives) — entrypoint.sh + // handles that case directly, see its statusMessage "Failed to compile: ...". let exitCode = 0; + let statusMessage = 'Script completed'; try { await run(makeApifyBinding(token, apiV2), captureConsole); } catch (err) { - stderr.push(err?.stack ?? err?.message ?? String(err)); + const message = err?.message ?? String(err); + stderr.push(err?.stack ?? message); exitCode = 1; + statusMessage = `Script threw: ${message}`; } - await pushOutput(apiV2, token, env, { stdout: stdout.join('\n'), stderr: stderr.join('\n'), exitCode }); + await pushOutput(apiV2, token, env, { + stdout: stdout.join('\n'), + stderr: stderr.join('\n'), + exitCode, + statusMessage, + }); return Response.json({ ok: true }); }, }; From 5bc33749af0e93fffa626189a612123a2d034637 Mon Sep 17 00:00:00 2001 From: MQ37 Date: Mon, 13 Jul 2026 14:57:52 +0200 Subject: [PATCH 16/38] refactor!: rename actor.getDetails() to actor.get() Council-reviewed rename (both a data-structure and an interface-design lens independently converged on this one): actor.get matches run.get's existing naming convention in this same binding, which getDetails did not. Breaking change, acceptable pre-release. docs/API.md and README.md's binding summary updated to match; the fuller docs/description self-containment pass (a separate, later PR) will carry this through the rest of the copy. --- README.md | 2 +- docs/API.md | 2 +- worker/runner.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9cd014e..8dbbf67 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ Every method takes one options object and returns parsed JSON ```js // Actors apify.actor.search({ query, limit?, category? }) // → actors[] -apify.actor.getDetails({ actorId }) // → actor +apify.actor.get({ actorId }) // → actor apify.actor.start({ actorId, input?, memoryMbytes?, timeoutSecs?, maxTotalChargeUsd?, maxItems? }) // → run apify.actor.run({ actorId, ...startOpts, waitForFinishSecs = 60 }) // → run (waits) apify.actor.runAndGetItems({ actorId, input?, fields?, limit?, ...runOpts }) // → { run, items } diff --git a/docs/API.md b/docs/API.md index c746fb1..b57da64 100644 --- a/docs/API.md +++ b/docs/API.md @@ -49,7 +49,7 @@ const actors = await apify.actor.search({ query: 'web scraper', limit: 5 }); console.log(actors.map((a) => `${a.username}/${a.name}`).join('\n')); ``` -### `actor.getDetails({ actorId })` → `Actor` +### `actor.get({ actorId })` → `Actor` Fetch the full record for one Actor. diff --git a/worker/runner.js b/worker/runner.js index 9a0dd79..6005ce0 100644 --- a/worker/runner.js +++ b/worker/runner.js @@ -91,7 +91,7 @@ function makeApifyBinding(token, apiV2) { apiData('GET', '/store', { searchParams: { search: query, limit, category } }) .then((page) => page.items), - getDetails: ({ actorId }) => + get: ({ actorId }) => apiData('GET', `/acts/${encodeURIComponent(actorId)}`), // Shared by run() and start(): both POST /acts/:id/runs, differing only in whether From d59cec3e1c9334a3c38908a8e5e6dcd4fcc5dcdb Mon Sep 17 00:00:00 2001 From: MQ37 Date: Mon, 13 Jul 2026 16:56:35 +0200 Subject: [PATCH 17/38] refactor: migrate worker/*.js to TypeScript, add CI type-checking worker/guard.js and worker/runner.js -> guard.ts / runner.ts, with real types for the apify binding's full public surface (every method's options object named and typed, not `any`), the Env/Run/ApifyRecord shapes this code reads, and the redirect/fetch-guard helpers. Compiles clean under strict:true. API response shapes stay honestly `any` at the api{Json,Data} boundary -- the Apify API's JSON envelope isn't something this repo has a verified schema for, and asserting a precise shape we haven't checked would be worse than not typing it. Everything this code actually authors (every destructured options object, every local helper's params/return) is fully typed. Dockerfile's builder stage already runs Node (to resolve the workerd binary path via require('workerd')) -- compiling TypeScript there is one more RUN line, not a new toolchain. The runtime stage still copies only the compiled JS + entrypoint.sh + config.capnp; no Node, no npm packages, no .ts sources ship in the final image. worker/runner.js and worker/guard.js are now build artifacts (pnpm build / pnpm typecheck), gitignored, generated from the .ts source at Docker build time -- matching how the workerd binary itself is already obtained via the same Node stage. Added .github/workflows/typecheck.yml (pnpm typecheck on push/PR) -- this is the actual CI gap the reviewer flagged; it needs no Apify credentials, unlike test.sh's live apify push + call, so it's safe to run automatically. Verified zero behavioral regression: reran the full live-workerd suite (valid script, usercode.js compile-failure diagnostic, runtime throw, run.abort scoping, console/apify-binding freeze, redirect-following allowlist bypass) against the freshly compiled output -- all six identical to the pre-migration JS. --- .github/workflows/typecheck.yml | 19 +++ .gitignore | 4 + Dockerfile | 21 ++- package.json | 7 + pnpm-lock.yaml | 11 ++ tsconfig.json | 10 ++ worker/{guard.js => guard.ts} | 20 +-- worker/{runner.js => runner.ts} | 287 +++++++++++++++++++++++++------- worker/usercode.d.ts | 7 + 9 files changed, 308 insertions(+), 78 deletions(-) create mode 100644 .github/workflows/typecheck.yml create mode 100644 tsconfig.json rename worker/{guard.js => guard.ts} (89%) rename worker/{runner.js => runner.ts} (60%) create mode 100644 worker/usercode.d.ts diff --git a/.github/workflows/typecheck.yml b/.github/workflows/typecheck.yml new file mode 100644 index 0000000..ccac2fd --- /dev/null +++ b/.github/workflows/typecheck.yml @@ -0,0 +1,19 @@ +name: Typecheck + +on: + push: + branches: [master] + pull_request: + +jobs: + typecheck: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: pnpm + - run: pnpm install --frozen-lockfile + - run: pnpm run typecheck diff --git a/.gitignore b/.gitignore index 1e13533..e347e03 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,8 @@ node_modules/ +# Generated at container startup by entrypoint.sh (never checked in): worker/usercode.js +# Compiled from worker/*.ts by `pnpm build` (tsconfig.json emits next to the source): +worker/runner.js +worker/guard.js *.log .DS_Store diff --git a/Dockerfile b/Dockerfile index b675bf7..9fe4169 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,21 +1,27 @@ # Two-stage build: drop the Node runtime entirely. workerd is a standalone # glibc binary; only libc + libm are needed at runtime (verified via `ldd`). # -# Stage 1: pull the workerd binary via a Node base. pnpm keeps it in the virtual -# store (not hoisted), so resolve the path through `require('workerd')`. +# Stage 1: pull the workerd binary + compile worker/*.ts -> worker/*.js, via a Node +# base. Node already runs here to resolve workerd's binary path, so compiling +# TypeScript is one more RUN line, not a new toolchain. pnpm keeps workerd in the +# virtual store (not hoisted), so resolve the path through `require('workerd')`. FROM node:24-bookworm-slim AS builder WORKDIR /build -COPY package.json pnpm-lock.yaml ./ +COPY package.json pnpm-lock.yaml tsconfig.json ./ +COPY worker/ ./worker/ # --ignore-scripts skips workerd's postinstall (a binary-download fallback we # don't need — the binary ships in the @cloudflare/workerd-linux-64 optional dep) -# and avoids pnpm's hard error on unapproved dependency build scripts. +# and avoids pnpm's hard error on unapproved dependency build scripts. Full +# (non --prod) install: typescript is a devDependency, needed by `pnpm build` below. RUN corepack enable \ - && pnpm install --prod --frozen-lockfile --ignore-scripts \ + && pnpm install --frozen-lockfile --ignore-scripts \ + && pnpm run build \ && BIN="$(node -e "process.stdout.write(require('workerd').default)")" \ && cp "$BIN" /workerd \ && chmod +x /workerd -# Stage 2: minimal runtime — debian + ca-certificates + the workerd binary. +# Stage 2: minimal runtime — debian + ca-certificates + the workerd binary + the +# compiled JS. No Node, no TypeScript, no npm packages in this image. FROM debian:bookworm-slim # curl: loopback HTTP client + Actor-input fetch; jq: extract `code` from the input JSON. @@ -26,6 +32,7 @@ RUN apt-get update \ COPY --from=builder /workerd /usr/local/bin/workerd WORKDIR /app -COPY worker/ ./worker/ +COPY worker/entrypoint.sh worker/config.capnp ./worker/ +COPY --from=builder /build/worker/runner.js /build/worker/guard.js ./worker/ ENTRYPOINT ["sh", "/app/worker/entrypoint.sh"] diff --git a/package.json b/package.json index 33190c6..8de1c9f 100644 --- a/package.json +++ b/package.json @@ -9,5 +9,12 @@ }, "engines": { "node": ">=24" + }, + "scripts": { + "build": "tsc -p tsconfig.json", + "typecheck": "tsc --noEmit -p tsconfig.json" + }, + "devDependencies": { + "typescript": "6.0.3" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 95ed99b..8d7fee7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11,6 +11,10 @@ importers: workerd: specifier: 1.20260402.1 version: 1.20260402.1 + devDependencies: + typescript: + specifier: 6.0.3 + version: 6.0.3 packages: @@ -44,6 +48,11 @@ packages: cpu: [x64] os: [win32] + typescript@6.0.3: + resolution: {integrity: sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==} + engines: {node: '>=14.17'} + hasBin: true + workerd@1.20260402.1: resolution: {integrity: sha512-Cg+OUlukdcCHrTTg0MBCIMFRE6XO3yGVGiWCnJPvfffy2Ga2girrEq3qF/YlHSTmbIyEE5ebCFxBYYYZueQ/Mg==} engines: {node: '>=16'} @@ -66,6 +75,8 @@ snapshots: '@cloudflare/workerd-windows-64@1.20260402.1': optional: true + typescript@6.0.3: {} + workerd@1.20260402.1: optionalDependencies: '@cloudflare/workerd-darwin-64': 1.20260402.1 diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..ac781ef --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,10 @@ +{ + "compilerOptions": { + "target": "es2022", + "module": "es2022", + "moduleResolution": "bundler", + "lib": ["es2022", "dom"], + "strict": true + }, + "include": ["worker/*.ts"] +} diff --git a/worker/guard.js b/worker/guard.ts similarity index 89% rename from worker/guard.js rename to worker/guard.ts index 662fddb..17840ef 100644 --- a/worker/guard.js +++ b/worker/guard.ts @@ -23,8 +23,8 @@ const realFetch = globalThis.fetch.bind(globalThis); // try to recover the unrestricted fetch, it gets this same cached instance — // but the value is already gone. A standing `export { realFetch }` would hand // it to that later import too; don't reintroduce one. -let unclaimedRealFetch = realFetch; -export function claimRealFetch() { +let unclaimedRealFetch: typeof realFetch | null = realFetch; +export function claimRealFetch(): typeof realFetch | null { const fetchFn = unclaimedRealFetch; unclaimedRealFetch = null; return fetchFn; @@ -33,12 +33,12 @@ export function claimRealFetch() { // Match apify.com exactly or any subdomain. The leading dot in the suffix is // what rejects look-alikes: `evilapify.com` (no dot) and `apify.com.evil.com` // (ends with `.evil.com`) both fail. -function isAllowedHost(hostname) { +function isAllowedHost(hostname: string): boolean { const host = hostname.toLowerCase().replace(/\.$/, ''); // strip FQDN trailing dot return host === 'apify.com' || host.endsWith('.apify.com'); } -function requestUrl(input) { +function requestUrl(input: RequestInfo | URL): string { if (typeof input === 'string') return input; if (input instanceof URL) return input.href; if (input && typeof input.url === 'string') return input.url; // Request @@ -47,8 +47,8 @@ function requestUrl(input) { // Parses and validates one URL against the allowlist. Returns the parsed URL // (callers use it to resolve a relative redirect Location) or throws. -function validateUrl(input) { - let url; +function validateUrl(input: RequestInfo | URL): URL { + let url: URL; try { // Parse to the real host — defeats userinfo (`apify.com@evil.com`), // path/query/fragment (`evil.com/apify.com`) and similar tricks. @@ -74,14 +74,14 @@ function validateUrl(input) { const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]); const MAX_REDIRECT_HOPS = 5; -function nextRedirectInit(init, status) { +function nextRedirectInit(init: RequestInit | undefined, status: number): RequestInit | undefined { const method = (init?.method ?? 'GET').toUpperCase(); const downgradeToGet = status === 303 || ((status === 301 || status === 302) && method === 'POST'); if (!downgradeToGet) return init; return { ...init, method: 'GET', body: undefined }; } -async function guardedFetch(input, init, hop = 0) { +async function guardedFetch(input: RequestInfo | URL, init: RequestInit | undefined, hop = 0): Promise { if (hop > MAX_REDIRECT_HOPS) { throw new Error(`Blocked fetch: exceeded ${MAX_REDIRECT_HOPS} redirects`); } @@ -99,7 +99,7 @@ async function guardedFetch(input, init, hop = 0) { // recover the ambient (real, unrestricted) fetch reference some engines // expose under a different name; locking it closes that off. Object.defineProperty(globalThis, 'fetch', { - value: (input, init) => guardedFetch(input, init), + value: (input: RequestInfo | URL, init?: RequestInit) => guardedFetch(input, init), writable: false, configurable: false, enumerable: true, @@ -109,7 +109,7 @@ Object.defineProperty(globalThis, 'fetch', { // even without nodejs_compat, and they connect directly (not through the fetch // guard), so a script could otherwise open a wss:// or SSE connection to any // public host and exfiltrate data around the *.apify.com allowlist. -function blockGlobal(name) { +function blockGlobal(name: string): void { const blocked = function () { throw new Error(`Blocked ${name}: only fetch() to apify.com and its subdomains is allowed`); }; diff --git a/worker/runner.js b/worker/runner.ts similarity index 60% rename from worker/runner.js rename to worker/runner.ts index 6005ce0..f8d71bb 100644 --- a/worker/runner.js +++ b/worker/runner.ts @@ -16,22 +16,172 @@ import { run } from './usercode.js'; // Must run before usercode.js's `run()` is ever invoked (it does, here — module // evaluation order puts this ahead of any dynamic import from inside `run()`). -const realFetch = claimRealFetch(); -if (!realFetch) throw new Error('realFetch already claimed — guard.js imported out of order.'); +// Factored into a function (rather than a bare `const` + `if (!x) throw`) so the +// non-null guarantee is encoded in the return type once, here — TS doesn't carry +// a narrowed-from-null check across the later function declarations that close +// over `realFetch`, but a return type with the `null` branch already thrown away +// needs no further narrowing anywhere downstream. +function requireRealFetch(): typeof globalThis.fetch { + const fetchFn = claimRealFetch(); + if (!fetchFn) throw new Error('realFetch already claimed — guard.js imported out of order.'); + return fetchFn; +} +const realFetch = requireRealFetch(); const DEFAULT_ITERATE_BATCH = 1000; const DEFAULT_GET_SCHEMA_SAMPLE = 5; -function stringify(x) { +// --- Types --------------------------------------------------------------- +// The Apify API returns many more fields per record than this code reads. Rather +// than inventing a full schema we don't have, ApifyRecord asserts nothing beyond +// "a JSON object" and each specific shape below only names the fields this code +// actually consumes. + +interface ApifyRecord { + [key: string]: unknown; +} + +interface RunRecord extends ApifyRecord { + id: string; +} + +type SearchParamValue = string | number | boolean | undefined | null; +type SearchParams = Record; + +interface ApiCallOptions { + searchParams?: SearchParams; + body?: unknown; + contentType?: string; +} + +interface SearchOptions { + query: string; + limit?: number; + category?: string; +} + +interface ActorIdOptions { + actorId: string; +} + +interface StartOptions { + actorId: string; + input?: unknown; + memoryMbytes?: number; + timeoutSecs?: number; + waitForFinishSecs?: number; + maxTotalChargeUsd?: number; + maxItems?: number; +} + +interface RunAndGetItemsOptions extends StartOptions { + fields?: string[]; + limit?: number; +} + +interface RunIdOptions { + runId: string; +} + +interface WaitOptions extends RunIdOptions { + waitForFinishSecs?: number; +} + +interface GetLogOptions extends RunIdOptions { + limit?: number; +} + +interface DatasetListOptions { + datasetId: string; + fields?: string[]; + omit?: string[]; + limit?: number; + offset?: number; + clean?: boolean; + desc?: boolean; +} + +interface DatasetIterateOptions extends Omit { + batchSize?: number; +} + +interface DatasetSchemaOptions { + datasetId: string; + sample?: number; +} + +interface DatasetSchema { + itemCount: unknown; + sampleSize: number; + fields: { name: string; types: string[]; nullable: boolean }[]; +} + +interface CreateOptions { + name?: string; +} + +interface PushItemsOptions { + datasetId: string; + items: unknown[]; +} + +interface KvsGetOptions { + storeId: string; + key: string; +} + +interface KvsSetOptions extends KvsGetOptions { + value: unknown; + contentType?: string; +} + +interface KvsListOptions { + storeId: string; + limit?: number; + exclusiveStartKey?: string; +} + +interface ConsoleLike { + log: (...args: unknown[]) => void; + error: (...args: unknown[]) => void; + warn: (...args: unknown[]) => void; + info: (...args: unknown[]) => void; +} + +interface Env { + APIFY_TOKEN?: string; + DEFAULT_DATASET_ID?: string; + DEFAULT_DATASET_ID_LEGACY?: string; + API_BASE_URL?: string; +} + +interface OutputItem { + stdout: string; + stderr: string; + exitCode: number; + statusMessage: string; +} + +// --------------------------------------------------------------------------- + +function stringify(x: unknown): string { if (typeof x === 'string') return x; try { return JSON.stringify(x); } catch { return String(x); } } -function makeApifyBinding(token, apiV2) { - const baseHeaders = { Authorization: `Bearer ${token}` }; +function errorMessage(err: unknown): string { + return err instanceof Error ? err.message : String(err); +} + +function errorDetail(err: unknown): string { + return err instanceof Error && err.stack ? err.stack : errorMessage(err); +} + +function makeApifyBinding(token: string, apiV2: string) { + const baseHeaders: Record = { Authorization: `Bearer ${token}` }; // Build a URL with optional query params; null/undefined values are dropped. - const buildUrl = (path, searchParams) => { + const buildUrl = (path: string, searchParams?: SearchParams): URL => { const url = new URL(`${apiV2}${path}`); if (searchParams) { for (const [key, value] of Object.entries(searchParams)) { @@ -43,26 +193,40 @@ function makeApifyBinding(token, apiV2) { // Single-source HTTP wrapper. Throws on non-2xx with the response body in the message. // `body`: string / Uint8Array passed through; objects are JSON.stringify'd. - const apiCall = async (method, path, { searchParams, body, contentType } = {}) => { - const init = { method, headers: { ...baseHeaders } }; + const apiCall = async (method: string, path: string, options: ApiCallOptions = {}): Promise => { + const { searchParams, body, contentType } = options; + const headers: Record = { ...baseHeaders }; + let requestBody: BodyInit | undefined; if (body !== undefined) { - const isRaw = typeof body === 'string' || body instanceof Uint8Array || body instanceof ArrayBuffer; - init.body = isRaw ? body : JSON.stringify(body); - init.headers['content-type'] = contentType ?? (isRaw ? 'application/octet-stream' : 'application/json'); + if (typeof body === 'string' || body instanceof Uint8Array || body instanceof ArrayBuffer) { + // Cast: this TS/DOM-lib pairing types Uint8Array generically over its buffer, + // which doesn't structurally match BodyInit here even though it's a valid + // fetch body at runtime (an ArrayBufferView). + requestBody = body as BodyInit; + headers['content-type'] = contentType ?? 'application/octet-stream'; + } else { + requestBody = JSON.stringify(body); + headers['content-type'] = contentType ?? 'application/json'; + } } - const response = await realFetch(buildUrl(path, searchParams), init); + const response = await realFetch(buildUrl(path, searchParams), { method, headers, body: requestBody }); if (!response.ok) throw new Error(`${method} ${path} failed: ${response.status} ${await response.text()}`); return response; }; - const apiJson = async (...args) => (await apiCall(...args)).json(); - const apiData = async (...args) => (await apiJson(...args)).data; + // The Apify API's JSON envelope (`{ data: ... }`) carries whatever shape the endpoint + // returns; there's no schema to check it against here, so this stays honestly `any` + // rather than asserting a shape we haven't verified. + const apiJson = async (method: string, path: string, options?: ApiCallOptions): Promise => + (await apiCall(method, path, options)).json(); + const apiData = async (method: string, path: string, options?: ApiCallOptions): Promise => + (await apiJson(method, path, options)).data; // Run IDs this script itself started, via actor.run() / actor.start() (and transitively // actor.runAndGetItems(), which calls actor.run()). run.abort() below is scoped to this // set — a script can only abort runs it started, not any account-wide runId it's handed // or guesses. - const startedRunIds = new Set(); + const startedRunIds = new Set(); // POST /acts/:id/runs, shared by actor.run() (start+wait, waitForFinishSecs defaults to 60, // capped at 60s per the Apify API — for longer runs use start() + apify.run.wait()) and @@ -70,7 +234,7 @@ function makeApifyBinding(token, apiV2) { // defaultDatasetId / defaultKeyValueStoreId. Intentionally does NOT use /run-sync, which // returns the OUTPUT KVS record (a pattern only some Actors follow) rather than the // structured run record. - const createRun = ({ actorId, input, memoryMbytes, timeoutSecs, waitForFinishSecs, maxTotalChargeUsd, maxItems }) => + const createRun = ({ actorId, input, memoryMbytes, timeoutSecs, waitForFinishSecs, maxTotalChargeUsd, maxItems }: StartOptions): Promise => apiData('POST', `/acts/${encodeURIComponent(actorId)}/runs`, { searchParams: { waitForFinish: waitForFinishSecs, @@ -80,37 +244,47 @@ function makeApifyBinding(token, apiV2) { maxItems, }, body: input ?? {}, - }).then((runRecord) => { + }).then((runRecord: RunRecord) => { startedRunIds.add(runRecord.id); return runRecord; }); const actor = { // GET /v2/store — Apify Store search. Returns the items array directly. - search: ({ query, limit, category }) => + search: ({ query, limit, category }: SearchOptions): Promise => apiData('GET', '/store', { searchParams: { search: query, limit, category } }) - .then((page) => page.items), + .then((page: { items: ApifyRecord[] }) => page.items), - get: ({ actorId }) => + get: ({ actorId }: ActorIdOptions): Promise => apiData('GET', `/acts/${encodeURIComponent(actorId)}`), // Shared by run() and start(): both POST /acts/:id/runs, differing only in whether // waitForFinish is set. Records the created run's ID in startedRunIds so run.abort() // can be scoped to runs this script itself started (see the run.abort definition below). - run: (opts) => createRun({ waitForFinishSecs: 60, ...opts }), + run: (opts: StartOptions): Promise => createRun({ waitForFinishSecs: 60, ...opts }), // Async kickoff. Returns immediately with a run record in READY/RUNNING state. - start: (opts) => createRun(opts), - // runAndGetItems is added below once `dataset.listItems` is defined. + start: (opts: StartOptions): Promise => createRun(opts), + + // Runs an Actor (same as run(), waitForFinishSecs defaults to 60) and returns its + // dataset items in one call. Calls createRun() directly rather than through + // `actor.run()` — same underlying request, no self-reference to `actor` needed. + runAndGetItems: async ({ actorId, input, fields, limit, ...runOpts }: RunAndGetItemsOptions): Promise<{ run: RunRecord; items: ApifyRecord[] }> => { + const runRecord = await createRun({ actorId, input, waitForFinishSecs: 60, ...runOpts }); + const items = await dataset.listItems({ + datasetId: runRecord.defaultDatasetId as string, fields, limit, + }); + return { run: runRecord, items }; + }, }; const run = { - get: ({ runId }) => + get: ({ runId }: RunIdOptions): Promise => apiData('GET', `/actor-runs/${encodeURIComponent(runId)}`), // Block until the run terminates or `waitForFinishSecs` elapses (whichever comes first). // The Apify API caps this at 60s per request; longer waits require a polling loop. - wait: ({ runId, waitForFinishSecs = 60 }) => + wait: ({ runId, waitForFinishSecs = 60 }: WaitOptions): Promise => apiData('GET', `/actor-runs/${encodeURIComponent(runId)}`, { searchParams: { waitForFinish: waitForFinishSecs }, }), @@ -118,7 +292,7 @@ function makeApifyBinding(token, apiV2) { // Scoped to runs this script itself started (see startedRunIds above) — without this, // any runId a script is handed (e.g. read from a dataset item, or guessed) could abort // an unrelated, account-wide run. - abort: ({ runId }) => { + abort: ({ runId }: RunIdOptions): Promise => { if (!startedRunIds.has(runId)) { throw new Error(`Blocked run.abort: "${runId}" was not started by this script`); } @@ -127,7 +301,7 @@ function makeApifyBinding(token, apiV2) { // Returns the full run log as text. `limit` tails the last N characters; the Apify API // does not paginate logs, so this is a client-side slice (the full body is fetched). - getLog: async ({ runId, limit }) => { + getLog: async ({ runId, limit }: GetLogOptions): Promise => { const response = await apiCall('GET', `/logs/${encodeURIComponent(runId)}`); const text = await response.text(); return limit && text.length > limit ? text.slice(-limit) : text; @@ -139,7 +313,7 @@ function makeApifyBinding(token, apiV2) { // `x-apify-pagination-total` header is unreliable for freshly-created datasets // (eventually consistent), so we don't surface a `total`. Use `getSchema` if you // need an item count, or iterate to consume the whole dataset. - listItems: async ({ datasetId, fields, omit, limit, offset, clean, desc }) => { + listItems: async ({ datasetId, fields, omit, limit, offset, clean, desc }: DatasetListOptions): Promise => { const response = await apiCall('GET', `/datasets/${encodeURIComponent(datasetId)}/items`, { searchParams: { fields: fields?.join(','), @@ -157,7 +331,7 @@ function makeApifyBinding(token, apiV2) { // so the user can `for await (const item of apify.dataset.iterate({...}))` without // worrying about offsets. Stops when a page returns fewer items than `batchSize` // (the natural end-of-data signal — pagination total is not used, see listItems). - iterate: async function* ({ datasetId, fields, omit, clean, desc, batchSize = DEFAULT_ITERATE_BATCH }) { + iterate: async function* ({ datasetId, fields, omit, clean, desc, batchSize = DEFAULT_ITERATE_BATCH }: DatasetIterateOptions): AsyncGenerator { let offset = 0; while (true) { const items = await dataset.listItems({ @@ -172,16 +346,15 @@ function makeApifyBinding(token, apiV2) { }, // Apify has no dedicated schema endpoint; we infer one from a small sample of items. - // Returns { itemCount, sampleSize, fields: [{ name, types, nullable }] }. - getSchema: async ({ datasetId, sample = DEFAULT_GET_SCHEMA_SAMPLE }) => { + getSchema: async ({ datasetId, sample = DEFAULT_GET_SCHEMA_SAMPLE }: DatasetSchemaOptions): Promise => { const meta = await apiData('GET', `/datasets/${encodeURIComponent(datasetId)}`); const items = await dataset.listItems({ datasetId, limit: sample }); - const fields = new Map(); + const fields = new Map>(); for (const item of items) { for (const [name, value] of Object.entries(item ?? {})) { const type = value === null ? 'null' : Array.isArray(value) ? 'array' : typeof value; if (!fields.has(name)) fields.set(name, new Set()); - fields.get(name).add(type); + fields.get(name)!.add(type); } } return { @@ -195,27 +368,19 @@ function makeApifyBinding(token, apiV2) { }; }, - create: ({ name } = {}) => + create: ({ name }: CreateOptions = {}): Promise => apiData('POST', '/datasets', { searchParams: { name } }), - pushItems: async ({ datasetId, items }) => { + pushItems: async ({ datasetId, items }: PushItemsOptions): Promise => { await apiCall('POST', `/datasets/${encodeURIComponent(datasetId)}/items`, { body: items }); }, }; - actor.runAndGetItems = async ({ actorId, input, fields, limit, ...runOpts }) => { - const runRecord = await actor.run({ actorId, input, ...runOpts }); - const items = await dataset.listItems({ - datasetId: runRecord.defaultDatasetId, fields, limit, - }); - return { run: runRecord, items }; - }; - const kvs = { // Returns the value directly (parsed when JSON, string when text/*, Uint8Array otherwise). // Returns null when the key does not exist (404), not an error — this matches the common // "lookup or default" pattern in code. - get: async ({ storeId, key }) => { + get: async ({ storeId, key }: KvsGetOptions): Promise => { const response = await realFetch(buildUrl(`/key-value-stores/${encodeURIComponent(storeId)}/records/${encodeURIComponent(key)}`), { headers: baseHeaders, }); @@ -229,11 +394,12 @@ function makeApifyBinding(token, apiV2) { // `value`: object → application/json; string → text/plain; Uint8Array/ArrayBuffer → // application/octet-stream (or whatever the caller passed via `contentType`). - set: async ({ storeId, key, value, contentType }) => { - let body; + set: async ({ storeId, key, value, contentType }: KvsSetOptions): Promise => { + let body: BodyInit; let resolvedContentType = contentType; if (value instanceof Uint8Array || value instanceof ArrayBuffer) { - body = value; + // Cast: see the equivalent Uint8Array-vs-BodyInit comment in apiCall() above. + body = value as BodyInit; resolvedContentType = resolvedContentType ?? 'application/octet-stream'; } else if (typeof value === 'string') { body = value; @@ -247,12 +413,12 @@ function makeApifyBinding(token, apiV2) { }); }, - list: ({ storeId, limit, exclusiveStartKey }) => + list: ({ storeId, limit, exclusiveStartKey }: KvsListOptions): Promise => apiData('GET', `/key-value-stores/${encodeURIComponent(storeId)}/keys`, { searchParams: { limit, exclusiveStartKey }, }), - create: ({ name } = {}) => + create: ({ name }: CreateOptions = {}): Promise => apiData('POST', '/key-value-stores', { searchParams: { name } }), }; @@ -267,7 +433,7 @@ function makeApifyBinding(token, apiV2) { } // Push the captured streams as a single item to the run's default dataset. -async function pushOutput(apiV2, token, env, item) { +async function pushOutput(apiV2: string, token: string, env: Env, item: OutputItem): Promise { const datasetId = env.DEFAULT_DATASET_ID || env.DEFAULT_DATASET_ID_LEGACY; if (!datasetId) throw new Error('Default dataset ID missing from Actor run environment.'); const response = await realFetch(`${apiV2}/datasets/${encodeURIComponent(datasetId)}/items`, { @@ -279,7 +445,7 @@ async function pushOutput(apiV2, token, env, item) { } export default { - async fetch(request, env) { + async fetch(request: Request, env: Env): Promise { const url = new URL(request.url); if (url.pathname === '/health') return new Response('ok'); if (url.pathname !== '/run') return new Response('Not found', { status: 404 }); @@ -289,14 +455,14 @@ export default { // APIFY_API_BASE_URL is the platform-internal API (may have a trailing slash). const apiV2 = `${(env.API_BASE_URL || 'https://api.apify.com').replace(/\/+$/, '')}/v2`; - const stdout = []; - const stderr = []; + const stdout: string[] = []; + const stderr: string[] = []; // Frozen so the script can't reassign e.g. console.log to corrupt its own capture. - const captureConsole = Object.freeze({ - log: (...args) => stdout.push(args.map(stringify).join(' ')), - error: (...args) => stderr.push(args.map(stringify).join(' ')), - warn: (...args) => stderr.push(args.map(stringify).join(' ')), - info: (...args) => stdout.push(args.map(stringify).join(' ')), + const captureConsole: ConsoleLike = Object.freeze({ + log: (...args: unknown[]) => stdout.push(args.map(stringify).join(' ')), + error: (...args: unknown[]) => stderr.push(args.map(stringify).join(' ')), + warn: (...args: unknown[]) => stderr.push(args.map(stringify).join(' ')), + info: (...args: unknown[]) => stdout.push(args.map(stringify).join(' ')), }); // A thrown program is a user-level failure: capture it in stderr and still @@ -316,10 +482,9 @@ export default { try { await run(makeApifyBinding(token, apiV2), captureConsole); } catch (err) { - const message = err?.message ?? String(err); - stderr.push(err?.stack ?? message); + stderr.push(errorDetail(err)); exitCode = 1; - statusMessage = `Script threw: ${message}`; + statusMessage = `Script threw: ${errorMessage(err)}`; } await pushOutput(apiV2, token, env, { diff --git a/worker/usercode.d.ts b/worker/usercode.d.ts new file mode 100644 index 0000000..f36a870 --- /dev/null +++ b/worker/usercode.d.ts @@ -0,0 +1,7 @@ +// usercode.js is generated at container startup by entrypoint.sh (gitignored, never +// checked in) — it wraps the run's `code` input in `export async function run(apify, +// console) { ...code... }`. This declaration lets `tsc` resolve runner.ts's import +// without the generated file present. The user's code is untyped JS text spliced +// into the function body, so `apify`/`console` stay `unknown` here on purpose — +// runner.ts's own ApifyBinding/ConsoleLike types describe what's actually passed in. +export function run(apify: unknown, consoleLike: unknown): Promise; From 90201416bbfcc6ec0272e430e9b5f68095e45b8a Mon Sep 17 00:00:00 2001 From: MQ37 Date: Mon, 13 Jul 2026 16:56:57 +0200 Subject: [PATCH 18/38] docs: drop TypeScript claim for the user's code input The code input is fetched at container runtime and wrapped verbatim into usercode.js (entrypoint.sh); nothing transpiles it and workerd doesn't strip types, so a TypeScript type annotation in user code is a SyntaxError at load today, not a supported input. actor.json's description/input schema and README's intro + input table claimed TypeScript/JavaScript; corrected to JavaScript only, with the SyntaxError-at-load reason stated so it's not mistaken for an oversight. Scoped to the user-facing code input contract only -- the separate question of what language the Actor's own source is written in was resolved in the previous commit (TypeScript, compiled at build time). --- .actor/actor.json | 4 ++-- README.md | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.actor/actor.json b/.actor/actor.json index 3b0434e..4da5e2f 100644 --- a/.actor/actor.json +++ b/.actor/actor.json @@ -2,7 +2,7 @@ "actorSpecification": 1, "name": "code-runtime", "title": "Code Runtime", - "description": "Runs an LLM-submitted TypeScript/JavaScript program in a sandboxed workerd V8 isolate with Apify bindings. One program per run; the captured { stdout, stderr } is pushed as a single item to the default dataset.", + "description": "Runs an LLM-submitted JavaScript program in a sandboxed workerd V8 isolate with Apify bindings. One program per run; the captured { stdout, stderr } is pushed as a single item to the default dataset.", "version": "0.1", "buildTag": "latest", "usesStandbyMode": false, @@ -15,7 +15,7 @@ "code": { "title": "Code", "type": "string", - "description": "TypeScript/JavaScript program executed inside the sandbox. It receives an `apify` binding and `console`; stdout and stderr are captured separately and pushed to the default dataset as { stdout, stderr }.", + "description": "JavaScript program executed inside the sandbox (JS only — nothing transpiles it, so a TypeScript type annotation is a SyntaxError at load). It receives an `apify` binding and `console`; stdout and stderr are captured separately and pushed to the default dataset as { stdout, stderr }.", "editor": "javascript" } }, diff --git a/README.md b/README.md index 8dbbf67..d58d97b 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,9 @@ ## What it does -This Actor executes TypeScript/JavaScript that an AI agent submits through the -Apify MCP Server's **Code Mode**, then returns whatever the script printed. +This Actor executes JavaScript that an AI agent submits through the Apify MCP +Server's **Code Mode**, then returns whatever the script printed. JS only — +nothing transpiles it, so a TypeScript type annotation is a SyntaxError at load. Code Mode exists so an agent can do many Apify operations in **one go** — search the Store, run an Actor, read its dataset, filter and aggregate the @@ -51,7 +52,7 @@ For full configuration options, use the configurator at | Field | Type | Description | |---|---|---| -| `code` | string | The TypeScript/JavaScript script to run. It receives the `apify` binding and `console`. | +| `code` | string | The JavaScript script to run (JS only, not transpiled). It receives the `apify` binding and `console`. | ## Output From 976d15157ce76d12c24ce56b6061be125ca2e8ff Mon Sep 17 00:00:00 2001 From: MQ37 Date: Mon, 13 Jul 2026 17:03:02 +0200 Subject: [PATCH 19/38] refactor!: rename actor.run/runAndGetItems, run.wait, dataset.getSchema Council-reviewed renames (both a data-structure and an interface-design lens independently kept these three, out of a larger proposed table): - actor.run() -> actor.call(): 'run' was doing double duty as a verb (start+wait) and as the top-level run-management namespace. call avoids the collision and matches the CLI (apify call) / MCP (call-actor) vocabulary. - actor.runAndGetItems() -> actor.callAndGetItems(): follows from the above. - run.wait() -> run.waitForFinish(): passes straight through to the REST waitForFinish query param; matches the platform vocabulary. The waitForFinishSecs *parameter* name is intentionally NOT touched -- pairing this method rename with dropping the unit suffix would produce waitForFinish({ waitForFinish: 30 }), read as a boolean, not a duration. - dataset.getSchema() -> dataset.inferFields(): 'schema' already means two other things in this Actor's own ecosystem (input schema, and this Actor's own *declared* dataset schema in actor.json) -- inferFields names what the method actually does (infer field types from a sample) without colliding. Breaking change, acceptable pre-release. Verified live: apify.actor.call, apify.actor.callAndGetItems, apify.run.waitForFinish, apify.dataset.inferFields all exercised end-to-end against a mocked Apify API via real workerd. docs/API.md and README.md still describe the old names -- updated in the next commit alongside the rest of the self-containment docs pass. --- worker/runner.ts | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/worker/runner.ts b/worker/runner.ts index f8d71bb..d762d4a 100644 --- a/worker/runner.ts +++ b/worker/runner.ts @@ -222,15 +222,15 @@ function makeApifyBinding(token: string, apiV2: string) { const apiData = async (method: string, path: string, options?: ApiCallOptions): Promise => (await apiJson(method, path, options)).data; - // Run IDs this script itself started, via actor.run() / actor.start() (and transitively - // actor.runAndGetItems(), which calls actor.run()). run.abort() below is scoped to this - // set — a script can only abort runs it started, not any account-wide runId it's handed - // or guesses. + // Run IDs this script itself started, via actor.call() / actor.start() (and transitively + // actor.callAndGetItems(), which shares createRun() below). run.abort() below is scoped to + // this set — a script can only abort runs it started, not any account-wide runId it's + // handed or guesses. const startedRunIds = new Set(); - // POST /acts/:id/runs, shared by actor.run() (start+wait, waitForFinishSecs defaults to 60, - // capped at 60s per the Apify API — for longer runs use start() + apify.run.wait()) and - // actor.start() (async kickoff, no wait). Returns the run record so the caller can read + // POST /acts/:id/runs, shared by actor.call() (start+wait, waitForFinishSecs defaults to 60, + // capped at 60s per the Apify API — for longer runs use start() + apify.run.waitForFinish()) + // and actor.start() (async kickoff, no wait). Returns the run record so the caller can read // defaultDatasetId / defaultKeyValueStoreId. Intentionally does NOT use /run-sync, which // returns the OUTPUT KVS record (a pattern only some Actors follow) rather than the // structured run record. @@ -261,15 +261,15 @@ function makeApifyBinding(token: string, apiV2: string) { // Shared by run() and start(): both POST /acts/:id/runs, differing only in whether // waitForFinish is set. Records the created run's ID in startedRunIds so run.abort() // can be scoped to runs this script itself started (see the run.abort definition below). - run: (opts: StartOptions): Promise => createRun({ waitForFinishSecs: 60, ...opts }), + call: (opts: StartOptions): Promise => createRun({ waitForFinishSecs: 60, ...opts }), // Async kickoff. Returns immediately with a run record in READY/RUNNING state. start: (opts: StartOptions): Promise => createRun(opts), - // Runs an Actor (same as run(), waitForFinishSecs defaults to 60) and returns its + // Runs an Actor (same as call(), waitForFinishSecs defaults to 60) and returns its // dataset items in one call. Calls createRun() directly rather than through - // `actor.run()` — same underlying request, no self-reference to `actor` needed. - runAndGetItems: async ({ actorId, input, fields, limit, ...runOpts }: RunAndGetItemsOptions): Promise<{ run: RunRecord; items: ApifyRecord[] }> => { + // `actor.call()` — same underlying request, no self-reference to `actor` needed. + callAndGetItems: async ({ actorId, input, fields, limit, ...runOpts }: RunAndGetItemsOptions): Promise<{ run: RunRecord; items: ApifyRecord[] }> => { const runRecord = await createRun({ actorId, input, waitForFinishSecs: 60, ...runOpts }); const items = await dataset.listItems({ datasetId: runRecord.defaultDatasetId as string, fields, limit, @@ -284,7 +284,7 @@ function makeApifyBinding(token: string, apiV2: string) { // Block until the run terminates or `waitForFinishSecs` elapses (whichever comes first). // The Apify API caps this at 60s per request; longer waits require a polling loop. - wait: ({ runId, waitForFinishSecs = 60 }: WaitOptions): Promise => + waitForFinish: ({ runId, waitForFinishSecs = 60 }: WaitOptions): Promise => apiData('GET', `/actor-runs/${encodeURIComponent(runId)}`, { searchParams: { waitForFinish: waitForFinishSecs }, }), @@ -311,7 +311,7 @@ function makeApifyBinding(token: string, apiV2: string) { const dataset = { // Returns the items array directly (no wrapper). The Apify API's // `x-apify-pagination-total` header is unreliable for freshly-created datasets - // (eventually consistent), so we don't surface a `total`. Use `getSchema` if you + // (eventually consistent), so we don't surface a `total`. Use `inferFields` if you // need an item count, or iterate to consume the whole dataset. listItems: async ({ datasetId, fields, omit, limit, offset, clean, desc }: DatasetListOptions): Promise => { const response = await apiCall('GET', `/datasets/${encodeURIComponent(datasetId)}/items`, { @@ -346,7 +346,9 @@ function makeApifyBinding(token: string, apiV2: string) { }, // Apify has no dedicated schema endpoint; we infer one from a small sample of items. - getSchema: async ({ datasetId, sample = DEFAULT_GET_SCHEMA_SAMPLE }: DatasetSchemaOptions): Promise => { + // Named inferFields (not getSchema) to avoid colliding with the Actor's own *declared* + // dataset schema (a different concept, described in this Actor's own actor.json). + inferFields: async ({ datasetId, sample = DEFAULT_GET_SCHEMA_SAMPLE }: DatasetSchemaOptions): Promise => { const meta = await apiData('GET', `/datasets/${encodeURIComponent(datasetId)}`); const items = await dataset.listItems({ datasetId, limit: sample }); const fields = new Map>(); From 8e18d6b5a62f6e8d23b9dd6d2065dc169b31e0c9 Mon Sep 17 00:00:00 2001 From: MQ37 Date: Mon, 13 Jul 2026 17:12:31 +0200 Subject: [PATCH 20/38] docs: self-contained actor.json description, dataset schema, README Final self-containment pass (previously planned as a separate stacked PR, landing directly on this branch per instruction). All content was council- reviewed earlier in the design phase; this lands it against the actual current code (post rename + statusMessage + TS migration). .actor/actor.json: - description (249/300 chars): what it does, the data-heavy/fan-out framing, sandbox limits moved out to README (kept here: run mechanics + billing). - input.code.description (375/500 chars): the correctness-critical facts -- only console output round-trips (a top-level return is NOT captured), call apify.actor.get() before running an Actor, print a small summary. - output.description: now includes exitCode + statusMessage. - defaultRunOptions: timeoutSecs 900, memoryMbytes 1024 -- this Actor had no default before; overridable per call. - storages.dataset: declared fields (stdout/stderr/exitCode/statusMessage, exitCode kept as enum:[0,1] -- not widened to nullable, see prior design notes) plus a dataset-level description stating the run-level-kill cardinality caveat (zero items is a distinct case from any field's value). No `views` block -- a single always-one-item dataset doesn't need one. README.md: - "Calling this Actor" replaces the stale `?tools=run-code,get-code-docs` section -- that mechanism doesn't exist; call-actor/search-actors/ fetch-actor-details are already default MCP tools, no opt-in needed. - Data-heavy/fan-out framing + the free-text-extraction anti-pattern (kept here, cut from the character-capped actor.json description). - Workflow tips (get-before-running, log storage IDs before processing, print-small-summary, return values aren't captured) folded into "How it works" rather than a new section duplicating the code-input description. - "Limits & failure modes": defaultRunOptions + override, and the exitCode/statusMessage-vs-run-status distinction for resource kills. - "Recipes": bounded parallel fan-out (this Actor's actual measured strength) and the >60s start+poll pattern. - "Limitations": sub-Actor runs aren't MCP-attributed -- known, unresolved, tracked separately, not fixed by this PR. - Egress-safety wording softened to "no direct fetch-based exfil", not "contained" -- actor.start()/dataset writes are still exfil paths outside this guard's scope. - Binding summary + Input example updated to the renamed methods (call/callAndGetItems/waitForFinish/inferFields/get). docs/API.md: renamed section headers/cross-references to match (actor.call, actor.callAndGetItems, run.waitForFinish, dataset.inferFields). package.json: description no longer claims "Worker Loader API" (this Actor is one static worker, not per-request isolate loading) -- stale since before this branch, corrected here alongside the rest of the accuracy pass. --- .actor/actor.json | 44 ++++++++++++++-- README.md | 130 +++++++++++++++++++++++++++++++++++++++------- docs/API.md | 20 +++---- package.json | 2 +- 4 files changed, 163 insertions(+), 33 deletions(-) diff --git a/.actor/actor.json b/.actor/actor.json index 4da5e2f..3458a96 100644 --- a/.actor/actor.json +++ b/.actor/actor.json @@ -2,10 +2,14 @@ "actorSpecification": 1, "name": "code-runtime", "title": "Code Runtime", - "description": "Runs an LLM-submitted JavaScript program in a sandboxed workerd V8 isolate with Apify bindings. One program per run; the captured { stdout, stderr } is pushed as a single item to the default dataset.", + "description": "Runs one JS script in a sandboxed Actor with an apify binding (run Actors, datasets & KV stores). Best for data-heavy jobs: scrape hundreds+ places/items, chain Actor outputs, and filter/sort/aggregate in one billed run. Results land in the dataset.", "version": "0.1", "buildTag": "latest", "usesStandbyMode": false, + "defaultRunOptions": { + "timeoutSecs": 900, + "memoryMbytes": 1024 + }, "input": { "title": "Code Runtime Input", "description": "The program to run inside the sandbox.", @@ -15,7 +19,7 @@ "code": { "title": "Code", "type": "string", - "description": "JavaScript program executed inside the sandbox (JS only — nothing transpiles it, so a TypeScript type annotation is a SyntaxError at load). It receives an `apify` binding and `console`; stdout and stderr are captured separately and pushed to the default dataset as { stdout, stderr }.", + "description": "JavaScript executed in the sandbox with `apify` and `console` globals; only console output is captured and pushed to the dataset as { stdout, stderr, exitCode, statusMessage } — a top-level `return` value is NOT captured. Call apify.actor.get({ actorId }) before running an Actor, to read its schema first. Print a small JSON summary of the result — never dump full datasets.", "editor": "javascript" } }, @@ -24,7 +28,7 @@ "output": { "actorOutputSchemaVersion": 1, "title": "Code Runtime Output", - "description": "One dataset item { stdout, stderr } with the program's captured output.", + "description": "One dataset item { stdout, stderr, exitCode, statusMessage } (exitCode: 0=returned, 1=threw); see the exitCode field description for the timeout/OOM case.", "type": "object", "properties": { "output": { @@ -34,5 +38,39 @@ } } }, + "storages": { + "dataset": { + "actorSpecification": 1, + "description": "Present only if the script ran to completion (returned or threw). A run-level timeout or OOM kill produces zero items for that run — the calling Actor-run's own status (SUCCEEDED vs FAILED/TIMED-OUT) is the signal for that case, not item presence.", + "fields": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "stdout": { + "type": "string", + "title": "stdout", + "description": "Captured console.log/console.info output." + }, + "stderr": { + "type": "string", + "title": "stderr", + "description": "Captured console.error/console.warn output, plus the thrown error (or compile failure) message when exitCode is 1." + }, + "exitCode": { + "type": "integer", + "enum": [0, 1], + "title": "Exit code", + "description": "0 = script returned normally, 1 = script threw (or failed to compile)." + }, + "statusMessage": { + "type": "string", + "title": "Status message", + "description": "Prose form of the same signal: 'Script completed' / 'Script threw: ...' / 'Failed to compile: ...'." + } + }, + "required": ["stdout", "stderr", "exitCode", "statusMessage"] + } + } + }, "dockerfile": "../Dockerfile" } diff --git a/README.md b/README.md index d58d97b..c028a08 100644 --- a/README.md +++ b/README.md @@ -15,17 +15,27 @@ search the Store, run an Actor, read its dataset, filter and aggregate the results — instead of sending every intermediate result back through the model and wasting tokens. This Actor is the sandbox that runs that script. -## Enabling Code Mode on the MCP Server +**Best suited for data-heavy jobs** — scraping hundreds or thousands of +places/items via an Actor, then filtering, sorting, or aggregating them +locally before returning a small summary. **Weaker fit** for steps that +require reading or judging free text (picking a fact out of an article, +choosing a search term) — keep the model in the loop there instead; a wrong +guess inside the sandbox fails silently until the whole script finishes. -Code Mode is opt-in. Add the Code Mode tools to the `tools` query parameter of -your mcp.apify.com connection URL: +## Calling this Actor + +Self-contained — no special MCP-server opt-in required. Any MCP client +already has `search-actors`, `fetch-actor-details`, and `call-actor` as +default tools: ``` -https://mcp.apify.com/?tools=run-code,get-code-docs +call-actor({ actor: "apify/code-runtime", input: { code: "..." } }) ``` -For full configuration options, use the configurator at -[mcp.apify.com](https://mcp.apify.com). +Or via the raw API: `POST /v2/acts/apify~code-runtime/runs` with `{ code }` +as the body. Results land in the run's default dataset, same as any Actor +call — follow the response's `nextStep` (or call `get-dataset-items`/ +`GET /v2/datasets/{datasetId}/items`) to read it. ## How it works @@ -41,12 +51,22 @@ For full configuration options, use the configurator at the current run's token (see below). - `console.log` / `console.info` go to **stdout**; `console.error` / `console.warn` go to **stderr**. The two streams are captured separately. +- Before running an Actor from your script, call `apify.actor.get({ actorId })` + once to read its input/output schema. +- As each nested run finishes, log its `run.id` / `defaultDatasetId` / + `defaultKeyValueStoreId` **before** processing its output — if the script + then throws, a re-run can read those existing storages instead of paying to + re-run the Actor (nothing persists between this Actor's own runs, but the + Actors it started keep their results). +- Print a small, JSON-stringified summary of the result — never dump full + datasets. Only what you `console.log`/`console.info` comes back; a + top-level `return` value is **not** captured. ## Input ```json { - "code": "const { items } = await apify.actor.runAndGetItems({ actorId: 'apify/rag-web-browser', input: { query: 'apify' }, limit: 3 });\nconsole.log(items.map((i) => i.metadata?.title).join('\\n'));" + "code": "const { items } = await apify.actor.callAndGetItems({ actorId: 'apify/rag-web-browser', input: { query: 'apify' }, limit: 3 });\nconsole.log(items.map((i) => i.metadata?.title).join('\\n'));" } ``` @@ -56,28 +76,100 @@ For full configuration options, use the configurator at ## Output -A single **dataset item** with the captured streams and the script's exit status: +A single **dataset item** with the captured streams, the script's exit +status, and a prose status message: ```json -{ "stdout": "Apify: Full-stack web scraping ...\n...", "stderr": "", "exitCode": 0 } +{ "stdout": "Apify: Full-stack web scraping ...\n...", "stderr": "", "exitCode": 0, "statusMessage": "Script completed" } ``` If the script throws, the error lands in `stderr`, `stdout` keeps whatever was -printed before the failure, and `exitCode` is `1`. The Actor run itself still -**succeeds** — `exitCode` is the reliable signal for a failed script (`0` = the -script returned normally, `1` = it threw), since `stderr` is also a legitimate -log channel (`console.error` / `console.warn`). +printed before the failure, `exitCode` is `1`, and `statusMessage` is +`"Script threw: ..."`. The Actor run itself still **succeeds** — check +`exitCode`/`statusMessage`, not `stderr` content, to detect a failed script, +since `stderr` is also a legitimate log channel (`console.error` / +`console.warn`). + +If the script fails to **compile** (a syntax error), the same contract +applies — `exitCode: 1`, `statusMessage: "Failed to compile: ..."` — pushed +by the container entrypoint directly, since a malformed script never reaches +the sandboxed worker at all. + +A run-level **timeout or out-of-memory kill** is a different, third outcome: +the container is killed before it can push anything, so this dataset item may +not exist for that run at all. That case is signaled by the Actor run's own +status (`SUCCEEDED` vs `FAILED`/`TIMED-OUT`), not by this item's absence — +see [Limits & failure modes](#limits--failure-modes). + +## Limits & failure modes + +- Default `defaultRunOptions`: `timeoutSecs: 900`, `memoryMbytes: 1024` + (`.actor/actor.json`). Override per call — e.g. the MCP `call-actor` tool's + `callOptions.timeout`/`callOptions.memory`, or the API's `timeout`/`memory` + run options — for scripts that chain several long-running Actor calls. +- `exitCode`/`statusMessage` signal the **script's** outcome only (returned / + threw / failed to compile). A resource-limit kill is a **run-level** + outcome instead — check the Actor run's own `status`, not this dataset + item, for that case (see [Output](#output) above). ## Permissions & safety -- Runs with **limited permissions**: the sandbox has no filesystem and can reach - only the Apify API (`*.apify.com`). +- Runs with **limited permissions**: the sandbox has no filesystem and + outbound `fetch` (including through redirects, which are re-validated per + hop) is limited to the Apify API (`*.apify.com`). - **No imports.** The isolate runs without workerd's `nodejs_compat`, so user code cannot import Node built-ins (`node:net`, `node:fs`, …) or npm packages. This removes `node:net` — a raw-socket egress path that would otherwise bypass the `fetch` allowlist — and keeps the run token out of `process.env` (which is not defined). - Each run is an isolated, single-use container — nothing persists between runs. +- This closes off **direct fetch-based exfil** from the container — it does not + stop every path to move data out (e.g. `actor.start({ input })` on an Actor + with its own open internet access, or writing to a dataset/key-value store). + +## Recipes + +### Bounded parallel fan-out + +This Actor's clearest win: run several independent Actors (or the same Actor +over several inputs) concurrently, then reduce before returning. Chunk the +fan-out (e.g. 5–10 at a time) — an unbounded `Promise.all` over many inputs +can hit your account's concurrent-run or memory limits. + +```js +const inputs = [{ query: 'a' }, { query: 'b' }, { query: 'c' } /* ... */]; +const CHUNK = 5; +const results = []; +for (let i = 0; i < inputs.length; i += CHUNK) { + const batch = inputs.slice(i, i + CHUNK); + const batchResults = await Promise.all( + batch.map((input) => apify.actor.callAndGetItems({ actorId: 'apify/rag-web-browser', input, limit: 5 })), + ); + results.push(...batchResults.flatMap((r) => r.items)); +} +console.log(JSON.stringify(results.slice(0, 5))); // small summary, not the full dump +``` + +### Runs longer than 60s: start, then poll + +`actor.call`'s wait is capped at 60s per request (a REST API limit, not this +Actor's). For a longer-running Actor, start it and poll: + +```js +let run = await apify.actor.start({ actorId, input }); +const TERMINAL = ['SUCCEEDED', 'FAILED', 'ABORTED', 'TIMED-OUT']; +while (!TERMINAL.includes(run.status)) { + run = await apify.run.waitForFinish({ runId: run.id, waitForFinishSecs: 60 }); +} +``` + +## Limitations + +Actor runs launched from inside the sandbox (via the `apify` binding) are +recorded as ordinary Actor runs — they aren't attributed back to the MCP +session that ultimately triggered them. If you're measuring "Actor runs +driven by MCP," Code Mode's sub-runs won't show up as such. Known, +unresolved, tracked separately from this Actor. ## The `apify` binding @@ -90,12 +182,12 @@ Every method takes one options object and returns parsed JSON apify.actor.search({ query, limit?, category? }) // → actors[] apify.actor.get({ actorId }) // → actor apify.actor.start({ actorId, input?, memoryMbytes?, timeoutSecs?, maxTotalChargeUsd?, maxItems? }) // → run -apify.actor.run({ actorId, ...startOpts, waitForFinishSecs = 60 }) // → run (waits) -apify.actor.runAndGetItems({ actorId, input?, fields?, limit?, ...runOpts }) // → { run, items } +apify.actor.call({ actorId, ...startOpts, waitForFinishSecs = 60 }) // → run (waits) +apify.actor.callAndGetItems({ actorId, input?, fields?, limit?, ...runOpts }) // → { run, items } // Runs apify.run.get({ runId }) // → run -apify.run.wait({ runId, waitForFinishSecs = 60 }) // → run +apify.run.waitForFinish({ runId, waitForFinishSecs = 60 }) // → run apify.run.abort({ runId }) // → run apify.run.getLog({ runId, limit? }) // → string @@ -104,7 +196,7 @@ apify.dataset.create({ name? }) // → dataset apify.dataset.pushItems({ datasetId, items }) // → void apify.dataset.listItems({ datasetId, fields?, omit?, limit?, offset?, clean?, desc? }) // → items[] apify.dataset.iterate({ datasetId, batchSize = 1000, ...filters }) // → async iterable -apify.dataset.getSchema({ datasetId, sample = 5 }) // → { itemCount, fields[] } +apify.dataset.inferFields({ datasetId, sample = 5 }) // → { itemCount, fields[] } // Key-value stores apify.kvs.create({ name? }) // → store diff --git a/docs/API.md b/docs/API.md index b57da64..ad3e5b3 100644 --- a/docs/API.md +++ b/docs/API.md @@ -77,7 +77,7 @@ Start an Actor **asynchronously** and return immediately with a run record in **Output:** the Run object (unwrapped `data`). **Apify API:** [`POST /v2/acts/{actorId}/runs`](https://docs.apify.com/api/v2/act-runs-post) -### `actor.run({ actorId, input?, waitForFinishSecs?, memoryMbytes?, timeoutSecs?, maxTotalChargeUsd?, maxItems? })` → `Run` +### `actor.call({ actorId, input?, waitForFinishSecs?, memoryMbytes?, timeoutSecs?, maxTotalChargeUsd?, maxItems? })` → `Run` Start an Actor and **wait** for it to finish (or until `waitForFinishSecs` elapses), then return the run record. @@ -86,7 +86,7 @@ elapses), then return the run record. |---|---|---|---|---| | `actorId` | `string` | yes | | `username/name` or Actor ID. | | `input` | `object` | no | `{}` | Actor input. | -| `waitForFinishSecs` | `number` | no | `60` | Seconds to wait (`waitForFinish`). **The Apify API caps a single wait at 60s** — for longer runs use `start()` + a `run.wait()` loop. | +| `waitForFinishSecs` | `number` | no | `60` | Seconds to wait (`waitForFinish`). **The Apify API caps a single wait at 60s** — for longer runs use `start()` + a `run.waitForFinish()` loop. | | `memoryMbytes` | `number` | no | | Memory limit. | | `timeoutSecs` | `number` | no | | Run timeout. | | `maxTotalChargeUsd` | `number` | no | | Cost cap. | @@ -97,9 +97,9 @@ elapses), then return the run record. (not `/run-sync`, which returns the output record instead of the run object). **Apify API:** [`POST /v2/acts/{actorId}/runs`](https://docs.apify.com/api/v2/act-runs-post) -### `actor.runAndGetItems({ actorId, input?, fields?, limit?, ...runOpts })` → `{ run, items }` +### `actor.callAndGetItems({ actorId, input?, fields?, limit?, ...runOpts })` → `{ run, items }` -Convenience wrapper: `actor.run(...)` followed by reading the run's default +Convenience wrapper: `actor.call(...)` followed by reading the run's default dataset via `dataset.listItems`. | Param | Type | Required | Description | @@ -108,13 +108,13 @@ dataset via `dataset.listItems`. | `input` | `object` | no | Actor input. | | `fields` | `string[]` | no | Restrict returned item fields. | | `limit` | `number` | no | Max items to fetch. | -| `...runOpts` | | no | Any `actor.run` option (`waitForFinishSecs`, `memoryMbytes`, `timeoutSecs`, `maxTotalChargeUsd`, `maxItems`). | +| `...runOpts` | | no | Any `actor.call` option (`waitForFinishSecs`, `memoryMbytes`, `timeoutSecs`, `maxTotalChargeUsd`, `maxItems`). | **Output (custom):** ```js { - run: Run, // the run object, as actor.run returns + run: Run, // the run object, as actor.call returns items: object[] // items from run.defaultDatasetId } ``` @@ -123,7 +123,7 @@ dataset via `dataset.listItems`. then [`GET /v2/datasets/{datasetId}/items`](https://docs.apify.com/api/v2/dataset-items-get) ```js -const { run, items } = await apify.actor.runAndGetItems({ +const { run, items } = await apify.actor.callAndGetItems({ actorId: 'apify/rag-web-browser', input: { query: 'apify' }, limit: 3, @@ -146,7 +146,7 @@ Fetch the current run record (status, stats, default storage IDs). **Output:** the Run object (unwrapped `data`). **Apify API:** [`GET /v2/actor-runs/{runId}`](https://docs.apify.com/api/v2/actor-run-get) -### `run.wait({ runId, waitForFinishSecs? })` → `Run` +### `run.waitForFinish({ runId, waitForFinishSecs? })` → `Run` Block until the run terminates or `waitForFinishSecs` elapses, whichever comes first, then return the run record. @@ -228,7 +228,7 @@ Read a page of items. **Output (custom):** the **items array directly** — this endpoint already returns a bare array (no `data`/pagination wrapper). A dataset's pagination total is eventually consistent right after creation, so no `total` is surfaced; -use [`getSchema`](#datasetgetschema--schema) for a count or +use [`inferFields`](#datasetinferfields--schema) for a count or [`iterate`](#datasetiterate--asyncgeneratorobject) to consume everything. **Apify API:** [`GET /v2/datasets/{datasetId}/items`](https://docs.apify.com/api/v2/dataset-items-get) @@ -255,7 +255,7 @@ for await (const item of apify.dataset.iterate({ datasetId })) count++; console.log('total items:', count); ``` -### `dataset.getSchema({ datasetId, sample? })` → `Schema` +### `dataset.inferFields({ datasetId, sample? })` → `Schema` Infer a lightweight schema from a sample of items (Apify has no schema endpoint). diff --git a/package.json b/package.json index 8de1c9f..b8152c7 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "code-runtime", "version": "0.1.0", - "description": "workerd as a normal (per-run) Apify Actor; one V8 isolate per run via the Worker Loader API.", + "description": "workerd as a normal (per-run) Apify Actor: one static worker, booted once per run, runs the submitted script and exits.", "private": true, "packageManager": "pnpm@11.1.3", "dependencies": { From 9d251d4001ff6d97bf0894ca4487fca998c989fb Mon Sep 17 00:00:00 2001 From: MQ37 Date: Tue, 14 Jul 2026 10:22:21 +0200 Subject: [PATCH 21/38] refactor: migrate tests/*.js probes to TypeScript, fix stale run.wait doc link MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - tests/binding-smoke.ts, tests/sandbox-isolation.ts: typed against the same ApifyBinding surface runner.ts exposes at runtime (exported type-only from runner.ts), and updated to the renamed methods (get, call, callAndGetItems, waitForFinish, inferFields) that a prior commit renamed but these probes still called under their old names. - tests/globals.d.ts: ambient apify/process/require declarations via `declare global` for these standalone-compiled probe files (apify/console are real function parameters at runtime once entrypoint.sh splices the compiled JS into the code input, not globals). - tsconfig.json: tests/*.ts now type-checked alongside worker/*.ts. - package.json build: strips the `export {};` marker tsc appends to each probe (needed so tsc treats each file as its own module — otherwise their top-level consts collide across files, and top-level await needs a module). Left in, that line would be a syntax error once spliced into the wrapping `async function run(apify, console) { ... }`. Verified with node --check against the actual wrapped shape. - test.sh: runs `pnpm build` before pushing/calling so probes compile first. - docs/API.md: fix a stale `run.wait` anchor missed by an earlier rename commit (actual method is `run.waitForFinish`). --- .gitignore | 3 +- docs/API.md | 2 +- package.json | 2 +- test.sh | 6 ++- tests/{binding-smoke.js => binding-smoke.ts} | 52 +++++++++++-------- tests/globals.d.ts | 19 +++++++ ...dbox-isolation.js => sandbox-isolation.ts} | 26 ++++++---- tsconfig.json | 2 +- worker/runner.ts | 5 ++ 9 files changed, 81 insertions(+), 36 deletions(-) rename tests/{binding-smoke.js => binding-smoke.ts} (64%) create mode 100644 tests/globals.d.ts rename tests/{sandbox-isolation.js => sandbox-isolation.ts} (81%) diff --git a/.gitignore b/.gitignore index e347e03..39046c7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,9 @@ node_modules/ # Generated at container startup by entrypoint.sh (never checked in): worker/usercode.js -# Compiled from worker/*.ts by `pnpm build` (tsconfig.json emits next to the source): +# Compiled from worker/*.ts and tests/*.ts by `pnpm build` (tsconfig.json emits next to source): worker/runner.js worker/guard.js +tests/*.js *.log .DS_Store diff --git a/docs/API.md b/docs/API.md index ad3e5b3..66331e6 100644 --- a/docs/API.md +++ b/docs/API.md @@ -63,7 +63,7 @@ Fetch the full record for one Actor. ### `actor.start({ actorId, input?, memoryMbytes?, timeoutSecs?, maxTotalChargeUsd?, maxItems? })` → `Run` Start an Actor **asynchronously** and return immediately with a run record in -`READY`/`RUNNING` state. Use [`run.wait`](#runwait--run) to block for the result. +`READY`/`RUNNING` state. Use [`run.waitForFinish`](#runwaitforfinish--run) to block for the result. | Param | Type | Required | Description | |---|---|---|---| diff --git a/package.json b/package.json index b8152c7..4bd7ecd 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "node": ">=24" }, "scripts": { - "build": "tsc -p tsconfig.json", + "build": "tsc -p tsconfig.json && sed -i '/^export {};$/d' tests/*.js", "typecheck": "tsc --noEmit -p tsconfig.json" }, "devDependencies": { diff --git a/test.sh b/test.sh index ed09ad5..2184a65 100755 --- a/test.sh +++ b/test.sh @@ -8,12 +8,16 @@ set -eu cd "$(dirname "$0")" -# Probes run against the built Actor. Add a file here to register a new probe. +# Probes are written in tests/*.ts and compiled by `pnpm build`; add a .ts file +# there to register a new probe. Run against the built Actor. PROBES="tests/binding-smoke.js tests/sandbox-isolation.js" command -v apify >/dev/null 2>&1 || { echo "apify CLI not found" >&2; exit 1; } command -v jq >/dev/null 2>&1 || { echo "jq not found" >&2; exit 1; } +echo "==> pnpm build" +pnpm build + input_json="$(mktemp)" trap 'rm -f "$input_json"' EXIT diff --git a/tests/binding-smoke.js b/tests/binding-smoke.ts similarity index 64% rename from tests/binding-smoke.js rename to tests/binding-smoke.ts index 98b5b55..c7b2369 100644 --- a/tests/binding-smoke.js +++ b/tests/binding-smoke.ts @@ -2,14 +2,22 @@ // the Actor's `code` input by test.sh and executed on the built Actor via // `apify call`. Exercises every binding method and prints a sentinel line // (ALL_TESTS_PASSED) that test.sh greps for. -const results = []; -async function check(name, fn) { +// +// `export {}` marks this file as its own ES module, so its top-level consts +// don't collide with the other probe's (both are type-checked in one tsc +// program) and top-level await below is legal. `pnpm build` strips this line +// post-compile (see package.json) -- left in, it would be a syntax error once +// spliced into the wrapping `async function run(apify, console) { ... }`. +export {}; + +const results: boolean[] = []; +async function check(name: string, fn: () => Promise): Promise { try { const out = await fn(); console.log(`PASS ${name}: ${out ?? ''}`); results.push(true); } catch (e) { - console.error(`FAIL ${name}: ${e.message}`); + console.error(`FAIL ${name}: ${(e as Error).message}`); results.push(false); } } @@ -22,15 +30,15 @@ await check('actor.search', async () => { if (!Array.isArray(items)) throw new Error('expected array'); return `${items.length} actors`; }); -await check('actor.getDetails', async () => { - const d = await apify.actor.getDetails({ actorId: ACTOR }); +await check('actor.get', async () => { + const d = await apify.actor.get({ actorId: ACTOR }); return `${d.username}/${d.name}`; }); // ---- dataset ---- -let datasetId; +let datasetId = ''; await check('dataset.create', async () => { - datasetId = (await apify.dataset.create()).id; + datasetId = (await apify.dataset.create()).id as string; return datasetId; }); await check('dataset.pushItems', async () => { @@ -41,8 +49,8 @@ await check('dataset.listItems', async () => { const items = await apify.dataset.listItems({ datasetId }); return `${items.length} items`; }); -await check('dataset.getSchema', async () => { - const s = await apify.dataset.getSchema({ datasetId }); +await check('dataset.inferFields', async () => { + const s = await apify.dataset.inferFields({ datasetId }); return `itemCount=${s.itemCount} fields=${s.fields.map((f) => f.name).join(',')}`; }); await check('dataset.iterate', async () => { @@ -52,9 +60,9 @@ await check('dataset.iterate', async () => { }); // ---- key-value store ---- -let storeId; +let storeId = ''; await check('kvs.create', async () => { - storeId = (await apify.kvs.create()).id; + storeId = (await apify.kvs.create()).id as string; return storeId; }); await check('kvs.set', async () => { @@ -63,46 +71,46 @@ await check('kvs.set', async () => { return 'set obj + txt'; }); await check('kvs.get', async () => { - const obj = await apify.kvs.get({ storeId, key: 'obj' }); + const obj = await apify.kvs.get({ storeId, key: 'obj' }) as { hello: string }; const txt = await apify.kvs.get({ storeId, key: 'txt' }); const missing = await apify.kvs.get({ storeId, key: 'nope' }); return `obj.hello=${obj.hello} txt=${txt} missing=${missing}`; }); await check('kvs.list', async () => { - const l = await apify.kvs.list({ storeId }); + const l = await apify.kvs.list({ storeId }) as { items: unknown[] }; return `${l.items.length} keys`; }); // ---- run lifecycle ---- -let runId; +let runId = ''; await check('actor.start', async () => { const run = await apify.actor.start({ actorId: ACTOR }); - runId = run.id; + runId = run.id as string; return `runId=${runId} status=${run.status}`; }); await check('run.get', async () => { return `status=${(await apify.run.get({ runId })).status}`; }); -await check('run.wait', async () => { - return `status=${(await apify.run.wait({ runId, waitForFinishSecs: 60 })).status}`; +await check('run.waitForFinish', async () => { + return `status=${(await apify.run.waitForFinish({ runId, waitForFinishSecs: 60 })).status}`; }); await check('run.getLog', async () => { return `${(await apify.run.getLog({ runId, limit: 200 })).length} chars`; }); // ---- run + get items (sync) ---- -await check('actor.run', async () => { - return `status=${(await apify.actor.run({ actorId: ACTOR, waitForFinishSecs: 60 })).status}`; +await check('actor.call', async () => { + return `status=${(await apify.actor.call({ actorId: ACTOR, waitForFinishSecs: 60 })).status}`; }); -await check('actor.runAndGetItems', async () => { - const { run, items } = await apify.actor.runAndGetItems({ actorId: ACTOR, limit: 5, waitForFinishSecs: 60 }); +await check('actor.callAndGetItems', async () => { + const { run, items } = await apify.actor.callAndGetItems({ actorId: ACTOR, limit: 5, waitForFinishSecs: 60 }); return `status=${run.status} items=${items.length}`; }); // ---- abort ---- await check('run.abort', async () => { const run = await apify.actor.start({ actorId: ACTOR }); - return `status=${(await apify.run.abort({ runId: run.id })).status}`; + return `status=${(await apify.run.abort({ runId: run.id as string })).status}`; }); const passed = results.filter(Boolean).length; diff --git a/tests/globals.d.ts b/tests/globals.d.ts new file mode 100644 index 0000000..2114679 --- /dev/null +++ b/tests/globals.d.ts @@ -0,0 +1,19 @@ +// Ambient declarations for probes in this directory. Each probe is compiled +// standalone by tsc, then submitted as the Actor's `code` input — entrypoint.sh +// splices that text into `export async function run(apify, console) { ... }` +// (see worker/usercode.d.ts), so `apify` and `console` are real function +// parameters at runtime, not globals. `declare global` here only satisfies the +// compiler for these standalone probe files; nothing in this file is emitted to JS. +import type { ApifyBinding } from '../worker/runner.js'; + +declare global { + const apify: ApifyBinding; + + // Sandbox-isolation probe checks `typeof process`/`typeof require` — both are + // genuinely absent at runtime (no nodejs_compat). `typeof` never throws on an + // undeclared identifier, so declaring these here doesn't change that: a + // `declare` emits no JS, so the runtime binding stays exactly as absent as + // the probe expects. + const process: unknown; + const require: unknown; +} diff --git a/tests/sandbox-isolation.js b/tests/sandbox-isolation.ts similarity index 81% rename from tests/sandbox-isolation.js rename to tests/sandbox-isolation.ts index fda8b3d..3019095 100644 --- a/tests/sandbox-isolation.js +++ b/tests/sandbox-isolation.ts @@ -8,8 +8,16 @@ // *.apify.com fetch allowlist — finding A) and process.env (which held the // run's APIFY_TOKEN — finding B), and makes the "no imports" docs accurate // (finding C). fetch() and the apify binding must still work. -const results = []; -function check(name, cond, detail = '') { +// +// `export {}` marks this file as its own ES module, so its top-level consts +// don't collide with the other probe's (both are type-checked in one tsc +// program) and top-level await below is legal. `pnpm build` strips this line +// post-compile (see package.json) -- left in, it would be a syntax error once +// spliced into the wrapping `async function run(apify, console) { ... }`. +export {}; + +const results: boolean[] = []; +function check(name: string, cond: boolean, detail = ''): void { if (cond) { console.log(`PASS ${name}: ${detail}`); results.push(true); @@ -38,12 +46,12 @@ check('fetch available', typeof fetch === 'function', `typeof fetch = ${typeof f // synchronously with a "Blocked fetch" message BEFORE any network I/O; anything // else (a real network/HTTP error) means guard let the request through. So we // classify by the error message, not by whether the request ultimately succeeds. -async function guardBlocks(url) { +async function guardBlocks(url: string): Promise { try { await fetch(url); return false; // request went out — guard allowed it } catch (e) { - return /Blocked fetch/.test(e.message); // guard rejection vs. network error + return /Blocked fetch/.test((e as Error).message); // guard rejection vs. network error } } @@ -69,14 +77,14 @@ for (const url of blockedTargets) { // are web-standard globals that connect directly (not through the fetch guard), // so a script could otherwise open a wss:// / SSE channel to any public host and // exfiltrate around the *.apify.com allowlist (apify/ai-team#216 finding A). -function blocksConstruct(name, url) { - const Ctor = globalThis[name]; +function blocksConstruct(name: string, url: string): boolean { + const Ctor = (globalThis as Record)[name]; if (typeof Ctor !== 'function') return true; // absent → not an egress path try { - new Ctor(url); + new (Ctor as new (u: string) => unknown)(url); return false; // constructed → egress opened } catch (e) { - return /Blocked/.test(e.message); // our guard rejection vs. any other error + return /Blocked/.test((e as Error).message); // our guard rejection vs. any other error } } check('WebSocket blocked', blocksConstruct('WebSocket', 'wss://echo.websocket.org'), 'no wss egress'); @@ -88,7 +96,7 @@ try { const found = await apify.actor.search({ query: 'hello world', limit: 1 }); bindingWorks = Array.isArray(found); } catch (e) { - console.error(`apify.actor.search threw: ${e.message}`); + console.error(`apify.actor.search threw: ${(e as Error).message}`); } check('apify binding works', bindingWorks, bindingWorks ? 'actor.search ok' : 'binding broken'); diff --git a/tsconfig.json b/tsconfig.json index ac781ef..704b2ad 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -6,5 +6,5 @@ "lib": ["es2022", "dom"], "strict": true }, - "include": ["worker/*.ts"] + "include": ["worker/*.ts", "tests/*.ts"] } diff --git a/worker/runner.ts b/worker/runner.ts index d762d4a..da55a5d 100644 --- a/worker/runner.ts +++ b/worker/runner.ts @@ -434,6 +434,11 @@ function makeApifyBinding(token: string, apiV2: string) { }); } +// The shape handed to user code as the `apify` binding. Exported (type-only — +// erased at compile time) so tests/*.ts can type-check probes against the same +// surface real usercode.js runs against, without importing runner.ts at runtime. +export type ApifyBinding = ReturnType; + // Push the captured streams as a single item to the run's default dataset. async function pushOutput(apiV2: string, token: string, env: Env, item: OutputItem): Promise { const datasetId = env.DEFAULT_DATASET_ID || env.DEFAULT_DATASET_ID_LEGACY; From 2169a10d524f5d77eaae42839ab6c894a3e18d5e Mon Sep 17 00:00:00 2001 From: MQ37 Date: Tue, 14 Jul 2026 11:06:03 +0200 Subject: [PATCH 22/38] refactor!: apify.store top-level binding, kvs -> keyValueStore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Revisits the two renames the council flagged as bikeshedding and deferred to v2 (now resolved per your call: breaking changes are fine, this is a POC). - api.apify.com/v2/openapi.json tags GET /v2/store as its own top-level 'Store' resource (operationId store_get), sibling to 'Actors', not a sub-resource of it. Moved actor.search() out of the actor.* namespace to a bare apify.store({ search, limit?, category? }) binding to match — the Store resource has exactly one operation, so a namespace object would be one method wrapping nothing. - Renamed the search param query -> search to match the API's own field name (GET /v2/store?search=...), so the options object no longer invents a name the wire format doesn't use. - Renamed kvs -> keyValueStore (namespace + all Kvs*Options interfaces) to match the API's own key-value-stores resource name instead of an abbreviation found nowhere in the Apify API itself. - README, docs/API.md, tests/*.ts updated to match. docs/API.md gets its own ## apify.store section ahead of ## apify.actor. --- README.md | 12 +++++++----- docs/API.md | 30 +++++++++++++++++++----------- tests/binding-smoke.ts | 26 +++++++++++++------------- tests/sandbox-isolation.ts | 6 +++--- worker/runner.ts | 35 +++++++++++++++++++---------------- 5 files changed, 61 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index c028a08..fe5cb9c 100644 --- a/README.md +++ b/README.md @@ -178,8 +178,10 @@ Every method takes one options object and returns parsed JSON [here](https://github.com/apify/actor-code-runtime/blob/master/docs/API.md). ```js +// Store — GET /v2/store, a top-level Apify API resource (not an Actor method) +apify.store({ search, limit?, category? }) // → actors[] + // Actors -apify.actor.search({ query, limit?, category? }) // → actors[] apify.actor.get({ actorId }) // → actor apify.actor.start({ actorId, input?, memoryMbytes?, timeoutSecs?, maxTotalChargeUsd?, maxItems? }) // → run apify.actor.call({ actorId, ...startOpts, waitForFinishSecs = 60 }) // → run (waits) @@ -199,10 +201,10 @@ apify.dataset.iterate({ datasetId, batchSize = 1000, ...filters }) // → async apify.dataset.inferFields({ datasetId, sample = 5 }) // → { itemCount, fields[] } // Key-value stores -apify.kvs.create({ name? }) // → store -apify.kvs.set({ storeId, key, value, contentType? }) // → void -apify.kvs.get({ storeId, key }) // → value | null -apify.kvs.list({ storeId, limit?, exclusiveStartKey? }) // → { items } +apify.keyValueStore.create({ name? }) // → store +apify.keyValueStore.set({ storeId, key, value, contentType? }) // → void +apify.keyValueStore.get({ storeId, key }) // → value | null +apify.keyValueStore.list({ storeId, limit?, exclusiveStartKey? }) // → { items } ``` ## Learn more diff --git a/docs/API.md b/docs/API.md index 66331e6..f5bd5b8 100644 --- a/docs/API.md +++ b/docs/API.md @@ -22,21 +22,25 @@ document describes every method in detail. live request/response schema. - **Errors.** A non-2xx API response throws an `Error` whose message is ` failed: `. The one exception is - [`kvs.get`](#kvsget--value--null), which returns `null` for a missing key. + [`keyValueStore.get`](#keyvaluestoreget--value--null), which returns `null` for a missing key. - **Network.** Outbound `fetch` from your script is restricted to `apify.com` and its subdomains. --- -## `apify.actor` +## `apify.store` + +Apify's own API tags this endpoint `Store` — a top-level resource, not an +Actor method — so the binding mirrors that: `apify.store(...)`, not +`apify.actor.store(...)`. -### `actor.search({ query, limit?, category? })` → `Actor[]` +### `apify.store({ search, limit?, category? })` → `Actor[]` Search the Apify Store. | Param | Type | Required | Description | |---|---|---|---| -| `query` | `string` | yes | Full-text search query. | +| `search` | `string` | yes | Full-text search query. | | `limit` | `number` | no | Maximum number of results. | | `category` | `string` | no | Restrict to a Store category. | @@ -45,10 +49,14 @@ is dropped) — i.e. an `Actor[]`. **Apify API:** [`GET /v2/store`](https://docs.apify.com/api/v2/store-get) ```js -const actors = await apify.actor.search({ query: 'web scraper', limit: 5 }); +const actors = await apify.store({ search: 'web scraper', limit: 5 }); console.log(actors.map((a) => `${a.username}/${a.name}`).join('\n')); ``` +--- + +## `apify.actor` + ### `actor.get({ actorId })` → `Actor` Fetch the full record for one Actor. @@ -285,9 +293,9 @@ Infer a lightweight schema from a sample of items (Apify has no schema endpoint) --- -## `apify.kvs` +## `apify.keyValueStore` -### `kvs.create({ name? })` → `Store` +### `keyValueStore.create({ name? })` → `KeyValueStore` Create a key-value store and return its record. @@ -295,10 +303,10 @@ Create a key-value store and return its record. |---|---|---|---| | `name` | `string` | no | Named (persistent) store; omit for an unnamed (temporary) one. | -**Output:** the Store object (unwrapped `data`). +**Output:** the key-value store's record object (unwrapped `data`). **Apify API:** [`POST /v2/key-value-stores`](https://docs.apify.com/api/v2/key-value-stores-post) -### `kvs.set({ storeId, key, value, contentType? })` → `void` +### `keyValueStore.set({ storeId, key, value, contentType? })` → `void` Write a record. The content type is inferred from `value`: @@ -318,7 +326,7 @@ Write a record. The content type is inferred from `value`: **Output:** none (resolves once the record is stored). **Apify API:** [`PUT /v2/key-value-stores/{storeId}/records/{key}`](https://docs.apify.com/api/v2/key-value-store-record-put) -### `kvs.get({ storeId, key })` → `value` \| `null` +### `keyValueStore.get({ storeId, key })` → `value` \| `null` Read a record. @@ -337,7 +345,7 @@ Returns **`null`** when the key does not exist (404) instead of throwing, so you can do lookup-or-default without a `try/catch`. **Apify API:** [`GET /v2/key-value-stores/{storeId}/records/{key}`](https://docs.apify.com/api/v2/key-value-store-record-get) -### `kvs.list({ storeId, limit?, exclusiveStartKey? })` → `{ items, … }` +### `keyValueStore.list({ storeId, limit?, exclusiveStartKey? })` → `{ items, … }` List keys in a store. diff --git a/tests/binding-smoke.ts b/tests/binding-smoke.ts index c7b2369..76fa9f5 100644 --- a/tests/binding-smoke.ts +++ b/tests/binding-smoke.ts @@ -25,8 +25,8 @@ async function check(name: string, fn: () => Promise): Promise { const ACTOR = 'apify/hello-world'; // ---- actor (read) ---- -await check('actor.search', async () => { - const items = await apify.actor.search({ query: 'hello world', limit: 3 }); +await check('store', async () => { + const items = await apify.store({ search: 'hello world', limit: 3 }); if (!Array.isArray(items)) throw new Error('expected array'); return `${items.length} actors`; }); @@ -61,23 +61,23 @@ await check('dataset.iterate', async () => { // ---- key-value store ---- let storeId = ''; -await check('kvs.create', async () => { - storeId = (await apify.kvs.create()).id as string; +await check('keyValueStore.create', async () => { + storeId = (await apify.keyValueStore.create()).id as string; return storeId; }); -await check('kvs.set', async () => { - await apify.kvs.set({ storeId, key: 'obj', value: { hello: 'world' } }); - await apify.kvs.set({ storeId, key: 'txt', value: 'plain' }); +await check('keyValueStore.set', async () => { + await apify.keyValueStore.set({ storeId, key: 'obj', value: { hello: 'world' } }); + await apify.keyValueStore.set({ storeId, key: 'txt', value: 'plain' }); return 'set obj + txt'; }); -await check('kvs.get', async () => { - const obj = await apify.kvs.get({ storeId, key: 'obj' }) as { hello: string }; - const txt = await apify.kvs.get({ storeId, key: 'txt' }); - const missing = await apify.kvs.get({ storeId, key: 'nope' }); +await check('keyValueStore.get', async () => { + const obj = await apify.keyValueStore.get({ storeId, key: 'obj' }) as { hello: string }; + const txt = await apify.keyValueStore.get({ storeId, key: 'txt' }); + const missing = await apify.keyValueStore.get({ storeId, key: 'nope' }); return `obj.hello=${obj.hello} txt=${txt} missing=${missing}`; }); -await check('kvs.list', async () => { - const l = await apify.kvs.list({ storeId }) as { items: unknown[] }; +await check('keyValueStore.list', async () => { + const l = await apify.keyValueStore.list({ storeId }) as { items: unknown[] }; return `${l.items.length} keys`; }); diff --git a/tests/sandbox-isolation.ts b/tests/sandbox-isolation.ts index 3019095..335032d 100644 --- a/tests/sandbox-isolation.ts +++ b/tests/sandbox-isolation.ts @@ -93,12 +93,12 @@ check('EventSource blocked', blocksConstruct('EventSource', 'https://example.com // The apify binding must still work (fetch to *.apify.com). let bindingWorks = false; try { - const found = await apify.actor.search({ query: 'hello world', limit: 1 }); + const found = await apify.store({ search: 'hello world', limit: 1 }); bindingWorks = Array.isArray(found); } catch (e) { - console.error(`apify.actor.search threw: ${(e as Error).message}`); + console.error(`apify.store threw: ${(e as Error).message}`); } -check('apify binding works', bindingWorks, bindingWorks ? 'actor.search ok' : 'binding broken'); +check('apify binding works', bindingWorks, bindingWorks ? 'store ok' : 'binding broken'); const passed = results.filter(Boolean).length; console.log(`\n=== SUMMARY: ${passed}/${results.length} passed ===`); diff --git a/worker/runner.ts b/worker/runner.ts index da55a5d..5b6fe41 100644 --- a/worker/runner.ts +++ b/worker/runner.ts @@ -54,8 +54,8 @@ interface ApiCallOptions { contentType?: string; } -interface SearchOptions { - query: string; +interface StoreSearchOptions { + search: string; limit?: number; category?: string; } @@ -125,17 +125,17 @@ interface PushItemsOptions { items: unknown[]; } -interface KvsGetOptions { +interface KeyValueStoreGetOptions { storeId: string; key: string; } -interface KvsSetOptions extends KvsGetOptions { +interface KeyValueStoreSetOptions extends KeyValueStoreGetOptions { value: unknown; contentType?: string; } -interface KvsListOptions { +interface KeyValueStoreListOptions { storeId: string; limit?: number; exclusiveStartKey?: string; @@ -250,11 +250,6 @@ function makeApifyBinding(token: string, apiV2: string) { }); const actor = { - // GET /v2/store — Apify Store search. Returns the items array directly. - search: ({ query, limit, category }: SearchOptions): Promise => - apiData('GET', '/store', { searchParams: { search: query, limit, category } }) - .then((page: { items: ApifyRecord[] }) => page.items), - get: ({ actorId }: ActorIdOptions): Promise => apiData('GET', `/acts/${encodeURIComponent(actorId)}`), @@ -378,16 +373,16 @@ function makeApifyBinding(token: string, apiV2: string) { }, }; - const kvs = { + const keyValueStore = { // Returns the value directly (parsed when JSON, string when text/*, Uint8Array otherwise). // Returns null when the key does not exist (404), not an error — this matches the common // "lookup or default" pattern in code. - get: async ({ storeId, key }: KvsGetOptions): Promise => { + get: async ({ storeId, key }: KeyValueStoreGetOptions): Promise => { const response = await realFetch(buildUrl(`/key-value-stores/${encodeURIComponent(storeId)}/records/${encodeURIComponent(key)}`), { headers: baseHeaders, }); if (response.status === 404) return null; - if (!response.ok) throw new Error(`GET kvs.get failed: ${response.status} ${await response.text()}`); + if (!response.ok) throw new Error(`GET keyValueStore.get failed: ${response.status} ${await response.text()}`); const contentType = response.headers.get('content-type') ?? ''; if (contentType.includes('application/json')) return response.json(); if (contentType.startsWith('text/')) return response.text(); @@ -396,7 +391,7 @@ function makeApifyBinding(token: string, apiV2: string) { // `value`: object → application/json; string → text/plain; Uint8Array/ArrayBuffer → // application/octet-stream (or whatever the caller passed via `contentType`). - set: async ({ storeId, key, value, contentType }: KvsSetOptions): Promise => { + set: async ({ storeId, key, value, contentType }: KeyValueStoreSetOptions): Promise => { let body: BodyInit; let resolvedContentType = contentType; if (value instanceof Uint8Array || value instanceof ArrayBuffer) { @@ -415,7 +410,7 @@ function makeApifyBinding(token: string, apiV2: string) { }); }, - list: ({ storeId, limit, exclusiveStartKey }: KvsListOptions): Promise => + list: ({ storeId, limit, exclusiveStartKey }: KeyValueStoreListOptions): Promise => apiData('GET', `/key-value-stores/${encodeURIComponent(storeId)}/keys`, { searchParams: { limit, exclusiveStartKey }, }), @@ -424,13 +419,21 @@ function makeApifyBinding(token: string, apiV2: string) { apiData('POST', '/key-value-stores', { searchParams: { name } }), }; + // GET /v2/store — Apify Store search (a top-level resource in the Apify API, + // tagged `Store`, distinct from `Actors` — hence a top-level binding rather + // than an `actor.*` method). Returns the items array directly. + const store = ({ search, limit, category }: StoreSearchOptions): Promise => + apiData('GET', '/store', { searchParams: { search, limit, category } }) + .then((page: { items: ApifyRecord[] }) => page.items); + // Freeze every namespace (and the wrapper) so the script can't reassign a method to // corrupt its own behavior or, for `console` below, its own output capture. return Object.freeze({ actor: Object.freeze(actor), + store, run: Object.freeze(run), dataset: Object.freeze(dataset), - kvs: Object.freeze(kvs), + keyValueStore: Object.freeze(keyValueStore), }); } From f23e4e3f738b754e966f83f94995c7b17afeda3f Mon Sep 17 00:00:00 2001 From: MQ37 Date: Tue, 14 Jul 2026 11:35:22 +0200 Subject: [PATCH 23/38] feat: forward X-Apify-Request-Origin: MCP to sub-runs when this run is MCP-started Closes (partially) the MCP-attribution gap flagged in README's Limitations. Verified end-to-end against apify-core / apify-worker source (not just docs): - apify-mcp-server sends X-Apify-Request-Origin: MCP on every API call (src/apify_client.ts). - apify-core's requestOriginParserMiddleware validates it against META_ORIGINS (includes MCP) and sets req.origin (src/api/src/middleware/request_origin_parser.ts). - actor_jobs.ts's createMeta(req.origin || defaultOrigin, ...) -- defaultOrigin is META_ORIGINS.ACTOR for actor-run-token callers -- stores it as the new run's meta.origin (src/api/src/lib/actor_jobs.ts:374-378). - apify-worker injects it back into the container as APIFY_META_ORIGIN (act2_run_job.ts:2212, APIFY_ENV_VARS.META_ORIGIN = "APIFY_META_ORIGIN", matches apify-docs' environment_variables.md). So this Actor's own container already receives APIFY_META_ORIGIN=MCP when apify-mcp-server started it -- no mcp-server change, no new Actor input field. Rejected a hidden `isMCPRun` input field: call-actor's tool schema is built directly from actor.json, so a hidden field is either LLM-visible-and-spoofable or never set by anyone. APIFY_META_ORIGIN is platform-injected and unspoofable from inside the sandbox. - worker/config.capnp: bind PARENT_ORIGIN from APIFY_META_ORIGIN (same pattern as the existing APIFY_TOKEN binding). - worker/runner.ts: makeApifyBinding() takes parentOrigin; sends X-Apify-Request-Origin: MCP on all its own API calls only when parentOrigin === 'MCP'. Also sets a plain User-Agent (previously unset). - README: rewrote Limitations to document the (already-existing, no-code-needed) meta.actorRunId parent-run link apify-core sets on every sub-run automatically, and what this change adds on top of it. Verified live with the real workerd binary + a mock Apify API asserting the header: absent when APIFY_META_ORIGIN is unset, absent when it's 'ACTOR', present as 'MCP' only when it's 'MCP'. --- README.md | 19 ++++++++++++++----- worker/config.capnp | 3 +++ worker/runner.ts | 30 +++++++++++++++++++++++++++--- 3 files changed, 44 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index fe5cb9c..b6ac87f 100644 --- a/README.md +++ b/README.md @@ -165,11 +165,20 @@ while (!TERMINAL.includes(run.status)) { ## Limitations -Actor runs launched from inside the sandbox (via the `apify` binding) are -recorded as ordinary Actor runs — they aren't attributed back to the MCP -session that ultimately triggered them. If you're measuring "Actor runs -driven by MCP," Code Mode's sub-runs won't show up as such. Known, -unresolved, tracked separately from this Actor. +Sub-runs started from inside the sandbox (via `apify.actor.start/call/callAndGetItems`) +get `meta.origin: 'MCP'` on the Apify platform, same as this Actor's own run, when +this Actor was itself started via the Apify MCP Server — this Actor reads its own +`APIFY_META_ORIGIN` env var (platform-set, not spoofable from inside the sandbox) +and forwards `X-Apify-Request-Origin: MCP` on its own API calls only when that's +`MCP`. Every sub-run also gets a platform-native `meta.actorRunId` link back to +this run regardless of origin (set automatically from the run-scoped token, no +code needed here) — so even a fully generic query can already walk sub-run → +`meta.actorRunId` → parent `meta.origin` to reconstruct the chain; the origin +forwarding above just makes single-field origin queries work without that join. + +What's still not attributed: the specific MCP *session* (which client, which +conversation) that triggered this Actor's own run in the first place — that +context isn't part of the platform's Run schema at all, on or off Code Mode. ## The `apify` binding diff --git a/worker/config.capnp b/worker/config.capnp index c5d380b..c78ebbb 100644 --- a/worker/config.capnp +++ b/worker/config.capnp @@ -34,6 +34,9 @@ const codeRuntime :Workerd.Worker = ( (name = "DEFAULT_DATASET_ID", fromEnvironment = "ACTOR_DEFAULT_DATASET_ID"), (name = "DEFAULT_DATASET_ID_LEGACY", fromEnvironment = "APIFY_DEFAULT_DATASET_ID"), (name = "API_BASE_URL", fromEnvironment = "APIFY_API_BASE_URL"), + # This run's own meta.origin (e.g. "MCP" when apify-mcp-server started it), + # forwarded to sub-runs this script starts — see PARENT_ORIGIN in runner.ts. + (name = "PARENT_ORIGIN", fromEnvironment = "APIFY_META_ORIGIN"), ], globalOutbound = "internet", compatibilityDate = "2026-01-15", diff --git a/worker/runner.ts b/worker/runner.ts index 5b6fe41..2a323c9 100644 --- a/worker/runner.ts +++ b/worker/runner.ts @@ -153,6 +153,13 @@ interface Env { DEFAULT_DATASET_ID?: string; DEFAULT_DATASET_ID_LEGACY?: string; API_BASE_URL?: string; + // APIFY_META_ORIGIN, forwarded from the platform's own env var of the same + // name (bound as PARENT_ORIGIN in config.capnp). Reflects this run's own + // meta.origin, set by apify-core from the X-Apify-Request-Origin request + // header the caller sent when creating THIS run — 'MCP' when apify-mcp-server + // started it. Platform-injected, not user-settable: unlike an Actor input + // field, a script running inside this Actor cannot spoof it. + PARENT_ORIGIN?: string; } interface OutputItem { @@ -177,8 +184,25 @@ function errorDetail(err: unknown): string { return err instanceof Error && err.stack ? err.stack : errorMessage(err); } -function makeApifyBinding(token: string, apiV2: string) { - const baseHeaders: Record = { Authorization: `Bearer ${token}` }; +// 'MCP' matches apify-core's META_ORIGINS.MCP / apify-mcp-server's own +// X-Apify-Request-Origin header value — reusing the platform's existing +// convention rather than inventing a new one. +const MCP_ORIGIN = 'MCP'; +const REQUEST_ORIGIN_HEADER = 'X-Apify-Request-Origin'; + +function makeApifyBinding(token: string, apiV2: string, parentOrigin: string | undefined) { + // Every request this Actor makes identifies itself; requests made while THIS + // run's own origin is MCP additionally forward that origin so runs started by + // apify.actor.start/call/callAndGetItems() below get meta.origin: 'MCP' too, + // instead of the platform's default meta.origin: 'ACTOR' for actor-to-actor + // calls. Gated on parentOrigin (verified server-side, see the Env.PARENT_ORIGIN + // comment) rather than any Actor input, so a script can't forge an origin this + // run wasn't actually started with. + const baseHeaders: Record = { + Authorization: `Bearer ${token}`, + 'User-Agent': 'apify-code-runtime', + ...(parentOrigin === MCP_ORIGIN ? { [REQUEST_ORIGIN_HEADER]: MCP_ORIGIN } : {}), + }; // Build a URL with optional query params; null/undefined values are dropped. const buildUrl = (path: string, searchParams?: SearchParams): URL => { @@ -490,7 +514,7 @@ export default { let exitCode = 0; let statusMessage = 'Script completed'; try { - await run(makeApifyBinding(token, apiV2), captureConsole); + await run(makeApifyBinding(token, apiV2, env.PARENT_ORIGIN), captureConsole); } catch (err) { stderr.push(errorDetail(err)); exitCode = 1; From 2b842803d34850402704857737f8b7d73bcb2276 Mon Sep 17 00:00:00 2001 From: MQ37 Date: Tue, 14 Jul 2026 13:06:07 +0200 Subject: [PATCH 24/38] fix(ci): allow workerd postinstall build script under pnpm 11 pnpm 11 blocks dependency install scripts by default (ERR_PNPM_IGNORED_BUILDS) unless explicitly allowed. Adds pnpm-workspace.yaml with allowBuilds: { workerd: true } -- workerd's postinstall fetches its platform-specific binary, which is what CI's pnpm typecheck (and every other pnpm command) needs present. --- pnpm-workspace.yaml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 pnpm-workspace.yaml diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml new file mode 100644 index 0000000..797a5c3 --- /dev/null +++ b/pnpm-workspace.yaml @@ -0,0 +1,2 @@ +allowBuilds: + workerd: true From 5fbbc5328ec31fd3d399ba923e652ce618ba2bb6 Mon Sep 17 00:00:00 2001 From: MQ37 Date: Tue, 14 Jul 2026 13:21:28 +0200 Subject: [PATCH 25/38] docs: add Chain Actors recipe, dataset/keyValueStore usage examples Closes 2 of the 4 doc-parity gaps found vs. apify-mcp-server#1044's original guide content (the other two -- discover-and-inspect snippet, Promise.allSettled fix for the fan-out recipe -- postponed): - README: Chain Actors recipe (one run's output feeds the next), using the renamed callAndGetItems. - docs/API.md: chained create -> pushItems -> listItems example for apify.dataset (previously signature tables only); chained create -> set -> get example for apify.keyValueStore (previously zero code examples in that section). --- README.md | 13 +++++++++++++ docs/API.md | 13 +++++++++++++ 2 files changed, 26 insertions(+) diff --git a/README.md b/README.md index b6ac87f..3c5d7c2 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,19 @@ see [Limits & failure modes](#limits--failure-modes). ## Recipes +### Chain Actors (one run's output feeds the next) + +```js +const { items: results } = await apify.actor.callAndGetItems({ + actorId: 'apify/google-search-scraper', input: { queries: 'apify' }, limit: 10, +}); +const startUrls = results.flatMap((r) => r.organicResults ?? []).map((r) => ({ url: r.url })); +const { items: pages } = await apify.actor.callAndGetItems({ + actorId: 'apify/website-content-crawler', input: { startUrls }, +}); +console.log(JSON.stringify(pages.slice(0, 3).map((p) => p.url))); +``` + ### Bounded parallel fan-out This Actor's clearest win: run several independent Actors (or the same Actor diff --git a/docs/API.md b/docs/API.md index f5bd5b8..10ad7ae 100644 --- a/docs/API.md +++ b/docs/API.md @@ -219,6 +219,12 @@ Append one or more items to a dataset. **Output:** none (resolves once the items are stored). **Apify API:** [`POST /v2/datasets/{datasetId}/items`](https://docs.apify.com/api/v2/dataset-items-post) +```js +const ds = await apify.dataset.create(); +await apify.dataset.pushItems({ datasetId: ds.id, items: [{ a: 1 }, { a: 2 }] }); +const items = await apify.dataset.listItems({ datasetId: ds.id }); +``` + ### `dataset.listItems({ datasetId, fields?, omit?, limit?, offset?, clean?, desc? })` → `object[]` Read a page of items. @@ -345,6 +351,13 @@ Returns **`null`** when the key does not exist (404) instead of throwing, so you can do lookup-or-default without a `try/catch`. **Apify API:** [`GET /v2/key-value-stores/{storeId}/records/{key}`](https://docs.apify.com/api/v2/key-value-store-record-get) +```js +const kv = await apify.keyValueStore.create(); +await apify.keyValueStore.set({ storeId: kv.id, key: 'state', value: { seen: [] } }); +const state = await apify.keyValueStore.get({ storeId: kv.id, key: 'state' }); // → { seen: [] } +const missing = await apify.keyValueStore.get({ storeId: kv.id, key: 'nope' }); // → null +``` + ### `keyValueStore.list({ storeId, limit?, exclusiveStartKey? })` → `{ items, … }` List keys in a store. From 0f610792e8f34d01f2970bc61f5f881c88d1e03d Mon Sep 17 00:00:00 2001 From: MQ37 Date: Thu, 16 Jul 2026 13:03:46 +0200 Subject: [PATCH 26/38] fix: use draft-07 dataset schema, not draft 2020-12 Actor build failed with: Dataset schema compilation failed with: no schema with key or ref "https://json-schema.org/draft/2020-12/schema" apify-core compiles storages.dataset.fields with a plain `new Ajv(...)` (src/packages/storages/src/dataset_validation.ts) - no Ajv2020 variant, no addMetaSchema for 2020-12, so that $schema value has no meta-schema to resolve against. Switch to draft-07, which this Ajv instance supports natively; the schema itself only uses draft-07-compatible keywords (type/properties/required/enum), so no semantic change. Also nudge the agent to read the README before using the Actor, added to the top-level description (277/300 chars). --- .actor/actor.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.actor/actor.json b/.actor/actor.json index 3458a96..466ea41 100644 --- a/.actor/actor.json +++ b/.actor/actor.json @@ -2,7 +2,7 @@ "actorSpecification": 1, "name": "code-runtime", "title": "Code Runtime", - "description": "Runs one JS script in a sandboxed Actor with an apify binding (run Actors, datasets & KV stores). Best for data-heavy jobs: scrape hundreds+ places/items, chain Actor outputs, and filter/sort/aggregate in one billed run. Results land in the dataset.", + "description": "Runs one JS script in a sandboxed Actor with an apify binding (run Actors, datasets & KV stores). Best for data-heavy jobs: scrape hundreds+ places/items, chain Actor outputs, and filter/sort/aggregate in one billed run. Results land in the dataset. Read the README before use.", "version": "0.1", "buildTag": "latest", "usesStandbyMode": false, @@ -43,7 +43,7 @@ "actorSpecification": 1, "description": "Present only if the script ran to completion (returned or threw). A run-level timeout or OOM kill produces zero items for that run — the calling Actor-run's own status (SUCCEEDED vs FAILED/TIMED-OUT) is the signal for that case, not item presence.", "fields": { - "$schema": "https://json-schema.org/draft/2020-12/schema", + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { "stdout": { From 854d5364e92d877fe3ce6b156cf87ed49bb8954a Mon Sep 17 00:00:00 2001 From: MQ37 Date: Thu, 16 Jul 2026 13:12:54 +0200 Subject: [PATCH 27/38] fix: Docker builder stage doesn't need tests/ compiled Build failed: sed: can't read tests/*.js: No such file or directory The builder stage only COPYs worker/ (tests/ is dev-only probe fixtures for test.sh, submitted as Actor input at run time - never part of the image, correctly never copied in). `pnpm run build` compiles both worker/*.ts and tests/*.ts per tsconfig, then sed's the tsc-appended `export {};` marker out of tests/*.js - that sed fails outright when the directory doesn't exist. Call tsc directly instead of the shared pnpm script: tsconfig's tests/*.ts include glob simply matches nothing when the directory is absent (no error), producing exactly worker/runner.js + worker/guard.js - the only two files stage 2 copies. Local dev / test.sh, where tests/ does exist, are unaffected. Verified: reproduced the exact original failure in an isolated copy of the builder context (no tests/), confirmed tsc alone succeeds and produces both worker/*.js files, then ran the full multi-stage Dockerfile build to completion with podman. --- Dockerfile | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9fe4169..fe47c06 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,10 +12,15 @@ COPY worker/ ./worker/ # --ignore-scripts skips workerd's postinstall (a binary-download fallback we # don't need — the binary ships in the @cloudflare/workerd-linux-64 optional dep) # and avoids pnpm's hard error on unapproved dependency build scripts. Full -# (non --prod) install: typescript is a devDependency, needed by `pnpm build` below. +# (non --prod) install: typescript is a devDependency, needed to compile below. +# Compile with tsc directly, not `pnpm run build`: that script also sed's +# tests/*.js (dev-only probe fixtures for test.sh, submitted as Actor input at +# run time — never part of the image), which isn't copied into this build +# context and doesn't need to be; tsconfig's tests/*.ts include glob simply +# matches nothing here. RUN corepack enable \ && pnpm install --frozen-lockfile --ignore-scripts \ - && pnpm run build \ + && pnpm exec tsc -p tsconfig.json \ && BIN="$(node -e "process.stdout.write(require('workerd').default)")" \ && cp "$BIN" /workerd \ && chmod +x /workerd From c9d05b5a5573d43df78bb2b82d1726eeafe30fdb Mon Sep 17 00:00:00 2001 From: MQ37 Date: Thu, 16 Jul 2026 13:40:35 +0200 Subject: [PATCH 28/38] docs: warn against dangling-promise pattern and wrong API shape in code field MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Agent transcripts show scripts wrapping logic in an unawaited 'async function main(){...}; main().catch(...)' pattern truncate silently after the top-level body returns — no error, no partial result, exitCode 0. Also confirmed agents defaulting to the public apify-client SDK's curried .actor(id).call() shape instead of this binding's flat { actorId, input } options-object shape. Both warnings added to the code field's own description since it's always in context (unlike the README, which agents often skip fetching before first use). --- .actor/actor.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.actor/actor.json b/.actor/actor.json index 466ea41..a78f9ee 100644 --- a/.actor/actor.json +++ b/.actor/actor.json @@ -19,7 +19,7 @@ "code": { "title": "Code", "type": "string", - "description": "JavaScript executed in the sandbox with `apify` and `console` globals; only console output is captured and pushed to the dataset as { stdout, stderr, exitCode, statusMessage } — a top-level `return` value is NOT captured. Call apify.actor.get({ actorId }) before running an Actor, to read its schema first. Print a small JSON summary of the result — never dump full datasets.", + "description": "JavaScript executed in the sandbox with `apify` and `console` globals; only console output is captured and pushed to the dataset as { stdout, stderr, exitCode, statusMessage } — a top-level `return` value is NOT captured. Call apify.actor.get({ actorId }) before running an Actor, to read its schema first. Print a small JSON summary of the result — never dump full datasets. Write top-level `await` statements directly in the script; do NOT wrap your logic in an async function you call without awaiting (e.g. `async function main(){...}; main()`) — the script returns as soon as the top-level body finishes, silently discarding anything still pending, with no error. Every apify.* method takes ONE options object keyed by id, e.g. apify.actor.call({ actorId, input }), apify.dataset.listItems({ datasetId, limit }) — this is NOT the public apify-client SDK's curried apify.actor(id).call(input) shape.", "editor": "javascript" } }, From 6c5225627bd59cce28599f6e003c9eb894a62c40 Mon Sep 17 00:00:00 2001 From: MQ37 Date: Thu, 16 Jul 2026 16:48:20 +0200 Subject: [PATCH 29/38] feat: opt into fullReadmeOnly, bypass the auto-generated summary This Actor's README documents an exact API contract (apify.* method names/shapes) -- the platform's auto-generated readmeSummary omits that section entirely, confirmed via a live eval trace where the agent fetched the README, got the summary, and guessed the wrong dataset method as a result. Requires the matching apify-mcp-server change (resolveReadmeContent honoring this flag). Unverified: whether this unrecognized actor.json key survives the platform's build validation into the API response -- needs a live check after this Actor is rebuilt. --- .actor/actor.json | 1 + 1 file changed, 1 insertion(+) diff --git a/.actor/actor.json b/.actor/actor.json index a78f9ee..615d94b 100644 --- a/.actor/actor.json +++ b/.actor/actor.json @@ -6,6 +6,7 @@ "version": "0.1", "buildTag": "latest", "usesStandbyMode": false, + "fullReadmeOnly": true, "defaultRunOptions": { "timeoutSecs": 900, "memoryMbytes": 1024 From 77a3d8de97b56d8cc6b2a53709da59c1491047fc Mon Sep 17 00:00:00 2001 From: MQ37 Date: Thu, 16 Jul 2026 17:05:59 +0200 Subject: [PATCH 30/38] docs: lead schema-check guidance with fetch-actor-details, not just in-sandbox check Live eval trace: agent wrote apify.actor.callAndGetItems({ actorId: 'apify/rag-web-browser', input: { url, maxPages } }) from memory -- wrong field names (real one is 'query') -- despite this field's existing 'call apify.actor.get() first' guidance. All 20 calls failed fast with a clear 400, so no time was wasted discovering it, but it still cost a full wasted round trip before the agent self-corrected. The in-sandbox-only phrasing was easy to skip since it costs an extra nested Actor call inside the script. Reworded to lead with the outer fetch-actor-details tool -- the same tool the agent already uses correctly for every other Actor it discovers via search-actors, just not yet for ones it only decides to call while writing code -- and keep apify.actor.get() as the fallback for Actors picked at runtime. --- .actor/actor.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.actor/actor.json b/.actor/actor.json index 615d94b..6727e94 100644 --- a/.actor/actor.json +++ b/.actor/actor.json @@ -20,7 +20,7 @@ "code": { "title": "Code", "type": "string", - "description": "JavaScript executed in the sandbox with `apify` and `console` globals; only console output is captured and pushed to the dataset as { stdout, stderr, exitCode, statusMessage } — a top-level `return` value is NOT captured. Call apify.actor.get({ actorId }) before running an Actor, to read its schema first. Print a small JSON summary of the result — never dump full datasets. Write top-level `await` statements directly in the script; do NOT wrap your logic in an async function you call without awaiting (e.g. `async function main(){...}; main()`) — the script returns as soon as the top-level body finishes, silently discarding anything still pending, with no error. Every apify.* method takes ONE options object keyed by id, e.g. apify.actor.call({ actorId, input }), apify.dataset.listItems({ datasetId, limit }) — this is NOT the public apify-client SDK's curried apify.actor(id).call(input) shape.", + "description": "JavaScript executed in the sandbox with `apify` and `console` globals; only console output is captured and pushed to the dataset as { stdout, stderr, exitCode, statusMessage } — a top-level `return` value is NOT captured. Before writing code that calls a specific Actor, check its real input field names first — via fetch-actor-details (outside this script, before you write it) or apify.actor.get({ actorId }) (inside it, for an Actor picked at runtime). Do not guess field names from memory; a wrong one throws a fast 400, but costs a wasted round trip. Print a small JSON summary of the result — never dump full datasets. Write top-level `await` statements directly in the script; do NOT wrap your logic in an async function you call without awaiting (e.g. `async function main(){...}; main()`) — the script returns as soon as the top-level body finishes, silently discarding anything still pending, with no error. Every apify.* method takes ONE options object keyed by id, e.g. apify.actor.call({ actorId, input }), apify.dataset.listItems({ datasetId, limit }) — this is NOT the public apify-client SDK's curried apify.actor(id).call(input) shape.", "editor": "javascript" } }, From dae5b1dea17b69e5185dc12dca26393a0d6ae6f2 Mon Sep 17 00:00:00 2001 From: MQ37 Date: Fri, 17 Jul 2026 12:43:43 +0200 Subject: [PATCH 31/38] docs: make dataset-ID reuse imperative, clarify non-terminal wait status Council review (Torvalds/Hotz/Pike/Hoare) + live eval traces: 1 of 3 recovery attempts wasted a full re-run of an already-succeeded nested Actor call instead of reusing its logged defaultDatasetId, because the guidance was advisory ("can read those existing storages") not a rule. Reworded to an imperative 'reuse it, do not re-run' in the code field description, README, and docs/API.md. Separately: actor.call/run.waitForFinish's 60s wait cap was already documented, but a trace showed an agent still treating a returned READY/RUNNING status as a hard failure right after a single wait -- the cap and its corollary (non-terminal isn't an error) lived in two different places. Added an inline note directly on the binding-table/method-doc lines themselves, not just the separate polling recipe. --- .actor/actor.json | 2 +- README.md | 14 ++++++++------ docs/API.md | 14 +++++++++++++- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/.actor/actor.json b/.actor/actor.json index 6727e94..95202c8 100644 --- a/.actor/actor.json +++ b/.actor/actor.json @@ -20,7 +20,7 @@ "code": { "title": "Code", "type": "string", - "description": "JavaScript executed in the sandbox with `apify` and `console` globals; only console output is captured and pushed to the dataset as { stdout, stderr, exitCode, statusMessage } — a top-level `return` value is NOT captured. Before writing code that calls a specific Actor, check its real input field names first — via fetch-actor-details (outside this script, before you write it) or apify.actor.get({ actorId }) (inside it, for an Actor picked at runtime). Do not guess field names from memory; a wrong one throws a fast 400, but costs a wasted round trip. Print a small JSON summary of the result — never dump full datasets. Write top-level `await` statements directly in the script; do NOT wrap your logic in an async function you call without awaiting (e.g. `async function main(){...}; main()`) — the script returns as soon as the top-level body finishes, silently discarding anything still pending, with no error. Every apify.* method takes ONE options object keyed by id, e.g. apify.actor.call({ actorId, input }), apify.dataset.listItems({ datasetId, limit }) — this is NOT the public apify-client SDK's curried apify.actor(id).call(input) shape.", + "description": "JavaScript executed in the sandbox with `apify` and `console` globals; only console output is captured and pushed to the dataset as { stdout, stderr, exitCode, statusMessage } — a top-level `return` value is NOT captured. Before writing code that calls a specific Actor, check its real input field names first — via fetch-actor-details (outside this script, before you write it) or apify.actor.get({ actorId }) (inside it, for an Actor picked at runtime). Do not guess field names from memory; a wrong one throws a fast 400, but costs a wasted round trip. Print a small JSON summary of the result — never dump full datasets. Write top-level `await` statements directly in the script; do NOT wrap your logic in an async function you call without awaiting (e.g. `async function main(){...}; main()`) — the script returns as soon as the top-level body finishes, silently discarding anything still pending, with no error. Every apify.* method takes ONE options object keyed by id, e.g. apify.actor.call({ actorId, input }), apify.dataset.listItems({ datasetId, limit }) — this is NOT the public apify-client SDK's curried apify.actor(id).call(input) shape. If a prior attempt already logged a nested run's defaultDatasetId/defaultKeyValueStoreId (visible in your own earlier turns), reuse it — do NOT re-run the same Actor call with identical input, that wastes compute on a call that already succeeded. apify.actor.call/run.waitForFinish may return non-terminal (READY/RUNNING) once the 60s wait cap elapses — that is NOT a failure, poll again instead of throwing.", "editor": "javascript" } }, diff --git a/README.md b/README.md index 3c5d7c2..112575c 100644 --- a/README.md +++ b/README.md @@ -54,10 +54,12 @@ call — follow the response's `nextStep` (or call `get-dataset-items`/ - Before running an Actor from your script, call `apify.actor.get({ actorId })` once to read its input/output schema. - As each nested run finishes, log its `run.id` / `defaultDatasetId` / - `defaultKeyValueStoreId` **before** processing its output — if the script - then throws, a re-run can read those existing storages instead of paying to - re-run the Actor (nothing persists between this Actor's own runs, but the - Actors it started keep their results). + `defaultKeyValueStoreId` **before** processing its output (nothing persists + between this Actor's own runs, but the Actors it started keep their + results). **If a prior attempt's `defaultDatasetId`/`defaultKeyValueStoreId` + is visible in your own earlier turns, reuse it — do not re-run the same + Actor call with identical input.** Re-running wastes the compute/cost of a + call that already succeeded. - Print a small, JSON-stringified summary of the result — never dump full datasets. Only what you `console.log`/`console.info` comes back; a top-level `return` value is **not** captured. @@ -206,12 +208,12 @@ apify.store({ search, limit?, category? }) // → actors[] // Actors apify.actor.get({ actorId }) // → actor apify.actor.start({ actorId, input?, memoryMbytes?, timeoutSecs?, maxTotalChargeUsd?, maxItems? }) // → run -apify.actor.call({ actorId, ...startOpts, waitForFinishSecs = 60 }) // → run (waits) +apify.actor.call({ actorId, ...startOpts, waitForFinishSecs = 60 }) // → run (waits; may return non-terminal READY/RUNNING if the 60s cap elapses first — not an error, poll run.waitForFinish) apify.actor.callAndGetItems({ actorId, input?, fields?, limit?, ...runOpts }) // → { run, items } // Runs apify.run.get({ runId }) // → run -apify.run.waitForFinish({ runId, waitForFinishSecs = 60 }) // → run +apify.run.waitForFinish({ runId, waitForFinishSecs = 60 }) // → run (same non-terminal caveat as actor.call above) apify.run.abort({ runId }) // → run apify.run.getLog({ runId, limit? }) // → string diff --git a/docs/API.md b/docs/API.md index 10ad7ae..58bec56 100644 --- a/docs/API.md +++ b/docs/API.md @@ -25,6 +25,10 @@ document describes every method in detail. [`keyValueStore.get`](#keyvaluestoreget--value--null), which returns `null` for a missing key. - **Network.** Outbound `fetch` from your script is restricted to `apify.com` and its subdomains. +- **Reuse, don't re-run.** If a prior attempt already logged a nested run's + `defaultDatasetId`/`defaultKeyValueStoreId` (visible in your own earlier + turns), reuse it — do not re-run the same Actor call with identical input. + Re-running wastes the compute/cost of a call that already succeeded. --- @@ -103,6 +107,10 @@ elapses), then return the run record. **Output:** the Run object (unwrapped `data`), exposing `defaultDatasetId` and `defaultKeyValueStoreId` for reading results. Uses the standard run endpoint (not `/run-sync`, which returns the output record instead of the run object). +**May return non-terminal** (`status: 'READY'`/`'RUNNING'`) if `waitForFinishSecs` +(capped at 60s by the API) elapses before the run finishes — this is **not** +an error; poll [`run.waitForFinish`](#runwaitforfinish--run) until `status` is +terminal (`SUCCEEDED`/`FAILED`/`ABORTED`/`TIMED-OUT`). **Apify API:** [`POST /v2/acts/{actorId}/runs`](https://docs.apify.com/api/v2/act-runs-post) ### `actor.callAndGetItems({ actorId, input?, fields?, limit?, ...runOpts })` → `{ run, items }` @@ -164,7 +172,11 @@ first, then return the run record. | `runId` | `string` | yes | | The run ID. | | `waitForFinishSecs` | `number` | no | `60` | Seconds to wait (`waitForFinish`). **Capped at 60s by the API**; poll in a loop for longer runs. | -**Output:** the Run object (unwrapped `data`). +**Output:** the Run object (unwrapped `data`). **May return non-terminal** +(`status: 'READY'`/`'RUNNING'`) if the cap elapses first — this is **not** an +error, it means keep polling; check `status` against the terminal set +(`SUCCEEDED`/`FAILED`/`ABORTED`/`TIMED-OUT`) before treating any other value +as a failure. **Apify API:** [`GET /v2/actor-runs/{runId}`](https://docs.apify.com/api/v2/actor-run-get) ### `run.abort({ runId })` → `Run` From 776c58141f29bef998a970102ff018db43859787 Mon Sep 17 00:00:00 2001 From: MQ37 Date: Fri, 17 Jul 2026 12:57:26 +0200 Subject: [PATCH 32/38] feat!: dual-mode listItems/store, remove dataset.iterate() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Grounded in a fresh clone of apify-client-js (the real Apify TS SDK) and a council review (Torvalds/Hotz/Pike/Hoare) — all four independently converged on the same root cause behind the highest-frequency bug in this project: listItems()/callAndGetItems()/store() each returned a different envelope for 'here are your records' (bare array / bare array / {run,items}), confirmed still recurring 3x even with the full (non-summarized) README available, so more prose wasn't going to fix it. dataset.listItems() and store() now return a value that's both a Promise (await -> one page, { items, count, offset, limit, desc }) and an AsyncIterable (for await -> every item, auto-paginated) -- the same dual nature as apify-client's own PaginatedIterator, implemented via the same mechanism (Object.defineProperty(promise, Symbol.asyncIterator, ...)), but as ONE shared implementation (makePaginatedList) instead of apify-client's own three independent, subtly-different copies of this trick. This makes dataset.iterate() redundant -- removed, along with DatasetIterateOptions and DEFAULT_ITERATE_BATCH. actor.callAndGetItems() and dataset.inferFields() updated for the new page shape (no behavior change -- both only ever consumed a single page). Verified beyond typecheck: a standalone runtime smoke test confirms the dual nature actually works (await resolves to one page; for-await auto-paginates across multiple pages; a call with no limit makes exactly one HTTP request instead of over-fetching). tests/binding-smoke.ts and tests/sandbox-isolation.ts updated for the new shapes. Explicitly NOT adopted from apify-client: its curried client.actor(id).call() shape (this binding's flat options-object convention already fixed a confirmed confusion bug this session and stays), and its own internal inconsistencies (mixed positional/options args, waitForFinish vs waitSecs naming split for the same concept). Breaking change for any script written against the old bare-array listItems()/store() shape -- every script is freshly generated per run (one script per Actor run, no shipped callers), so no migration path is needed. --- README.md | 34 +++++++-- docs/API.md | 94 ++++++++++++++----------- tests/binding-smoke.ts | 19 +++-- tests/sandbox-isolation.ts | 2 +- worker/runner.ts | 138 ++++++++++++++++++++++++++----------- 5 files changed, 193 insertions(+), 94 deletions(-) diff --git a/README.md b/README.md index 112575c..b878660 100644 --- a/README.md +++ b/README.md @@ -165,6 +165,24 @@ for (let i = 0; i < inputs.length; i += CHUNK) { console.log(JSON.stringify(results.slice(0, 5))); // small summary, not the full dump ``` +### Read an entire dataset without managing offsets + +`dataset.listItems` (and `store`) return a value that's both a `Promise` (one +page) and an `AsyncIterable` (every item, auto-paginated) — pick whichever +you need: + +```js +// One page — e.g. a quick peek +const { items, count } = await apify.dataset.listItems({ datasetId, limit: 10 }); + +// Every item, however many pages that takes +let matches = 0; +for await (const item of apify.dataset.listItems({ datasetId })) { + if (item.rating >= 4.5) matches++; +} +console.log(`${matches} matching items`); +``` + ### Runs longer than 60s: start, then poll `actor.call`'s wait is capped at 60s per request (a REST API limit, not this @@ -197,13 +215,18 @@ context isn't part of the platform's Run schema at all, on or off Code Mode. ## The `apify` binding -Every method takes one options object and returns parsed JSON -(`?` = optional, `= x` = default). Full API documentation is available +Every method takes one options object and returns parsed JSON — except +`store` and `dataset.listItems`, which return a value that's both a `Promise` +(one page) and an `AsyncIterable` (every match/item, auto-paginated) — see +their own lines below (`?` = optional, `= x` = default). Full API +documentation is available [here](https://github.com/apify/actor-code-runtime/blob/master/docs/API.md). ```js // Store — GET /v2/store, a top-level Apify API resource (not an Actor method) -apify.store({ search, limit?, category? }) // → actors[] +// `await` for one page, `for await` to walk every match (same dual nature as +// dataset.listItems below). +apify.store({ search, limit?, offset?, category? }) // → { items, count, offset, limit } // Actors apify.actor.get({ actorId }) // → actor @@ -220,8 +243,9 @@ apify.run.getLog({ runId, limit? }) // → string // Datasets apify.dataset.create({ name? }) // → dataset apify.dataset.pushItems({ datasetId, items }) // → void -apify.dataset.listItems({ datasetId, fields?, omit?, limit?, offset?, clean?, desc? }) // → items[] -apify.dataset.iterate({ datasetId, batchSize = 1000, ...filters }) // → async iterable +// `await` for one page: { items, count, offset, limit, desc }. `for await` auto-paginates +// through the whole dataset, one item at a time — no separate iterate() method needed. +apify.dataset.listItems({ datasetId, fields?, omit?, limit?, offset?, clean?, desc? }) apify.dataset.inferFields({ datasetId, sample = 5 }) // → { itemCount, fields[] } // Key-value stores diff --git a/docs/API.md b/docs/API.md index 58bec56..691ea88 100644 --- a/docs/API.md +++ b/docs/API.md @@ -6,10 +6,19 @@ document describes every method in detail. ## Conventions -- **Every method is `async`** — `await` the result (or `for await` for - `dataset.iterate`). +- **Every method is `async`** — except `dataset.listItems`/`store` (see + below), which are plain functions that return an already-awaitable value; + `await`ing one behaves identically to awaiting a true `async` call. - **One options object.** Each method takes a single object argument; there are no positional parameters. +- **Paginated methods are dual-mode.** [`dataset.listItems`](#datasetlistitems--object-count-offset-limit-desc-) + and [`store`](#apifystore--object-count-offset-limit-) return a value that's + both a `Promise` and an `AsyncIterable`: `await` it for one page (`{ items, + count, offset, limit, ... }`); `for await (const item of ...)` it to + auto-paginate through everything, one item at a time. One call, one name, + two ways to consume it — matching the official + [`apify-client`](https://github.com/apify/apify-client-js) SDK's + `PaginatedIterator` convention. There's no separate "iterate" method. - **`actorId`** accepts either `username/name` (e.g. `apify/rag-web-browser`) or the Actor's ID. - **Return values.** The Apify API wraps most responses in a `{ "data": … }` @@ -38,23 +47,31 @@ Apify's own API tags this endpoint `Store` — a top-level resource, not an Actor method — so the binding mirrors that: `apify.store(...)`, not `apify.actor.store(...)`. -### `apify.store({ search, limit?, category? })` → `Actor[]` +### `apify.store({ search, limit?, offset?, category? })` → `{ items, count, offset, limit }` -Search the Apify Store. +Search the Apify Store. Dual-mode — see [Conventions](#conventions). | Param | Type | Required | Description | |---|---|---|---| | `search` | `string` | yes | Full-text search query. | -| `limit` | `number` | no | Maximum number of results. | +| `limit` | `number` | no | Page size (`await` mode) / items per page (`for await` mode). | +| `offset` | `number` | no | Starting offset. | | `category` | `string` | no | Restrict to a Store category. | -**Output:** the `data.items` array of the Store listing (the pagination wrapper -is dropped) — i.e. an `Actor[]`. +**Output (custom):** one page — `{ items: Actor[], count, offset, limit }` +(`count` is this page's actual item count; `offset`/`limit` echo the request). **Apify API:** [`GET /v2/store`](https://docs.apify.com/api/v2/store-get) ```js -const actors = await apify.store({ search: 'web scraper', limit: 5 }); -console.log(actors.map((a) => `${a.username}/${a.name}`).join('\n')); +// One page +const { items } = await apify.store({ search: 'web scraper', limit: 5 }); +console.log(items.map((a) => `${a.username}/${a.name}`).join('\n')); + +// Every match +const names = []; +for await (const actor of apify.store({ search: 'web scraper', limit: 20 })) { + names.push(`${actor.username}/${actor.name}`); +} ``` --- @@ -234,51 +251,48 @@ Append one or more items to a dataset. ```js const ds = await apify.dataset.create(); await apify.dataset.pushItems({ datasetId: ds.id, items: [{ a: 1 }, { a: 2 }] }); -const items = await apify.dataset.listItems({ datasetId: ds.id }); +const { items } = await apify.dataset.listItems({ datasetId: ds.id }); ``` -### `dataset.listItems({ datasetId, fields?, omit?, limit?, offset?, clean?, desc? })` → `object[]` +### `dataset.listItems({ datasetId, fields?, omit?, limit?, offset?, clean?, desc? })` → `{ items, count, offset, limit, desc }` -Read a page of items. +Read the dataset. Dual-mode — see [Conventions](#conventions): `await` for one +page, `for await` to auto-paginate through everything (replaces what used to +be a separate `iterate()` method — one name, one method, both jobs). | Param | Type | Required | Description | |---|---|---|---| | `datasetId` | `string` | yes | Dataset ID. | | `fields` | `string[]` | no | Only include these fields (joined into `fields`). | | `omit` | `string[]` | no | Exclude these fields. | -| `limit` | `number` | no | Page size. | -| `offset` | `number` | no | Starting offset. | +| `limit` | `number` | no | Page size (`await` mode) / items fetched per page (`for await` mode). Omit for the API's own default (effectively unbounded — a single `await` then returns everything in one page). | +| `offset` | `number` | no | Starting offset. Default `0`. | | `clean` | `boolean` | no | Skip empty items / hidden fields (`clean=1`). | | `desc` | `boolean` | no | Reverse (newest first, `desc=1`). | -**Output (custom):** the **items array directly** — this endpoint already -returns a bare array (no `data`/pagination wrapper). A dataset's pagination -total is eventually consistent right after creation, so no `total` is surfaced; -use [`inferFields`](#datasetinferfields--schema) for a count or -[`iterate`](#datasetiterate--asyncgeneratorobject) to consume everything. -**Apify API:** [`GET /v2/datasets/{datasetId}/items`](https://docs.apify.com/api/v2/dataset-items-get) - -### `dataset.iterate({ datasetId, fields?, omit?, clean?, desc?, batchSize? })` → `AsyncGenerator` - -Async-iterate the **entire** dataset, paging internally so you don't manage -offsets. Stops when a page returns fewer than `batchSize` items. - -| Param | Type | Required | Default | Description | -|---|---|---|---|---| -| `datasetId` | `string` | yes | | Dataset ID. | -| `fields` | `string[]` | no | | Only include these fields. | -| `omit` | `string[]` | no | | Exclude these fields. | -| `clean` | `boolean` | no | | Skip empty items / hidden fields. | -| `desc` | `boolean` | no | | Reverse order. | -| `batchSize` | `number` | no | `1000` | Items fetched per page. | - -**Output (custom):** an async generator yielding one item (`object`) at a time. -**Apify API:** [`GET /v2/datasets/{datasetId}/items`](https://docs.apify.com/api/v2/dataset-items-get) (paged internally) +**Output (custom):** +- `await apify.dataset.listItems({...})` → one page: `{ items: object[], count, + offset, limit, desc }`. `offset`/`limit`/`desc` echo what the API actually + applied (read from its `x-apify-pagination-*` response headers, not just + the request); `count` is this page's actual item count. **No `total`** — + the API's `x-apify-pagination-total` header is unreliable for + freshly-created datasets (eventually consistent); use + [`inferFields`](#datasetinferfields--schema) for an approximate count. +- `for await (const item of apify.dataset.listItems({...}))` → every item in + the dataset, one at a time, paging internally. Stops when a page comes back + shorter than requested (the natural end-of-data signal). + +**Apify API:** [`GET /v2/datasets/{datasetId}/items`](https://docs.apify.com/api/v2/dataset-items-get) (paged internally for `for await`) ```js -let count = 0; -for await (const item of apify.dataset.iterate({ datasetId })) count++; -console.log('total items:', count); +// One page +const { items, count } = await apify.dataset.listItems({ datasetId, limit: 100 }); +console.log(`${count} items on this page`); + +// Every item +let total = 0; +for await (const item of apify.dataset.listItems({ datasetId })) total++; +console.log('total items:', total); ``` ### `dataset.inferFields({ datasetId, sample? })` → `Schema` diff --git a/tests/binding-smoke.ts b/tests/binding-smoke.ts index 76fa9f5..c43ab83 100644 --- a/tests/binding-smoke.ts +++ b/tests/binding-smoke.ts @@ -26,9 +26,14 @@ const ACTOR = 'apify/hello-world'; // ---- actor (read) ---- await check('store', async () => { - const items = await apify.store({ search: 'hello world', limit: 3 }); - if (!Array.isArray(items)) throw new Error('expected array'); - return `${items.length} actors`; + const page = await apify.store({ search: 'hello world', limit: 3 }); + if (!Array.isArray(page.items)) throw new Error('expected items array'); + return `${page.count} actors`; +}); +await check('store (for await)', async () => { + let n = 0; + for await (const _ of apify.store({ search: 'hello world', limit: 3 })) n++; + return `${n} actors iterated`; }); await check('actor.get', async () => { const d = await apify.actor.get({ actorId: ACTOR }); @@ -46,16 +51,16 @@ await check('dataset.pushItems', async () => { return '2 pushed'; }); await check('dataset.listItems', async () => { - const items = await apify.dataset.listItems({ datasetId }); - return `${items.length} items`; + const page = await apify.dataset.listItems({ datasetId }); + return `${page.count} items, offset=${page.offset}, limit=${page.limit}`; }); await check('dataset.inferFields', async () => { const s = await apify.dataset.inferFields({ datasetId }); return `itemCount=${s.itemCount} fields=${s.fields.map((f) => f.name).join(',')}`; }); -await check('dataset.iterate', async () => { +await check('dataset.listItems (for await)', async () => { let n = 0; - for await (const _ of apify.dataset.iterate({ datasetId, batchSize: 1 })) n++; + for await (const _ of apify.dataset.listItems({ datasetId, limit: 1 })) n++; return `${n} iterated`; }); diff --git a/tests/sandbox-isolation.ts b/tests/sandbox-isolation.ts index 335032d..bf877df 100644 --- a/tests/sandbox-isolation.ts +++ b/tests/sandbox-isolation.ts @@ -94,7 +94,7 @@ check('EventSource blocked', blocksConstruct('EventSource', 'https://example.com let bindingWorks = false; try { const found = await apify.store({ search: 'hello world', limit: 1 }); - bindingWorks = Array.isArray(found); + bindingWorks = Array.isArray(found.items); } catch (e) { console.error(`apify.store threw: ${(e as Error).message}`); } diff --git a/worker/runner.ts b/worker/runner.ts index 2a323c9..f90acbf 100644 --- a/worker/runner.ts +++ b/worker/runner.ts @@ -28,7 +28,6 @@ function requireRealFetch(): typeof globalThis.fetch { } const realFetch = requireRealFetch(); -const DEFAULT_ITERATE_BATCH = 1000; const DEFAULT_GET_SCHEMA_SAMPLE = 5; // --- Types --------------------------------------------------------------- @@ -57,6 +56,7 @@ interface ApiCallOptions { interface StoreSearchOptions { search: string; limit?: number; + offset?: number; category?: string; } @@ -101,10 +101,24 @@ interface DatasetListOptions { desc?: boolean; } -interface DatasetIterateOptions extends Omit { - batchSize?: number; +// One page, from a single request. Mirrors apify-client's PaginatedList shape (items, count, +// offset, limit, desc) except `total`, which stays deliberately unsurfaced — see makePaginatedList. +interface ItemsPage { + items: T[]; + count: number; + offset: number; + limit: number; +} + +interface DatasetItemsPage extends ItemsPage { + desc: boolean; } +// Awaiting this value resolves to one page (ItemsPage); `for await`-ing it auto-paginates +// through every item, one at a time. Same dual nature as apify-client's own PaginatedIterator +// (one call, one name, two ways to consume it) — see makePaginatedList for the mechanism. +type PaginatedItems> = Promise & AsyncIterable; + interface DatasetSchemaOptions { datasetId: string; sample?: number; @@ -184,6 +198,44 @@ function errorDetail(err: unknown): string { return err instanceof Error && err.stack ? err.stack : errorMessage(err); } +// Wraps a page-fetcher into a value that's both a Promise (awaits to the first page) and an +// AsyncIterable (walks every page, yielding one item at a time) — the same dual nature as +// apify-client's own PaginatedIterator, implemented via the same trick it uses: attach +// Symbol.asyncIterator to a live Promise object (a Promise is a plain object at runtime, so +// this is legal, no class or wrapper needed). +// +// One shared implementation, unlike apify-client itself, which independently re-implements +// this exact trick three times (dataset items, key-value-store keys, request-queue requests) +// with three subtly different cursor/offset conventions — see docs/API.md's Conventions +// section for that comparison. Every paginated method in this binding goes through this one +// function instead. +// +// Continuation stops the same way the old dataset.iterate() did: a page shorter than the +// limit it was asked for is the natural end-of-data signal (no dataset `total` is trusted — +// see the listItems comment below for why). +function makePaginatedList>( + fetchPage: (offset: number, limit: number | undefined) => Promise, + offset: number, + limit: number | undefined, +): PaginatedItems { + const firstPagePromise = fetchPage(offset, limit); + + async function* iterateAll(): AsyncGenerator { + let page = await firstPagePromise; + yield* page.items; + let nextOffset = page.offset + page.items.length; + while (page.items.length > 0 && page.items.length >= page.limit) { + page = await fetchPage(nextOffset, page.limit); + yield* page.items; + nextOffset += page.items.length; + } + } + + return Object.defineProperty(firstPagePromise, Symbol.asyncIterator, { + value: iterateAll, + }) as PaginatedItems; +} + // 'MCP' matches apify-core's META_ORIGINS.MCP / apify-mcp-server's own // X-Apify-Request-Origin header value — reusing the platform's existing // convention rather than inventing a new one. @@ -290,7 +342,7 @@ function makeApifyBinding(token: string, apiV2: string, parentOrigin: string | u // `actor.call()` — same underlying request, no self-reference to `actor` needed. callAndGetItems: async ({ actorId, input, fields, limit, ...runOpts }: RunAndGetItemsOptions): Promise<{ run: RunRecord; items: ApifyRecord[] }> => { const runRecord = await createRun({ actorId, input, waitForFinishSecs: 60, ...runOpts }); - const items = await dataset.listItems({ + const { items } = await dataset.listItems({ datasetId: runRecord.defaultDatasetId as string, fields, limit, }); return { run: runRecord, items }; @@ -328,40 +380,36 @@ function makeApifyBinding(token: string, apiV2: string, parentOrigin: string | u }; const dataset = { - // Returns the items array directly (no wrapper). The Apify API's + // `await` resolves to one page: { items, count, offset, limit, desc }. `for await` + // auto-paginates through the entire dataset, one item at a time (replaces the old, + // separate dataset.iterate() method — see makePaginatedList). offset/limit/desc echo + // back what the API actually applied (read from its x-apify-pagination-* response + // headers, not just the request), except `total`: the Apify API's // `x-apify-pagination-total` header is unreliable for freshly-created datasets - // (eventually consistent), so we don't surface a `total`. Use `inferFields` if you - // need an item count, or iterate to consume the whole dataset. - listItems: async ({ datasetId, fields, omit, limit, offset, clean, desc }: DatasetListOptions): Promise => { - const response = await apiCall('GET', `/datasets/${encodeURIComponent(datasetId)}/items`, { - searchParams: { - fields: fields?.join(','), - omit: omit?.join(','), - limit, - offset, - clean: clean ? '1' : undefined, - desc: desc ? '1' : undefined, - }, - }); - return response.json(); - }, - - // Async generator over the entire dataset. Pages internally in `batchSize` chunks - // so the user can `for await (const item of apify.dataset.iterate({...}))` without - // worrying about offsets. Stops when a page returns fewer items than `batchSize` - // (the natural end-of-data signal — pagination total is not used, see listItems). - iterate: async function* ({ datasetId, fields, omit, clean, desc, batchSize = DEFAULT_ITERATE_BATCH }: DatasetIterateOptions): AsyncGenerator { - let offset = 0; - while (true) { - const items = await dataset.listItems({ - datasetId, fields, omit, clean, desc, - limit: batchSize, offset, + // (eventually consistent), so it's never surfaced — `count` (this page's actual item + // count) is what you want instead. Use `inferFields` if you need an approximate total. + listItems: ({ datasetId, fields, omit, limit, offset = 0, clean, desc }: DatasetListOptions): PaginatedItems => { + const fetchPage = async (pageOffset: number, pageLimit?: number): Promise => { + const response = await apiCall('GET', `/datasets/${encodeURIComponent(datasetId)}/items`, { + searchParams: { + fields: fields?.join(','), + omit: omit?.join(','), + limit: pageLimit, + offset: pageOffset, + clean: clean ? '1' : undefined, + desc: desc ? '1' : undefined, + }, }); - if (items.length === 0) break; - for (const item of items) yield item; - if (items.length < batchSize) break; - offset += items.length; - } + const items: ApifyRecord[] = await response.json(); + return { + items, + count: items.length, + offset: Number(response.headers.get('x-apify-pagination-offset') ?? pageOffset), + limit: Number(response.headers.get('x-apify-pagination-limit') ?? pageLimit ?? items.length), + desc: response.headers.get('x-apify-pagination-desc') === 'true', + }; + }; + return makePaginatedList(fetchPage, offset, limit); }, // Apify has no dedicated schema endpoint; we infer one from a small sample of items. @@ -369,7 +417,7 @@ function makeApifyBinding(token: string, apiV2: string, parentOrigin: string | u // dataset schema (a different concept, described in this Actor's own actor.json). inferFields: async ({ datasetId, sample = DEFAULT_GET_SCHEMA_SAMPLE }: DatasetSchemaOptions): Promise => { const meta = await apiData('GET', `/datasets/${encodeURIComponent(datasetId)}`); - const items = await dataset.listItems({ datasetId, limit: sample }); + const { items } = await dataset.listItems({ datasetId, limit: sample }); const fields = new Map>(); for (const item of items) { for (const [name, value] of Object.entries(item ?? {})) { @@ -445,10 +493,18 @@ function makeApifyBinding(token: string, apiV2: string, parentOrigin: string | u // GET /v2/store — Apify Store search (a top-level resource in the Apify API, // tagged `Store`, distinct from `Actors` — hence a top-level binding rather - // than an `actor.*` method). Returns the items array directly. - const store = ({ search, limit, category }: StoreSearchOptions): Promise => - apiData('GET', '/store', { searchParams: { search, limit, category } }) - .then((page: { items: ApifyRecord[] }) => page.items); + // than an `actor.*` method). Same dual nature as dataset.listItems: `await` for one + // page, `for await` to walk every match. offset/limit echo back the request (the + // endpoint's own JSON body doesn't carry pagination metadata beyond `items`, unlike + // dataset's header-based pagination — see makePaginatedList). + const store = ({ search, limit, offset = 0, category }: StoreSearchOptions): PaginatedItems> => { + const fetchPage = async (pageOffset: number, pageLimit: number | undefined): Promise> => { + const page = await apiData('GET', '/store', { searchParams: { search, limit: pageLimit, offset: pageOffset, category } }); + const items: ApifyRecord[] = page.items; + return { items, count: items.length, offset: pageOffset, limit: pageLimit ?? items.length }; + }; + return makePaginatedList(fetchPage, offset, limit); + }; // Freeze every namespace (and the wrapper) so the script can't reassign a method to // corrupt its own behavior or, for `console` below, its own output capture. From 83eb3ae625063ddc30a24d017d482c11ae305505 Mon Sep 17 00:00:00 2001 From: MQ37 Date: Fri, 17 Jul 2026 14:55:46 +0200 Subject: [PATCH 33/38] docs: nudge Actor description/README with data-backed usage thresholds A/B eval (run 2, apify-mcp-server evals/workflows) shows code mode's worth-it point isn't item count alone but item count x per-item payload size, given the sandbox's fixed ~20K token round-trip overhead: - 5-item single lookup: +78% slower, +46% more tokens (loses) - 100-record filter/sort/aggregate: -35% faster, -19% fewer tokens (wins) - 20-item fan-out over per-item web pages: -59% faster, -75% fewer tokens (wins decisively even at low item count) Encode this as concrete thresholds in both the actor.json description (agent-facing at discovery time) and README (agent-facing after fetch-actor-details). --- .actor/actor.json | 2 +- README.md | 24 ++++++++++++++++++------ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/.actor/actor.json b/.actor/actor.json index 95202c8..e715d0d 100644 --- a/.actor/actor.json +++ b/.actor/actor.json @@ -2,7 +2,7 @@ "actorSpecification": 1, "name": "code-runtime", "title": "Code Runtime", - "description": "Runs one JS script in a sandboxed Actor with an apify binding (run Actors, datasets & KV stores). Best for data-heavy jobs: scrape hundreds+ places/items, chain Actor outputs, and filter/sort/aggregate in one billed run. Results land in the dataset. Read the README before use.", + "description": "Runs one JS script in a sandboxed Actor with an apify binding (run Actors, datasets & KV stores). Worth it only for bulk work: filtering/sorting/aggregating 50+ dataset records, or fanning out over 10+ sub-resources (e.g. visiting many pages) per run. Skip it for a single small (<10 item) lookup \u2014 a direct Actor call is faster and cheaper there. Results land in the dataset. Read the README before use.", "version": "0.1", "buildTag": "latest", "usesStandbyMode": false, diff --git a/README.md b/README.md index b878660..d6499b1 100644 --- a/README.md +++ b/README.md @@ -15,12 +15,24 @@ search the Store, run an Actor, read its dataset, filter and aggregate the results — instead of sending every intermediate result back through the model and wasting tokens. This Actor is the sandbox that runs that script. -**Best suited for data-heavy jobs** — scraping hundreds or thousands of -places/items via an Actor, then filtering, sorting, or aggregating them -locally before returning a small summary. **Weaker fit** for steps that -require reading or judging free text (picking a fact out of an article, -choosing a search term) — keep the model in the loop there instead; a wrong -guess inside the sandbox fails silently until the whole script finishes. +**Worth it only for bulk work**, measured empirically (A/B eval, tokens/duration +vs. calling Actor tools directly): + +- **Filtering/sorting/aggregating ~50+ dataset records** in one dataset — + modest win (~20-35% less time, ~20% fewer tokens at 100 records). +- **Fanning out over ~10+ sub-resources with a sizeable payload each** + (e.g. visiting many pages, chaining Actor outputs) — decisive win even at + small counts, since every skipped round trip avoids funneling a whole raw + page/document through the model (~60% less time, ~75% fewer tokens at 20 + places × 20 page fetches). +- **Below ~10 items with no fan-out** (a single small lookup) — **don't use + this Actor**. The sandbox's own round-trip overhead (~20K tokens) isn't + paid back; a direct Actor call is both faster and cheaper. + +**Weaker fit** for steps that require reading or judging free text (picking a +fact out of an article, choosing a search term) — keep the model in the loop +there instead; a wrong guess inside the sandbox fails silently until the +whole script finishes. ## Calling this Actor From 73468849ec91dbcf3d65b951330d93f085ef660a Mon Sep 17 00:00:00 2001 From: MQ37 Date: Fri, 17 Jul 2026 14:57:03 +0200 Subject: [PATCH 34/38] fix: trim actor.json description to fit under 300 chars --- .actor/actor.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.actor/actor.json b/.actor/actor.json index e715d0d..cf3d551 100644 --- a/.actor/actor.json +++ b/.actor/actor.json @@ -2,7 +2,7 @@ "actorSpecification": 1, "name": "code-runtime", "title": "Code Runtime", - "description": "Runs one JS script in a sandboxed Actor with an apify binding (run Actors, datasets & KV stores). Worth it only for bulk work: filtering/sorting/aggregating 50+ dataset records, or fanning out over 10+ sub-resources (e.g. visiting many pages) per run. Skip it for a single small (<10 item) lookup \u2014 a direct Actor call is faster and cheaper there. Results land in the dataset. Read the README before use.", + "description": "Runs one JS script in a sandboxed Actor with an apify binding (Actors, datasets, KV stores). Worth it for bulk work: 50+ record filter/sort/aggregate, or 10+ item fan-out (e.g. many page visits). Skip for a single <10-item lookup \u2014 call the Actor directly instead. Read the README first.", "version": "0.1", "buildTag": "latest", "usesStandbyMode": false, From a767a0eae1520c0dbcf5e88e9c2f6b98a749199e Mon Sep 17 00:00:00 2001 From: MQ37 Date: Fri, 17 Jul 2026 15:05:52 +0200 Subject: [PATCH 35/38] docs: cut internal-telemetry section, dedupe repeated facts, tighten prose MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Council review (Pike, Davis, Hotz) flagged the README as over-explained: same facts stated 2-4x across sections with circular cross-references. - Remove 'Limitations' section (meta.origin/meta.actorRunId telemetry attribution) entirely — internal plumbing irrelevant to a caller deciding how to use this Actor. - Merge 'Limits & failure modes' into Output — both described the same exitCode/statusMessage vs run-status distinction, cross-referencing each other in a circle. - 'No imports' facts now live once in Permissions & safety; How it works just points there instead of restating. - Split the nested-run bullet (was one sentence doing four jobs) into two. - 'Worth it only for bulk work' bullets -> table. - Dual Promise/AsyncIterable explanation for listItems/store stated once (apify binding section); Recipes and inline comments now just reference it. - Fixed a real inconsistency: Output's terminal-status list was missing ABORTED (present in the Recipes poll-loop code right below it). 273 -> 221 lines, same information. --- README.md | 204 ++++++++++++++++++++---------------------------------- 1 file changed, 76 insertions(+), 128 deletions(-) diff --git a/README.md b/README.md index d6499b1..a764b23 100644 --- a/README.md +++ b/README.md @@ -6,33 +6,24 @@ ## What it does -This Actor executes JavaScript that an AI agent submits through the Apify MCP +Executes one JS script that an AI agent submits through the Apify MCP Server's **Code Mode**, then returns whatever the script printed. JS only — -nothing transpiles it, so a TypeScript type annotation is a SyntaxError at load. - -Code Mode exists so an agent can do many Apify operations in **one go** — -search the Store, run an Actor, read its dataset, filter and aggregate the -results — instead of sending every intermediate result back through the model -and wasting tokens. This Actor is the sandbox that runs that script. - -**Worth it only for bulk work**, measured empirically (A/B eval, tokens/duration -vs. calling Actor tools directly): - -- **Filtering/sorting/aggregating ~50+ dataset records** in one dataset — - modest win (~20-35% less time, ~20% fewer tokens at 100 records). -- **Fanning out over ~10+ sub-resources with a sizeable payload each** - (e.g. visiting many pages, chaining Actor outputs) — decisive win even at - small counts, since every skipped round trip avoids funneling a whole raw - page/document through the model (~60% less time, ~75% fewer tokens at 20 - places × 20 page fetches). -- **Below ~10 items with no fan-out** (a single small lookup) — **don't use - this Actor**. The sandbox's own round-trip overhead (~20K tokens) isn't - paid back; a direct Actor call is both faster and cheaper. - -**Weaker fit** for steps that require reading or judging free text (picking a -fact out of an article, choosing a search term) — keep the model in the loop -there instead; a wrong guess inside the sandbox fails silently until the -whole script finishes. +a TypeScript annotation is a SyntaxError at load, nothing transpiles it. + +Code Mode lets an agent do many Apify operations in **one call** — search +the Store, run an Actor, read its dataset, filter and aggregate — instead of +sending every intermediate result back through the model. This Actor is the +sandbox that runs that script. + +**Worth it only for bulk work** (measured, A/B eval vs. calling Actor tools +directly): + +| Workload | Verdict | +|---|---| +| Filter/sort/aggregate 50+ dataset records | Modest win — ~20-35% less time, ~20% fewer tokens | +| Fan out over 10+ sub-resources with a sizeable payload each (visit many pages, chain Actors) | Decisive win — ~60% less time, ~75% fewer tokens | +| Under 10 items, no fan-out | **Don't use this Actor** — ~20K-token sandbox overhead isn't paid back | +| Reading/judging free text (pick a fact, choose a search term) | Keep the model in the loop — a wrong guess fails silently until the script ends | ## Calling this Actor @@ -49,32 +40,30 @@ as the body. Results land in the run's default dataset, same as any Actor call — follow the response's `nextStep` (or call `get-dataset-items`/ `GET /v2/datasets/{datasetId}/items`) to read it. +Default `timeoutSecs: 900`, `memoryMbytes: 1024` (`.actor/actor.json`) — +override per call for scripts chaining several long Actor runs (MCP +`call-actor`'s `callOptions.timeout`/`callOptions.memory`, or the API's +`timeout`/`memory`). + ## How it works -- **One script per run.** The Actor reads your `code`, runs it once, writes the - result, and exits. -- The code runs inside a [`workerd`](https://github.com/cloudflare/workerd) V8 - isolate: **no imports** — neither npm packages nor Node built-in `node:*` - modules are available (`import`/`require` of any module fails); web-standard - globals such as `fetch` are present. Outbound network is restricted to - `*.apify.com`. -- Inside the script a global **`apify`** object exposes a small, typed subset of - the Apify API — run Actors, read/write datasets and key-value stores — using - the current run's token (see below). -- `console.log` / `console.info` go to **stdout**; `console.error` / - `console.warn` go to **stderr**. The two streams are captured separately. -- Before running an Actor from your script, call `apify.actor.get({ actorId })` - once to read its input/output schema. -- As each nested run finishes, log its `run.id` / `defaultDatasetId` / - `defaultKeyValueStoreId` **before** processing its output (nothing persists - between this Actor's own runs, but the Actors it started keep their - results). **If a prior attempt's `defaultDatasetId`/`defaultKeyValueStoreId` - is visible in your own earlier turns, reuse it — do not re-run the same - Actor call with identical input.** Re-running wastes the compute/cost of a - call that already succeeded. -- Print a small, JSON-stringified summary of the result — never dump full - datasets. Only what you `console.log`/`console.info` comes back; a - top-level `return` value is **not** captured. +- **One script per run.** Reads `code`, runs it once, writes the result, exits. +- Runs inside a sandboxed [`workerd`](https://github.com/cloudflare/workerd) + V8 isolate — see [Permissions & safety](#permissions--safety) for what's allowed. +- A global **`apify`** object exposes a small, typed subset of the Apify API + — run Actors, read/write datasets and key-value stores — using the + current run's token. +- `console.log`/`console.info` → **stdout**; `console.error`/`console.warn` + → **stderr**, captured separately. +- Call `apify.actor.get({ actorId })` before running an Actor you haven't + checked — don't guess its input schema. +- Log a nested run's `run.id`/`defaultDatasetId`/`defaultKeyValueStoreId` + before processing its output — nothing persists between this Actor's own + runs, but the Actors it started keep theirs. +- Already have a dataset/store ID from an earlier turn? Reuse it — don't + re-run an identical call, it wastes cost. +- Print a small JSON summary, never a full dataset — only `console.log`/ + `console.info` output comes back; a top-level `return` is **not** captured. ## Input @@ -90,56 +79,37 @@ call — follow the response's `nextStep` (or call `get-dataset-items`/ ## Output -A single **dataset item** with the captured streams, the script's exit -status, and a prose status message: +A single **dataset item**: ```json { "stdout": "Apify: Full-stack web scraping ...\n...", "stderr": "", "exitCode": 0, "statusMessage": "Script completed" } ``` -If the script throws, the error lands in `stderr`, `stdout` keeps whatever was -printed before the failure, `exitCode` is `1`, and `statusMessage` is -`"Script threw: ..."`. The Actor run itself still **succeeds** — check -`exitCode`/`statusMessage`, not `stderr` content, to detect a failed script, -since `stderr` is also a legitimate log channel (`console.error` / -`console.warn`). - -If the script fails to **compile** (a syntax error), the same contract -applies — `exitCode: 1`, `statusMessage: "Failed to compile: ..."` — pushed -by the container entrypoint directly, since a malformed script never reaches -the sandboxed worker at all. - -A run-level **timeout or out-of-memory kill** is a different, third outcome: -the container is killed before it can push anything, so this dataset item may -not exist for that run at all. That case is signaled by the Actor run's own -status (`SUCCEEDED` vs `FAILED`/`TIMED-OUT`), not by this item's absence — -see [Limits & failure modes](#limits--failure-modes). - -## Limits & failure modes - -- Default `defaultRunOptions`: `timeoutSecs: 900`, `memoryMbytes: 1024` - (`.actor/actor.json`). Override per call — e.g. the MCP `call-actor` tool's - `callOptions.timeout`/`callOptions.memory`, or the API's `timeout`/`memory` - run options — for scripts that chain several long-running Actor calls. -- `exitCode`/`statusMessage` signal the **script's** outcome only (returned / - threw / failed to compile). A resource-limit kill is a **run-level** - outcome instead — check the Actor run's own `status`, not this dataset - item, for that case (see [Output](#output) above). +| Outcome | `exitCode` | `statusMessage` | +|---|---|---| +| Script returned | `0` | `Script completed` | +| Script threw | `1` | `Script threw: ...` | +| Failed to compile (syntax error) | `1` | `Failed to compile: ...` | +| Run-level timeout / OOM kill | — | item may not exist for this run at all | + +Check `exitCode`/`statusMessage`, not `stderr` content, to detect a failed +script — `stderr` also carries `console.error`/`console.warn` output, so its +presence alone isn't failure. A timeout/OOM kill is signaled by the Actor +run's own status (`SUCCEEDED` vs `FAILED`/`ABORTED`/`TIMED-OUT`), not by this +item's absence. ## Permissions & safety -- Runs with **limited permissions**: the sandbox has no filesystem and - outbound `fetch` (including through redirects, which are re-validated per - hop) is limited to the Apify API (`*.apify.com`). -- **No imports.** The isolate runs without workerd's `nodejs_compat`, so user - code cannot import Node built-ins (`node:net`, `node:fs`, …) or npm packages. - This removes `node:net` — a raw-socket egress path that would otherwise bypass - the `fetch` allowlist — and keeps the run token out of `process.env` (which is - not defined). +- Sandbox has **no filesystem**; outbound `fetch` (redirects re-validated + per hop) is limited to the Apify API (`*.apify.com`). +- **No imports** — runs without workerd's `nodejs_compat`, so no Node + built-ins (`node:net`, `node:fs`, …) or npm packages. This also removes + `node:net` (a raw-socket path that would bypass the `fetch` allowlist) and + keeps the run token out of `process.env` (undefined here). - Each run is an isolated, single-use container — nothing persists between runs. -- This closes off **direct fetch-based exfil** from the container — it does not - stop every path to move data out (e.g. `actor.start({ input })` on an Actor - with its own open internet access, or writing to a dataset/key-value store). +- This closes **direct fetch-based exfil** — it does not close every path to + move data out (e.g. `actor.start({ input })` on an Actor with its own + internet access, or writing to a dataset/key-value store). ## Recipes @@ -158,10 +128,10 @@ console.log(JSON.stringify(pages.slice(0, 3).map((p) => p.url))); ### Bounded parallel fan-out -This Actor's clearest win: run several independent Actors (or the same Actor -over several inputs) concurrently, then reduce before returning. Chunk the -fan-out (e.g. 5–10 at a time) — an unbounded `Promise.all` over many inputs -can hit your account's concurrent-run or memory limits. +This Actor's clearest win: run several Actors (or the same Actor over +several inputs) concurrently, then reduce before returning. Chunk it (5–10 +at a time) — an unbounded `Promise.all` can hit your account's +concurrent-run or memory limits. ```js const inputs = [{ query: 'a' }, { query: 'b' }, { query: 'c' } /* ... */]; @@ -179,9 +149,8 @@ console.log(JSON.stringify(results.slice(0, 5))); // small summary, not the full ### Read an entire dataset without managing offsets -`dataset.listItems` (and `store`) return a value that's both a `Promise` (one -page) and an `AsyncIterable` (every item, auto-paginated) — pick whichever -you need: +`dataset.listItems`/`store` work two ways — `await` for one page, `for +await` to auto-paginate every item (see [the apify binding](#the-apify-binding)): ```js // One page — e.g. a quick peek @@ -208,56 +177,35 @@ while (!TERMINAL.includes(run.status)) { } ``` -## Limitations - -Sub-runs started from inside the sandbox (via `apify.actor.start/call/callAndGetItems`) -get `meta.origin: 'MCP'` on the Apify platform, same as this Actor's own run, when -this Actor was itself started via the Apify MCP Server — this Actor reads its own -`APIFY_META_ORIGIN` env var (platform-set, not spoofable from inside the sandbox) -and forwards `X-Apify-Request-Origin: MCP` on its own API calls only when that's -`MCP`. Every sub-run also gets a platform-native `meta.actorRunId` link back to -this run regardless of origin (set automatically from the run-scoped token, no -code needed here) — so even a fully generic query can already walk sub-run → -`meta.actorRunId` → parent `meta.origin` to reconstruct the chain; the origin -forwarding above just makes single-field origin queries work without that join. - -What's still not attributed: the specific MCP *session* (which client, which -conversation) that triggered this Actor's own run in the first place — that -context isn't part of the platform's Run schema at all, on or off Code Mode. - ## The `apify` binding Every method takes one options object and returns parsed JSON — except -`store` and `dataset.listItems`, which return a value that's both a `Promise` -(one page) and an `AsyncIterable` (every match/item, auto-paginated) — see -their own lines below (`?` = optional, `= x` = default). Full API -documentation is available -[here](https://github.com/apify/actor-code-runtime/blob/master/docs/API.md). +`store` and `dataset.listItems`, which return a value that's both a +`Promise` (one page) and an `AsyncIterable` (every match/item, +auto-paginated). Full API docs: +[API.md](https://github.com/apify/actor-code-runtime/blob/master/docs/API.md). +(`?` = optional, `= x` = default) ```js // Store — GET /v2/store, a top-level Apify API resource (not an Actor method) -// `await` for one page, `for await` to walk every match (same dual nature as -// dataset.listItems below). -apify.store({ search, limit?, offset?, category? }) // → { items, count, offset, limit } +apify.store({ search, limit?, offset?, category? }) // → { items, count, offset, limit }; dual Promise/AsyncIterable, see above // Actors apify.actor.get({ actorId }) // → actor apify.actor.start({ actorId, input?, memoryMbytes?, timeoutSecs?, maxTotalChargeUsd?, maxItems? }) // → run -apify.actor.call({ actorId, ...startOpts, waitForFinishSecs = 60 }) // → run (waits; may return non-terminal READY/RUNNING if the 60s cap elapses first — not an error, poll run.waitForFinish) +apify.actor.call({ actorId, ...startOpts, waitForFinishSecs = 60 }) // → run (may be non-terminal READY/RUNNING past the 60s cap — not an error, see Recipes) apify.actor.callAndGetItems({ actorId, input?, fields?, limit?, ...runOpts }) // → { run, items } // Runs apify.run.get({ runId }) // → run -apify.run.waitForFinish({ runId, waitForFinishSecs = 60 }) // → run (same non-terminal caveat as actor.call above) +apify.run.waitForFinish({ runId, waitForFinishSecs = 60 }) // → run (same non-terminal caveat) apify.run.abort({ runId }) // → run apify.run.getLog({ runId, limit? }) // → string // Datasets apify.dataset.create({ name? }) // → dataset apify.dataset.pushItems({ datasetId, items }) // → void -// `await` for one page: { items, count, offset, limit, desc }. `for await` auto-paginates -// through the whole dataset, one item at a time — no separate iterate() method needed. -apify.dataset.listItems({ datasetId, fields?, omit?, limit?, offset?, clean?, desc? }) +apify.dataset.listItems({ datasetId, fields?, omit?, limit?, offset?, clean?, desc? }) // → { items, count, offset, limit, desc }; dual Promise/AsyncIterable, see above apify.dataset.inferFields({ datasetId, sample = 5 }) // → { itemCount, fields[] } // Key-value stores From 2302300db5fb72fcb6490da23e18de544803bc73 Mon Sep 17 00:00:00 2001 From: MQ37 Date: Fri, 17 Jul 2026 15:14:41 +0200 Subject: [PATCH 36/38] docs: trim Code Mode/API-plumbing mentions from README MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - What it does: drop 'Code Mode' naming and the TypeScript-SyntaxError aside (JS-only is already stated in the Input field description). - Calling this Actor: drop the raw-API POST/nextStep alternative — the call-actor example is the one path that matters here. - Learn more: drop the Code Mode design PR link, keep just the MCP Server link. - Also dropped the earlier free-text-judgment table row per feedback. --- README.md | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index a764b23..c8e2886 100644 --- a/README.md +++ b/README.md @@ -7,13 +7,12 @@ ## What it does Executes one JS script that an AI agent submits through the Apify MCP -Server's **Code Mode**, then returns whatever the script printed. JS only — -a TypeScript annotation is a SyntaxError at load, nothing transpiles it. +Server, then returns whatever the script printed. -Code Mode lets an agent do many Apify operations in **one call** — search -the Store, run an Actor, read its dataset, filter and aggregate — instead of -sending every intermediate result back through the model. This Actor is the -sandbox that runs that script. +Lets an agent do many Apify operations in **one call** — search the Store, +run an Actor, read its dataset, filter and aggregate — instead of sending +every intermediate result back through the model. This Actor is the sandbox +that runs that script. **Worth it only for bulk work** (measured, A/B eval vs. calling Actor tools directly): @@ -23,7 +22,6 @@ directly): | Filter/sort/aggregate 50+ dataset records | Modest win — ~20-35% less time, ~20% fewer tokens | | Fan out over 10+ sub-resources with a sizeable payload each (visit many pages, chain Actors) | Decisive win — ~60% less time, ~75% fewer tokens | | Under 10 items, no fan-out | **Don't use this Actor** — ~20K-token sandbox overhead isn't paid back | -| Reading/judging free text (pick a fact, choose a search term) | Keep the model in the loop — a wrong guess fails silently until the script ends | ## Calling this Actor @@ -35,11 +33,6 @@ default tools: call-actor({ actor: "apify/code-runtime", input: { code: "..." } }) ``` -Or via the raw API: `POST /v2/acts/apify~code-runtime/runs` with `{ code }` -as the body. Results land in the run's default dataset, same as any Actor -call — follow the response's `nextStep` (or call `get-dataset-items`/ -`GET /v2/datasets/{datasetId}/items`) to read it. - Default `timeoutSecs: 900`, `memoryMbytes: 1024` (`.actor/actor.json`) — override per call for scripts chaining several long Actor runs (MCP `call-actor`'s `callOptions.timeout`/`callOptions.memory`, or the API's @@ -218,4 +211,3 @@ apify.keyValueStore.list({ storeId, limit?, exclusiveStartKey? }) // → { item ## Learn more - Apify MCP Server: -- Code Mode design: [apify/apify-mcp-server#794](https://github.com/apify/apify-mcp-server/pull/794) From 2812c9ca3695a855f3e94e533caf4509f063bde7 Mon Sep 17 00:00:00 2001 From: MQ37 Date: Fri, 17 Jul 2026 16:10:21 +0200 Subject: [PATCH 37/38] chore: remove fullReadmeOnly actor.json flag MCP server now hardcodes this Actor's ID to always return the full README, instead of trusting a self-declared actor.json flag any Actor could set to opt itself out of the auto-generated summary. --- .actor/actor.json | 1 - 1 file changed, 1 deletion(-) diff --git a/.actor/actor.json b/.actor/actor.json index cf3d551..a17a10b 100644 --- a/.actor/actor.json +++ b/.actor/actor.json @@ -6,7 +6,6 @@ "version": "0.1", "buildTag": "latest", "usesStandbyMode": false, - "fullReadmeOnly": true, "defaultRunOptions": { "timeoutSecs": 900, "memoryMbytes": 1024 From 8930b95acc4aa46299c7da56309ccb2e224a2a8d Mon Sep 17 00:00:00 2001 From: MQ37 Date: Wed, 22 Jul 2026 09:57:06 +0200 Subject: [PATCH 38/38] fix: gate realFetch claim on request handling, not import order MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #1 review comment (2026-07-21, issuecomment-5037390847) found the claimRealFetch() one-shot handoff added in 5461aee didn't guarantee runner.ts claims first. entrypoint.sh splices `code` verbatim (no escaping) into `export async function run(apify, console) { }`; a bare `}` in `code` closes that function early, and everything after runs as ordinary MODULE-SCOPE code in usercode.js. ES module evaluation order puts that ahead of runner.ts's own top-level code (usercode.js is the import evaluated immediately before runner.ts's own body runs), so injected top-level code using `(await import('./guard.js')).claimRealFetch()` reliably claimed the unrestricted fetch first, making runner.ts's own claim get null and throw -- crashing the whole Actor run. Verified live against the deployed Actor before this fix: workerd exits with 'Uncaught Error: realFetch already claimed — guard.js imported out of order', run FAILS. Root cause isn't import order, it's that claimRealFetch() was claimable at all during module evaluation, before any request. Fix: gate it on genuine request handling instead. guard.ts adds requestHandlingStarted (false until markRequestHandlingStarted() is called) and claimRealFetch() refuses (returns null without consuming the resource) until that flag is set. runner.ts calls markRequestHandlingStarted() + claims realFetch as the first, synchronous statements of the /run path in its fetch handler -- before any `await`, so no attacker-scheduled microtask from usercode.js's module scope can race it. Module evaluation (guard.js, usercode.js, runner.ts's own top level) always completes before workerd dispatches the first request, so this holds regardless of any module-graph ordering, unlike the previous design. realFetch itself moved from a module-top-level const (claimed at runner.ts's own top-level, which is what lost the race) to a module-level definite-assigned `let`, assigned inside the /run handler before any of the functions that close over it run. Also fixes the stale comments at guard.ts (claimRealFetch) and runner.ts (top-of-file + requireRealFetch) asserting import order made runner.ts claim first -- disproved by the live PoC above. Adds tests/fixtures/realfetch-escape.js: a regression probe using the exact escape shape from the review comment (real top-level await, not an async IIFE -- top-level await is what forces usercode.js's evaluation to fully settle before runner.ts's own top-level code runs; an IIFE's internal await doesn't carry that guarantee). Wired into test.sh as a new live check asserting `apify call`'s own exit status (this probe's run() body is an empty no-op post-escape, so it has no captured console to report a sentinel through, unlike the existing binding-smoke/sandbox-isolation probes). Verified against the real deployed Actor, before/after, same probe: - before (reverted worker/*.ts, same probe file): Actor run FAILS, workerd crash log matches the predicted 'guard.js imported out of order' message. - after (this commit): Actor run SUCCEEDS, exitCode 0, statusMessage 'Script completed' -- same as running an empty script. Full suite (binding-smoke, sandbox-isolation, this new regression probe) run live via ./test.sh: 19/19, 17/17, and the regression probe all pass. --- test.sh | 18 ++++++++++++ tests/fixtures/realfetch-escape.js | 43 +++++++++++++++++++++++++++++ worker/guard.ts | 44 ++++++++++++++++++++++++------ worker/runner.ts | 38 ++++++++++++++++---------- 4 files changed, 120 insertions(+), 23 deletions(-) create mode 100644 tests/fixtures/realfetch-escape.js diff --git a/test.sh b/test.sh index 2184a65..650b042 100755 --- a/test.sh +++ b/test.sh @@ -40,3 +40,21 @@ done [ "$failed" -eq 0 ] || { echo "==> some probes FAILED" >&2; exit 1; } echo "==> all probes passed" + +# Regression probe for the realFetch claim-ordering bug (PR #1 review, +# 2026-07-21): tests/fixtures/realfetch-escape.js escapes usercode.js's +# wrapper into module scope and tries to steal the internal-only realFetch +# before runner.ts's own claim. It has no captured console to report through +# (see the file's own comment), so success/failure is the Actor run itself +# succeeding vs. failing -- not a printed sentinel like the probes above. +echo "==> apify call: tests/fixtures/realfetch-escape.js (regression: realFetch claim ordering)" +jq -n --arg code "$(cat tests/fixtures/realfetch-escape.js)" '{ code: $code }' > "$input_json" +if apify call -f "$input_json" -o; then + echo "==> realfetch-escape passed (run succeeded — module-scope steal attempt did not hijack/crash the internal claim)" +else + echo "==> realfetch-escape FAILED (run crashed — realFetch claim-ordering regression, see guard.ts's requestHandlingStarted gate)" >&2 + failed=1 +fi + +[ "$failed" -eq 0 ] || { echo "==> some probes FAILED" >&2; exit 1; } +echo "==> all probes (including regressions) passed" diff --git a/tests/fixtures/realfetch-escape.js b/tests/fixtures/realfetch-escape.js new file mode 100644 index 0000000..5d591f5 --- /dev/null +++ b/tests/fixtures/realfetch-escape.js @@ -0,0 +1,43 @@ +// Regression probe for the realFetch claim-ordering bug found in PR #1's +// review (2026-07-21): +// https://github.com/apify/actor-code-runtime/pull/1#issuecomment-5037390847 +// +// entrypoint.sh splices `code` verbatim (no escaping) into +// `export async function run(apify, console) { }`. The bare `}` right +// after this comment block closes that function early -- deliberately, to +// reproduce the escape -- so `run` becomes a harmless no-op (nothing is left +// in its body once the comments end) and everything after runs as ordinary +// MODULE-SCOPE code in usercode.js: workerd evaluates that unconditionally, +// before this worker ever calls runner.ts's request handler. That used to be +// enough to `await import('./guard.js')` and call claimRealFetch() directly, +// stealing the unrestricted, un-allowlisted fetch before runner.ts's own +// claim (previously made at runner.ts's own module top level) ever ran -- +// which made THAT claim get null, throw, and crash the whole Actor run +// (self-DoS; the real exploit payoff for an attacker would be using the +// stolen fetch directly, not reported here). +// +// Not valid JS on its own (it opens with an unbalanced `}`) -- intentionally, +// same shape as the reported PoC. Not TypeScript, not compiled, not +// typechecked (lives under tests/fixtures/, outside tsconfig's `include` and +// outside the `tests/*.js` build-artifact glob): see test.sh, which pushes +// this file's raw content directly as the `code` input. +// +// Expected result with the fix (guard.ts's requestHandlingStarted gate): +// claimRealFetch() called from module scope returns null without consuming +// the resource, so nothing crashes -- this Actor run completes normally +// (exitCode 0, "Script completed"), same as any run of an empty script. +// Before the fix: this run FAILS outright (workerd crashes during module +// evaluation; entrypoint.sh can't tell that apart from a real infra failure +// and fails the whole Actor run). test.sh asserts on `apify call`'s own exit +// status, not a printed sentinel -- this probe's `run` body never executes +// any of its own code, so it has no captured console to report through. +// +// Must be genuine top-level await, not an async IIFE: a module containing +// top-level await defers the evaluation of modules that depend on it (here, +// runner.ts) until that await settles (see MDN/TC39 "Top-level await", +// Asynchronous Module Evaluation) -- an IIFE's internal await does not carry +// that guarantee, so it would not reliably win the race this probe exists to +// exercise. +} +globalThis.__stolenRealFetch = (await import('./guard.js')).claimRealFetch(); +;{ diff --git a/worker/guard.ts b/worker/guard.ts index 17840ef..46c62ab 100644 --- a/worker/guard.ts +++ b/worker/guard.ts @@ -3,7 +3,7 @@ // runs at module-evaluation time. Our own Apify API calls use the real, // unrestricted fetch (the internal API is a private IP, not *.apify.com) — // see claimRealFetch() below for how runner.js gets it without leaving it -// reachable from user code. +// usable by user code. // // Egress surface (workerd, no nodejs_compat): the only JS-reachable outbound // primitives are fetch, WebSocket, and EventSource. Raw sockets (node:net, @@ -15,16 +15,42 @@ const realFetch = globalThis.fetch.bind(globalThis); -// One-shot handoff of the unrestricted fetch to runner.js. ES modules are -// evaluated once and cached, so `guard.js` is the same module instance no -// matter who imports it. runner.js imports this module (and calls -// claimRealFetch()) before usercode.js is ever imported, so it always claims -// first. If the sandboxed script later does `await import('./guard.js')` to -// try to recover the unrestricted fetch, it gets this same cached instance — -// but the value is already gone. A standing `export { realFetch }` would hand -// it to that later import too; don't reintroduce one. +// One-shot handoff of the unrestricted fetch to runner.js — gated on genuine +// request handling, not on import order between guard.js and usercode.js. +// +// entrypoint.sh splices the `code` input verbatim (no escaping) into +// `export async function run(apify, console) { }`. A bare `}` in +// `code` closes that function early; everything after it runs as ordinary +// MODULE-SCOPE code in usercode.js, executed during module evaluation — +// i.e. unconditionally, before workerd ever calls this worker's own +// `fetch(request, env)` handler. That escaped code can `await +// import('./guard.js')` and call claimRealFetch() itself. +// +// This module previously assumed import order made runner.js's own claim run +// first (guard.js has no dependency on usercode.js, so it evaluates before +// it) — that protects nothing here, because it's usercode.js's top-level code +// racing runner.ts's top-level code, and usercode.js is the import evaluated +// immediately before runner.ts's own body runs. Verified live (PR #1 review, +// 2026-07-21): a `code` input using the escape above reliably wins that race, +// so runner.ts's own claim got null and threw, crashing the whole Actor run. +// +// The actual invariant that holds regardless of any module-graph ordering: +// usercode.js's module scope never runs *inside* a `fetch(request, env)` +// call — module evaluation always finishes before workerd dispatches the +// first request. So claimRealFetch() refuses to hand anything out until +// markRequestHandlingStarted() has been called; runner.ts calls it (and +// claims) as the first, synchronous statement of its `/run` handling, before +// any `await`, so no attacker-scheduled microtask can race it either. A claim +// attempted before that (legitimate or injected) gets null without consuming +// the resource, so the real claim still succeeds afterward. +let requestHandlingStarted = false; +export function markRequestHandlingStarted(): void { + requestHandlingStarted = true; +} + let unclaimedRealFetch: typeof realFetch | null = realFetch; export function claimRealFetch(): typeof realFetch | null { + if (!requestHandlingStarted) return null; // too early to be a trusted caller const fetchFn = unclaimedRealFetch; unclaimedRealFetch = null; return fetchFn; diff --git a/worker/runner.ts b/worker/runner.ts index f90acbf..ad3d1ec 100644 --- a/worker/runner.ts +++ b/worker/runner.ts @@ -7,26 +7,31 @@ // Single-tenant: one run = one container = one program = one token. No Worker // Loader / per-request isolate is needed — the program runs in this worker, // which is itself the sandbox (no filesystem, restricted outbound network). -// guard.js must be imported before usercode.js: it overrides globalThis.fetch -// to allow only apify.com, and hands us the real, unrestricted fetch via a -// one-shot claimRealFetch() for our own (internal) API calls — see guard.js -// for why this is a claim, not a standing export. -import { claimRealFetch } from './guard.js'; +// guard.js overrides globalThis.fetch to allow only apify.com, and hands us +// the real, unrestricted fetch via a one-shot claimRealFetch() for our own +// (internal) API calls. That claim is gated on markRequestHandlingStarted() +// (called below, first thing in the `fetch` handler's `/run` path), not on +// import order — usercode.js's module scope can run attacker-controlled code +// before this module's own top-level code does, so the claim must not +// happen at this module's top level either. See guard.ts's comment on +// claimRealFetch for the full reasoning. +import { claimRealFetch, markRequestHandlingStarted } from './guard.js'; import { run } from './usercode.js'; -// Must run before usercode.js's `run()` is ever invoked (it does, here — module -// evaluation order puts this ahead of any dynamic import from inside `run()`). -// Factored into a function (rather than a bare `const` + `if (!x) throw`) so the -// non-null guarantee is encoded in the return type once, here — TS doesn't carry -// a narrowed-from-null check across the later function declarations that close -// over `realFetch`, but a return type with the `null` branch already thrown away -// needs no further narrowing anywhere downstream. +// Called as the first, synchronous statement of the `/run` path in the +// `fetch` handler below — before any `await`, so no attacker-scheduled +// microtask from usercode.js's module scope can call claimRealFetch() in +// between. See guard.ts. function requireRealFetch(): typeof globalThis.fetch { const fetchFn = claimRealFetch(); - if (!fetchFn) throw new Error('realFetch already claimed — guard.js imported out of order.'); + if (!fetchFn) throw new Error('realFetch already claimed.'); return fetchFn; } -const realFetch = requireRealFetch(); +// Assigned inside the `fetch` handler's `/run` path (the only call path into +// the functions below) before any of them run — definite-assignment +// asserted rather than left `| undefined` so call sites here don't need +// null checks. +let realFetch!: typeof globalThis.fetch; const DEFAULT_GET_SCHEMA_SAMPLE = 5; @@ -540,6 +545,11 @@ export default { if (url.pathname === '/health') return new Response('ok'); if (url.pathname !== '/run') return new Response('Not found', { status: 404 }); + // First, synchronous statements of the /run path — see guard.ts and the + // requireRealFetch comment above for why this can't happen any earlier. + markRequestHandlingStarted(); + realFetch = requireRealFetch(); + const token = env.APIFY_TOKEN; if (!token) throw new Error('APIFY_TOKEN missing from Actor run environment.'); // APIFY_API_BASE_URL is the platform-internal API (may have a trailing slash).