diff --git a/.actor/actor.json b/.actor/actor.json new file mode 100644 index 0000000..a17a10b --- /dev/null +++ b/.actor/actor.json @@ -0,0 +1,76 @@ +{ + "actorSpecification": 1, + "name": "code-runtime", + "title": "Code Runtime", + "description": "Runs one JS script in a sandboxed Actor with an apify binding (Actors, datasets, KV stores). Worth it for bulk work: 50+ record filter/sort/aggregate, or 10+ item fan-out (e.g. many page visits). Skip for a single <10-item lookup \u2014 call the Actor directly instead. Read the README first.", + "version": "0.1", + "buildTag": "latest", + "usesStandbyMode": false, + "defaultRunOptions": { + "timeoutSecs": 900, + "memoryMbytes": 1024 + }, + "input": { + "title": "Code Runtime Input", + "description": "The program to run inside the sandbox.", + "type": "object", + "schemaVersion": 1, + "properties": { + "code": { + "title": "Code", + "type": "string", + "description": "JavaScript executed in the sandbox with `apify` and `console` globals; only console output is captured and pushed to the dataset as { stdout, stderr, exitCode, statusMessage } — a top-level `return` value is NOT captured. Before writing code that calls a specific Actor, check its real input field names first — via fetch-actor-details (outside this script, before you write it) or apify.actor.get({ actorId }) (inside it, for an Actor picked at runtime). Do not guess field names from memory; a wrong one throws a fast 400, but costs a wasted round trip. Print a small JSON summary of the result — never dump full datasets. Write top-level `await` statements directly in the script; do NOT wrap your logic in an async function you call without awaiting (e.g. `async function main(){...}; main()`) — the script returns as soon as the top-level body finishes, silently discarding anything still pending, with no error. Every apify.* method takes ONE options object keyed by id, e.g. apify.actor.call({ actorId, input }), apify.dataset.listItems({ datasetId, limit }) — this is NOT the public apify-client SDK's curried apify.actor(id).call(input) shape. If a prior attempt already logged a nested run's defaultDatasetId/defaultKeyValueStoreId (visible in your own earlier turns), reuse it — do NOT re-run the same Actor call with identical input, that wastes compute on a call that already succeeded. apify.actor.call/run.waitForFinish may return non-terminal (READY/RUNNING) once the 60s wait cap elapses — that is NOT a failure, poll again instead of throwing.", + "editor": "javascript" + } + }, + "required": ["code"] + }, + "output": { + "actorOutputSchemaVersion": 1, + "title": "Code Runtime Output", + "description": "One dataset item { stdout, stderr, exitCode, statusMessage } (exitCode: 0=returned, 1=threw); see the exitCode field description for the timeout/OOM case.", + "type": "object", + "properties": { + "output": { + "type": "string", + "title": "Execution output", + "template": "{{links.apiDefaultDatasetUrl}}/items" + } + } + }, + "storages": { + "dataset": { + "actorSpecification": 1, + "description": "Present only if the script ran to completion (returned or threw). A run-level timeout or OOM kill produces zero items for that run — the calling Actor-run's own status (SUCCEEDED vs FAILED/TIMED-OUT) is the signal for that case, not item presence.", + "fields": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "stdout": { + "type": "string", + "title": "stdout", + "description": "Captured console.log/console.info output." + }, + "stderr": { + "type": "string", + "title": "stderr", + "description": "Captured console.error/console.warn output, plus the thrown error (or compile failure) message when exitCode is 1." + }, + "exitCode": { + "type": "integer", + "enum": [0, 1], + "title": "Exit code", + "description": "0 = script returned normally, 1 = script threw (or failed to compile)." + }, + "statusMessage": { + "type": "string", + "title": "Status message", + "description": "Prose form of the same signal: 'Script completed' / 'Script threw: ...' / 'Failed to compile: ...'." + } + }, + "required": ["stdout", "stderr", "exitCode", "statusMessage"] + } + } + }, + "dockerfile": "../Dockerfile" +} diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..ec15a3b --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +node_modules +.git +*.log +.DS_Store diff --git a/.github/workflows/typecheck.yml b/.github/workflows/typecheck.yml new file mode 100644 index 0000000..ccac2fd --- /dev/null +++ b/.github/workflows/typecheck.yml @@ -0,0 +1,19 @@ +name: Typecheck + +on: + push: + branches: [master] + pull_request: + +jobs: + typecheck: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: pnpm + - run: pnpm install --frozen-lockfile + - run: pnpm run typecheck diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..39046c7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +node_modules/ +# Generated at container startup by entrypoint.sh (never checked in): +worker/usercode.js +# Compiled from worker/*.ts and tests/*.ts by `pnpm build` (tsconfig.json emits next to source): +worker/runner.js +worker/guard.js +tests/*.js +*.log +.DS_Store diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..fe47c06 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,43 @@ +# Two-stage build: drop the Node runtime entirely. workerd is a standalone +# glibc binary; only libc + libm are needed at runtime (verified via `ldd`). +# +# Stage 1: pull the workerd binary + compile worker/*.ts -> worker/*.js, via a Node +# base. Node already runs here to resolve workerd's binary path, so compiling +# TypeScript is one more RUN line, not a new toolchain. pnpm keeps workerd in the +# virtual store (not hoisted), so resolve the path through `require('workerd')`. +FROM node:24-bookworm-slim AS builder +WORKDIR /build +COPY package.json pnpm-lock.yaml tsconfig.json ./ +COPY worker/ ./worker/ +# --ignore-scripts skips workerd's postinstall (a binary-download fallback we +# don't need — the binary ships in the @cloudflare/workerd-linux-64 optional dep) +# and avoids pnpm's hard error on unapproved dependency build scripts. Full +# (non --prod) install: typescript is a devDependency, needed to compile below. +# Compile with tsc directly, not `pnpm run build`: that script also sed's +# tests/*.js (dev-only probe fixtures for test.sh, submitted as Actor input at +# run time — never part of the image), which isn't copied into this build +# context and doesn't need to be; tsconfig's tests/*.ts include glob simply +# matches nothing here. +RUN corepack enable \ + && pnpm install --frozen-lockfile --ignore-scripts \ + && pnpm exec tsc -p tsconfig.json \ + && BIN="$(node -e "process.stdout.write(require('workerd').default)")" \ + && cp "$BIN" /workerd \ + && chmod +x /workerd + +# Stage 2: minimal runtime — debian + ca-certificates + the workerd binary + the +# compiled JS. No Node, no TypeScript, no npm packages in this image. +FROM debian:bookworm-slim + +# curl: loopback HTTP client + Actor-input fetch; jq: extract `code` from the input JSON. +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates curl jq \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /workerd /usr/local/bin/workerd + +WORKDIR /app +COPY worker/entrypoint.sh worker/config.capnp ./worker/ +COPY --from=builder /build/worker/runner.js /build/worker/guard.js ./worker/ + +ENTRYPOINT ["sh", "/app/worker/entrypoint.sh"] diff --git a/README.md b/README.md index e1d04ba..c8e2886 100644 --- a/README.md +++ b/README.md @@ -1 +1,213 @@ # Code Runtime (experimental) + +> ⚠️ **Experimental infrastructure Actor.** It powers **Code Mode** on +> [mcp.apify.com](https://mcp.apify.com) and is normally invoked by the Apify +> MCP Server, not run by hand. Its behaviour and API may change without notice. + +## What it does + +Executes one JS script that an AI agent submits through the Apify MCP +Server, then returns whatever the script printed. + +Lets an agent do many Apify operations in **one call** — search the Store, +run an Actor, read its dataset, filter and aggregate — instead of sending +every intermediate result back through the model. This Actor is the sandbox +that runs that script. + +**Worth it only for bulk work** (measured, A/B eval vs. calling Actor tools +directly): + +| Workload | Verdict | +|---|---| +| Filter/sort/aggregate 50+ dataset records | Modest win — ~20-35% less time, ~20% fewer tokens | +| Fan out over 10+ sub-resources with a sizeable payload each (visit many pages, chain Actors) | Decisive win — ~60% less time, ~75% fewer tokens | +| Under 10 items, no fan-out | **Don't use this Actor** — ~20K-token sandbox overhead isn't paid back | + +## Calling this Actor + +Self-contained — no special MCP-server opt-in required. Any MCP client +already has `search-actors`, `fetch-actor-details`, and `call-actor` as +default tools: + +``` +call-actor({ actor: "apify/code-runtime", input: { code: "..." } }) +``` + +Default `timeoutSecs: 900`, `memoryMbytes: 1024` (`.actor/actor.json`) — +override per call for scripts chaining several long Actor runs (MCP +`call-actor`'s `callOptions.timeout`/`callOptions.memory`, or the API's +`timeout`/`memory`). + +## How it works + +- **One script per run.** Reads `code`, runs it once, writes the result, exits. +- Runs inside a sandboxed [`workerd`](https://github.com/cloudflare/workerd) + V8 isolate — see [Permissions & safety](#permissions--safety) for what's allowed. +- A global **`apify`** object exposes a small, typed subset of the Apify API + — run Actors, read/write datasets and key-value stores — using the + current run's token. +- `console.log`/`console.info` → **stdout**; `console.error`/`console.warn` + → **stderr**, captured separately. +- Call `apify.actor.get({ actorId })` before running an Actor you haven't + checked — don't guess its input schema. +- Log a nested run's `run.id`/`defaultDatasetId`/`defaultKeyValueStoreId` + before processing its output — nothing persists between this Actor's own + runs, but the Actors it started keep theirs. +- Already have a dataset/store ID from an earlier turn? Reuse it — don't + re-run an identical call, it wastes cost. +- Print a small JSON summary, never a full dataset — only `console.log`/ + `console.info` output comes back; a top-level `return` is **not** captured. + +## Input + +```json +{ + "code": "const { items } = await apify.actor.callAndGetItems({ actorId: 'apify/rag-web-browser', input: { query: 'apify' }, limit: 3 });\nconsole.log(items.map((i) => i.metadata?.title).join('\\n'));" +} +``` + +| Field | Type | Description | +|---|---|---| +| `code` | string | The JavaScript script to run (JS only, not transpiled). It receives the `apify` binding and `console`. | + +## Output + +A single **dataset item**: + +```json +{ "stdout": "Apify: Full-stack web scraping ...\n...", "stderr": "", "exitCode": 0, "statusMessage": "Script completed" } +``` + +| Outcome | `exitCode` | `statusMessage` | +|---|---|---| +| Script returned | `0` | `Script completed` | +| Script threw | `1` | `Script threw: ...` | +| Failed to compile (syntax error) | `1` | `Failed to compile: ...` | +| Run-level timeout / OOM kill | — | item may not exist for this run at all | + +Check `exitCode`/`statusMessage`, not `stderr` content, to detect a failed +script — `stderr` also carries `console.error`/`console.warn` output, so its +presence alone isn't failure. A timeout/OOM kill is signaled by the Actor +run's own status (`SUCCEEDED` vs `FAILED`/`ABORTED`/`TIMED-OUT`), not by this +item's absence. + +## Permissions & safety + +- Sandbox has **no filesystem**; outbound `fetch` (redirects re-validated + per hop) is limited to the Apify API (`*.apify.com`). +- **No imports** — runs without workerd's `nodejs_compat`, so no Node + built-ins (`node:net`, `node:fs`, …) or npm packages. This also removes + `node:net` (a raw-socket path that would bypass the `fetch` allowlist) and + keeps the run token out of `process.env` (undefined here). +- Each run is an isolated, single-use container — nothing persists between runs. +- This closes **direct fetch-based exfil** — it does not close every path to + move data out (e.g. `actor.start({ input })` on an Actor with its own + internet access, or writing to a dataset/key-value store). + +## Recipes + +### Chain Actors (one run's output feeds the next) + +```js +const { items: results } = await apify.actor.callAndGetItems({ + actorId: 'apify/google-search-scraper', input: { queries: 'apify' }, limit: 10, +}); +const startUrls = results.flatMap((r) => r.organicResults ?? []).map((r) => ({ url: r.url })); +const { items: pages } = await apify.actor.callAndGetItems({ + actorId: 'apify/website-content-crawler', input: { startUrls }, +}); +console.log(JSON.stringify(pages.slice(0, 3).map((p) => p.url))); +``` + +### Bounded parallel fan-out + +This Actor's clearest win: run several Actors (or the same Actor over +several inputs) concurrently, then reduce before returning. Chunk it (5–10 +at a time) — an unbounded `Promise.all` can hit your account's +concurrent-run or memory limits. + +```js +const inputs = [{ query: 'a' }, { query: 'b' }, { query: 'c' } /* ... */]; +const CHUNK = 5; +const results = []; +for (let i = 0; i < inputs.length; i += CHUNK) { + const batch = inputs.slice(i, i + CHUNK); + const batchResults = await Promise.all( + batch.map((input) => apify.actor.callAndGetItems({ actorId: 'apify/rag-web-browser', input, limit: 5 })), + ); + results.push(...batchResults.flatMap((r) => r.items)); +} +console.log(JSON.stringify(results.slice(0, 5))); // small summary, not the full dump +``` + +### Read an entire dataset without managing offsets + +`dataset.listItems`/`store` work two ways — `await` for one page, `for +await` to auto-paginate every item (see [the apify binding](#the-apify-binding)): + +```js +// One page — e.g. a quick peek +const { items, count } = await apify.dataset.listItems({ datasetId, limit: 10 }); + +// Every item, however many pages that takes +let matches = 0; +for await (const item of apify.dataset.listItems({ datasetId })) { + if (item.rating >= 4.5) matches++; +} +console.log(`${matches} matching items`); +``` + +### Runs longer than 60s: start, then poll + +`actor.call`'s wait is capped at 60s per request (a REST API limit, not this +Actor's). For a longer-running Actor, start it and poll: + +```js +let run = await apify.actor.start({ actorId, input }); +const TERMINAL = ['SUCCEEDED', 'FAILED', 'ABORTED', 'TIMED-OUT']; +while (!TERMINAL.includes(run.status)) { + run = await apify.run.waitForFinish({ runId: run.id, waitForFinishSecs: 60 }); +} +``` + +## The `apify` binding + +Every method takes one options object and returns parsed JSON — except +`store` and `dataset.listItems`, which return a value that's both a +`Promise` (one page) and an `AsyncIterable` (every match/item, +auto-paginated). Full API docs: +[API.md](https://github.com/apify/actor-code-runtime/blob/master/docs/API.md). +(`?` = optional, `= x` = default) + +```js +// Store — GET /v2/store, a top-level Apify API resource (not an Actor method) +apify.store({ search, limit?, offset?, category? }) // → { items, count, offset, limit }; dual Promise/AsyncIterable, see above + +// Actors +apify.actor.get({ actorId }) // → actor +apify.actor.start({ actorId, input?, memoryMbytes?, timeoutSecs?, maxTotalChargeUsd?, maxItems? }) // → run +apify.actor.call({ actorId, ...startOpts, waitForFinishSecs = 60 }) // → run (may be non-terminal READY/RUNNING past the 60s cap — not an error, see Recipes) +apify.actor.callAndGetItems({ actorId, input?, fields?, limit?, ...runOpts }) // → { run, items } + +// Runs +apify.run.get({ runId }) // → run +apify.run.waitForFinish({ runId, waitForFinishSecs = 60 }) // → run (same non-terminal caveat) +apify.run.abort({ runId }) // → run +apify.run.getLog({ runId, limit? }) // → string + +// Datasets +apify.dataset.create({ name? }) // → dataset +apify.dataset.pushItems({ datasetId, items }) // → void +apify.dataset.listItems({ datasetId, fields?, omit?, limit?, offset?, clean?, desc? }) // → { items, count, offset, limit, desc }; dual Promise/AsyncIterable, see above +apify.dataset.inferFields({ datasetId, sample = 5 }) // → { itemCount, fields[] } + +// Key-value stores +apify.keyValueStore.create({ name? }) // → store +apify.keyValueStore.set({ storeId, key, value, contentType? }) // → void +apify.keyValueStore.get({ storeId, key }) // → value | null +apify.keyValueStore.list({ storeId, limit?, exclusiveStartKey? }) // → { items } +``` + +## Learn more + +- Apify MCP Server: diff --git a/docs/API.md b/docs/API.md new file mode 100644 index 0000000..691ea88 --- /dev/null +++ b/docs/API.md @@ -0,0 +1,431 @@ +# The `apify` binding — API reference + +Inside a Code Mode script a global **`apify`** object exposes a small, typed +subset of the Apify API, authenticated with the current run's token. This +document describes every method in detail. + +## Conventions + +- **Every method is `async`** — except `dataset.listItems`/`store` (see + below), which are plain functions that return an already-awaitable value; + `await`ing one behaves identically to awaiting a true `async` call. +- **One options object.** Each method takes a single object argument; there are + no positional parameters. +- **Paginated methods are dual-mode.** [`dataset.listItems`](#datasetlistitems--object-count-offset-limit-desc-) + and [`store`](#apifystore--object-count-offset-limit-) return a value that's + both a `Promise` and an `AsyncIterable`: `await` it for one page (`{ items, + count, offset, limit, ... }`); `for await (const item of ...)` it to + auto-paginate through everything, one item at a time. One call, one name, + two ways to consume it — matching the official + [`apify-client`](https://github.com/apify/apify-client-js) SDK's + `PaginatedIterator` convention. There's no separate "iterate" method. +- **`actorId`** accepts either `username/name` (e.g. `apify/rag-web-browser`) or + the Actor's ID. +- **Return values.** The Apify API wraps most responses in a `{ "data": … }` + envelope. These methods **unwrap it for you** and return the inner value; the + linked API page describes that inner `data` shape. Where a method transforms + the response further (extracts an array, parses by content type, infers a + schema, returns plain text), the exact output is spelled out below. +- **`Apify API:`** each method links to its underlying endpoint on + [docs.apify.com/api/v2](https://docs.apify.com/api/v2) so you can inspect the + live request/response schema. +- **Errors.** A non-2xx API response throws an `Error` whose message is + ` failed: `. The one exception is + [`keyValueStore.get`](#keyvaluestoreget--value--null), which returns `null` for a missing key. +- **Network.** Outbound `fetch` from your script is restricted to `apify.com` + and its subdomains. +- **Reuse, don't re-run.** If a prior attempt already logged a nested run's + `defaultDatasetId`/`defaultKeyValueStoreId` (visible in your own earlier + turns), reuse it — do not re-run the same Actor call with identical input. + Re-running wastes the compute/cost of a call that already succeeded. + +--- + +## `apify.store` + +Apify's own API tags this endpoint `Store` — a top-level resource, not an +Actor method — so the binding mirrors that: `apify.store(...)`, not +`apify.actor.store(...)`. + +### `apify.store({ search, limit?, offset?, category? })` → `{ items, count, offset, limit }` + +Search the Apify Store. Dual-mode — see [Conventions](#conventions). + +| Param | Type | Required | Description | +|---|---|---|---| +| `search` | `string` | yes | Full-text search query. | +| `limit` | `number` | no | Page size (`await` mode) / items per page (`for await` mode). | +| `offset` | `number` | no | Starting offset. | +| `category` | `string` | no | Restrict to a Store category. | + +**Output (custom):** one page — `{ items: Actor[], count, offset, limit }` +(`count` is this page's actual item count; `offset`/`limit` echo the request). +**Apify API:** [`GET /v2/store`](https://docs.apify.com/api/v2/store-get) + +```js +// One page +const { items } = await apify.store({ search: 'web scraper', limit: 5 }); +console.log(items.map((a) => `${a.username}/${a.name}`).join('\n')); + +// Every match +const names = []; +for await (const actor of apify.store({ search: 'web scraper', limit: 20 })) { + names.push(`${actor.username}/${actor.name}`); +} +``` + +--- + +## `apify.actor` + +### `actor.get({ actorId })` → `Actor` + +Fetch the full record for one Actor. + +| Param | Type | Required | Description | +|---|---|---|---| +| `actorId` | `string` | yes | `username/name` or Actor ID. | + +**Output:** the Actor object (unwrapped `data`). +**Apify API:** [`GET /v2/acts/{actorId}`](https://docs.apify.com/api/v2/act-get) + +### `actor.start({ actorId, input?, memoryMbytes?, timeoutSecs?, maxTotalChargeUsd?, maxItems? })` → `Run` + +Start an Actor **asynchronously** and return immediately with a run record in +`READY`/`RUNNING` state. Use [`run.waitForFinish`](#runwaitforfinish--run) to block for the result. + +| Param | Type | Required | Description | +|---|---|---|---| +| `actorId` | `string` | yes | `username/name` or Actor ID. | +| `input` | `object` | no | Actor input (defaults to `{}`). | +| `memoryMbytes` | `number` | no | Memory limit for the run (`memory` query param). | +| `timeoutSecs` | `number` | no | Run timeout in seconds (`timeout`). | +| `maxTotalChargeUsd` | `number` | no | Hard cap on the run's cost. | +| `maxItems` | `number` | no | Max dataset items for pay-per-result Actors. | + +**Output:** the Run object (unwrapped `data`). +**Apify API:** [`POST /v2/acts/{actorId}/runs`](https://docs.apify.com/api/v2/act-runs-post) + +### `actor.call({ actorId, input?, waitForFinishSecs?, memoryMbytes?, timeoutSecs?, maxTotalChargeUsd?, maxItems? })` → `Run` + +Start an Actor and **wait** for it to finish (or until `waitForFinishSecs` +elapses), then return the run record. + +| Param | Type | Required | Default | Description | +|---|---|---|---|---| +| `actorId` | `string` | yes | | `username/name` or Actor ID. | +| `input` | `object` | no | `{}` | Actor input. | +| `waitForFinishSecs` | `number` | no | `60` | Seconds to wait (`waitForFinish`). **The Apify API caps a single wait at 60s** — for longer runs use `start()` + a `run.waitForFinish()` loop. | +| `memoryMbytes` | `number` | no | | Memory limit. | +| `timeoutSecs` | `number` | no | | Run timeout. | +| `maxTotalChargeUsd` | `number` | no | | Cost cap. | +| `maxItems` | `number` | no | | Max items (pay-per-result). | + +**Output:** the Run object (unwrapped `data`), exposing `defaultDatasetId` and +`defaultKeyValueStoreId` for reading results. Uses the standard run endpoint +(not `/run-sync`, which returns the output record instead of the run object). +**May return non-terminal** (`status: 'READY'`/`'RUNNING'`) if `waitForFinishSecs` +(capped at 60s by the API) elapses before the run finishes — this is **not** +an error; poll [`run.waitForFinish`](#runwaitforfinish--run) until `status` is +terminal (`SUCCEEDED`/`FAILED`/`ABORTED`/`TIMED-OUT`). +**Apify API:** [`POST /v2/acts/{actorId}/runs`](https://docs.apify.com/api/v2/act-runs-post) + +### `actor.callAndGetItems({ actorId, input?, fields?, limit?, ...runOpts })` → `{ run, items }` + +Convenience wrapper: `actor.call(...)` followed by reading the run's default +dataset via `dataset.listItems`. + +| Param | Type | Required | Description | +|---|---|---|---| +| `actorId` | `string` | yes | `username/name` or Actor ID. | +| `input` | `object` | no | Actor input. | +| `fields` | `string[]` | no | Restrict returned item fields. | +| `limit` | `number` | no | Max items to fetch. | +| `...runOpts` | | no | Any `actor.call` option (`waitForFinishSecs`, `memoryMbytes`, `timeoutSecs`, `maxTotalChargeUsd`, `maxItems`). | + +**Output (custom):** + +```js +{ + run: Run, // the run object, as actor.call returns + items: object[] // items from run.defaultDatasetId +} +``` + +**Apify API:** [`POST /v2/acts/{actorId}/runs`](https://docs.apify.com/api/v2/act-runs-post) +then [`GET /v2/datasets/{datasetId}/items`](https://docs.apify.com/api/v2/dataset-items-get) + +```js +const { run, items } = await apify.actor.callAndGetItems({ + actorId: 'apify/rag-web-browser', + input: { query: 'apify' }, + limit: 3, +}); +console.log(run.status, items.length); +``` + +--- + +## `apify.run` + +### `run.get({ runId })` → `Run` + +Fetch the current run record (status, stats, default storage IDs). + +| Param | Type | Required | Description | +|---|---|---|---| +| `runId` | `string` | yes | The run ID. | + +**Output:** the Run object (unwrapped `data`). +**Apify API:** [`GET /v2/actor-runs/{runId}`](https://docs.apify.com/api/v2/actor-run-get) + +### `run.waitForFinish({ runId, waitForFinishSecs? })` → `Run` + +Block until the run terminates or `waitForFinishSecs` elapses, whichever comes +first, then return the run record. + +| Param | Type | Required | Default | Description | +|---|---|---|---|---| +| `runId` | `string` | yes | | The run ID. | +| `waitForFinishSecs` | `number` | no | `60` | Seconds to wait (`waitForFinish`). **Capped at 60s by the API**; poll in a loop for longer runs. | + +**Output:** the Run object (unwrapped `data`). **May return non-terminal** +(`status: 'READY'`/`'RUNNING'`) if the cap elapses first — this is **not** an +error, it means keep polling; check `status` against the terminal set +(`SUCCEEDED`/`FAILED`/`ABORTED`/`TIMED-OUT`) before treating any other value +as a failure. +**Apify API:** [`GET /v2/actor-runs/{runId}`](https://docs.apify.com/api/v2/actor-run-get) + +### `run.abort({ runId })` → `Run` + +Abort a running run. Returns the run record (status transitions to `ABORTING`). +Aborting a finished run is an API error. + +| Param | Type | Required | Description | +|---|---|---|---| +| `runId` | `string` | yes | The run ID. | + +**Output:** the Run object (unwrapped `data`). +**Apify API:** [`POST /v2/actor-runs/{runId}/abort`](https://docs.apify.com/api/v2/act-run-abort-post) + +### `run.getLog({ runId, limit? })` → `string` + +Return the run's log as text. + +| Param | Type | Required | Description | +|---|---|---|---| +| `runId` | `string` | yes | The run ID. | +| `limit` | `number` | no | If set, return only the **last** `limit` characters (client-side tail; the full log is fetched). | + +**Output (custom):** the raw log **text** (not JSON). With `limit`, the last +`limit` characters. +**Apify API:** [`GET /v2/logs/{runId}`](https://docs.apify.com/api/v2/log-get) + +--- + +## `apify.dataset` + +### `dataset.create({ name? })` → `Dataset` + +Create a dataset and return its record (use `.id` for subsequent calls). + +| Param | Type | Required | Description | +|---|---|---|---| +| `name` | `string` | no | Named (persistent) dataset; omit for an unnamed (temporary) one. | + +**Output:** the Dataset object (unwrapped `data`). +**Apify API:** [`POST /v2/datasets`](https://docs.apify.com/api/v2/datasets-post) + +### `dataset.pushItems({ datasetId, items })` → `void` + +Append one or more items to a dataset. + +| Param | Type | Required | Description | +|---|---|---|---| +| `datasetId` | `string` | yes | Target dataset ID. | +| `items` | `object \| object[]` | yes | A single item or an array of items. | + +**Output:** none (resolves once the items are stored). +**Apify API:** [`POST /v2/datasets/{datasetId}/items`](https://docs.apify.com/api/v2/dataset-items-post) + +```js +const ds = await apify.dataset.create(); +await apify.dataset.pushItems({ datasetId: ds.id, items: [{ a: 1 }, { a: 2 }] }); +const { items } = await apify.dataset.listItems({ datasetId: ds.id }); +``` + +### `dataset.listItems({ datasetId, fields?, omit?, limit?, offset?, clean?, desc? })` → `{ items, count, offset, limit, desc }` + +Read the dataset. Dual-mode — see [Conventions](#conventions): `await` for one +page, `for await` to auto-paginate through everything (replaces what used to +be a separate `iterate()` method — one name, one method, both jobs). + +| Param | Type | Required | Description | +|---|---|---|---| +| `datasetId` | `string` | yes | Dataset ID. | +| `fields` | `string[]` | no | Only include these fields (joined into `fields`). | +| `omit` | `string[]` | no | Exclude these fields. | +| `limit` | `number` | no | Page size (`await` mode) / items fetched per page (`for await` mode). Omit for the API's own default (effectively unbounded — a single `await` then returns everything in one page). | +| `offset` | `number` | no | Starting offset. Default `0`. | +| `clean` | `boolean` | no | Skip empty items / hidden fields (`clean=1`). | +| `desc` | `boolean` | no | Reverse (newest first, `desc=1`). | + +**Output (custom):** +- `await apify.dataset.listItems({...})` → one page: `{ items: object[], count, + offset, limit, desc }`. `offset`/`limit`/`desc` echo what the API actually + applied (read from its `x-apify-pagination-*` response headers, not just + the request); `count` is this page's actual item count. **No `total`** — + the API's `x-apify-pagination-total` header is unreliable for + freshly-created datasets (eventually consistent); use + [`inferFields`](#datasetinferfields--schema) for an approximate count. +- `for await (const item of apify.dataset.listItems({...}))` → every item in + the dataset, one at a time, paging internally. Stops when a page comes back + shorter than requested (the natural end-of-data signal). + +**Apify API:** [`GET /v2/datasets/{datasetId}/items`](https://docs.apify.com/api/v2/dataset-items-get) (paged internally for `for await`) + +```js +// One page +const { items, count } = await apify.dataset.listItems({ datasetId, limit: 100 }); +console.log(`${count} items on this page`); + +// Every item +let total = 0; +for await (const item of apify.dataset.listItems({ datasetId })) total++; +console.log('total items:', total); +``` + +### `dataset.inferFields({ datasetId, sample? })` → `Schema` + +Infer a lightweight schema from a sample of items (Apify has no schema endpoint). + +| Param | Type | Required | Default | Description | +|---|---|---|---|---| +| `datasetId` | `string` | yes | | Dataset ID. | +| `sample` | `number` | no | `5` | Number of items to inspect. | + +**Output (custom):** + +```js +{ + itemCount, // number | undefined — from the dataset metadata (eventually consistent) + sampleSize, // number of items actually inspected + fields: [ // one entry per field seen across the sample + { + name, // field name + types, // string[], e.g. ['string'] or ['number','null'] + nullable // boolean — true if any sampled value was null + } + ] +} +``` + +**Apify API:** [`GET /v2/datasets/{datasetId}`](https://docs.apify.com/api/v2/dataset-get) +(for `itemCount`) + [`GET /v2/datasets/{datasetId}/items`](https://docs.apify.com/api/v2/dataset-items-get) (the sample) + +--- + +## `apify.keyValueStore` + +### `keyValueStore.create({ name? })` → `KeyValueStore` + +Create a key-value store and return its record. + +| Param | Type | Required | Description | +|---|---|---|---| +| `name` | `string` | no | Named (persistent) store; omit for an unnamed (temporary) one. | + +**Output:** the key-value store's record object (unwrapped `data`). +**Apify API:** [`POST /v2/key-value-stores`](https://docs.apify.com/api/v2/key-value-stores-post) + +### `keyValueStore.set({ storeId, key, value, contentType? })` → `void` + +Write a record. The content type is inferred from `value`: + +| `value` type | Stored as | +|---|---| +| `object` | `application/json; charset=utf-8` | +| `string` | `text/plain; charset=utf-8` | +| `Uint8Array` / `ArrayBuffer` | `application/octet-stream` | + +| Param | Type | Required | Description | +|---|---|---|---| +| `storeId` | `string` | yes | Store ID. | +| `key` | `string` | yes | Record key. | +| `value` | `object \| string \| Uint8Array \| ArrayBuffer` | yes | Value to store. | +| `contentType` | `string` | no | Override the inferred content type. | + +**Output:** none (resolves once the record is stored). +**Apify API:** [`PUT /v2/key-value-stores/{storeId}/records/{key}`](https://docs.apify.com/api/v2/key-value-store-record-put) + +### `keyValueStore.get({ storeId, key })` → `value` \| `null` + +Read a record. + +| Param | Type | Required | Description | +|---|---|---|---| +| `storeId` | `string` | yes | Store ID. | +| `key` | `string` | yes | Record key. | + +**Output (custom):** the value, typed by the stored content type — + +- `application/json` → parsed **object** +- `text/*` → **string** +- anything else → **`Uint8Array`** + +Returns **`null`** when the key does not exist (404) instead of throwing, so you +can do lookup-or-default without a `try/catch`. +**Apify API:** [`GET /v2/key-value-stores/{storeId}/records/{key}`](https://docs.apify.com/api/v2/key-value-store-record-get) + +```js +const kv = await apify.keyValueStore.create(); +await apify.keyValueStore.set({ storeId: kv.id, key: 'state', value: { seen: [] } }); +const state = await apify.keyValueStore.get({ storeId: kv.id, key: 'state' }); // → { seen: [] } +const missing = await apify.keyValueStore.get({ storeId: kv.id, key: 'nope' }); // → null +``` + +### `keyValueStore.list({ storeId, limit?, exclusiveStartKey? })` → `{ items, … }` + +List keys in a store. + +| Param | Type | Required | Description | +|---|---|---|---| +| `storeId` | `string` | yes | Store ID. | +| `limit` | `number` | no | Max keys to return. | +| `exclusiveStartKey` | `string` | no | Continue listing after this key (pagination). | + +**Output:** the unwrapped `data`: `{ items: [{ key, size }], count, limit, +isTruncated, exclusiveStartKey, nextExclusiveStartKey }`. +**Apify API:** [`GET /v2/key-value-stores/{storeId}/keys`](https://docs.apify.com/api/v2/key-value-store-keys-get) + +--- + +## `console` + +`console` is captured, not printed live: + +| Method | Stream | +|---|---| +| `console.log`, `console.info` | **stdout** | +| `console.error`, `console.warn` | **stderr** | + +Non-string arguments are `JSON.stringify`'d. When the script finishes, both +streams and the script's exit status are written to the run's default dataset as +a single item: + +```json +{ "stdout": "...", "stderr": "...", "exitCode": 0 } +``` + +`exitCode` is the script's effective exit status: `0` when it returns normally, +`1` when it throws. It is distinct from the Actor run's status — see below. + +## Error handling + +- A non-2xx API response throws `Error: failed: `. +- If your script throws, the error (stack/message) is appended to **stderr**, + `exitCode` is set to `1`, and the run still **succeeds** with whatever was + printed beforehand — so failures are observable in the output rather than + crashing the run. Check `exitCode` (not `stderr`) to detect a failed script: + `stderr` may be non-empty from ordinary `console.error` / `console.warn` + logging even on a successful run. diff --git a/package.json b/package.json new file mode 100644 index 0000000..4bd7ecd --- /dev/null +++ b/package.json @@ -0,0 +1,20 @@ +{ + "name": "code-runtime", + "version": "0.1.0", + "description": "workerd as a normal (per-run) Apify Actor: one static worker, booted once per run, runs the submitted script and exits.", + "private": true, + "packageManager": "pnpm@11.1.3", + "dependencies": { + "workerd": "1.20260402.1" + }, + "engines": { + "node": ">=24" + }, + "scripts": { + "build": "tsc -p tsconfig.json && sed -i '/^export {};$/d' tests/*.js", + "typecheck": "tsc --noEmit -p tsconfig.json" + }, + "devDependencies": { + "typescript": "6.0.3" + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml new file mode 100644 index 0000000..8d7fee7 --- /dev/null +++ b/pnpm-lock.yaml @@ -0,0 +1,86 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + workerd: + specifier: 1.20260402.1 + version: 1.20260402.1 + devDependencies: + typescript: + specifier: 6.0.3 + version: 6.0.3 + +packages: + + '@cloudflare/workerd-darwin-64@1.20260402.1': + resolution: {integrity: sha512-/kyyZ5HjOPT202Vsw3P+vICsVulldC9ym+W5UyKl9dufxfHgcdeT4EYtT+tkk+3SlkB1RmoVxgr3VRlqP2ojjw==} + engines: {node: '>=16'} + cpu: [x64] + os: [darwin] + + '@cloudflare/workerd-darwin-arm64@1.20260402.1': + resolution: {integrity: sha512-nr+AgUonmepiuD4utn4KQl2fIn/aDuWEO+/B2fzjMnjLSgLpN1IQADUW1uD4FVJdd0ss9ugK1dztFJJWpImCaQ==} + engines: {node: '>=16'} + cpu: [arm64] + os: [darwin] + + '@cloudflare/workerd-linux-64@1.20260402.1': + resolution: {integrity: sha512-0vJj0pO6ARpCmvyLgbQk204dogL72geEyC+rOnkFgwcDJI4e8oxKrTQZWjPHra3BHUknEomUd37i3K4L4JfU4w==} + engines: {node: '>=16'} + cpu: [x64] + os: [linux] + + '@cloudflare/workerd-linux-arm64@1.20260402.1': + resolution: {integrity: sha512-aRsxuv1bmwkkX4sG8igutkwVHgsJ1I6mH4/u/1Jfhpb7Bj3xr9p9N2fjDwax+Dhy7xt2tC8TS/ZWLeFFjK2NMA==} + engines: {node: '>=16'} + cpu: [arm64] + os: [linux] + + '@cloudflare/workerd-windows-64@1.20260402.1': + resolution: {integrity: sha512-aFKYAuIYTPsuWyHxv39yOFi9bmIvAsIuNmHGYkYltHZUoN4by6tJVCOQUp6PDlZO7TjHSD2XjjM8JBwK/D6tVA==} + engines: {node: '>=16'} + cpu: [x64] + os: [win32] + + typescript@6.0.3: + resolution: {integrity: sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==} + engines: {node: '>=14.17'} + hasBin: true + + workerd@1.20260402.1: + resolution: {integrity: sha512-Cg+OUlukdcCHrTTg0MBCIMFRE6XO3yGVGiWCnJPvfffy2Ga2girrEq3qF/YlHSTmbIyEE5ebCFxBYYYZueQ/Mg==} + engines: {node: '>=16'} + hasBin: true + +snapshots: + + '@cloudflare/workerd-darwin-64@1.20260402.1': + optional: true + + '@cloudflare/workerd-darwin-arm64@1.20260402.1': + optional: true + + '@cloudflare/workerd-linux-64@1.20260402.1': + optional: true + + '@cloudflare/workerd-linux-arm64@1.20260402.1': + optional: true + + '@cloudflare/workerd-windows-64@1.20260402.1': + optional: true + + typescript@6.0.3: {} + + workerd@1.20260402.1: + optionalDependencies: + '@cloudflare/workerd-darwin-64': 1.20260402.1 + '@cloudflare/workerd-darwin-arm64': 1.20260402.1 + '@cloudflare/workerd-linux-64': 1.20260402.1 + '@cloudflare/workerd-linux-arm64': 1.20260402.1 + '@cloudflare/workerd-windows-64': 1.20260402.1 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml new file mode 100644 index 0000000..797a5c3 --- /dev/null +++ b/pnpm-workspace.yaml @@ -0,0 +1,2 @@ +allowBuilds: + workerd: true diff --git a/test.sh b/test.sh new file mode 100755 index 0000000..650b042 --- /dev/null +++ b/test.sh @@ -0,0 +1,60 @@ +#!/bin/sh +# Deploy the Actor (apify push) once, then run each test probe on the freshly +# built version via `apify call`. Each probe is a script submitted as the `code` +# input; it prints ALL_TESTS_PASSED on success. Exits non-zero if any probe fails. +# +# Usage: ./test.sh +set -eu + +cd "$(dirname "$0")" + +# Probes are written in tests/*.ts and compiled by `pnpm build`; add a .ts file +# there to register a new probe. Run against the built Actor. +PROBES="tests/binding-smoke.js tests/sandbox-isolation.js" + +command -v apify >/dev/null 2>&1 || { echo "apify CLI not found" >&2; exit 1; } +command -v jq >/dev/null 2>&1 || { echo "jq not found" >&2; exit 1; } + +echo "==> pnpm build" +pnpm build + +input_json="$(mktemp)" +trap 'rm -f "$input_json"' EXIT + +echo "==> apify push" +apify push + +failed=0 +for probe in $PROBES; do + echo "==> apify call: ${probe}" + jq -n --arg code "$(cat "$probe")" '{ code: $code }' > "$input_json" + output="$(apify call -f "$input_json" -o 2>&1)" + echo "$output" + if printf '%s' "$output" | grep -q 'ALL_TESTS_PASSED'; then + echo "==> ${probe} passed" + else + echo "==> ${probe} FAILED" >&2 + failed=1 + fi +done + +[ "$failed" -eq 0 ] || { echo "==> some probes FAILED" >&2; exit 1; } +echo "==> all probes passed" + +# Regression probe for the realFetch claim-ordering bug (PR #1 review, +# 2026-07-21): tests/fixtures/realfetch-escape.js escapes usercode.js's +# wrapper into module scope and tries to steal the internal-only realFetch +# before runner.ts's own claim. It has no captured console to report through +# (see the file's own comment), so success/failure is the Actor run itself +# succeeding vs. failing -- not a printed sentinel like the probes above. +echo "==> apify call: tests/fixtures/realfetch-escape.js (regression: realFetch claim ordering)" +jq -n --arg code "$(cat tests/fixtures/realfetch-escape.js)" '{ code: $code }' > "$input_json" +if apify call -f "$input_json" -o; then + echo "==> realfetch-escape passed (run succeeded — module-scope steal attempt did not hijack/crash the internal claim)" +else + echo "==> realfetch-escape FAILED (run crashed — realFetch claim-ordering regression, see guard.ts's requestHandlingStarted gate)" >&2 + failed=1 +fi + +[ "$failed" -eq 0 ] || { echo "==> some probes FAILED" >&2; exit 1; } +echo "==> all probes (including regressions) passed" diff --git a/tests/binding-smoke.ts b/tests/binding-smoke.ts new file mode 100644 index 0000000..c43ab83 --- /dev/null +++ b/tests/binding-smoke.ts @@ -0,0 +1,124 @@ +// Smoke test for the `apify` binding exposed to Code Mode programs. Submitted as +// the Actor's `code` input by test.sh and executed on the built Actor via +// `apify call`. Exercises every binding method and prints a sentinel line +// (ALL_TESTS_PASSED) that test.sh greps for. +// +// `export {}` marks this file as its own ES module, so its top-level consts +// don't collide with the other probe's (both are type-checked in one tsc +// program) and top-level await below is legal. `pnpm build` strips this line +// post-compile (see package.json) -- left in, it would be a syntax error once +// spliced into the wrapping `async function run(apify, console) { ... }`. +export {}; + +const results: boolean[] = []; +async function check(name: string, fn: () => Promise): Promise { + try { + const out = await fn(); + console.log(`PASS ${name}: ${out ?? ''}`); + results.push(true); + } catch (e) { + console.error(`FAIL ${name}: ${(e as Error).message}`); + results.push(false); + } +} + +const ACTOR = 'apify/hello-world'; + +// ---- actor (read) ---- +await check('store', async () => { + const page = await apify.store({ search: 'hello world', limit: 3 }); + if (!Array.isArray(page.items)) throw new Error('expected items array'); + return `${page.count} actors`; +}); +await check('store (for await)', async () => { + let n = 0; + for await (const _ of apify.store({ search: 'hello world', limit: 3 })) n++; + return `${n} actors iterated`; +}); +await check('actor.get', async () => { + const d = await apify.actor.get({ actorId: ACTOR }); + return `${d.username}/${d.name}`; +}); + +// ---- dataset ---- +let datasetId = ''; +await check('dataset.create', async () => { + datasetId = (await apify.dataset.create()).id as string; + return datasetId; +}); +await check('dataset.pushItems', async () => { + await apify.dataset.pushItems({ datasetId, items: [{ a: 1, b: 'x' }, { a: 2, b: 'y' }] }); + return '2 pushed'; +}); +await check('dataset.listItems', async () => { + const page = await apify.dataset.listItems({ datasetId }); + return `${page.count} items, offset=${page.offset}, limit=${page.limit}`; +}); +await check('dataset.inferFields', async () => { + const s = await apify.dataset.inferFields({ datasetId }); + return `itemCount=${s.itemCount} fields=${s.fields.map((f) => f.name).join(',')}`; +}); +await check('dataset.listItems (for await)', async () => { + let n = 0; + for await (const _ of apify.dataset.listItems({ datasetId, limit: 1 })) n++; + return `${n} iterated`; +}); + +// ---- key-value store ---- +let storeId = ''; +await check('keyValueStore.create', async () => { + storeId = (await apify.keyValueStore.create()).id as string; + return storeId; +}); +await check('keyValueStore.set', async () => { + await apify.keyValueStore.set({ storeId, key: 'obj', value: { hello: 'world' } }); + await apify.keyValueStore.set({ storeId, key: 'txt', value: 'plain' }); + return 'set obj + txt'; +}); +await check('keyValueStore.get', async () => { + const obj = await apify.keyValueStore.get({ storeId, key: 'obj' }) as { hello: string }; + const txt = await apify.keyValueStore.get({ storeId, key: 'txt' }); + const missing = await apify.keyValueStore.get({ storeId, key: 'nope' }); + return `obj.hello=${obj.hello} txt=${txt} missing=${missing}`; +}); +await check('keyValueStore.list', async () => { + const l = await apify.keyValueStore.list({ storeId }) as { items: unknown[] }; + return `${l.items.length} keys`; +}); + +// ---- run lifecycle ---- +let runId = ''; +await check('actor.start', async () => { + const run = await apify.actor.start({ actorId: ACTOR }); + runId = run.id as string; + return `runId=${runId} status=${run.status}`; +}); +await check('run.get', async () => { + return `status=${(await apify.run.get({ runId })).status}`; +}); +await check('run.waitForFinish', async () => { + return `status=${(await apify.run.waitForFinish({ runId, waitForFinishSecs: 60 })).status}`; +}); +await check('run.getLog', async () => { + return `${(await apify.run.getLog({ runId, limit: 200 })).length} chars`; +}); + +// ---- run + get items (sync) ---- +await check('actor.call', async () => { + return `status=${(await apify.actor.call({ actorId: ACTOR, waitForFinishSecs: 60 })).status}`; +}); +await check('actor.callAndGetItems', async () => { + const { run, items } = await apify.actor.callAndGetItems({ actorId: ACTOR, limit: 5, waitForFinishSecs: 60 }); + return `status=${run.status} items=${items.length}`; +}); + +// ---- abort ---- +await check('run.abort', async () => { + const run = await apify.actor.start({ actorId: ACTOR }); + return `status=${(await apify.run.abort({ runId: run.id as string })).status}`; +}); + +const passed = results.filter(Boolean).length; +console.log(`\n=== SUMMARY: ${passed}/${results.length} passed ===`); +if (passed === results.length) console.log('ALL_TESTS_PASSED'); +else console.error(`SOME_TESTS_FAILED (${results.length - passed} failed)`); diff --git a/tests/fixtures/realfetch-escape.js b/tests/fixtures/realfetch-escape.js new file mode 100644 index 0000000..5d591f5 --- /dev/null +++ b/tests/fixtures/realfetch-escape.js @@ -0,0 +1,43 @@ +// Regression probe for the realFetch claim-ordering bug found in PR #1's +// review (2026-07-21): +// https://github.com/apify/actor-code-runtime/pull/1#issuecomment-5037390847 +// +// entrypoint.sh splices `code` verbatim (no escaping) into +// `export async function run(apify, console) { }`. The bare `}` right +// after this comment block closes that function early -- deliberately, to +// reproduce the escape -- so `run` becomes a harmless no-op (nothing is left +// in its body once the comments end) and everything after runs as ordinary +// MODULE-SCOPE code in usercode.js: workerd evaluates that unconditionally, +// before this worker ever calls runner.ts's request handler. That used to be +// enough to `await import('./guard.js')` and call claimRealFetch() directly, +// stealing the unrestricted, un-allowlisted fetch before runner.ts's own +// claim (previously made at runner.ts's own module top level) ever ran -- +// which made THAT claim get null, throw, and crash the whole Actor run +// (self-DoS; the real exploit payoff for an attacker would be using the +// stolen fetch directly, not reported here). +// +// Not valid JS on its own (it opens with an unbalanced `}`) -- intentionally, +// same shape as the reported PoC. Not TypeScript, not compiled, not +// typechecked (lives under tests/fixtures/, outside tsconfig's `include` and +// outside the `tests/*.js` build-artifact glob): see test.sh, which pushes +// this file's raw content directly as the `code` input. +// +// Expected result with the fix (guard.ts's requestHandlingStarted gate): +// claimRealFetch() called from module scope returns null without consuming +// the resource, so nothing crashes -- this Actor run completes normally +// (exitCode 0, "Script completed"), same as any run of an empty script. +// Before the fix: this run FAILS outright (workerd crashes during module +// evaluation; entrypoint.sh can't tell that apart from a real infra failure +// and fails the whole Actor run). test.sh asserts on `apify call`'s own exit +// status, not a printed sentinel -- this probe's `run` body never executes +// any of its own code, so it has no captured console to report through. +// +// Must be genuine top-level await, not an async IIFE: a module containing +// top-level await defers the evaluation of modules that depend on it (here, +// runner.ts) until that await settles (see MDN/TC39 "Top-level await", +// Asynchronous Module Evaluation) -- an IIFE's internal await does not carry +// that guarantee, so it would not reliably win the race this probe exists to +// exercise. +} +globalThis.__stolenRealFetch = (await import('./guard.js')).claimRealFetch(); +;{ diff --git a/tests/globals.d.ts b/tests/globals.d.ts new file mode 100644 index 0000000..2114679 --- /dev/null +++ b/tests/globals.d.ts @@ -0,0 +1,19 @@ +// Ambient declarations for probes in this directory. Each probe is compiled +// standalone by tsc, then submitted as the Actor's `code` input — entrypoint.sh +// splices that text into `export async function run(apify, console) { ... }` +// (see worker/usercode.d.ts), so `apify` and `console` are real function +// parameters at runtime, not globals. `declare global` here only satisfies the +// compiler for these standalone probe files; nothing in this file is emitted to JS. +import type { ApifyBinding } from '../worker/runner.js'; + +declare global { + const apify: ApifyBinding; + + // Sandbox-isolation probe checks `typeof process`/`typeof require` — both are + // genuinely absent at runtime (no nodejs_compat). `typeof` never throws on an + // undeclared identifier, so declaring these here doesn't change that: a + // `declare` emits no JS, so the runtime binding stays exactly as absent as + // the probe expects. + const process: unknown; + const require: unknown; +} diff --git a/tests/sandbox-isolation.ts b/tests/sandbox-isolation.ts new file mode 100644 index 0000000..bf877df --- /dev/null +++ b/tests/sandbox-isolation.ts @@ -0,0 +1,106 @@ +// Sandbox isolation test. Submitted as the Actor's `code` input by test.sh and +// executed on the built Actor via `apify call`. Asserts the sandbox boundary +// holds and prints a sentinel line (ALL_TESTS_PASSED) that test.sh greps for. +// +// Guards github.com/apify/ai-team#216 (findings A, B, C): the isolate runs +// WITHOUT workerd's nodejs_compat, so user code cannot reach Node built-ins. +// That removes node:net (a raw-socket egress path that bypassed guard.js's +// *.apify.com fetch allowlist — finding A) and process.env (which held the +// run's APIFY_TOKEN — finding B), and makes the "no imports" docs accurate +// (finding C). fetch() and the apify binding must still work. +// +// `export {}` marks this file as its own ES module, so its top-level consts +// don't collide with the other probe's (both are type-checked in one tsc +// program) and top-level await below is legal. `pnpm build` strips this line +// post-compile (see package.json) -- left in, it would be a syntax error once +// spliced into the wrapping `async function run(apify, console) { ... }`. +export {}; + +const results: boolean[] = []; +function check(name: string, cond: boolean, detail = ''): void { + if (cond) { + console.log(`PASS ${name}: ${detail}`); + results.push(true); + } else { + console.error(`FAIL ${name}: ${detail}`); + results.push(false); + } +} + +// Node built-ins must NOT be importable (no nodejs_compat). +for (const mod of ['node:net', 'node:fs', 'node:dns', 'node:child_process']) { + let imported = false; + try { await import(mod); imported = true; } catch { /* expected */ } + check(`import ${mod} blocked`, !imported, imported ? 'IMPORTED (leak!)' : 'blocked'); +} + +// The run token lives in process.env under nodejs_compat; without it, process +// and require must be undefined so user code can't read the credential. +check('process undefined', typeof process === 'undefined', `typeof process = ${typeof process}`); +check('require undefined', typeof require === 'undefined', `typeof require = ${typeof require}`); + +// fetch must remain — the apify binding depends on it. +check('fetch available', typeof fetch === 'function', `typeof fetch = ${typeof fetch}`); + +// guard.js allowlist: apify.com and *.apify.com only. A guard rejection throws +// synchronously with a "Blocked fetch" message BEFORE any network I/O; anything +// else (a real network/HTTP error) means guard let the request through. So we +// classify by the error message, not by whether the request ultimately succeeds. +async function guardBlocks(url: string): Promise { + try { + await fetch(url); + return false; // request went out — guard allowed it + } catch (e) { + return /Blocked fetch/.test((e as Error).message); // guard rejection vs. network error + } +} + +// Allowed: the main domain and any subdomain must NOT be guard-blocked. +for (const url of ['https://apify.com/', 'https://api.apify.com/v2/browser-info']) { + check(`allow ${url}`, !(await guardBlocks(url)), 'not blocked by guard'); +} + +// Blocked: other public hosts, subdomain look-alikes, userinfo/host tricks, and +// the cloud metadata IP must all be guard-blocked. +const blockedTargets = [ + 'https://example.com/', // unrelated public host + 'https://evilapify.com/', // suffix without the dot — must not match .apify.com + 'https://apify.com.evil.com/', // real host is evil.com + 'https://apify.com@evil.com/', // userinfo trick — real host is evil.com + 'http://169.254.169.254/', // cloud link-local metadata +]; +for (const url of blockedTargets) { + check(`block ${url}`, await guardBlocks(url), 'blocked by guard'); +} + +// Non-fetch egress primitives must be neutralized: WebSocket and EventSource +// are web-standard globals that connect directly (not through the fetch guard), +// so a script could otherwise open a wss:// / SSE channel to any public host and +// exfiltrate around the *.apify.com allowlist (apify/ai-team#216 finding A). +function blocksConstruct(name: string, url: string): boolean { + const Ctor = (globalThis as Record)[name]; + if (typeof Ctor !== 'function') return true; // absent → not an egress path + try { + new (Ctor as new (u: string) => unknown)(url); + return false; // constructed → egress opened + } catch (e) { + return /Blocked/.test((e as Error).message); // our guard rejection vs. any other error + } +} +check('WebSocket blocked', blocksConstruct('WebSocket', 'wss://echo.websocket.org'), 'no wss egress'); +check('EventSource blocked', blocksConstruct('EventSource', 'https://example.com/sse'), 'no SSE egress'); + +// The apify binding must still work (fetch to *.apify.com). +let bindingWorks = false; +try { + const found = await apify.store({ search: 'hello world', limit: 1 }); + bindingWorks = Array.isArray(found.items); +} catch (e) { + console.error(`apify.store threw: ${(e as Error).message}`); +} +check('apify binding works', bindingWorks, bindingWorks ? 'store ok' : 'binding broken'); + +const passed = results.filter(Boolean).length; +console.log(`\n=== SUMMARY: ${passed}/${results.length} passed ===`); +if (passed === results.length) console.log('ALL_TESTS_PASSED'); +else console.error(`SOME_TESTS_FAILED (${results.length - passed} failed)`); diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..704b2ad --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,10 @@ +{ + "compilerOptions": { + "target": "es2022", + "module": "es2022", + "moduleResolution": "bundler", + "lib": ["es2022", "dom"], + "strict": true + }, + "include": ["worker/*.ts", "tests/*.ts"] +} diff --git a/worker/config.capnp b/worker/config.capnp new file mode 100644 index 0000000..c78ebbb --- /dev/null +++ b/worker/config.capnp @@ -0,0 +1,48 @@ +using Workerd = import "/workerd/workerd.capnp"; + +const config :Workerd.Config = ( + services = [ + (name = "main", worker = .codeRuntime), + # Outbound for fetch(). Apify's api.apify.com may resolve to a private + # address inside the platform network, so allow private/local too. + # tlsOptions enables HTTPS egress (trust workerd's built-in CA set); + # the hostname allowlist is enforced by guard.js, not here. + (name = "internet", network = (allow = ["public", "private", "local"], tlsOptions = (trustBrowserCas = true))), + ], + # __PORT__ is substituted by entrypoint.sh from its own $PORT before workerd + # starts — single source of truth, see entrypoint.sh. + sockets = [ + ( + name = "http", + address = "127.0.0.1:__PORT__", + http = (), + service = "main", + ), + ], +); + +# runner.js is the entrypoint module; usercode.js is generated at container +# start by entrypoint.sh (the Actor input wrapped as `export async function run`). +const codeRuntime :Workerd.Worker = ( + modules = [ + (name = "runner.js", esModule = embed "runner.js"), + (name = "guard.js", esModule = embed "guard.js"), + (name = "usercode.js", esModule = embed "usercode.js"), + ], + bindings = [ + (name = "APIFY_TOKEN", fromEnvironment = "APIFY_TOKEN"), + (name = "DEFAULT_DATASET_ID", fromEnvironment = "ACTOR_DEFAULT_DATASET_ID"), + (name = "DEFAULT_DATASET_ID_LEGACY", fromEnvironment = "APIFY_DEFAULT_DATASET_ID"), + (name = "API_BASE_URL", fromEnvironment = "APIFY_API_BASE_URL"), + # This run's own meta.origin (e.g. "MCP" when apify-mcp-server started it), + # forwarded to sub-runs this script starts — see PARENT_ORIGIN in runner.ts. + (name = "PARENT_ORIGIN", fromEnvironment = "APIFY_META_ORIGIN"), + ], + globalOutbound = "internet", + compatibilityDate = "2026-01-15", + # No nodejs_compat: user code runs with web-standard APIs only. This is a + # security boundary, not a convenience toggle — the flag would expose node:net + # (a raw-socket egress path that bypasses guard.js's *.apify.com fetch allowlist) + # and process.env (which holds the run's APIFY_TOKEN). runner.js and guard.js use + # only web-standard APIs (fetch/URL/Response/Uint8Array), so they need nothing here. +); diff --git a/worker/entrypoint.sh b/worker/entrypoint.sh new file mode 100755 index 0000000..bcdb6ce --- /dev/null +++ b/worker/entrypoint.sh @@ -0,0 +1,92 @@ +#!/bin/sh +# Normal-mode (non-standby) Apify Actor entrypoint. Single-tenant per run: +# 1. read the Actor input and wrap its `code` into the runnable usercode.js module +# 2. boot workerd on loopback (it embeds runner.js + usercode.js) +# 3. trigger /run once, then exit +# workerd hosts the sandboxed worker and is reached only over loopback. +set -eu + +PORT=8787 +READINESS_ATTEMPTS=100 # 100 * 0.1s = 10s budget for workerd to bind the socket + +API_BASE="${APIFY_API_BASE_URL:-https://api.apify.com}" +API_BASE="${API_BASE%/}" # APIFY_API_BASE_URL ships with a trailing slash +STORE_ID="${ACTOR_DEFAULT_KEY_VALUE_STORE_ID:-${APIFY_DEFAULT_KEY_VALUE_STORE_ID:-}}" +DATASET_ID="${ACTOR_DEFAULT_DATASET_ID:-${APIFY_DEFAULT_DATASET_ID:-}}" +INPUT_KEY="${APIFY_INPUT_KEY:-INPUT}" +WORKERD_STDERR=/tmp/workerd.err + +if [ -z "${APIFY_TOKEN:-}" ] || [ -z "$STORE_ID" ] || [ -z "$DATASET_ID" ]; then + echo "[code-runtime] missing APIFY_TOKEN or default key-value store / dataset ID" >&2 + exit 1 +fi + +# Fetch the Actor input and wrap its `code` into the runnable module. The `code` +# is inserted as code between the wrapper lines (not as a string) — no escaping. +INPUT_URL="${API_BASE}/v2/key-value-stores/${STORE_ID}/records/${INPUT_KEY}" +input_status=$(curl -sS -o /tmp/input.json -w '%{http_code}' \ + -H "Authorization: Bearer ${APIFY_TOKEN}" "$INPUT_URL") +if [ "$input_status" != "200" ]; then + echo "[code-runtime] failed to read Actor input from ${INPUT_URL} (HTTP ${input_status})" >&2 + exit 1 +fi + +{ + printf 'export async function run(apify, console) {\n' + jq -r '.code // ""' < /tmp/input.json + printf '\n}\n' +} > /app/worker/usercode.js + +# config.capnp hardcodes __PORT__ as a placeholder so the port has one source ($PORT above). +sed -i "s/__PORT__/${PORT}/" /app/worker/config.capnp + +/usr/local/bin/workerd serve --experimental /app/worker/config.capnp 2>"$WORKERD_STDERR" & +workerd_pid=$! +trap 'kill "$workerd_pid" 2>/dev/null || true' EXIT + +# push_compile_failure reports a usercode.js syntax error as a normal, SUCCEEDED script +# result (same contract as a script that throws at runtime) instead of failing the whole +# Actor run. usercode.js wraps the user's `code` inside `export async function run(...) { +# ... }` with nothing else at module scope, so nothing in it can fail to *parse* except +# that inserted code — a startup crash naming usercode.js is therefore always a syntax +# error in the user's script, never our own code. See detection below. +push_compile_failure() { + crash_log=$(cat "$WORKERD_STDERR" 2>/dev/null || true) + echo "[code-runtime] usercode.js failed to compile: $crash_log" >&2 + item=$(jq -n --arg stderr "$crash_log" \ + '{stdout: "", stderr: $stderr, exitCode: 1, statusMessage: ("Failed to compile: " + $stderr)}') + push_status=$(curl -sS -o /dev/null -w '%{http_code}' -X POST \ + "${API_BASE}/v2/datasets/${DATASET_ID}/items" \ + -H "Authorization: Bearer ${APIFY_TOKEN}" \ + -H 'content-type: application/json; charset=utf-8' \ + --data-binary "$item") + case "$push_status" in + 2??) exit 0 ;; + *) + echo "[code-runtime] failed to push compile-failure diagnostic (HTTP $push_status)" >&2 + exit 1 + ;; + esac +} + +attempt=0 +until curl -sf "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; do + if ! kill -0 "$workerd_pid" 2>/dev/null; then + if grep -q 'usercode\.js' "$WORKERD_STDERR" 2>/dev/null; then + push_compile_failure + fi + echo "[code-runtime] workerd exited before startup: $(cat "$WORKERD_STDERR" 2>/dev/null)" >&2 + exit 1 + fi + attempt=$((attempt + 1)) + if [ "$attempt" -ge "$READINESS_ATTEMPTS" ]; then + echo "[code-runtime] workerd did not become ready in time" >&2 + exit 1 + fi + sleep 0.1 +done + +# Trigger the single run. The worker runs the program and pushes { stdout, stderr, +# exitCode, statusMessage } to the default dataset. A non-2xx response fails the Actor +# run (curl -f). +curl -fsS -X POST "http://127.0.0.1:${PORT}/run" diff --git a/worker/guard.ts b/worker/guard.ts new file mode 100644 index 0000000..46c62ab --- /dev/null +++ b/worker/guard.ts @@ -0,0 +1,151 @@ +// Restrict the user program's outbound network to apify.com and its subdomains. +// Imported before usercode.js so the overrides are in place even for code that +// runs at module-evaluation time. Our own Apify API calls use the real, +// unrestricted fetch (the internal API is a private IP, not *.apify.com) — +// see claimRealFetch() below for how runner.js gets it without leaving it +// usable by user code. +// +// Egress surface (workerd, no nodejs_compat): the only JS-reachable outbound +// primitives are fetch, WebSocket, and EventSource. Raw sockets (node:net, +// cloudflare:sockets connect()) need module imports, which are already blocked. +// fetch is allowlisted below; WebSocket and EventSource are removed outright +// because runner.js and the apify binding never use them — leaving them would +// be a non-fetch egress path around the allowlist (apify/ai-team#216 finding A, +// via WebSocket). If a future need arises, wrap them like fetch instead. + +const realFetch = globalThis.fetch.bind(globalThis); + +// One-shot handoff of the unrestricted fetch to runner.js — gated on genuine +// request handling, not on import order between guard.js and usercode.js. +// +// entrypoint.sh splices the `code` input verbatim (no escaping) into +// `export async function run(apify, console) { }`. A bare `}` in +// `code` closes that function early; everything after it runs as ordinary +// MODULE-SCOPE code in usercode.js, executed during module evaluation — +// i.e. unconditionally, before workerd ever calls this worker's own +// `fetch(request, env)` handler. That escaped code can `await +// import('./guard.js')` and call claimRealFetch() itself. +// +// This module previously assumed import order made runner.js's own claim run +// first (guard.js has no dependency on usercode.js, so it evaluates before +// it) — that protects nothing here, because it's usercode.js's top-level code +// racing runner.ts's top-level code, and usercode.js is the import evaluated +// immediately before runner.ts's own body runs. Verified live (PR #1 review, +// 2026-07-21): a `code` input using the escape above reliably wins that race, +// so runner.ts's own claim got null and threw, crashing the whole Actor run. +// +// The actual invariant that holds regardless of any module-graph ordering: +// usercode.js's module scope never runs *inside* a `fetch(request, env)` +// call — module evaluation always finishes before workerd dispatches the +// first request. So claimRealFetch() refuses to hand anything out until +// markRequestHandlingStarted() has been called; runner.ts calls it (and +// claims) as the first, synchronous statement of its `/run` handling, before +// any `await`, so no attacker-scheduled microtask can race it either. A claim +// attempted before that (legitimate or injected) gets null without consuming +// the resource, so the real claim still succeeds afterward. +let requestHandlingStarted = false; +export function markRequestHandlingStarted(): void { + requestHandlingStarted = true; +} + +let unclaimedRealFetch: typeof realFetch | null = realFetch; +export function claimRealFetch(): typeof realFetch | null { + if (!requestHandlingStarted) return null; // too early to be a trusted caller + const fetchFn = unclaimedRealFetch; + unclaimedRealFetch = null; + return fetchFn; +} + +// Match apify.com exactly or any subdomain. The leading dot in the suffix is +// what rejects look-alikes: `evilapify.com` (no dot) and `apify.com.evil.com` +// (ends with `.evil.com`) both fail. +function isAllowedHost(hostname: string): boolean { + const host = hostname.toLowerCase().replace(/\.$/, ''); // strip FQDN trailing dot + return host === 'apify.com' || host.endsWith('.apify.com'); +} + +function requestUrl(input: RequestInfo | URL): string { + if (typeof input === 'string') return input; + if (input instanceof URL) return input.href; + if (input && typeof input.url === 'string') return input.url; // Request + return String(input); +} + +// Parses and validates one URL against the allowlist. Returns the parsed URL +// (callers use it to resolve a relative redirect Location) or throws. +function validateUrl(input: RequestInfo | URL): URL { + let url: URL; + try { + // Parse to the real host — defeats userinfo (`apify.com@evil.com`), + // path/query/fragment (`evil.com/apify.com`) and similar tricks. + url = new URL(requestUrl(input)); + } catch { + throw new Error('Blocked fetch: only absolute http(s) URLs to apify.com are allowed'); + } + if (url.protocol !== 'https:' && url.protocol !== 'http:') { + throw new Error(`Blocked fetch: protocol "${url.protocol}" is not allowed`); + } + if (!isAllowedHost(url.hostname)) { + throw new Error(`Blocked fetch to "${url.hostname}": only apify.com and its subdomains are allowed`); + } + return url; +} + +// fetch() follows redirects internally by default, invisibly to a wrapper that +// only checks the initial URL — an allowlisted host could 302 to anywhere. +// Follow redirects ourselves, one hop at a time, and re-validate each Location +// against the allowlist before following it. Status-to-method mapping matches +// the WHATWG fetch spec: 303 always downgrades to GET; 301/302 downgrade to +// GET only when the original method was POST; 307/308 preserve method + body. +const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]); +const MAX_REDIRECT_HOPS = 5; + +function nextRedirectInit(init: RequestInit | undefined, status: number): RequestInit | undefined { + const method = (init?.method ?? 'GET').toUpperCase(); + const downgradeToGet = status === 303 || ((status === 301 || status === 302) && method === 'POST'); + if (!downgradeToGet) return init; + return { ...init, method: 'GET', body: undefined }; +} + +async function guardedFetch(input: RequestInfo | URL, init: RequestInit | undefined, hop = 0): Promise { + if (hop > MAX_REDIRECT_HOPS) { + throw new Error(`Blocked fetch: exceeded ${MAX_REDIRECT_HOPS} redirects`); + } + const url = validateUrl(input); + const response = await realFetch(input, { ...init, redirect: 'manual' }); + if (!REDIRECT_STATUSES.has(response.status)) return response; + const location = response.headers.get('location'); + if (!location) return response; // redirect status with no Location: nothing to follow + const nextUrl = new URL(location, url); // resolves a relative Location against the current URL + return guardedFetch(nextUrl.href, nextRedirectInit(init, response.status), hop + 1); +} + +// writable:false + configurable:false, matching blockGlobal() below — a plain +// assignment could be overwritten or deleted by the sandboxed script to +// recover the ambient (real, unrestricted) fetch reference some engines +// expose under a different name; locking it closes that off. +Object.defineProperty(globalThis, 'fetch', { + value: (input: RequestInfo | URL, init?: RequestInit) => guardedFetch(input, init), + writable: false, + configurable: false, + enumerable: true, +}); + +// Remove the non-fetch egress primitives. These are web-standard globals present +// even without nodejs_compat, and they connect directly (not through the fetch +// guard), so a script could otherwise open a wss:// or SSE connection to any +// public host and exfiltrate data around the *.apify.com allowlist. +function blockGlobal(name: string): void { + const blocked = function () { + throw new Error(`Blocked ${name}: only fetch() to apify.com and its subdomains is allowed`); + }; + Object.defineProperty(globalThis, name, { + value: blocked, + writable: false, + configurable: false, + enumerable: false, + }); +} +for (const name of ['WebSocket', 'EventSource']) { + if (name in globalThis) blockGlobal(name); +} diff --git a/worker/runner.ts b/worker/runner.ts new file mode 100644 index 0000000..ad3d1ec --- /dev/null +++ b/worker/runner.ts @@ -0,0 +1,598 @@ +// Single worker for the per-run code-runtime Actor. It runs the user's program +// (imported from the generated `usercode.js` module) with the `apify` REST +// binding and a captured `console`, then pushes `{ stdout, stderr, exitCode, +// statusMessage }` to the run's default dataset. The container entrypoint +// generates `usercode.js`, boots workerd, and triggers `/run` once. +// +// Single-tenant: one run = one container = one program = one token. No Worker +// Loader / per-request isolate is needed — the program runs in this worker, +// which is itself the sandbox (no filesystem, restricted outbound network). +// guard.js overrides globalThis.fetch to allow only apify.com, and hands us +// the real, unrestricted fetch via a one-shot claimRealFetch() for our own +// (internal) API calls. That claim is gated on markRequestHandlingStarted() +// (called below, first thing in the `fetch` handler's `/run` path), not on +// import order — usercode.js's module scope can run attacker-controlled code +// before this module's own top-level code does, so the claim must not +// happen at this module's top level either. See guard.ts's comment on +// claimRealFetch for the full reasoning. +import { claimRealFetch, markRequestHandlingStarted } from './guard.js'; +import { run } from './usercode.js'; + +// Called as the first, synchronous statement of the `/run` path in the +// `fetch` handler below — before any `await`, so no attacker-scheduled +// microtask from usercode.js's module scope can call claimRealFetch() in +// between. See guard.ts. +function requireRealFetch(): typeof globalThis.fetch { + const fetchFn = claimRealFetch(); + if (!fetchFn) throw new Error('realFetch already claimed.'); + return fetchFn; +} +// Assigned inside the `fetch` handler's `/run` path (the only call path into +// the functions below) before any of them run — definite-assignment +// asserted rather than left `| undefined` so call sites here don't need +// null checks. +let realFetch!: typeof globalThis.fetch; + +const DEFAULT_GET_SCHEMA_SAMPLE = 5; + +// --- Types --------------------------------------------------------------- +// The Apify API returns many more fields per record than this code reads. Rather +// than inventing a full schema we don't have, ApifyRecord asserts nothing beyond +// "a JSON object" and each specific shape below only names the fields this code +// actually consumes. + +interface ApifyRecord { + [key: string]: unknown; +} + +interface RunRecord extends ApifyRecord { + id: string; +} + +type SearchParamValue = string | number | boolean | undefined | null; +type SearchParams = Record; + +interface ApiCallOptions { + searchParams?: SearchParams; + body?: unknown; + contentType?: string; +} + +interface StoreSearchOptions { + search: string; + limit?: number; + offset?: number; + category?: string; +} + +interface ActorIdOptions { + actorId: string; +} + +interface StartOptions { + actorId: string; + input?: unknown; + memoryMbytes?: number; + timeoutSecs?: number; + waitForFinishSecs?: number; + maxTotalChargeUsd?: number; + maxItems?: number; +} + +interface RunAndGetItemsOptions extends StartOptions { + fields?: string[]; + limit?: number; +} + +interface RunIdOptions { + runId: string; +} + +interface WaitOptions extends RunIdOptions { + waitForFinishSecs?: number; +} + +interface GetLogOptions extends RunIdOptions { + limit?: number; +} + +interface DatasetListOptions { + datasetId: string; + fields?: string[]; + omit?: string[]; + limit?: number; + offset?: number; + clean?: boolean; + desc?: boolean; +} + +// One page, from a single request. Mirrors apify-client's PaginatedList shape (items, count, +// offset, limit, desc) except `total`, which stays deliberately unsurfaced — see makePaginatedList. +interface ItemsPage { + items: T[]; + count: number; + offset: number; + limit: number; +} + +interface DatasetItemsPage extends ItemsPage { + desc: boolean; +} + +// Awaiting this value resolves to one page (ItemsPage); `for await`-ing it auto-paginates +// through every item, one at a time. Same dual nature as apify-client's own PaginatedIterator +// (one call, one name, two ways to consume it) — see makePaginatedList for the mechanism. +type PaginatedItems> = Promise & AsyncIterable; + +interface DatasetSchemaOptions { + datasetId: string; + sample?: number; +} + +interface DatasetSchema { + itemCount: unknown; + sampleSize: number; + fields: { name: string; types: string[]; nullable: boolean }[]; +} + +interface CreateOptions { + name?: string; +} + +interface PushItemsOptions { + datasetId: string; + items: unknown[]; +} + +interface KeyValueStoreGetOptions { + storeId: string; + key: string; +} + +interface KeyValueStoreSetOptions extends KeyValueStoreGetOptions { + value: unknown; + contentType?: string; +} + +interface KeyValueStoreListOptions { + storeId: string; + limit?: number; + exclusiveStartKey?: string; +} + +interface ConsoleLike { + log: (...args: unknown[]) => void; + error: (...args: unknown[]) => void; + warn: (...args: unknown[]) => void; + info: (...args: unknown[]) => void; +} + +interface Env { + APIFY_TOKEN?: string; + DEFAULT_DATASET_ID?: string; + DEFAULT_DATASET_ID_LEGACY?: string; + API_BASE_URL?: string; + // APIFY_META_ORIGIN, forwarded from the platform's own env var of the same + // name (bound as PARENT_ORIGIN in config.capnp). Reflects this run's own + // meta.origin, set by apify-core from the X-Apify-Request-Origin request + // header the caller sent when creating THIS run — 'MCP' when apify-mcp-server + // started it. Platform-injected, not user-settable: unlike an Actor input + // field, a script running inside this Actor cannot spoof it. + PARENT_ORIGIN?: string; +} + +interface OutputItem { + stdout: string; + stderr: string; + exitCode: number; + statusMessage: string; +} + +// --------------------------------------------------------------------------- + +function stringify(x: unknown): string { + if (typeof x === 'string') return x; + try { return JSON.stringify(x); } catch { return String(x); } +} + +function errorMessage(err: unknown): string { + return err instanceof Error ? err.message : String(err); +} + +function errorDetail(err: unknown): string { + return err instanceof Error && err.stack ? err.stack : errorMessage(err); +} + +// Wraps a page-fetcher into a value that's both a Promise (awaits to the first page) and an +// AsyncIterable (walks every page, yielding one item at a time) — the same dual nature as +// apify-client's own PaginatedIterator, implemented via the same trick it uses: attach +// Symbol.asyncIterator to a live Promise object (a Promise is a plain object at runtime, so +// this is legal, no class or wrapper needed). +// +// One shared implementation, unlike apify-client itself, which independently re-implements +// this exact trick three times (dataset items, key-value-store keys, request-queue requests) +// with three subtly different cursor/offset conventions — see docs/API.md's Conventions +// section for that comparison. Every paginated method in this binding goes through this one +// function instead. +// +// Continuation stops the same way the old dataset.iterate() did: a page shorter than the +// limit it was asked for is the natural end-of-data signal (no dataset `total` is trusted — +// see the listItems comment below for why). +function makePaginatedList>( + fetchPage: (offset: number, limit: number | undefined) => Promise, + offset: number, + limit: number | undefined, +): PaginatedItems { + const firstPagePromise = fetchPage(offset, limit); + + async function* iterateAll(): AsyncGenerator { + let page = await firstPagePromise; + yield* page.items; + let nextOffset = page.offset + page.items.length; + while (page.items.length > 0 && page.items.length >= page.limit) { + page = await fetchPage(nextOffset, page.limit); + yield* page.items; + nextOffset += page.items.length; + } + } + + return Object.defineProperty(firstPagePromise, Symbol.asyncIterator, { + value: iterateAll, + }) as PaginatedItems; +} + +// 'MCP' matches apify-core's META_ORIGINS.MCP / apify-mcp-server's own +// X-Apify-Request-Origin header value — reusing the platform's existing +// convention rather than inventing a new one. +const MCP_ORIGIN = 'MCP'; +const REQUEST_ORIGIN_HEADER = 'X-Apify-Request-Origin'; + +function makeApifyBinding(token: string, apiV2: string, parentOrigin: string | undefined) { + // Every request this Actor makes identifies itself; requests made while THIS + // run's own origin is MCP additionally forward that origin so runs started by + // apify.actor.start/call/callAndGetItems() below get meta.origin: 'MCP' too, + // instead of the platform's default meta.origin: 'ACTOR' for actor-to-actor + // calls. Gated on parentOrigin (verified server-side, see the Env.PARENT_ORIGIN + // comment) rather than any Actor input, so a script can't forge an origin this + // run wasn't actually started with. + const baseHeaders: Record = { + Authorization: `Bearer ${token}`, + 'User-Agent': 'apify-code-runtime', + ...(parentOrigin === MCP_ORIGIN ? { [REQUEST_ORIGIN_HEADER]: MCP_ORIGIN } : {}), + }; + + // Build a URL with optional query params; null/undefined values are dropped. + const buildUrl = (path: string, searchParams?: SearchParams): URL => { + const url = new URL(`${apiV2}${path}`); + if (searchParams) { + for (const [key, value] of Object.entries(searchParams)) { + if (value !== undefined && value !== null) url.searchParams.set(key, String(value)); + } + } + return url; + }; + + // Single-source HTTP wrapper. Throws on non-2xx with the response body in the message. + // `body`: string / Uint8Array passed through; objects are JSON.stringify'd. + const apiCall = async (method: string, path: string, options: ApiCallOptions = {}): Promise => { + const { searchParams, body, contentType } = options; + const headers: Record = { ...baseHeaders }; + let requestBody: BodyInit | undefined; + if (body !== undefined) { + if (typeof body === 'string' || body instanceof Uint8Array || body instanceof ArrayBuffer) { + // Cast: this TS/DOM-lib pairing types Uint8Array generically over its buffer, + // which doesn't structurally match BodyInit here even though it's a valid + // fetch body at runtime (an ArrayBufferView). + requestBody = body as BodyInit; + headers['content-type'] = contentType ?? 'application/octet-stream'; + } else { + requestBody = JSON.stringify(body); + headers['content-type'] = contentType ?? 'application/json'; + } + } + const response = await realFetch(buildUrl(path, searchParams), { method, headers, body: requestBody }); + if (!response.ok) throw new Error(`${method} ${path} failed: ${response.status} ${await response.text()}`); + return response; + }; + + // The Apify API's JSON envelope (`{ data: ... }`) carries whatever shape the endpoint + // returns; there's no schema to check it against here, so this stays honestly `any` + // rather than asserting a shape we haven't verified. + const apiJson = async (method: string, path: string, options?: ApiCallOptions): Promise => + (await apiCall(method, path, options)).json(); + const apiData = async (method: string, path: string, options?: ApiCallOptions): Promise => + (await apiJson(method, path, options)).data; + + // Run IDs this script itself started, via actor.call() / actor.start() (and transitively + // actor.callAndGetItems(), which shares createRun() below). run.abort() below is scoped to + // this set — a script can only abort runs it started, not any account-wide runId it's + // handed or guesses. + const startedRunIds = new Set(); + + // POST /acts/:id/runs, shared by actor.call() (start+wait, waitForFinishSecs defaults to 60, + // capped at 60s per the Apify API — for longer runs use start() + apify.run.waitForFinish()) + // and actor.start() (async kickoff, no wait). Returns the run record so the caller can read + // defaultDatasetId / defaultKeyValueStoreId. Intentionally does NOT use /run-sync, which + // returns the OUTPUT KVS record (a pattern only some Actors follow) rather than the + // structured run record. + const createRun = ({ actorId, input, memoryMbytes, timeoutSecs, waitForFinishSecs, maxTotalChargeUsd, maxItems }: StartOptions): Promise => + apiData('POST', `/acts/${encodeURIComponent(actorId)}/runs`, { + searchParams: { + waitForFinish: waitForFinishSecs, + memory: memoryMbytes, + timeout: timeoutSecs, + maxTotalChargeUsd, + maxItems, + }, + body: input ?? {}, + }).then((runRecord: RunRecord) => { + startedRunIds.add(runRecord.id); + return runRecord; + }); + + const actor = { + get: ({ actorId }: ActorIdOptions): Promise => + apiData('GET', `/acts/${encodeURIComponent(actorId)}`), + + // Shared by run() and start(): both POST /acts/:id/runs, differing only in whether + // waitForFinish is set. Records the created run's ID in startedRunIds so run.abort() + // can be scoped to runs this script itself started (see the run.abort definition below). + call: (opts: StartOptions): Promise => createRun({ waitForFinishSecs: 60, ...opts }), + + // Async kickoff. Returns immediately with a run record in READY/RUNNING state. + start: (opts: StartOptions): Promise => createRun(opts), + + // Runs an Actor (same as call(), waitForFinishSecs defaults to 60) and returns its + // dataset items in one call. Calls createRun() directly rather than through + // `actor.call()` — same underlying request, no self-reference to `actor` needed. + callAndGetItems: async ({ actorId, input, fields, limit, ...runOpts }: RunAndGetItemsOptions): Promise<{ run: RunRecord; items: ApifyRecord[] }> => { + const runRecord = await createRun({ actorId, input, waitForFinishSecs: 60, ...runOpts }); + const { items } = await dataset.listItems({ + datasetId: runRecord.defaultDatasetId as string, fields, limit, + }); + return { run: runRecord, items }; + }, + }; + + const run = { + get: ({ runId }: RunIdOptions): Promise => + apiData('GET', `/actor-runs/${encodeURIComponent(runId)}`), + + // Block until the run terminates or `waitForFinishSecs` elapses (whichever comes first). + // The Apify API caps this at 60s per request; longer waits require a polling loop. + waitForFinish: ({ runId, waitForFinishSecs = 60 }: WaitOptions): Promise => + apiData('GET', `/actor-runs/${encodeURIComponent(runId)}`, { + searchParams: { waitForFinish: waitForFinishSecs }, + }), + + // Scoped to runs this script itself started (see startedRunIds above) — without this, + // any runId a script is handed (e.g. read from a dataset item, or guessed) could abort + // an unrelated, account-wide run. + abort: ({ runId }: RunIdOptions): Promise => { + if (!startedRunIds.has(runId)) { + throw new Error(`Blocked run.abort: "${runId}" was not started by this script`); + } + return apiData('POST', `/actor-runs/${encodeURIComponent(runId)}/abort`); + }, + + // Returns the full run log as text. `limit` tails the last N characters; the Apify API + // does not paginate logs, so this is a client-side slice (the full body is fetched). + getLog: async ({ runId, limit }: GetLogOptions): Promise => { + const response = await apiCall('GET', `/logs/${encodeURIComponent(runId)}`); + const text = await response.text(); + return limit && text.length > limit ? text.slice(-limit) : text; + }, + }; + + const dataset = { + // `await` resolves to one page: { items, count, offset, limit, desc }. `for await` + // auto-paginates through the entire dataset, one item at a time (replaces the old, + // separate dataset.iterate() method — see makePaginatedList). offset/limit/desc echo + // back what the API actually applied (read from its x-apify-pagination-* response + // headers, not just the request), except `total`: the Apify API's + // `x-apify-pagination-total` header is unreliable for freshly-created datasets + // (eventually consistent), so it's never surfaced — `count` (this page's actual item + // count) is what you want instead. Use `inferFields` if you need an approximate total. + listItems: ({ datasetId, fields, omit, limit, offset = 0, clean, desc }: DatasetListOptions): PaginatedItems => { + const fetchPage = async (pageOffset: number, pageLimit?: number): Promise => { + const response = await apiCall('GET', `/datasets/${encodeURIComponent(datasetId)}/items`, { + searchParams: { + fields: fields?.join(','), + omit: omit?.join(','), + limit: pageLimit, + offset: pageOffset, + clean: clean ? '1' : undefined, + desc: desc ? '1' : undefined, + }, + }); + const items: ApifyRecord[] = await response.json(); + return { + items, + count: items.length, + offset: Number(response.headers.get('x-apify-pagination-offset') ?? pageOffset), + limit: Number(response.headers.get('x-apify-pagination-limit') ?? pageLimit ?? items.length), + desc: response.headers.get('x-apify-pagination-desc') === 'true', + }; + }; + return makePaginatedList(fetchPage, offset, limit); + }, + + // Apify has no dedicated schema endpoint; we infer one from a small sample of items. + // Named inferFields (not getSchema) to avoid colliding with the Actor's own *declared* + // dataset schema (a different concept, described in this Actor's own actor.json). + inferFields: async ({ datasetId, sample = DEFAULT_GET_SCHEMA_SAMPLE }: DatasetSchemaOptions): Promise => { + const meta = await apiData('GET', `/datasets/${encodeURIComponent(datasetId)}`); + const { items } = await dataset.listItems({ datasetId, limit: sample }); + const fields = new Map>(); + for (const item of items) { + for (const [name, value] of Object.entries(item ?? {})) { + const type = value === null ? 'null' : Array.isArray(value) ? 'array' : typeof value; + if (!fields.has(name)) fields.set(name, new Set()); + fields.get(name)!.add(type); + } + } + return { + itemCount: meta?.itemCount, + sampleSize: items.length, + fields: [...fields.entries()].map(([name, types]) => ({ + name, + types: [...types], + nullable: types.has('null'), + })), + }; + }, + + create: ({ name }: CreateOptions = {}): Promise => + apiData('POST', '/datasets', { searchParams: { name } }), + + pushItems: async ({ datasetId, items }: PushItemsOptions): Promise => { + await apiCall('POST', `/datasets/${encodeURIComponent(datasetId)}/items`, { body: items }); + }, + }; + + const keyValueStore = { + // Returns the value directly (parsed when JSON, string when text/*, Uint8Array otherwise). + // Returns null when the key does not exist (404), not an error — this matches the common + // "lookup or default" pattern in code. + get: async ({ storeId, key }: KeyValueStoreGetOptions): Promise => { + const response = await realFetch(buildUrl(`/key-value-stores/${encodeURIComponent(storeId)}/records/${encodeURIComponent(key)}`), { + headers: baseHeaders, + }); + if (response.status === 404) return null; + if (!response.ok) throw new Error(`GET keyValueStore.get failed: ${response.status} ${await response.text()}`); + const contentType = response.headers.get('content-type') ?? ''; + if (contentType.includes('application/json')) return response.json(); + if (contentType.startsWith('text/')) return response.text(); + return new Uint8Array(await response.arrayBuffer()); + }, + + // `value`: object → application/json; string → text/plain; Uint8Array/ArrayBuffer → + // application/octet-stream (or whatever the caller passed via `contentType`). + set: async ({ storeId, key, value, contentType }: KeyValueStoreSetOptions): Promise => { + let body: BodyInit; + let resolvedContentType = contentType; + if (value instanceof Uint8Array || value instanceof ArrayBuffer) { + // Cast: see the equivalent Uint8Array-vs-BodyInit comment in apiCall() above. + body = value as BodyInit; + resolvedContentType = resolvedContentType ?? 'application/octet-stream'; + } else if (typeof value === 'string') { + body = value; + resolvedContentType = resolvedContentType ?? 'text/plain; charset=utf-8'; + } else { + body = JSON.stringify(value); + resolvedContentType = resolvedContentType ?? 'application/json; charset=utf-8'; + } + await apiCall('PUT', `/key-value-stores/${encodeURIComponent(storeId)}/records/${encodeURIComponent(key)}`, { + body, contentType: resolvedContentType, + }); + }, + + list: ({ storeId, limit, exclusiveStartKey }: KeyValueStoreListOptions): Promise => + apiData('GET', `/key-value-stores/${encodeURIComponent(storeId)}/keys`, { + searchParams: { limit, exclusiveStartKey }, + }), + + create: ({ name }: CreateOptions = {}): Promise => + apiData('POST', '/key-value-stores', { searchParams: { name } }), + }; + + // GET /v2/store — Apify Store search (a top-level resource in the Apify API, + // tagged `Store`, distinct from `Actors` — hence a top-level binding rather + // than an `actor.*` method). Same dual nature as dataset.listItems: `await` for one + // page, `for await` to walk every match. offset/limit echo back the request (the + // endpoint's own JSON body doesn't carry pagination metadata beyond `items`, unlike + // dataset's header-based pagination — see makePaginatedList). + const store = ({ search, limit, offset = 0, category }: StoreSearchOptions): PaginatedItems> => { + const fetchPage = async (pageOffset: number, pageLimit: number | undefined): Promise> => { + const page = await apiData('GET', '/store', { searchParams: { search, limit: pageLimit, offset: pageOffset, category } }); + const items: ApifyRecord[] = page.items; + return { items, count: items.length, offset: pageOffset, limit: pageLimit ?? items.length }; + }; + return makePaginatedList(fetchPage, offset, limit); + }; + + // Freeze every namespace (and the wrapper) so the script can't reassign a method to + // corrupt its own behavior or, for `console` below, its own output capture. + return Object.freeze({ + actor: Object.freeze(actor), + store, + run: Object.freeze(run), + dataset: Object.freeze(dataset), + keyValueStore: Object.freeze(keyValueStore), + }); +} + +// The shape handed to user code as the `apify` binding. Exported (type-only — +// erased at compile time) so tests/*.ts can type-check probes against the same +// surface real usercode.js runs against, without importing runner.ts at runtime. +export type ApifyBinding = ReturnType; + +// Push the captured streams as a single item to the run's default dataset. +async function pushOutput(apiV2: string, token: string, env: Env, item: OutputItem): Promise { + const datasetId = env.DEFAULT_DATASET_ID || env.DEFAULT_DATASET_ID_LEGACY; + if (!datasetId) throw new Error('Default dataset ID missing from Actor run environment.'); + const response = await realFetch(`${apiV2}/datasets/${encodeURIComponent(datasetId)}/items`, { + method: 'POST', + headers: { Authorization: `Bearer ${token}`, 'content-type': 'application/json; charset=utf-8' }, + body: JSON.stringify(item), + }); + if (!response.ok) throw new Error(`Failed to push dataset item: ${response.status} ${await response.text()}`); +} + +export default { + async fetch(request: Request, env: Env): Promise { + const url = new URL(request.url); + if (url.pathname === '/health') return new Response('ok'); + if (url.pathname !== '/run') return new Response('Not found', { status: 404 }); + + // First, synchronous statements of the /run path — see guard.ts and the + // requireRealFetch comment above for why this can't happen any earlier. + markRequestHandlingStarted(); + realFetch = requireRealFetch(); + + const token = env.APIFY_TOKEN; + if (!token) throw new Error('APIFY_TOKEN missing from Actor run environment.'); + // APIFY_API_BASE_URL is the platform-internal API (may have a trailing slash). + const apiV2 = `${(env.API_BASE_URL || 'https://api.apify.com').replace(/\/+$/, '')}/v2`; + + const stdout: string[] = []; + const stderr: string[] = []; + // Frozen so the script can't reassign e.g. console.log to corrupt its own capture. + const captureConsole: ConsoleLike = Object.freeze({ + log: (...args: unknown[]) => stdout.push(args.map(stringify).join(' ')), + error: (...args: unknown[]) => stderr.push(args.map(stringify).join(' ')), + warn: (...args: unknown[]) => stderr.push(args.map(stringify).join(' ')), + info: (...args: unknown[]) => stdout.push(args.map(stringify).join(' ')), + }); + + // A thrown program is a user-level failure: capture it in stderr and still + // push the output, so the run SUCCEEDS with diagnostics. Infra failures + // (missing env, dataset push) throw and fail the run. + // + // exitCode is the user script's effective status, distinct from the Actor run's + // status: 0 when the script returns normally, 1 when it throws. The run itself + // still SUCCEEDS on a throw, so callers detect a failed script via this field + // rather than heuristics on stderr (console.error is a legitimate log channel). + // statusMessage carries the same signal in prose, for callers that don't want to + // branch on exitCode. A script that fails to *compile* never reaches this handler at + // all (workerd fails the whole run before any request arrives) — entrypoint.sh + // handles that case directly, see its statusMessage "Failed to compile: ...". + let exitCode = 0; + let statusMessage = 'Script completed'; + try { + await run(makeApifyBinding(token, apiV2, env.PARENT_ORIGIN), captureConsole); + } catch (err) { + stderr.push(errorDetail(err)); + exitCode = 1; + statusMessage = `Script threw: ${errorMessage(err)}`; + } + + await pushOutput(apiV2, token, env, { + stdout: stdout.join('\n'), + stderr: stderr.join('\n'), + exitCode, + statusMessage, + }); + return Response.json({ ok: true }); + }, +}; diff --git a/worker/usercode.d.ts b/worker/usercode.d.ts new file mode 100644 index 0000000..f36a870 --- /dev/null +++ b/worker/usercode.d.ts @@ -0,0 +1,7 @@ +// usercode.js is generated at container startup by entrypoint.sh (gitignored, never +// checked in) — it wraps the run's `code` input in `export async function run(apify, +// console) { ...code... }`. This declaration lets `tsc` resolve runner.ts's import +// without the generated file present. The user's code is untyped JS text spliced +// into the function body, so `apify`/`console` stay `unknown` here on purpose — +// runner.ts's own ApifyBinding/ConsoleLike types describe what's actually passed in. +export function run(apify: unknown, consoleLike: unknown): Promise;