diff --git a/README.md b/README.md
index 0ff4d16..15e3b21 100755
--- a/README.md
+++ b/README.md
@@ -91,6 +91,7 @@ What each offers:
| Managed embeddings: managed indexes, `records.vector_search` by text `query` | ✓ | — |
| Natural-language search on SQL (`ai.search`, `ai.run_sql`) | ✓ | — |
| Relationship-pattern suggestions (`relationships.patterns.*`) and join suggestions (`ai.get_join_candidates`) | ✓ | — |
+| Background file imports (`imports.*`) and SaaS connections (`connections.*`) | ✓ | — |
| Project settings (`settings.get`), dashboard, team access | ✓ | — |
In direct mode a Cloud-only method raises `CloudOnlyError` (with `method` and `reason`) before
@@ -241,6 +242,110 @@ db.records.import_csv(
---
+## File imports and connections (RushDB Cloud)
+
+Large files and SaaS data load in the background: the client hands the data over and the run
+continues on the server. Every file import and every connection sync is an **import run**; follow it
+with `db.imports.get(id)` or `db.imports.wait(id)` until its status is `completed`, `partial`,
+`failed` or `cancelled`.
+
+### Import files
+
+```python
+# One call: create → upload every file → start (→ wait)
+run = db.imports.import_files(
+ [
+ {'data': 'customers.csv', 'label': 'Customer'}, # a path: name and size come from it
+ open('orders.csv.gz', 'rb'), # a binary file object works too
+ ],
+ mappings={0: {'keyColumn': 'id'}}, # re-importing upserts by `id`
+ links=[{'source': 1, 'sourceColumn': 'customer_id', 'target': 0, 'targetColumn': 'id', 'type': 'PLACED_BY'}],
+ parallelism=4,
+ wait=True,
+ wait_interval=2,
+ on_progress=lambda p: print(f"{p['loaded']}/{p['total']}" if p['phase'] == 'upload' else p['run']['status']),
+)
+print(run.data['recordsWritten'], run.data['summary'])
+```
+
+Or step by step, to show a preview before starting:
+
+```python
+import os
+
+created = db.imports.create([{'name': 'orders.csv', 'size': os.path.getsize('orders.csv')}]).data
+db.imports.upload(created['id'], created['files'][0], 'orders.csv', parallelism=4, retries=3)
+preview = db.imports.preview(created['id']).data # column types, suggested key and links, personal data
+columns = preview['files'][0]['columns'] # each has `suggestion` ('include' | 'skip' | 'hash') and `pii` when it looks personal
+mapping = {
+ 'skip': [c['name'] for c in columns if c['suggestion'] == 'skip'],
+ 'hash': [c['name'] for c in columns if c['suggestion'] == 'hash'], # keyed hash: joins, not readable
+}
+if preview['files'][0].get('suggestedKey'):
+ mapping['keyColumn'] = preview['files'][0]['suggestedKey']
+db.imports.start(created['id'], files=[{'id': created['files'][0]['id'], 'mapping': mapping}])
+finished = db.imports.wait(created['id'], timeout=30 * 60).data # TimeoutError after 30 minutes
+```
+
+`upload` follows the upload mode the server returns: `s3-multipart` sends parts to their presigned
+URLs (`parallelism` at a time on a thread pool, default 4; each failed part retried, `retries`
+default 3) and completes with their ETags; `direct` sends the bytes through the API. Upload data can
+be a file path (`str` or `Path`), `bytes`/`bytearray`/`memoryview`, or a binary file object (read
+from its current position). A non-seekable stream (a pipe, a socket) needs its `size` up front in
+`import_files`, and a direct upload of one is not retried.
+
+Also: `imports.list(kind=, connection_id=, limit=, skip=)`, `cancel`, `retry`, `delete`,
+`add_files`, `remove_file`.
+
+### Connections
+
+```python
+import os
+
+catalog = db.connections.catalog().data # connectors with their streams and config fields
+
+created = db.connections.create({
+ 'connectorId': 'hubspot',
+ 'auth': {'type': 'bearer', 'token': os.environ['HUBSPOT_TOKEN']},
+ 'mode': 'scheduled',
+ 'scheduleMinutes': 60,
+ 'streams': [{'stream': 'contacts', 'policy': {'mode': 'auto', 'exclude': ['mobilephone']}}],
+}).data
+connection_id = created['connection']['id']
+db.connections.test(connection_id)
+db.connections.preview(connection_id, 'contacts') # first records, nothing written
+
+# Field review: profiles with personal-data findings (masked examples), the policy, unreviewed fields
+review = db.connections.fields(connection_id, 'contacts', limit=100).data
+db.connections.review_fields(
+ connection_id,
+ 'contacts',
+ [{'name': f['name'], 'action': f['suggestion']} for f in review['fields']], # or your own choices
+)
+
+sync_run = db.connections.sync(connection_id).data # or sync(connection_id, streams=['contacts'], full=True)
+db.imports.wait(sync_run['id'])
+```
+
+A stream's `policy` decides which fields reach the graph: `auto` keeps everything except `exclude`
+and `hash`; `review_fields` switches it to an `allowlist` of the fields marked `include` or `hash`,
+and fields that appear later wait in the stream's `unreviewed` list until the next review.
+
+On-prem databases sync through `rushdb-agent`: create an agent connection and hand the token
+(shown once) to the agent.
+
+```python
+agent = db.connections.create({'connectorId': 'agent', 'name': 'Warehouse', 'agent': {'intervalMinutes': 15}}).data
+print(agent['agentToken']) # connection['agent'] shows what the agent last reported
+```
+
+Also: `list`, `get`, `update`, `delete(id, delete_data=True)`, `runs(id, limit=, skip=)`, `pause`,
+`resume`, `rotate_webhook_token` (a new webhook URL, or a new agent token), `push_templates()`
+(Zapier, Make and n8n recipes for push connections). Credentials are sealed on the server and never
+returned.
+
+---
+
## SearchResult
`db.records.find()` returns a `SearchResult` — a list-like container with pagination metadata.
diff --git a/pyproject.toml b/pyproject.toml
index a687121..cb58158 100755
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
[project]
name = "rushdb"
-version = "3.3.0"
+version = "3.4.0"
description = "RushDB SDK for Python: the client for RushDB, the database for AI (RushDB Cloud or self-hosted)"
authors = [
{name = "RushDB Team", email = "hi@rushdb.com"}
diff --git a/src/rushdb/__init__.py b/src/rushdb/__init__.py
index 84694d9..4ccbba7 100755
--- a/src/rushdb/__init__.py
+++ b/src/rushdb/__init__.py
@@ -4,6 +4,8 @@
"""
from .api.ai import AIAPI
+from .api.connections import ConnectionsAPI
+from .api.imports import ImportsAPI
from .api.query import QueryAPI
from .api.records import pick_record_id
from .api.relationship_patterns import RelationshipPatternsAPI
@@ -25,6 +27,7 @@
ValueMatch,
ValueMatchesData,
)
+from .models.ingest import FINISHED_IMPORT_RUN_STATUSES
from .models.property import (
Property,
PropertyValuesData,
@@ -91,4 +94,7 @@
"ApiResponse",
"AIAPI",
"TextIndexesAPI",
+ "ImportsAPI",
+ "ConnectionsAPI",
+ "FINISHED_IMPORT_RUN_STATUSES",
]
diff --git a/src/rushdb/api/__init__.py b/src/rushdb/api/__init__.py
index 6c6de94..a07063c 100755
--- a/src/rushdb/api/__init__.py
+++ b/src/rushdb/api/__init__.py
@@ -1,4 +1,6 @@
from .ai import AIAPI
+from .connections import ConnectionsAPI
+from .imports import ImportsAPI
from .labels import LabelsAPI
from .properties import PropertiesAPI
from .query import QueryAPI
@@ -18,4 +20,6 @@
"RelationsAPI",
"RelationshipPatternsAPI",
"TextIndexesAPI",
+ "ImportsAPI",
+ "ConnectionsAPI",
]
diff --git a/src/rushdb/api/connections.py b/src/rushdb/api/connections.py
new file mode 100644
index 0000000..d9844a0
--- /dev/null
+++ b/src/rushdb/api/connections.py
@@ -0,0 +1,258 @@
+"""SaaS connections for the RushDB Python SDK (RushDB Cloud only)."""
+
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
+
+from ..models.api_response import ApiResponse
+from ..models.ingest import (
+ CatalogConnector,
+ Connection,
+ ConnectionFieldsReview,
+ ConnectionPreview,
+ ConnectionTestResult,
+ ConnectionWebhookToken,
+ CreateConnectionParams,
+ CreatedConnection,
+ DeletedConnection,
+ FieldChoice,
+ ImportRun,
+ PushTemplate,
+ UpdateConnectionParams,
+)
+from .base import BaseAPI
+from .imports import _guard, _wrap
+
+if TYPE_CHECKING:
+ from ..client import RushDB
+
+
+class ConnectionsAPI(BaseAPI):
+ """The project's connections to SaaS tools (CRM, billing, support, issue
+ trackers, docs, spreadsheets), custom APIs, webhooks and on-prem agents.
+ Every sync is an import run: follow it with ``db.imports.get(run_id)`` or
+ ``db.imports.wait(run_id)``.
+
+ Accessed via ``db.connections``. **RushDB Cloud only**: in direct mode
+ every method raises ``CloudOnlyError`` before sending anything.
+ """
+
+ def __init__(self, client: "RushDB"):
+ super().__init__(client)
+
+ def _guard(self, method: str) -> None:
+ _guard(self.client, "connections", method)
+
+ def catalog(self) -> "ApiResponse[List[CatalogConnector]]":
+ """Built-in connectors and the workspace's own, with their streams,
+ auth types and config fields."""
+ self._guard("catalog")
+ return _wrap(self.client._make_request("GET", "/connections/catalog"))
+
+ def push_templates(self) -> "ApiResponse[List[PushTemplate]]":
+ """Zapier, Make and n8n recipes for pushing records into a webhook
+ (push) connection."""
+ self._guard("push_templates")
+ return _wrap(self.client._make_request("GET", "/connections/push-templates"))
+
+ def list(self) -> "ApiResponse[List[Connection]]":
+ """The project's connections with their streams, status and last sync."""
+ self._guard("list")
+ return _wrap(self.client._make_request("GET", "/connections"))
+
+ def get(self, connection_id: str) -> "ApiResponse[Connection]":
+ """One connection."""
+ self._guard("get")
+ return _wrap(self.client._make_request("GET", f"/connections/{connection_id}"))
+
+ def create(
+ self, params: CreateConnectionParams
+ ) -> "ApiResponse[CreatedConnection]":
+ """Create a connection.
+
+ Credentials are sealed on the server and never returned. A push or
+ trigger connection's ``webhookToken`` and an agent connection's
+ (``connectorId: "agent"``) ``agentToken`` are returned once.
+
+ Args:
+ params: ``{"connectorId", "name"?, "config"?, "auth"?, "oauthApp"?,
+ "webhookSecret"?, "mode"?, "scheduleMinutes"?, "streams"?,
+ "push"?, "agent"?, "specId"?}``.
+
+ Returns:
+ ApiResponse: ``data`` is ``{"connection", "webhookToken"?, "agentToken"?}``.
+ """
+ self._guard("create")
+ return _wrap(self.client._make_request("POST", "/connections", dict(params)))
+
+ def update(
+ self, connection_id: str, params: UpdateConnectionParams
+ ) -> "ApiResponse[Connection]":
+ """Change name, credentials, mode, schedule or stream settings
+ (labels, delete mode, field policy)."""
+ self._guard("update")
+ return _wrap(
+ self.client._make_request(
+ "PATCH", f"/connections/{connection_id}", dict(params)
+ )
+ )
+
+ def delete(
+ self, connection_id: str, delete_data: bool = False
+ ) -> "ApiResponse[DeletedConnection]":
+ """Delete a connection; ``delete_data=True`` also deletes the records
+ it wrote."""
+ self._guard("delete")
+ params = {"deleteData": "true"} if delete_data else None
+ return _wrap(
+ self.client._make_request(
+ "DELETE", f"/connections/{connection_id}", params=params
+ )
+ )
+
+ def test(self, connection_id: str) -> "ApiResponse[ConnectionTestResult]":
+ """Check the credentials against the provider."""
+ self._guard("test")
+ return _wrap(
+ self.client._make_request("POST", f"/connections/{connection_id}/test", {})
+ )
+
+ def preview(
+ self, connection_id: str, stream: str, limit: Optional[int] = None
+ ) -> "ApiResponse[ConnectionPreview]":
+ """The first records of a stream as they would be written (up to 50).
+ Writes nothing."""
+ self._guard("preview")
+ body: Dict[str, Any] = {"stream": stream}
+ if limit is not None:
+ body["limit"] = limit
+ return _wrap(
+ self.client._make_request(
+ "POST", f"/connections/{connection_id}/preview", body
+ )
+ )
+
+ def fields(
+ self, connection_id: str, stream: str, limit: Optional[int] = None
+ ) -> "ApiResponse[ConnectionFieldsReview]":
+ """The field review of one stream: sampled field profiles with
+ personal-data findings (values of fields that look personal are
+ masked), the stream's current policy and the fields held back since
+ the last review. Writes nothing.
+
+ Args:
+ stream: The stream name.
+ limit: Records to sample, 1–200.
+ """
+ self._guard("fields")
+ body: Dict[str, Any] = {"stream": stream}
+ if limit is not None:
+ body["limit"] = limit
+ return _wrap(
+ self.client._make_request(
+ "POST", f"/connections/{connection_id}/fields", body
+ )
+ )
+
+ def review_fields(
+ self, connection_id: str, stream: str, choices: List[FieldChoice]
+ ) -> "ApiResponse[Connection]":
+ """Save a field review: the stream switches to an allowlist of the
+ fields marked ``include`` or ``hash`` (hashed ones are kept as a keyed
+ hash); ``exclude``d fields are left out, and new fields wait for the
+ next review.
+
+ Args:
+ choices: ``[{"name": "email", "action": "include" | "exclude" | "hash"}]``.
+ """
+ self._guard("review_fields")
+
+ def pick(action: str) -> List[str]:
+ return [choice["name"] for choice in choices if choice["action"] == action]
+
+ body = {
+ "streams": [
+ {
+ "stream": stream,
+ "policy": {
+ "mode": "allowlist",
+ "include": pick("include"),
+ "exclude": pick("exclude"),
+ "hash": pick("hash"),
+ },
+ }
+ ]
+ }
+ return _wrap(
+ self.client._make_request("PATCH", f"/connections/{connection_id}", body)
+ )
+
+ def sync(
+ self,
+ connection_id: str,
+ streams: Optional[List[str]] = None,
+ full: Optional[bool] = None,
+ ) -> "ApiResponse[ImportRun]":
+ """Sync now and return the queued run.
+
+ Args:
+ streams: Stream names; every enabled stream when omitted.
+ full: Ignore stored cursors and read everything again.
+ """
+ self._guard("sync")
+ body: Dict[str, Any] = {}
+ if streams is not None:
+ body["streams"] = streams
+ if full is not None:
+ body["full"] = full
+ return _wrap(
+ self.client._make_request(
+ "POST", f"/connections/{connection_id}/runs", body
+ )
+ )
+
+ def runs(
+ self,
+ connection_id: str,
+ limit: Optional[int] = None,
+ skip: Optional[int] = None,
+ ) -> "ApiResponse[List[ImportRun]]":
+ """The connection's sync runs, newest first; ``total`` is the count."""
+ self._guard("runs")
+ params = {
+ key: value
+ for key, value in (("limit", limit), ("skip", skip))
+ if value is not None
+ }
+ return _wrap(
+ self.client._make_request(
+ "GET", f"/connections/{connection_id}/runs", params=params or None
+ )
+ )
+
+ def pause(self, connection_id: str) -> "ApiResponse[Connection]":
+ """Stop scheduled and webhook-triggered syncs until resumed."""
+ self._guard("pause")
+ return _wrap(
+ self.client._make_request("POST", f"/connections/{connection_id}/pause", {})
+ )
+
+ def resume(self, connection_id: str) -> "ApiResponse[Connection]":
+ """Resume scheduled and webhook-triggered syncs."""
+ self._guard("resume")
+ return _wrap(
+ self.client._make_request(
+ "POST", f"/connections/{connection_id}/resume", {}
+ )
+ )
+
+ def rotate_webhook_token(
+ self, connection_id: str
+ ) -> "ApiResponse[ConnectionWebhookToken]":
+ """A new token, shown once; the previous one stops working. Trigger and
+ push connections get a new webhook URL, agent connections a new agent
+ token."""
+ self._guard("rotate_webhook_token")
+ return _wrap(
+ self.client._make_request(
+ "POST", f"/connections/{connection_id}/webhook-token", {}
+ )
+ )
diff --git a/src/rushdb/api/imports.py b/src/rushdb/api/imports.py
new file mode 100644
index 0000000..f7213e7
--- /dev/null
+++ b/src/rushdb/api/imports.py
@@ -0,0 +1,492 @@
+"""Background file imports for the RushDB Python SDK (RushDB Cloud only)."""
+
+import os
+import threading
+import time
+from typing import (
+ TYPE_CHECKING,
+ Any,
+ Callable,
+ Dict,
+ List,
+ Mapping,
+ Optional,
+ Sequence,
+ Union,
+ cast,
+)
+
+from ..common import CloudOnlyError
+from ..direct.capabilities import CLOUD_ONLY_METHODS
+from ..models.api_response import ApiResponse
+from ..models.ingest import (
+ FINISHED_IMPORT_RUN_STATUSES,
+ CreatedImport,
+ ImportFile,
+ ImportFileInput,
+ ImportFileLink,
+ ImportFileMapping,
+ ImportFileSettings,
+ ImportFilesLink,
+ ImportFilesProgress,
+ ImportFileUpload,
+ ImportPreview,
+ ImportRun,
+ ImportRunKind,
+)
+from .base import BaseAPI
+from .uploads import (
+ DEFAULT_UPLOAD_PARALLELISM,
+ DEFAULT_UPLOAD_RETRIES,
+ DEFAULT_UPLOAD_TIMEOUT,
+ ProgressCallback,
+ UploadData,
+ source_of,
+ upload_file_content,
+)
+
+if TYPE_CHECKING:
+ from ..client import RushDB
+
+ImportFileSource = Union[str, "os.PathLike[str]", Any, Dict[str, Any]]
+"""A file for ``imports.import_files``: a path, a binary file object, or a dict
+``{"data", "name"?, "size"?, "label"?, "mapping"?}`` (``data`` is a path, bytes
+or a binary file object)."""
+
+
+def _wrap(response: Dict[str, Any]) -> ApiResponse:
+ return ApiResponse(
+ data=response.get("data"),
+ success=response.get("success", True),
+ total=response.get("total"),
+ )
+
+
+def _guard(client: "RushDB", namespace: str, method: str) -> None:
+ if getattr(client, "direct", False) is True:
+ name = f"{namespace}.{method}"
+ raise CloudOnlyError(name, CLOUD_ONLY_METHODS[name])
+
+
+class ImportsAPI(BaseAPI):
+ """Background file imports (CSV, JSON, NDJSON, Parquet, optionally
+ gzip-compressed). Create an import with its files, upload them, preview the
+ mapping, start; the run continues on the server without the client.
+
+ Accessed via ``db.imports``. **RushDB Cloud only**: in direct mode every
+ method raises ``CloudOnlyError`` before sending anything.
+
+ Example::
+
+ >>> run = db.imports.import_files(
+ ... ["customers.csv", "orders.csv.gz"],
+ ... mappings={0: {"keyColumn": "id"}},
+ ... wait=True,
+ ... )
+ >>> run.data["status"], run.data["recordsWritten"]
+ """
+
+ def __init__(self, client: "RushDB"):
+ super().__init__(client)
+ # Upload instructions from `create`/`add_files`, so `upload` can take ids.
+ self._uploads: Dict[str, ImportFileUpload] = {}
+ self._uploads_lock = threading.Lock()
+
+ def _guard(self, method: str) -> None:
+ _guard(self.client, "imports", method)
+
+ def _remember(self, import_id: str, files: Sequence[ImportFileUpload]) -> None:
+ with self._uploads_lock:
+ for file in files or []:
+ self._uploads[f"{import_id}/{file['id']}"] = file
+
+ def _forget(self, import_id: str, file_id: str) -> None:
+ with self._uploads_lock:
+ self._uploads.pop(f"{import_id}/{file_id}", None)
+
+ def create(
+ self, files: List[ImportFileInput], name: Optional[str] = None
+ ) -> "ApiResponse[CreatedImport]":
+ """Create a draft import with its files (name and exact size in bytes;
+ up to 50). Each returned file carries its ``upload`` instructions.
+
+ Args:
+ files: ``[{"name": "orders.csv", "size": 1024, "label"?: "Order"}]``.
+ The extension picks the parser.
+ name: A name for the import.
+ """
+ self._guard("create")
+ body: Dict[str, Any] = {"files": files}
+ if name is not None:
+ body["name"] = name
+ response = _wrap(self.client._make_request("POST", "/imports", body))
+ if isinstance(response.data, dict):
+ self._remember(response.data["id"], response.data.get("files") or [])
+ return response
+
+ def add_files(
+ self, import_id: str, files: List[ImportFileInput]
+ ) -> "ApiResponse[List[ImportFileUpload]]":
+ """Add files to a draft import."""
+ self._guard("add_files")
+ response = _wrap(
+ self.client._make_request(
+ "POST", f"/imports/{import_id}/files", {"files": files}
+ )
+ )
+ if isinstance(response.data, list):
+ self._remember(import_id, response.data)
+ return response
+
+ def remove_file(self, import_id: str, file_id: str) -> ApiResponse:
+ """Remove a file from a draft import. ``data`` is ``{"deleted": True}``."""
+ self._guard("remove_file")
+ self._forget(import_id, file_id)
+ return _wrap(
+ self.client._make_request("DELETE", f"/imports/{import_id}/files/{file_id}")
+ )
+
+ def upload(
+ self,
+ import_id: str,
+ file: Union[str, ImportFileUpload],
+ data: UploadData,
+ *,
+ parallelism: int = DEFAULT_UPLOAD_PARALLELISM,
+ retries: int = DEFAULT_UPLOAD_RETRIES,
+ timeout: float = DEFAULT_UPLOAD_TIMEOUT,
+ on_progress: Optional[ProgressCallback] = None,
+ ) -> "ApiResponse[ImportFile]":
+ """Upload one file's bytes and complete it.
+
+ With ``s3-multipart`` upload instructions, parts go to their presigned
+ URLs ``parallelism`` at a time and their ETags are sent on completion;
+ with ``direct``, the bytes go through the API. Failed requests (network
+ errors, 429, 5xx) are retried; a non-seekable stream sent ``direct`` is
+ read once and not retried.
+
+ Args:
+ import_id: The import's id.
+ file: A file from ``create``/``add_files`` (with its ``upload``
+ instructions), or its id when this client created it.
+ data: A file path (``str`` or ``Path``), ``bytes``/``bytearray``/
+ ``memoryview``, or a binary file object. Its size must match
+ the size announced for the file.
+ parallelism: Parts uploaded at once (``s3-multipart``). Default 4.
+ retries: Attempts per part (or per direct upload). Default 3.
+ timeout: Per-request timeout in seconds. Default 600.
+ on_progress: Called with ``{"fileId", "loaded", "total"}`` as
+ bytes are sent (from upload threads).
+
+ Returns:
+ ApiResponse: The completed file.
+ """
+ self._guard("upload")
+ if isinstance(file, str):
+ with self._uploads_lock:
+ target = self._uploads.get(f"{import_id}/{file}")
+ if target is None:
+ raise ValueError(
+ f"No upload instructions for file {file}: pass the file "
+ "returned by imports.create() or imports.add_files()"
+ )
+ else:
+ target = file
+ parts = upload_file_content(
+ self.client,
+ target,
+ data,
+ parallelism=parallelism,
+ retries=retries,
+ timeout=timeout,
+ on_progress=on_progress,
+ )
+ response = _wrap(
+ self.client._make_request(
+ "POST",
+ f"/imports/{import_id}/files/{target['id']}/complete",
+ {"parts": parts} if parts is not None else {},
+ )
+ )
+ self._forget(import_id, target["id"])
+ return response
+
+ def preview(
+ self,
+ import_id: str,
+ files: Optional[List[ImportFileSettings]] = None,
+ ) -> "ApiResponse[ImportPreview]":
+ """Column types, samples, personal-data hints, a suggested key per file
+ and suggested links between files. Writes nothing.
+
+ Args:
+ files: Optional settings per file (``{"id", "label"?, "mapping"?}``).
+ """
+ self._guard("preview")
+ body: Dict[str, Any] = {} if files is None else {"files": files}
+ return _wrap(
+ self.client._make_request("POST", f"/imports/{import_id}/preview", body)
+ )
+
+ def start(
+ self,
+ import_id: str,
+ files: Optional[List[ImportFileSettings]] = None,
+ links: Optional[List[ImportFileLink]] = None,
+ ) -> "ApiResponse[ImportRun]":
+ """Start the run with the files' labels and mappings and the links
+ between files.
+
+ Args:
+ files: Settings per file: ``{"id", "label"?, "mapping"?}``, where a
+ mapping is ``{"keyColumn", "skip", "hash", "rename", "types",
+ "delimiter"}``.
+ links: ``{"sourceFileId", "sourceColumn", "targetFileId",
+ "targetColumn", "type"}``.
+ """
+ self._guard("start")
+ body: Dict[str, Any] = {}
+ if files is not None:
+ body["files"] = files
+ if links:
+ body["links"] = links
+ return _wrap(
+ self.client._make_request("POST", f"/imports/{import_id}/start", body)
+ )
+
+ def get(self, import_id: str) -> "ApiResponse[ImportRun]":
+ """One import run (a file import or a connection sync) with its progress."""
+ self._guard("get")
+ return _wrap(self.client._make_request("GET", f"/imports/{import_id}"))
+
+ def list(
+ self,
+ kind: Optional[ImportRunKind] = None,
+ connection_id: Optional[str] = None,
+ limit: Optional[int] = None,
+ skip: Optional[int] = None,
+ ) -> "ApiResponse[List[ImportRun]]":
+ """Import runs of the project, newest first; ``total`` is the count.
+
+ Args:
+ kind: ``"file"`` or ``"sync"``.
+ connection_id: Only the syncs of this connection.
+ limit: 1–100.
+ skip: Runs to skip.
+ """
+ self._guard("list")
+ params: Dict[str, Any] = {
+ key: value
+ for key, value in (
+ ("kind", kind),
+ ("connectionId", connection_id),
+ ("limit", limit),
+ ("skip", skip),
+ )
+ if value is not None
+ }
+ return _wrap(
+ self.client._make_request("GET", "/imports", params=params or None)
+ )
+
+ def cancel(self, import_id: str) -> "ApiResponse[ImportRun]":
+ """Ask a queued or running run to stop; it ends ``cancelled``."""
+ self._guard("cancel")
+ return _wrap(
+ self.client._make_request("POST", f"/imports/{import_id}/cancel", {})
+ )
+
+ def retry(self, import_id: str) -> "ApiResponse[ImportRun]":
+ """Run a failed or partial file import again."""
+ self._guard("retry")
+ return _wrap(
+ self.client._make_request("POST", f"/imports/{import_id}/retry", {})
+ )
+
+ def delete(self, import_id: str) -> ApiResponse:
+ """Delete an import and its uploaded files (records already written stay)."""
+ self._guard("delete")
+ return _wrap(self.client._make_request("DELETE", f"/imports/{import_id}"))
+
+ def wait(
+ self,
+ import_id: str,
+ interval: float = 2.0,
+ timeout: Optional[float] = None,
+ on_progress: Optional[Callable[[ImportRun], None]] = None,
+ ) -> "ApiResponse[ImportRun]":
+ """Poll a run until it is ``completed``, ``partial``, ``failed`` or
+ ``cancelled``, and return it.
+
+ Args:
+ interval: Seconds between polls. Default 2.
+ timeout: Give up after this many seconds. Default: no limit.
+ on_progress: Called with the run after every poll.
+
+ Raises:
+ TimeoutError: The run had not finished within ``timeout``.
+ """
+ self._guard("wait")
+ interval = max(0.0, interval)
+ deadline = None if timeout is None else time.monotonic() + timeout
+ while True:
+ response = self.get(import_id)
+ run = cast(ImportRun, response.data)
+ if on_progress is not None:
+ on_progress(run)
+ if run["status"] in FINISHED_IMPORT_RUN_STATUSES:
+ return response
+ if deadline is not None and time.monotonic() + interval > deadline:
+ raise TimeoutError(
+ f"Import {import_id} is still {run['status']} after {timeout} s"
+ )
+ time.sleep(interval)
+
+ def import_files(
+ self,
+ files: Sequence[ImportFileSource],
+ *,
+ name: Optional[str] = None,
+ links: Optional[List[ImportFilesLink]] = None,
+ mappings: Optional[Mapping[Union[str, int], ImportFileMapping]] = None,
+ wait: bool = False,
+ wait_interval: float = 2.0,
+ wait_timeout: Optional[float] = None,
+ parallelism: int = DEFAULT_UPLOAD_PARALLELISM,
+ retries: int = DEFAULT_UPLOAD_RETRIES,
+ timeout: float = DEFAULT_UPLOAD_TIMEOUT,
+ on_progress: Optional[Callable[[ImportFilesProgress], None]] = None,
+ ) -> "ApiResponse[ImportRun]":
+ """Import files in one call: create → upload every file → start
+ (→ wait when ``wait=True``).
+
+ Args:
+ files: Each a file path, a binary file object, or a dict
+ ``{"data", "name"?, "size"?, "label"?, "mapping"?}``. Names
+ and sizes come from the path, the file object or the bytes
+ when not given; a non-seekable stream needs ``size``. The
+ name's extension picks the parser.
+ name: A name for the import.
+ links: Links between files, naming them by position or name:
+ ``{"source", "sourceColumn", "target", "targetColumn", "type"}``.
+ mappings: Mappings by file name or position, in addition to
+ ``mapping`` on each file.
+ wait: Wait for the run to finish (default: return once started).
+ wait_interval: Seconds between polls while waiting.
+ wait_timeout: Give up waiting after this many seconds
+ (``TimeoutError``).
+ parallelism, retries, timeout: As for ``upload``.
+ on_progress: Called with ``{"phase": "upload", "fileId", "loaded",
+ "total"}`` while uploading, then ``{"phase": "run", "run"}``.
+
+ Returns:
+ ApiResponse: The started run, or the finished one with ``wait``.
+ """
+ self._guard("import_files")
+ sources: List[Dict[str, Any]] = []
+ for index, file in enumerate(files):
+ entry: Dict[str, Any] = (
+ dict(file) if isinstance(file, dict) else {"data": file}
+ )
+ data = entry.get("data")
+ file_name = entry.get("name")
+ if file_name is None:
+ if isinstance(data, (str, os.PathLike)):
+ file_name = os.path.basename(os.fspath(data))
+ elif isinstance(getattr(data, "name", None), str):
+ file_name = os.path.basename(data.name) # type: ignore[union-attr]
+ if not isinstance(file_name, str) or not file_name:
+ raise ValueError(
+ f"File {index} needs a name (its extension picks the parser)"
+ )
+ size = entry.get("size")
+ if size is None:
+ size = source_of(data).size # type: ignore[arg-type]
+ if not isinstance(size, int):
+ raise ValueError(
+ f"File {index} ({file_name}) is a stream: pass its size in bytes"
+ )
+ entry["name"] = file_name
+ entry["size"] = size
+ sources.append(entry)
+
+ announced: List[ImportFileInput] = []
+ for source in sources:
+ item: ImportFileInput = {"name": source["name"], "size": source["size"]}
+ if source.get("label"):
+ item["label"] = source["label"]
+ announced.append(item)
+ created = self.create(announced, name=name)
+ run = created.data
+ run_files = run["files"]
+
+ def file_id_of(ref: Union[int, str]) -> str:
+ position = (
+ ref
+ if isinstance(ref, int)
+ else next(
+ (i for i, source in enumerate(sources) if source["name"] == ref),
+ -1,
+ )
+ )
+ if not 0 <= position < len(run_files):
+ raise ValueError(f"No file {ref!r} in this import")
+ return run_files[position]["id"]
+
+ resolved_links: List[ImportFileLink] = []
+ for link in links or []:
+ resolved_links.append(
+ {
+ "sourceFileId": file_id_of(link["source"]),
+ "sourceColumn": link["sourceColumn"],
+ "targetFileId": file_id_of(link["target"]),
+ "targetColumn": link["targetColumn"],
+ "type": link["type"],
+ }
+ )
+
+ def upload_progress(progress: Any) -> None:
+ if on_progress is not None:
+ on_progress({"phase": "upload", **progress})
+
+ for index, source in enumerate(sources):
+ self.upload(
+ run["id"],
+ run_files[index],
+ source["data"],
+ parallelism=parallelism,
+ retries=retries,
+ timeout=timeout,
+ on_progress=upload_progress,
+ )
+
+ settings: List[ImportFileSettings] = []
+ for index, source in enumerate(sources):
+ mapping = source.get("mapping")
+ if mapping is None and mappings:
+ mapping = mappings.get(source["name"])
+ if mapping is None:
+ mapping = mappings.get(index)
+ if mapping is None:
+ mapping = mappings.get(str(index))
+ setting: ImportFileSettings = {"id": run_files[index]["id"]}
+ if mapping:
+ setting["mapping"] = mapping
+ settings.append(setting)
+
+ started = self.start(run["id"], files=settings, links=resolved_links or None)
+ if on_progress is not None:
+ on_progress({"phase": "run", "run": started.data})
+ if not wait:
+ return started
+
+ def run_progress(current: ImportRun) -> None:
+ if on_progress is not None:
+ on_progress({"phase": "run", "run": current})
+
+ return self.wait(
+ run["id"],
+ interval=wait_interval,
+ timeout=wait_timeout,
+ on_progress=run_progress,
+ )
diff --git a/src/rushdb/api/uploads.py b/src/rushdb/api/uploads.py
new file mode 100644
index 0000000..df17a15
--- /dev/null
+++ b/src/rushdb/api/uploads.py
@@ -0,0 +1,333 @@
+"""Sending an import file's bytes as the server's upload instructions say.
+
+``s3-multipart`` puts parts to presigned URLs (no API auth header) a few at a
+time and returns their ETags for the completion call; ``direct`` puts the
+bytes to the API with the usual auth header.
+"""
+
+import functools
+import os
+import threading
+import time
+import urllib.parse
+from concurrent.futures import FIRST_COMPLETED, Future, ThreadPoolExecutor, wait
+from pathlib import Path
+from typing import (
+ IO,
+ TYPE_CHECKING,
+ Any,
+ Callable,
+ Dict,
+ Iterator,
+ List,
+ Optional,
+ Set,
+ Tuple,
+ TypeVar,
+ Union,
+)
+
+from ..common import RushDBError
+from ..models.ingest import ImportFileUpload, ImportUploadProgress
+
+if TYPE_CHECKING:
+ from ..client import RushDB
+
+UploadData = Union[str, "os.PathLike[str]", bytes, bytearray, memoryview, IO[bytes]]
+"""File contents for an upload: a file path, bytes, or a binary file object."""
+
+ProgressCallback = Callable[[ImportUploadProgress], None]
+
+DEFAULT_UPLOAD_PARALLELISM = 4
+DEFAULT_UPLOAD_RETRIES = 3
+DEFAULT_UPLOAD_TIMEOUT = 600.0
+"""Per-request timeout in seconds."""
+
+T = TypeVar("T")
+
+
+def _backoff(attempt: int) -> None:
+ time.sleep(min(8.0, 0.25 * 2**attempt))
+
+
+def _read_exactly(stream: IO[bytes], size: int) -> bytes:
+ """Up to ``size`` bytes; fewer only at the end of the stream."""
+ chunks: List[bytes] = []
+ remaining = size
+ while remaining > 0:
+ chunk = stream.read(remaining)
+ if not chunk:
+ break
+ if not isinstance(chunk, (bytes, bytearray)):
+ raise TypeError("Upload file objects must be opened in binary mode ('rb')")
+ chunks.append(bytes(chunk))
+ remaining -= len(chunk)
+ return b"".join(chunks)
+
+
+class _Source:
+ """Upload data: random access (can be re-read for a retry) or a stream."""
+
+ size: Optional[int] = None
+ random_access = True
+
+ def read_range(self, start: int, end: int) -> bytes:
+ raise NotImplementedError
+
+ def stream(self) -> IO[bytes]:
+ raise NotImplementedError
+
+
+class _BytesSource(_Source):
+ def __init__(self, data: Union[bytes, bytearray, memoryview]):
+ self._view = memoryview(data).cast("B")
+ self.size = self._view.nbytes
+
+ def read_range(self, start: int, end: int) -> bytes:
+ return bytes(self._view[start:end])
+
+
+class _PathSource(_Source):
+ def __init__(self, path: Union[str, "os.PathLike[str]"]):
+ self._path = Path(path)
+ self.size = self._path.stat().st_size
+
+ def read_range(self, start: int, end: int) -> bytes:
+ with open(self._path, "rb") as f:
+ f.seek(start)
+ return _read_exactly(f, end - start)
+
+
+class _FileSource(_Source):
+ def __init__(self, f: IO[bytes]):
+ self._file = f
+ self._lock = threading.Lock()
+ seekable = False
+ try:
+ seekable = bool(f.seekable())
+ except (AttributeError, OSError, ValueError):
+ pass
+ if seekable:
+ self._start = f.tell()
+ end = f.seek(0, os.SEEK_END)
+ f.seek(self._start)
+ self.size = end - self._start
+ else:
+ self._start = 0
+ self.size = None
+ self.random_access = False
+
+ def read_range(self, start: int, end: int) -> bytes:
+ with self._lock:
+ self._file.seek(self._start + start)
+ return _read_exactly(self._file, end - start)
+
+ def stream(self) -> IO[bytes]:
+ return self._file
+
+
+def size_of_file_object(f: Any) -> Optional[int]:
+ """Bytes left in a seekable file object, or ``None``."""
+ return _FileSource(f).size
+
+
+def source_of(data: UploadData) -> _Source:
+ if isinstance(data, (bytes, bytearray, memoryview)):
+ return _BytesSource(data)
+ if isinstance(data, (str, os.PathLike)):
+ return _PathSource(data)
+ if hasattr(data, "read"):
+ return _FileSource(data)
+ raise TypeError(
+ "Unsupported upload data: pass a file path, bytes or a binary file object"
+ )
+
+
+def _is_retryable(error: BaseException) -> bool:
+ if not isinstance(error, RushDBError):
+ return False
+ status = error.status
+ return status is None or status == 429 or status >= 500
+
+
+def _with_retries(attempts: int, run: Callable[[], T]) -> T:
+ attempts = max(1, attempts)
+ for attempt in range(attempts):
+ try:
+ return run()
+ except RushDBError as error:
+ if not _is_retryable(error) or attempt == attempts - 1:
+ raise
+ _backoff(attempt)
+ raise AssertionError("unreachable")
+
+
+def _header(headers: Dict[str, str], name: str) -> Optional[str]:
+ for key, value in headers.items():
+ if key.lower() == name.lower():
+ return value
+ return None
+
+
+def upload_file_content(
+ client: "RushDB",
+ file: ImportFileUpload,
+ data: UploadData,
+ parallelism: int = DEFAULT_UPLOAD_PARALLELISM,
+ retries: int = DEFAULT_UPLOAD_RETRIES,
+ timeout: float = DEFAULT_UPLOAD_TIMEOUT,
+ on_progress: Optional[ProgressCallback] = None,
+) -> Optional[List[Dict[str, Any]]]:
+ """Sends one file's bytes as its upload instructions say.
+
+ Returns the parts with their ETags for ``s3-multipart``, ``None`` for
+ ``direct``.
+ """
+ source = source_of(data)
+ total = file["size"]
+ if source.size is not None and source.size != total:
+ raise ValueError(
+ f"File {file['id']} was announced as {total} bytes "
+ f"but the data has {source.size}"
+ )
+
+ progress_lock = threading.Lock()
+ loaded = 0
+
+ def progress(sent: int) -> None:
+ nonlocal loaded
+ with progress_lock:
+ loaded += sent
+ if on_progress is not None:
+ on_progress({"fileId": file["id"], "loaded": loaded, "total": total})
+
+ upload = file["upload"]
+ if upload["mode"] == "direct":
+ url = urllib.parse.urljoin(client.base_url + "/", upload["path"])
+ headers = {
+ "Authorization": f"Bearer {client.api_key}",
+ "Content-Type": "application/octet-stream",
+ "Content-Length": str(total),
+ }
+ method = upload.get("method") or "PUT"
+
+ if source.random_access:
+
+ def send_all() -> None:
+ if isinstance(source, _PathSource):
+ with open(source._path, "rb") as f:
+ client._send_raw(
+ method, url, f, headers, timeout, "upload failed"
+ )
+ elif isinstance(source, _FileSource):
+ with source._lock:
+ source._file.seek(source._start)
+ client._send_raw(
+ method, url, source._file, headers, timeout, "upload failed"
+ )
+ else:
+ client._send_raw(
+ method,
+ url,
+ source.read_range(0, total),
+ headers,
+ timeout,
+ "upload failed",
+ )
+
+ _with_retries(retries, send_all)
+ else:
+ # A stream is read once: no retry.
+ client._send_raw(
+ method, url, source.stream(), headers, timeout, "upload failed"
+ )
+ progress(total)
+ return None
+
+ part_size = upload["partSize"]
+ urls = {part["partNumber"]: part["url"] for part in upload["parts"]}
+
+ def jobs() -> Iterator[Tuple[int, Callable[[], bytes]]]:
+ if source.random_access:
+ for part in upload["parts"]:
+ number = part["partNumber"]
+ start = (number - 1) * part_size
+ end = min(start + part_size, total)
+ yield number, functools.partial(source.read_range, start, end)
+ return
+ stream = source.stream()
+ number = 0
+ sent = 0
+ while True:
+ body = _read_exactly(stream, part_size)
+ if not body:
+ break
+ number += 1
+ sent += len(body)
+ if number not in urls or sent > total:
+ raise ValueError(
+ f"File {file['id']} has more data than the {total} bytes announced"
+ )
+ yield number, functools.partial(bytes, body)
+ if sent != total:
+ raise ValueError(
+ f"File {file['id']} was announced as {total} bytes "
+ f"but the stream had {sent}"
+ )
+
+ def put_part(number: int, read: Callable[[], bytes]) -> Dict[str, Any]:
+ body = read()
+
+ def attempt() -> str:
+ _, headers, _ = client._send_raw(
+ "PUT",
+ urls[number],
+ body,
+ {"Content-Type": "application/octet-stream"},
+ timeout,
+ f"upload of part {number} failed",
+ )
+ etag = _header(headers, "ETag")
+ if not etag:
+ raise RushDBError(
+ f"Part {number} uploaded but the storage returned no ETag header",
+ status=400,
+ )
+ return etag
+
+ etag = _with_retries(retries, attempt)
+ progress(len(body))
+ return {"partNumber": number, "etag": etag}
+
+ done: List[Dict[str, Any]] = []
+ failure: Optional[BaseException] = None
+ running: Set["Future[Dict[str, Any]]"] = set()
+
+ def collect(finished: Set["Future[Dict[str, Any]]"]) -> None:
+ nonlocal failure
+ for future in finished:
+ running.discard(future)
+ error = future.exception()
+ if error is not None:
+ failure = failure or error
+ else:
+ done.append(future.result())
+
+ with ThreadPoolExecutor(max_workers=max(1, parallelism)) as pool:
+ try:
+ for number, read in jobs():
+ running.add(pool.submit(put_part, number, read))
+ if len(running) >= max(1, parallelism):
+ finished, _ = wait(running, return_when=FIRST_COMPLETED)
+ collect(finished)
+ if failure is not None:
+ break
+ except (
+ BaseException
+ ) as error: # the source failed: stop after the parts in flight
+ failure = failure or error
+ finished, _ = wait(running)
+ collect(finished)
+ if failure is not None:
+ raise failure
+ return sorted(done, key=lambda part: part["partNumber"])
diff --git a/src/rushdb/client.py b/src/rushdb/client.py
index 89ac3ff..c0585b6 100755
--- a/src/rushdb/client.py
+++ b/src/rushdb/client.py
@@ -9,9 +9,11 @@
import urllib.error
import urllib.parse
import urllib.request
-from typing import Any, Dict, Optional
+from typing import Any, Dict, Optional, Tuple
from .api.ai import AIAPI
+from .api.connections import ConnectionsAPI
+from .api.imports import ImportsAPI
from .api.labels import LabelsAPI
from .api.properties import PropertiesAPI
from .api.query import QueryAPI
@@ -65,6 +67,9 @@ class RushDB:
properties (PropertiesAPI): API interface for property operations
labels (LabelsAPI): API interface for label operations
tx (TransactionsAPI): API interface for transaction operations
+ imports (ImportsAPI): Background file imports (RushDB Cloud only)
+ connections (ConnectionsAPI): SaaS connections and their syncs
+ (RushDB Cloud only)
settings (_SettingsNamespace): API interface for project settings
Example:
@@ -186,6 +191,8 @@ def __init__(
self.relationships = RelationsAPI(self)
self.ai = AIAPI(self)
self.text_indexes = TextIndexesAPI(self)
+ self.imports = ImportsAPI(self)
+ self.connections = ConnectionsAPI(self)
self.settings = _SettingsNamespace(self)
def _make_request(
@@ -297,6 +304,58 @@ def _make_request(
except json.JSONDecodeError as e:
raise RushDBError(f"Invalid JSON response: {str(e)}")
+ def _send_raw(
+ self,
+ method: str,
+ url: str,
+ body: Any,
+ headers: Optional[Dict[str, str]] = None,
+ timeout: Optional[float] = None,
+ what: str = "request failed",
+ ) -> Tuple[int, Dict[str, str], bytes]:
+ """Send a raw (non-JSON) body to an absolute URL.
+
+ Used for file uploads: presigned storage URLs (no auth header) and the
+ API's direct upload path (the caller adds the auth header). ``body`` is
+ bytes or a binary file object; a file object needs a
+ ``Content-Length`` header.
+
+ Returns:
+ The status code, the response headers and the response body.
+
+ Raises:
+ RushDBError: With ``status`` set on an HTTP error status, and
+ without it on a network error or timeout.
+ """
+ request = urllib.request.Request(
+ url, data=body, headers=dict(headers or {}), method=method
+ )
+ try:
+ with urllib.request.urlopen(request, timeout=timeout) as response:
+ return (
+ response.status,
+ {key: value for key, value in response.headers.items()},
+ response.read(),
+ )
+ except urllib.error.HTTPError as e:
+ # Storage answers XML; the status says enough then.
+ try:
+ error_body = json.loads(e.read().decode("utf-8"))
+ except (json.JSONDecodeError, UnicodeDecodeError, OSError):
+ error_body = {}
+ if not isinstance(error_body, dict):
+ error_body = {}
+ message = error_body.get("message") or error_body.get("error")
+ if isinstance(message, list):
+ message = "; ".join(str(item) for item in message)
+ raise RushDBError(
+ str(message) if message else f"{what} (HTTP {e.code})",
+ error_body,
+ status=e.code,
+ )
+ except (urllib.error.URLError, OSError) as e:
+ raise RushDBError(f"{what}: {e}")
+
def ping(self) -> bool:
"""Test connectivity to the RushDB server.
diff --git a/src/rushdb/direct/capabilities.py b/src/rushdb/direct/capabilities.py
index d0fef88..2733a1f 100644
--- a/src/rushdb/direct/capabilities.py
+++ b/src/rushdb/direct/capabilities.py
@@ -8,6 +8,8 @@
from typing import Dict
_PATTERNS = "relationship-pattern suggestions are produced and stored by the platform"
+_IMPORTS = "file imports run on the platform"
+_CONNECTIONS = "connections and their syncs run on the platform"
CLOUD_ONLY_METHODS: Dict[str, str] = {
"ai.search": "natural-language search turns the prompt into a SQL statement with an LLM",
@@ -22,6 +24,35 @@
"relationships.patterns.ignore": _PATTERNS,
"relationships.patterns.delete": _PATTERNS,
"settings.get": "project settings belong to the platform",
+ "imports.create": _IMPORTS,
+ "imports.add_files": _IMPORTS,
+ "imports.remove_file": _IMPORTS,
+ "imports.upload": _IMPORTS,
+ "imports.preview": _IMPORTS,
+ "imports.start": _IMPORTS,
+ "imports.get": _IMPORTS,
+ "imports.list": _IMPORTS,
+ "imports.wait": _IMPORTS,
+ "imports.cancel": _IMPORTS,
+ "imports.retry": _IMPORTS,
+ "imports.delete": _IMPORTS,
+ "imports.import_files": _IMPORTS,
+ "connections.catalog": _CONNECTIONS,
+ "connections.push_templates": _CONNECTIONS,
+ "connections.list": _CONNECTIONS,
+ "connections.get": _CONNECTIONS,
+ "connections.create": _CONNECTIONS,
+ "connections.update": _CONNECTIONS,
+ "connections.delete": _CONNECTIONS,
+ "connections.test": _CONNECTIONS,
+ "connections.preview": _CONNECTIONS,
+ "connections.fields": _CONNECTIONS,
+ "connections.review_fields": _CONNECTIONS,
+ "connections.sync": _CONNECTIONS,
+ "connections.runs": _CONNECTIONS,
+ "connections.pause": _CONNECTIONS,
+ "connections.resume": _CONNECTIONS,
+ "connections.rotate_webhook_token": _CONNECTIONS,
}
"""Methods that need RushDB Cloud, with the reason."""
diff --git a/src/rushdb/models/ingest.py b/src/rushdb/models/ingest.py
new file mode 100644
index 0000000..defc841
--- /dev/null
+++ b/src/rushdb/models/ingest.py
@@ -0,0 +1,497 @@
+"""Background file imports and SaaS connections (RushDB Cloud).
+
+Both write through import runs: an import run is created for every file
+import and every connection sync, and continues on the server without the
+client. Poll it with ``db.imports.get(id)`` or ``db.imports.wait(id)``.
+
+The types mirror the JSON the API sends and receives, so their keys are
+camelCase.
+"""
+
+from typing import Any, Dict, List, Literal, Optional, Tuple, TypedDict, Union
+
+# ── import runs ──────────────────────────────────────────────────────────────
+
+ImportRunKind = Literal["file", "sync"]
+ImportRunTrigger = Literal["manual", "schedule", "webhook", "api", "retry", "once"]
+ImportRunStatus = Literal[
+ "draft",
+ "queued",
+ "running",
+ "paused_billing",
+ "completed",
+ "partial",
+ "failed",
+ "cancelled",
+]
+
+FINISHED_IMPORT_RUN_STATUSES: Tuple[str, ...] = (
+ "completed",
+ "partial",
+ "failed",
+ "cancelled",
+)
+"""Statuses after which a run no longer changes (until it is retried)."""
+
+
+class _ImportRunPartRequired(TypedDict):
+ status: Literal["pending", "running", "done", "failed", "skipped"]
+ records: int
+
+
+class ImportRunPart(_ImportRunPartRequired, total=False):
+ """Progress of one file or one stream of a run."""
+
+ label: str
+ title: str
+ deleted: int
+ links: int
+ bytes: int
+ totalBytes: int
+ errors: int
+ error: str
+
+
+class ImportRunError(TypedDict, total=False):
+ part: str
+ row: int
+ externalId: str
+ message: str
+
+
+class ImportRunSummary(TypedDict, total=False):
+ """What a run found after it finished."""
+
+ labels: List[str]
+ suggestions: int
+ awaitingEmbeddings: int
+ durationMs: int
+
+
+# ── personal data ────────────────────────────────────────────────────────────
+
+PiiKind = Literal[
+ "email",
+ "phone",
+ "name",
+ "address",
+ "birthdate",
+ "national_id",
+ "tax_id",
+ "payment_card",
+ "bank_account",
+ "ip_address",
+ "credentials",
+ "compensation",
+ "health",
+ "location",
+ "free_text",
+]
+
+
+class PiiFinding(TypedDict):
+ """Why a field or column looks like personal data."""
+
+ kind: PiiKind
+ source: Literal["name", "value"]
+ confidence: float
+ reason: str
+
+
+# ── file imports ─────────────────────────────────────────────────────────────
+
+ImportColumnType = Literal["string", "number", "boolean", "datetime"]
+
+
+class ImportFileMapping(TypedDict, total=False):
+ """How one file maps to the graph."""
+
+ keyColumn: str
+ """Column whose value identifies a row across imports: re-importing upserts."""
+ skip: List[str]
+ hash: List[str]
+ """Columns kept as a keyed hash: joinable across sources, not readable."""
+ rename: Dict[str, str]
+ types: Dict[str, ImportColumnType]
+ delimiter: str
+ """CSV only."""
+
+
+class ImportFileLink(TypedDict):
+ """A link between two files of one import: rows whose ``sourceColumn``
+ equals the target's ``targetColumn``."""
+
+ sourceFileId: str
+ sourceColumn: str
+ targetFileId: str
+ targetColumn: str
+ type: str
+
+
+class ImportFile(TypedDict):
+ id: str
+ name: str
+ format: str
+ compression: Optional[str]
+ size: int
+ status: str
+ label: str
+ mapping: Optional[ImportFileMapping]
+ rowsDone: int
+ bytesDone: int
+ errorCount: int
+ error: Optional[str]
+
+
+class S3PartUrl(TypedDict):
+ partNumber: int
+ url: str
+
+
+class S3MultipartUpload(TypedDict):
+ """Parts go to presigned PUT URLs, without the API auth header."""
+
+ mode: Literal["s3-multipart"]
+ partSize: int
+ parts: List[S3PartUrl]
+ expiresAt: str
+
+
+class DirectUpload(TypedDict):
+ """Through the API: ``PUT`` the raw bytes to ``path`` with the usual auth header."""
+
+ mode: Literal["direct"]
+ method: Literal["PUT"]
+ path: str
+
+
+ImportUploadInstructions = Union[S3MultipartUpload, DirectUpload]
+
+
+class ImportFileUpload(ImportFile):
+ """A file with its upload instructions, as ``create``/``add_files`` return it."""
+
+ upload: ImportUploadInstructions
+
+
+class _ImportRunRequired(TypedDict):
+ id: str
+ kind: ImportRunKind
+ trigger: ImportRunTrigger
+ status: ImportRunStatus
+ name: Optional[str]
+ connectionId: Optional[str]
+ recordsWritten: int
+ recordsDeleted: int
+ linksCreated: int
+ errorCount: int
+ progress: Dict[str, ImportRunPart]
+ """Per file (file imports) or per stream (syncs)."""
+ errorSample: List[ImportRunError]
+ summary: Optional[ImportRunSummary]
+ error: Optional[str]
+ startedBy: Optional[Dict[str, Any]]
+ startedAt: Optional[str]
+ finishedAt: Optional[str]
+ createdAt: str
+ updatedAt: str
+
+
+class ImportRun(_ImportRunRequired, total=False):
+ """One import run (a file import or a connection sync) with its progress."""
+
+ files: List[ImportFile]
+ options: Optional[Dict[str, Any]]
+
+
+class CreatedImport(_ImportRunRequired):
+ """A draft import whose files carry their upload instructions."""
+
+ files: List[ImportFileUpload]
+
+
+class _ImportFileInputRequired(TypedDict):
+ name: str
+ size: int
+
+
+class ImportFileInput(_ImportFileInputRequired, total=False):
+ """A file announced to ``imports.create``: its name (the extension picks
+ the parser) and exact size in bytes."""
+
+ label: str
+
+
+class _ImportFileSettingsRequired(TypedDict):
+ id: str
+
+
+class ImportFileSettings(_ImportFileSettingsRequired, total=False):
+ """Settings of one uploaded file for ``preview`` and ``start``."""
+
+ label: str
+ mapping: ImportFileMapping
+
+
+class ImportColumnProfile(TypedDict, total=False):
+ name: str
+ type: ImportColumnType
+ samples: List[Any]
+ nulls: int
+ distinct: int
+ unique: bool
+ pii: PiiFinding
+ """Looks like personal data; its samples are masked then."""
+ suggestion: Literal["include", "skip", "hash"]
+
+
+class ImportFilePreview(TypedDict, total=False):
+ fileId: str
+ name: str
+ label: str
+ rows: int
+ columns: List[ImportColumnProfile]
+ suggestedKey: str
+ errors: List[str]
+
+
+class ImportLinkSuggestion(ImportFileLink):
+ confidence: float
+ reason: str
+
+
+class ImportPreview(TypedDict):
+ files: List[ImportFilePreview]
+ links: List[ImportLinkSuggestion]
+
+
+class ImportUploadProgress(TypedDict):
+ fileId: str
+ loaded: int
+ """Bytes sent so far for this file."""
+ total: int
+
+
+class ImportFilesLink(TypedDict):
+ """A link for ``imports.import_files``, naming files by position or name."""
+
+ source: Union[int, str]
+ """The source file, by position in the list or by name."""
+ sourceColumn: str
+ target: Union[int, str]
+ targetColumn: str
+ type: str
+
+
+class ImportFilesProgress(TypedDict, total=False):
+ """``{"phase": "upload", "fileId", "loaded", "total"}`` while uploading,
+ then ``{"phase": "run", "run": ImportRun}``."""
+
+ phase: Literal["upload", "run"]
+ fileId: str
+ loaded: int
+ total: int
+ run: ImportRun
+
+
+# ── connections ──────────────────────────────────────────────────────────────
+
+ConnectionMode = Literal[
+ "once", "scheduled", "manual", "webhook_trigger", "webhook_push", "agent"
+]
+ConnectionStatus = Literal["draft", "ready", "paused", "needs_reauth"]
+ConnectionDeleteMode = Literal["soft", "hard", "ignore"]
+
+
+class FieldPolicy(TypedDict, total=False):
+ """Which fields of a stream reach the graph. ``auto`` keeps every field
+ except what is excluded or hashed; ``allowlist`` keeps only reviewed
+ fields (``include``, ``hash``), and new fields wait in ``unreviewed``."""
+
+ mode: Literal["auto", "allowlist"]
+ include: List[str]
+ exclude: List[str]
+ hash: List[str]
+ reviewedAt: str
+ reviewedBy: str
+
+
+class FieldProfile(TypedDict, total=False):
+ """One field of a stream as sampled for the field review."""
+
+ name: str
+ type: Literal["string", "number", "boolean", "datetime", "array", "null"]
+ coverage: float
+ pii: PiiFinding
+ example: str
+ """A masked example (never a raw value of a field that looks personal)."""
+ suggestion: Literal["include", "exclude", "hash"]
+
+
+class ConnectionFieldsReview(TypedDict):
+ """``connections.fields``: the field review of one stream."""
+
+ stream: str
+ label: str
+ sampled: int
+ fields: List[FieldProfile]
+ policy: FieldPolicy
+ unreviewed: List[str]
+
+
+class FieldChoice(TypedDict):
+ """A reviewer's choice for one field, for ``connections.review_fields``."""
+
+ name: str
+ action: Literal["include", "exclude", "hash"]
+
+
+class PushTemplate(TypedDict, total=False):
+ """How Zapier, Make or n8n push records into a webhook (push) connection."""
+
+ id: str
+ name: str
+ description: str
+ docsUrl: str
+ steps: List[str]
+ """Steps in order; ``{{pushUrl}}`` stands for the connection's push URL."""
+ samplePayload: Dict[str, Any]
+ logo: str
+
+
+class ConnectionStream(TypedDict, total=False):
+ stream: str
+ enabled: bool
+ label: str
+ deleteMode: ConnectionDeleteMode
+ policy: FieldPolicy
+ reviewed: bool
+ unreviewed: List[str]
+ records: int
+ lastSyncedAt: Optional[str]
+ cursor: Union[str, int, None]
+ description: str
+ optional: bool
+
+
+class Connection(TypedDict, total=False):
+ id: str
+ name: str
+ connectorId: str
+ specId: Optional[str]
+ connectorName: str
+ status: ConnectionStatus
+ mode: ConnectionMode
+ scheduleMinutes: Optional[int]
+ nextRunAt: Optional[str]
+ config: Dict[str, Any]
+ authType: Optional[str]
+ account: Optional[str]
+ hasCredentials: bool
+ hasWebhookSecret: bool
+ webhookUrl: Optional[str]
+ customWebhookUrl: Optional[str]
+ hasCustomWebhookToken: bool
+ webhookSetup: Optional[str]
+ lastRunAt: Optional[str]
+ lastSuccessAt: Optional[str]
+ lastWebhookAt: Optional[str]
+ lastError: Optional[str]
+ activeRunId: Optional[str]
+ agent: Optional[Dict[str, Any]]
+ """Agent connections: what the agent last reported."""
+ logo: str
+ streams: List[ConnectionStream]
+ createdAt: str
+ updatedAt: str
+
+
+class CatalogConnector(TypedDict, total=False):
+ id: str
+ name: str
+ category: str
+ description: str
+ docsUrl: str
+ auth: List[str]
+ streams: List[Dict[str, Any]]
+ config: Dict[str, Dict[str, Any]]
+ webhooks: bool
+ incremental: bool
+ beta: bool
+ oauthAvailable: bool
+ custom: Dict[str, Any]
+ logo: str
+
+
+class ConnectionStreamSettings(TypedDict, total=False):
+ stream: str
+ enabled: bool
+ label: str
+ deleteMode: ConnectionDeleteMode
+ policy: FieldPolicy
+
+
+class UpdateConnectionParams(TypedDict, total=False):
+ """Fields of a connection that can be set on create and update."""
+
+ name: str
+ config: Dict[str, Union[str, List[str]]]
+ auth: Dict[str, str]
+ """``{"type": "bearer", "token"}``, ``{"type": "apiKey", "apiKey"}``,
+ ``{"type": "basic", "username", "password"}`` or ``{"type": "none"}``."""
+ oauthApp: Dict[str, str]
+ webhookSecret: str
+ mode: ConnectionMode
+ scheduleMinutes: int
+ """5 minutes to 7 days."""
+ streams: List[ConnectionStreamSettings]
+ push: Dict[str, str]
+ """Push connections: ``{"label", "keyField"}``."""
+ agent: Dict[str, int]
+ """Agent connections: ``{"intervalMinutes"}`` (1–1440)."""
+ specId: str
+
+
+class CreateConnectionParams(UpdateConnectionParams, total=False):
+ connectorId: str
+ """A built-in connector id (see ``catalog()``), ``custom`` (with
+ ``specId``), ``webhook`` (push) or ``agent`` (on-prem agent). Required."""
+
+
+class _CreatedConnectionRequired(TypedDict):
+ connection: Connection
+
+
+class CreatedConnection(_CreatedConnectionRequired, total=False):
+ """Tokens are returned once: ``webhookToken`` for push and trigger
+ connections, ``agentToken`` (for ``rushdb-agent``) for agent connections."""
+
+ webhookToken: str
+ agentToken: str
+
+
+class ConnectionTestResult(TypedDict, total=False):
+ ok: bool
+ account: str
+ error: str
+
+
+class ConnectionPreview(TypedDict):
+ stream: str
+ label: str
+ records: List[Dict[str, Any]]
+
+
+class _DeletedConnectionRequired(TypedDict):
+ deleted: bool
+
+
+class DeletedConnection(_DeletedConnectionRequired, total=False):
+ recordsDeleted: int
+
+
+class ConnectionWebhookToken(TypedDict):
+ """The new token with its URL; shown once."""
+
+ token: str
+ url: str
diff --git a/tests/test_connections.py b/tests/test_connections.py
new file mode 100644
index 0000000..f611a1d
--- /dev/null
+++ b/tests/test_connections.py
@@ -0,0 +1,177 @@
+"""db.connections against a mocked HTTP layer."""
+
+import json
+from unittest import mock
+
+import pytest
+
+from rushdb import RushDB
+
+API = "https://api.test/api/v1"
+
+
+class _Resp:
+ def __init__(self, body):
+ self._body = json.dumps(body).encode("utf-8")
+ self.status = 200
+ self.headers = {}
+
+ def read(self):
+ return self._body
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *args):
+ return False
+
+
+@pytest.fixture
+def calls():
+ recorded = []
+
+ def fake_urlopen(request, timeout=None):
+ headers = {key.lower(): value for key, value in request.header_items()}
+ recorded.append(
+ {
+ "method": request.get_method(),
+ "path": request.full_url[len(API) :],
+ "body": (
+ json.loads(request.data.decode("utf-8")) if request.data else None
+ ),
+ "auth": headers.get("authorization"),
+ }
+ )
+ return _Resp({"success": True, "data": {"ok": True}})
+
+ with mock.patch("urllib.request.urlopen", side_effect=fake_urlopen):
+ yield recorded
+
+
+def db():
+ return RushDB("token", url="https://api.test")
+
+
+def test_every_method_maps_to_its_route(calls):
+ connections = db().connections
+
+ connections.catalog()
+ connections.list()
+ connections.get("conn-1")
+ connections.create(
+ {
+ "connectorId": "hubspot",
+ "name": "HubSpot",
+ "auth": {"type": "bearer", "token": "token"},
+ "mode": "scheduled",
+ "scheduleMinutes": 60,
+ "streams": [
+ {"stream": "contacts", "policy": {"mode": "auto", "exclude": ["phone"]}}
+ ],
+ }
+ )
+ connections.update("conn-1", {"scheduleMinutes": 30})
+ connections.delete("conn-1")
+ connections.delete("conn-1", delete_data=True)
+ connections.test("conn-1")
+ connections.preview("conn-1", "contacts", limit=5)
+ connections.sync("conn-1", streams=["contacts"], full=True)
+ connections.sync("conn-1")
+ connections.runs("conn-1", limit=5, skip=10)
+ connections.pause("conn-1")
+ connections.resume("conn-1")
+ connections.rotate_webhook_token("conn-1")
+ connections.push_templates()
+ connections.fields("conn-1", "contacts", limit=100)
+ connections.fields("conn-1", "deals")
+
+ assert [f"{call['method']} {call['path']}" for call in calls] == [
+ "GET /connections/catalog",
+ "GET /connections",
+ "GET /connections/conn-1",
+ "POST /connections",
+ "PATCH /connections/conn-1",
+ "DELETE /connections/conn-1",
+ "DELETE /connections/conn-1?deleteData=true",
+ "POST /connections/conn-1/test",
+ "POST /connections/conn-1/preview",
+ "POST /connections/conn-1/runs",
+ "POST /connections/conn-1/runs",
+ "GET /connections/conn-1/runs?limit=5&skip=10",
+ "POST /connections/conn-1/pause",
+ "POST /connections/conn-1/resume",
+ "POST /connections/conn-1/webhook-token",
+ "GET /connections/push-templates",
+ "POST /connections/conn-1/fields",
+ "POST /connections/conn-1/fields",
+ ]
+ assert all(call["auth"] == "Bearer token" for call in calls)
+ assert calls[3]["body"] == {
+ "connectorId": "hubspot",
+ "name": "HubSpot",
+ "auth": {"type": "bearer", "token": "token"},
+ "mode": "scheduled",
+ "scheduleMinutes": 60,
+ "streams": [
+ {"stream": "contacts", "policy": {"mode": "auto", "exclude": ["phone"]}}
+ ],
+ }
+ assert calls[4]["body"] == {"scheduleMinutes": 30}
+ assert calls[8]["body"] == {"stream": "contacts", "limit": 5}
+ assert calls[9]["body"] == {"streams": ["contacts"], "full": True}
+ # Writes without parameters still send a JSON body.
+ assert calls[10]["body"] == {}
+ assert calls[7]["body"] == {}
+ assert calls[16]["body"] == {"stream": "contacts", "limit": 100}
+ assert calls[17]["body"] == {"stream": "deals"}
+
+
+def test_a_field_review_is_saved_as_an_allowlist_policy(calls):
+ db().connections.review_fields(
+ "conn-1",
+ "contacts",
+ [
+ {"name": "company", "action": "include"},
+ {"name": "email", "action": "hash"},
+ {"name": "phone", "action": "exclude"},
+ {"name": "title", "action": "include"},
+ ],
+ )
+ assert len(calls) == 1
+ assert f"{calls[0]['method']} {calls[0]['path']}" == "PATCH /connections/conn-1"
+ assert calls[0]["body"] == {
+ "streams": [
+ {
+ "stream": "contacts",
+ "policy": {
+ "mode": "allowlist",
+ "include": ["company", "title"],
+ "exclude": ["phone"],
+ "hash": ["email"],
+ },
+ }
+ ]
+ }
+
+
+def test_an_agent_connection_is_created_and_its_token_rotated(calls):
+ connections = db().connections
+ connections.create(
+ {"connectorId": "agent", "name": "Warehouse", "agent": {"intervalMinutes": 15}}
+ )
+ connections.rotate_webhook_token("agent-1")
+ assert [f"{call['method']} {call['path']}" for call in calls] == [
+ "POST /connections",
+ "POST /connections/agent-1/webhook-token",
+ ]
+ assert calls[0]["body"] == {
+ "connectorId": "agent",
+ "name": "Warehouse",
+ "agent": {"intervalMinutes": 15},
+ }
+
+
+def test_responses_are_wrapped(calls):
+ response = db().connections.test("conn-1")
+ assert response.success is True
+ assert response.data == {"ok": True}
diff --git a/tests/test_direct_mode.py b/tests/test_direct_mode.py
index 0f90f60..7cef883 100755
--- a/tests/test_direct_mode.py
+++ b/tests/test_direct_mode.py
@@ -300,6 +300,55 @@ def operations(calls):
"ai.resolve_names": lambda db: db.ai.resolve_names("books"),
"ai.get_join_candidates": lambda db: db.ai.get_join_candidates(),
"settings.get": lambda db: db.settings.get(),
+ "imports.create": lambda db: db.imports.create([{"name": "a.csv", "size": 1}]),
+ "imports.add_files": lambda db: db.imports.add_files(
+ "imp-1", [{"name": "b.csv", "size": 1}]
+ ),
+ "imports.remove_file": lambda db: db.imports.remove_file("imp-1", "f1"),
+ "imports.upload": lambda db: db.imports.upload(
+ "imp-1",
+ {
+ "id": "f1",
+ "size": 1,
+ "upload": {"mode": "direct", "method": "PUT", "path": "/x"},
+ },
+ b"x",
+ ),
+ "imports.preview": lambda db: db.imports.preview("imp-1"),
+ "imports.start": lambda db: db.imports.start("imp-1"),
+ "imports.get": lambda db: db.imports.get("imp-1"),
+ "imports.list": lambda db: db.imports.list(),
+ "imports.wait": lambda db: db.imports.wait("imp-1"),
+ "imports.cancel": lambda db: db.imports.cancel("imp-1"),
+ "imports.retry": lambda db: db.imports.retry("imp-1"),
+ "imports.delete": lambda db: db.imports.delete("imp-1"),
+ "imports.import_files": lambda db: db.imports.import_files(
+ [{"data": b"x", "name": "a.csv"}]
+ ),
+ "connections.catalog": lambda db: db.connections.catalog(),
+ "connections.push_templates": lambda db: db.connections.push_templates(),
+ "connections.list": lambda db: db.connections.list(),
+ "connections.get": lambda db: db.connections.get("conn-1"),
+ "connections.create": lambda db: db.connections.create(
+ {"connectorId": "agent", "name": "Warehouse"}
+ ),
+ "connections.update": lambda db: db.connections.update(
+ "conn-1", {"scheduleMinutes": 30}
+ ),
+ "connections.delete": lambda db: db.connections.delete("conn-1", delete_data=True),
+ "connections.test": lambda db: db.connections.test("conn-1"),
+ "connections.preview": lambda db: db.connections.preview("conn-1", "contacts"),
+ "connections.fields": lambda db: db.connections.fields("conn-1", "contacts"),
+ "connections.review_fields": lambda db: db.connections.review_fields(
+ "conn-1", "contacts", [{"name": "email", "action": "hash"}]
+ ),
+ "connections.sync": lambda db: db.connections.sync("conn-1"),
+ "connections.runs": lambda db: db.connections.runs("conn-1"),
+ "connections.pause": lambda db: db.connections.pause("conn-1"),
+ "connections.resume": lambda db: db.connections.resume("conn-1"),
+ "connections.rotate_webhook_token": lambda db: db.connections.rotate_webhook_token(
+ "conn-1"
+ ),
}
# Answered without the engine (it has no operation for them).
@@ -329,6 +378,8 @@ def walk(obj, path, depth):
"ai",
"text_indexes",
"settings",
+ "imports",
+ "connections",
):
walk(getattr(db, namespace), [namespace], 0)
return found
@@ -358,6 +409,20 @@ def test_method_talks_to_the_engine_only_or_is_refused_as_cloud_only(
assert engine_calls
+def test_imports_and_connections_are_cloud_only_in_direct_mode():
+ for name, method in EVERY_METHOD.items():
+ if name.startswith(("imports.", "connections.")):
+ assert name in CLOUD_ONLY_METHODS, name
+ with mock.patch("urllib.request.urlopen") as urlopen:
+ with pytest.raises(CloudOnlyError) as raised:
+ direct_db().imports.import_files([{"data": b"x", "name": "a.csv"}])
+ assert raised.value.method == "imports.import_files"
+ with pytest.raises(CloudOnlyError) as raised:
+ direct_db().connections.sync("conn-1")
+ assert raised.value.method == "connections.sync"
+ urlopen.assert_not_called()
+
+
def test_direct_mode_needs_an_engine_url():
with pytest.raises(ValueError):
RushDB("ENGINE-SECRET", direct=True)
diff --git a/tests/test_imports.py b/tests/test_imports.py
new file mode 100644
index 0000000..f106497
--- /dev/null
+++ b/tests/test_imports.py
@@ -0,0 +1,583 @@
+"""db.imports against a mocked HTTP layer: S3 multipart and direct uploads,
+waiting on a run, and import_files end to end."""
+
+import io
+import json
+import threading
+import urllib.error
+import urllib.parse
+from unittest import mock
+
+import pytest
+
+from rushdb import RushDB, RushDBError
+
+API = "https://api.test/api/v1"
+S3 = "https://bucket.s3.test/"
+
+
+class _Resp:
+ def __init__(self, status=200, body=b"", headers=None):
+ self.status = status
+ self._body = body
+ self.headers = headers or {}
+
+ def read(self):
+ return self._body
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *args):
+ return False
+
+
+def json_answer(data, total=None):
+ body = {"success": True, "data": data}
+ if total is not None:
+ body["total"] = total
+ return _Resp(200, json.dumps(body).encode("utf-8"))
+
+
+def http_error(url, status, body=b""):
+ return urllib.error.HTTPError(url, status, "error", {}, io.BytesIO(body))
+
+
+class Calls:
+ """Records every request and answers it with ``answer(call, calls)``."""
+
+ def __init__(self, answer):
+ self.answer = answer
+ self.calls = []
+ self.lock = threading.Lock()
+
+ def __call__(self, request, timeout=None):
+ data = request.data
+ if data is not None and hasattr(data, "read"):
+ data = data.read()
+ headers = {key.lower(): value for key, value in request.header_items()}
+ body = data
+ if headers.get("content-type") == "application/json" and data:
+ body = json.loads(data.decode("utf-8"))
+ call = {
+ "url": request.full_url,
+ "method": request.get_method(),
+ "headers": headers,
+ "body": body,
+ "timeout": timeout,
+ }
+ with self.lock:
+ self.calls.append(call)
+ snapshot = list(self.calls)
+ result = self.answer(call, snapshot)
+ if isinstance(result, Exception):
+ raise result
+ return result
+
+ def api(self):
+ return [call for call in self.calls if call["url"].startswith(API)]
+
+ def s3(self):
+ return [call for call in self.calls if call["url"].startswith(S3)]
+
+
+@pytest.fixture
+def http():
+ """Patch urlopen; ``http(answer)`` installs the answer and returns the calls."""
+ patches = []
+
+ def install(answer):
+ calls = Calls(answer)
+ patcher = mock.patch("urllib.request.urlopen", side_effect=calls)
+ patcher.start()
+ patches.append(patcher)
+ return calls
+
+ with mock.patch("rushdb.api.uploads._backoff"):
+ yield install
+ for patcher in patches:
+ patcher.stop()
+
+
+def db():
+ return RushDB("token", url="https://api.test")
+
+
+def run(**patch):
+ base = {
+ "id": "imp-1",
+ "kind": "file",
+ "trigger": "api",
+ "status": "draft",
+ "name": None,
+ "connectionId": None,
+ "recordsWritten": 0,
+ "recordsDeleted": 0,
+ "linksCreated": 0,
+ "errorCount": 0,
+ "progress": {},
+ "errorSample": [],
+ "summary": None,
+ "error": None,
+ "startedBy": None,
+ "startedAt": None,
+ "finishedAt": None,
+ "createdAt": "2026-01-01T00:00:00.000Z",
+ "updatedAt": "2026-01-01T00:00:00.000Z",
+ }
+ base.update(patch)
+ return base
+
+
+def file_dto(file_id, name, size):
+ return {
+ "id": file_id,
+ "name": name,
+ "format": "csv",
+ "compression": None,
+ "size": size,
+ "status": "uploading",
+ "label": "Row",
+ "mapping": None,
+ "rowsDone": 0,
+ "bytesDone": 0,
+ "errorCount": 0,
+ "error": None,
+ }
+
+
+def s3_file(file_id, size, part_size):
+ count = max(1, -(-size // part_size))
+ return {
+ **file_dto(file_id, f"{file_id}.csv", size),
+ "upload": {
+ "mode": "s3-multipart",
+ "partSize": part_size,
+ "expiresAt": "2026-01-01T01:00:00.000Z",
+ "parts": [
+ {
+ "partNumber": n,
+ "url": f"{S3}{file_id}?partNumber={n}&X-Amz-Signature=sig",
+ }
+ for n in range(1, count + 1)
+ ],
+ },
+ }
+
+
+def direct_file(file_id, size):
+ return {
+ **file_dto(file_id, f"{file_id}.csv", size),
+ "upload": {
+ "mode": "direct",
+ "method": "PUT",
+ "path": f"/api/v1/imports/imp-1/files/{file_id}/content",
+ },
+ }
+
+
+def data_of(length):
+ return bytes(index % 251 for index in range(length))
+
+
+def part_number(url):
+ return int(urllib.parse.parse_qs(urllib.parse.urlparse(url).query)["partNumber"][0])
+
+
+def s3_and_api(api, fail_first=None):
+ """Answers part PUTs with an ETag (``fail_first`` fails once with 503)."""
+
+ def answer(call, calls):
+ if call["url"].startswith(S3):
+ number = part_number(call["url"])
+ attempts = len([c for c in calls if c["url"] == call["url"]])
+ if number == fail_first and attempts == 1:
+ return http_error(call["url"], 503, b"SlowDown")
+ return _Resp(200, b"", {"ETag": f'"etag-{number}"'})
+ return api(call)
+
+ return answer
+
+
+# ── s3-multipart ─────────────────────────────────────────────────────────────
+
+
+def test_multipart_puts_every_part_retries_a_failed_one_and_completes(http):
+ file = s3_file("f1", 10, 4)
+
+ def api(call):
+ if call["url"] == f"{API}/imports":
+ return json_answer({**run(), "files": [file]})
+ if call["url"].endswith("/complete"):
+ return json_answer({**file_dto("f1", "f1.csv", 10), "status": "uploaded"})
+ return http_error(call["url"], 404)
+
+ calls = http(s3_and_api(api, fail_first=2))
+ client = db()
+
+ created = client.imports.create([{"name": "f1.csv", "size": 10}])
+ assert calls.calls[0]["method"] == "POST"
+ assert calls.calls[0]["body"] == {"files": [{"name": "f1.csv", "size": 10}]}
+
+ progress = []
+ data = data_of(10)
+ result = client.imports.upload(
+ created.data["id"],
+ "f1",
+ data,
+ parallelism=2,
+ on_progress=lambda p: progress.append(p["loaded"]),
+ )
+
+ puts = calls.s3()
+ assert len(puts) == 4 # three parts, one retried
+ for put in puts:
+ assert put["method"] == "PUT"
+ assert "authorization" not in put["headers"]
+ sizes = {part_number(put["url"]): len(put["body"]) for put in puts}
+ assert sizes == {1: 4, 2: 4, 3: 2}
+ third = next(put for put in puts if part_number(put["url"]) == 3)
+ assert third["body"] == data[8:]
+
+ complete = calls.calls[-1]
+ assert complete["url"] == f"{API}/imports/imp-1/files/f1/complete"
+ assert complete["method"] == "POST"
+ assert complete["headers"]["authorization"] == "Bearer token"
+ assert complete["body"] == {
+ "parts": [
+ {"partNumber": 1, "etag": '"etag-1"'},
+ {"partNumber": 2, "etag": '"etag-2"'},
+ {"partNumber": 3, "etag": '"etag-3"'},
+ ]
+ }
+ assert progress[-1] == 10
+ assert result.data["status"] == "uploaded"
+
+
+def test_multipart_reads_parts_from_a_path(http, tmp_path):
+ path = tmp_path / "f1.csv"
+ data = data_of(9)
+ path.write_bytes(data)
+ calls = http(s3_and_api(lambda call: json_answer(file_dto("f1", "f1.csv", 9))))
+
+ db().imports.upload("imp-1", s3_file("f1", 9, 4), path)
+
+ bodies = {part_number(put["url"]): put["body"] for put in calls.s3()}
+ assert bodies == {1: data[:4], 2: data[4:8], 3: data[8:]}
+ assert len(calls.calls[-1]["body"]["parts"]) == 3
+
+
+def test_multipart_cuts_a_non_seekable_stream_into_parts(http):
+ data = data_of(9)
+
+ class Pipe(io.RawIOBase):
+ """Unseekable, and returns short reads."""
+
+ def __init__(self):
+ self._buffer = io.BytesIO(data)
+
+ def readable(self):
+ return True
+
+ def seekable(self):
+ return False
+
+ def read(self, size=-1):
+ return self._buffer.read(min(size, 3) if size > 0 else size)
+
+ calls = http(s3_and_api(lambda call: json_answer(file_dto("f1", "f1.csv", 9))))
+
+ db().imports.upload("imp-1", s3_file("f1", 9, 4), Pipe(), parallelism=1)
+
+ bodies = sorted((part_number(put["url"]), put["body"]) for put in calls.s3())
+ assert bodies == [(1, data[:4]), (2, data[4:8]), (3, data[8:])]
+
+
+def test_multipart_uploads_a_file_object_from_its_position(http):
+ data = data_of(10)
+ f = io.BytesIO(b"xx" + data)
+ f.read(2)
+ calls = http(s3_and_api(lambda call: json_answer(file_dto("f1", "f1.csv", 10))))
+
+ db().imports.upload("imp-1", s3_file("f1", 10, 4), f)
+
+ bodies = {part_number(put["url"]): put["body"] for put in calls.s3()}
+ assert b"".join(bodies[n] for n in sorted(bodies)) == data
+
+
+def test_a_size_mismatch_fails_before_sending_anything(http):
+ calls = http(s3_and_api(lambda call: json_answer({})))
+ with pytest.raises(ValueError, match="announced as 10 bytes"):
+ db().imports.upload("imp-1", s3_file("f1", 10, 4), data_of(9))
+ assert calls.calls == []
+
+
+def test_a_4xx_part_upload_is_not_retried(http):
+ calls = http(lambda call, calls: http_error(call["url"], 403, b""))
+ with pytest.raises(RushDBError) as raised:
+ db().imports.upload("imp-1", s3_file("f1", 4, 4), data_of(4), retries=3)
+ assert raised.value.status == 403
+ assert len(calls.calls) == 1
+
+
+def test_a_part_failing_every_attempt_stops_the_upload(http):
+ calls = http(lambda call, calls: http_error(call["url"], 500))
+ with pytest.raises(RushDBError):
+ db().imports.upload("imp-1", s3_file("f1", 4, 4), data_of(4), retries=2)
+ assert len(calls.calls) == 2 # no completion call
+
+
+def test_upload_needs_the_file_object_for_files_this_client_did_not_create(http):
+ http(lambda call, calls: json_answer({}))
+ with pytest.raises(ValueError, match="No upload instructions"):
+ db().imports.upload("imp-1", "f1", data_of(1))
+
+
+# ── direct ───────────────────────────────────────────────────────────────────
+
+
+def test_direct_puts_the_raw_bytes_with_the_auth_header_then_completes(http):
+ calls = http(lambda call, calls: json_answer(file_dto("f1", "f1.csv", 5)))
+
+ db().imports.upload("imp-1", direct_file("f1", 5), data_of(5))
+
+ put, complete = calls.calls
+ assert put["url"] == f"{API}/imports/imp-1/files/f1/content"
+ assert put["method"] == "PUT"
+ assert put["headers"]["authorization"] == "Bearer token"
+ assert put["headers"]["content-type"] == "application/octet-stream"
+ assert put["body"] == data_of(5)
+ assert complete["url"] == f"{API}/imports/imp-1/files/f1/complete"
+ assert complete["method"] == "POST"
+ assert complete["body"] == {}
+
+
+def test_direct_streams_a_path_and_retries_a_5xx(http, tmp_path):
+ path = tmp_path / "f1.csv"
+ path.write_bytes(b"a,b\n1,2")
+
+ def answer(call, calls):
+ if call["url"].endswith("/content") and len(calls) == 1:
+ return http_error(call["url"], 502)
+ return json_answer(file_dto("f1", "f1.csv", 7))
+
+ calls = http(answer)
+
+ db().imports.upload("imp-1", direct_file("f1", 7), str(path))
+
+ puts = [call for call in calls.calls if call["url"].endswith("/content")]
+ assert [put["body"] for put in puts] == [b"a,b\n1,2", b"a,b\n1,2"]
+ assert puts[1]["headers"]["content-length"] == "7"
+
+
+# ── runs ─────────────────────────────────────────────────────────────────────
+
+
+def test_wait_polls_until_the_run_finishes(http):
+ statuses = iter(["queued", "running", "completed"])
+ calls = http(lambda call, calls: json_answer(run(status=next(statuses))))
+ seen = []
+
+ result = db().imports.wait(
+ "imp-1", interval=0.001, on_progress=lambda r: seen.append(r["status"])
+ )
+
+ assert result.data["status"] == "completed"
+ assert seen == ["queued", "running", "completed"]
+ assert all(
+ call["method"] == "GET" and call["url"] == f"{API}/imports/imp-1"
+ for call in calls.calls
+ )
+
+
+def test_wait_gives_up_after_the_timeout(http):
+ http(lambda call, calls: json_answer(run(status="running")))
+ with pytest.raises(TimeoutError, match="still running"):
+ db().imports.wait("imp-1", interval=0.005, timeout=0.012)
+
+
+def test_list_filters_and_run_actions_map_to_their_routes(http):
+ calls = http(lambda call, calls: json_answer([], total=0))
+ client = db()
+
+ listed = client.imports.list(kind="sync", connection_id="c 1", limit=10)
+ client.imports.cancel("imp-1")
+ client.imports.retry("imp-1")
+ client.imports.delete("imp-1")
+ client.imports.preview("imp-1", files=[{"id": "f1", "label": "Order"}])
+ client.imports.start("imp-1")
+ client.imports.add_files("imp-1", [{"name": "b.csv", "size": 3}])
+ client.imports.remove_file("imp-1", "f2")
+
+ assert listed.total == 0
+ assert [f"{c['method']} {c['url'][len(API):]}" for c in calls.calls] == [
+ "GET /imports?kind=sync&connectionId=c+1&limit=10",
+ "POST /imports/imp-1/cancel",
+ "POST /imports/imp-1/retry",
+ "DELETE /imports/imp-1",
+ "POST /imports/imp-1/preview",
+ "POST /imports/imp-1/start",
+ "POST /imports/imp-1/files",
+ "DELETE /imports/imp-1/files/f2",
+ ]
+ assert calls.calls[4]["body"] == {"files": [{"id": "f1", "label": "Order"}]}
+ assert calls.calls[5]["body"] == {}
+ assert calls.calls[6]["body"] == {"files": [{"name": "b.csv", "size": 3}]}
+
+
+def test_start_passes_hashed_columns_from_the_preview(http):
+ preview = {
+ "files": [
+ {
+ "fileId": "f1",
+ "name": "customers.csv",
+ "label": "Customer",
+ "rows": 2,
+ "columns": [
+ {"name": "id", "type": "number", "suggestion": "include"},
+ {
+ "name": "email",
+ "type": "string",
+ "samples": ["a***@example.com"],
+ "pii": {
+ "kind": "email",
+ "source": "name",
+ "confidence": 0.9,
+ "reason": "The name suggests an email",
+ },
+ "suggestion": "hash",
+ },
+ ],
+ "errors": [],
+ }
+ ],
+ "links": [],
+ }
+ calls = http(lambda call, calls: json_answer(preview))
+ client = db()
+
+ data = client.imports.preview("imp-1").data
+ email = next(c for c in data["files"][0]["columns"] if c["suggestion"] == "hash")
+ assert email["pii"]["kind"] == "email"
+ client.imports.start(
+ "imp-1",
+ files=[{"id": "f1", "mapping": {"keyColumn": "id", "hash": [email["name"]]}}],
+ )
+ assert calls.calls[1]["body"] == {
+ "files": [{"id": "f1", "mapping": {"keyColumn": "id", "hash": ["email"]}}]
+ }
+
+
+# ── import_files ─────────────────────────────────────────────────────────────
+
+
+def test_import_files_creates_uploads_starts_with_mappings_and_links_then_waits(
+ http, tmp_path
+):
+ customers = tmp_path / "customers.csv"
+ customers.write_bytes(data_of(3))
+ files = [direct_file("f1", 3), direct_file("f2", 4)]
+ polls = iter(["running", "completed"])
+
+ def answer(call, calls):
+ path = call["url"][len(API) :]
+ if call["method"] == "POST" and path == "/imports":
+ return json_answer({**run(), "files": files})
+ if path.endswith("/content") or path.endswith("/complete"):
+ return json_answer(file_dto("f", "f.csv", 1))
+ if path.endswith("/start"):
+ return json_answer(run(status="queued"))
+ if call["method"] == "GET":
+ return json_answer(run(status=next(polls)))
+ return http_error(call["url"], 404)
+
+ calls = http(answer)
+ phases = []
+
+ result = db().imports.import_files(
+ [
+ {"data": customers, "label": "Customer"},
+ {"data": data_of(4), "name": "orders.csv"},
+ ],
+ name="CRM export",
+ mappings={"customers.csv": {"keyColumn": "id"}},
+ links=[
+ {
+ "source": "orders.csv",
+ "sourceColumn": "customer_id",
+ "target": 0,
+ "targetColumn": "id",
+ "type": "PLACED_BY",
+ }
+ ],
+ wait=True,
+ wait_interval=0.001,
+ on_progress=lambda p: phases.append(p["phase"]),
+ )
+
+ assert result.data["status"] == "completed"
+ assert calls.calls[0]["body"] == {
+ "name": "CRM export",
+ "files": [
+ {"name": "customers.csv", "size": 3, "label": "Customer"},
+ {"name": "orders.csv", "size": 4},
+ ],
+ }
+ contents = [c["body"] for c in calls.calls if c["url"].endswith("/content")]
+ assert contents == [data_of(3), data_of(4)]
+ start = next(c for c in calls.calls if c["url"].endswith("/start"))
+ assert start["body"] == {
+ "files": [{"id": "f1", "mapping": {"keyColumn": "id"}}, {"id": "f2"}],
+ "links": [
+ {
+ "sourceFileId": "f2",
+ "sourceColumn": "customer_id",
+ "targetFileId": "f1",
+ "targetColumn": "id",
+ "type": "PLACED_BY",
+ }
+ ],
+ }
+ assert "upload" in phases
+ assert phases[-1] == "run"
+
+
+def test_import_files_takes_paths_and_returns_once_started(http, tmp_path):
+ path = tmp_path / "rows.ndjson"
+ path.write_bytes(b'{"a":1}\n')
+
+ def answer(call, calls):
+ path_ = call["url"][len(API) :]
+ if path_ == "/imports":
+ return json_answer({**run(), "files": [direct_file("f1", 8)]})
+ if path_.endswith("/start"):
+ return json_answer(run(status="queued"))
+ return json_answer(file_dto("f1", "rows.ndjson", 8))
+
+ calls = http(answer)
+
+ result = db().imports.import_files([path], mappings={0: {"keyColumn": "a"}})
+
+ assert result.data["status"] == "queued"
+ assert calls.calls[0]["body"] == {"files": [{"name": "rows.ndjson", "size": 8}]}
+ assert calls.calls[-1]["body"] == {
+ "files": [{"id": "f1", "mapping": {"keyColumn": "a"}}]
+ }
+ assert not any(c["method"] == "GET" for c in calls.calls)
+
+
+def test_import_files_needs_a_size_for_a_stream_and_a_name_for_bytes(http):
+ calls = http(lambda call, calls: json_answer({}))
+
+ class Pipe(io.RawIOBase):
+ def readable(self):
+ return True
+
+ def seekable(self):
+ return False
+
+ with pytest.raises(ValueError, match="pass its size"):
+ db().imports.import_files([{"data": Pipe(), "name": "a.csv"}])
+ with pytest.raises(ValueError, match="needs a name"):
+ db().imports.import_files([{"data": b"x"}])
+ assert calls.calls == []
diff --git a/uv.lock b/uv.lock
index 18cf2b7..bc19fe6 100755
--- a/uv.lock
+++ b/uv.lock
@@ -710,7 +710,7 @@ wheels = [
[[package]]
name = "rushdb"
-version = "3.3.0"
+version = "3.4.0"
source = { editable = "." }
[package.dev-dependencies]