diff --git a/README.md b/README.md index 0ff4d16..15e3b21 100755 --- a/README.md +++ b/README.md @@ -91,6 +91,7 @@ What each offers: | Managed embeddings: managed indexes, `records.vector_search` by text `query` | ✓ | — | | Natural-language search on SQL (`ai.search`, `ai.run_sql`) | ✓ | — | | Relationship-pattern suggestions (`relationships.patterns.*`) and join suggestions (`ai.get_join_candidates`) | ✓ | — | +| Background file imports (`imports.*`) and SaaS connections (`connections.*`) | ✓ | — | | Project settings (`settings.get`), dashboard, team access | ✓ | — | In direct mode a Cloud-only method raises `CloudOnlyError` (with `method` and `reason`) before @@ -241,6 +242,110 @@ db.records.import_csv( --- +## File imports and connections (RushDB Cloud) + +Large files and SaaS data load in the background: the client hands the data over and the run +continues on the server. Every file import and every connection sync is an **import run**; follow it +with `db.imports.get(id)` or `db.imports.wait(id)` until its status is `completed`, `partial`, +`failed` or `cancelled`. + +### Import files + +```python +# One call: create → upload every file → start (→ wait) +run = db.imports.import_files( + [ + {'data': 'customers.csv', 'label': 'Customer'}, # a path: name and size come from it + open('orders.csv.gz', 'rb'), # a binary file object works too + ], + mappings={0: {'keyColumn': 'id'}}, # re-importing upserts by `id` + links=[{'source': 1, 'sourceColumn': 'customer_id', 'target': 0, 'targetColumn': 'id', 'type': 'PLACED_BY'}], + parallelism=4, + wait=True, + wait_interval=2, + on_progress=lambda p: print(f"{p['loaded']}/{p['total']}" if p['phase'] == 'upload' else p['run']['status']), +) +print(run.data['recordsWritten'], run.data['summary']) +``` + +Or step by step, to show a preview before starting: + +```python +import os + +created = db.imports.create([{'name': 'orders.csv', 'size': os.path.getsize('orders.csv')}]).data +db.imports.upload(created['id'], created['files'][0], 'orders.csv', parallelism=4, retries=3) +preview = db.imports.preview(created['id']).data # column types, suggested key and links, personal data +columns = preview['files'][0]['columns'] # each has `suggestion` ('include' | 'skip' | 'hash') and `pii` when it looks personal +mapping = { + 'skip': [c['name'] for c in columns if c['suggestion'] == 'skip'], + 'hash': [c['name'] for c in columns if c['suggestion'] == 'hash'], # keyed hash: joins, not readable +} +if preview['files'][0].get('suggestedKey'): + mapping['keyColumn'] = preview['files'][0]['suggestedKey'] +db.imports.start(created['id'], files=[{'id': created['files'][0]['id'], 'mapping': mapping}]) +finished = db.imports.wait(created['id'], timeout=30 * 60).data # TimeoutError after 30 minutes +``` + +`upload` follows the upload mode the server returns: `s3-multipart` sends parts to their presigned +URLs (`parallelism` at a time on a thread pool, default 4; each failed part retried, `retries` +default 3) and completes with their ETags; `direct` sends the bytes through the API. Upload data can +be a file path (`str` or `Path`), `bytes`/`bytearray`/`memoryview`, or a binary file object (read +from its current position). A non-seekable stream (a pipe, a socket) needs its `size` up front in +`import_files`, and a direct upload of one is not retried. + +Also: `imports.list(kind=, connection_id=, limit=, skip=)`, `cancel`, `retry`, `delete`, +`add_files`, `remove_file`. + +### Connections + +```python +import os + +catalog = db.connections.catalog().data # connectors with their streams and config fields + +created = db.connections.create({ + 'connectorId': 'hubspot', + 'auth': {'type': 'bearer', 'token': os.environ['HUBSPOT_TOKEN']}, + 'mode': 'scheduled', + 'scheduleMinutes': 60, + 'streams': [{'stream': 'contacts', 'policy': {'mode': 'auto', 'exclude': ['mobilephone']}}], +}).data +connection_id = created['connection']['id'] +db.connections.test(connection_id) +db.connections.preview(connection_id, 'contacts') # first records, nothing written + +# Field review: profiles with personal-data findings (masked examples), the policy, unreviewed fields +review = db.connections.fields(connection_id, 'contacts', limit=100).data +db.connections.review_fields( + connection_id, + 'contacts', + [{'name': f['name'], 'action': f['suggestion']} for f in review['fields']], # or your own choices +) + +sync_run = db.connections.sync(connection_id).data # or sync(connection_id, streams=['contacts'], full=True) +db.imports.wait(sync_run['id']) +``` + +A stream's `policy` decides which fields reach the graph: `auto` keeps everything except `exclude` +and `hash`; `review_fields` switches it to an `allowlist` of the fields marked `include` or `hash`, +and fields that appear later wait in the stream's `unreviewed` list until the next review. + +On-prem databases sync through `rushdb-agent`: create an agent connection and hand the token +(shown once) to the agent. + +```python +agent = db.connections.create({'connectorId': 'agent', 'name': 'Warehouse', 'agent': {'intervalMinutes': 15}}).data +print(agent['agentToken']) # connection['agent'] shows what the agent last reported +``` + +Also: `list`, `get`, `update`, `delete(id, delete_data=True)`, `runs(id, limit=, skip=)`, `pause`, +`resume`, `rotate_webhook_token` (a new webhook URL, or a new agent token), `push_templates()` +(Zapier, Make and n8n recipes for push connections). Credentials are sealed on the server and never +returned. + +--- + ## SearchResult `db.records.find()` returns a `SearchResult` — a list-like container with pagination metadata. diff --git a/pyproject.toml b/pyproject.toml index a687121..cb58158 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "rushdb" -version = "3.3.0" +version = "3.4.0" description = "RushDB SDK for Python: the client for RushDB, the database for AI (RushDB Cloud or self-hosted)" authors = [ {name = "RushDB Team", email = "hi@rushdb.com"} diff --git a/src/rushdb/__init__.py b/src/rushdb/__init__.py index 84694d9..4ccbba7 100755 --- a/src/rushdb/__init__.py +++ b/src/rushdb/__init__.py @@ -4,6 +4,8 @@ """ from .api.ai import AIAPI +from .api.connections import ConnectionsAPI +from .api.imports import ImportsAPI from .api.query import QueryAPI from .api.records import pick_record_id from .api.relationship_patterns import RelationshipPatternsAPI @@ -25,6 +27,7 @@ ValueMatch, ValueMatchesData, ) +from .models.ingest import FINISHED_IMPORT_RUN_STATUSES from .models.property import ( Property, PropertyValuesData, @@ -91,4 +94,7 @@ "ApiResponse", "AIAPI", "TextIndexesAPI", + "ImportsAPI", + "ConnectionsAPI", + "FINISHED_IMPORT_RUN_STATUSES", ] diff --git a/src/rushdb/api/__init__.py b/src/rushdb/api/__init__.py index 6c6de94..a07063c 100755 --- a/src/rushdb/api/__init__.py +++ b/src/rushdb/api/__init__.py @@ -1,4 +1,6 @@ from .ai import AIAPI +from .connections import ConnectionsAPI +from .imports import ImportsAPI from .labels import LabelsAPI from .properties import PropertiesAPI from .query import QueryAPI @@ -18,4 +20,6 @@ "RelationsAPI", "RelationshipPatternsAPI", "TextIndexesAPI", + "ImportsAPI", + "ConnectionsAPI", ] diff --git a/src/rushdb/api/connections.py b/src/rushdb/api/connections.py new file mode 100644 index 0000000..d9844a0 --- /dev/null +++ b/src/rushdb/api/connections.py @@ -0,0 +1,258 @@ +"""SaaS connections for the RushDB Python SDK (RushDB Cloud only).""" + +from typing import TYPE_CHECKING, Any, Dict, List, Optional + +from ..models.api_response import ApiResponse +from ..models.ingest import ( + CatalogConnector, + Connection, + ConnectionFieldsReview, + ConnectionPreview, + ConnectionTestResult, + ConnectionWebhookToken, + CreateConnectionParams, + CreatedConnection, + DeletedConnection, + FieldChoice, + ImportRun, + PushTemplate, + UpdateConnectionParams, +) +from .base import BaseAPI +from .imports import _guard, _wrap + +if TYPE_CHECKING: + from ..client import RushDB + + +class ConnectionsAPI(BaseAPI): + """The project's connections to SaaS tools (CRM, billing, support, issue + trackers, docs, spreadsheets), custom APIs, webhooks and on-prem agents. + Every sync is an import run: follow it with ``db.imports.get(run_id)`` or + ``db.imports.wait(run_id)``. + + Accessed via ``db.connections``. **RushDB Cloud only**: in direct mode + every method raises ``CloudOnlyError`` before sending anything. + """ + + def __init__(self, client: "RushDB"): + super().__init__(client) + + def _guard(self, method: str) -> None: + _guard(self.client, "connections", method) + + def catalog(self) -> "ApiResponse[List[CatalogConnector]]": + """Built-in connectors and the workspace's own, with their streams, + auth types and config fields.""" + self._guard("catalog") + return _wrap(self.client._make_request("GET", "/connections/catalog")) + + def push_templates(self) -> "ApiResponse[List[PushTemplate]]": + """Zapier, Make and n8n recipes for pushing records into a webhook + (push) connection.""" + self._guard("push_templates") + return _wrap(self.client._make_request("GET", "/connections/push-templates")) + + def list(self) -> "ApiResponse[List[Connection]]": + """The project's connections with their streams, status and last sync.""" + self._guard("list") + return _wrap(self.client._make_request("GET", "/connections")) + + def get(self, connection_id: str) -> "ApiResponse[Connection]": + """One connection.""" + self._guard("get") + return _wrap(self.client._make_request("GET", f"/connections/{connection_id}")) + + def create( + self, params: CreateConnectionParams + ) -> "ApiResponse[CreatedConnection]": + """Create a connection. + + Credentials are sealed on the server and never returned. A push or + trigger connection's ``webhookToken`` and an agent connection's + (``connectorId: "agent"``) ``agentToken`` are returned once. + + Args: + params: ``{"connectorId", "name"?, "config"?, "auth"?, "oauthApp"?, + "webhookSecret"?, "mode"?, "scheduleMinutes"?, "streams"?, + "push"?, "agent"?, "specId"?}``. + + Returns: + ApiResponse: ``data`` is ``{"connection", "webhookToken"?, "agentToken"?}``. + """ + self._guard("create") + return _wrap(self.client._make_request("POST", "/connections", dict(params))) + + def update( + self, connection_id: str, params: UpdateConnectionParams + ) -> "ApiResponse[Connection]": + """Change name, credentials, mode, schedule or stream settings + (labels, delete mode, field policy).""" + self._guard("update") + return _wrap( + self.client._make_request( + "PATCH", f"/connections/{connection_id}", dict(params) + ) + ) + + def delete( + self, connection_id: str, delete_data: bool = False + ) -> "ApiResponse[DeletedConnection]": + """Delete a connection; ``delete_data=True`` also deletes the records + it wrote.""" + self._guard("delete") + params = {"deleteData": "true"} if delete_data else None + return _wrap( + self.client._make_request( + "DELETE", f"/connections/{connection_id}", params=params + ) + ) + + def test(self, connection_id: str) -> "ApiResponse[ConnectionTestResult]": + """Check the credentials against the provider.""" + self._guard("test") + return _wrap( + self.client._make_request("POST", f"/connections/{connection_id}/test", {}) + ) + + def preview( + self, connection_id: str, stream: str, limit: Optional[int] = None + ) -> "ApiResponse[ConnectionPreview]": + """The first records of a stream as they would be written (up to 50). + Writes nothing.""" + self._guard("preview") + body: Dict[str, Any] = {"stream": stream} + if limit is not None: + body["limit"] = limit + return _wrap( + self.client._make_request( + "POST", f"/connections/{connection_id}/preview", body + ) + ) + + def fields( + self, connection_id: str, stream: str, limit: Optional[int] = None + ) -> "ApiResponse[ConnectionFieldsReview]": + """The field review of one stream: sampled field profiles with + personal-data findings (values of fields that look personal are + masked), the stream's current policy and the fields held back since + the last review. Writes nothing. + + Args: + stream: The stream name. + limit: Records to sample, 1–200. + """ + self._guard("fields") + body: Dict[str, Any] = {"stream": stream} + if limit is not None: + body["limit"] = limit + return _wrap( + self.client._make_request( + "POST", f"/connections/{connection_id}/fields", body + ) + ) + + def review_fields( + self, connection_id: str, stream: str, choices: List[FieldChoice] + ) -> "ApiResponse[Connection]": + """Save a field review: the stream switches to an allowlist of the + fields marked ``include`` or ``hash`` (hashed ones are kept as a keyed + hash); ``exclude``d fields are left out, and new fields wait for the + next review. + + Args: + choices: ``[{"name": "email", "action": "include" | "exclude" | "hash"}]``. + """ + self._guard("review_fields") + + def pick(action: str) -> List[str]: + return [choice["name"] for choice in choices if choice["action"] == action] + + body = { + "streams": [ + { + "stream": stream, + "policy": { + "mode": "allowlist", + "include": pick("include"), + "exclude": pick("exclude"), + "hash": pick("hash"), + }, + } + ] + } + return _wrap( + self.client._make_request("PATCH", f"/connections/{connection_id}", body) + ) + + def sync( + self, + connection_id: str, + streams: Optional[List[str]] = None, + full: Optional[bool] = None, + ) -> "ApiResponse[ImportRun]": + """Sync now and return the queued run. + + Args: + streams: Stream names; every enabled stream when omitted. + full: Ignore stored cursors and read everything again. + """ + self._guard("sync") + body: Dict[str, Any] = {} + if streams is not None: + body["streams"] = streams + if full is not None: + body["full"] = full + return _wrap( + self.client._make_request( + "POST", f"/connections/{connection_id}/runs", body + ) + ) + + def runs( + self, + connection_id: str, + limit: Optional[int] = None, + skip: Optional[int] = None, + ) -> "ApiResponse[List[ImportRun]]": + """The connection's sync runs, newest first; ``total`` is the count.""" + self._guard("runs") + params = { + key: value + for key, value in (("limit", limit), ("skip", skip)) + if value is not None + } + return _wrap( + self.client._make_request( + "GET", f"/connections/{connection_id}/runs", params=params or None + ) + ) + + def pause(self, connection_id: str) -> "ApiResponse[Connection]": + """Stop scheduled and webhook-triggered syncs until resumed.""" + self._guard("pause") + return _wrap( + self.client._make_request("POST", f"/connections/{connection_id}/pause", {}) + ) + + def resume(self, connection_id: str) -> "ApiResponse[Connection]": + """Resume scheduled and webhook-triggered syncs.""" + self._guard("resume") + return _wrap( + self.client._make_request( + "POST", f"/connections/{connection_id}/resume", {} + ) + ) + + def rotate_webhook_token( + self, connection_id: str + ) -> "ApiResponse[ConnectionWebhookToken]": + """A new token, shown once; the previous one stops working. Trigger and + push connections get a new webhook URL, agent connections a new agent + token.""" + self._guard("rotate_webhook_token") + return _wrap( + self.client._make_request( + "POST", f"/connections/{connection_id}/webhook-token", {} + ) + ) diff --git a/src/rushdb/api/imports.py b/src/rushdb/api/imports.py new file mode 100644 index 0000000..f7213e7 --- /dev/null +++ b/src/rushdb/api/imports.py @@ -0,0 +1,492 @@ +"""Background file imports for the RushDB Python SDK (RushDB Cloud only).""" + +import os +import threading +import time +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + List, + Mapping, + Optional, + Sequence, + Union, + cast, +) + +from ..common import CloudOnlyError +from ..direct.capabilities import CLOUD_ONLY_METHODS +from ..models.api_response import ApiResponse +from ..models.ingest import ( + FINISHED_IMPORT_RUN_STATUSES, + CreatedImport, + ImportFile, + ImportFileInput, + ImportFileLink, + ImportFileMapping, + ImportFileSettings, + ImportFilesLink, + ImportFilesProgress, + ImportFileUpload, + ImportPreview, + ImportRun, + ImportRunKind, +) +from .base import BaseAPI +from .uploads import ( + DEFAULT_UPLOAD_PARALLELISM, + DEFAULT_UPLOAD_RETRIES, + DEFAULT_UPLOAD_TIMEOUT, + ProgressCallback, + UploadData, + source_of, + upload_file_content, +) + +if TYPE_CHECKING: + from ..client import RushDB + +ImportFileSource = Union[str, "os.PathLike[str]", Any, Dict[str, Any]] +"""A file for ``imports.import_files``: a path, a binary file object, or a dict +``{"data", "name"?, "size"?, "label"?, "mapping"?}`` (``data`` is a path, bytes +or a binary file object).""" + + +def _wrap(response: Dict[str, Any]) -> ApiResponse: + return ApiResponse( + data=response.get("data"), + success=response.get("success", True), + total=response.get("total"), + ) + + +def _guard(client: "RushDB", namespace: str, method: str) -> None: + if getattr(client, "direct", False) is True: + name = f"{namespace}.{method}" + raise CloudOnlyError(name, CLOUD_ONLY_METHODS[name]) + + +class ImportsAPI(BaseAPI): + """Background file imports (CSV, JSON, NDJSON, Parquet, optionally + gzip-compressed). Create an import with its files, upload them, preview the + mapping, start; the run continues on the server without the client. + + Accessed via ``db.imports``. **RushDB Cloud only**: in direct mode every + method raises ``CloudOnlyError`` before sending anything. + + Example:: + + >>> run = db.imports.import_files( + ... ["customers.csv", "orders.csv.gz"], + ... mappings={0: {"keyColumn": "id"}}, + ... wait=True, + ... ) + >>> run.data["status"], run.data["recordsWritten"] + """ + + def __init__(self, client: "RushDB"): + super().__init__(client) + # Upload instructions from `create`/`add_files`, so `upload` can take ids. + self._uploads: Dict[str, ImportFileUpload] = {} + self._uploads_lock = threading.Lock() + + def _guard(self, method: str) -> None: + _guard(self.client, "imports", method) + + def _remember(self, import_id: str, files: Sequence[ImportFileUpload]) -> None: + with self._uploads_lock: + for file in files or []: + self._uploads[f"{import_id}/{file['id']}"] = file + + def _forget(self, import_id: str, file_id: str) -> None: + with self._uploads_lock: + self._uploads.pop(f"{import_id}/{file_id}", None) + + def create( + self, files: List[ImportFileInput], name: Optional[str] = None + ) -> "ApiResponse[CreatedImport]": + """Create a draft import with its files (name and exact size in bytes; + up to 50). Each returned file carries its ``upload`` instructions. + + Args: + files: ``[{"name": "orders.csv", "size": 1024, "label"?: "Order"}]``. + The extension picks the parser. + name: A name for the import. + """ + self._guard("create") + body: Dict[str, Any] = {"files": files} + if name is not None: + body["name"] = name + response = _wrap(self.client._make_request("POST", "/imports", body)) + if isinstance(response.data, dict): + self._remember(response.data["id"], response.data.get("files") or []) + return response + + def add_files( + self, import_id: str, files: List[ImportFileInput] + ) -> "ApiResponse[List[ImportFileUpload]]": + """Add files to a draft import.""" + self._guard("add_files") + response = _wrap( + self.client._make_request( + "POST", f"/imports/{import_id}/files", {"files": files} + ) + ) + if isinstance(response.data, list): + self._remember(import_id, response.data) + return response + + def remove_file(self, import_id: str, file_id: str) -> ApiResponse: + """Remove a file from a draft import. ``data`` is ``{"deleted": True}``.""" + self._guard("remove_file") + self._forget(import_id, file_id) + return _wrap( + self.client._make_request("DELETE", f"/imports/{import_id}/files/{file_id}") + ) + + def upload( + self, + import_id: str, + file: Union[str, ImportFileUpload], + data: UploadData, + *, + parallelism: int = DEFAULT_UPLOAD_PARALLELISM, + retries: int = DEFAULT_UPLOAD_RETRIES, + timeout: float = DEFAULT_UPLOAD_TIMEOUT, + on_progress: Optional[ProgressCallback] = None, + ) -> "ApiResponse[ImportFile]": + """Upload one file's bytes and complete it. + + With ``s3-multipart`` upload instructions, parts go to their presigned + URLs ``parallelism`` at a time and their ETags are sent on completion; + with ``direct``, the bytes go through the API. Failed requests (network + errors, 429, 5xx) are retried; a non-seekable stream sent ``direct`` is + read once and not retried. + + Args: + import_id: The import's id. + file: A file from ``create``/``add_files`` (with its ``upload`` + instructions), or its id when this client created it. + data: A file path (``str`` or ``Path``), ``bytes``/``bytearray``/ + ``memoryview``, or a binary file object. Its size must match + the size announced for the file. + parallelism: Parts uploaded at once (``s3-multipart``). Default 4. + retries: Attempts per part (or per direct upload). Default 3. + timeout: Per-request timeout in seconds. Default 600. + on_progress: Called with ``{"fileId", "loaded", "total"}`` as + bytes are sent (from upload threads). + + Returns: + ApiResponse: The completed file. + """ + self._guard("upload") + if isinstance(file, str): + with self._uploads_lock: + target = self._uploads.get(f"{import_id}/{file}") + if target is None: + raise ValueError( + f"No upload instructions for file {file}: pass the file " + "returned by imports.create() or imports.add_files()" + ) + else: + target = file + parts = upload_file_content( + self.client, + target, + data, + parallelism=parallelism, + retries=retries, + timeout=timeout, + on_progress=on_progress, + ) + response = _wrap( + self.client._make_request( + "POST", + f"/imports/{import_id}/files/{target['id']}/complete", + {"parts": parts} if parts is not None else {}, + ) + ) + self._forget(import_id, target["id"]) + return response + + def preview( + self, + import_id: str, + files: Optional[List[ImportFileSettings]] = None, + ) -> "ApiResponse[ImportPreview]": + """Column types, samples, personal-data hints, a suggested key per file + and suggested links between files. Writes nothing. + + Args: + files: Optional settings per file (``{"id", "label"?, "mapping"?}``). + """ + self._guard("preview") + body: Dict[str, Any] = {} if files is None else {"files": files} + return _wrap( + self.client._make_request("POST", f"/imports/{import_id}/preview", body) + ) + + def start( + self, + import_id: str, + files: Optional[List[ImportFileSettings]] = None, + links: Optional[List[ImportFileLink]] = None, + ) -> "ApiResponse[ImportRun]": + """Start the run with the files' labels and mappings and the links + between files. + + Args: + files: Settings per file: ``{"id", "label"?, "mapping"?}``, where a + mapping is ``{"keyColumn", "skip", "hash", "rename", "types", + "delimiter"}``. + links: ``{"sourceFileId", "sourceColumn", "targetFileId", + "targetColumn", "type"}``. + """ + self._guard("start") + body: Dict[str, Any] = {} + if files is not None: + body["files"] = files + if links: + body["links"] = links + return _wrap( + self.client._make_request("POST", f"/imports/{import_id}/start", body) + ) + + def get(self, import_id: str) -> "ApiResponse[ImportRun]": + """One import run (a file import or a connection sync) with its progress.""" + self._guard("get") + return _wrap(self.client._make_request("GET", f"/imports/{import_id}")) + + def list( + self, + kind: Optional[ImportRunKind] = None, + connection_id: Optional[str] = None, + limit: Optional[int] = None, + skip: Optional[int] = None, + ) -> "ApiResponse[List[ImportRun]]": + """Import runs of the project, newest first; ``total`` is the count. + + Args: + kind: ``"file"`` or ``"sync"``. + connection_id: Only the syncs of this connection. + limit: 1–100. + skip: Runs to skip. + """ + self._guard("list") + params: Dict[str, Any] = { + key: value + for key, value in ( + ("kind", kind), + ("connectionId", connection_id), + ("limit", limit), + ("skip", skip), + ) + if value is not None + } + return _wrap( + self.client._make_request("GET", "/imports", params=params or None) + ) + + def cancel(self, import_id: str) -> "ApiResponse[ImportRun]": + """Ask a queued or running run to stop; it ends ``cancelled``.""" + self._guard("cancel") + return _wrap( + self.client._make_request("POST", f"/imports/{import_id}/cancel", {}) + ) + + def retry(self, import_id: str) -> "ApiResponse[ImportRun]": + """Run a failed or partial file import again.""" + self._guard("retry") + return _wrap( + self.client._make_request("POST", f"/imports/{import_id}/retry", {}) + ) + + def delete(self, import_id: str) -> ApiResponse: + """Delete an import and its uploaded files (records already written stay).""" + self._guard("delete") + return _wrap(self.client._make_request("DELETE", f"/imports/{import_id}")) + + def wait( + self, + import_id: str, + interval: float = 2.0, + timeout: Optional[float] = None, + on_progress: Optional[Callable[[ImportRun], None]] = None, + ) -> "ApiResponse[ImportRun]": + """Poll a run until it is ``completed``, ``partial``, ``failed`` or + ``cancelled``, and return it. + + Args: + interval: Seconds between polls. Default 2. + timeout: Give up after this many seconds. Default: no limit. + on_progress: Called with the run after every poll. + + Raises: + TimeoutError: The run had not finished within ``timeout``. + """ + self._guard("wait") + interval = max(0.0, interval) + deadline = None if timeout is None else time.monotonic() + timeout + while True: + response = self.get(import_id) + run = cast(ImportRun, response.data) + if on_progress is not None: + on_progress(run) + if run["status"] in FINISHED_IMPORT_RUN_STATUSES: + return response + if deadline is not None and time.monotonic() + interval > deadline: + raise TimeoutError( + f"Import {import_id} is still {run['status']} after {timeout} s" + ) + time.sleep(interval) + + def import_files( + self, + files: Sequence[ImportFileSource], + *, + name: Optional[str] = None, + links: Optional[List[ImportFilesLink]] = None, + mappings: Optional[Mapping[Union[str, int], ImportFileMapping]] = None, + wait: bool = False, + wait_interval: float = 2.0, + wait_timeout: Optional[float] = None, + parallelism: int = DEFAULT_UPLOAD_PARALLELISM, + retries: int = DEFAULT_UPLOAD_RETRIES, + timeout: float = DEFAULT_UPLOAD_TIMEOUT, + on_progress: Optional[Callable[[ImportFilesProgress], None]] = None, + ) -> "ApiResponse[ImportRun]": + """Import files in one call: create → upload every file → start + (→ wait when ``wait=True``). + + Args: + files: Each a file path, a binary file object, or a dict + ``{"data", "name"?, "size"?, "label"?, "mapping"?}``. Names + and sizes come from the path, the file object or the bytes + when not given; a non-seekable stream needs ``size``. The + name's extension picks the parser. + name: A name for the import. + links: Links between files, naming them by position or name: + ``{"source", "sourceColumn", "target", "targetColumn", "type"}``. + mappings: Mappings by file name or position, in addition to + ``mapping`` on each file. + wait: Wait for the run to finish (default: return once started). + wait_interval: Seconds between polls while waiting. + wait_timeout: Give up waiting after this many seconds + (``TimeoutError``). + parallelism, retries, timeout: As for ``upload``. + on_progress: Called with ``{"phase": "upload", "fileId", "loaded", + "total"}`` while uploading, then ``{"phase": "run", "run"}``. + + Returns: + ApiResponse: The started run, or the finished one with ``wait``. + """ + self._guard("import_files") + sources: List[Dict[str, Any]] = [] + for index, file in enumerate(files): + entry: Dict[str, Any] = ( + dict(file) if isinstance(file, dict) else {"data": file} + ) + data = entry.get("data") + file_name = entry.get("name") + if file_name is None: + if isinstance(data, (str, os.PathLike)): + file_name = os.path.basename(os.fspath(data)) + elif isinstance(getattr(data, "name", None), str): + file_name = os.path.basename(data.name) # type: ignore[union-attr] + if not isinstance(file_name, str) or not file_name: + raise ValueError( + f"File {index} needs a name (its extension picks the parser)" + ) + size = entry.get("size") + if size is None: + size = source_of(data).size # type: ignore[arg-type] + if not isinstance(size, int): + raise ValueError( + f"File {index} ({file_name}) is a stream: pass its size in bytes" + ) + entry["name"] = file_name + entry["size"] = size + sources.append(entry) + + announced: List[ImportFileInput] = [] + for source in sources: + item: ImportFileInput = {"name": source["name"], "size": source["size"]} + if source.get("label"): + item["label"] = source["label"] + announced.append(item) + created = self.create(announced, name=name) + run = created.data + run_files = run["files"] + + def file_id_of(ref: Union[int, str]) -> str: + position = ( + ref + if isinstance(ref, int) + else next( + (i for i, source in enumerate(sources) if source["name"] == ref), + -1, + ) + ) + if not 0 <= position < len(run_files): + raise ValueError(f"No file {ref!r} in this import") + return run_files[position]["id"] + + resolved_links: List[ImportFileLink] = [] + for link in links or []: + resolved_links.append( + { + "sourceFileId": file_id_of(link["source"]), + "sourceColumn": link["sourceColumn"], + "targetFileId": file_id_of(link["target"]), + "targetColumn": link["targetColumn"], + "type": link["type"], + } + ) + + def upload_progress(progress: Any) -> None: + if on_progress is not None: + on_progress({"phase": "upload", **progress}) + + for index, source in enumerate(sources): + self.upload( + run["id"], + run_files[index], + source["data"], + parallelism=parallelism, + retries=retries, + timeout=timeout, + on_progress=upload_progress, + ) + + settings: List[ImportFileSettings] = [] + for index, source in enumerate(sources): + mapping = source.get("mapping") + if mapping is None and mappings: + mapping = mappings.get(source["name"]) + if mapping is None: + mapping = mappings.get(index) + if mapping is None: + mapping = mappings.get(str(index)) + setting: ImportFileSettings = {"id": run_files[index]["id"]} + if mapping: + setting["mapping"] = mapping + settings.append(setting) + + started = self.start(run["id"], files=settings, links=resolved_links or None) + if on_progress is not None: + on_progress({"phase": "run", "run": started.data}) + if not wait: + return started + + def run_progress(current: ImportRun) -> None: + if on_progress is not None: + on_progress({"phase": "run", "run": current}) + + return self.wait( + run["id"], + interval=wait_interval, + timeout=wait_timeout, + on_progress=run_progress, + ) diff --git a/src/rushdb/api/uploads.py b/src/rushdb/api/uploads.py new file mode 100644 index 0000000..df17a15 --- /dev/null +++ b/src/rushdb/api/uploads.py @@ -0,0 +1,333 @@ +"""Sending an import file's bytes as the server's upload instructions say. + +``s3-multipart`` puts parts to presigned URLs (no API auth header) a few at a +time and returns their ETags for the completion call; ``direct`` puts the +bytes to the API with the usual auth header. +""" + +import functools +import os +import threading +import time +import urllib.parse +from concurrent.futures import FIRST_COMPLETED, Future, ThreadPoolExecutor, wait +from pathlib import Path +from typing import ( + IO, + TYPE_CHECKING, + Any, + Callable, + Dict, + Iterator, + List, + Optional, + Set, + Tuple, + TypeVar, + Union, +) + +from ..common import RushDBError +from ..models.ingest import ImportFileUpload, ImportUploadProgress + +if TYPE_CHECKING: + from ..client import RushDB + +UploadData = Union[str, "os.PathLike[str]", bytes, bytearray, memoryview, IO[bytes]] +"""File contents for an upload: a file path, bytes, or a binary file object.""" + +ProgressCallback = Callable[[ImportUploadProgress], None] + +DEFAULT_UPLOAD_PARALLELISM = 4 +DEFAULT_UPLOAD_RETRIES = 3 +DEFAULT_UPLOAD_TIMEOUT = 600.0 +"""Per-request timeout in seconds.""" + +T = TypeVar("T") + + +def _backoff(attempt: int) -> None: + time.sleep(min(8.0, 0.25 * 2**attempt)) + + +def _read_exactly(stream: IO[bytes], size: int) -> bytes: + """Up to ``size`` bytes; fewer only at the end of the stream.""" + chunks: List[bytes] = [] + remaining = size + while remaining > 0: + chunk = stream.read(remaining) + if not chunk: + break + if not isinstance(chunk, (bytes, bytearray)): + raise TypeError("Upload file objects must be opened in binary mode ('rb')") + chunks.append(bytes(chunk)) + remaining -= len(chunk) + return b"".join(chunks) + + +class _Source: + """Upload data: random access (can be re-read for a retry) or a stream.""" + + size: Optional[int] = None + random_access = True + + def read_range(self, start: int, end: int) -> bytes: + raise NotImplementedError + + def stream(self) -> IO[bytes]: + raise NotImplementedError + + +class _BytesSource(_Source): + def __init__(self, data: Union[bytes, bytearray, memoryview]): + self._view = memoryview(data).cast("B") + self.size = self._view.nbytes + + def read_range(self, start: int, end: int) -> bytes: + return bytes(self._view[start:end]) + + +class _PathSource(_Source): + def __init__(self, path: Union[str, "os.PathLike[str]"]): + self._path = Path(path) + self.size = self._path.stat().st_size + + def read_range(self, start: int, end: int) -> bytes: + with open(self._path, "rb") as f: + f.seek(start) + return _read_exactly(f, end - start) + + +class _FileSource(_Source): + def __init__(self, f: IO[bytes]): + self._file = f + self._lock = threading.Lock() + seekable = False + try: + seekable = bool(f.seekable()) + except (AttributeError, OSError, ValueError): + pass + if seekable: + self._start = f.tell() + end = f.seek(0, os.SEEK_END) + f.seek(self._start) + self.size = end - self._start + else: + self._start = 0 + self.size = None + self.random_access = False + + def read_range(self, start: int, end: int) -> bytes: + with self._lock: + self._file.seek(self._start + start) + return _read_exactly(self._file, end - start) + + def stream(self) -> IO[bytes]: + return self._file + + +def size_of_file_object(f: Any) -> Optional[int]: + """Bytes left in a seekable file object, or ``None``.""" + return _FileSource(f).size + + +def source_of(data: UploadData) -> _Source: + if isinstance(data, (bytes, bytearray, memoryview)): + return _BytesSource(data) + if isinstance(data, (str, os.PathLike)): + return _PathSource(data) + if hasattr(data, "read"): + return _FileSource(data) + raise TypeError( + "Unsupported upload data: pass a file path, bytes or a binary file object" + ) + + +def _is_retryable(error: BaseException) -> bool: + if not isinstance(error, RushDBError): + return False + status = error.status + return status is None or status == 429 or status >= 500 + + +def _with_retries(attempts: int, run: Callable[[], T]) -> T: + attempts = max(1, attempts) + for attempt in range(attempts): + try: + return run() + except RushDBError as error: + if not _is_retryable(error) or attempt == attempts - 1: + raise + _backoff(attempt) + raise AssertionError("unreachable") + + +def _header(headers: Dict[str, str], name: str) -> Optional[str]: + for key, value in headers.items(): + if key.lower() == name.lower(): + return value + return None + + +def upload_file_content( + client: "RushDB", + file: ImportFileUpload, + data: UploadData, + parallelism: int = DEFAULT_UPLOAD_PARALLELISM, + retries: int = DEFAULT_UPLOAD_RETRIES, + timeout: float = DEFAULT_UPLOAD_TIMEOUT, + on_progress: Optional[ProgressCallback] = None, +) -> Optional[List[Dict[str, Any]]]: + """Sends one file's bytes as its upload instructions say. + + Returns the parts with their ETags for ``s3-multipart``, ``None`` for + ``direct``. + """ + source = source_of(data) + total = file["size"] + if source.size is not None and source.size != total: + raise ValueError( + f"File {file['id']} was announced as {total} bytes " + f"but the data has {source.size}" + ) + + progress_lock = threading.Lock() + loaded = 0 + + def progress(sent: int) -> None: + nonlocal loaded + with progress_lock: + loaded += sent + if on_progress is not None: + on_progress({"fileId": file["id"], "loaded": loaded, "total": total}) + + upload = file["upload"] + if upload["mode"] == "direct": + url = urllib.parse.urljoin(client.base_url + "/", upload["path"]) + headers = { + "Authorization": f"Bearer {client.api_key}", + "Content-Type": "application/octet-stream", + "Content-Length": str(total), + } + method = upload.get("method") or "PUT" + + if source.random_access: + + def send_all() -> None: + if isinstance(source, _PathSource): + with open(source._path, "rb") as f: + client._send_raw( + method, url, f, headers, timeout, "upload failed" + ) + elif isinstance(source, _FileSource): + with source._lock: + source._file.seek(source._start) + client._send_raw( + method, url, source._file, headers, timeout, "upload failed" + ) + else: + client._send_raw( + method, + url, + source.read_range(0, total), + headers, + timeout, + "upload failed", + ) + + _with_retries(retries, send_all) + else: + # A stream is read once: no retry. + client._send_raw( + method, url, source.stream(), headers, timeout, "upload failed" + ) + progress(total) + return None + + part_size = upload["partSize"] + urls = {part["partNumber"]: part["url"] for part in upload["parts"]} + + def jobs() -> Iterator[Tuple[int, Callable[[], bytes]]]: + if source.random_access: + for part in upload["parts"]: + number = part["partNumber"] + start = (number - 1) * part_size + end = min(start + part_size, total) + yield number, functools.partial(source.read_range, start, end) + return + stream = source.stream() + number = 0 + sent = 0 + while True: + body = _read_exactly(stream, part_size) + if not body: + break + number += 1 + sent += len(body) + if number not in urls or sent > total: + raise ValueError( + f"File {file['id']} has more data than the {total} bytes announced" + ) + yield number, functools.partial(bytes, body) + if sent != total: + raise ValueError( + f"File {file['id']} was announced as {total} bytes " + f"but the stream had {sent}" + ) + + def put_part(number: int, read: Callable[[], bytes]) -> Dict[str, Any]: + body = read() + + def attempt() -> str: + _, headers, _ = client._send_raw( + "PUT", + urls[number], + body, + {"Content-Type": "application/octet-stream"}, + timeout, + f"upload of part {number} failed", + ) + etag = _header(headers, "ETag") + if not etag: + raise RushDBError( + f"Part {number} uploaded but the storage returned no ETag header", + status=400, + ) + return etag + + etag = _with_retries(retries, attempt) + progress(len(body)) + return {"partNumber": number, "etag": etag} + + done: List[Dict[str, Any]] = [] + failure: Optional[BaseException] = None + running: Set["Future[Dict[str, Any]]"] = set() + + def collect(finished: Set["Future[Dict[str, Any]]"]) -> None: + nonlocal failure + for future in finished: + running.discard(future) + error = future.exception() + if error is not None: + failure = failure or error + else: + done.append(future.result()) + + with ThreadPoolExecutor(max_workers=max(1, parallelism)) as pool: + try: + for number, read in jobs(): + running.add(pool.submit(put_part, number, read)) + if len(running) >= max(1, parallelism): + finished, _ = wait(running, return_when=FIRST_COMPLETED) + collect(finished) + if failure is not None: + break + except ( + BaseException + ) as error: # the source failed: stop after the parts in flight + failure = failure or error + finished, _ = wait(running) + collect(finished) + if failure is not None: + raise failure + return sorted(done, key=lambda part: part["partNumber"]) diff --git a/src/rushdb/client.py b/src/rushdb/client.py index 89ac3ff..c0585b6 100755 --- a/src/rushdb/client.py +++ b/src/rushdb/client.py @@ -9,9 +9,11 @@ import urllib.error import urllib.parse import urllib.request -from typing import Any, Dict, Optional +from typing import Any, Dict, Optional, Tuple from .api.ai import AIAPI +from .api.connections import ConnectionsAPI +from .api.imports import ImportsAPI from .api.labels import LabelsAPI from .api.properties import PropertiesAPI from .api.query import QueryAPI @@ -65,6 +67,9 @@ class RushDB: properties (PropertiesAPI): API interface for property operations labels (LabelsAPI): API interface for label operations tx (TransactionsAPI): API interface for transaction operations + imports (ImportsAPI): Background file imports (RushDB Cloud only) + connections (ConnectionsAPI): SaaS connections and their syncs + (RushDB Cloud only) settings (_SettingsNamespace): API interface for project settings Example: @@ -186,6 +191,8 @@ def __init__( self.relationships = RelationsAPI(self) self.ai = AIAPI(self) self.text_indexes = TextIndexesAPI(self) + self.imports = ImportsAPI(self) + self.connections = ConnectionsAPI(self) self.settings = _SettingsNamespace(self) def _make_request( @@ -297,6 +304,58 @@ def _make_request( except json.JSONDecodeError as e: raise RushDBError(f"Invalid JSON response: {str(e)}") + def _send_raw( + self, + method: str, + url: str, + body: Any, + headers: Optional[Dict[str, str]] = None, + timeout: Optional[float] = None, + what: str = "request failed", + ) -> Tuple[int, Dict[str, str], bytes]: + """Send a raw (non-JSON) body to an absolute URL. + + Used for file uploads: presigned storage URLs (no auth header) and the + API's direct upload path (the caller adds the auth header). ``body`` is + bytes or a binary file object; a file object needs a + ``Content-Length`` header. + + Returns: + The status code, the response headers and the response body. + + Raises: + RushDBError: With ``status`` set on an HTTP error status, and + without it on a network error or timeout. + """ + request = urllib.request.Request( + url, data=body, headers=dict(headers or {}), method=method + ) + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + return ( + response.status, + {key: value for key, value in response.headers.items()}, + response.read(), + ) + except urllib.error.HTTPError as e: + # Storage answers XML; the status says enough then. + try: + error_body = json.loads(e.read().decode("utf-8")) + except (json.JSONDecodeError, UnicodeDecodeError, OSError): + error_body = {} + if not isinstance(error_body, dict): + error_body = {} + message = error_body.get("message") or error_body.get("error") + if isinstance(message, list): + message = "; ".join(str(item) for item in message) + raise RushDBError( + str(message) if message else f"{what} (HTTP {e.code})", + error_body, + status=e.code, + ) + except (urllib.error.URLError, OSError) as e: + raise RushDBError(f"{what}: {e}") + def ping(self) -> bool: """Test connectivity to the RushDB server. diff --git a/src/rushdb/direct/capabilities.py b/src/rushdb/direct/capabilities.py index d0fef88..2733a1f 100644 --- a/src/rushdb/direct/capabilities.py +++ b/src/rushdb/direct/capabilities.py @@ -8,6 +8,8 @@ from typing import Dict _PATTERNS = "relationship-pattern suggestions are produced and stored by the platform" +_IMPORTS = "file imports run on the platform" +_CONNECTIONS = "connections and their syncs run on the platform" CLOUD_ONLY_METHODS: Dict[str, str] = { "ai.search": "natural-language search turns the prompt into a SQL statement with an LLM", @@ -22,6 +24,35 @@ "relationships.patterns.ignore": _PATTERNS, "relationships.patterns.delete": _PATTERNS, "settings.get": "project settings belong to the platform", + "imports.create": _IMPORTS, + "imports.add_files": _IMPORTS, + "imports.remove_file": _IMPORTS, + "imports.upload": _IMPORTS, + "imports.preview": _IMPORTS, + "imports.start": _IMPORTS, + "imports.get": _IMPORTS, + "imports.list": _IMPORTS, + "imports.wait": _IMPORTS, + "imports.cancel": _IMPORTS, + "imports.retry": _IMPORTS, + "imports.delete": _IMPORTS, + "imports.import_files": _IMPORTS, + "connections.catalog": _CONNECTIONS, + "connections.push_templates": _CONNECTIONS, + "connections.list": _CONNECTIONS, + "connections.get": _CONNECTIONS, + "connections.create": _CONNECTIONS, + "connections.update": _CONNECTIONS, + "connections.delete": _CONNECTIONS, + "connections.test": _CONNECTIONS, + "connections.preview": _CONNECTIONS, + "connections.fields": _CONNECTIONS, + "connections.review_fields": _CONNECTIONS, + "connections.sync": _CONNECTIONS, + "connections.runs": _CONNECTIONS, + "connections.pause": _CONNECTIONS, + "connections.resume": _CONNECTIONS, + "connections.rotate_webhook_token": _CONNECTIONS, } """Methods that need RushDB Cloud, with the reason.""" diff --git a/src/rushdb/models/ingest.py b/src/rushdb/models/ingest.py new file mode 100644 index 0000000..defc841 --- /dev/null +++ b/src/rushdb/models/ingest.py @@ -0,0 +1,497 @@ +"""Background file imports and SaaS connections (RushDB Cloud). + +Both write through import runs: an import run is created for every file +import and every connection sync, and continues on the server without the +client. Poll it with ``db.imports.get(id)`` or ``db.imports.wait(id)``. + +The types mirror the JSON the API sends and receives, so their keys are +camelCase. +""" + +from typing import Any, Dict, List, Literal, Optional, Tuple, TypedDict, Union + +# ── import runs ────────────────────────────────────────────────────────────── + +ImportRunKind = Literal["file", "sync"] +ImportRunTrigger = Literal["manual", "schedule", "webhook", "api", "retry", "once"] +ImportRunStatus = Literal[ + "draft", + "queued", + "running", + "paused_billing", + "completed", + "partial", + "failed", + "cancelled", +] + +FINISHED_IMPORT_RUN_STATUSES: Tuple[str, ...] = ( + "completed", + "partial", + "failed", + "cancelled", +) +"""Statuses after which a run no longer changes (until it is retried).""" + + +class _ImportRunPartRequired(TypedDict): + status: Literal["pending", "running", "done", "failed", "skipped"] + records: int + + +class ImportRunPart(_ImportRunPartRequired, total=False): + """Progress of one file or one stream of a run.""" + + label: str + title: str + deleted: int + links: int + bytes: int + totalBytes: int + errors: int + error: str + + +class ImportRunError(TypedDict, total=False): + part: str + row: int + externalId: str + message: str + + +class ImportRunSummary(TypedDict, total=False): + """What a run found after it finished.""" + + labels: List[str] + suggestions: int + awaitingEmbeddings: int + durationMs: int + + +# ── personal data ──────────────────────────────────────────────────────────── + +PiiKind = Literal[ + "email", + "phone", + "name", + "address", + "birthdate", + "national_id", + "tax_id", + "payment_card", + "bank_account", + "ip_address", + "credentials", + "compensation", + "health", + "location", + "free_text", +] + + +class PiiFinding(TypedDict): + """Why a field or column looks like personal data.""" + + kind: PiiKind + source: Literal["name", "value"] + confidence: float + reason: str + + +# ── file imports ───────────────────────────────────────────────────────────── + +ImportColumnType = Literal["string", "number", "boolean", "datetime"] + + +class ImportFileMapping(TypedDict, total=False): + """How one file maps to the graph.""" + + keyColumn: str + """Column whose value identifies a row across imports: re-importing upserts.""" + skip: List[str] + hash: List[str] + """Columns kept as a keyed hash: joinable across sources, not readable.""" + rename: Dict[str, str] + types: Dict[str, ImportColumnType] + delimiter: str + """CSV only.""" + + +class ImportFileLink(TypedDict): + """A link between two files of one import: rows whose ``sourceColumn`` + equals the target's ``targetColumn``.""" + + sourceFileId: str + sourceColumn: str + targetFileId: str + targetColumn: str + type: str + + +class ImportFile(TypedDict): + id: str + name: str + format: str + compression: Optional[str] + size: int + status: str + label: str + mapping: Optional[ImportFileMapping] + rowsDone: int + bytesDone: int + errorCount: int + error: Optional[str] + + +class S3PartUrl(TypedDict): + partNumber: int + url: str + + +class S3MultipartUpload(TypedDict): + """Parts go to presigned PUT URLs, without the API auth header.""" + + mode: Literal["s3-multipart"] + partSize: int + parts: List[S3PartUrl] + expiresAt: str + + +class DirectUpload(TypedDict): + """Through the API: ``PUT`` the raw bytes to ``path`` with the usual auth header.""" + + mode: Literal["direct"] + method: Literal["PUT"] + path: str + + +ImportUploadInstructions = Union[S3MultipartUpload, DirectUpload] + + +class ImportFileUpload(ImportFile): + """A file with its upload instructions, as ``create``/``add_files`` return it.""" + + upload: ImportUploadInstructions + + +class _ImportRunRequired(TypedDict): + id: str + kind: ImportRunKind + trigger: ImportRunTrigger + status: ImportRunStatus + name: Optional[str] + connectionId: Optional[str] + recordsWritten: int + recordsDeleted: int + linksCreated: int + errorCount: int + progress: Dict[str, ImportRunPart] + """Per file (file imports) or per stream (syncs).""" + errorSample: List[ImportRunError] + summary: Optional[ImportRunSummary] + error: Optional[str] + startedBy: Optional[Dict[str, Any]] + startedAt: Optional[str] + finishedAt: Optional[str] + createdAt: str + updatedAt: str + + +class ImportRun(_ImportRunRequired, total=False): + """One import run (a file import or a connection sync) with its progress.""" + + files: List[ImportFile] + options: Optional[Dict[str, Any]] + + +class CreatedImport(_ImportRunRequired): + """A draft import whose files carry their upload instructions.""" + + files: List[ImportFileUpload] + + +class _ImportFileInputRequired(TypedDict): + name: str + size: int + + +class ImportFileInput(_ImportFileInputRequired, total=False): + """A file announced to ``imports.create``: its name (the extension picks + the parser) and exact size in bytes.""" + + label: str + + +class _ImportFileSettingsRequired(TypedDict): + id: str + + +class ImportFileSettings(_ImportFileSettingsRequired, total=False): + """Settings of one uploaded file for ``preview`` and ``start``.""" + + label: str + mapping: ImportFileMapping + + +class ImportColumnProfile(TypedDict, total=False): + name: str + type: ImportColumnType + samples: List[Any] + nulls: int + distinct: int + unique: bool + pii: PiiFinding + """Looks like personal data; its samples are masked then.""" + suggestion: Literal["include", "skip", "hash"] + + +class ImportFilePreview(TypedDict, total=False): + fileId: str + name: str + label: str + rows: int + columns: List[ImportColumnProfile] + suggestedKey: str + errors: List[str] + + +class ImportLinkSuggestion(ImportFileLink): + confidence: float + reason: str + + +class ImportPreview(TypedDict): + files: List[ImportFilePreview] + links: List[ImportLinkSuggestion] + + +class ImportUploadProgress(TypedDict): + fileId: str + loaded: int + """Bytes sent so far for this file.""" + total: int + + +class ImportFilesLink(TypedDict): + """A link for ``imports.import_files``, naming files by position or name.""" + + source: Union[int, str] + """The source file, by position in the list or by name.""" + sourceColumn: str + target: Union[int, str] + targetColumn: str + type: str + + +class ImportFilesProgress(TypedDict, total=False): + """``{"phase": "upload", "fileId", "loaded", "total"}`` while uploading, + then ``{"phase": "run", "run": ImportRun}``.""" + + phase: Literal["upload", "run"] + fileId: str + loaded: int + total: int + run: ImportRun + + +# ── connections ────────────────────────────────────────────────────────────── + +ConnectionMode = Literal[ + "once", "scheduled", "manual", "webhook_trigger", "webhook_push", "agent" +] +ConnectionStatus = Literal["draft", "ready", "paused", "needs_reauth"] +ConnectionDeleteMode = Literal["soft", "hard", "ignore"] + + +class FieldPolicy(TypedDict, total=False): + """Which fields of a stream reach the graph. ``auto`` keeps every field + except what is excluded or hashed; ``allowlist`` keeps only reviewed + fields (``include``, ``hash``), and new fields wait in ``unreviewed``.""" + + mode: Literal["auto", "allowlist"] + include: List[str] + exclude: List[str] + hash: List[str] + reviewedAt: str + reviewedBy: str + + +class FieldProfile(TypedDict, total=False): + """One field of a stream as sampled for the field review.""" + + name: str + type: Literal["string", "number", "boolean", "datetime", "array", "null"] + coverage: float + pii: PiiFinding + example: str + """A masked example (never a raw value of a field that looks personal).""" + suggestion: Literal["include", "exclude", "hash"] + + +class ConnectionFieldsReview(TypedDict): + """``connections.fields``: the field review of one stream.""" + + stream: str + label: str + sampled: int + fields: List[FieldProfile] + policy: FieldPolicy + unreviewed: List[str] + + +class FieldChoice(TypedDict): + """A reviewer's choice for one field, for ``connections.review_fields``.""" + + name: str + action: Literal["include", "exclude", "hash"] + + +class PushTemplate(TypedDict, total=False): + """How Zapier, Make or n8n push records into a webhook (push) connection.""" + + id: str + name: str + description: str + docsUrl: str + steps: List[str] + """Steps in order; ``{{pushUrl}}`` stands for the connection's push URL.""" + samplePayload: Dict[str, Any] + logo: str + + +class ConnectionStream(TypedDict, total=False): + stream: str + enabled: bool + label: str + deleteMode: ConnectionDeleteMode + policy: FieldPolicy + reviewed: bool + unreviewed: List[str] + records: int + lastSyncedAt: Optional[str] + cursor: Union[str, int, None] + description: str + optional: bool + + +class Connection(TypedDict, total=False): + id: str + name: str + connectorId: str + specId: Optional[str] + connectorName: str + status: ConnectionStatus + mode: ConnectionMode + scheduleMinutes: Optional[int] + nextRunAt: Optional[str] + config: Dict[str, Any] + authType: Optional[str] + account: Optional[str] + hasCredentials: bool + hasWebhookSecret: bool + webhookUrl: Optional[str] + customWebhookUrl: Optional[str] + hasCustomWebhookToken: bool + webhookSetup: Optional[str] + lastRunAt: Optional[str] + lastSuccessAt: Optional[str] + lastWebhookAt: Optional[str] + lastError: Optional[str] + activeRunId: Optional[str] + agent: Optional[Dict[str, Any]] + """Agent connections: what the agent last reported.""" + logo: str + streams: List[ConnectionStream] + createdAt: str + updatedAt: str + + +class CatalogConnector(TypedDict, total=False): + id: str + name: str + category: str + description: str + docsUrl: str + auth: List[str] + streams: List[Dict[str, Any]] + config: Dict[str, Dict[str, Any]] + webhooks: bool + incremental: bool + beta: bool + oauthAvailable: bool + custom: Dict[str, Any] + logo: str + + +class ConnectionStreamSettings(TypedDict, total=False): + stream: str + enabled: bool + label: str + deleteMode: ConnectionDeleteMode + policy: FieldPolicy + + +class UpdateConnectionParams(TypedDict, total=False): + """Fields of a connection that can be set on create and update.""" + + name: str + config: Dict[str, Union[str, List[str]]] + auth: Dict[str, str] + """``{"type": "bearer", "token"}``, ``{"type": "apiKey", "apiKey"}``, + ``{"type": "basic", "username", "password"}`` or ``{"type": "none"}``.""" + oauthApp: Dict[str, str] + webhookSecret: str + mode: ConnectionMode + scheduleMinutes: int + """5 minutes to 7 days.""" + streams: List[ConnectionStreamSettings] + push: Dict[str, str] + """Push connections: ``{"label", "keyField"}``.""" + agent: Dict[str, int] + """Agent connections: ``{"intervalMinutes"}`` (1–1440).""" + specId: str + + +class CreateConnectionParams(UpdateConnectionParams, total=False): + connectorId: str + """A built-in connector id (see ``catalog()``), ``custom`` (with + ``specId``), ``webhook`` (push) or ``agent`` (on-prem agent). Required.""" + + +class _CreatedConnectionRequired(TypedDict): + connection: Connection + + +class CreatedConnection(_CreatedConnectionRequired, total=False): + """Tokens are returned once: ``webhookToken`` for push and trigger + connections, ``agentToken`` (for ``rushdb-agent``) for agent connections.""" + + webhookToken: str + agentToken: str + + +class ConnectionTestResult(TypedDict, total=False): + ok: bool + account: str + error: str + + +class ConnectionPreview(TypedDict): + stream: str + label: str + records: List[Dict[str, Any]] + + +class _DeletedConnectionRequired(TypedDict): + deleted: bool + + +class DeletedConnection(_DeletedConnectionRequired, total=False): + recordsDeleted: int + + +class ConnectionWebhookToken(TypedDict): + """The new token with its URL; shown once.""" + + token: str + url: str diff --git a/tests/test_connections.py b/tests/test_connections.py new file mode 100644 index 0000000..f611a1d --- /dev/null +++ b/tests/test_connections.py @@ -0,0 +1,177 @@ +"""db.connections against a mocked HTTP layer.""" + +import json +from unittest import mock + +import pytest + +from rushdb import RushDB + +API = "https://api.test/api/v1" + + +class _Resp: + def __init__(self, body): + self._body = json.dumps(body).encode("utf-8") + self.status = 200 + self.headers = {} + + def read(self): + return self._body + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + +@pytest.fixture +def calls(): + recorded = [] + + def fake_urlopen(request, timeout=None): + headers = {key.lower(): value for key, value in request.header_items()} + recorded.append( + { + "method": request.get_method(), + "path": request.full_url[len(API) :], + "body": ( + json.loads(request.data.decode("utf-8")) if request.data else None + ), + "auth": headers.get("authorization"), + } + ) + return _Resp({"success": True, "data": {"ok": True}}) + + with mock.patch("urllib.request.urlopen", side_effect=fake_urlopen): + yield recorded + + +def db(): + return RushDB("token", url="https://api.test") + + +def test_every_method_maps_to_its_route(calls): + connections = db().connections + + connections.catalog() + connections.list() + connections.get("conn-1") + connections.create( + { + "connectorId": "hubspot", + "name": "HubSpot", + "auth": {"type": "bearer", "token": "token"}, + "mode": "scheduled", + "scheduleMinutes": 60, + "streams": [ + {"stream": "contacts", "policy": {"mode": "auto", "exclude": ["phone"]}} + ], + } + ) + connections.update("conn-1", {"scheduleMinutes": 30}) + connections.delete("conn-1") + connections.delete("conn-1", delete_data=True) + connections.test("conn-1") + connections.preview("conn-1", "contacts", limit=5) + connections.sync("conn-1", streams=["contacts"], full=True) + connections.sync("conn-1") + connections.runs("conn-1", limit=5, skip=10) + connections.pause("conn-1") + connections.resume("conn-1") + connections.rotate_webhook_token("conn-1") + connections.push_templates() + connections.fields("conn-1", "contacts", limit=100) + connections.fields("conn-1", "deals") + + assert [f"{call['method']} {call['path']}" for call in calls] == [ + "GET /connections/catalog", + "GET /connections", + "GET /connections/conn-1", + "POST /connections", + "PATCH /connections/conn-1", + "DELETE /connections/conn-1", + "DELETE /connections/conn-1?deleteData=true", + "POST /connections/conn-1/test", + "POST /connections/conn-1/preview", + "POST /connections/conn-1/runs", + "POST /connections/conn-1/runs", + "GET /connections/conn-1/runs?limit=5&skip=10", + "POST /connections/conn-1/pause", + "POST /connections/conn-1/resume", + "POST /connections/conn-1/webhook-token", + "GET /connections/push-templates", + "POST /connections/conn-1/fields", + "POST /connections/conn-1/fields", + ] + assert all(call["auth"] == "Bearer token" for call in calls) + assert calls[3]["body"] == { + "connectorId": "hubspot", + "name": "HubSpot", + "auth": {"type": "bearer", "token": "token"}, + "mode": "scheduled", + "scheduleMinutes": 60, + "streams": [ + {"stream": "contacts", "policy": {"mode": "auto", "exclude": ["phone"]}} + ], + } + assert calls[4]["body"] == {"scheduleMinutes": 30} + assert calls[8]["body"] == {"stream": "contacts", "limit": 5} + assert calls[9]["body"] == {"streams": ["contacts"], "full": True} + # Writes without parameters still send a JSON body. + assert calls[10]["body"] == {} + assert calls[7]["body"] == {} + assert calls[16]["body"] == {"stream": "contacts", "limit": 100} + assert calls[17]["body"] == {"stream": "deals"} + + +def test_a_field_review_is_saved_as_an_allowlist_policy(calls): + db().connections.review_fields( + "conn-1", + "contacts", + [ + {"name": "company", "action": "include"}, + {"name": "email", "action": "hash"}, + {"name": "phone", "action": "exclude"}, + {"name": "title", "action": "include"}, + ], + ) + assert len(calls) == 1 + assert f"{calls[0]['method']} {calls[0]['path']}" == "PATCH /connections/conn-1" + assert calls[0]["body"] == { + "streams": [ + { + "stream": "contacts", + "policy": { + "mode": "allowlist", + "include": ["company", "title"], + "exclude": ["phone"], + "hash": ["email"], + }, + } + ] + } + + +def test_an_agent_connection_is_created_and_its_token_rotated(calls): + connections = db().connections + connections.create( + {"connectorId": "agent", "name": "Warehouse", "agent": {"intervalMinutes": 15}} + ) + connections.rotate_webhook_token("agent-1") + assert [f"{call['method']} {call['path']}" for call in calls] == [ + "POST /connections", + "POST /connections/agent-1/webhook-token", + ] + assert calls[0]["body"] == { + "connectorId": "agent", + "name": "Warehouse", + "agent": {"intervalMinutes": 15}, + } + + +def test_responses_are_wrapped(calls): + response = db().connections.test("conn-1") + assert response.success is True + assert response.data == {"ok": True} diff --git a/tests/test_direct_mode.py b/tests/test_direct_mode.py index 0f90f60..7cef883 100755 --- a/tests/test_direct_mode.py +++ b/tests/test_direct_mode.py @@ -300,6 +300,55 @@ def operations(calls): "ai.resolve_names": lambda db: db.ai.resolve_names("books"), "ai.get_join_candidates": lambda db: db.ai.get_join_candidates(), "settings.get": lambda db: db.settings.get(), + "imports.create": lambda db: db.imports.create([{"name": "a.csv", "size": 1}]), + "imports.add_files": lambda db: db.imports.add_files( + "imp-1", [{"name": "b.csv", "size": 1}] + ), + "imports.remove_file": lambda db: db.imports.remove_file("imp-1", "f1"), + "imports.upload": lambda db: db.imports.upload( + "imp-1", + { + "id": "f1", + "size": 1, + "upload": {"mode": "direct", "method": "PUT", "path": "/x"}, + }, + b"x", + ), + "imports.preview": lambda db: db.imports.preview("imp-1"), + "imports.start": lambda db: db.imports.start("imp-1"), + "imports.get": lambda db: db.imports.get("imp-1"), + "imports.list": lambda db: db.imports.list(), + "imports.wait": lambda db: db.imports.wait("imp-1"), + "imports.cancel": lambda db: db.imports.cancel("imp-1"), + "imports.retry": lambda db: db.imports.retry("imp-1"), + "imports.delete": lambda db: db.imports.delete("imp-1"), + "imports.import_files": lambda db: db.imports.import_files( + [{"data": b"x", "name": "a.csv"}] + ), + "connections.catalog": lambda db: db.connections.catalog(), + "connections.push_templates": lambda db: db.connections.push_templates(), + "connections.list": lambda db: db.connections.list(), + "connections.get": lambda db: db.connections.get("conn-1"), + "connections.create": lambda db: db.connections.create( + {"connectorId": "agent", "name": "Warehouse"} + ), + "connections.update": lambda db: db.connections.update( + "conn-1", {"scheduleMinutes": 30} + ), + "connections.delete": lambda db: db.connections.delete("conn-1", delete_data=True), + "connections.test": lambda db: db.connections.test("conn-1"), + "connections.preview": lambda db: db.connections.preview("conn-1", "contacts"), + "connections.fields": lambda db: db.connections.fields("conn-1", "contacts"), + "connections.review_fields": lambda db: db.connections.review_fields( + "conn-1", "contacts", [{"name": "email", "action": "hash"}] + ), + "connections.sync": lambda db: db.connections.sync("conn-1"), + "connections.runs": lambda db: db.connections.runs("conn-1"), + "connections.pause": lambda db: db.connections.pause("conn-1"), + "connections.resume": lambda db: db.connections.resume("conn-1"), + "connections.rotate_webhook_token": lambda db: db.connections.rotate_webhook_token( + "conn-1" + ), } # Answered without the engine (it has no operation for them). @@ -329,6 +378,8 @@ def walk(obj, path, depth): "ai", "text_indexes", "settings", + "imports", + "connections", ): walk(getattr(db, namespace), [namespace], 0) return found @@ -358,6 +409,20 @@ def test_method_talks_to_the_engine_only_or_is_refused_as_cloud_only( assert engine_calls +def test_imports_and_connections_are_cloud_only_in_direct_mode(): + for name, method in EVERY_METHOD.items(): + if name.startswith(("imports.", "connections.")): + assert name in CLOUD_ONLY_METHODS, name + with mock.patch("urllib.request.urlopen") as urlopen: + with pytest.raises(CloudOnlyError) as raised: + direct_db().imports.import_files([{"data": b"x", "name": "a.csv"}]) + assert raised.value.method == "imports.import_files" + with pytest.raises(CloudOnlyError) as raised: + direct_db().connections.sync("conn-1") + assert raised.value.method == "connections.sync" + urlopen.assert_not_called() + + def test_direct_mode_needs_an_engine_url(): with pytest.raises(ValueError): RushDB("ENGINE-SECRET", direct=True) diff --git a/tests/test_imports.py b/tests/test_imports.py new file mode 100644 index 0000000..f106497 --- /dev/null +++ b/tests/test_imports.py @@ -0,0 +1,583 @@ +"""db.imports against a mocked HTTP layer: S3 multipart and direct uploads, +waiting on a run, and import_files end to end.""" + +import io +import json +import threading +import urllib.error +import urllib.parse +from unittest import mock + +import pytest + +from rushdb import RushDB, RushDBError + +API = "https://api.test/api/v1" +S3 = "https://bucket.s3.test/" + + +class _Resp: + def __init__(self, status=200, body=b"", headers=None): + self.status = status + self._body = body + self.headers = headers or {} + + def read(self): + return self._body + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + +def json_answer(data, total=None): + body = {"success": True, "data": data} + if total is not None: + body["total"] = total + return _Resp(200, json.dumps(body).encode("utf-8")) + + +def http_error(url, status, body=b""): + return urllib.error.HTTPError(url, status, "error", {}, io.BytesIO(body)) + + +class Calls: + """Records every request and answers it with ``answer(call, calls)``.""" + + def __init__(self, answer): + self.answer = answer + self.calls = [] + self.lock = threading.Lock() + + def __call__(self, request, timeout=None): + data = request.data + if data is not None and hasattr(data, "read"): + data = data.read() + headers = {key.lower(): value for key, value in request.header_items()} + body = data + if headers.get("content-type") == "application/json" and data: + body = json.loads(data.decode("utf-8")) + call = { + "url": request.full_url, + "method": request.get_method(), + "headers": headers, + "body": body, + "timeout": timeout, + } + with self.lock: + self.calls.append(call) + snapshot = list(self.calls) + result = self.answer(call, snapshot) + if isinstance(result, Exception): + raise result + return result + + def api(self): + return [call for call in self.calls if call["url"].startswith(API)] + + def s3(self): + return [call for call in self.calls if call["url"].startswith(S3)] + + +@pytest.fixture +def http(): + """Patch urlopen; ``http(answer)`` installs the answer and returns the calls.""" + patches = [] + + def install(answer): + calls = Calls(answer) + patcher = mock.patch("urllib.request.urlopen", side_effect=calls) + patcher.start() + patches.append(patcher) + return calls + + with mock.patch("rushdb.api.uploads._backoff"): + yield install + for patcher in patches: + patcher.stop() + + +def db(): + return RushDB("token", url="https://api.test") + + +def run(**patch): + base = { + "id": "imp-1", + "kind": "file", + "trigger": "api", + "status": "draft", + "name": None, + "connectionId": None, + "recordsWritten": 0, + "recordsDeleted": 0, + "linksCreated": 0, + "errorCount": 0, + "progress": {}, + "errorSample": [], + "summary": None, + "error": None, + "startedBy": None, + "startedAt": None, + "finishedAt": None, + "createdAt": "2026-01-01T00:00:00.000Z", + "updatedAt": "2026-01-01T00:00:00.000Z", + } + base.update(patch) + return base + + +def file_dto(file_id, name, size): + return { + "id": file_id, + "name": name, + "format": "csv", + "compression": None, + "size": size, + "status": "uploading", + "label": "Row", + "mapping": None, + "rowsDone": 0, + "bytesDone": 0, + "errorCount": 0, + "error": None, + } + + +def s3_file(file_id, size, part_size): + count = max(1, -(-size // part_size)) + return { + **file_dto(file_id, f"{file_id}.csv", size), + "upload": { + "mode": "s3-multipart", + "partSize": part_size, + "expiresAt": "2026-01-01T01:00:00.000Z", + "parts": [ + { + "partNumber": n, + "url": f"{S3}{file_id}?partNumber={n}&X-Amz-Signature=sig", + } + for n in range(1, count + 1) + ], + }, + } + + +def direct_file(file_id, size): + return { + **file_dto(file_id, f"{file_id}.csv", size), + "upload": { + "mode": "direct", + "method": "PUT", + "path": f"/api/v1/imports/imp-1/files/{file_id}/content", + }, + } + + +def data_of(length): + return bytes(index % 251 for index in range(length)) + + +def part_number(url): + return int(urllib.parse.parse_qs(urllib.parse.urlparse(url).query)["partNumber"][0]) + + +def s3_and_api(api, fail_first=None): + """Answers part PUTs with an ETag (``fail_first`` fails once with 503).""" + + def answer(call, calls): + if call["url"].startswith(S3): + number = part_number(call["url"]) + attempts = len([c for c in calls if c["url"] == call["url"]]) + if number == fail_first and attempts == 1: + return http_error(call["url"], 503, b"SlowDown") + return _Resp(200, b"", {"ETag": f'"etag-{number}"'}) + return api(call) + + return answer + + +# ── s3-multipart ───────────────────────────────────────────────────────────── + + +def test_multipart_puts_every_part_retries_a_failed_one_and_completes(http): + file = s3_file("f1", 10, 4) + + def api(call): + if call["url"] == f"{API}/imports": + return json_answer({**run(), "files": [file]}) + if call["url"].endswith("/complete"): + return json_answer({**file_dto("f1", "f1.csv", 10), "status": "uploaded"}) + return http_error(call["url"], 404) + + calls = http(s3_and_api(api, fail_first=2)) + client = db() + + created = client.imports.create([{"name": "f1.csv", "size": 10}]) + assert calls.calls[0]["method"] == "POST" + assert calls.calls[0]["body"] == {"files": [{"name": "f1.csv", "size": 10}]} + + progress = [] + data = data_of(10) + result = client.imports.upload( + created.data["id"], + "f1", + data, + parallelism=2, + on_progress=lambda p: progress.append(p["loaded"]), + ) + + puts = calls.s3() + assert len(puts) == 4 # three parts, one retried + for put in puts: + assert put["method"] == "PUT" + assert "authorization" not in put["headers"] + sizes = {part_number(put["url"]): len(put["body"]) for put in puts} + assert sizes == {1: 4, 2: 4, 3: 2} + third = next(put for put in puts if part_number(put["url"]) == 3) + assert third["body"] == data[8:] + + complete = calls.calls[-1] + assert complete["url"] == f"{API}/imports/imp-1/files/f1/complete" + assert complete["method"] == "POST" + assert complete["headers"]["authorization"] == "Bearer token" + assert complete["body"] == { + "parts": [ + {"partNumber": 1, "etag": '"etag-1"'}, + {"partNumber": 2, "etag": '"etag-2"'}, + {"partNumber": 3, "etag": '"etag-3"'}, + ] + } + assert progress[-1] == 10 + assert result.data["status"] == "uploaded" + + +def test_multipart_reads_parts_from_a_path(http, tmp_path): + path = tmp_path / "f1.csv" + data = data_of(9) + path.write_bytes(data) + calls = http(s3_and_api(lambda call: json_answer(file_dto("f1", "f1.csv", 9)))) + + db().imports.upload("imp-1", s3_file("f1", 9, 4), path) + + bodies = {part_number(put["url"]): put["body"] for put in calls.s3()} + assert bodies == {1: data[:4], 2: data[4:8], 3: data[8:]} + assert len(calls.calls[-1]["body"]["parts"]) == 3 + + +def test_multipart_cuts_a_non_seekable_stream_into_parts(http): + data = data_of(9) + + class Pipe(io.RawIOBase): + """Unseekable, and returns short reads.""" + + def __init__(self): + self._buffer = io.BytesIO(data) + + def readable(self): + return True + + def seekable(self): + return False + + def read(self, size=-1): + return self._buffer.read(min(size, 3) if size > 0 else size) + + calls = http(s3_and_api(lambda call: json_answer(file_dto("f1", "f1.csv", 9)))) + + db().imports.upload("imp-1", s3_file("f1", 9, 4), Pipe(), parallelism=1) + + bodies = sorted((part_number(put["url"]), put["body"]) for put in calls.s3()) + assert bodies == [(1, data[:4]), (2, data[4:8]), (3, data[8:])] + + +def test_multipart_uploads_a_file_object_from_its_position(http): + data = data_of(10) + f = io.BytesIO(b"xx" + data) + f.read(2) + calls = http(s3_and_api(lambda call: json_answer(file_dto("f1", "f1.csv", 10)))) + + db().imports.upload("imp-1", s3_file("f1", 10, 4), f) + + bodies = {part_number(put["url"]): put["body"] for put in calls.s3()} + assert b"".join(bodies[n] for n in sorted(bodies)) == data + + +def test_a_size_mismatch_fails_before_sending_anything(http): + calls = http(s3_and_api(lambda call: json_answer({}))) + with pytest.raises(ValueError, match="announced as 10 bytes"): + db().imports.upload("imp-1", s3_file("f1", 10, 4), data_of(9)) + assert calls.calls == [] + + +def test_a_4xx_part_upload_is_not_retried(http): + calls = http(lambda call, calls: http_error(call["url"], 403, b"")) + with pytest.raises(RushDBError) as raised: + db().imports.upload("imp-1", s3_file("f1", 4, 4), data_of(4), retries=3) + assert raised.value.status == 403 + assert len(calls.calls) == 1 + + +def test_a_part_failing_every_attempt_stops_the_upload(http): + calls = http(lambda call, calls: http_error(call["url"], 500)) + with pytest.raises(RushDBError): + db().imports.upload("imp-1", s3_file("f1", 4, 4), data_of(4), retries=2) + assert len(calls.calls) == 2 # no completion call + + +def test_upload_needs_the_file_object_for_files_this_client_did_not_create(http): + http(lambda call, calls: json_answer({})) + with pytest.raises(ValueError, match="No upload instructions"): + db().imports.upload("imp-1", "f1", data_of(1)) + + +# ── direct ─────────────────────────────────────────────────────────────────── + + +def test_direct_puts_the_raw_bytes_with_the_auth_header_then_completes(http): + calls = http(lambda call, calls: json_answer(file_dto("f1", "f1.csv", 5))) + + db().imports.upload("imp-1", direct_file("f1", 5), data_of(5)) + + put, complete = calls.calls + assert put["url"] == f"{API}/imports/imp-1/files/f1/content" + assert put["method"] == "PUT" + assert put["headers"]["authorization"] == "Bearer token" + assert put["headers"]["content-type"] == "application/octet-stream" + assert put["body"] == data_of(5) + assert complete["url"] == f"{API}/imports/imp-1/files/f1/complete" + assert complete["method"] == "POST" + assert complete["body"] == {} + + +def test_direct_streams_a_path_and_retries_a_5xx(http, tmp_path): + path = tmp_path / "f1.csv" + path.write_bytes(b"a,b\n1,2") + + def answer(call, calls): + if call["url"].endswith("/content") and len(calls) == 1: + return http_error(call["url"], 502) + return json_answer(file_dto("f1", "f1.csv", 7)) + + calls = http(answer) + + db().imports.upload("imp-1", direct_file("f1", 7), str(path)) + + puts = [call for call in calls.calls if call["url"].endswith("/content")] + assert [put["body"] for put in puts] == [b"a,b\n1,2", b"a,b\n1,2"] + assert puts[1]["headers"]["content-length"] == "7" + + +# ── runs ───────────────────────────────────────────────────────────────────── + + +def test_wait_polls_until_the_run_finishes(http): + statuses = iter(["queued", "running", "completed"]) + calls = http(lambda call, calls: json_answer(run(status=next(statuses)))) + seen = [] + + result = db().imports.wait( + "imp-1", interval=0.001, on_progress=lambda r: seen.append(r["status"]) + ) + + assert result.data["status"] == "completed" + assert seen == ["queued", "running", "completed"] + assert all( + call["method"] == "GET" and call["url"] == f"{API}/imports/imp-1" + for call in calls.calls + ) + + +def test_wait_gives_up_after_the_timeout(http): + http(lambda call, calls: json_answer(run(status="running"))) + with pytest.raises(TimeoutError, match="still running"): + db().imports.wait("imp-1", interval=0.005, timeout=0.012) + + +def test_list_filters_and_run_actions_map_to_their_routes(http): + calls = http(lambda call, calls: json_answer([], total=0)) + client = db() + + listed = client.imports.list(kind="sync", connection_id="c 1", limit=10) + client.imports.cancel("imp-1") + client.imports.retry("imp-1") + client.imports.delete("imp-1") + client.imports.preview("imp-1", files=[{"id": "f1", "label": "Order"}]) + client.imports.start("imp-1") + client.imports.add_files("imp-1", [{"name": "b.csv", "size": 3}]) + client.imports.remove_file("imp-1", "f2") + + assert listed.total == 0 + assert [f"{c['method']} {c['url'][len(API):]}" for c in calls.calls] == [ + "GET /imports?kind=sync&connectionId=c+1&limit=10", + "POST /imports/imp-1/cancel", + "POST /imports/imp-1/retry", + "DELETE /imports/imp-1", + "POST /imports/imp-1/preview", + "POST /imports/imp-1/start", + "POST /imports/imp-1/files", + "DELETE /imports/imp-1/files/f2", + ] + assert calls.calls[4]["body"] == {"files": [{"id": "f1", "label": "Order"}]} + assert calls.calls[5]["body"] == {} + assert calls.calls[6]["body"] == {"files": [{"name": "b.csv", "size": 3}]} + + +def test_start_passes_hashed_columns_from_the_preview(http): + preview = { + "files": [ + { + "fileId": "f1", + "name": "customers.csv", + "label": "Customer", + "rows": 2, + "columns": [ + {"name": "id", "type": "number", "suggestion": "include"}, + { + "name": "email", + "type": "string", + "samples": ["a***@example.com"], + "pii": { + "kind": "email", + "source": "name", + "confidence": 0.9, + "reason": "The name suggests an email", + }, + "suggestion": "hash", + }, + ], + "errors": [], + } + ], + "links": [], + } + calls = http(lambda call, calls: json_answer(preview)) + client = db() + + data = client.imports.preview("imp-1").data + email = next(c for c in data["files"][0]["columns"] if c["suggestion"] == "hash") + assert email["pii"]["kind"] == "email" + client.imports.start( + "imp-1", + files=[{"id": "f1", "mapping": {"keyColumn": "id", "hash": [email["name"]]}}], + ) + assert calls.calls[1]["body"] == { + "files": [{"id": "f1", "mapping": {"keyColumn": "id", "hash": ["email"]}}] + } + + +# ── import_files ───────────────────────────────────────────────────────────── + + +def test_import_files_creates_uploads_starts_with_mappings_and_links_then_waits( + http, tmp_path +): + customers = tmp_path / "customers.csv" + customers.write_bytes(data_of(3)) + files = [direct_file("f1", 3), direct_file("f2", 4)] + polls = iter(["running", "completed"]) + + def answer(call, calls): + path = call["url"][len(API) :] + if call["method"] == "POST" and path == "/imports": + return json_answer({**run(), "files": files}) + if path.endswith("/content") or path.endswith("/complete"): + return json_answer(file_dto("f", "f.csv", 1)) + if path.endswith("/start"): + return json_answer(run(status="queued")) + if call["method"] == "GET": + return json_answer(run(status=next(polls))) + return http_error(call["url"], 404) + + calls = http(answer) + phases = [] + + result = db().imports.import_files( + [ + {"data": customers, "label": "Customer"}, + {"data": data_of(4), "name": "orders.csv"}, + ], + name="CRM export", + mappings={"customers.csv": {"keyColumn": "id"}}, + links=[ + { + "source": "orders.csv", + "sourceColumn": "customer_id", + "target": 0, + "targetColumn": "id", + "type": "PLACED_BY", + } + ], + wait=True, + wait_interval=0.001, + on_progress=lambda p: phases.append(p["phase"]), + ) + + assert result.data["status"] == "completed" + assert calls.calls[0]["body"] == { + "name": "CRM export", + "files": [ + {"name": "customers.csv", "size": 3, "label": "Customer"}, + {"name": "orders.csv", "size": 4}, + ], + } + contents = [c["body"] for c in calls.calls if c["url"].endswith("/content")] + assert contents == [data_of(3), data_of(4)] + start = next(c for c in calls.calls if c["url"].endswith("/start")) + assert start["body"] == { + "files": [{"id": "f1", "mapping": {"keyColumn": "id"}}, {"id": "f2"}], + "links": [ + { + "sourceFileId": "f2", + "sourceColumn": "customer_id", + "targetFileId": "f1", + "targetColumn": "id", + "type": "PLACED_BY", + } + ], + } + assert "upload" in phases + assert phases[-1] == "run" + + +def test_import_files_takes_paths_and_returns_once_started(http, tmp_path): + path = tmp_path / "rows.ndjson" + path.write_bytes(b'{"a":1}\n') + + def answer(call, calls): + path_ = call["url"][len(API) :] + if path_ == "/imports": + return json_answer({**run(), "files": [direct_file("f1", 8)]}) + if path_.endswith("/start"): + return json_answer(run(status="queued")) + return json_answer(file_dto("f1", "rows.ndjson", 8)) + + calls = http(answer) + + result = db().imports.import_files([path], mappings={0: {"keyColumn": "a"}}) + + assert result.data["status"] == "queued" + assert calls.calls[0]["body"] == {"files": [{"name": "rows.ndjson", "size": 8}]} + assert calls.calls[-1]["body"] == { + "files": [{"id": "f1", "mapping": {"keyColumn": "a"}}] + } + assert not any(c["method"] == "GET" for c in calls.calls) + + +def test_import_files_needs_a_size_for_a_stream_and_a_name_for_bytes(http): + calls = http(lambda call, calls: json_answer({})) + + class Pipe(io.RawIOBase): + def readable(self): + return True + + def seekable(self): + return False + + with pytest.raises(ValueError, match="pass its size"): + db().imports.import_files([{"data": Pipe(), "name": "a.csv"}]) + with pytest.raises(ValueError, match="needs a name"): + db().imports.import_files([{"data": b"x"}]) + assert calls.calls == [] diff --git a/uv.lock b/uv.lock index 18cf2b7..bc19fe6 100755 --- a/uv.lock +++ b/uv.lock @@ -710,7 +710,7 @@ wheels = [ [[package]] name = "rushdb" -version = "3.3.0" +version = "3.4.0" source = { editable = "." } [package.dev-dependencies]