DreamDB

import dreamdb_dataset as vd

# Define a schema with image + CLIP embedding + label (chainable)
schema = (vd.Schema()
          .add_image("image", mime="jpeg")
          .add_embedding("embedding", dim=512, algorithm="dreamdb.ivf-cosine", rerank=True)
          .add_scalar_categorical("label"))

# (Or imperative if you prefer: schema = vd.Schema(); schema.add_image(...); ...)

# Create a Dataset on MinIO/S3
backend = "http://minio:9000/imagenet-1k"
ds = vd.Dataset.create("imagenet-1k", schema, backend=backend)

# Append a batch of samples (one dict per row, keyed by field name)
ds.append_many([
    {"image": jpeg_bytes, "embedding": clip_vec, "label": "tabby cat"},
    # … one dict per sample
])

# Snapshot the dataset at a point in time (for reproducible training)
v = ds.snapshot("baseline-2026-05-18")

# Query: top-K nearest neighbors via spatial indexing
hits = ds.iter_vector(field="embedding", query=clip_embed("a brown dog"), top_k=10)

# Sharded parallel ingest across N workers, then merge back
# (each worker calls ds.branch("worker-N") + append_many on its slice)
ds.merge_many(["worker-0", "worker-1", "worker-2"])

# GDPR-style deletion (read-side suppression; storage compaction is a separate op)
ds.delete(anchors=[1779083474791115000], reason="gdpr")

# Time-travel: open any prior snapshot
old = vd.Dataset.open_at(v, backend=backend)

# 1. Clone + build
git clone https://github.com/<your-org>/dreamdb
cd dreamdb
cargo build --release            # ~5 min cold; <30s incremental

# 2. Start a MinIO backend
docker run -d --name dreamdb-minio \
  -p 9000:9000 -p 9001:9001 \
  -e MINIO_ROOT_USER=dreamdb \
  -e MINIO_ROOT_PASSWORD=dreamdbsecret \
  minio/minio:latest server /data --console-address ":9001"
docker exec dreamdb-minio mc alias set local http://localhost:9000 dreamdb dreamdbsecret
docker exec dreamdb-minio mc mb local/demo
docker exec dreamdb-minio mc anonymous set public local/demo

# 3. Build + install the Python wheel
cd dreamdb-dataset-python
uv pip install maturin
maturin develop --release

# 4. A 30-line "hello world" — write 1000 random vectors, query nearest

#!/usr/bin/env -S uv run
# /// script
# dependencies = ["numpy", "dreamdb-dataset"]
# ///

import numpy as np, dreamdb_dataset as vd

s = vd.Schema().add_embedding("v", dim=32, algorithm="dreamdb.lsh-cosine")
ds = vd.Dataset.create("hello", s, backend="http://localhost:9000/demo")

samples = [{"v": np.random.randn(32).astype(np.float32)} for _ in range(1000)]
ds.append_many(samples)

q = np.random.randn(32).astype(np.float32).tolist()
batches = ds.iter_vector(field="v", query=q, top_k=5)
for batch in batches:
    for anchor in batch["_time_anchors"]:
        print(f"hit: anchor={anchor}")

# 5. Inspect the Manifest history
./target/release/dreamdb inspect \
  --backend http://localhost:9000/demo --ref-name hello --max-depth 10

dreamdb/
├── spec/                   # 21 protocol spec docs (start with spec/0000)
├── INDEX.md                # spec navigation + Open Questions audit
├── design/                 # implementation design docs (operator-facing)
├── dreamdb-core/            # addressing, hashing, CBOR primitives — shared by every layer
├── dreamdb-protocol/        # Object types (Manifest, Track, Bucket, ...) + verbs (Open/Append/Query/Stream)
├── dreamdb-connector/       # storage trait + memory backend
├── dreamdb-connector-http/  # MinIO/S3 implementation
├── dreamdb-dataset/         # high-level SDK (Schema, Sample, Dataset, Filter, Batch, MergeStrategy)
├── dreamdb-dataset-python/  # PyO3 bindings + Python wrapper + browser JS demo
├── dreamdb-cli/             # operator binaries (rebuild-ivf, ada-ivf-step, gc, delete, merge-many)
├── dreamdb-conformance/     # JSON test vectors per spec/0009 §3
└── dreamdb-bench/           # microbenchmarks (storage / latency / recall)