Benchmarks

# export SYNAP_API_KEY=synap_...   SYNAP_INSTANCE_ID=inst_...
import statistics
import time
import uuid
from maximem_synap import MaximemSynapSDK

# (conversation that states the fact, question, string the answer must contain)
CASES = [
    ("User: I'm moving to Lisbon in March.\nAssistant: Congratulations!", "Where is the user moving?", "lisbon"),
    ("User: My daughter is allergic to peanuts.\nAssistant: Noted.", "What is the user's daughter allergic to?", "peanut"),
    ("User: Our standup is at 9:30 every weekday.\nAssistant: Got it.", "When is the user's standup?", "9:30"),
]

sdk = MaximemSynapSDK()
await sdk.initialize()
user_id = f"recall-test-{uuid.uuid4()}"
# B2B instance: also pass customer_id to create() and fetch().

for conversation, _, _ in CASES:
    result = await sdk.memories.create(
        document=conversation, document_type="ai-chat-conversation", user_id=user_id
    )
    await sdk.memories.wait_for_completion(result.ingestion_id)

hits, latencies_ms = 0, []
for _, question, expected in CASES:
    start = time.perf_counter()
    ctx = await sdk.user.context.fetch(
        user_id=user_id, search_query=[question], response_shape="ranked"
    )
    latencies_ms.append((time.perf_counter() - start) * 1000)
    found = any(expected in item.content.lower() for item in ctx.items)
    hits += found
    print("HIT " if found else "MISS", question)

print(f"recall {hits}/{len(CASES)}, median fetch {statistics.median(latencies_ms):.0f} ms")
await sdk.shutdown()

// export SYNAP_API_KEY=synap_...   SYNAP_INSTANCE_ID=inst_...
import { randomUUID } from 'node:crypto';
import { SynapClient } from '@maximem/synap-js-sdk';

// [conversation that states the fact, question, string the answer must contain]
const CASES: [string, string, string][] = [
  ["User: I'm moving to Lisbon in March.\nAssistant: Congratulations!", 'Where is the user moving?', 'lisbon'],
  ['User: My daughter is allergic to peanuts.\nAssistant: Noted.', "What is the user's daughter allergic to?", 'peanut'],
  ['User: Our standup is at 9:30 every weekday.\nAssistant: Got it.', "When is the user's standup?", '9:30'],
];

const sdk = new SynapClient();
await sdk.initialize();
const userId = `recall-test-${randomUUID()}`;
// B2B instance: also pass customer_id to create() and fetch().

for (const [conversation] of CASES) {
  const result = await sdk.memories.create({
    document: conversation,
    document_type: 'ai-chat-conversation',
    user_id: userId,
  });
  await sdk.memories.wait_for_completion(result.ingestion_id);
}

let hits = 0;
const latenciesMs: number[] = [];
for (const [, question, expected] of CASES) {
  const start = performance.now();
  const ctx = await sdk.user.context.fetch({
    user_id: userId,
    search_query: [question],
    response_shape: 'ranked',
  });
  latenciesMs.push(performance.now() - start);
  const found = (ctx.items ?? []).some((item) => (item.content ?? '').toLowerCase().includes(expected));
  if (found) hits += 1;
  console.log(found ? 'HIT ' : 'MISS', question);
}

const sorted = [...latenciesMs].sort((a, b) => a - b);
const mid = Math.floor(sorted.length / 2);
const median = sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2;
console.log(`recall ${hits}/${CASES.length}, median fetch ${median.toFixed(0)} ms`);
await sdk.shutdown();