biondizzle/context-paging
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit

Browse Source
This commit is contained in:
biondizzle
2026-03-28 08:54:57 +00:00
commit 26f8b33db7
154 changed files with 23075 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
src/CacheInterface.php Normal file
View File

@@ -0,0 +1,48 @@
<?php
declare(strict_types=1);
namespace ContextPaging;
/**
* Interface for caching arbitrary data.
*/
interface CacheInterface
{
/**
* Get a value from the cache.
*
* @param string $key The cache key
* @return mixed|null The cached value or null if not found
*/
public function get(string $key): mixed;
/**
* Set a value in the cache.
*
* @param string $key The cache key
* @param mixed $value The value to cache (must be serializable)
* @param int|null $ttl Optional TTL in seconds
*/
public function set(string $key, mixed $value, ?int $ttl = null): void;
/**
* Check if a key exists.
*
* @param string $key The cache key
* @return bool
*/
public function has(string $key): bool;
/**
* Delete a key from the cache.
*
* @param string $key The cache key
*/
public function delete(string $key): void;
/**
* Clear all keys with the configured prefix.
*/
public function clear(): void;
}

29
src/CompletionsClientInterface.php Normal file
View File

@@ -0,0 +1,29 @@
<?php
declare(strict_types=1);
namespace ContextPaging;
use Psr\Http\Message\ResponseInterface;
/**
* Interface for OpenAI-compatible chat completions API.
*/
interface CompletionsClientInterface
{
/**
* Create a chat completion.
*
* @param array $messages OpenAI-format messages array
* @param array $options Additional options (model, temperature, stream, etc.)
* @return ResponseInterface PSR-7 response (streaming or regular)
*/
public function chat(array $messages, array $options = []): ResponseInterface;
/**
* List available models.
*
* @return ResponseInterface
*/
public function listModels(): ResponseInterface;
}

614
src/ContextPaging.php Normal file
View File

@@ -0,0 +1,614 @@
<?php
declare(strict_types=1);
namespace ContextPaging;
use Psr\Http\Message\ServerRequestInterface;
use Psr\Http\Message\ResponseInterface;
/**
* Context Paging — Virtual memory for LLM context windows.
*
* Loop 2: fit() — compress messages until they fit the context window.
* Loop 3: execute() — run LLM, handle dereference tool calls.
*/
class ContextPaging
{
/**
* Maximum context tokens for the model.
*/
private int $maxContextTokens = 128000;
/**
* Log file path for internal events (optional).
*/
private ?string $logFile = null;
/**
* Request ID for correlating log entries.
*/
private string $requestId;
/**
* Tokens reserved for the response.
*/
private int $responseReserve = 4096;
/**
* Safety margin for token counting discrepancies.
* Different tokenizers (tiktoken vs vLLM) may count slightly differently,
* plus there's overhead for message formatting. This buffer prevents
* edge cases where we think we fit but the API rejects us.
*/
private int $safetyMargin = 500;
/**
* Cache for original messages (the "disk" backing virtual memory).
* Keyed by MD5 hash → full message array.
*/
private CacheInterface $messageStore;
/**
* Summary cache (MD5 of original → summary text).
*/
private CacheInterface $summaryCache;
/**
* Token counter instance.
*/
private TokenCounter $tokenCounter;
/**
* Tool call parser instance.
*/
private ToolCallParser $toolCallParser;
/**
* Tool formatter instance.
*/
private ToolFormatter $toolFormatter;
/**
* Tool call mode (NATIVE, RAW, or AUTO).
*/
private ToolCallMode $toolCallMode = ToolCallMode::AUTO;
/**
* Summarizer instance (optional).
*/
private ?SummarizerInterface $summarizer = null;
/**
* @param TokenCounter|null $tokenCounter
* @param SummarizerInterface|null $summarizer
* @param CacheInterface|null $messageStore Cache for original messages (default: in-memory)
* @param CacheInterface|null $summaryCache Cache for summaries (default: in-memory)
*/
public function __construct(
?TokenCounter $tokenCounter = null,
?SummarizerInterface $summarizer = null,
?CacheInterface $messageStore = null,
?CacheInterface $summaryCache = null
) {
$this->tokenCounter = $tokenCounter ?? new TokenCounter();
$this->summarizer = $summarizer;
$this->messageStore = $messageStore ?? new InMemoryCache();
$this->summaryCache = $summaryCache ?? new InMemoryCache();
$this->toolCallParser = new ToolCallParser($this->toolCallMode);
$this->toolFormatter = new ToolFormatter($this->toolCallMode);
$this->requestId = substr(md5(uniqid('', true)), 0, 8);
}
/**
* Set the log file path.
*/
public function setLogFile(string $path): self
{
$this->logFile = $path;
return $this;
}
/**
* Log an event to the log file.
*/
private function log(string $event, array $data = []): void
{
if ($this->logFile === null) {
return;
}
$entry = json_encode(array_merge(
['timestamp' => date('Y-m-d H:i:s'), 'request_id' => $this->requestId, 'event' => $event],
$data
)) . "\n";
file_put_contents($this->logFile, $entry, FILE_APPEND | LOCK_EX);
}
/**
* Set the summarizer.
*/
public function setSummarizer(SummarizerInterface $summarizer): self
{
$this->summarizer = $summarizer;
return $this;
}
/**
* Set the message store cache.
*/
public function setMessageStore(CacheInterface $cache): self
{
$this->messageStore = $cache;
return $this;
}
/**
* Set the summary cache.
*/
public function setSummaryCache(CacheInterface $cache): self
{
$this->summaryCache = $cache;
return $this;
}
/**
* Set the tool call mode.
*/
public function setToolCallMode(ToolCallMode $mode): self
{
$this->toolCallMode = $mode;
$this->toolCallParser->setMode($mode);
$this->toolFormatter->setMode($mode);
return $this;
}
/**
* Get the current tool call mode.
*/
public function getToolCallMode(): ToolCallMode
{
return $this->toolCallMode;
}
/**
* LOOP 2 — Fit the context to the window.
*/
public function fit(ServerRequestInterface $request): ServerRequestInterface
{
$body = $request->getParsedBody();
$messages = $body['messages'] ?? [];
if (empty($messages)) {
return $request;
}
// Store originals for dereferencing
$this->storeOriginals($messages);
// Get max_tokens from request, fall back to responseReserve
$maxTokens = $body['max_tokens'] ?? $this->responseReserve;
// Calculate current token count and budget
// Safety margin accounts for tokenizer discrepancies and message overhead
$tokens = $this->countTokens($messages);
$budget = $this->maxContextTokens - $maxTokens - $this->safetyMargin;
$this->log('fit_start', [
'message_count' => count($messages),
'original_tokens' => $tokens,
'budget' => $budget,
'max_context' => $this->maxContextTokens,
'response_reserve' => $maxTokens,
'needs_compression' => $tokens > $budget,
]);
// Already fits? Done.
if ($tokens <= $budget) {
$this->log('fit_skip', ['reason' => 'already_within_budget']);
return $request->withAttribute('context_fitted', true)
->withAttribute('context_tokens', $tokens)
->withAttribute('context_budget', $budget);
}
// Summarize oldest messages until we fit
$messages = $this->summarizeToFit($messages, $budget, $tokens);
// Rebuild the request with fitted messages
$body['messages'] = $messages;
$newTokens = $this->countTokens($messages);
$this->log('fit_complete', [
'original_tokens' => $tokens,
'fitted_tokens' => $newTokens,
'saved_tokens' => $tokens - $newTokens,
'compression_ratio' => round(($tokens - $newTokens) / $tokens * 100, 1) . '%',
]);
return $request->withParsedBody($body)
->withAttribute('context_fitted', true)
->withAttribute('context_tokens', $newTokens)
->withAttribute('context_budget', $budget)
->withAttribute('original_token_count', $tokens);
}
/**
* LOOP 3 — Execute with dereference handling.
*/
public function execute(ServerRequestInterface $request, callable $llmInvoker): ResponseInterface
{
$messages = $request->getParsedBody()['messages'] ?? [];
$options = $this->extractOptions($request);
// Add the fetch_message tool to the request
$payload = $this->toolFormatter->buildPayload(
$messages,
$options,
[ToolFormatter::FETCH_MESSAGE_TOOL],
$this->toolCallMode
);
$iteration = 0;
$maxIterations = 10;
$response = null;
$this->log('execute_start', [
'message_count' => count($messages),
'tool_mode' => $this->toolCallMode->value,
]);
while ($iteration < $maxIterations) {
// Memory dump: log context state before each LLM call
$this->logMemoryDump($payload['messages'], $iteration);
$response = $llmInvoker($payload['messages'], $payload);
$responseBody = $response->getBody()->getContents();
$responseData = json_decode($responseBody, true);
$response = new \GuzzleHttp\Psr7\Response(
$response->getStatusCode(),
$response->getHeaders(),
$responseBody
);
if ($iteration === 0 && $this->toolCallMode === ToolCallMode::AUTO) {
$detectedMode = $this->toolCallParser->detectMode($responseData ?? []);
$this->toolCallParser->setMode($detectedMode);
$this->toolFormatter->setMode($detectedMode);
$this->log('tool_mode_detected', ['mode' => $detectedMode->value]);
}
$toolCalls = $this->toolCallParser->extract($responseData ?? []);
if ($toolCalls === null) {
$this->log('execute_complete', [
'iterations' => $iteration,
'had_dereferences' => $iteration > 0,
]);
return $response;
}
$fetchCall = null;
foreach ($toolCalls as $call) {
if (($call['name'] ?? null) === 'fetch_message') {
$fetchCall = $call;
break;
}
}
if ($fetchCall === null) {
$this->log('execute_complete', [
'iterations' => $iteration,
'had_dereferences' => $iteration > 0,
'other_tool_calls' => count($toolCalls),
]);
return $response;
}
$md5 = $fetchCall['arguments']['md5'] ?? null;
if ($md5 === null) {
$this->log('dereference_error', ['reason' => 'missing_md5']);
return $response;
}
$this->log('dereference_start', [
'md5' => $md5,
'iteration' => $iteration + 1,
]);
$fullMessage = $this->dereference($md5);
if ($fullMessage === null) {
$this->log('dereference_error', [
'md5' => $md5,
'reason' => 'message_not_found',
]);
return $response;
}
$fullContent = $fullMessage['content'] ?? '';
$fullTokens = $this->tokenCounter->count($fullContent);
$payload['messages'] = $this->injectDereferenced($payload['messages'], $md5, $fullMessage);
$payload['messages'][] = [
'role' => 'tool',
'content' => json_encode([
'status' => 'success',
'message' => 'Full message retrieved and injected into context.',
]),
'tool_call_id' => $fetchCall['id'],
];
$this->log('dereference_success', [
'md5' => $md5,
'role' => $fullMessage['role'] ?? 'unknown',
'content_chars' => is_string($fullContent) ? strlen($fullContent) : 0,
'content_tokens' => $fullTokens,
'new_message_count' => count($payload['messages']),
]);
$iteration++;
}
$this->log('execute_error', ['reason' => 'max_iterations_reached', 'iterations' => $iteration]);
return $response ?? new \GuzzleHttp\Psr7\Response(
500,
['Content-Type' => 'application/json'],
json_encode(['error' => ['message' => 'Max dereference iterations reached']])
);
}
// -----------------------------------------------------------------
// PRIVATE: Loop 2 helpers
// -----------------------------------------------------------------
private function extractOptions(ServerRequestInterface $request): array
{
$body = $request->getParsedBody();
$options = $body;
unset($options['messages']);
return $options;
}
/**
* Store original messages keyed by MD5 hash.
*/
private function storeOriginals(array $messages): void
{
foreach ($messages as $message) {
$content = $message['content'] ?? '';
if (is_string($content)) {
$md5 = md5($content);
$this->messageStore->set("msg:{$md5}", $message);
}
}
}
/**
* Summarize messages until we fit the budget.
*/
private function summarizeToFit(array $messages, int $budget, int $originalTokens): array
{
$lastIndex = count($messages) - 1;
$summarizedCount = 0;
while ($this->countTokens($messages) > $budget) {
$summarizedIndex = null;
for ($i = 0; $i < $lastIndex; $i++) {
if (!$this->isSummarized($messages[$i])) {
$summarizedIndex = $i;
break;
}
}
if ($summarizedIndex === null) {
$this->log('fit_error', [
'reason' => 'all_messages_summarized',
'current_tokens' => $this->countTokens($messages),
'budget' => $budget,
]);
throw new \RuntimeException(
'Context still over budget after all messages summarized. ' .
'Last message is too large.'
);
}
$original = $messages[$summarizedIndex];
$originalContent = $original['content'] ?? '';
$originalLen = is_string($originalContent) ? strlen($originalContent) : 0;
$originalMsgTokens = $this->tokenCounter->count($originalContent);
$messages[$summarizedIndex] = $this->summarizeMessage($messages[$summarizedIndex]);
$summarizedCount++;
$summaryContent = $messages[$summarizedIndex]['content'];
$summaryMsgTokens = $this->tokenCounter->count($summaryContent);
$currentTokens = $this->countTokens($messages);
$this->log('summarize', [
'index' => $summarizedIndex,
'role' => $original['role'] ?? 'unknown',
'original_chars' => $originalLen,
'original_tokens' => $originalMsgTokens,
'summary_tokens' => $summaryMsgTokens,
'tokens_saved' => $originalMsgTokens - $summaryMsgTokens,
'running_total_tokens' => $currentTokens,
'budget' => $budget,
'md5' => $messages[$summarizedIndex]['_original_md5'] ?? null,
]);
}
$this->log('fit_summarized', [
'total_summarized' => $summarizedCount,
'original_tokens' => $originalTokens,
'final_tokens' => $this->countTokens($messages),
]);
return $messages;
}
/**
* Summarize a single message.
*/
private function summarizeMessage(array $message): array
{
$content = $message['content'] ?? '';
$md5 = is_string($content) ? md5($content) : md5(json_encode($content));
// Check cache first
$cacheKey = "summary:{$md5}";
$summary = $this->summaryCache->get($cacheKey);
if ($summary === null) {
$summary = $this->generateSummary($content);
$this->summaryCache->set($cacheKey, $summary);
}
return [
'role' => $message['role'] ?? 'user',
'content' => "[md5:{$md5}] {$summary}",
'_summarized' => true,
'_original_md5' => $md5,
];
}
private function isSummarized(array $message): bool
{
return isset($message['_summarized']) && $message['_summarized'] === true;
}
private function countTokens(array $messages): int
{
return $this->tokenCounter->contextSize($messages);
}
/**
* Generate a summary for a message.
*/
private function generateSummary(string $content): string
{
if ($this->summarizer !== null) {
return $this->summarizer->summarize($content);
}
if (strlen($content) > 100) {
return substr($content, 0, 100) . '...';
}
return $content;
}
// -----------------------------------------------------------------
// PRIVATE: Loop 3 helpers
// -----------------------------------------------------------------
/**
* Log a memory dump of the current context state.
* Like dumping CPU registers each cycle - shows what the model "sees".
*/
private function logMemoryDump(array $messages, int $iteration): void
{
$summarized = 0;
$original = 0;
$messageSummary = [];
foreach ($messages as $i => $msg) {
$isSummarized = isset($msg['_summarized']) && $msg['_summarized'] === true;
if ($isSummarized) {
$summarized++;
} else {
$original++;
}
$content = $msg['content'] ?? '';
$preview = is_string($content)
? (strlen($content) > 80 ? substr($content, 0, 80) . '...' : $content)
: '(non-string content)';
$messageSummary[] = [
'idx' => $i,
'role' => $msg['role'] ?? 'unknown',
'summarized' => $isSummarized,
'md5' => $msg['_original_md5'] ?? null,
'tokens' => $this->tokenCounter->count($content),
'preview' => $preview,
];
}
$totalTokens = $this->countTokens($messages);
$this->log('memory_dump', [
'iteration' => $iteration,
'total_messages' => count($messages),
'summarized_count' => $summarized,
'original_count' => $original,
'total_tokens' => $totalTokens,
'budget' => $this->maxContextTokens - $this->responseReserve - $this->safetyMargin,
'messages' => $messageSummary,
]);
}
/**
* Dereference an MD5 hash to get the original message.
*/
private function dereference(string $md5): ?array
{
return $this->messageStore->get("msg:{$md5}");
}
/**
* Replace a summarized message with the full message.
*/
private function injectDereferenced(array $messages, string $md5, array $fullMessage): array
{
foreach ($messages as $i => $message) {
if (($message['_original_md5'] ?? null) === $md5) {
$messages[$i] = $fullMessage;
break;
}
}
return $messages;
}
// -----------------------------------------------------------------
// Configuration
// -----------------------------------------------------------------
public function setMaxContextTokens(int $tokens): self
{
$this->maxContextTokens = $tokens;
return $this;
}
public function setResponseReserve(int $tokens): self
{
$this->responseReserve = $tokens;
return $this;
}
public function getToolFormatter(): ToolFormatter
{
return $this->toolFormatter;
}
public function getToolCallParser(): ToolCallParser
{
return $this->toolCallParser;
}
public function getMessageStore(): CacheInterface
{
return $this->messageStore;
}
public function getSummaryCache(): CacheInterface
{
return $this->summaryCache;
}
}

102
src/ContextRequest.php Normal file
View File

@@ -0,0 +1,102 @@
<?php
declare(strict_types=1);
namespace ContextPaging;
use GuzzleHttp\Psr7\ServerRequest;
use Psr\Http\Message\ServerRequestInterface;
/**
* Extended ServerRequest with context size tracking.
*/
class ContextRequest extends ServerRequest
{
private ?TokenCounter $tokenCounter = null;
/**
* Create from a standard ServerRequestInterface.
*/
public static function fromRequest(ServerRequestInterface $request): self
{
return new self(
method: $request->getMethod(),
uri: $request->getUri(),
headers: $request->getHeaders(),
body: $request->getBody(),
version: $request->getProtocolVersion(),
serverParams: $request->getServerParams()
);
}
/**
* Set the token counter instance.
*/
public function withTokenCounter(TokenCounter $counter): self
{
$new = clone $this;
$new->tokenCounter = $counter;
return $new;
}
/**
* Get the token counter (lazy-load default if not set).
*/
public function getTokenCounter(): TokenCounter
{
if ($this->tokenCounter === null) {
$this->tokenCounter = new TokenCounter();
}
return $this->tokenCounter;
}
/**
* Calculate and store context size for the given messages.
*
* @param array $messages Array of ['role' => '...', 'content' => '...']
* @param string $encoding Encoding name (cl100k_base or o200k_base).
* @param int $perMessage Overhead tokens per message.
* @param int $replyPrimer Tokens added after all messages.
* @return self New request instance with context_size attribute.
*/
public function contextSize(
array $messages,
string $encoding = 'cl100k_base',
int $perMessage = 4,
int $replyPrimer = 3
): self {
$count = $this->getTokenCounter()->contextSize(
$messages,
$encoding,
$perMessage,
$replyPrimer
);
return $this->withAttribute('context_size', [
'tokens' => $count,
'encoding' => $encoding,
'message_count' => count($messages),
'per_message' => $perMessage,
'reply_primer' => $replyPrimer,
]);
}
/**
* Get the stored context size, if any.
*
* @return array|null ['tokens' => int, 'encoding' => string, ...]
*/
public function getContextSize(): ?array
{
return $this->getAttribute('context_size');
}
/**
* Get just the token count from stored context size.
*/
public function getContextTokenCount(): ?int
{
$ctx = $this->getContextSize();
return $ctx['tokens'] ?? null;
}
}

39
src/InMemoryCache.php Normal file
View File

@@ -0,0 +1,39 @@
<?php
declare(strict_types=1);
namespace ContextPaging;
/**
* In-memory cache implementation (default, for single-request usage).
*/
class InMemoryCache implements CacheInterface
{
private array $data = [];
public function get(string $key): mixed
{
return $this->data[$key] ?? null;
}
public function set(string $key, mixed $value, ?int $ttl = null): void
{
$this->data[$key] = $value;
// TTL ignored for in-memory (single request scope)
}
public function has(string $key): bool
{
return array_key_exists($key, $this->data);
}
public function delete(string $key): void
{
unset($this->data[$key]);
}
public function clear(): void
{
$this->data = [];
}
}

135
src/LLMSummarizer.php Normal file
View File

@@ -0,0 +1,135 @@
<?php
declare(strict_types=1);
namespace ContextPaging;
use Psr\Http\Message\ResponseInterface;
/**
* LLM-backed summarizer using OpenAI-compatible API.
*/
class LLMSummarizer implements SummarizerInterface
{
private CompletionsClientInterface $client;
private string $model;
private string $systemPrompt;
private int $maxTokens;
private float $temperature;
/**
* @param CompletionsClientInterface $client The LLM client
* @param string $model Model to use for summarization
* @param string $systemPrompt System prompt for summarization
* @param int $maxTokens Max tokens for summary output
* @param float $temperature Temperature for generation
*/
public function __construct(
CompletionsClientInterface $client,
string $model = 'HuggingFaceTB/SmolLM3-3B',
string $systemPrompt = 'You are a summarization assistant. Summarize the given text concisely, preserving key information. Be brief but comprehensive.',
int $maxTokens = 200,
float $temperature = 0.3
) {
$this->client = $client;
$this->model = $model;
$this->systemPrompt = $systemPrompt;
$this->maxTokens = $maxTokens;
$this->temperature = $temperature;
}
/**
* Generate a summary of the content.
*
* @param string $content The content to summarize
* @param array $context Optional context (role, previous messages, etc.)
* @return string The summary
*/
public function summarize(string $content, array $context = []): string
{
// Build the prompt
$userPrompt = $this->buildUserPrompt($content, $context);
$messages = [
['role' => 'system', 'content' => $this->systemPrompt],
['role' => 'user', 'content' => $userPrompt],
];
$response = $this->client->chat($messages, [
'model' => $this->model,
'max_tokens' => $this->maxTokens,
'temperature' => $this->temperature,
]);
return $this->extractContent($response);
}
/**
* Build the user prompt for summarization.
*/
private function buildUserPrompt(string $content, array $context): string
{
$role = $context['role'] ?? 'unknown';
$instruction = $context['instruction'] ?? 'Summarize in 2-3 sentences:';
$prompt = "{$instruction}\n\n";
if ($role !== 'unknown') {
$prompt .= "[Role: {$role}]\n";
}
$prompt .= $content;
return $prompt;
}
/**
* Extract content from the response.
*/
private function extractContent(ResponseInterface $response): string
{
$body = json_decode($response->getBody()->getContents(), true);
if (!isset($body['choices'][0]['message']['content'])) {
throw new \RuntimeException('No content in summarizer response');
}
return trim($body['choices'][0]['message']['content']);
}
/**
* Set the model.
*/
public function setModel(string $model): self
{
$this->model = $model;
return $this;
}
/**
* Set the max tokens for output.
*/
public function setMaxTokens(int $maxTokens): self
{
$this->maxTokens = $maxTokens;
return $this;
}
/**
* Set the temperature.
*/
public function setTemperature(float $temperature): self
{
$this->temperature = $temperature;
return $this;
}
/**
* Set the system prompt.
*/
public function setSystemPrompt(string $prompt): self
{
$this->systemPrompt = $prompt;
return $this;
}
}

133
src/Middleware/ModelQuirksMiddleware.php Normal file
View File

@@ -0,0 +1,133 @@
<?php
declare(strict_types=1);
namespace ContextPaging\Middleware;
use Psr\Http\Message\ResponseInterface;
use Psr\Http\Message\ServerRequestInterface;
use Psr\Http\Server\MiddlewareInterface;
use Psr\Http\Server\RequestHandlerInterface;
/**
* Model Quirks Middleware
*
* Handles model-specific behavior adjustments so the core logic stays model-agnostic.
* Mutates the request payload based on the model being used.
*
* Example quirks:
* - SmolLM3: Inject `/no_think` to suppress reasoning tokens
* - Some models: Strip tool definitions to avoid confusion
*/
class ModelQuirksMiddleware implements MiddlewareInterface
{
/**
* Model quirk configurations.
* Keys are regex patterns to match model names.
*/
private const QUIRK_CONFIGS = [
// SmolLM3 - small model that outputs reasoning tokens and gets confused by tools
'/SmolLM3/i' => [
'no_think' => true, // Inject /no_think into system prompt
'strip_tools' => true, // Remove tool definitions (doesn't use them correctly)
'tool_mode' => 'raw', // Use raw mode if tools are kept
],
// Add more model quirks here as needed
// '/Llama-3/i' => [...],
// '/Qwen/i' => [...],
];
/**
* Process the request and apply model quirks.
*/
public function process(ServerRequestInterface $request, RequestHandlerInterface $handler): ResponseInterface
{
$body = $request->getParsedBody();
if (!is_array($body)) {
return $handler->handle($request);
}
$model = $body['model'] ?? '';
$quirks = $this->getQuirksForModel($model);
if (empty($quirks)) {
return $handler->handle($request);
}
// Apply quirks to the request body
$body = $this->applyQuirks($body, $quirks);
// Store original body for reference
$request = $request->withAttribute('original_body', $request->getParsedBody());
$request = $request->withParsedBody($body);
// Also update the raw body for downstream consumers
$request = $request->withBody(new \Slim\Psr7\Stream(fopen('php://temp', 'r+')));
$request->getBody()->write(json_encode($body));
$request->getBody()->rewind();
return $handler->handle($request);
}
/**
* Get quirks for a specific model.
*/
public function getQuirksForModel(string $model): array
{
foreach (self::QUIRK_CONFIGS as $pattern => $quirks) {
if (preg_match($pattern, $model)) {
return $quirks;
}
}
return [];
}
/**
* Apply quirks to the request body.
*/
private function applyQuirks(array $body, array $quirks): array
{
// Inject /no_think into system prompt
if ($quirks['no_think'] ?? false) {
$body = $this->injectNoThink($body);
}
// Store quirks for later middleware/handlers
$body['_quirks'] = $quirks;
return $body;
}
/**
* Inject /no_think into the system prompt.
*/
private function injectNoThink(array $body): array
{
$messages = $body['messages'] ?? [];
foreach ($messages as $i => $message) {
if (($message['role'] ?? null) === 'system') {
$content = $message['content'] ?? '';
// Only inject if not already present
if (!str_contains($content, '/no_think')) {
$messages[$i]['content'] = rtrim($content) . ' /no_think';
}
$body['messages'] = $messages;
return $body;
}
}
// No system message found, prepend one
array_unshift($messages, [
'role' => 'system',
'content' => '/no_think',
]);
$body['messages'] = $messages;
return $body;
}
}

349
src/OpenAICompatibleClient.php Normal file
View File

@@ -0,0 +1,349 @@
<?php
declare(strict_types=1);
namespace ContextPaging;
use GuzzleHttp\Client;
use GuzzleHttp\Exception\ConnectException;
use GuzzleHttp\Exception\RequestException;
use GuzzleHttp\Psr7\Response;
use Psr\Http\Message\ResponseInterface;
/**
* OpenAI-compatible chat completions client.
*
* Works with OpenAI, Vultr Inference, vLLM, and any OpenAI-compatible API.
*/
class OpenAICompatibleClient implements CompletionsClientInterface
{
private Client $httpClient;
private string $baseUrl;
private ?string $apiKey;
private int $timeout;
private bool $verifySsl;
/**
* Patterns to sanitize from error messages (internal URLs, etc.)
*/
private const SANITIZE_PATTERNS = [
'#https?://[^\s]*vultrinference\.com[^\s]*#i',
'#https?://prod\.[^\s]*#i',
'#vllm[^\s]*#i',
];
/**
* Fields that should never be forwarded to the API.
*/
private const INTERNAL_FIELDS = [
'collection',
'_original_md5',
'_summarized',
];
/**
* @param string $baseUrl API base URL (e.g., "https://api.openai.com/v1")
* @param string|null $apiKey API key (optional for local/dev endpoints)
* @param int $timeout Request timeout in seconds
* @param bool $verifySsl Whether to verify SSL certificates
*/
public function __construct(
string $baseUrl,
?string $apiKey = null,
int $timeout = 300,
bool $verifySsl = true
) {
$this->baseUrl = rtrim($baseUrl, '/');
$this->apiKey = $apiKey;
$this->timeout = $timeout;
$this->verifySsl = $verifySsl;
$this->httpClient = new Client([
'timeout' => $timeout,
'verify' => $verifySsl,
]);
}
/**
* Create a chat completion (non-streaming).
*
* @param array $messages OpenAI-format messages
* @param array $options Additional options (model, temperature, etc.)
* @return ResponseInterface
*/
public function chat(array $messages, array $options = []): ResponseInterface
{
$payload = $this->buildPayload($messages, $options);
// Force non-streaming
$payload['stream'] = false;
return $this->sendRequest('/chat/completions', $payload);
}
/**
* Create a streaming chat completion.
*
* Returns a generator that yields SSE data chunks.
* Each chunk is the decoded JSON data (not the raw "data: " line).
*
* @param array $messages OpenAI-format messages
* @param array $options Additional options (model, temperature, etc.)
* @return \Generator Yields arrays: ['delta' => ..., 'finish_reason' => ..., 'usage' => ...]
*/
public function chatStream(array $messages, array $options = []): \Generator
{
$payload = $this->buildPayload($messages, $options);
// Force streaming
$payload['stream'] = true;
if (!isset($payload['stream_options'])) {
$payload['stream_options'] = ['include_usage' => true];
}
$response = $this->sendStreamingRequest('/chat/completions', $payload);
foreach ($response as $chunk) {
yield $chunk;
}
}
/**
* List available models.
*/
public function listModels(): ResponseInterface
{
return $this->sendRequest('/models', [], 'GET');
}
// -----------------------------------------------------------------
// Internal
// -----------------------------------------------------------------
/**
* Build the request payload.
*/
private function buildPayload(array $messages, array $options): array
{
// Strip internal fields
$payload = array_diff_key($options, array_flip(self::INTERNAL_FIELDS));
// Set messages
$payload['messages'] = $this->cleanMessages($messages);
// Ensure model is set
if (!isset($payload['model'])) {
$payload['model'] = 'default';
}
return $payload;
}
/**
* Clean messages for API consumption.
*/
private function cleanMessages(array $messages): array
{
return array_map(function (array $msg): array {
// Remove internal tracking fields
unset($msg['_summarized'], $msg['_original_md5']);
// Ensure content is present
if (!isset($msg['content'])) {
$msg['content'] = '';
}
return $msg;
}, $messages);
}
/**
* Send a non-streaming request.
*/
private function sendRequest(string $endpoint, array $payload, string $method = 'POST'): ResponseInterface
{
$options = $this->buildRequestOptions($payload);
try {
$response = $this->httpClient->request($method, $this->baseUrl . $endpoint, $options);
// Clean up the response body
$body = json_decode($response->getBody()->getContents(), true);
// Remove internal fields we don't expose
unset($body['prompt_logprobs']);
foreach ($body['choices'] ?? [] as $i => $choice) {
unset($body['choices'][$i]['stop_reason']);
}
return new Response(
$response->getStatusCode(),
['Content-Type' => 'application/json'],
json_encode($body)
);
} catch (ConnectException $e) {
return $this->errorResponse(503, 'The AI service is currently unavailable. Please try again later.');
} catch (RequestException $e) {
$response = $e->getResponse();
if ($response) {
$body = json_decode($response->getBody()->getContents(), true);
$body = $this->sanitizeErrorBody($body);
return new Response(
$response->getStatusCode(),
['Content-Type' => 'application/json'],
json_encode($body)
);
}
return $this->errorResponse(500, $this->sanitizeMessage($e->getMessage()));
} catch (\Throwable $e) {
return $this->errorResponse(500, $this->sanitizeMessage($e->getMessage()));
}
}
/**
* Send a streaming request.
*
* @return \Generator Yields decoded JSON chunks
*/
private function sendStreamingRequest(string $endpoint, array $payload): \Generator
{
$options = $this->buildRequestOptions($payload);
$options['stream'] = true;
try {
$response = $this->httpClient->request('POST', $this->baseUrl . $endpoint, $options);
$body = $response->getBody();
$buffer = '';
while (!$body->eof()) {
$chunk = $body->read(8192);
if (empty($chunk)) {
continue;
}
$buffer .= $chunk;
// Process complete lines
while (($newlinePos = strpos($buffer, "\n")) !== false) {
$line = substr($buffer, 0, $newlinePos);
$buffer = substr($buffer, $newlinePos + 1);
$line = trim($line);
if (empty($line)) {
continue;
}
if (!str_starts_with($line, 'data: ')) {
continue;
}
$data = substr($line, 6);
if ($data === '[DONE]') {
return; // Generator complete
}
$decoded = json_decode($data, true);
if (!$decoded) {
continue;
}
// Clean up internal fields
foreach ($decoded['choices'] ?? [] as $i => $choice) {
unset($decoded['choices'][$i]['stop_reason']);
}
yield $decoded;
}
}
} catch (ConnectException $e) {
yield [
'error' => [
'message' => 'The AI service is currently unavailable.',
'type' => 'connection_error',
],
];
} catch (\Throwable $e) {
yield [
'error' => [
'message' => $this->sanitizeMessage($e->getMessage()),
'type' => 'server_error',
],
];
}
}
/**
* Build Guzzle request options.
*/
private function buildRequestOptions(array $payload): array
{
$options = [
'headers' => [
'Content-Type' => 'application/json',
'Accept' => 'application/json',
],
];
if ($this->apiKey) {
$options['headers']['Authorization'] = 'Bearer ' . $this->apiKey;
}
if (!empty($payload)) {
$options['json'] = $payload;
}
return $options;
}
/**
* Create an error response.
*/
private function errorResponse(int $status, string $message): ResponseInterface
{
return new Response(
$status,
['Content-Type' => 'application/json'],
json_encode([
'error' => [
'message' => $message,
'type' => 'server_error',
],
])
);
}
/**
* Sanitize error response body.
*/
private function sanitizeErrorBody(?array $body): array
{
if ($body === null) {
return ['error' => ['message' => 'Unknown error', 'type' => 'server_error']];
}
if (isset($body['error']['message'])) {
$body['error']['message'] = $this->sanitizeMessage($body['error']['message']);
}
if (isset($body['message'])) {
$body['message'] = $this->sanitizeMessage($body['message']);
}
return $body;
}
/**
* Sanitize error messages to remove internal URLs/patterns.
*/
private function sanitizeMessage(string $message): string
{
$cleaned = preg_replace(self::SANITIZE_PATTERNS, '', $message);
return trim(preg_replace('/\s+/', ' ', $cleaned)) ?: 'An error occurred';
}
}

38
src/OpenAICompletionsInterface.php Normal file
View File

@@ -0,0 +1,38 @@
<?php
declare(strict_types=1);
namespace ContextPaging;
/**
* Interface for OpenAI token counting operations.
*/
interface OpenAICompletionsInterface
{
/**
* Count tokens in a string.
*
* @param string $text The text to tokenize.
* @param string $encoding One of: cl100k_base (GPT-4/3.5), o200k_base (GPT-4o/o1).
* @return int Token count.
* @throws \RuntimeException on process failure.
*/
public function count(string $text, string $encoding = 'cl100k_base'): int;
/**
* Count tokens for an array of messages (chat format).
* Concatenates all content with separator tokens for a rough count.
*
* @param array $messages Array of ['role' => '...', 'content' => '...']
* @param string $encoding Encoding name.
* @param int $perMessage Overhead tokens per message (4 for cl100k models).
* @param int $replyPrimer Tokens added after all messages (3 for cl100k models).
* @return int
*/
public function contextSize(
array $messages,
string $encoding = 'cl100k_base',
int $perMessage = 4,
int $replyPrimer = 3
): int;
}

103
src/RedisCache.php Normal file
View File

@@ -0,0 +1,103 @@
<?php
declare(strict_types=1);
namespace ContextPaging;
use Predis\ClientInterface;
/**
* Redis-backed cache implementation for persistent storage.
*/
class RedisCache implements CacheInterface
{
private ClientInterface $redis;
private string $prefix;
private ?int $defaultTtl;
/**
* @param ClientInterface $redis Predis client instance
* @param string $prefix Key prefix for namespacing (e.g., 'context_paging:')
* @param int|null $defaultTtl Default TTL in seconds (null = no expiry)
*/
public function __construct(
ClientInterface $redis,
string $prefix = 'ctx:',
?int $defaultTtl = null
) {
$this->redis = $redis;
$this->prefix = $prefix;
$this->defaultTtl = $defaultTtl;
}
/**
* Create from a Redis URL.
*
* @param string $url Redis connection URL (redis:// or rediss://)
* @param string $prefix Key prefix
* @param int|null $defaultTtl Default TTL
*/
public static function fromUrl(
string $url,
string $prefix = 'ctx:',
?int $defaultTtl = null
): self {
$redis = new \Predis\Client($url);
return new self($redis, $prefix, $defaultTtl);
}
public function get(string $key): mixed
{
$value = $this->redis->get($this->prefix . $key);
if ($value === null) {
return null;
}
return json_decode($value, true);
}
public function set(string $key, mixed $value, ?int $ttl = null): void
{
$fullKey = $this->prefix . $key;
$serialized = json_encode($value, JSON_UNESCAPED_UNICODE);
$ttl = $ttl ?? $this->defaultTtl;
if ($ttl !== null) {
$this->redis->setex($fullKey, $ttl, $serialized);
} else {
$this->redis->set($fullKey, $serialized);
}
}
public function has(string $key): bool
{
return (bool) $this->redis->exists($this->prefix . $key);
}
public function delete(string $key): void
{
$this->redis->del([$this->prefix . $key]);
}
public function clear(): void
{
// Use KEYS for simplicity (fine for dev/small datasets)
// For production with large datasets, use SCAN with proper cursor handling
$pattern = $this->prefix . '*';
$keys = $this->redis->keys($pattern);
if (!empty($keys)) {
$this->redis->del($keys);
}
}
/**
* Get the underlying Redis client.
*/
public function getRedis(): ClientInterface
{
return $this->redis;
}
}

20
src/SummarizerInterface.php Normal file
View File

@@ -0,0 +1,20 @@
<?php
declare(strict_types=1);
namespace ContextPaging;
/**
* Interface for message summarization.
*/
interface SummarizerInterface
{
/**
* Generate a concise summary of the given content.
*
* @param string $content The content to summarize
* @param array $context Optional context (e.g., conversation role, surrounding messages)
* @return string The summary
*/
public function summarize(string $content, array $context = []): string;
}

116
src/TokenCounter.php Normal file
View File

@@ -0,0 +1,116 @@
<?php
declare(strict_types=1);
namespace ContextPaging;
/**
* Token counter implementation using the token-counter binary.
*/
class TokenCounter implements OpenAICompletionsInterface
{
private string $binaryPath;
public function __construct(?string $binaryPath = null)
{
$this->binaryPath = $binaryPath ?? __DIR__ . '/../token-counter';
}
/**
* Count tokens in a string.
*
* @param string $text The text to tokenize.
* @param string $encoding One of: cl100k_base (GPT-4/3.5), o200k_base (GPT-4o/o1).
* @return int Token count.
* @throws \RuntimeException on process failure.
*/
public function count(string $text, string $encoding = 'cl100k_base'): int
{
$cmd = escapeshellarg($this->binaryPath) . ' ' . escapeshellarg($encoding);
$descriptors = [
0 => ['pipe', 'r'], // stdin - we write the text here
1 => ['pipe', 'w'], // stdout - we read the count here
2 => ['pipe', 'w'], // stderr - capture errors
];
$process = proc_open($cmd, $descriptors, $pipes);
if (!is_resource($process)) {
throw new \RuntimeException('Failed to start token-counter process');
}
// Write text to stdin in chunks to avoid pipe buffer deadlocks on large inputs.
$this->writeToStdin($pipes[0], $text);
fclose($pipes[0]);
// Read stdout (the token count)
$stdout = stream_get_contents($pipes[1]);
fclose($pipes[1]);
// Read stderr (any errors)
$stderr = stream_get_contents($pipes[2]);
fclose($pipes[2]);
$exitCode = proc_close($process);
if ($exitCode !== 0) {
throw new \RuntimeException("token-counter failed (exit {$exitCode}): {$stderr}");
}
$count = trim($stdout);
if (!ctype_digit($count)) {
throw new \RuntimeException("Unexpected output from token-counter: {$stdout}");
}
return (int) $count;
}
/**
* Count tokens for an array of messages (chat format).
* Concatenates all content with separator tokens for a rough count.
*
* @param array $messages Array of ['role' => '...', 'content' => '...']
* @param string $encoding Encoding name.
* @param int $perMessage Overhead tokens per message (4 for cl100k models).
* @param int $replyPrimer Tokens added after all messages (3 for cl100k models).
* @return int
*/
public function contextSize(
array $messages,
string $encoding = 'cl100k_base',
int $perMessage = 4,
int $replyPrimer = 3
): int {
// Concatenate all role + content so we only fork once.
$parts = [];
foreach ($messages as $msg) {
$parts[] = ($msg['role'] ?? '') . "\n" . ($msg['content'] ?? '');
}
$blob = implode("\n", $parts);
$rawTokens = $this->count($blob, $encoding);
// Add per-message overhead + reply primer
return $rawTokens + (count($messages) * $perMessage) + $replyPrimer;
}
/**
* Write large strings to stdin without deadlocking.
*/
private function writeToStdin($pipe, string $data): void
{
$chunkSize = 8192; // 8KB chunks
$offset = 0;
$length = strlen($data);
while ($offset < $length) {
$chunk = substr($data, $offset, $chunkSize);
$written = fwrite($pipe, $chunk);
if ($written === false) {
throw new \RuntimeException('Failed to write to token-counter stdin');
}
$offset += $written;
}
}
}

15
src/ToolCallMode.php Normal file
View File

@@ -0,0 +1,15 @@
<?php
declare(strict_types=1);
namespace ContextPaging;
/**
* Tool call parsing/formatting modes.
*/
enum ToolCallMode: string
{
case NATIVE = 'native'; // OpenAI-style tool_calls array
case RAW = 'raw'; // Raw XML/markers in content
case AUTO = 'auto'; // Auto-detect from response
}

211
src/ToolCallParser.php Normal file
View File

@@ -0,0 +1,211 @@
<?php
declare(strict_types=1);
namespace ContextPaging;
/**
* Parses tool calls from LLM responses.
*
* Supports two modes:
* - NATIVE: OpenAI-style tool_calls array in the response
* - RAW: Tool calls embedded in content as markers
*/
class ToolCallParser
{
private ToolCallMode $mode;
/**
* Regex patterns for raw tool calls.
* SmolLM3 and similar models output tool calls as markers in content.
*/
private const RAW_PATTERNS = [
// Primary pattern: markers with JSON
'/<tool_call>\s*(\{.*?\})\s*<\/tool_call>/s',
// Alternative: code block format
'/```tool_call\s*\n?\s*(\{.*?\})\s*\n?\s*```/s',
// Alternative: just JSON with tool_call context
'/\{"name"\s*:\s*"[^"]+"\s*,\s*"arguments"\s*:\s*\{.*?\}\s*\}/s',
];
public function __construct(ToolCallMode $mode = ToolCallMode::AUTO)
{
$this->mode = $mode;
}
/**
* Extract tool calls from a response.
*
* @param array $response The full API response (decoded JSON)
* @return array|null Array of tool calls, or null if none found
*/
public function extract(array $response): ?array
{
$message = $response['choices'][0]['message'] ?? null;
if (!$message) {
return null;
}
// Try native tool_calls first (if mode is NATIVE or AUTO)
if ($this->mode === ToolCallMode::NATIVE || $this->mode === ToolCallMode::AUTO) {
$nativeCalls = $this->extractNative($message);
if (!empty($nativeCalls)) {
return $nativeCalls;
}
}
// Try raw tool calls in content (if mode is RAW or AUTO)
if ($this->mode === ToolCallMode::RAW || $this->mode === ToolCallMode::AUTO) {
$rawCalls = $this->extractRaw($message);
if (!empty($rawCalls)) {
return $rawCalls;
}
}
return null;
}
/**
* Check if a response contains tool calls.
*/
public function hasToolCalls(array $response): bool
{
return $this->extract($response) !== null;
}
/**
* Get the detected mode from a response.
*
* @param array $response The API response
* @return ToolCallMode The detected mode, or current mode if set explicitly
*/
public function detectMode(array $response): ToolCallMode
{
if ($this->mode !== ToolCallMode::AUTO) {
return $this->mode;
}
$message = $response['choices'][0]['message'] ?? null;
if (!$message) {
return ToolCallMode::RAW; // Default fallback
}
// Check for native tool_calls
if (!empty($message['tool_calls'])) {
return ToolCallMode::NATIVE;
}
// Check for raw tool calls in content
$content = $message['content'] ?? '';
if ($this->contentHasRawToolCall($content)) {
return ToolCallMode::RAW;
}
return ToolCallMode::RAW; // No tool calls found
}
/**
* Check if content contains raw tool call markers.
*/
private function contentHasRawToolCall(string $content): bool
{
foreach (self::RAW_PATTERNS as $pattern) {
if (preg_match($pattern, $content)) {
return true;
}
}
return false;
}
/**
* Extract native OpenAI-style tool calls.
*/
private function extractNative(array $message): ?array
{
$toolCalls = $message['tool_calls'] ?? [];
if (empty($toolCalls)) {
return null;
}
$result = [];
foreach ($toolCalls as $call) {
$fn = $call['function'] ?? [];
$args = $fn['arguments'] ?? '{}';
// Parse JSON arguments
if (is_string($args)) {
$args = json_decode($args, true) ?? [];
}
$result[] = [
'id' => $call['id'] ?? null,
'name' => $fn['name'] ?? null,
'arguments' => $args,
];
}
return $result;
}
/**
* Extract raw tool calls from message content.
*/
private function extractRaw(array $message): ?array
{
$content = $message['content'] ?? '';
if (empty($content)) {
return null;
}
$result = [];
$seen = []; // Track MD5 of JSON to deduplicate
foreach (self::RAW_PATTERNS as $pattern) {
if (preg_match_all($pattern, $content, $matches, PREG_SET_ORDER)) {
foreach ($matches as $match) {
$json = $match[1] ?? $match[0];
// Deduplicate by content hash
$hash = md5($json);
if (isset($seen[$hash])) {
continue;
}
$seen[$hash] = true;
$decoded = json_decode($json, true);
if ($decoded && isset($decoded['name'])) {
$result[] = [
'id' => null,
'name' => $decoded['name'],
'arguments' => $decoded['arguments'] ?? [],
];
}
}
}
}
return empty($result) ? null : $result;
}
/**
* Set the parsing mode.
*/
public function setMode(ToolCallMode $mode): self
{
$this->mode = $mode;
return $this;
}
/**
* Get the current parsing mode.
*/
public function getMode(): ToolCallMode
{
return $this->mode;
}
}

236
src/ToolFormatter.php Normal file
View File

@@ -0,0 +1,236 @@
<?php
declare(strict_types=1);
namespace ContextPaging;
/**
* Formats tools for LLM requests.
*
* Supports two modes:
* - NATIVE: OpenAI-style tools array in the request
* - RAW: Tools described in system prompt with raw output format
*/
class ToolFormatter
{
private ToolCallMode $mode;
/**
* Tools available for the context paging dereference operation.
*/
public const FETCH_MESSAGE_TOOL = [
'name' => 'fetch_message',
'description' => 'Retrieve the full content of a summarized message when you need complete details to answer the current request.',
'parameters' => [
'type' => 'object',
'properties' => [
'md5' => [
'type' => 'string',
'description' => 'The MD5 hash of the message to retrieve (from the [md5:...] pointer)',
],
],
'required' => ['md5'],
],
];
public function __construct(ToolCallMode $mode = ToolCallMode::AUTO)
{
$this->mode = $mode;
}
/**
* Format tools for the request payload.
*
* @param array $tools Array of tool definitions
* @param ToolCallMode|null $mode Override mode
* @return array For NATIVE: ['tools' => [...], 'tool_choice' => 'auto']. For RAW: empty array (use buildPayload instead)
*/
public function formatForRequest(array $tools, ?ToolCallMode $mode = null): array
{
$useMode = $mode ?? $this->mode;
if ($useMode === ToolCallMode::NATIVE) {
return $this->formatNative($tools);
}
// RAW mode: tools are injected into system prompt, not the payload
// Return empty array - caller should use buildPayload() or injectToolsIntoSystemPrompt()
return [];
}
/**
* Format tools in OpenAI native format.
*
* Returns an array to be merged into the request payload:
* ['tools' => [...], 'tool_choice' => 'auto']
*/
public function formatNative(array $tools): array
{
$formatted = [];
foreach ($tools as $tool) {
$formatted[] = [
'type' => 'function',
'function' => [
'name' => $tool['name'],
'description' => $tool['description'] ?? '',
'parameters' => $tool['parameters'] ?? ['type' => 'object', 'properties' => new \stdClass()],
],
];
}
return [
'tools' => $formatted,
'tool_choice' => 'auto',
];
}
/**
* Format tools as raw text for system prompt.
*
* Returns a string to be appended to the system prompt.
*/
public function formatRaw(array $tools): string
{
$lines = [
'## Tools',
'You have access to the following tools:',
'',
];
foreach ($tools as $tool) {
$params = $tool['parameters'] ?? ['properties' => []];
$paramsJson = json_encode($params, JSON_PRETTY_PRINT);
$lines[] = "<tools>";
$lines[] = json_encode([
'name' => $tool['name'],
'description' => $tool['description'] ?? '',
'parameters' => $params,
], JSON_UNESCAPED_SLASHES);
$lines[] = "</tools>";
$lines[] = '';
}
$lines[] = 'When you need to use a tool, you MUST respond with ONLY a tool call in this exact format:';
$lines[] = '`<tool_call>{"name": "tool_name", "arguments": {"arg": "value"}}</tool_call>`';
$lines[] = '';
$lines[] = 'Do not explain how to use tools. Actually call them. The tool will be executed and you will receive the result.';
return implode("\n", $lines);
}
/**
* Build a system prompt with tools injected.
*
* @param string $existingSystemPrompt The existing system prompt
* @param array $tools Tools to inject
* @param ToolCallMode|null $mode Override mode
* @return string Modified system prompt
*/
public function injectToolsIntoSystemPrompt(
string $existingSystemPrompt,
array $tools,
?ToolCallMode $mode = null
): string {
$useMode = $mode ?? $this->mode;
// Native mode doesn't modify the system prompt
if ($useMode === ToolCallMode::NATIVE) {
return $existingSystemPrompt;
}
// Raw mode appends tool definitions to the system prompt
$toolText = $this->formatRaw($tools);
// Append with separator
return rtrim($existingSystemPrompt) . "\n\n" . $toolText;
}
/**
* Build the full request payload with tools.
*
* @param array $messages The messages array
* @param array $options Additional options (model, temperature, etc.)
* @param array $tools Tools to include
* @param ToolCallMode|null $mode Override mode
* @return array Complete request payload
*/
public function buildPayload(
array $messages,
array $options = [],
array $tools = [],
?ToolCallMode $mode = null
): array {
$useMode = $mode ?? $this->mode;
$payload = $options;
$payload['messages'] = $messages;
if (empty($tools)) {
return $payload;
}
if ($useMode === ToolCallMode::NATIVE) {
// Add tools to payload
$nativeTools = $this->formatNative($tools);
$payload['tools'] = $nativeTools['tools'];
$payload['tool_choice'] = $nativeTools['tool_choice'];
} else {
// Inject tools into system prompt (no tools key in payload)
$payload['messages'] = $this->injectToolsIntoMessages($messages, $tools);
}
return $payload;
}
/**
* Inject tools into the messages array by modifying the system prompt.
*/
public function injectToolsIntoMessages(array $messages, array $tools): array
{
$result = [];
$injected = false;
$toolText = $this->formatRaw($tools);
foreach ($messages as $message) {
if (!$injected && ($message['role'] ?? null) === 'system') {
// Inject tools into the first system message
$result[] = [
'role' => 'system',
'content' => rtrim($message['content'] ?? '') . "\n\n" . $toolText,
];
$injected = true;
} else {
$result[] = $message;
}
}
// If no system message, prepend one
if (!$injected) {
array_unshift($result, [
'role' => 'system',
'content' => $toolText,
]);
}
return $result;
}
/**
* Set the formatting mode.
*/
public function setMode(ToolCallMode $mode): self
{
$this->mode = $mode;
return $this;
}
/**
* Get the current mode.
*/
public function getMode(): ToolCallMode
{
return $this->mode;
}
}