feat: add windows ingest client for mcp server
This commit is contained in:
parent
a0e0bd293a
commit
2416692e8d
|
|
@ -2,6 +2,7 @@
|
|||
__pycache__/
|
||||
*.pyc
|
||||
.pytest_cache/
|
||||
.pytest-bt/
|
||||
.mypy_cache/
|
||||
.ruff_cache/
|
||||
.venv/
|
||||
|
|
@ -10,4 +11,4 @@ env/
|
|||
build/
|
||||
dist/
|
||||
*.egg-info/
|
||||
|
||||
tmp*/
|
||||
|
|
|
|||
|
|
@ -0,0 +1,43 @@
|
|||
{
|
||||
"version": "0.2.0",
|
||||
"configurations": [
|
||||
{
|
||||
"name": "TechLib Ingest Client",
|
||||
"type": "python",
|
||||
"request": "launch",
|
||||
"module": "techlib_mcpserver.client",
|
||||
"cwd": "${workspaceFolder}",
|
||||
"env": {
|
||||
"PYTHONPATH": "${workspaceFolder}\\src"
|
||||
},
|
||||
"console": "integratedTerminal",
|
||||
"justMyCode": true,
|
||||
"args": [
|
||||
"--source",
|
||||
"${workspaceFolder}\\in",
|
||||
"--recursive",
|
||||
"--server-url",
|
||||
"http://192.168.201.91:8888"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "TechLib Ingest Client Prompt",
|
||||
"type": "python",
|
||||
"request": "launch",
|
||||
"module": "techlib_mcpserver.client",
|
||||
"cwd": "${workspaceFolder}",
|
||||
"env": {
|
||||
"PYTHONPATH": "${workspaceFolder}\\src"
|
||||
},
|
||||
"console": "integratedTerminal",
|
||||
"justMyCode": true,
|
||||
"args": [
|
||||
"--source",
|
||||
"${workspaceFolder}\\in\\Huawei CloudEngine 6820 Switch Datasheet.pdf",
|
||||
"--emit-prompt",
|
||||
"--server-url",
|
||||
"http://192.168.201.91:8888"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
{
|
||||
"version": "2.0.0",
|
||||
"tasks": [
|
||||
{
|
||||
"label": "TechLib: ingest folder",
|
||||
"type": "shell",
|
||||
"command": "powershell.exe",
|
||||
"args": [
|
||||
"-ExecutionPolicy",
|
||||
"Bypass",
|
||||
"-File",
|
||||
"${workspaceFolder}\\scripts\\windows\\run_ingest_client.ps1",
|
||||
"-Source",
|
||||
"${workspaceFolder}\\in",
|
||||
"-Recursive",
|
||||
"-ServerUrl",
|
||||
"http://192.168.201.91:8888"
|
||||
],
|
||||
"problemMatcher": []
|
||||
},
|
||||
{
|
||||
"label": "TechLib: emit prompt",
|
||||
"type": "shell",
|
||||
"command": "powershell.exe",
|
||||
"args": [
|
||||
"-ExecutionPolicy",
|
||||
"Bypass",
|
||||
"-File",
|
||||
"${workspaceFolder}\\scripts\\windows\\run_ingest_client.ps1",
|
||||
"-Source",
|
||||
"${workspaceFolder}\\in\\Huawei CloudEngine 6820 Switch Datasheet.pdf",
|
||||
"-EmitPrompt",
|
||||
"-ServerUrl",
|
||||
"http://192.168.201.91:8888"
|
||||
],
|
||||
"problemMatcher": []
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -75,6 +75,7 @@ The project integrates with:
|
|||
## Documentation Map
|
||||
|
||||
- [Usage Guide](docs/usage-guide.md)
|
||||
- [Windows Ingest Client](docs/windows-ingest-client.md)
|
||||
- [Deployment Guide](docs/deployment-guide.md)
|
||||
- [PostgreSQL Operations](docs/postgresql.md)
|
||||
- [Content Delivery](docs/content-delivery.md)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
# Usage Guide
|
||||
# Usage Guide
|
||||
|
||||
This guide covers local usage of TechDoc Library MCP Server v1.
|
||||
|
||||
|
|
@ -25,6 +25,8 @@ The service exposes two compatible MCP entry points:
|
|||
- Native MCP transport at `POST /mcp` for JSON-RPC `initialize`, `tools/list`, and `tools/call`
|
||||
- HTTP facade at `/mcp/*` for Swagger, `curl`, and manual diagnostics
|
||||
|
||||
For Windows/VSCode-based regular ingest, use the dedicated client described in [Windows Ingest Client](windows-ingest-client.md).
|
||||
|
||||
## MCP Tool Overview
|
||||
|
||||
The MCP facade exposes version-aware document library tools:
|
||||
|
|
@ -110,4 +112,5 @@ curl http://127.0.0.1:8000/health
|
|||
|
||||
- [Deployment Guide](docs/deployment-guide.md)
|
||||
- [PostgreSQL Operations](docs/postgresql.md)
|
||||
- [Content Delivery](docs/content-delivery.md)
|
||||
- [Content Delivery](docs/content-delivery.md)
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,87 @@
|
|||
# Windows Ingest Client
|
||||
|
||||
This repository includes a Windows-friendly ingest client for running from PowerShell or VSCode.
|
||||
|
||||
## What It Does
|
||||
|
||||
The client:
|
||||
|
||||
- reads one file or a folder of files
|
||||
- extracts text locally
|
||||
- normalizes document metainfo
|
||||
- probes the live MCP server capabilities
|
||||
- ingests through native MCP when available, otherwise falls back to the HTTP facade
|
||||
- skips duplicates using a local ledger file
|
||||
|
||||
## Running From PowerShell
|
||||
|
||||
Example:
|
||||
|
||||
```powershell
|
||||
$env:PYTHONPATH = "$PWD\src"
|
||||
$env:TECHLIB_INGEST_SERVER_URL = 'http://192.168.201.91:8888'
|
||||
python -m techlib_mcpserver.client --source .\in --recursive
|
||||
```
|
||||
|
||||
If you use the bundled wrapper script, it sets `PYTHONPATH` automatically:
|
||||
|
||||
```powershell
|
||||
powershell.exe -ExecutionPolicy Bypass -File .\scripts\windows\run_ingest_client.ps1 -Source .\in -Recursive -ServerUrl http://192.168.201.91:8888
|
||||
```
|
||||
|
||||
Optional prompt-assisted flow for a single file:
|
||||
|
||||
```powershell
|
||||
$env:PYTHONPATH = "$PWD\src"
|
||||
python -m techlib_mcpserver.client --source .\in\Huawei CloudEngine 6820 Switch Datasheet.pdf --emit-prompt > prompt.txt
|
||||
```
|
||||
|
||||
Feed the prompt into your preferred model, save strict JSON metainfo to a file, then ingest with:
|
||||
|
||||
```powershell
|
||||
$env:PYTHONPATH = "$PWD\src"
|
||||
python -m techlib_mcpserver.client --source .\in\Huawei CloudEngine 6820 Switch Datasheet.pdf --prompt-json .\metainfo.json
|
||||
```
|
||||
|
||||
## Environment Variables
|
||||
|
||||
The client reads `.env.local` and `.env` files automatically.
|
||||
Use these variables for repeatable runs:
|
||||
|
||||
- `TECHLIB_INGEST_SERVER_URL`
|
||||
- `TECHLIB_INGEST_TRANSPORT`
|
||||
- `TECHLIB_INGEST_STATE_FILE`
|
||||
- `TECHLIB_INGEST_REQUEST_TIMEOUT_SECONDS`
|
||||
- `TECHLIB_INGEST_PROMPT_PREVIEW_CHARS`
|
||||
|
||||
## VSCode
|
||||
|
||||
Use the provided launch and task files for debugging or repeated runs. They call the same Python CLI, so behavior matches PowerShell runs.
|
||||
|
||||
## Prompt Contract
|
||||
|
||||
Prompt output must be strict JSON and must not include `subject_payload`.
|
||||
The expected shape is:
|
||||
|
||||
```json
|
||||
{
|
||||
"title": "string or null",
|
||||
"document_type_name": "string",
|
||||
"content_type": "string or null",
|
||||
"subjects": [
|
||||
{
|
||||
"vendor_name": "string or null",
|
||||
"equipment_type": "string or null",
|
||||
"model": "string or null",
|
||||
"model_family": "string or null"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
## Notes
|
||||
|
||||
- If the live server does not support structured subjects yet, the client falls back to the legacy flat ingest payload.
|
||||
- Duplicate suppression is local to the client and keyed by `source_uri + checksum + server_url`.
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
# Ingest client settings for Windows / VSCode
|
||||
# Copy to .env.local or set in your shell profile.
|
||||
|
||||
TECHLIB_INGEST_SERVER_URL=http://192.168.201.91:8888
|
||||
TECHLIB_INGEST_TRANSPORT=auto
|
||||
TECHLIB_INGEST_STATE_FILE=C:\Users\%USERNAME%\AppData\Roaming\techlib_mcpserver_ingest\state.json
|
||||
TECHLIB_INGEST_REQUEST_TIMEOUT_SECONDS=60
|
||||
TECHLIB_INGEST_PROMPT_PREVIEW_CHARS=12000
|
||||
# TECHLIB_INGEST_LOG_FILE=C:\Users\%USERNAME%\AppData\Roaming\techlib_mcpserver_ingest\ingest.log
|
||||
|
|
@ -24,9 +24,13 @@ dependencies = [
|
|||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
client = ["pypdf>=6,<7"]
|
||||
test = ["pytest>=8,<9"]
|
||||
dev = ["pytest>=8,<9"]
|
||||
|
||||
[project.scripts]
|
||||
techlib-ingest-client = "techlib_mcpserver.client.cli:main"
|
||||
|
||||
[tool.setuptools]
|
||||
package-dir = {"" = "src"}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,48 @@
|
|||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$Source,
|
||||
[string]$ServerUrl = $env:TECHLIB_INGEST_SERVER_URL,
|
||||
[ValidateSet('auto', 'mcp', 'http')]
|
||||
[string]$Transport = 'auto',
|
||||
[switch]$Recursive,
|
||||
[switch]$Force,
|
||||
[string]$StateFile = $env:TECHLIB_INGEST_STATE_FILE,
|
||||
[string]$PromptJson,
|
||||
[switch]$EmitPrompt,
|
||||
[string]$LogFile = $env:TECHLIB_INGEST_LOG_FILE,
|
||||
[int]$PreviewChars = 12000
|
||||
)
|
||||
|
||||
$root = Resolve-Path (Join-Path $PSScriptRoot '..\..')
|
||||
$srcPath = Join-Path $root 'src'
|
||||
if ($env:PYTHONPATH) {
|
||||
$env:PYTHONPATH = "$srcPath;$env:PYTHONPATH"
|
||||
} else {
|
||||
$env:PYTHONPATH = $srcPath
|
||||
}
|
||||
|
||||
$pythonCandidates = @(
|
||||
(Join-Path $root '.venv\Scripts\python.exe'),
|
||||
'python'
|
||||
)
|
||||
$python = $pythonCandidates | Where-Object { Test-Path $_ } | Select-Object -First 1
|
||||
if (-not $python) {
|
||||
$python = 'python'
|
||||
}
|
||||
|
||||
$argsList = @(
|
||||
'-m', 'techlib_mcpserver.client',
|
||||
'--source', $Source,
|
||||
'--transport', $Transport,
|
||||
'--preview-chars', $PreviewChars
|
||||
)
|
||||
if ($ServerUrl) { $argsList += @('--server-url', $ServerUrl) }
|
||||
if ($Recursive) { $argsList += '--recursive' }
|
||||
if ($Force) { $argsList += '--force' }
|
||||
if ($StateFile) { $argsList += @('--state-file', $StateFile) }
|
||||
if ($PromptJson) { $argsList += @('--prompt-json', $PromptJson) }
|
||||
if ($EmitPrompt) { $argsList += '--emit-prompt' }
|
||||
if ($LogFile) { $argsList += @('--log-file', $LogFile) }
|
||||
|
||||
& $python @argsList
|
||||
exit $LASTEXITCODE
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
"""Windows-friendly ingest client package."""
|
||||
|
||||
__all__ = []
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
"""Module entrypoint for the ingest client."""
|
||||
|
||||
from .cli import main
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -0,0 +1,232 @@
|
|||
"""CLI entrypoint for the Windows-friendly ingest client."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from techlib_mcpserver.app.logging import StructuredFormatter, configure_logging, get_logger
|
||||
|
||||
from .config import IngestClientSettings
|
||||
from .metadata import build_ingest_plan, canonical_subject_lines, iter_source_documents
|
||||
from .prompting import build_metadata_prompt, load_prompt_metainfo
|
||||
from .state import IngestLedger
|
||||
from .transport import MCPIngestClient
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="techlib-ingest-client",
|
||||
description="Windows-friendly ingest client for the TechDoc Library MCP server.",
|
||||
)
|
||||
parser.add_argument("--source", required=True, help="File or directory to ingest.")
|
||||
parser.add_argument(
|
||||
"--server-url",
|
||||
default=None,
|
||||
help="MCP server base URL, for example http://192.168.201.91:8888.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--transport",
|
||||
choices=("auto", "mcp", "http"),
|
||||
default=None,
|
||||
help="Preferred transport mode.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--recursive",
|
||||
action="store_true",
|
||||
help="Recurse into subdirectories when source is a directory.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--force",
|
||||
action="store_true",
|
||||
help="Reingest even if source URI and checksum were already seen.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--state-file",
|
||||
default=None,
|
||||
help="Path to the local ingest ledger JSON file.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--prompt-json",
|
||||
default=None,
|
||||
help="Path to strict JSON metainfo output from a prompt or external model.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--emit-prompt",
|
||||
action="store_true",
|
||||
help="Print a normalization prompt and exit instead of ingesting.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--log-file",
|
||||
default=None,
|
||||
help="Optional log file path.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--preview-chars",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Number of characters to include in prompt previews.",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def _apply_overrides(settings: IngestClientSettings, args: argparse.Namespace) -> IngestClientSettings:
|
||||
if args.server_url is not None:
|
||||
settings.server_url = args.server_url
|
||||
if args.transport is not None:
|
||||
settings.transport = args.transport
|
||||
if args.recursive:
|
||||
settings.recursive = True
|
||||
if args.force:
|
||||
settings.force = True
|
||||
if args.state_file is not None:
|
||||
settings.state_file = Path(args.state_file)
|
||||
if args.log_file is not None:
|
||||
settings.log_file = Path(args.log_file)
|
||||
if args.preview_chars is not None:
|
||||
settings.prompt_preview_chars = args.preview_chars
|
||||
return settings
|
||||
|
||||
|
||||
def _configure_file_logging(log_file: Path) -> None:
|
||||
log_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
handler = logging.FileHandler(log_file, encoding="utf-8")
|
||||
handler.setFormatter(StructuredFormatter(datefmt="%Y-%m-%dT%H:%M:%S%z"))
|
||||
logging.getLogger().addHandler(handler)
|
||||
|
||||
|
||||
def _configure_stdio() -> None:
|
||||
for stream_name in ("stdout", "stderr"):
|
||||
stream = getattr(sys, stream_name)
|
||||
if hasattr(stream, "reconfigure"):
|
||||
stream.reconfigure(encoding="utf-8", errors="replace")
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
_configure_stdio()
|
||||
parser = build_parser()
|
||||
args = parser.parse_args(argv)
|
||||
settings = _apply_overrides(IngestClientSettings.from_env(), args)
|
||||
|
||||
configure_logging()
|
||||
if settings.log_file is not None:
|
||||
_configure_file_logging(settings.log_file)
|
||||
logger = get_logger(
|
||||
__name__,
|
||||
app_name="techlib_ingest_client",
|
||||
app_env="windows-vscode",
|
||||
)
|
||||
|
||||
source_path = Path(args.source).expanduser().resolve()
|
||||
if not source_path.exists():
|
||||
parser.error(f"source does not exist: {source_path}")
|
||||
|
||||
prompt_metainfo = None
|
||||
if args.prompt_json:
|
||||
prompt_metainfo = load_prompt_metainfo(args.prompt_json)
|
||||
|
||||
client = MCPIngestClient(
|
||||
settings.server_url,
|
||||
transport_preference=settings.transport,
|
||||
timeout_seconds=settings.request_timeout_seconds,
|
||||
)
|
||||
capabilities = client.probe()
|
||||
logger.info(
|
||||
"server capabilities probed: %s",
|
||||
json.dumps(
|
||||
{
|
||||
"server_url": capabilities.server_url,
|
||||
"native_mcp_available": capabilities.native_mcp_available,
|
||||
"http_facade_available": capabilities.http_facade_available,
|
||||
"supports_structured_subjects": capabilities.supports_structured_subjects,
|
||||
"native_tool_names": capabilities.native_tool_names,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
)
|
||||
|
||||
ledger = IngestLedger(settings.state_file)
|
||||
paths = iter_source_documents(source_path, settings.recursive, settings.source_extensions)
|
||||
if not paths:
|
||||
logger.warning("no ingestable documents found under %s", source_path)
|
||||
return 0
|
||||
|
||||
if args.emit_prompt:
|
||||
if len(paths) != 1:
|
||||
parser.error("--emit-prompt currently supports exactly one file")
|
||||
document = build_ingest_plan(paths[0], prompt_metainfo=prompt_metainfo)
|
||||
print(
|
||||
build_metadata_prompt(
|
||||
source_path=document.source.path,
|
||||
source_preview=document.source.text_content,
|
||||
prompt_preview_chars=settings.prompt_preview_chars,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
ingested = 0
|
||||
skipped = 0
|
||||
for path in paths:
|
||||
plan = build_ingest_plan(path, prompt_metainfo=prompt_metainfo)
|
||||
source = plan.source
|
||||
if not args.force and ledger.seen(source.source_uri, source.checksum_sha256, settings.server_url):
|
||||
logger.info(
|
||||
"skip duplicate source_uri=%s checksum=%s",
|
||||
source.source_uri,
|
||||
source.checksum_sha256,
|
||||
)
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
logger.info(
|
||||
"ingesting %s",
|
||||
json.dumps(
|
||||
{
|
||||
"source_uri": source.source_uri,
|
||||
"title": plan.metainfo.title,
|
||||
"document_type_name": plan.metainfo.document_type_name,
|
||||
"subjects": canonical_subject_lines(plan.metainfo.subjects),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
)
|
||||
result = client.ingest(plan)
|
||||
payload = result.payload
|
||||
if "error" in payload:
|
||||
raise SystemExit(json.dumps(payload, ensure_ascii=False, indent=2))
|
||||
|
||||
structured = payload.get("result", {}).get("structuredContent")
|
||||
document_id = None
|
||||
version_id = None
|
||||
if isinstance(structured, dict):
|
||||
document = structured.get("document", {})
|
||||
version = structured.get("version", {})
|
||||
if isinstance(document, dict):
|
||||
document_id = document.get("document_id")
|
||||
if isinstance(version, dict):
|
||||
version_id = version.get("version_id")
|
||||
|
||||
ledger.record(
|
||||
source_uri=source.source_uri,
|
||||
checksum_sha256=source.checksum_sha256,
|
||||
server_url=settings.server_url,
|
||||
document_id=document_id,
|
||||
version_id=version_id,
|
||||
)
|
||||
ingested += 1
|
||||
logger.info(
|
||||
"ingest completed transport=%s document_id=%s version_id=%s",
|
||||
result.transport,
|
||||
document_id,
|
||||
version_id,
|
||||
)
|
||||
|
||||
logger.info("run completed ingested=%s skipped=%s", ingested, skipped)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
|
|
@ -0,0 +1,98 @@
|
|||
"""Runtime settings for the Windows-friendly ingest client."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
|
||||
def _parse_env_file(path: Path) -> dict[str, str]:
|
||||
if not path.exists():
|
||||
return {}
|
||||
values: dict[str, str] = {}
|
||||
for raw_line in path.read_text(encoding="utf-8").splitlines():
|
||||
line = raw_line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, value = line.split("=", 1)
|
||||
values[key.strip()] = value.strip().strip('"').strip("'")
|
||||
return values
|
||||
|
||||
|
||||
def _expand_path(value: str) -> Path:
|
||||
return Path(os.path.expandvars(value)).expanduser()
|
||||
|
||||
|
||||
def _as_bool(value: str | None, default: bool) -> bool:
|
||||
if value is None:
|
||||
return default
|
||||
normalized = value.strip().lower()
|
||||
if normalized in {"1", "true", "yes", "on"}:
|
||||
return True
|
||||
if normalized in {"0", "false", "no", "off"}:
|
||||
return False
|
||||
return default
|
||||
|
||||
|
||||
def _as_transport(value: str | None, default: str) -> Literal["auto", "mcp", "http"]:
|
||||
if value is None:
|
||||
return default # type: ignore[return-value]
|
||||
normalized = value.strip().lower()
|
||||
if normalized in {"auto", "mcp", "http"}:
|
||||
return normalized # type: ignore[return-value]
|
||||
return default # type: ignore[return-value]
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class IngestClientSettings:
|
||||
server_url: str = "http://127.0.0.1:8888"
|
||||
transport: Literal["auto", "mcp", "http"] = "auto"
|
||||
recursive: bool = False
|
||||
force: bool = False
|
||||
state_file: Path = field(
|
||||
default_factory=lambda: Path.home() / ".techlib_mcpserver_ingest" / "state.json"
|
||||
)
|
||||
log_file: Path | None = None
|
||||
prompt_preview_chars: int = 12000
|
||||
source_extensions: tuple[str, ...] = (
|
||||
".pdf",
|
||||
".txt",
|
||||
".md",
|
||||
".rst",
|
||||
".json",
|
||||
".yaml",
|
||||
".yml",
|
||||
".csv",
|
||||
".log",
|
||||
)
|
||||
request_timeout_seconds: float = 60.0
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> "IngestClientSettings":
|
||||
env = {}
|
||||
for filename in (".env.local", ".env"):
|
||||
env.update(_parse_env_file(Path.cwd() / filename))
|
||||
env.update(os.environ)
|
||||
settings = cls()
|
||||
settings.server_url = env.get("TECHLIB_INGEST_SERVER_URL", settings.server_url)
|
||||
settings.transport = _as_transport(env.get("TECHLIB_INGEST_TRANSPORT"), settings.transport)
|
||||
settings.recursive = _as_bool(env.get("TECHLIB_INGEST_RECURSIVE"), settings.recursive)
|
||||
settings.force = _as_bool(env.get("TECHLIB_INGEST_FORCE"), settings.force)
|
||||
if env.get("TECHLIB_INGEST_STATE_FILE"):
|
||||
settings.state_file = _expand_path(env["TECHLIB_INGEST_STATE_FILE"])
|
||||
if env.get("TECHLIB_INGEST_LOG_FILE"):
|
||||
settings.log_file = _expand_path(env["TECHLIB_INGEST_LOG_FILE"])
|
||||
if env.get("TECHLIB_INGEST_PROMPT_PREVIEW_CHARS"):
|
||||
settings.prompt_preview_chars = int(env["TECHLIB_INGEST_PROMPT_PREVIEW_CHARS"])
|
||||
if env.get("TECHLIB_INGEST_REQUEST_TIMEOUT_SECONDS"):
|
||||
settings.request_timeout_seconds = float(env["TECHLIB_INGEST_REQUEST_TIMEOUT_SECONDS"])
|
||||
if env.get("TECHLIB_INGEST_SOURCE_EXTENSIONS"):
|
||||
settings.source_extensions = tuple(
|
||||
part.strip() for part in env["TECHLIB_INGEST_SOURCE_EXTENSIONS"].split(",") if part.strip()
|
||||
)
|
||||
return settings
|
||||
|
||||
|
||||
__all__ = ["IngestClientSettings"]
|
||||
|
|
@ -0,0 +1,284 @@
|
|||
"""Deterministic metadata extraction helpers for ingest planning."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import mimetypes
|
||||
import re
|
||||
from hashlib import sha256
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
|
||||
from .models import IngestPlan, PromptMetainfoInput, PromptSubjectInput, SourceDocument
|
||||
|
||||
KNOWN_VENDORS = (
|
||||
"Huawei",
|
||||
"Eltex",
|
||||
"Cisco",
|
||||
"Juniper",
|
||||
"HPE",
|
||||
"Aruba",
|
||||
"MikroTik",
|
||||
"D-Link",
|
||||
"Ruijie",
|
||||
"Extreme",
|
||||
"Nokia",
|
||||
"Mellanox",
|
||||
)
|
||||
|
||||
DOCUMENT_TYPE_KEYWORDS = (
|
||||
("datasheet", "datasheet"),
|
||||
("spec sheet", "datasheet"),
|
||||
("manual", "manual"),
|
||||
("quick start", "quick_start_guide"),
|
||||
("quickstart", "quick_start_guide"),
|
||||
("install", "installation_guide"),
|
||||
("configuration", "configuration_guide"),
|
||||
("troubleshooting", "troubleshooting_guide"),
|
||||
("release notes", "release_notes"),
|
||||
("specification", "specification"),
|
||||
("spec", "specification"),
|
||||
("guide", "technical_description"),
|
||||
)
|
||||
|
||||
EQUIPMENT_KEYWORDS = (
|
||||
("switch", "switch"),
|
||||
("router", "router"),
|
||||
("firewall", "firewall"),
|
||||
("access point", "access_point"),
|
||||
("transceiver", "transceiver"),
|
||||
("module", "module"),
|
||||
("controller", "controller"),
|
||||
)
|
||||
|
||||
PDF_MODEL_RE = re.compile(r"\b([A-Z]{1,4}\d{3,5}(?:-[A-Z0-9]+)*)\b")
|
||||
HUAWEI_MODEL_RE = re.compile(r"\b(CE\d{4,5}-[A-Z0-9-]+)\b")
|
||||
|
||||
|
||||
def checksum_sha256(data: bytes) -> str:
|
||||
return sha256(data).hexdigest()
|
||||
|
||||
|
||||
def guess_content_type(path: Path) -> str:
|
||||
content_type, _ = mimetypes.guess_type(path.name)
|
||||
if content_type:
|
||||
return content_type
|
||||
if path.suffix.lower() == ".pdf":
|
||||
return "application/pdf"
|
||||
return "text/plain"
|
||||
|
||||
|
||||
def infer_title(path: Path, text: str | None = None, *, pdf_title: str | None = None) -> str:
|
||||
if pdf_title and pdf_title.strip():
|
||||
return pdf_title.strip()
|
||||
if text:
|
||||
first_line = text.strip().splitlines()[0].strip()
|
||||
if first_line:
|
||||
return first_line[:160]
|
||||
stem = path.stem.replace("_", " ").replace("-", " ")
|
||||
stem = re.sub(r"\s+", " ", stem).strip()
|
||||
return stem[:160]
|
||||
|
||||
|
||||
def extract_document_text(path: Path, raw_bytes: bytes) -> tuple[str, str, str]:
|
||||
suffix = path.suffix.lower()
|
||||
if suffix == ".pdf":
|
||||
return _extract_pdf_text(path)
|
||||
text = raw_bytes.decode("utf-8", errors="replace").strip()
|
||||
return text, "text/plain", infer_title(path, text)
|
||||
|
||||
|
||||
def read_document(path: Path) -> SourceDocument:
|
||||
raw_bytes = path.read_bytes()
|
||||
extracted_text, source_content_type, title = extract_document_text(path, raw_bytes)
|
||||
return SourceDocument(
|
||||
path=path,
|
||||
checksum_sha256=checksum_sha256(raw_bytes),
|
||||
raw_bytes=raw_bytes,
|
||||
text_content=extracted_text,
|
||||
extracted_content_type=source_content_type,
|
||||
source_uri=path.resolve().as_uri(),
|
||||
source_file_name=path.name,
|
||||
title=title,
|
||||
)
|
||||
|
||||
|
||||
def infer_document_type_name(text: str, title: str) -> str:
|
||||
haystack = f"{title}\n{text}".lower()
|
||||
for keyword, document_type_name in DOCUMENT_TYPE_KEYWORDS:
|
||||
if keyword in haystack:
|
||||
return document_type_name
|
||||
return "technical_description"
|
||||
|
||||
|
||||
def infer_vendor_name(text: str, title: str) -> str | None:
|
||||
haystack = f"{title}\n{text}"
|
||||
for vendor in KNOWN_VENDORS:
|
||||
if re.search(rf"\b{re.escape(vendor)}\b", haystack, flags=re.IGNORECASE):
|
||||
return vendor
|
||||
return None
|
||||
|
||||
|
||||
def infer_equipment_type(text: str, title: str) -> str | None:
|
||||
haystack = f"{title}\n{text}".lower()
|
||||
for keyword, equipment_type in EQUIPMENT_KEYWORDS:
|
||||
if keyword in haystack:
|
||||
return equipment_type
|
||||
return None
|
||||
|
||||
|
||||
def infer_model(text: str, title: str, vendor_name: str | None = None) -> str | None:
|
||||
haystack = f"{title}\n{text}"
|
||||
if vendor_name and vendor_name.lower() == "huawei":
|
||||
match = HUAWEI_MODEL_RE.search(haystack)
|
||||
if match:
|
||||
return match.group(1)
|
||||
match = PDF_MODEL_RE.search(haystack)
|
||||
if match:
|
||||
return match.group(1)
|
||||
return None
|
||||
|
||||
|
||||
def infer_model_family(text: str, title: str, model: str | None = None) -> str | None:
|
||||
haystack = f"{title}\n{text}"
|
||||
if "CloudEngine 6820" in haystack:
|
||||
return "CloudEngine 6820 series"
|
||||
if model:
|
||||
cleaned = model
|
||||
if "-" in cleaned:
|
||||
cleaned = cleaned.split("-")[0]
|
||||
return f"{cleaned} family"
|
||||
return None
|
||||
|
||||
|
||||
def build_subjects(
|
||||
text: str,
|
||||
title: str,
|
||||
*,
|
||||
fallback_vendor: str | None = None,
|
||||
) -> tuple[PromptSubjectInput, ...]:
|
||||
vendor_name = infer_vendor_name(text, title) or fallback_vendor
|
||||
equipment_type = infer_equipment_type(text, title)
|
||||
model = infer_model(text, title, vendor_name=vendor_name)
|
||||
model_family = infer_model_family(text, title, model=model)
|
||||
return (
|
||||
PromptSubjectInput(
|
||||
vendor_name=vendor_name,
|
||||
equipment_type=equipment_type,
|
||||
model=model,
|
||||
model_family=model_family,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def infer_metainfo_from_document(
|
||||
path: Path,
|
||||
*,
|
||||
text: str,
|
||||
title: str,
|
||||
content_type: str,
|
||||
fallback_vendor: str | None = None,
|
||||
prompt_metainfo: PromptMetainfoInput | None = None,
|
||||
) -> PromptMetainfoInput:
|
||||
inferred_subjects = build_subjects(text, title, fallback_vendor=fallback_vendor)
|
||||
metainfo = PromptMetainfoInput(
|
||||
title=title,
|
||||
document_type_name=infer_document_type_name(text, title),
|
||||
content_type=content_type,
|
||||
subjects=list(inferred_subjects),
|
||||
)
|
||||
if prompt_metainfo is None:
|
||||
return metainfo
|
||||
return PromptMetainfoInput(
|
||||
title=prompt_metainfo.title or metainfo.title,
|
||||
document_type_name=prompt_metainfo.document_type_name or metainfo.document_type_name,
|
||||
content_type=prompt_metainfo.content_type or metainfo.content_type,
|
||||
subjects=prompt_metainfo.subjects or metainfo.subjects,
|
||||
)
|
||||
|
||||
|
||||
def build_ingest_plan(
|
||||
path: Path,
|
||||
*,
|
||||
prompt_metainfo: PromptMetainfoInput | None = None,
|
||||
fallback_vendor: str | None = None,
|
||||
) -> IngestPlan:
|
||||
source = read_document(path)
|
||||
metainfo = infer_metainfo_from_document(
|
||||
path,
|
||||
text=source.text_content,
|
||||
title=source.title,
|
||||
content_type=source.extracted_content_type,
|
||||
fallback_vendor=fallback_vendor,
|
||||
prompt_metainfo=prompt_metainfo,
|
||||
)
|
||||
return IngestPlan(source=source, metainfo=metainfo)
|
||||
|
||||
|
||||
def iter_source_documents(source: Path, recursive: bool, extensions: Iterable[str]) -> list[Path]:
|
||||
if source.is_file():
|
||||
return [source]
|
||||
extension_set = {ext.lower() for ext in extensions}
|
||||
candidates = source.rglob("*") if recursive else source.iterdir()
|
||||
return [
|
||||
candidate
|
||||
for candidate in candidates
|
||||
if candidate.is_file() and candidate.suffix.lower() in extension_set
|
||||
]
|
||||
|
||||
|
||||
def canonical_subject_lines(subjects: Iterable[PromptSubjectInput]) -> list[str]:
|
||||
lines: list[str] = []
|
||||
for subject in subjects:
|
||||
parts = []
|
||||
if subject.vendor_name:
|
||||
parts.append(subject.vendor_name)
|
||||
if subject.equipment_type:
|
||||
parts.append(subject.equipment_type)
|
||||
if subject.model:
|
||||
parts.append(subject.model)
|
||||
if subject.model_family:
|
||||
parts.append(subject.model_family)
|
||||
if parts:
|
||||
lines.append(" / ".join(parts))
|
||||
return lines
|
||||
|
||||
|
||||
def _extract_pdf_text(path: Path) -> tuple[str, str, str]:
|
||||
try:
|
||||
from pypdf import PdfReader
|
||||
except ImportError as exc: # pragma: no cover - exercised manually if dependency missing
|
||||
raise RuntimeError(
|
||||
"PDF ingest requires the optional 'pypdf' dependency. Install techlib_mcpserver[client]."
|
||||
) from exc
|
||||
|
||||
reader = PdfReader(path)
|
||||
text_parts: list[str] = []
|
||||
for page in reader.pages:
|
||||
page_text = (page.extract_text() or "").strip()
|
||||
if page_text:
|
||||
text_parts.append(page_text)
|
||||
|
||||
pdf_title = None
|
||||
metadata = getattr(reader, "metadata", None)
|
||||
if metadata is not None:
|
||||
pdf_title = getattr(metadata, "title", None)
|
||||
if not pdf_title and isinstance(metadata, dict):
|
||||
pdf_title = metadata.get("/Title")
|
||||
|
||||
extracted_text = "\n\n".join(text_parts).strip()
|
||||
return extracted_text, "text/plain", infer_title(path, extracted_text, pdf_title=pdf_title)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"build_ingest_plan",
|
||||
"build_subjects",
|
||||
"canonical_subject_lines",
|
||||
"checksum_sha256",
|
||||
"extract_document_text",
|
||||
"guess_content_type",
|
||||
"infer_document_type_name",
|
||||
"infer_metainfo_from_document",
|
||||
"infer_title",
|
||||
"iter_source_documents",
|
||||
"read_document",
|
||||
]
|
||||
|
|
@ -0,0 +1,111 @@
|
|||
"""Client-side ingest models and validation helpers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
|
||||
class ClientModel(BaseModel):
|
||||
"""Strict base model for client-side JSON payloads."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class PromptSubjectInput(ClientModel):
|
||||
"""Structured subject metainfo produced by a prompt or heuristic inference."""
|
||||
|
||||
vendor_name: str | None = None
|
||||
equipment_type: str | None = None
|
||||
model: str | None = None
|
||||
model_family: str | None = None
|
||||
|
||||
@field_validator("vendor_name", "equipment_type", "model", "model_family", mode="before")
|
||||
@classmethod
|
||||
def _blank_strings_become_none(cls, value: object) -> object:
|
||||
if isinstance(value, str) and not value.strip():
|
||||
return None
|
||||
return value
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _at_least_one_field(self) -> "PromptSubjectInput":
|
||||
if not any((self.vendor_name, self.equipment_type, self.model, self.model_family)):
|
||||
raise ValueError(
|
||||
"subject must include at least one of vendor_name, equipment_type, model, or model_family"
|
||||
)
|
||||
return self
|
||||
|
||||
|
||||
class PromptMetainfoInput(ClientModel):
|
||||
"""Prompt-normalized metainfo that can be used to build ingest payloads."""
|
||||
|
||||
title: str | None = None
|
||||
document_type_name: str | None = None
|
||||
content_type: str | None = None
|
||||
subjects: list[PromptSubjectInput] = Field(default_factory=list)
|
||||
|
||||
@field_validator("title", "document_type_name", "content_type", mode="before")
|
||||
@classmethod
|
||||
def _blank_strings_become_none(cls, value: object) -> object:
|
||||
if isinstance(value, str) and not value.strip():
|
||||
return None
|
||||
return value
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _validate_subjects(self) -> "PromptMetainfoInput":
|
||||
if not self.subjects:
|
||||
raise ValueError("subjects must not be empty")
|
||||
if not self.document_type_name:
|
||||
raise ValueError("document_type_name is required")
|
||||
return self
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class SourceDocument:
|
||||
"""A document discovered by the client before ingest."""
|
||||
|
||||
path: Path
|
||||
checksum_sha256: str
|
||||
raw_bytes: bytes
|
||||
text_content: str
|
||||
extracted_content_type: str
|
||||
source_uri: str
|
||||
source_file_name: str
|
||||
title: str
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class IngestPlan:
|
||||
"""Canonical ingest plan before transport mapping."""
|
||||
|
||||
source: SourceDocument
|
||||
metainfo: PromptMetainfoInput
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class ServerCapabilities:
|
||||
"""Transport and payload capabilities detected from the live server."""
|
||||
|
||||
server_url: str
|
||||
native_mcp_available: bool
|
||||
http_facade_available: bool
|
||||
supports_structured_subjects: bool
|
||||
native_tool_names: tuple[str, ...] = ()
|
||||
|
||||
@property
|
||||
def preferred_transport(self) -> str:
|
||||
if self.native_mcp_available:
|
||||
return "mcp"
|
||||
return "http"
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ClientModel",
|
||||
"IngestPlan",
|
||||
"PromptMetainfoInput",
|
||||
"PromptSubjectInput",
|
||||
"ServerCapabilities",
|
||||
"SourceDocument",
|
||||
]
|
||||
|
|
@ -0,0 +1,61 @@
|
|||
"""Prompt generation and prompt-output validation helpers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from .models import PromptMetainfoInput
|
||||
|
||||
|
||||
def build_metadata_prompt(
|
||||
*,
|
||||
source_path: Path,
|
||||
source_preview: str,
|
||||
prompt_preview_chars: int,
|
||||
) -> str:
|
||||
"""Create a strict prompt for manual or external LLM normalization."""
|
||||
|
||||
preview = source_preview[:prompt_preview_chars]
|
||||
return (
|
||||
"You are preparing metainfo for ingest into TechDoc Library MCP Server.\n"
|
||||
"Return STRICT JSON only. No markdown, no code fences, no commentary.\n"
|
||||
"Do not include subject_payload.\n\n"
|
||||
"Required JSON shape:\n"
|
||||
"{\n"
|
||||
' "title": string | null,\n'
|
||||
' "document_type_name": string,\n'
|
||||
' "content_type": string | null,\n'
|
||||
' "subjects": [\n'
|
||||
" {\n"
|
||||
' "vendor_name": string | null,\n'
|
||||
' "equipment_type": string | null,\n'
|
||||
' "model": string | null,\n'
|
||||
' "model_family": string | null\n'
|
||||
" }\n"
|
||||
" ]\n"
|
||||
"}\n\n"
|
||||
"Rules:\n"
|
||||
"- subjects must not be empty.\n"
|
||||
"- Each subject must describe one equipment context.\n"
|
||||
"- Use lower_snake_case for document_type_name.\n"
|
||||
"- Keep null for unknown fields.\n"
|
||||
"- Prefer one subject for one model family or model set.\n\n"
|
||||
f"Source file: {source_path.name}\n\n"
|
||||
"Document preview:\n"
|
||||
f"{preview}\n"
|
||||
)
|
||||
|
||||
|
||||
def load_prompt_metainfo(source: str) -> PromptMetainfoInput:
|
||||
"""Validate prompt output from a JSON string or JSON file."""
|
||||
|
||||
path = Path(source)
|
||||
if path.exists():
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
else:
|
||||
payload = json.loads(source)
|
||||
return PromptMetainfoInput.model_validate(payload)
|
||||
|
||||
|
||||
__all__ = ["build_metadata_prompt", "load_prompt_metainfo"]
|
||||
|
|
@ -0,0 +1,91 @@
|
|||
"""Duplicate suppression state for regular ingest runs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class LedgerEntry:
|
||||
source_uri: str
|
||||
checksum_sha256: str
|
||||
server_url: str
|
||||
document_id: str | None
|
||||
version_id: str | None
|
||||
ingested_at: str
|
||||
|
||||
|
||||
class IngestLedger:
|
||||
"""Persist a lightweight ingest history to skip duplicates safely."""
|
||||
|
||||
def __init__(self, path: Path) -> None:
|
||||
self.path = path
|
||||
self._entries: dict[tuple[str, str, str], LedgerEntry] = {}
|
||||
self.load()
|
||||
|
||||
def load(self) -> None:
|
||||
if not self.path.exists():
|
||||
return
|
||||
payload = json.loads(self.path.read_text(encoding="utf-8"))
|
||||
for item in payload.get("entries", []):
|
||||
entry = LedgerEntry(
|
||||
source_uri=item["source_uri"],
|
||||
checksum_sha256=item["checksum_sha256"],
|
||||
server_url=item["server_url"],
|
||||
document_id=item.get("document_id"),
|
||||
version_id=item.get("version_id"),
|
||||
ingested_at=item["ingested_at"],
|
||||
)
|
||||
self._entries[(entry.source_uri, entry.checksum_sha256, entry.server_url)] = entry
|
||||
|
||||
def seen(self, source_uri: str, checksum_sha256: str, server_url: str) -> bool:
|
||||
return (source_uri, checksum_sha256, server_url) in self._entries
|
||||
|
||||
def record(
|
||||
self,
|
||||
*,
|
||||
source_uri: str,
|
||||
checksum_sha256: str,
|
||||
server_url: str,
|
||||
document_id: str | None,
|
||||
version_id: str | None,
|
||||
) -> None:
|
||||
entry = LedgerEntry(
|
||||
source_uri=source_uri,
|
||||
checksum_sha256=checksum_sha256,
|
||||
server_url=server_url,
|
||||
document_id=document_id,
|
||||
version_id=version_id,
|
||||
ingested_at=datetime.now(timezone.utc).isoformat(),
|
||||
)
|
||||
self._entries[(source_uri, checksum_sha256, server_url)] = entry
|
||||
self.save()
|
||||
|
||||
def save(self) -> None:
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
payload: dict[str, Any] = {
|
||||
"entries": [
|
||||
{
|
||||
"source_uri": entry.source_uri,
|
||||
"checksum_sha256": entry.checksum_sha256,
|
||||
"server_url": entry.server_url,
|
||||
"document_id": entry.document_id,
|
||||
"version_id": entry.version_id,
|
||||
"ingested_at": entry.ingested_at,
|
||||
}
|
||||
for entry in sorted(
|
||||
self._entries.values(),
|
||||
key=lambda item: (item.server_url, item.source_uri, item.checksum_sha256),
|
||||
)
|
||||
]
|
||||
}
|
||||
temp_path = self.path.with_suffix(self.path.suffix + ".tmp")
|
||||
temp_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
temp_path.replace(self.path)
|
||||
|
||||
|
||||
__all__ = ["IngestLedger", "LedgerEntry"]
|
||||
|
|
@ -0,0 +1,206 @@
|
|||
"""Transport helpers for the ingest client."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
from .models import IngestPlan, PromptSubjectInput, ServerCapabilities
|
||||
|
||||
SUPPORTED_PROTOCOL_VERSION = "2025-11-25"
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class TransportResponse:
|
||||
"""Normalized transport response from either HTTP facade or native MCP."""
|
||||
|
||||
transport: str
|
||||
payload: dict[str, Any]
|
||||
|
||||
|
||||
class MCPIngestClient:
|
||||
"""Probe a live server and send ingest requests using the best available path."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
server_url: str,
|
||||
*,
|
||||
transport_preference: str = "auto",
|
||||
timeout_seconds: float = 60.0,
|
||||
) -> None:
|
||||
self.server_url = server_url.rstrip("/")
|
||||
self.transport_preference = transport_preference
|
||||
self.timeout_seconds = timeout_seconds
|
||||
self._capabilities: ServerCapabilities | None = None
|
||||
|
||||
def probe(self) -> ServerCapabilities:
|
||||
if self._capabilities is not None:
|
||||
return self._capabilities
|
||||
|
||||
http_available, supports_structured = self._probe_openapi()
|
||||
native_available, native_tools = self._probe_native_mcp()
|
||||
native_ready = native_available and "ingest_document" in native_tools
|
||||
self._capabilities = ServerCapabilities(
|
||||
server_url=self.server_url,
|
||||
native_mcp_available=native_ready,
|
||||
http_facade_available=http_available,
|
||||
supports_structured_subjects=supports_structured,
|
||||
native_tool_names=native_tools,
|
||||
)
|
||||
return self._capabilities
|
||||
|
||||
def ingest(self, plan: IngestPlan) -> TransportResponse:
|
||||
capabilities = self.probe()
|
||||
if self.transport_preference == "mcp" and capabilities.native_mcp_available:
|
||||
return self._native_ingest(plan, structured=capabilities.supports_structured_subjects)
|
||||
if self.transport_preference == "http" and capabilities.http_facade_available:
|
||||
return self._http_ingest(plan, structured=capabilities.supports_structured_subjects)
|
||||
|
||||
if capabilities.native_mcp_available:
|
||||
try:
|
||||
return self._native_ingest(
|
||||
plan, structured=capabilities.supports_structured_subjects
|
||||
)
|
||||
except Exception:
|
||||
if capabilities.http_facade_available:
|
||||
return self._http_ingest(
|
||||
plan, structured=capabilities.supports_structured_subjects
|
||||
)
|
||||
raise
|
||||
|
||||
return self._http_ingest(plan, structured=capabilities.supports_structured_subjects)
|
||||
|
||||
def _probe_openapi(self) -> tuple[bool, bool]:
|
||||
try:
|
||||
payload = self._request_json(f"{self.server_url}/openapi.json", method="GET")
|
||||
except Exception:
|
||||
return False, False
|
||||
|
||||
paths = payload.get("paths", {})
|
||||
http_available = "/mcp/ingest_document" in paths
|
||||
schemas = payload.get("components", {}).get("schemas", {})
|
||||
metadata_schema = schemas.get("DocumentBaseMetadataInput", {})
|
||||
properties = metadata_schema.get("properties", {})
|
||||
supports_structured = "subjects" in properties
|
||||
return http_available, supports_structured
|
||||
|
||||
def _probe_native_mcp(self) -> tuple[bool, tuple[str, ...]]:
|
||||
initialize_payload = {
|
||||
"jsonrpc": "2.0",
|
||||
"id": 1,
|
||||
"method": "initialize",
|
||||
"params": {
|
||||
"protocolVersion": SUPPORTED_PROTOCOL_VERSION,
|
||||
"capabilities": {"tools": {}},
|
||||
"clientInfo": {"name": "techlib_ingest_client", "version": "0.1.0"},
|
||||
},
|
||||
}
|
||||
try:
|
||||
initialize_response = self._request_json(
|
||||
f"{self.server_url}/mcp",
|
||||
method="POST",
|
||||
payload=initialize_payload,
|
||||
)
|
||||
except Exception:
|
||||
return False, ()
|
||||
|
||||
if "error" in initialize_response:
|
||||
return False, ()
|
||||
|
||||
try:
|
||||
tools_response = self._request_json(
|
||||
f"{self.server_url}/mcp",
|
||||
method="POST",
|
||||
payload={
|
||||
"jsonrpc": "2.0",
|
||||
"id": 2,
|
||||
"method": "tools/list",
|
||||
"params": {},
|
||||
},
|
||||
)
|
||||
except Exception:
|
||||
return True, ()
|
||||
|
||||
tool_names: list[str] = []
|
||||
result = tools_response.get("result", {})
|
||||
for tool in result.get("tools", []):
|
||||
name = tool.get("name")
|
||||
if isinstance(name, str):
|
||||
tool_names.append(name)
|
||||
return True, tuple(tool_names)
|
||||
|
||||
def _build_payload(self, plan: IngestPlan, *, structured: bool) -> dict[str, Any]:
|
||||
subject = plan.metainfo.subjects[0]
|
||||
metadata: dict[str, Any] = {
|
||||
"source_uri": plan.source.source_uri,
|
||||
"source_file_name": plan.source.source_file_name,
|
||||
"content_type": plan.metainfo.content_type or plan.source.extracted_content_type,
|
||||
"document_type_name": plan.metainfo.document_type_name,
|
||||
"title": plan.metainfo.title or plan.source.title,
|
||||
}
|
||||
if structured:
|
||||
metadata["subjects"] = [_subject_payload(item) for item in plan.metainfo.subjects]
|
||||
else:
|
||||
metadata["vendor_name"] = subject.vendor_name
|
||||
return {"content": plan.source.text_content, "metadata": metadata}
|
||||
|
||||
def _http_ingest(self, plan: IngestPlan, *, structured: bool) -> TransportResponse:
|
||||
payload = self._build_payload(plan, structured=structured)
|
||||
response = self._request_json(
|
||||
f"{self.server_url}/mcp/ingest_document",
|
||||
method="POST",
|
||||
payload=payload,
|
||||
)
|
||||
return TransportResponse(transport="http", payload=response)
|
||||
|
||||
def _native_ingest(self, plan: IngestPlan, *, structured: bool) -> TransportResponse:
|
||||
payload = self._build_payload(plan, structured=structured)
|
||||
response = self._request_json(
|
||||
f"{self.server_url}/mcp",
|
||||
method="POST",
|
||||
payload={
|
||||
"jsonrpc": "2.0",
|
||||
"id": 3,
|
||||
"method": "tools/call",
|
||||
"params": {"name": "ingest_document", "arguments": payload},
|
||||
},
|
||||
)
|
||||
return TransportResponse(transport="mcp", payload=response)
|
||||
|
||||
def _request_json(
|
||||
self,
|
||||
url: str,
|
||||
*,
|
||||
method: str,
|
||||
payload: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
data = None
|
||||
headers = {"Accept": "application/json"}
|
||||
if payload is not None:
|
||||
data = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
||||
headers["Content-Type"] = "application/json"
|
||||
request = Request(url, data=data, headers=headers, method=method)
|
||||
try:
|
||||
with urlopen(request, timeout=self.timeout_seconds) as response:
|
||||
body = response.read().decode("utf-8")
|
||||
except HTTPError as exc:
|
||||
body = exc.read().decode("utf-8")
|
||||
try:
|
||||
return json.loads(body)
|
||||
except json.JSONDecodeError:
|
||||
raise RuntimeError(f"HTTP {exc.code} from {url}: {body}") from exc
|
||||
except URLError as exc:
|
||||
raise RuntimeError(f"failed to reach {url}: {exc}") from exc
|
||||
return json.loads(body) if body else {}
|
||||
|
||||
|
||||
def _subject_payload(subject: PromptSubjectInput) -> dict[str, Any]:
|
||||
return subject.model_dump(exclude_none=True)
|
||||
|
||||
|
||||
__all__ = ["MCPIngestClient", "SUPPORTED_PROTOCOL_VERSION", "TransportResponse"]
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,157 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from techlib_mcpserver.client.metadata import build_ingest_plan
|
||||
from techlib_mcpserver.client.models import ServerCapabilities
|
||||
from techlib_mcpserver.client.prompting import build_metadata_prompt, load_prompt_metainfo
|
||||
from techlib_mcpserver.client.state import IngestLedger
|
||||
from techlib_mcpserver.client.transport import MCPIngestClient, TransportResponse
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _workspace_dir() -> Path:
|
||||
root = Path.cwd() / ".tmp" / "client-tests"
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
path = root / uuid4().hex
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
yield path
|
||||
finally:
|
||||
shutil.rmtree(path, ignore_errors=True)
|
||||
|
||||
|
||||
def _plan(tmp_dir: Path):
|
||||
source = tmp_dir / "Huawei CloudEngine 6820 Switch Datasheet.txt"
|
||||
source.write_text("Huawei CloudEngine 6820 Switch Datasheet\nCE6820-48S6CQ", encoding="utf-8")
|
||||
return build_ingest_plan(source)
|
||||
|
||||
|
||||
def test_prompt_generation_excludes_subject_payload() -> None:
|
||||
with _workspace_dir() as tmp_dir:
|
||||
plan = _plan(tmp_dir)
|
||||
prompt = build_metadata_prompt(
|
||||
source_path=plan.source.path,
|
||||
source_preview=plan.source.text_content,
|
||||
prompt_preview_chars=400,
|
||||
)
|
||||
assert "\"subject_payload\"" not in prompt
|
||||
assert "subjects must not be empty" in prompt
|
||||
|
||||
|
||||
def test_prompt_json_validation_rejects_extra_fields() -> None:
|
||||
with _workspace_dir() as tmp_dir:
|
||||
payload = tmp_dir / "metainfo.json"
|
||||
payload.write_text(
|
||||
"""
|
||||
{
|
||||
"title": "Huawei CloudEngine 6820 Switch Datasheet",
|
||||
"document_type_name": "datasheet",
|
||||
"content_type": "text/plain",
|
||||
"subjects": [
|
||||
{
|
||||
"vendor_name": "Huawei",
|
||||
"equipment_type": "switch",
|
||||
"model": "CE6820-48S6CQ",
|
||||
"model_family": "CloudEngine 6820 series",
|
||||
"subject_payload": {}
|
||||
}
|
||||
]
|
||||
}
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
with pytest.raises(ValidationError):
|
||||
load_prompt_metainfo(str(payload))
|
||||
|
||||
|
||||
def test_client_probe_detects_structured_subjects_from_openapi() -> None:
|
||||
client = MCPIngestClient("http://example.invalid")
|
||||
|
||||
def fake_request_json(url: str, *, method: str, payload=None):
|
||||
if url.endswith("/openapi.json"):
|
||||
return {
|
||||
"paths": {"/mcp/ingest_document": {}},
|
||||
"components": {
|
||||
"schemas": {
|
||||
"DocumentBaseMetadataInput": {
|
||||
"properties": {"subjects": {"type": "array"}}
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
if url.endswith("/mcp") and payload and payload.get("method") == "tools/list":
|
||||
return {"result": {"tools": [{"name": "ingest_document"}]}}
|
||||
if url.endswith("/mcp") and payload and payload.get("method") == "initialize":
|
||||
return {"result": {"protocolVersion": "2025-11-25"}}
|
||||
raise AssertionError(url)
|
||||
|
||||
client._request_json = fake_request_json # type: ignore[method-assign]
|
||||
capabilities = client.probe()
|
||||
assert capabilities.http_facade_available is True
|
||||
assert capabilities.native_mcp_available is True
|
||||
assert capabilities.supports_structured_subjects is True
|
||||
assert "ingest_document" in capabilities.native_tool_names
|
||||
|
||||
|
||||
def test_client_builds_legacy_payload_without_subjects() -> None:
|
||||
with _workspace_dir() as tmp_dir:
|
||||
plan = _plan(tmp_dir)
|
||||
client = MCPIngestClient("http://example.invalid")
|
||||
payload = client._build_payload(plan, structured=False)
|
||||
assert payload["metadata"]["vendor_name"] == "Huawei"
|
||||
assert "subjects" not in payload["metadata"]
|
||||
assert payload["metadata"]["document_type_name"] == plan.metainfo.document_type_name
|
||||
|
||||
|
||||
def test_client_prefers_native_transport_when_available() -> None:
|
||||
with _workspace_dir() as tmp_dir:
|
||||
plan = _plan(tmp_dir)
|
||||
client = MCPIngestClient("http://example.invalid")
|
||||
client._capabilities = ServerCapabilities(
|
||||
server_url=client.server_url,
|
||||
native_mcp_available=True,
|
||||
http_facade_available=True,
|
||||
supports_structured_subjects=False,
|
||||
native_tool_names=("ingest_document",),
|
||||
)
|
||||
|
||||
recorded: list[str] = []
|
||||
|
||||
def fake_native_ingest(plan_arg, *, structured: bool):
|
||||
recorded.append(f"native:{structured}")
|
||||
return TransportResponse("mcp", {"result": {"ok": True}})
|
||||
|
||||
def fake_http_ingest(plan_arg, *, structured: bool):
|
||||
recorded.append(f"http:{structured}")
|
||||
return TransportResponse("http", {"result": {"ok": True}})
|
||||
|
||||
client._native_ingest = fake_native_ingest # type: ignore[method-assign]
|
||||
client._http_ingest = fake_http_ingest # type: ignore[method-assign]
|
||||
client.probe = lambda: client._capabilities # type: ignore[method-assign]
|
||||
|
||||
response = client.ingest(plan)
|
||||
assert response.transport == "mcp"
|
||||
assert recorded == ["native:False"]
|
||||
|
||||
|
||||
def test_ingest_ledger_skips_duplicates() -> None:
|
||||
with _workspace_dir() as tmp_dir:
|
||||
ledger = IngestLedger(tmp_dir / "state.json")
|
||||
assert ledger.seen("file:///docs/a.txt", "abc", "http://example.invalid") is False
|
||||
ledger.record(
|
||||
source_uri="file:///docs/a.txt",
|
||||
checksum_sha256="abc",
|
||||
server_url="http://example.invalid",
|
||||
document_id="1",
|
||||
version_id="2",
|
||||
)
|
||||
assert ledger.seen("file:///docs/a.txt", "abc", "http://example.invalid") is True
|
||||
|
||||
Loading…
Reference in New Issue