feat: add windows ingest client for mcp server

This commit is contained in:
Ilgiz Kalmetyev 2026-04-10 18:11:13 +05:00
parent a0e0bd293a
commit 2416692e8d
19 changed files with 1488 additions and 3 deletions

3
.gitignore vendored
View File

@ -2,6 +2,7 @@
__pycache__/
*.pyc
.pytest_cache/
.pytest-bt/
.mypy_cache/
.ruff_cache/
.venv/
@ -10,4 +11,4 @@ env/
build/
dist/
*.egg-info/
tmp*/

43
.vscode/launch.json vendored Normal file
View File

@ -0,0 +1,43 @@
{
"version": "0.2.0",
"configurations": [
{
"name": "TechLib Ingest Client",
"type": "python",
"request": "launch",
"module": "techlib_mcpserver.client",
"cwd": "${workspaceFolder}",
"env": {
"PYTHONPATH": "${workspaceFolder}\\src"
},
"console": "integratedTerminal",
"justMyCode": true,
"args": [
"--source",
"${workspaceFolder}\\in",
"--recursive",
"--server-url",
"http://192.168.201.91:8888"
]
},
{
"name": "TechLib Ingest Client Prompt",
"type": "python",
"request": "launch",
"module": "techlib_mcpserver.client",
"cwd": "${workspaceFolder}",
"env": {
"PYTHONPATH": "${workspaceFolder}\\src"
},
"console": "integratedTerminal",
"justMyCode": true,
"args": [
"--source",
"${workspaceFolder}\\in\\Huawei CloudEngine 6820 Switch Datasheet.pdf",
"--emit-prompt",
"--server-url",
"http://192.168.201.91:8888"
]
}
]
}

39
.vscode/tasks.json vendored Normal file
View File

@ -0,0 +1,39 @@
{
"version": "2.0.0",
"tasks": [
{
"label": "TechLib: ingest folder",
"type": "shell",
"command": "powershell.exe",
"args": [
"-ExecutionPolicy",
"Bypass",
"-File",
"${workspaceFolder}\\scripts\\windows\\run_ingest_client.ps1",
"-Source",
"${workspaceFolder}\\in",
"-Recursive",
"-ServerUrl",
"http://192.168.201.91:8888"
],
"problemMatcher": []
},
{
"label": "TechLib: emit prompt",
"type": "shell",
"command": "powershell.exe",
"args": [
"-ExecutionPolicy",
"Bypass",
"-File",
"${workspaceFolder}\\scripts\\windows\\run_ingest_client.ps1",
"-Source",
"${workspaceFolder}\\in\\Huawei CloudEngine 6820 Switch Datasheet.pdf",
"-EmitPrompt",
"-ServerUrl",
"http://192.168.201.91:8888"
],
"problemMatcher": []
}
]
}

View File

@ -75,6 +75,7 @@ The project integrates with:
## Documentation Map
- [Usage Guide](docs/usage-guide.md)
- [Windows Ingest Client](docs/windows-ingest-client.md)
- [Deployment Guide](docs/deployment-guide.md)
- [PostgreSQL Operations](docs/postgresql.md)
- [Content Delivery](docs/content-delivery.md)

View File

@ -1,4 +1,4 @@
# Usage Guide
# Usage Guide
This guide covers local usage of TechDoc Library MCP Server v1.
@ -25,6 +25,8 @@ The service exposes two compatible MCP entry points:
- Native MCP transport at `POST /mcp` for JSON-RPC `initialize`, `tools/list`, and `tools/call`
- HTTP facade at `/mcp/*` for Swagger, `curl`, and manual diagnostics
For Windows/VSCode-based regular ingest, use the dedicated client described in [Windows Ingest Client](windows-ingest-client.md).
## MCP Tool Overview
The MCP facade exposes version-aware document library tools:
@ -110,4 +112,5 @@ curl http://127.0.0.1:8000/health
- [Deployment Guide](docs/deployment-guide.md)
- [PostgreSQL Operations](docs/postgresql.md)
- [Content Delivery](docs/content-delivery.md)
- [Content Delivery](docs/content-delivery.md)

View File

@ -0,0 +1,87 @@
# Windows Ingest Client
This repository includes a Windows-friendly ingest client for running from PowerShell or VSCode.
## What It Does
The client:
- reads one file or a folder of files
- extracts text locally
- normalizes document metainfo
- probes the live MCP server capabilities
- ingests through native MCP when available, otherwise falls back to the HTTP facade
- skips duplicates using a local ledger file
## Running From PowerShell
Example:
```powershell
$env:PYTHONPATH = "$PWD\src"
$env:TECHLIB_INGEST_SERVER_URL = 'http://192.168.201.91:8888'
python -m techlib_mcpserver.client --source .\in --recursive
```
If you use the bundled wrapper script, it sets `PYTHONPATH` automatically:
```powershell
powershell.exe -ExecutionPolicy Bypass -File .\scripts\windows\run_ingest_client.ps1 -Source .\in -Recursive -ServerUrl http://192.168.201.91:8888
```
Optional prompt-assisted flow for a single file:
```powershell
$env:PYTHONPATH = "$PWD\src"
python -m techlib_mcpserver.client --source .\in\Huawei CloudEngine 6820 Switch Datasheet.pdf --emit-prompt > prompt.txt
```
Feed the prompt into your preferred model, save strict JSON metainfo to a file, then ingest with:
```powershell
$env:PYTHONPATH = "$PWD\src"
python -m techlib_mcpserver.client --source .\in\Huawei CloudEngine 6820 Switch Datasheet.pdf --prompt-json .\metainfo.json
```
## Environment Variables
The client reads `.env.local` and `.env` files automatically.
Use these variables for repeatable runs:
- `TECHLIB_INGEST_SERVER_URL`
- `TECHLIB_INGEST_TRANSPORT`
- `TECHLIB_INGEST_STATE_FILE`
- `TECHLIB_INGEST_REQUEST_TIMEOUT_SECONDS`
- `TECHLIB_INGEST_PROMPT_PREVIEW_CHARS`
## VSCode
Use the provided launch and task files for debugging or repeated runs. They call the same Python CLI, so behavior matches PowerShell runs.
## Prompt Contract
Prompt output must be strict JSON and must not include `subject_payload`.
The expected shape is:
```json
{
"title": "string or null",
"document_type_name": "string",
"content_type": "string or null",
"subjects": [
{
"vendor_name": "string or null",
"equipment_type": "string or null",
"model": "string or null",
"model_family": "string or null"
}
]
}
```
## Notes
- If the live server does not support structured subjects yet, the client falls back to the legacy flat ingest payload.
- Duplicate suppression is local to the client and keyed by `source_uri + checksum + server_url`.

View File

@ -0,0 +1,9 @@
# Ingest client settings for Windows / VSCode
# Copy to .env.local or set in your shell profile.
TECHLIB_INGEST_SERVER_URL=http://192.168.201.91:8888
TECHLIB_INGEST_TRANSPORT=auto
TECHLIB_INGEST_STATE_FILE=C:\Users\%USERNAME%\AppData\Roaming\techlib_mcpserver_ingest\state.json
TECHLIB_INGEST_REQUEST_TIMEOUT_SECONDS=60
TECHLIB_INGEST_PROMPT_PREVIEW_CHARS=12000
# TECHLIB_INGEST_LOG_FILE=C:\Users\%USERNAME%\AppData\Roaming\techlib_mcpserver_ingest\ingest.log

View File

@ -24,9 +24,13 @@ dependencies = [
]
[project.optional-dependencies]
client = ["pypdf>=6,<7"]
test = ["pytest>=8,<9"]
dev = ["pytest>=8,<9"]
[project.scripts]
techlib-ingest-client = "techlib_mcpserver.client.cli:main"
[tool.setuptools]
package-dir = {"" = "src"}

View File

@ -0,0 +1,48 @@
param(
[Parameter(Mandatory = $true)]
[string]$Source,
[string]$ServerUrl = $env:TECHLIB_INGEST_SERVER_URL,
[ValidateSet('auto', 'mcp', 'http')]
[string]$Transport = 'auto',
[switch]$Recursive,
[switch]$Force,
[string]$StateFile = $env:TECHLIB_INGEST_STATE_FILE,
[string]$PromptJson,
[switch]$EmitPrompt,
[string]$LogFile = $env:TECHLIB_INGEST_LOG_FILE,
[int]$PreviewChars = 12000
)
$root = Resolve-Path (Join-Path $PSScriptRoot '..\..')
$srcPath = Join-Path $root 'src'
if ($env:PYTHONPATH) {
$env:PYTHONPATH = "$srcPath;$env:PYTHONPATH"
} else {
$env:PYTHONPATH = $srcPath
}
$pythonCandidates = @(
(Join-Path $root '.venv\Scripts\python.exe'),
'python'
)
$python = $pythonCandidates | Where-Object { Test-Path $_ } | Select-Object -First 1
if (-not $python) {
$python = 'python'
}
$argsList = @(
'-m', 'techlib_mcpserver.client',
'--source', $Source,
'--transport', $Transport,
'--preview-chars', $PreviewChars
)
if ($ServerUrl) { $argsList += @('--server-url', $ServerUrl) }
if ($Recursive) { $argsList += '--recursive' }
if ($Force) { $argsList += '--force' }
if ($StateFile) { $argsList += @('--state-file', $StateFile) }
if ($PromptJson) { $argsList += @('--prompt-json', $PromptJson) }
if ($EmitPrompt) { $argsList += '--emit-prompt' }
if ($LogFile) { $argsList += @('--log-file', $LogFile) }
& $python @argsList
exit $LASTEXITCODE

View File

@ -0,0 +1,3 @@
"""Windows-friendly ingest client package."""
__all__ = []

View File

@ -0,0 +1,7 @@
"""Module entrypoint for the ingest client."""
from .cli import main
if __name__ == "__main__":
main()

View File

@ -0,0 +1,232 @@
"""CLI entrypoint for the Windows-friendly ingest client."""
from __future__ import annotations
import argparse
import json
import logging
import sys
from pathlib import Path
from techlib_mcpserver.app.logging import StructuredFormatter, configure_logging, get_logger
from .config import IngestClientSettings
from .metadata import build_ingest_plan, canonical_subject_lines, iter_source_documents
from .prompting import build_metadata_prompt, load_prompt_metainfo
from .state import IngestLedger
from .transport import MCPIngestClient
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
prog="techlib-ingest-client",
description="Windows-friendly ingest client for the TechDoc Library MCP server.",
)
parser.add_argument("--source", required=True, help="File or directory to ingest.")
parser.add_argument(
"--server-url",
default=None,
help="MCP server base URL, for example http://192.168.201.91:8888.",
)
parser.add_argument(
"--transport",
choices=("auto", "mcp", "http"),
default=None,
help="Preferred transport mode.",
)
parser.add_argument(
"--recursive",
action="store_true",
help="Recurse into subdirectories when source is a directory.",
)
parser.add_argument(
"--force",
action="store_true",
help="Reingest even if source URI and checksum were already seen.",
)
parser.add_argument(
"--state-file",
default=None,
help="Path to the local ingest ledger JSON file.",
)
parser.add_argument(
"--prompt-json",
default=None,
help="Path to strict JSON metainfo output from a prompt or external model.",
)
parser.add_argument(
"--emit-prompt",
action="store_true",
help="Print a normalization prompt and exit instead of ingesting.",
)
parser.add_argument(
"--log-file",
default=None,
help="Optional log file path.",
)
parser.add_argument(
"--preview-chars",
type=int,
default=None,
help="Number of characters to include in prompt previews.",
)
return parser
def _apply_overrides(settings: IngestClientSettings, args: argparse.Namespace) -> IngestClientSettings:
if args.server_url is not None:
settings.server_url = args.server_url
if args.transport is not None:
settings.transport = args.transport
if args.recursive:
settings.recursive = True
if args.force:
settings.force = True
if args.state_file is not None:
settings.state_file = Path(args.state_file)
if args.log_file is not None:
settings.log_file = Path(args.log_file)
if args.preview_chars is not None:
settings.prompt_preview_chars = args.preview_chars
return settings
def _configure_file_logging(log_file: Path) -> None:
log_file.parent.mkdir(parents=True, exist_ok=True)
handler = logging.FileHandler(log_file, encoding="utf-8")
handler.setFormatter(StructuredFormatter(datefmt="%Y-%m-%dT%H:%M:%S%z"))
logging.getLogger().addHandler(handler)
def _configure_stdio() -> None:
for stream_name in ("stdout", "stderr"):
stream = getattr(sys, stream_name)
if hasattr(stream, "reconfigure"):
stream.reconfigure(encoding="utf-8", errors="replace")
def main(argv: list[str] | None = None) -> int:
_configure_stdio()
parser = build_parser()
args = parser.parse_args(argv)
settings = _apply_overrides(IngestClientSettings.from_env(), args)
configure_logging()
if settings.log_file is not None:
_configure_file_logging(settings.log_file)
logger = get_logger(
__name__,
app_name="techlib_ingest_client",
app_env="windows-vscode",
)
source_path = Path(args.source).expanduser().resolve()
if not source_path.exists():
parser.error(f"source does not exist: {source_path}")
prompt_metainfo = None
if args.prompt_json:
prompt_metainfo = load_prompt_metainfo(args.prompt_json)
client = MCPIngestClient(
settings.server_url,
transport_preference=settings.transport,
timeout_seconds=settings.request_timeout_seconds,
)
capabilities = client.probe()
logger.info(
"server capabilities probed: %s",
json.dumps(
{
"server_url": capabilities.server_url,
"native_mcp_available": capabilities.native_mcp_available,
"http_facade_available": capabilities.http_facade_available,
"supports_structured_subjects": capabilities.supports_structured_subjects,
"native_tool_names": capabilities.native_tool_names,
},
ensure_ascii=False,
),
)
ledger = IngestLedger(settings.state_file)
paths = iter_source_documents(source_path, settings.recursive, settings.source_extensions)
if not paths:
logger.warning("no ingestable documents found under %s", source_path)
return 0
if args.emit_prompt:
if len(paths) != 1:
parser.error("--emit-prompt currently supports exactly one file")
document = build_ingest_plan(paths[0], prompt_metainfo=prompt_metainfo)
print(
build_metadata_prompt(
source_path=document.source.path,
source_preview=document.source.text_content,
prompt_preview_chars=settings.prompt_preview_chars,
)
)
return 0
ingested = 0
skipped = 0
for path in paths:
plan = build_ingest_plan(path, prompt_metainfo=prompt_metainfo)
source = plan.source
if not args.force and ledger.seen(source.source_uri, source.checksum_sha256, settings.server_url):
logger.info(
"skip duplicate source_uri=%s checksum=%s",
source.source_uri,
source.checksum_sha256,
)
skipped += 1
continue
logger.info(
"ingesting %s",
json.dumps(
{
"source_uri": source.source_uri,
"title": plan.metainfo.title,
"document_type_name": plan.metainfo.document_type_name,
"subjects": canonical_subject_lines(plan.metainfo.subjects),
},
ensure_ascii=False,
),
)
result = client.ingest(plan)
payload = result.payload
if "error" in payload:
raise SystemExit(json.dumps(payload, ensure_ascii=False, indent=2))
structured = payload.get("result", {}).get("structuredContent")
document_id = None
version_id = None
if isinstance(structured, dict):
document = structured.get("document", {})
version = structured.get("version", {})
if isinstance(document, dict):
document_id = document.get("document_id")
if isinstance(version, dict):
version_id = version.get("version_id")
ledger.record(
source_uri=source.source_uri,
checksum_sha256=source.checksum_sha256,
server_url=settings.server_url,
document_id=document_id,
version_id=version_id,
)
ingested += 1
logger.info(
"ingest completed transport=%s document_id=%s version_id=%s",
result.transport,
document_id,
version_id,
)
logger.info("run completed ingested=%s skipped=%s", ingested, skipped)
return 0
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -0,0 +1,98 @@
"""Runtime settings for the Windows-friendly ingest client."""
from __future__ import annotations
import os
from dataclasses import dataclass, field
from pathlib import Path
from typing import Literal
def _parse_env_file(path: Path) -> dict[str, str]:
if not path.exists():
return {}
values: dict[str, str] = {}
for raw_line in path.read_text(encoding="utf-8").splitlines():
line = raw_line.strip()
if not line or line.startswith("#") or "=" not in line:
continue
key, value = line.split("=", 1)
values[key.strip()] = value.strip().strip('"').strip("'")
return values
def _expand_path(value: str) -> Path:
return Path(os.path.expandvars(value)).expanduser()
def _as_bool(value: str | None, default: bool) -> bool:
if value is None:
return default
normalized = value.strip().lower()
if normalized in {"1", "true", "yes", "on"}:
return True
if normalized in {"0", "false", "no", "off"}:
return False
return default
def _as_transport(value: str | None, default: str) -> Literal["auto", "mcp", "http"]:
if value is None:
return default # type: ignore[return-value]
normalized = value.strip().lower()
if normalized in {"auto", "mcp", "http"}:
return normalized # type: ignore[return-value]
return default # type: ignore[return-value]
@dataclass(slots=True)
class IngestClientSettings:
server_url: str = "http://127.0.0.1:8888"
transport: Literal["auto", "mcp", "http"] = "auto"
recursive: bool = False
force: bool = False
state_file: Path = field(
default_factory=lambda: Path.home() / ".techlib_mcpserver_ingest" / "state.json"
)
log_file: Path | None = None
prompt_preview_chars: int = 12000
source_extensions: tuple[str, ...] = (
".pdf",
".txt",
".md",
".rst",
".json",
".yaml",
".yml",
".csv",
".log",
)
request_timeout_seconds: float = 60.0
@classmethod
def from_env(cls) -> "IngestClientSettings":
env = {}
for filename in (".env.local", ".env"):
env.update(_parse_env_file(Path.cwd() / filename))
env.update(os.environ)
settings = cls()
settings.server_url = env.get("TECHLIB_INGEST_SERVER_URL", settings.server_url)
settings.transport = _as_transport(env.get("TECHLIB_INGEST_TRANSPORT"), settings.transport)
settings.recursive = _as_bool(env.get("TECHLIB_INGEST_RECURSIVE"), settings.recursive)
settings.force = _as_bool(env.get("TECHLIB_INGEST_FORCE"), settings.force)
if env.get("TECHLIB_INGEST_STATE_FILE"):
settings.state_file = _expand_path(env["TECHLIB_INGEST_STATE_FILE"])
if env.get("TECHLIB_INGEST_LOG_FILE"):
settings.log_file = _expand_path(env["TECHLIB_INGEST_LOG_FILE"])
if env.get("TECHLIB_INGEST_PROMPT_PREVIEW_CHARS"):
settings.prompt_preview_chars = int(env["TECHLIB_INGEST_PROMPT_PREVIEW_CHARS"])
if env.get("TECHLIB_INGEST_REQUEST_TIMEOUT_SECONDS"):
settings.request_timeout_seconds = float(env["TECHLIB_INGEST_REQUEST_TIMEOUT_SECONDS"])
if env.get("TECHLIB_INGEST_SOURCE_EXTENSIONS"):
settings.source_extensions = tuple(
part.strip() for part in env["TECHLIB_INGEST_SOURCE_EXTENSIONS"].split(",") if part.strip()
)
return settings
__all__ = ["IngestClientSettings"]

View File

@ -0,0 +1,284 @@
"""Deterministic metadata extraction helpers for ingest planning."""
from __future__ import annotations
import mimetypes
import re
from hashlib import sha256
from pathlib import Path
from typing import Iterable
from .models import IngestPlan, PromptMetainfoInput, PromptSubjectInput, SourceDocument
KNOWN_VENDORS = (
"Huawei",
"Eltex",
"Cisco",
"Juniper",
"HPE",
"Aruba",
"MikroTik",
"D-Link",
"Ruijie",
"Extreme",
"Nokia",
"Mellanox",
)
DOCUMENT_TYPE_KEYWORDS = (
("datasheet", "datasheet"),
("spec sheet", "datasheet"),
("manual", "manual"),
("quick start", "quick_start_guide"),
("quickstart", "quick_start_guide"),
("install", "installation_guide"),
("configuration", "configuration_guide"),
("troubleshooting", "troubleshooting_guide"),
("release notes", "release_notes"),
("specification", "specification"),
("spec", "specification"),
("guide", "technical_description"),
)
EQUIPMENT_KEYWORDS = (
("switch", "switch"),
("router", "router"),
("firewall", "firewall"),
("access point", "access_point"),
("transceiver", "transceiver"),
("module", "module"),
("controller", "controller"),
)
PDF_MODEL_RE = re.compile(r"\b([A-Z]{1,4}\d{3,5}(?:-[A-Z0-9]+)*)\b")
HUAWEI_MODEL_RE = re.compile(r"\b(CE\d{4,5}-[A-Z0-9-]+)\b")
def checksum_sha256(data: bytes) -> str:
return sha256(data).hexdigest()
def guess_content_type(path: Path) -> str:
content_type, _ = mimetypes.guess_type(path.name)
if content_type:
return content_type
if path.suffix.lower() == ".pdf":
return "application/pdf"
return "text/plain"
def infer_title(path: Path, text: str | None = None, *, pdf_title: str | None = None) -> str:
if pdf_title and pdf_title.strip():
return pdf_title.strip()
if text:
first_line = text.strip().splitlines()[0].strip()
if first_line:
return first_line[:160]
stem = path.stem.replace("_", " ").replace("-", " ")
stem = re.sub(r"\s+", " ", stem).strip()
return stem[:160]
def extract_document_text(path: Path, raw_bytes: bytes) -> tuple[str, str, str]:
suffix = path.suffix.lower()
if suffix == ".pdf":
return _extract_pdf_text(path)
text = raw_bytes.decode("utf-8", errors="replace").strip()
return text, "text/plain", infer_title(path, text)
def read_document(path: Path) -> SourceDocument:
raw_bytes = path.read_bytes()
extracted_text, source_content_type, title = extract_document_text(path, raw_bytes)
return SourceDocument(
path=path,
checksum_sha256=checksum_sha256(raw_bytes),
raw_bytes=raw_bytes,
text_content=extracted_text,
extracted_content_type=source_content_type,
source_uri=path.resolve().as_uri(),
source_file_name=path.name,
title=title,
)
def infer_document_type_name(text: str, title: str) -> str:
haystack = f"{title}\n{text}".lower()
for keyword, document_type_name in DOCUMENT_TYPE_KEYWORDS:
if keyword in haystack:
return document_type_name
return "technical_description"
def infer_vendor_name(text: str, title: str) -> str | None:
haystack = f"{title}\n{text}"
for vendor in KNOWN_VENDORS:
if re.search(rf"\b{re.escape(vendor)}\b", haystack, flags=re.IGNORECASE):
return vendor
return None
def infer_equipment_type(text: str, title: str) -> str | None:
haystack = f"{title}\n{text}".lower()
for keyword, equipment_type in EQUIPMENT_KEYWORDS:
if keyword in haystack:
return equipment_type
return None
def infer_model(text: str, title: str, vendor_name: str | None = None) -> str | None:
haystack = f"{title}\n{text}"
if vendor_name and vendor_name.lower() == "huawei":
match = HUAWEI_MODEL_RE.search(haystack)
if match:
return match.group(1)
match = PDF_MODEL_RE.search(haystack)
if match:
return match.group(1)
return None
def infer_model_family(text: str, title: str, model: str | None = None) -> str | None:
haystack = f"{title}\n{text}"
if "CloudEngine 6820" in haystack:
return "CloudEngine 6820 series"
if model:
cleaned = model
if "-" in cleaned:
cleaned = cleaned.split("-")[0]
return f"{cleaned} family"
return None
def build_subjects(
text: str,
title: str,
*,
fallback_vendor: str | None = None,
) -> tuple[PromptSubjectInput, ...]:
vendor_name = infer_vendor_name(text, title) or fallback_vendor
equipment_type = infer_equipment_type(text, title)
model = infer_model(text, title, vendor_name=vendor_name)
model_family = infer_model_family(text, title, model=model)
return (
PromptSubjectInput(
vendor_name=vendor_name,
equipment_type=equipment_type,
model=model,
model_family=model_family,
),
)
def infer_metainfo_from_document(
path: Path,
*,
text: str,
title: str,
content_type: str,
fallback_vendor: str | None = None,
prompt_metainfo: PromptMetainfoInput | None = None,
) -> PromptMetainfoInput:
inferred_subjects = build_subjects(text, title, fallback_vendor=fallback_vendor)
metainfo = PromptMetainfoInput(
title=title,
document_type_name=infer_document_type_name(text, title),
content_type=content_type,
subjects=list(inferred_subjects),
)
if prompt_metainfo is None:
return metainfo
return PromptMetainfoInput(
title=prompt_metainfo.title or metainfo.title,
document_type_name=prompt_metainfo.document_type_name or metainfo.document_type_name,
content_type=prompt_metainfo.content_type or metainfo.content_type,
subjects=prompt_metainfo.subjects or metainfo.subjects,
)
def build_ingest_plan(
path: Path,
*,
prompt_metainfo: PromptMetainfoInput | None = None,
fallback_vendor: str | None = None,
) -> IngestPlan:
source = read_document(path)
metainfo = infer_metainfo_from_document(
path,
text=source.text_content,
title=source.title,
content_type=source.extracted_content_type,
fallback_vendor=fallback_vendor,
prompt_metainfo=prompt_metainfo,
)
return IngestPlan(source=source, metainfo=metainfo)
def iter_source_documents(source: Path, recursive: bool, extensions: Iterable[str]) -> list[Path]:
if source.is_file():
return [source]
extension_set = {ext.lower() for ext in extensions}
candidates = source.rglob("*") if recursive else source.iterdir()
return [
candidate
for candidate in candidates
if candidate.is_file() and candidate.suffix.lower() in extension_set
]
def canonical_subject_lines(subjects: Iterable[PromptSubjectInput]) -> list[str]:
lines: list[str] = []
for subject in subjects:
parts = []
if subject.vendor_name:
parts.append(subject.vendor_name)
if subject.equipment_type:
parts.append(subject.equipment_type)
if subject.model:
parts.append(subject.model)
if subject.model_family:
parts.append(subject.model_family)
if parts:
lines.append(" / ".join(parts))
return lines
def _extract_pdf_text(path: Path) -> tuple[str, str, str]:
try:
from pypdf import PdfReader
except ImportError as exc: # pragma: no cover - exercised manually if dependency missing
raise RuntimeError(
"PDF ingest requires the optional 'pypdf' dependency. Install techlib_mcpserver[client]."
) from exc
reader = PdfReader(path)
text_parts: list[str] = []
for page in reader.pages:
page_text = (page.extract_text() or "").strip()
if page_text:
text_parts.append(page_text)
pdf_title = None
metadata = getattr(reader, "metadata", None)
if metadata is not None:
pdf_title = getattr(metadata, "title", None)
if not pdf_title and isinstance(metadata, dict):
pdf_title = metadata.get("/Title")
extracted_text = "\n\n".join(text_parts).strip()
return extracted_text, "text/plain", infer_title(path, extracted_text, pdf_title=pdf_title)
__all__ = [
"build_ingest_plan",
"build_subjects",
"canonical_subject_lines",
"checksum_sha256",
"extract_document_text",
"guess_content_type",
"infer_document_type_name",
"infer_metainfo_from_document",
"infer_title",
"iter_source_documents",
"read_document",
]

View File

@ -0,0 +1,111 @@
"""Client-side ingest models and validation helpers."""
from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
class ClientModel(BaseModel):
"""Strict base model for client-side JSON payloads."""
model_config = ConfigDict(extra="forbid")
class PromptSubjectInput(ClientModel):
"""Structured subject metainfo produced by a prompt or heuristic inference."""
vendor_name: str | None = None
equipment_type: str | None = None
model: str | None = None
model_family: str | None = None
@field_validator("vendor_name", "equipment_type", "model", "model_family", mode="before")
@classmethod
def _blank_strings_become_none(cls, value: object) -> object:
if isinstance(value, str) and not value.strip():
return None
return value
@model_validator(mode="after")
def _at_least_one_field(self) -> "PromptSubjectInput":
if not any((self.vendor_name, self.equipment_type, self.model, self.model_family)):
raise ValueError(
"subject must include at least one of vendor_name, equipment_type, model, or model_family"
)
return self
class PromptMetainfoInput(ClientModel):
"""Prompt-normalized metainfo that can be used to build ingest payloads."""
title: str | None = None
document_type_name: str | None = None
content_type: str | None = None
subjects: list[PromptSubjectInput] = Field(default_factory=list)
@field_validator("title", "document_type_name", "content_type", mode="before")
@classmethod
def _blank_strings_become_none(cls, value: object) -> object:
if isinstance(value, str) and not value.strip():
return None
return value
@model_validator(mode="after")
def _validate_subjects(self) -> "PromptMetainfoInput":
if not self.subjects:
raise ValueError("subjects must not be empty")
if not self.document_type_name:
raise ValueError("document_type_name is required")
return self
@dataclass(slots=True)
class SourceDocument:
"""A document discovered by the client before ingest."""
path: Path
checksum_sha256: str
raw_bytes: bytes
text_content: str
extracted_content_type: str
source_uri: str
source_file_name: str
title: str
@dataclass(slots=True)
class IngestPlan:
"""Canonical ingest plan before transport mapping."""
source: SourceDocument
metainfo: PromptMetainfoInput
@dataclass(slots=True)
class ServerCapabilities:
"""Transport and payload capabilities detected from the live server."""
server_url: str
native_mcp_available: bool
http_facade_available: bool
supports_structured_subjects: bool
native_tool_names: tuple[str, ...] = ()
@property
def preferred_transport(self) -> str:
if self.native_mcp_available:
return "mcp"
return "http"
__all__ = [
"ClientModel",
"IngestPlan",
"PromptMetainfoInput",
"PromptSubjectInput",
"ServerCapabilities",
"SourceDocument",
]

View File

@ -0,0 +1,61 @@
"""Prompt generation and prompt-output validation helpers."""
from __future__ import annotations
import json
from pathlib import Path
from .models import PromptMetainfoInput
def build_metadata_prompt(
*,
source_path: Path,
source_preview: str,
prompt_preview_chars: int,
) -> str:
"""Create a strict prompt for manual or external LLM normalization."""
preview = source_preview[:prompt_preview_chars]
return (
"You are preparing metainfo for ingest into TechDoc Library MCP Server.\n"
"Return STRICT JSON only. No markdown, no code fences, no commentary.\n"
"Do not include subject_payload.\n\n"
"Required JSON shape:\n"
"{\n"
' "title": string | null,\n'
' "document_type_name": string,\n'
' "content_type": string | null,\n'
' "subjects": [\n'
" {\n"
' "vendor_name": string | null,\n'
' "equipment_type": string | null,\n'
' "model": string | null,\n'
' "model_family": string | null\n'
" }\n"
" ]\n"
"}\n\n"
"Rules:\n"
"- subjects must not be empty.\n"
"- Each subject must describe one equipment context.\n"
"- Use lower_snake_case for document_type_name.\n"
"- Keep null for unknown fields.\n"
"- Prefer one subject for one model family or model set.\n\n"
f"Source file: {source_path.name}\n\n"
"Document preview:\n"
f"{preview}\n"
)
def load_prompt_metainfo(source: str) -> PromptMetainfoInput:
"""Validate prompt output from a JSON string or JSON file."""
path = Path(source)
if path.exists():
payload = json.loads(path.read_text(encoding="utf-8"))
else:
payload = json.loads(source)
return PromptMetainfoInput.model_validate(payload)
__all__ = ["build_metadata_prompt", "load_prompt_metainfo"]

View File

@ -0,0 +1,91 @@
"""Duplicate suppression state for regular ingest runs."""
from __future__ import annotations
import json
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
@dataclass(slots=True)
class LedgerEntry:
source_uri: str
checksum_sha256: str
server_url: str
document_id: str | None
version_id: str | None
ingested_at: str
class IngestLedger:
"""Persist a lightweight ingest history to skip duplicates safely."""
def __init__(self, path: Path) -> None:
self.path = path
self._entries: dict[tuple[str, str, str], LedgerEntry] = {}
self.load()
def load(self) -> None:
if not self.path.exists():
return
payload = json.loads(self.path.read_text(encoding="utf-8"))
for item in payload.get("entries", []):
entry = LedgerEntry(
source_uri=item["source_uri"],
checksum_sha256=item["checksum_sha256"],
server_url=item["server_url"],
document_id=item.get("document_id"),
version_id=item.get("version_id"),
ingested_at=item["ingested_at"],
)
self._entries[(entry.source_uri, entry.checksum_sha256, entry.server_url)] = entry
def seen(self, source_uri: str, checksum_sha256: str, server_url: str) -> bool:
return (source_uri, checksum_sha256, server_url) in self._entries
def record(
self,
*,
source_uri: str,
checksum_sha256: str,
server_url: str,
document_id: str | None,
version_id: str | None,
) -> None:
entry = LedgerEntry(
source_uri=source_uri,
checksum_sha256=checksum_sha256,
server_url=server_url,
document_id=document_id,
version_id=version_id,
ingested_at=datetime.now(timezone.utc).isoformat(),
)
self._entries[(source_uri, checksum_sha256, server_url)] = entry
self.save()
def save(self) -> None:
self.path.parent.mkdir(parents=True, exist_ok=True)
payload: dict[str, Any] = {
"entries": [
{
"source_uri": entry.source_uri,
"checksum_sha256": entry.checksum_sha256,
"server_url": entry.server_url,
"document_id": entry.document_id,
"version_id": entry.version_id,
"ingested_at": entry.ingested_at,
}
for entry in sorted(
self._entries.values(),
key=lambda item: (item.server_url, item.source_uri, item.checksum_sha256),
)
]
}
temp_path = self.path.with_suffix(self.path.suffix + ".tmp")
temp_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
temp_path.replace(self.path)
__all__ = ["IngestLedger", "LedgerEntry"]

View File

@ -0,0 +1,206 @@
"""Transport helpers for the ingest client."""
from __future__ import annotations
import json
from dataclasses import dataclass
from typing import Any
from urllib.error import HTTPError, URLError
from urllib.request import Request, urlopen
from .models import IngestPlan, PromptSubjectInput, ServerCapabilities
SUPPORTED_PROTOCOL_VERSION = "2025-11-25"
@dataclass(slots=True)
class TransportResponse:
"""Normalized transport response from either HTTP facade or native MCP."""
transport: str
payload: dict[str, Any]
class MCPIngestClient:
"""Probe a live server and send ingest requests using the best available path."""
def __init__(
self,
server_url: str,
*,
transport_preference: str = "auto",
timeout_seconds: float = 60.0,
) -> None:
self.server_url = server_url.rstrip("/")
self.transport_preference = transport_preference
self.timeout_seconds = timeout_seconds
self._capabilities: ServerCapabilities | None = None
def probe(self) -> ServerCapabilities:
if self._capabilities is not None:
return self._capabilities
http_available, supports_structured = self._probe_openapi()
native_available, native_tools = self._probe_native_mcp()
native_ready = native_available and "ingest_document" in native_tools
self._capabilities = ServerCapabilities(
server_url=self.server_url,
native_mcp_available=native_ready,
http_facade_available=http_available,
supports_structured_subjects=supports_structured,
native_tool_names=native_tools,
)
return self._capabilities
def ingest(self, plan: IngestPlan) -> TransportResponse:
capabilities = self.probe()
if self.transport_preference == "mcp" and capabilities.native_mcp_available:
return self._native_ingest(plan, structured=capabilities.supports_structured_subjects)
if self.transport_preference == "http" and capabilities.http_facade_available:
return self._http_ingest(plan, structured=capabilities.supports_structured_subjects)
if capabilities.native_mcp_available:
try:
return self._native_ingest(
plan, structured=capabilities.supports_structured_subjects
)
except Exception:
if capabilities.http_facade_available:
return self._http_ingest(
plan, structured=capabilities.supports_structured_subjects
)
raise
return self._http_ingest(plan, structured=capabilities.supports_structured_subjects)
def _probe_openapi(self) -> tuple[bool, bool]:
try:
payload = self._request_json(f"{self.server_url}/openapi.json", method="GET")
except Exception:
return False, False
paths = payload.get("paths", {})
http_available = "/mcp/ingest_document" in paths
schemas = payload.get("components", {}).get("schemas", {})
metadata_schema = schemas.get("DocumentBaseMetadataInput", {})
properties = metadata_schema.get("properties", {})
supports_structured = "subjects" in properties
return http_available, supports_structured
def _probe_native_mcp(self) -> tuple[bool, tuple[str, ...]]:
initialize_payload = {
"jsonrpc": "2.0",
"id": 1,
"method": "initialize",
"params": {
"protocolVersion": SUPPORTED_PROTOCOL_VERSION,
"capabilities": {"tools": {}},
"clientInfo": {"name": "techlib_ingest_client", "version": "0.1.0"},
},
}
try:
initialize_response = self._request_json(
f"{self.server_url}/mcp",
method="POST",
payload=initialize_payload,
)
except Exception:
return False, ()
if "error" in initialize_response:
return False, ()
try:
tools_response = self._request_json(
f"{self.server_url}/mcp",
method="POST",
payload={
"jsonrpc": "2.0",
"id": 2,
"method": "tools/list",
"params": {},
},
)
except Exception:
return True, ()
tool_names: list[str] = []
result = tools_response.get("result", {})
for tool in result.get("tools", []):
name = tool.get("name")
if isinstance(name, str):
tool_names.append(name)
return True, tuple(tool_names)
def _build_payload(self, plan: IngestPlan, *, structured: bool) -> dict[str, Any]:
subject = plan.metainfo.subjects[0]
metadata: dict[str, Any] = {
"source_uri": plan.source.source_uri,
"source_file_name": plan.source.source_file_name,
"content_type": plan.metainfo.content_type or plan.source.extracted_content_type,
"document_type_name": plan.metainfo.document_type_name,
"title": plan.metainfo.title or plan.source.title,
}
if structured:
metadata["subjects"] = [_subject_payload(item) for item in plan.metainfo.subjects]
else:
metadata["vendor_name"] = subject.vendor_name
return {"content": plan.source.text_content, "metadata": metadata}
def _http_ingest(self, plan: IngestPlan, *, structured: bool) -> TransportResponse:
payload = self._build_payload(plan, structured=structured)
response = self._request_json(
f"{self.server_url}/mcp/ingest_document",
method="POST",
payload=payload,
)
return TransportResponse(transport="http", payload=response)
def _native_ingest(self, plan: IngestPlan, *, structured: bool) -> TransportResponse:
payload = self._build_payload(plan, structured=structured)
response = self._request_json(
f"{self.server_url}/mcp",
method="POST",
payload={
"jsonrpc": "2.0",
"id": 3,
"method": "tools/call",
"params": {"name": "ingest_document", "arguments": payload},
},
)
return TransportResponse(transport="mcp", payload=response)
def _request_json(
self,
url: str,
*,
method: str,
payload: dict[str, Any] | None = None,
) -> dict[str, Any]:
data = None
headers = {"Accept": "application/json"}
if payload is not None:
data = json.dumps(payload, ensure_ascii=False).encode("utf-8")
headers["Content-Type"] = "application/json"
request = Request(url, data=data, headers=headers, method=method)
try:
with urlopen(request, timeout=self.timeout_seconds) as response:
body = response.read().decode("utf-8")
except HTTPError as exc:
body = exc.read().decode("utf-8")
try:
return json.loads(body)
except json.JSONDecodeError:
raise RuntimeError(f"HTTP {exc.code} from {url}: {body}") from exc
except URLError as exc:
raise RuntimeError(f"failed to reach {url}: {exc}") from exc
return json.loads(body) if body else {}
def _subject_payload(subject: PromptSubjectInput) -> dict[str, Any]:
return subject.model_dump(exclude_none=True)
__all__ = ["MCPIngestClient", "SUPPORTED_PROTOCOL_VERSION", "TransportResponse"]

157
tests/test_ingest_client.py Normal file
View File

@ -0,0 +1,157 @@
from __future__ import annotations
import shutil
from contextlib import contextmanager
from pathlib import Path
from uuid import uuid4
import pytest
from pydantic import ValidationError
from techlib_mcpserver.client.metadata import build_ingest_plan
from techlib_mcpserver.client.models import ServerCapabilities
from techlib_mcpserver.client.prompting import build_metadata_prompt, load_prompt_metainfo
from techlib_mcpserver.client.state import IngestLedger
from techlib_mcpserver.client.transport import MCPIngestClient, TransportResponse
@contextmanager
def _workspace_dir() -> Path:
root = Path.cwd() / ".tmp" / "client-tests"
root.mkdir(parents=True, exist_ok=True)
path = root / uuid4().hex
path.mkdir(parents=True, exist_ok=True)
try:
yield path
finally:
shutil.rmtree(path, ignore_errors=True)
def _plan(tmp_dir: Path):
source = tmp_dir / "Huawei CloudEngine 6820 Switch Datasheet.txt"
source.write_text("Huawei CloudEngine 6820 Switch Datasheet\nCE6820-48S6CQ", encoding="utf-8")
return build_ingest_plan(source)
def test_prompt_generation_excludes_subject_payload() -> None:
with _workspace_dir() as tmp_dir:
plan = _plan(tmp_dir)
prompt = build_metadata_prompt(
source_path=plan.source.path,
source_preview=plan.source.text_content,
prompt_preview_chars=400,
)
assert "\"subject_payload\"" not in prompt
assert "subjects must not be empty" in prompt
def test_prompt_json_validation_rejects_extra_fields() -> None:
with _workspace_dir() as tmp_dir:
payload = tmp_dir / "metainfo.json"
payload.write_text(
"""
{
"title": "Huawei CloudEngine 6820 Switch Datasheet",
"document_type_name": "datasheet",
"content_type": "text/plain",
"subjects": [
{
"vendor_name": "Huawei",
"equipment_type": "switch",
"model": "CE6820-48S6CQ",
"model_family": "CloudEngine 6820 series",
"subject_payload": {}
}
]
}
""",
encoding="utf-8",
)
with pytest.raises(ValidationError):
load_prompt_metainfo(str(payload))
def test_client_probe_detects_structured_subjects_from_openapi() -> None:
client = MCPIngestClient("http://example.invalid")
def fake_request_json(url: str, *, method: str, payload=None):
if url.endswith("/openapi.json"):
return {
"paths": {"/mcp/ingest_document": {}},
"components": {
"schemas": {
"DocumentBaseMetadataInput": {
"properties": {"subjects": {"type": "array"}}
}
}
},
}
if url.endswith("/mcp") and payload and payload.get("method") == "tools/list":
return {"result": {"tools": [{"name": "ingest_document"}]}}
if url.endswith("/mcp") and payload and payload.get("method") == "initialize":
return {"result": {"protocolVersion": "2025-11-25"}}
raise AssertionError(url)
client._request_json = fake_request_json # type: ignore[method-assign]
capabilities = client.probe()
assert capabilities.http_facade_available is True
assert capabilities.native_mcp_available is True
assert capabilities.supports_structured_subjects is True
assert "ingest_document" in capabilities.native_tool_names
def test_client_builds_legacy_payload_without_subjects() -> None:
with _workspace_dir() as tmp_dir:
plan = _plan(tmp_dir)
client = MCPIngestClient("http://example.invalid")
payload = client._build_payload(plan, structured=False)
assert payload["metadata"]["vendor_name"] == "Huawei"
assert "subjects" not in payload["metadata"]
assert payload["metadata"]["document_type_name"] == plan.metainfo.document_type_name
def test_client_prefers_native_transport_when_available() -> None:
with _workspace_dir() as tmp_dir:
plan = _plan(tmp_dir)
client = MCPIngestClient("http://example.invalid")
client._capabilities = ServerCapabilities(
server_url=client.server_url,
native_mcp_available=True,
http_facade_available=True,
supports_structured_subjects=False,
native_tool_names=("ingest_document",),
)
recorded: list[str] = []
def fake_native_ingest(plan_arg, *, structured: bool):
recorded.append(f"native:{structured}")
return TransportResponse("mcp", {"result": {"ok": True}})
def fake_http_ingest(plan_arg, *, structured: bool):
recorded.append(f"http:{structured}")
return TransportResponse("http", {"result": {"ok": True}})
client._native_ingest = fake_native_ingest # type: ignore[method-assign]
client._http_ingest = fake_http_ingest # type: ignore[method-assign]
client.probe = lambda: client._capabilities # type: ignore[method-assign]
response = client.ingest(plan)
assert response.transport == "mcp"
assert recorded == ["native:False"]
def test_ingest_ledger_skips_duplicates() -> None:
with _workspace_dir() as tmp_dir:
ledger = IngestLedger(tmp_dir / "state.json")
assert ledger.seen("file:///docs/a.txt", "abc", "http://example.invalid") is False
ledger.record(
source_uri="file:///docs/a.txt",
checksum_sha256="abc",
server_url="http://example.invalid",
document_id="1",
version_id="2",
)
assert ledger.seen("file:///docs/a.txt", "abc", "http://example.invalid") is True