Julian Carlson
338 lines
14 KiB
Python
338 lines
14 KiB
Python
from fastapi import FastAPI, UploadFile, File, Form, HTTPException
|
|
from fastapi.responses import HTMLResponse
|
|
from pydantic import BaseModel
|
|
from typing import Optional
|
|
from pathlib import Path
|
|
import shutil
|
|
from .config import CONFIG
|
|
from .embed import embed_one
|
|
from .ingest import ingest_file
|
|
from . import db
|
|
|
|
app = FastAPI(title="KB Search")
|
|
|
|
|
|
class SearchRequest(BaseModel):
|
|
query: str
|
|
category: Optional[str] = None
|
|
k: int = 10
|
|
expand_neighbors: bool = True
|
|
neighbor_window: int = 1
|
|
rerank: bool = True
|
|
rerank_pool: int = 25
|
|
|
|
|
|
@app.post("/search")
|
|
def search(req: SearchRequest):
|
|
query_emb = embed_one(req.query)
|
|
|
|
# Fetch larger pool when reranking
|
|
initial_k = max(req.rerank_pool, req.k) if req.rerank else req.k
|
|
|
|
if req.expand_neighbors:
|
|
results = db.search_with_neighbors(
|
|
query_emb, k=initial_k, category=req.category,
|
|
neighbor_window=req.neighbor_window
|
|
)
|
|
else:
|
|
results = db.search(query_emb, k=initial_k, category=req.category)
|
|
|
|
rerank_used = False
|
|
if req.rerank and results:
|
|
try:
|
|
from .rerank import rerank as do_rerank
|
|
results = do_rerank(req.query, results, top_k=req.k)
|
|
rerank_used = True
|
|
except Exception as e:
|
|
print(f"Rerank failed, falling back to vector: {e}", flush=True)
|
|
results = results[:req.k]
|
|
else:
|
|
results = results[:req.k]
|
|
|
|
return {
|
|
"query": req.query,
|
|
"count": len(results),
|
|
"rerank_used": rerank_used,
|
|
"results": results,
|
|
}
|
|
|
|
|
|
@app.get("/stats")
|
|
def stats():
|
|
return db.get_stats()
|
|
|
|
|
|
@app.post("/upload")
|
|
async def upload(file: UploadFile = File(...), category: str = Form("uploads")):
|
|
if not file.filename:
|
|
raise HTTPException(400, "No filename")
|
|
|
|
safe_name = "".join(c for c in file.filename if c.isalnum() or c in "._- ")
|
|
target_dir = Path(CONFIG.corpus_root) / category
|
|
target_dir.mkdir(parents=True, exist_ok=True)
|
|
target = target_dir / safe_name
|
|
|
|
with open(target, "wb") as f:
|
|
shutil.copyfileobj(file.file, f)
|
|
|
|
result = ingest_file(target)
|
|
return result
|
|
|
|
|
|
@app.get("/upload", response_class=HTMLResponse)
|
|
def upload_page():
|
|
return """
|
|
<!DOCTYPE html>
|
|
<html>
|
|
<head>
|
|
<title>KB Upload</title>
|
|
<style>
|
|
body { font-family: -apple-system, sans-serif; max-width: 600px; margin: 50px auto; padding: 20px; }
|
|
.drop-zone { border: 2px dashed #888; padding: 40px; text-align: center; border-radius: 8px; cursor: pointer; }
|
|
.drop-zone.dragover { background: #f0f8ff; border-color: #4a9; }
|
|
select, input { padding: 8px; margin: 8px 0; width: 100%; box-sizing: border-box; }
|
|
button { padding: 10px 20px; background: #4a9; color: white; border: none; border-radius: 4px; cursor: pointer; }
|
|
#status { margin-top: 20px; padding: 10px; }
|
|
.ok { background: #efe; }
|
|
.err { background: #fee; }
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<h1>KB Upload</h1>
|
|
<form id="form" enctype="multipart/form-data">
|
|
<label>Category:</label>
|
|
<select name="category">
|
|
<option value="uploads">uploads (default)</option>
|
|
<option value="transcripts/claude">transcripts/claude</option>
|
|
<option value="transcripts/openclaw">transcripts/openclaw</option>
|
|
<option value="research/strategies">research/strategies</option>
|
|
<option value="research/notes">research/notes</option>
|
|
<option value="papers">papers</option>
|
|
<option value="courses">courses</option>
|
|
<option value="code-context">code-context</option>
|
|
</select>
|
|
<div class="drop-zone" id="dropzone">
|
|
<p>Drop files here or click to select</p>
|
|
<input type="file" name="file" id="file" multiple style="display:none">
|
|
</div>
|
|
<button type="submit">Upload</button>
|
|
</form>
|
|
<div id="status"></div>
|
|
<script>
|
|
const form = document.getElementById('form');
|
|
const fileInput = document.getElementById('file');
|
|
const dropzone = document.getElementById('dropzone');
|
|
const status = document.getElementById('status');
|
|
dropzone.onclick = () => fileInput.click();
|
|
dropzone.ondragover = (e) => { e.preventDefault(); dropzone.classList.add('dragover'); };
|
|
dropzone.ondragleave = () => dropzone.classList.remove('dragover');
|
|
dropzone.ondrop = (e) => {
|
|
e.preventDefault();
|
|
dropzone.classList.remove('dragover');
|
|
fileInput.files = e.dataTransfer.files;
|
|
dropzone.querySelector('p').textContent = `${fileInput.files.length} file(s) selected`;
|
|
};
|
|
fileInput.onchange = () => {
|
|
dropzone.querySelector('p').textContent = `${fileInput.files.length} file(s) selected`;
|
|
};
|
|
form.onsubmit = async (e) => {
|
|
e.preventDefault();
|
|
status.innerHTML = 'Uploading...';
|
|
status.className = '';
|
|
const category = form.querySelector('[name=category]').value;
|
|
const results = [];
|
|
for (const f of fileInput.files) {
|
|
const fd = new FormData();
|
|
fd.append('file', f);
|
|
fd.append('category', category);
|
|
try {
|
|
const r = await fetch('/upload', { method: 'POST', body: fd });
|
|
const j = await r.json();
|
|
results.push(`${f.name}: ${j.status}` + (j.chunks ? ` (${j.chunks} chunks)` : ''));
|
|
} catch (err) {
|
|
results.push(`${f.name}: error - ${err}`);
|
|
}
|
|
}
|
|
status.innerHTML = results.join('<br>');
|
|
status.className = 'ok';
|
|
};
|
|
</script>
|
|
</body>
|
|
</html>
|
|
"""
|
|
|
|
|
|
SEARCH_UI_HTML = """\
|
|
<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
<title>KB Search</title>
|
|
<style>
|
|
* { box-sizing: border-box; margin: 0; padding: 0; }
|
|
body { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; background: #0f1117; color: #e2e8f0; min-height: 100vh; }
|
|
.header { padding: 24px 32px 0; display: flex; align-items: center; justify-content: space-between; }
|
|
.header h1 { font-size: 1.4rem; font-weight: 600; letter-spacing: -0.02em; color: #f8fafc; }
|
|
.header a { color: #64748b; text-decoration: none; font-size: 0.85rem; }
|
|
.header a:hover { color: #94a3b8; }
|
|
.search-box { padding: 24px 32px; display: flex; gap: 10px; align-items: flex-start; flex-wrap: wrap; }
|
|
.search-box input[type=text] { flex: 1; min-width: 260px; padding: 10px 14px; background: #1e2130; border: 1px solid #334155; border-radius: 8px; color: #e2e8f0; font-size: 0.95rem; outline: none; }
|
|
.search-box input[type=text]:focus { border-color: #6366f1; }
|
|
.search-box button { padding: 10px 20px; background: #6366f1; color: #fff; border: none; border-radius: 8px; cursor: pointer; font-size: 0.95rem; white-space: nowrap; }
|
|
.search-box button:hover { background: #4f46e5; }
|
|
.search-box button:disabled { opacity: 0.5; cursor: default; }
|
|
.filters { padding: 0 32px 16px; display: flex; gap: 10px; flex-wrap: wrap; align-items: center; }
|
|
.filters label { font-size: 0.8rem; color: #64748b; }
|
|
.filters select, .filters input[type=number] { padding: 6px 10px; background: #1e2130; border: 1px solid #334155; border-radius: 6px; color: #e2e8f0; font-size: 0.82rem; }
|
|
.filters input[type=checkbox] { accent-color: #6366f1; }
|
|
.status { padding: 0 32px; font-size: 0.85rem; color: #64748b; min-height: 20px; }
|
|
.results { padding: 16px 32px 40px; display: flex; flex-direction: column; gap: 12px; }
|
|
.card { background: #1e2130; border: 1px solid #2d3748; border-radius: 10px; overflow: hidden; }
|
|
.card-header { padding: 12px 16px; display: flex; align-items: center; gap: 10px; cursor: pointer; user-select: none; }
|
|
.card-header:hover { background: #252a3a; }
|
|
.card-rank { font-size: 0.75rem; color: #64748b; min-width: 24px; }
|
|
.card-source { font-size: 0.78rem; color: #6366f1; font-family: monospace; flex: 1; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
|
.card-topic { font-size: 0.82rem; color: #94a3b8; flex: 2; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
|
.card-score { font-size: 0.75rem; color: #475569; white-space: nowrap; }
|
|
.card-badges { display: flex; gap: 4px; }
|
|
.badge { font-size: 0.68rem; padding: 2px 6px; border-radius: 4px; font-weight: 500; }
|
|
.badge-vec { background: #1e3a5f; color: #60a5fa; }
|
|
.badge-kw { background: #1a3a2a; color: #4ade80; }
|
|
.card-body { padding: 0 16px 16px; display: none; }
|
|
.card-body.open { display: block; }
|
|
.card-body pre { white-space: pre-wrap; word-break: break-word; font-size: 0.84rem; color: #cbd5e1; line-height: 1.6; font-family: inherit; margin-top: 10px; padding: 12px; background: #141720; border-radius: 6px; border: 1px solid #2d3748; max-height: 400px; overflow-y: auto; }
|
|
.card-meta { margin-top: 8px; font-size: 0.75rem; color: #475569; }
|
|
.empty { padding: 48px 32px; text-align: center; color: #475569; }
|
|
.spinner { display: inline-block; width: 16px; height: 16px; border: 2px solid #334155; border-top-color: #6366f1; border-radius: 50%; animation: spin 0.7s linear infinite; vertical-align: middle; margin-right: 6px; }
|
|
@keyframes spin { to { transform: rotate(360deg); } }
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<div class="header">
|
|
<h1>🔍 KB Search</h1>
|
|
<div style="display:flex;gap:16px">
|
|
<a href="/upload">Upload</a>
|
|
<a href="/stats">Stats</a>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="search-box">
|
|
<input type="text" id="q" placeholder="Search your knowledge base..." autofocus>
|
|
<button id="btn" onclick="doSearch()">Search</button>
|
|
</div>
|
|
|
|
<div class="filters">
|
|
<label>Category:</label>
|
|
<select id="cat">
|
|
<option value="">All</option>
|
|
<option value="transcripts/openclaw">OpenClaw transcripts</option>
|
|
<option value="transcripts/claude">Claude transcripts</option>
|
|
<option value="research">Research</option>
|
|
<option value="papers">Papers</option>
|
|
<option value="courses">Courses</option>
|
|
<option value="code-context">Code</option>
|
|
<option value="uploads">Uploads</option>
|
|
</select>
|
|
<label>Results:</label>
|
|
<input type="number" id="k" value="10" min="1" max="50" style="width:60px">
|
|
<label><input type="checkbox" id="rerank" checked> Rerank</label>
|
|
<label><input type="checkbox" id="expand" checked> Expand neighbors</label>
|
|
</div>
|
|
|
|
<div class="status" id="status"></div>
|
|
<div class="results" id="results"></div>
|
|
|
|
<script>
|
|
const q = document.getElementById('q');
|
|
const btn = document.getElementById('btn');
|
|
const status = document.getElementById('status');
|
|
const res = document.getElementById('results');
|
|
|
|
q.addEventListener('keydown', function(e) { if (e.key === 'Enter') doSearch(); });
|
|
|
|
async function doSearch() {
|
|
var query = q.value.trim();
|
|
if (!query) return;
|
|
btn.disabled = true;
|
|
status.innerHTML = '<span class="spinner"></span>Searching...';
|
|
res.innerHTML = '';
|
|
|
|
try {
|
|
var body = {
|
|
query: query,
|
|
k: parseInt(document.getElementById('k').value) || 10,
|
|
rerank: document.getElementById('rerank').checked,
|
|
expand_neighbors: document.getElementById('expand').checked,
|
|
neighbor_window: 1,
|
|
rerank_pool: 25
|
|
};
|
|
var cat = document.getElementById('cat').value;
|
|
if (cat) body.category = cat;
|
|
|
|
var r = await fetch('/search', {
|
|
method: 'POST',
|
|
headers: {'Content-Type': 'application/json'},
|
|
body: JSON.stringify(body)
|
|
});
|
|
var data = await r.json();
|
|
|
|
if (!r.ok) { status.textContent = 'Error: ' + (data.detail || r.status); return; }
|
|
|
|
var count = data.count || 0;
|
|
var reranked = data.rerank_used ? ' \u00b7 reranked' : '';
|
|
status.textContent = count === 0 ? 'No results.' : (count + ' result' + (count !== 1 ? 's' : '') + reranked);
|
|
|
|
if (count === 0) { res.innerHTML = '<div class="empty">No matching chunks found.</div>'; return; }
|
|
|
|
res.innerHTML = data.results.map(function(result, i) {
|
|
var src = result.source_path || (result.metadata && result.metadata.source_path) || '\u2014';
|
|
var srcShort = src.split('/').slice(-2).join('/');
|
|
var topic = result.topic || (result.metadata && result.metadata.topic) || '';
|
|
var score = result.score != null ? result.score.toFixed(3) : '';
|
|
var isVec = result.matched_vector;
|
|
var isKw = result.matched_keyword;
|
|
var content = (result.content || '').replace(/</g,'<').replace(/>/g,'>');
|
|
var tokens = result.token_count ? (result.token_count + ' tokens') : '';
|
|
var chunk = result.chunk_index != null ? ('chunk ' + result.chunk_index) : '';
|
|
|
|
var badges = (isVec ? '<span class="badge badge-vec">vec</span>' : '') +
|
|
(isKw ? '<span class="badge badge-kw">kw</span>' : '');
|
|
var meta = [srcShort, chunk, tokens].filter(Boolean).join(' \u00b7 ');
|
|
|
|
return '<div class="card">' +
|
|
'<div class="card-header" onclick="toggle(this)">' +
|
|
'<span class="card-rank">#' + (i+1) + '</span>' +
|
|
'<span class="card-source" title="' + src + '">' + srcShort + '</span>' +
|
|
'<span class="card-topic" title="' + topic + '">' + topic + '</span>' +
|
|
'<div class="card-badges">' + badges + '</div>' +
|
|
'<span class="card-score">' + score + '</span>' +
|
|
'</div>' +
|
|
'<div class="card-body">' +
|
|
'<pre>' + content + '</pre>' +
|
|
'<div class="card-meta">' + meta + '</div>' +
|
|
'</div>' +
|
|
'</div>';
|
|
}).join('');
|
|
|
|
} catch(e) {
|
|
status.textContent = 'Error: ' + e.message;
|
|
} finally {
|
|
btn.disabled = false;
|
|
}
|
|
}
|
|
|
|
function toggle(header) {
|
|
var body = header.nextElementSibling;
|
|
body.classList.toggle('open');
|
|
}
|
|
</script>
|
|
</body>
|
|
</html>
|
|
"""
|
|
|
|
|
|
@app.get("/", response_class=HTMLResponse)
|
|
def search_ui():
|
|
return SEARCH_UI_HTML
|