Editing Openai/691c1dba-9228-800f-8463-13b3a9006306 (section)

=== JSON schema file path (you can place the schema provided earlier as schema.json) ===
SCHEMA_PATH = "hplm_bridge_schema.json"

def load_schema():
    with open(SCHEMA_PATH,'r') as f:
        return json.load(f)

def canonical_json_bytes(obj):
    # canonical: sort keys, ensure separators, ensure utf-8
    return json.dumps(obj, sort_keys=True, ensure_ascii=False, separators=(',',':')).encode('utf-8')

def sha256(raw):
    import hashlib
    h = hashlib.sha256()
    h.update(raw)
    return h.hexdigest()

def hmac_sign(raw, key):
    return hmac.new(key, raw, hashlib.sha256).hexdigest()

def validate_file(path, schema, hmac_key=None):
    raw = path.read_bytes()
    try:
        obj = json.loads(raw.decode('utf-8'))
    except Exception as e:
        return False, f"JSON decode error: {e}", None
    # schema validation
    try:
        jsonschema.validate(instance=obj, schema=schema)
    except Exception as e:
        return False, f"Schema validation error: {e}", None
    # canonicalize and hash
    cj = canonical_json_bytes(obj)
    sha = sha256(cj)
    sig = None
    if hmac_key:
        sig = hmac_sign(cj, hmac_key)
    meta = {"path": str(path), "sha256": sha, "hmac": sig, "loaded_title": obj.get("title")}
    return True, "ok", meta

def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("indir", help="directory with json capsules")
    ap.add_argument("--hmac_key", help="optional hex key")
    args = ap.parse_args()
    key = bytes.fromhex(args.hmac_key) if args.hmac_key else None
    schema = load_schema()
    results=[]
    p = Path(args.indir)
    for f in p.glob("*.json"):
        ok,msg,meta = validate_file(f,schema,key)
        results.append({"file":str(f), "ok":ok, "msg":msg, "meta":meta})
    out = {"dir":str(p), "checked_at":datetime.datetime.utcnow().isoformat()+"Z", "results":results}
    print(json.dumps(out, indent=2))
if __name__=='__main__':
    main()

* Put the JSON Schema (from earlier) as hplm_bridge_schema.json.
* Run: python3 ingest_validate.py /path/to/dump --hmac_key <hex> (hmac key optional).
* Output: JSON report of validated files.

==== 2) Summarize & normalize (chunked summarizer) ====

If files are large or not summarized, run chunking + summarizer. If you want to use local LLMs, replace call to call_local_model with your webui endpoints.

Save summarize_normalize.py:

python
#!/usr/bin/env python3