Editing Openai/691c1dba-9228-800f-8463-13b3a9006306 (section)

=== compare_score.py ===
import json, os, argparse
from pathlib import Path
from collections import defaultdict
try:
    from sentence_transformers import SentenceTransformer, util
    EMB = SentenceTransformer('all-MiniLM-L6-v2')
except Exception:
    EMB = None

def sim(a,b):
    if EMB:
        v1 = EMB.encode(a, convert_to_tensor=True)
        v2 = EMB.encode(b, convert_to_tensor=True)
        return float(util.cos_sim(v1,v2).item())
    # fallback token overlap
    sa=set(a.split())
    sb=set(b.split())
    if not sa or not sb: return 0.0
    return len(sa & sb)/len(sa | sb)

def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("summaries_dir")
    args = ap.parse_args()
    p = Path(args.summaries_dir)
    files = list(p.glob("*_summary.json"))
    data=[]
    for f in files:
        obj=json.loads(f.read_text(encoding='utf-8'))
        data.append((str(f), obj['summary']))
    # pairwise similarity
    sims=[]
    for i in range(len(data)):
        for j in range(i+1,len(data)):
            s=sim(data[i][1], data[j][1])
            sims.append({"a":data[i][0],"b":data[j][0],"sim":s})
    out = {"pairs":sims}
    print(json.dumps(out, indent=2))

if __name__=='__main__':
    main()

* Run: python3 compare_score.py /path/to/files/summaries

==== 4) Selection & merge (rules) ====

I’ll give you an automatic merge algorithm (pick highest-scoring, union key facts, keep provenance).

merge_selector.py:

python
#!/usr/bin/env python3
import json, argparse
from pathlib import Path
def merge_summaries(summary_files, outpath="merged_capsule.json"):
    merged = {"title":"Merged Capsule", "version":"1.0", "author":"merged", "timestamp":None,
              "hplm":{}, "sources":[], "state":{}}
    # naive merge: collect summaries and ranked sentences
    texts=[]
    for f in summary_files:
        obj=json.loads(Path(f).read_text(encoding='utf-8'))
        texts.append(obj['summary'])
        merged['sources'].append({"file":f})
    # for simplicity, take top N sentences from each summary
    merged_text = "\n".join([t for t in texts])
    merged['hplm']={"merged_summary": merged_text[:2000]}
    Path(outpath).write_text(json.dumps(merged, indent=2), encoding='utf-8')
    print("Wrote merged capsule to", outpath)

if __name__=='__main__':
    import sys
    merge_summaries(sys.argv[1:])

==== 5) Distillation / dataset generation ====

Produce (prompt, best_response) pairs by extracting Q&A logs or calling ensemble and selecting best. Save as JSONL for training.

Example make_dataset.py:

python
#!/usr/bin/env python3