#!/usr/bin/env python3 """Second SO4AG 0.1 reader (Python, standard library only, Apache-2.0). Written separately from the reference implementation in validator/ and shares no code with it. It reads a source from SO4AG JSON, from a JSON-LD document or from an HTML page with embedded JSON-LD, and answers one question: may this claim be used for this task at this time? python so4ag_reader.py conformance run the conformance suite python so4ag_reader.py ask --property [--scope k=v ...] [--now ISO] [--review] """ import hashlib import json import re import sys from datetime import datetime, timezone from pathlib import Path SOURCE_TYPES = ("so4ag:Source", "https://soqver.com/spec/so4ag/0.1/terms#Source") # ---------- canonical JSON (RFC 8785 for the values SO4AG uses) and claim digest ---------- def canonical(v): if isinstance(v, dict): keys = sorted(v, key=lambda k: k.encode("utf-16-be")) return "{" + ",".join(json.dumps(k, ensure_ascii=False) + ":" + canonical(v[k]) for k in keys) + "}" if isinstance(v, list): return "[" + ",".join(canonical(x) for x in v) + "]" if isinstance(v, float) and v == int(v) and abs(v) < 1e15: return str(int(v)) return json.dumps(v, ensure_ascii=False) def claim_digest(claim): body = {k: v for k, v in claim.items() if k != "attestations"} return "sha256:" + hashlib.sha256(canonical(body).encode("utf-8")).hexdigest() # ---------- time ---------- def instant(text): t = datetime.fromisoformat(text.replace("Z", "+00:00")) return t if t.tzinfo else t.replace(tzinfo=timezone.utc) # ---------- the consumer rules ---------- def scope_keys_of(property_uri, profiles): for p in profiles: ns = p["namespace"] if property_uri.startswith(ns) and property_uri[len(ns):] in p["properties"]: return set(p.get("scope_keys", {})) return set() def evaluate(claim, task, now, profiles): """Returns (usable, reasons, value). `disputed` is reported but does not block.""" reasons = [] if task.get("property") and task["property"] != claim["property"]: reasons.append("property_mismatch") if task.get("subject") and task["subject"] != claim["subject"]: reasons.append("subject_mismatch") understood = set(task["understoodScopeKeys"]) if "understoodScopeKeys" in task else scope_keys_of(claim["property"], profiles) asked = task.get("scope") or {} for key, limit in claim["scope"].items(): if key not in understood: reasons.append("scope_not_comparable:" + key) elif key not in asked: reasons.append("scope_not_in_task:" + key) elif limit != "any" and scope_text(asked[key]) != limit: reasons.append("scope_mismatch:" + key) if now < instant(claim["valid_from"]): reasons.append("not_yet_valid") if now >= instant(claim["valid_until"]): reasons.append("expired") if claim.get("dispute"): reasons.append("disputed") if claim["knowledge_state"] != "known": reasons.append(claim["knowledge_state"]) if task.get("requireReview"): digest = claim_digest(claim) trusted = task.get("trustedIssuers") accepted = any( a.get("claim_digest") == digest and a.get("issuer_relation") == "independent" and instant(a["expires_at"]) > now and (trusted is None or a.get("issuer") in trusted) for a in claim.get("attestations", []) ) if not accepted: reasons.append("no_valid_independent_attestation") usable = all(r == "disputed" for r in reasons) return usable, sorted(reasons), (claim.get("value") if usable else None) def scope_text(v): if isinstance(v, bool): return "true" if v else "false" if isinstance(v, float) and v == int(v): return str(int(v)) return str(v) # ---------- reading JSON-LD ---------- def identifiers(node_list): return [{"scheme": i["propertyID"], "value": i["value"]} for i in node_list] def value_from_node(n): if isinstance(n, bool): return {"type": "boolean", "value": n} if "@value" in n: out = {"type": "text", "value": n["@value"]} if "@language" in n: out["lang"] = n["@language"] return out kind = n.get("@type") if kind == "MonetaryAmount": out = {"type": "money", "amount": n["minValue"] if "minValue" in n else n["value"], "currency": n["currency"]} if "so4ag:vatIncluded" in n: out["vat_included"] = n["so4ag:vatIncluded"] if "minValue" in n: out["bound"] = "minimum" elif "so4ag:bound" in n: out["bound"] = n["so4ag:bound"] return out if kind == "QuantitativeValue": return {"type": "quantity", "value": n["value"], "unit": n["unitText"]} if kind == "DefinedTerm": return {"type": "enum", "value": n["termCode"]} if kind == "ItemList": return {"type": "enum_set", "values": [x["termCode"] for x in n["itemListElement"]]} if kind == "Organization": out = {"type": "organization", "name": n["name"]} if "identifier" in n: out["identifiers"] = identifiers(n["identifier"]) if "url" in n: out["url"] = n["url"] return out if kind == "Place": out = {"type": "place", "name": n["name"]} address = n.get("address") or {} if "streetAddress" in address: out["address"] = address["streetAddress"] if "addressCountry" in address: out["country"] = address["addressCountry"] if "geo" in n: out["geo"] = {"lat": n["geo"]["latitude"], "lon": n["geo"]["longitude"]} if "openingHours" in n: out["opening_hours"] = n["openingHours"] return out raise ValueError("unknown value node") def copy_fields(src, pairs): return {to: src[frm] for frm, to in pairs if frm in src} def claim_from_node(pv, subject): c = {"id": pv["@id"], "subject": subject, "property": pv["propertyID"], "knowledge_state": pv["so4ag:knowledgeState"], "assertion_basis": pv["so4ag:assertionBasis"], "valid_from": pv["validFrom"], "valid_until": pv["validThrough"], "scope": {s["propertyID"]: s["value"] for s in pv.get("so4ag:scope", [])}} if "so4ag:revision" in pv: c["revision"] = pv["so4ag:revision"] if "value" in pv: c["value"] = value_from_node(pv["value"]) if "so4ag:evidence" in pv: c["evidence"] = [copy_fields(e, [("url", "url"), ("so4ag:observedAt", "observed_at"), ("so4ag:kind", "kind"), ("so4ag:issuer", "issuer"), ("so4ag:quote", "quote"), ("so4ag:digest", "digest")]) for e in pv["so4ag:evidence"]] if "so4ag:note" in pv: c["note"] = pv["so4ag:note"] if "so4ag:dispute" in pv: d = pv["so4ag:dispute"] c["dispute"] = copy_fields(d, [("so4ag:since", "since"), ("so4ag:note", "note")]) if "so4ag:conflictingClaim" in d: c["dispute"]["conflicting_claims"] = [x["@id"] for x in d["so4ag:conflictingClaim"]] if "so4ag:attestation" in pv: c["attestations"] = [] for a in pv["so4ag:attestation"]: item = copy_fields(a, [("@id", "id"), ("so4ag:issuerRelation", "issuer_relation"), ("so4ag:method", "method"), ("so4ag:claimDigest", "claim_digest"), ("so4ag:issuedAt", "issued_at"), ("so4ag:expiresAt", "expires_at"), ("so4ag:statusUrl", "status_url"), ("so4ag:credentialUrl", "credential_url")]) if "so4ag:issuer" in a: item["issuer"] = a["so4ag:issuer"]["@id"] c["attestations"].append(item) if "so4ag:extensions" in pv: c["extensions"] = pv["so4ag:extensions"]["@value"] return c def claims_from_jsonld(doc): graph = doc.get("@graph") if not isinstance(graph, list) or not any(n.get("additionalType") in SOURCE_TYPES for n in graph): raise ValueError("not a SO4AG source") claims = [] for node in graph: if node.get("additionalType") in SOURCE_TYPES: continue for pv in node.get("additionalProperty", []): if "so4ag:knowledgeState" in pv: claims.append(claim_from_node(pv, node["@id"])) return claims DEFAULT_KEYS = ("subject", "assertion_basis", "valid_from", "valid_until", "evidence", "scope") def expand02(doc): """SO4AG 0.2 -> claims written out in full: file defaults applied, null = the claim has no such field.""" defaults = doc.get("defaults") or {} out = [] for claim in doc.get("claims", []): c = dict(claim) for k in DEFAULT_KEYS: if k in c and c[k] is None: del c[k] elif k not in c and k in defaults: c[k] = json.loads(json.dumps(defaults[k])) c.setdefault("scope", {}) out.append(c) return out def claims_from_text(text): """SO4AG JSON (0.1 or 0.2), JSON-LD, or an HTML page with JSON-LD blocks.""" stripped = text.lstrip() if stripped.startswith("{"): doc = json.loads(text) if doc.get("so4ag") == "0.2-draft": return expand02(doc) return doc["claims"] if "claims" in doc else claims_from_jsonld(doc) for block in re.findall(r']*type\s*=\s*["\']application/ld\+json["\'][^>]*>(.*?)', text, re.S | re.I): try: doc = json.loads(block) except ValueError: continue for d in doc if isinstance(doc, list) else [doc]: try: return claims_from_jsonld(d) except (ValueError, AttributeError): pass raise ValueError("no SO4AG source found") # ---------- command line ---------- def load_profiles(root): return [json.loads(p.read_text(encoding="utf-8")) for p in sorted(Path(root).glob("*.json"))] def run_conformance(folder): folder = Path(folder) profiles = load_profiles(folder.parent.parent / "profiles") failed = total = 0 def check(ok, label): nonlocal failed, total total += 1 if not ok: failed += 1 print("FAIL", label) digests = json.loads((folder / "digests.json").read_text(encoding="utf-8")) sources = {} for name in ("shop", "rental"): native = json.loads((folder / f"{name}.json").read_text(encoding="utf-8"))["claims"] from_ld = claims_from_text((folder / f"{name}.jsonld").read_text(encoding="utf-8")) html = '" from_html = claims_from_text(html) sources[name + ".json"] = {"so4ag": native, "jsonld": from_ld, "html": from_html} for form, claims in sources[name + ".json"].items(): check(len(claims) == len(native), f"{name} {form}: number of claims") for c in claims: check(claim_digest(c) == digests.get(c["id"]), f"{name} {form}: digest of {c['id']}") cases = json.loads((folder / "evaluation.json").read_text(encoding="utf-8"))["cases"] for case in cases: for form, claims in sources[case["manifest"]].items(): claim = next(c for c in claims if c["id"] == case["claim"]) usable, reasons, value = evaluate(claim, case["task"], instant(case["now"]), profiles) want = case["expect"] check(usable == want["usable"] and reasons == want["reasons"] and value == want["value"], f"{case['id']} ({form}) {case['title']}: got {usable} {reasons}") print(f"{total - failed}/{total} checks passed ({len(cases)} evaluation cases x 3 input forms, digests of every claim)") return 1 if failed else 0 def run_ask(argv): path, prop, scope, now, review = argv[0], None, {}, datetime.now(timezone.utc), False i = 1 while i < len(argv): if argv[i] == "--property": prop = argv[i + 1]; i += 2 elif argv[i] == "--scope": k, _, v = argv[i + 1].partition("="); scope[k] = v; i += 2 elif argv[i] == "--now": now = instant(argv[i + 1]); i += 2 elif argv[i] == "--review": review = True; i += 1 else: raise SystemExit("unknown argument " + argv[i]) here = Path(__file__).resolve().parent profiles = load_profiles(here.parent.parent / "spec" / "profiles") claims = claims_from_text(Path(path).read_text(encoding="utf-8")) out = [] for c in claims: if prop and c["property"] != prop and not c["property"].endswith("#" + prop): continue task = {"property": c["property"], "scope": scope, "requireReview": review} usable, reasons, value = evaluate(c, task, now, profiles) out.append({"claim": c["id"], "usable": usable, "reasons": reasons, "value": value, "scope": c["scope"], "valid_until": c["valid_until"]}) out.sort(key=lambda r: not r["usable"]) print(json.dumps(out, ensure_ascii=False, indent=2)) return 0 if __name__ == "__main__": if len(sys.argv) >= 3 and sys.argv[1] == "conformance": sys.exit(run_conformance(sys.argv[2])) if len(sys.argv) >= 3 and sys.argv[1] == "ask": sys.exit(run_ask(sys.argv[2:])) print(__doc__) sys.exit(2)