Improve GVL explorer vendor evidence workflow

src/background.js

@@ -73,6 +73,18 @@ async function handleVendorGetMessage(message, sender) {
     return handleGetGvlSnapshotSummaryMessage(message);
   }
 
+  if (message.type === "rebuild_gvl_snapshot_normalized_data") {
+    return handleRebuildGvlSnapshotNormalizedDataMessage(message);
+  }
+
+  if (message.type === "list_gvl_vendors_for_snapshot") {
+    return handleListGvlVendorsForSnapshotMessage(message);
+  }
+
+  if (message.type === "get_gvl_vendor_detail") {
+    return handleGetGvlVendorDetailMessage(message);
+  }
+
   if (message.type === "get_latest_consent_state") {
     return handleGetLatestConsentStateMessage();
   }
@@ -280,12 +292,15 @@ async function handleGetGvlSnapshotSummaryMessage(message) {
       eventType: event?.eventType ?? null,
       eventCapturedAt: event?.capturedAt ?? null,
       vendorCount: snapshot.vendorCount ?? counts.vendorCount,
+      snapshotVendorCount: snapshot.vendorCount ?? null,
+      normalizedVendorCount: counts.vendorCount,
       purposeCount: snapshot.purposeCount ?? counts.purposeCount,
       specialPurposeCount: counts.specialPurposeCount,
       featureCount: counts.featureCount,
       specialFeatureCount: counts.specialFeatureCount,
       dataCategoryCount: counts.dataCategoryCount,
       vendorRelationshipCount: counts.vendorRelationshipCount,
+      normalizedVendorRelationshipCount: counts.vendorRelationshipCount,
       technicalFields: {
         snapshotStore: "gvl_snapshots",
         vendorListVersion: "vendorListVersion",
@@ -300,6 +315,267 @@ async function handleGetGvlSnapshotSummaryMessage(message) {
   };
 }
 
+async function handleRebuildGvlSnapshotNormalizedDataMessage(message) {
+  const db = await openVendorGetDb();
+  const payload = message?.payload ?? {};
+  const snapshot = await getGvlSnapshotByIdentifier(db, {
+    sha256: payload.sha256 ?? null,
+    vendorListVersion: payload.vendorListVersion ?? null
+  });
+
+  if (!snapshot) {
+    return {
+      success: false,
+      error: "gvl_snapshot_not_found"
+    };
+  }
+
+  const normalizationSummary = await normalizeGvlSnapshotWithExistingPipeline(
+    snapshot
+  );
+  const counts = await countGvlNormalizedRecordsForVersion(
+    db,
+    snapshot.vendorListVersion ?? null
+  );
+
+  return {
+    success: true,
+    snapshotSha256: snapshot.sha256 ?? null,
+    vendorListVersion: snapshot.vendorListVersion ?? null,
+    normalizationSummary,
+    counts
+  };
+}
+
+async function handleListGvlVendorsForSnapshotMessage(message) {
+  const db = await openVendorGetDb();
+  const payload = message?.payload ?? {};
+  const snapshot = await getGvlSnapshotByIdentifier(db, {
+    sha256: payload.sha256 ?? null,
+    vendorListVersion: payload.vendorListVersion ?? null
+  });
+
+  if (!snapshot) {
+    return {
+      success: false,
+      error: "gvl_snapshot_not_found"
+    };
+  }
+
+  const vendors = await listGvlVendorsForSnapshot(db, snapshot);
+
+  return {
+    success: true,
+    snapshotSha256: snapshot.sha256 ?? null,
+    vendorListVersion: snapshot.vendorListVersion ?? null,
+    vendors
+  };
+}
+
+function listGvlVendorsForSnapshot(db, snapshot) {
+  return new Promise((resolve, reject) => {
+    const tx = db.transaction([VENDORGET_STORE_NAMES.gvlVendors], "readonly");
+    const vendorsStore = tx.objectStore(VENDORGET_STORE_NAMES.gvlVendors);
+    const vendorListVersionIndex = vendorsStore.index("vendorListVersion");
+    const cursorRequest = vendorListVersionIndex.openCursor(
+      IDBKeyRange.only(snapshot.vendorListVersion)
+    );
+    const vendors = [];
+
+    cursorRequest.onerror = () => reject(cursorRequest.error);
+    cursorRequest.onsuccess = () => {
+      const cursor = cursorRequest.result;
+
+      if (!cursor) {
+        return;
+      }
+
+      if (cursor.value?.snapshotSha256 === snapshot.sha256) {
+        vendors.push({
+          vendorId: cursor.value.vendorId ?? null,
+          name: cursor.value.name ?? null,
+          deletedDate: cursor.value.deletedDate ?? null,
+          snapshotSha256: cursor.value.snapshotSha256 ?? null
+        });
+      }
+
+      cursor.continue();
+    };
+
+    tx.onerror = () => reject(tx.error);
+    tx.onabort = () => reject(tx.error);
+    tx.oncomplete = () => {
+      resolve(
+        vendors.sort((left, right) => {
+          return (
+            toComparableNumber(left.vendorId) -
+            toComparableNumber(right.vendorId)
+          );
+        })
+      );
+    };
+  });
+}
+
+async function handleGetGvlVendorDetailMessage(message) {
+  const db = await openVendorGetDb();
+  const vendorId = parseGvlVendorDetailId(message?.payload?.vendorId);
+
+  if (vendorId === null) {
+    return {
+      success: false,
+      error: "invalid_vendor_id"
+    };
+  }
+
+  const vendorRecord = await getLatestGvlVendorByVendorId(db, vendorId);
+
+  if (!vendorRecord) {
+    return {
+      success: false,
+      error: "gvl_vendor_not_found"
+    };
+  }
+
+  const snapshotSha256 = vendorRecord.snapshotSha256 ?? null;
+  const snapshot = snapshotSha256
+    ? await getGvlSnapshotBySha256(db, snapshotSha256)
+    : null;
+  const rawGvlSha256 = snapshot?.rawGvlSha256 ?? null;
+  const rawEvidence = rawGvlSha256
+    ? await getGvlRawEvidenceBySha256(db, rawGvlSha256)
+    : null;
+
+  return {
+    success: true,
+    vendorDetail: {
+      vendor: vendorRecord,
+      snapshot: buildGvlVendorDetailSnapshotSummary(snapshot, snapshotSha256),
+      rawEvidence: buildGvlVendorDetailRawEvidenceSummary(
+        rawEvidence,
+        rawGvlSha256
+      )
+    }
+  };
+}
+
+function parseGvlVendorDetailId(value) {
+  const vendorId = Number(value);
+
+  if (!Number.isInteger(vendorId) || vendorId <= 0) {
+    return null;
+  }
+
+  return vendorId;
+}
+
+function getLatestGvlVendorByVendorId(db, vendorId) {
+  return new Promise((resolve, reject) => {
+    const tx = db.transaction([VENDORGET_STORE_NAMES.gvlVendors], "readonly");
+    const vendorsStore = tx.objectStore(VENDORGET_STORE_NAMES.gvlVendors);
+    const vendorIdIndex = vendorsStore.index("vendorId");
+    const cursorRequest = vendorIdIndex.openCursor(IDBKeyRange.only(vendorId));
+    const vendors = [];
+
+    cursorRequest.onerror = () => reject(cursorRequest.error);
+    cursorRequest.onsuccess = () => {
+      const cursor = cursorRequest.result;
+
+      if (!cursor) {
+        return;
+      }
+
+      vendors.push(cursor.value);
+      cursor.continue();
+    };
+
+    tx.onerror = () => reject(tx.error);
+    tx.onabort = () => reject(tx.error);
+    tx.oncomplete = () => {
+      resolve(sortGvlVendorRecordsNewestFirst(vendors)[0] ?? null);
+    };
+  });
+}
+
+function sortGvlVendorRecordsNewestFirst(vendorRecords) {
+  return vendorRecords.slice().sort((left, right) => {
+    const fetchedAtComparison =
+      toComparableTime(right.snapshotFetchedAt) -
+      toComparableTime(left.snapshotFetchedAt);
+
+    if (fetchedAtComparison !== 0) {
+      return fetchedAtComparison;
+    }
+
+    return (
+      toComparableNumber(right.vendorListVersion) -
+      toComparableNumber(left.vendorListVersion)
+    );
+  });
+}
+
+function getGvlRawEvidenceBySha256(db, rawGvlSha256) {
+  return new Promise((resolve, reject) => {
+    const tx = db.transaction([VENDORGET_STORE_NAMES.gvlRawEvidence], "readonly");
+    const rawEvidenceStore = tx.objectStore(VENDORGET_STORE_NAMES.gvlRawEvidence);
+    const getRequest = rawEvidenceStore.get(rawGvlSha256);
+    let rawEvidenceOrNull = null;
+
+    getRequest.onerror = () => reject(getRequest.error);
+    getRequest.onsuccess = () => {
+      rawEvidenceOrNull = getRequest.result ?? null;
+    };
+
+    tx.onerror = () => reject(tx.error);
+    tx.onabort = () => reject(tx.error);
+    tx.oncomplete = () => resolve(rawEvidenceOrNull);
+  });
+}
+
+function buildGvlVendorDetailSnapshotSummary(snapshot, snapshotSha256) {
+  if (!snapshot) {
+    return {
+      snapshotSha256: snapshotSha256 ?? null,
+      rawGvlSha256: null,
+      vendorListVersion: null,
+      tcfPolicyVersion: null,
+      fetchedAt: null,
+      createdAt: null
+    };
+  }
+
+  return {
+    snapshotSha256: snapshot.sha256 ?? snapshotSha256 ?? null,
+    rawGvlSha256: snapshot.rawGvlSha256 ?? null,
+    vendorListVersion: snapshot.vendorListVersion ?? null,
+    tcfPolicyVersion: snapshot.tcfPolicyVersion ?? null,
+    fetchedAt: snapshot.fetchedAt ?? null,
+    createdAt: snapshot.createdAt ?? null
+  };
+}
+
+function buildGvlVendorDetailRawEvidenceSummary(rawEvidence, rawGvlSha256) {
+  if (!rawEvidence) {
+    return {
+      rawGvlSha256: rawGvlSha256 ?? null,
+      sourceUrl: null,
+      fetchedAt: null,
+      httpStatus: null,
+      contentType: null,
+      hasRawBody: false
+    };
+  }
+
+  return {
+    rawGvlSha256: rawEvidence.rawGvlSha256 ?? rawGvlSha256 ?? null,
+    sourceUrl: rawEvidence.sourceUrl ?? null,
+    fetchedAt: rawEvidence.fetchedAt ?? null,
+    httpStatus: rawEvidence.httpStatus ?? null,
+    contentType: rawEvidence.contentType ?? null,
+    hasRawBody: typeof rawEvidence.rawBody === "string"
+  };
+}
+
 async function handleGetLatestConsentStateMessage() {
   const db = await openVendorGetDb();
   const latestStateOrNull = await getLatestConsentState(db);
@@ -631,21 +907,55 @@ async function handleFetchOfficialGvlMessage() {
     }
 
     const db = await openVendorGetDb();
-    const result = await VendorGetGvlService.ingestGvlSnapshot(db, rawJson, {
-      sourceUrl: OFFICIAL_IAB_GVL_URL,
-      fetchedAt: new Date().toISOString(),
-      rawGvlSha256: rawGvlSha256,
-      diagnostics: {
-        ingestionSource: "official_iab_fetch",
-        responseStatus: responseStatus
-      }
-    });
+    const currentVendorListVersion = rawJson.vendorListVersion ?? null;
+    const existingSnapshot = await getGvlSnapshotByVendorListVersion(
+      db,
+      currentVendorListVersion
+    );
+    const ingestResult = existingSnapshot
+      ? null
+      : await VendorGetGvlService.ingestGvlSnapshot(db, rawJson, {
+          sourceUrl: OFFICIAL_IAB_GVL_URL,
+          fetchedAt: new Date().toISOString(),
+          rawGvlSha256: rawGvlSha256,
+          diagnostics: {
+            ingestionSource: "official_iab_fetch",
+            responseStatus: responseStatus
+          }
+        });
+    const snapshot = existingSnapshot ?? ingestResult.snapshot;
+    const completeness = await getGvlSnapshotNormalizedCompleteness(
+      db,
+      snapshot
+    );
+    let normalizationSummary = null;
+    let syncStatus = "current_and_locally_available";
+
+    if (!existingSnapshot) {
+      normalizationSummary = await normalizeGvlSnapshotWithExistingPipeline(
+        snapshot
+      );
+      syncStatus = "new_gvl_revision_stored_and_normalized";
+    } else if (!completeness.complete) {
+      normalizationSummary = await normalizeGvlSnapshotWithExistingPipeline(
+        existingSnapshot
+      );
+      syncStatus = "known_gvl_rebuilt_from_local_evidence";
+    }
+
+    const counts = await countGvlNormalizedRecordsForVersion(
+      db,
+      snapshot.vendorListVersion ?? null
+    );
 
     return {
       success: true,
-      alreadyKnown: result.alreadyKnown,
-      vendorListVersion: result.snapshot.vendorListVersion,
-      sha256: result.snapshot.sha256
+      alreadyKnown: Boolean(existingSnapshot),
+      syncStatus,
+      vendorListVersion: snapshot.vendorListVersion,
+      sha256: snapshot.sha256,
+      normalizationSummary,
+      counts
     };
   } catch (error) {
     console.warn("VG-Observe official GVL fetch failed", error);
@@ -657,6 +967,24 @@ async function handleFetchOfficialGvlMessage() {
   }
 }
 
+async function getGvlSnapshotNormalizedCompleteness(db, snapshot) {
+  const counts = await countGvlNormalizedRecordsForVersion(
+    db,
+    snapshot?.vendorListVersion ?? null
+  );
+  const expectedVendorCount = Number(snapshot?.vendorCount ?? 0);
+  const hasVendors =
+    expectedVendorCount > 0
+      ? counts.vendorCount >= expectedVendorCount
+      : counts.vendorCount > 0;
+  const hasVendorRelationships = counts.vendorRelationshipCount > 0;
+
+  return {
+    complete: hasVendors && hasVendorRelationships,
+    counts
+  };
+}
+
 async function fetchOfficialGvlJson() {
   const response = await fetch(OFFICIAL_IAB_GVL_URL, {
     method: "GET",