Initialize VG-Environment from stable VG-IV baseline

2026-05-21 19:58:08 +02:00
Commit a1a8147ae2
@@ -0,0 +1,569 @@
+console.log("VendorGet-IV background loaded");
+
+const OFFICIAL_IAB_GVL_URL =
+  "https://vendor-list.consensu.org/v3/vendor-list.json";
+const EVIDENCE_RECORDING_SOURCE = "vendorget_background_mirror";
+
+browser.runtime.onMessage.addListener((message, sender) =>
+  handleVendorGetMessage(message, sender)
+);
+
+browser.webRequest.onBeforeRequest.addListener(
+  handleObservedRequest,
+  { urls: ["<all_urls>"] }
+);
+
+async function handleVendorGetMessage(message, sender) {
+  if (!message) {
+    return null;
+  }
+
+  if (message.type === "gvl_import_json") {
+    return handleGvlImportJsonMessage(message);
+  }
+
+  if (message.type === "fetch_official_gvl") {
+    return handleFetchOfficialGvlMessage();
+  }
+
+  if (message.type === "start_evidence_maintenance_session") {
+    return startEvidenceMaintenanceSession(message?.payload?.source);
+  }
+
+  if (message.type === "refresh_evidence_maintenance_session") {
+    return refreshEvidenceMaintenanceSession(message?.payload?.source);
+  }
+
+  if (message.type === "end_evidence_maintenance_session") {
+    return endEvidenceMaintenanceSession(message?.payload?.source);
+  }
+
+  if (message.type === "get_evidence_maintenance_status") {
+    return getEvidenceMaintenanceStatus();
+  }
+
+  if (message.type === "get_evidence_retention_status") {
+    return handleGetEvidenceRetentionStatusMessage();
+  }
+
+  if (message.type === "purge_unlocked_evidence_records") {
+    return handlePurgeUnlockedEvidenceRecordsMessage();
+  }
+
+  if (message.type === "delete_all_evidence_database") {
+    return handleDeleteAllEvidenceDatabaseMessage();
+  }
+
+  if (message.type === "lock_all_evidence_records") {
+    return lockAllEvidenceRecords(
+      message?.payload?.reason ?? "dsar_used",
+      message?.payload?.note ?? null
+    );
+  }
+
+  if (message.type === "unlock_all_evidence_records") {
+    return unlockAllEvidenceRecords();
+  }
+
+  if (message.type !== "vendorget_capture") {
+    return;
+  }
+
+  if (!(await isConsentCaptureEnabled())) {
+    return;
+  }
+
+  const eventName = message?.payload?.eventName ?? null;
+  const tabId = sender?.tab?.id ?? null;
+
+  if (eventName === "tcf_ping") {
+    const pingData = message?.payload?.payload?.data ?? null;
+
+    if (tabId !== null && pingData) {
+      rememberLatestTcfPing(tabId, pingData);
+    }
+
+    console.log("VendorGet-IV tcf ping", {
+      payload: message.payload.payload,
+      sender
+    });
+
+    return;
+  }
+
+  if (eventName !== "consent_capture") {
+    console.log("VendorGet-IV ignored event", message);
+    return;
+  }
+
+  if (isEvidenceWriteSuspended()) {
+    console.info("VendorGet-IV evidence write skipped: maintenance mode");
+    return;
+  }
+
+  const latestPingData = tabId !== null ? getLatestTcfPing(tabId) : null;
+
+  const consentState = buildConsentStateV1(
+    message.payload.payload,
+    sender,
+    latestPingData
+  );
+
+  consentState.stateFingerprint = await sha256Hex(
+    stableStringify(consentState.fingerprintSource)
+  );
+
+  rememberLatestConsentState(consentState);
+
+  const result = await persistConsentState(
+    consentState,
+    message.payload.payload?.rawTcData ?? null
+  );
+
+  console.log("VendorGet-IV consent state persisted", result);
+}
+
+async function handleGetEvidenceRetentionStatusMessage() {
+  const db = await openVendorGetDb();
+  const totalCount = await countEvidenceRecords(db);
+  const lockedCount = await countLockedEvidenceRecords(db);
+  const storeCounts = await getEvidenceStoreCounts(db);
+
+  return {
+    success: true,
+    totalCount,
+    lockedCount,
+    unlockedCount: totalCount - lockedCount,
+    storeCounts
+  };
+}
+
+async function handlePurgeUnlockedEvidenceRecordsMessage() {
+  const db = await openVendorGetDb();
+
+  return purgeUnlockedEvidenceRecords(db);
+}
+
+function handleDeleteAllEvidenceDatabaseMessage() {
+  return deleteVendorGetDatabase();
+}
+
+async function handleGvlImportJsonMessage(message) {
+  const rawJson = message?.payload?.rawJson ?? null;
+
+  if (!isGvlImportCandidate(rawJson)) {
+    return {
+      success: false,
+      error: "invalid_gvl_json"
+    };
+  }
+
+  const db = await openVendorGetDb();
+  const result = await VendorGetGvlService.ingestGvlSnapshot(db, rawJson, {
+    sourceUrl: message?.payload?.sourceUrl ?? null,
+    diagnostics: {
+      importSource: "local_file"
+    }
+  });
+
+  return {
+    success: true,
+    alreadyKnown: result.alreadyKnown,
+    vendorListVersion: result.snapshot.vendorListVersion,
+    sha256: result.snapshot.sha256
+  };
+}
+
+function isGvlImportCandidate(value) {
+  return (
+    value &&
+    typeof value === "object" &&
+    !Array.isArray(value) &&
+    value.vendorListVersion !== undefined &&
+    value.vendors &&
+    typeof value.vendors === "object" &&
+    !Array.isArray(value.vendors)
+  );
+}
+
+async function handleFetchOfficialGvlMessage() {
+  try {
+    const response = await fetch(OFFICIAL_IAB_GVL_URL, {
+      method: "GET",
+      cache: "no-store"
+    });
+
+    if (!response.ok) {
+      return {
+        success: false,
+        error: "official_gvl_fetch_failed",
+        responseStatus: response.status
+      };
+    }
+
+    const rawJson = await response.json();
+
+    if (!isGvlImportCandidate(rawJson)) {
+      return {
+        success: false,
+        error: "invalid_gvl_json",
+        responseStatus: response.status
+      };
+    }
+
+    const db = await openVendorGetDb();
+    const result = await VendorGetGvlService.ingestGvlSnapshot(db, rawJson, {
+      sourceUrl: OFFICIAL_IAB_GVL_URL,
+      fetchedAt: new Date().toISOString(),
+      diagnostics: {
+        ingestionSource: "official_iab_fetch",
+        responseStatus: response.status
+      }
+    });
+
+    return {
+      success: true,
+      alreadyKnown: result.alreadyKnown,
+      vendorListVersion: result.snapshot.vendorListVersion,
+      sha256: result.snapshot.sha256
+    };
+  } catch (error) {
+    console.warn("VendorGet-IV official GVL fetch failed", error);
+
+    return {
+      success: false,
+      error: "official_gvl_fetch_failed"
+    };
+  }
+}
+
+function buildConsentStateV1(rawCapture, sender, latestPingData) {
+  const decodedTcString = decodeTcStringCoreMetadata(rawCapture?.tcString ?? null);
+
+  // Firefox/CMP/browser storage remains the consent source of truth; VG-IV
+  // stores an evidentiary mirror of the observed TCF state.
+  const state = {
+    schemaVersion: 1,
+    capturedAt: new Date().toISOString(),
+
+    page: {
+      url: rawCapture?.url ?? sender?.tab?.url ?? sender?.url ?? null,
+      origin: rawCapture?.origin ?? sender?.origin ?? null,
+      tabId: sender?.tab?.id ?? null,
+      frameId: sender?.frameId ?? null,
+      incognito: sender?.tab?.incognito ?? null,
+      cookieStoreId: sender?.tab?.cookieStoreId ?? null
+    },
+
+    cmp: {
+      cmpId:
+        rawCapture?.cmpId ??
+        decodedTcString?.cmpId ??
+        latestPingData?.cmpId ??
+        null,
+
+      cmpVersion:
+        rawCapture?.cmpVersion ??
+        decodedTcString?.cmpVersion ??
+        latestPingData?.cmpVersion ??
+        null,
+
+      tcfPolicyVersion:
+        rawCapture?.tcfPolicyVersion ??
+        decodedTcString?.tcfPolicyVersion ??
+        latestPingData?.tcfPolicyVersion ??
+        null,
+
+      gdprApplies: rawCapture?.gdprApplies ?? latestPingData?.gdprApplies ?? null,
+
+      isServiceSpecific:
+        rawCapture?.isServiceSpecific ??
+        decodedTcString?.isServiceSpecific ??
+        null,
+
+      useNonStandardTexts: rawCapture?.useNonStandardTexts ?? null,
+
+      publisherCC:
+        rawCapture?.publisherCC ??
+        decodedTcString?.publisherCC ??
+        null,
+
+      purposeOneTreatment:
+        rawCapture?.purposeOneTreatment ??
+        decodedTcString?.purposeOneTreatment ??
+        null
+    },
+
+    observation: {
+      eventStatus: rawCapture?.eventStatus ?? null,
+      cmpStatus: rawCapture?.cmpStatus ?? latestPingData?.cmpStatus ?? null
+    },
+
+    gvl: {
+      vendorListVersion:
+        rawCapture?.vendorListVersion ??
+        rawCapture?.gvlVersion ??
+        latestPingData?.gvlVersion ??
+        latestPingData?.vendorListVersion ??
+        decodedTcString?.vendorListVersion ??
+        null
+    },
+
+    consent: {
+      tcString: rawCapture?.tcString ?? null,
+      addtlConsent: rawCapture?.addtlConsent ?? null
+    },
+
+    purposes: {
+      consents: rawCapture?.purpose?.consents ?? {},
+      legitimateInterests: rawCapture?.purpose?.legitimateInterests ?? {}
+    },
+
+    vendors: {
+      consents: rawCapture?.vendor?.consents ?? {},
+      legitimateInterests: rawCapture?.vendor?.legitimateInterests ?? {},
+      disclosedVendors:
+        rawCapture?.vendor?.disclosedVendors ??
+        rawCapture?.disclosedVendors ??
+        {}
+    },
+
+    specialFeatureOptins: rawCapture?.specialFeatureOptins ?? {},
+
+    publisher: {
+      restrictions: rawCapture?.publisher?.restrictions ?? {},
+      consents: rawCapture?.publisher?.consents ?? {},
+      legitimateInterests: rawCapture?.publisher?.legitimateInterests ?? {}
+    },
+
+    diagnostics: {
+      bridgeTimestampUtc: rawCapture?.timestampUtc ?? null,
+      rawTopLevelKeys: Object.keys(rawCapture ?? {}),
+      decodedTcStringCore: decodedTcString,
+      latestPingData: latestPingData
+    },
+
+    fingerprintSource: null,
+    stateFingerprint: null
+  };
+
+  state.fingerprintSource = buildFingerprintSource(state);
+
+  return state;
+}
+
+function buildFingerprintSource(consentState) {
+  return {
+    cmp: consentState.cmp,
+    gvl: consentState.gvl,
+    consent: consentState.consent,
+    purposes: consentState.purposes,
+    vendors: consentState.vendors,
+    specialFeatureOptins: consentState.specialFeatureOptins,
+    publisher: consentState.publisher
+  };
+}
+
+async function persistConsentState(consentState, rawTcData) {
+  const db = await openVendorGetDb();
+  const now = new Date().toISOString();
+
+  return new Promise((resolve, reject) => {
+    const tx = db.transaction(["consent_states", "consent_events"], "readwrite");
+
+    const statesStore = tx.objectStore("consent_states");
+    const eventsStore = tx.objectStore("consent_events");
+
+    const getRequest = statesStore.get(consentState.stateFingerprint);
+
+    getRequest.onerror = () => reject(getRequest.error);
+
+    getRequest.onsuccess = () => {
+      const existingState = getRequest.result;
+
+      if (existingState) {
+        existingState.recordedAt =
+          existingState.recordedAt ?? existingState.createdAt ?? now;
+        existingState.recordingSource =
+          existingState.recordingSource ?? EVIDENCE_RECORDING_SOURCE;
+        existingState.lastSeenAt = now;
+        existingState.seenCount = (existingState.seenCount ?? 1) + 1;
+        existingState.updatedAt = now;
+
+        statesStore.put(existingState);
+
+        eventsStore.add({
+          eventType: "duplicate_state",
+          capturedAt: consentState.capturedAt,
+          recordedAt: now,
+          recordingSource: EVIDENCE_RECORDING_SOURCE,
+          stateFingerprint: consentState.stateFingerprint,
+          page: consentState.page,
+          rawEventName: "consent_capture",
+          rawTcData: rawTcData,
+          diagnostics: consentState.diagnostics
+        });
+
+        resolve({
+          action: "duplicate_state_updated",
+          stateFingerprint: consentState.stateFingerprint,
+          seenCount: existingState.seenCount
+        });
+
+        return;
+      }
+
+      const newStateRecord = {
+        ...consentState,
+        recordedAt: now,
+        recordingSource: EVIDENCE_RECORDING_SOURCE,
+        firstSeenAt: now,
+        lastSeenAt: now,
+        seenCount: 1,
+        createdAt: now,
+        updatedAt: now
+      };
+
+      statesStore.add(newStateRecord);
+
+      eventsStore.add({
+        eventType: "new_state",
+        capturedAt: consentState.capturedAt,
+        recordedAt: now,
+        recordingSource: EVIDENCE_RECORDING_SOURCE,
+        stateFingerprint: consentState.stateFingerprint,
+        page: consentState.page,
+        rawEventName: "consent_capture",
+        rawTcData: rawTcData,
+        diagnostics: consentState.diagnostics
+      });
+
+      resolve({
+        action: "new_state_inserted",
+        stateFingerprint: consentState.stateFingerprint,
+        seenCount: 1
+      });
+    };
+
+    tx.onerror = () => reject(tx.error);
+  });
+}
+
+async function persistObservedRequest(observedRequest) {
+  const db = await openVendorGetDb();
+
+  return new Promise((resolve, reject) => {
+    const tx = db.transaction(["observed_requests"], "readwrite");
+    const requestsStore = tx.objectStore("observed_requests");
+    const getRequest = requestsStore.get(observedRequest.requestFingerprint);
+
+    getRequest.onerror = () => reject(getRequest.error);
+
+    getRequest.onsuccess = () => {
+      const existingRequest = getRequest.result;
+
+      if (existingRequest) {
+        const updatedRequest = {
+          ...existingRequest,
+          requestFingerprintSource: observedRequest.requestFingerprintSource,
+          recordedAt: existingRequest.recordedAt ?? observedRequest.recordedAt,
+          recordingSource:
+            existingRequest.recordingSource ?? observedRequest.recordingSource,
+          lastSeenAt: observedRequest.lastSeenAt,
+          seenCount: (existingRequest.seenCount ?? 1) + 1,
+          request: observedRequest.request,
+          consentParams: observedRequest.consentParams,
+          context: observedRequest.context,
+          correlation: observedRequest.correlation
+        };
+
+        requestsStore.put(updatedRequest);
+
+        resolve({
+          action: "observed_request_updated",
+          requestFingerprint: observedRequest.requestFingerprint,
+          seenCount: updatedRequest.seenCount
+        });
+
+        return;
+      }
+
+      requestsStore.add(observedRequest);
+
+      resolve({
+        action: "observed_request_inserted",
+        requestFingerprint: observedRequest.requestFingerprint,
+        seenCount: observedRequest.seenCount
+      });
+    };
+
+    tx.onerror = () => reject(tx.error);
+  });
+}
+
+function decodeTcStringCoreMetadata(tcString) {
+  if (!tcString || typeof tcString !== "string") {
+    return null;
+  }
+
+  const coreSegment = tcString.split(".")[0];
+
+  try {
+    const bits = base64UrlToBits(coreSegment);
+
+    return {
+      version: bitsToInt(bits, 0, 6),
+      cmpId: bitsToInt(bits, 78, 12),
+      cmpVersion: bitsToInt(bits, 90, 12),
+      consentScreen: bitsToInt(bits, 102, 6),
+      consentLanguage: bitsToString(bits, 108, 12),
+      vendorListVersion: bitsToInt(bits, 120, 12),
+      tcfPolicyVersion: bitsToInt(bits, 132, 6),
+      isServiceSpecific: bitsToBoolean(bits, 138),
+      useNonStandardStacks: bitsToBoolean(bits, 139),
+      purposeOneTreatment: bitsToBoolean(bits, 200),
+      publisherCC: bitsToString(bits, 201, 12)
+    };
+  } catch (error) {
+    console.warn("VendorGet-IV could not decode TC string core metadata", error);
+    return null;
+  }
+}
+
+function base64UrlToBits(value) {
+  const base64 = value
+    .replace(/-/g, "+")
+    .replace(/_/g, "/")
+    .padEnd(Math.ceil(value.length / 4) * 4, "=");
+
+  const binary = atob(base64);
+  let bits = "";
+
+  for (let index = 0; index < binary.length; index += 1) {
+    bits += binary
+      .charCodeAt(index)
+      .toString(2)
+      .padStart(8, "0");
+  }
+
+  return bits;
+}
+
+function bitsToInt(bits, start, length) {
+  return parseInt(bits.slice(start, start + length), 2);
+}
+
+function bitsToBoolean(bits, index) {
+  return bits[index] === "1";
+}
+
+function bitsToString(bits, start, length) {
+  let result = "";
+
+  for (let index = start; index < start + length; index += 6) {
+    const charCode = bitsToInt(bits, index, 6) + 65;
+    result += String.fromCharCode(charCode);
+  }
+
+  return result;
+}