diff --git a/src/background.js b/src/background.js index f5c3bb3..6c1fd0e 100644 --- a/src/background.js +++ b/src/background.js @@ -41,6 +41,14 @@ async function handleVendorGetMessage(message, sender) { return handleVerifyGvlRevisionEvidenceJsonMessage(message); } + if (message.type === "import_gvl_revision_evidence_json") { + return handleImportGvlRevisionEvidenceJsonMessage(message); + } + + if (message.type === "mark_gvl_revision_evidence_vault_copy") { + return handleMarkGvlRevisionEvidenceVaultCopyMessage(message); + } + if (message.type === "import_gvl_evidence_json") { return handleImportGvlEvidenceJsonMessage(message); } @@ -265,6 +273,42 @@ async function handleImportGvlEvidenceJsonMessage(message) { } } +async function handleImportGvlRevisionEvidenceJsonMessage(message) { + try { + const importResult = await importVendorGetGvlRevisionEvidenceJson( + message?.payload?.export ?? null + ); + + return { + success: importResult.imported, + import: importResult, + verification: importResult.verification, + error: importResult.imported ? null : "invalid_gvl_revision_evidence" + }; + } catch (error) { + return { + success: false, + error: error?.message ?? String(error) + }; + } +} + +async function handleMarkGvlRevisionEvidenceVaultCopyMessage(message) { + try { + return { + success: true, + mark: await markVendorGetGvlRevisionEvidenceVaultCopy( + message?.payload?.snapshotSha256 ?? null + ) + }; + } catch (error) { + return { + success: false, + error: error?.message ?? String(error) + }; + } +} + async function handleGetLatestGvlUpdateStatusMessage() { const db = await openVendorGetDb(); const latestSnapshot = await getLatestGvlSnapshotByVendorListVersion(db); @@ -311,12 +355,17 @@ async function handleListGvlSnapshotsMessage() { const snapshotsWithEvents = await Promise.all( snapshots.map(async (snapshot) => { const event = await getAnyGvlSnapshotEventBySha256(db, snapshot.sha256); + const provenanceState = getGvlEvidenceProvenanceState(snapshot); return { vendorListVersion: snapshot.vendorListVersion ?? null, sha256: snapshot.sha256 ?? null, fetchedAt: snapshot.fetchedAt ?? null, sourceUrl: snapshot.sourceUrl ?? null, + provenance: provenanceState.provenance, + vaultCopyAvailable: provenanceState.vaultCopyAvailable, + workspaceDeleteAllowed: provenanceState.workspaceDeleteAllowed, + workspaceDeleteProtected: provenanceState.workspaceDeleteProtected, eventType: event?.eventType ?? null, eventCapturedAt: event?.capturedAt ?? null }; @@ -350,6 +399,7 @@ async function handleGetGvlSnapshotSummaryMessage(message) { db, vendorListVersion ); + const provenanceState = getGvlEvidenceProvenanceState(snapshot); return { success: true, @@ -360,6 +410,10 @@ async function handleGetGvlSnapshotSummaryMessage(message) { sourceUrl: snapshot.sourceUrl ?? null, eventType: event?.eventType ?? null, eventCapturedAt: event?.capturedAt ?? null, + provenance: provenanceState.provenance, + vaultCopyAvailable: provenanceState.vaultCopyAvailable, + workspaceDeleteAllowed: provenanceState.workspaceDeleteAllowed, + workspaceDeleteProtected: provenanceState.workspaceDeleteProtected, vendorCount: snapshot.vendorCount ?? counts.vendorCount, snapshotVendorCount: snapshot.vendorCount ?? null, normalizedVendorCount: counts.vendorCount, @@ -1215,7 +1269,14 @@ function isGvlImportCandidate(value) { async function handleFetchOfficialGvlMessage() { try { - const { rawJson, rawGvlSha256, responseStatus } = + const { + rawBody, + rawJson, + rawGvlSha256, + fetchedAt, + contentType, + responseStatus + } = await fetchOfficialGvlJson(); if (!isGvlImportCandidate(rawJson)) { @@ -1232,16 +1293,50 @@ async function handleFetchOfficialGvlMessage() { db, currentVendorListVersion ); + const currentSnapshotSha256 = + await VendorGetGvlService.calculateGvlSnapshotSha256(rawJson); + let webProvenanceMark = null; + + if (existingSnapshot?.sha256) { + if ( + existingSnapshot.sha256 !== currentSnapshotSha256 || + existingSnapshot.rawGvlSha256 !== rawGvlSha256 + ) { + return { + success: false, + error: "gvl_revision_evidence_conflict", + vendorListVersion: currentVendorListVersion, + existingSnapshotSha256: existingSnapshot.sha256 ?? null, + fetchedSnapshotSha256: currentSnapshotSha256, + existingRawGvlSha256: existingSnapshot.rawGvlSha256 ?? null, + fetchedRawGvlSha256: rawGvlSha256 + }; + } + + await VendorGetGvlService.storeGvlRawEvidenceIfNew(db, { + rawGvlSha256, + sourceUrl: OFFICIAL_IAB_GVL_URL, + fetchedAt, + httpStatus: responseStatus, + contentType, + rawBody + }); + webProvenanceMark = await markGvlRevisionEvidenceWebSource( + db, + existingSnapshot.sha256 + ); + } + const ingestResult = existingSnapshot ? null - : await VendorGetGvlService.ingestGvlSnapshot(db, rawJson, { - sourceUrl: OFFICIAL_IAB_GVL_URL, - fetchedAt: new Date().toISOString(), - rawGvlSha256: rawGvlSha256, - diagnostics: { - ingestionSource: "official_iab_fetch", - responseStatus: responseStatus - } + : await ingestOfficialGvlSnapshotFromFetchedEvidence(db, { + rawBody, + rawJson, + rawGvlSha256, + fetchedAt, + contentType, + responseStatus, + ingestionSource: "official_iab_fetch" }); const snapshot = existingSnapshot ?? ingestResult.snapshot; const completeness = await getGvlSnapshotNormalizedCompleteness( @@ -1274,6 +1369,7 @@ async function handleFetchOfficialGvlMessage() { syncStatus, vendorListVersion: snapshot.vendorListVersion, sha256: snapshot.sha256, + webProvenanceMark, normalizationSummary, counts }; @@ -1326,22 +1422,49 @@ async function fetchOfficialGvlJson() { const fetchedAt = new Date().toISOString(); const contentType = response.headers.get("Content-Type"); const rawGvlSha256 = await VendorGetGvlService.calculateRawGvlSha256(rawBody); - const db = await openVendorGetDb(); + return { + rawBody, + rawJson: JSON.parse(rawBody), + rawGvlSha256: rawGvlSha256, + fetchedAt, + contentType, + responseStatus: response.status + }; +} + +async function ingestOfficialGvlSnapshotFromFetchedEvidence( + db, + { + rawBody, + rawJson, + rawGvlSha256, + fetchedAt, + contentType, + responseStatus, + ingestionSource, + diagnostics + } +) { await VendorGetGvlService.storeGvlRawEvidenceIfNew(db, { rawGvlSha256, sourceUrl: OFFICIAL_IAB_GVL_URL, fetchedAt, - httpStatus: response.status, + httpStatus: responseStatus, contentType, rawBody }); - return { - rawJson: JSON.parse(rawBody), + return VendorGetGvlService.ingestGvlSnapshot(db, rawJson, { + sourceUrl: OFFICIAL_IAB_GVL_URL, + fetchedAt, rawGvlSha256: rawGvlSha256, - responseStatus: response.status - }; + diagnostics: { + ingestionSource, + responseStatus: responseStatus, + ...(diagnostics ?? {}) + } + }); } async function runStartupGvlAutoUpdateCheck() { @@ -1424,7 +1547,14 @@ async function runStartupGvlAutoUpdateCheck() { result: "started" }); - const { rawJson, rawGvlSha256, responseStatus } = + const { + rawBody, + rawJson, + rawGvlSha256, + fetchedAt, + contentType, + responseStatus + } = await fetchOfficialGvlJson(); if (!isGvlImportCandidate(rawJson)) { @@ -1439,15 +1569,17 @@ async function runStartupGvlAutoUpdateCheck() { previousVendorListVersion ); - const ingestResult = await VendorGetGvlService.ingestGvlSnapshot( + const ingestResult = await ingestOfficialGvlSnapshotFromFetchedEvidence( db, - rawJson, { - sourceUrl: OFFICIAL_IAB_GVL_URL, - fetchedAt: new Date().toISOString(), - rawGvlSha256: rawGvlSha256, + rawBody, + rawJson, + rawGvlSha256, + fetchedAt, + contentType, + responseStatus, + ingestionSource: "official_iab_auto_update", diagnostics: { - ingestionSource: "official_iab_auto_update", responseStatus: responseStatus, updateCheckSource: GVL_AUTO_UPDATE_SOURCE, checkedAt: lastAutoGvlCheckStartedAt, diff --git a/src/background/db/db-retention.js b/src/background/db/db-retention.js index b93b2b2..6116b42 100644 --- a/src/background/db/db-retention.js +++ b/src/background/db/db-retention.js @@ -29,10 +29,13 @@ function getEvidenceStoreCounts(db) { }); } -function purgeUnlockedEvidenceRecords(db) { +async function purgeUnlockedEvidenceRecords(db) { + const gvlWorkspaceProtection = await buildGvlWorkspaceProtectionIndex(db); + return new Promise((resolve, reject) => { let deletedCount = 0; let keptLockedCount = 0; + let keptGvlWorkspaceProtectedCount = 0; const tx = db.transaction(VENDORGET_EVIDENCE_STORE_NAMES, "readwrite"); tx.onerror = () => reject(tx.error); @@ -41,7 +44,12 @@ function purgeUnlockedEvidenceRecords(db) { resolve({ success: true, deletedCount, - keptLockedCount + keptLockedCount, + keptGvlWorkspaceProtectedCount, + gvlWorkspaceProtectionNotice: + keptGvlWorkspaceProtectedCount > 0 + ? "Diese GVL-Evidence wurde noch nicht in den Vault exportiert." + : null }); }; @@ -61,6 +69,18 @@ function purgeUnlockedEvidenceRecords(db) { return; } + if ( + isGvlWorkspaceProtectedRecord( + storeName, + cursor.value, + gvlWorkspaceProtection + ) + ) { + keptGvlWorkspaceProtectedCount += 1; + cursor.continue(); + return; + } + deletedCount += 1; cursor.delete(); cursor.continue(); @@ -69,6 +89,80 @@ function purgeUnlockedEvidenceRecords(db) { }); } +function buildGvlWorkspaceProtectionIndex(db) { + return new Promise((resolve, reject) => { + const protectedSnapshotSha256 = new Set(); + const protectedRawGvlSha256 = new Set(); + const tx = db.transaction([VENDORGET_STORE_NAMES.gvlSnapshots], "readonly"); + const cursorRequest = tx + .objectStore(VENDORGET_STORE_NAMES.gvlSnapshots) + .openCursor(); + + cursorRequest.onerror = () => reject(cursorRequest.error); + cursorRequest.onsuccess = () => { + const cursor = cursorRequest.result; + + if (!cursor) { + return; + } + + const snapshot = cursor.value; + const provenanceState = getGvlEvidenceProvenanceState(snapshot); + + if (provenanceState.workspaceDeleteProtected) { + if (snapshot.sha256) { + protectedSnapshotSha256.add(snapshot.sha256); + } + + if (snapshot.rawGvlSha256) { + protectedRawGvlSha256.add(snapshot.rawGvlSha256); + } + } + + cursor.continue(); + }; + + tx.onerror = () => reject(tx.error); + tx.onabort = () => reject(tx.error); + tx.oncomplete = () => { + resolve({ + protectedSnapshotSha256, + protectedRawGvlSha256 + }); + }; + }); +} + +function isGvlWorkspaceProtectedRecord(storeName, record, protectionIndex) { + if (!record || typeof record !== "object") { + return false; + } + + if (storeName === VENDORGET_STORE_NAMES.gvlSnapshots) { + return protectionIndex.protectedSnapshotSha256.has(record.sha256); + } + + if (storeName === VENDORGET_STORE_NAMES.gvlRawEvidence) { + return protectionIndex.protectedRawGvlSha256.has(record.rawGvlSha256); + } + + if ( + [ + VENDORGET_STORE_NAMES.gvlVendors, + VENDORGET_STORE_NAMES.gvlPurposes, + VENDORGET_STORE_NAMES.gvlSpecialPurposes, + VENDORGET_STORE_NAMES.gvlFeatures, + VENDORGET_STORE_NAMES.gvlSpecialFeatures, + VENDORGET_STORE_NAMES.gvlDataCategories, + VENDORGET_STORE_NAMES.gvlVendorRelationships + ].includes(storeName) + ) { + return protectionIndex.protectedSnapshotSha256.has(record.snapshotSha256); + } + + return false; +} + function countRecordsInStores(db, storeNames) { return new Promise((resolve, reject) => { let totalCount = 0; diff --git a/src/background/gvl-service.js b/src/background/gvl-service.js index be795ad..d35a0d6 100644 --- a/src/background/gvl-service.js +++ b/src/background/gvl-service.js @@ -23,25 +23,32 @@ function storeGvlRawEvidenceIfNew(db, rawEvidence) { return new Promise((resolve, reject) => { const tx = db.transaction(["gvl_raw_evidence"], "readwrite"); const rawEvidenceStore = tx.objectStore("gvl_raw_evidence"); - const getRequest = rawEvidenceStore.get(rawEvidence.rawGvlSha256); + const evidenceRecord = annotateGvlEvidenceRecordProvenance( + rawEvidence, + "web" + ); + const getRequest = rawEvidenceStore.get(evidenceRecord.rawGvlSha256); let result = null; getRequest.onerror = () => reject(getRequest.error); getRequest.onsuccess = () => { if (getRequest.result) { + rawEvidenceStore.put( + annotateGvlEvidenceRecordProvenance(getRequest.result, "web") + ); result = { stored: false, - rawGvlSha256: rawEvidence.rawGvlSha256 + rawGvlSha256: evidenceRecord.rawGvlSha256 }; return; } - rawEvidenceStore.add(rawEvidence); + rawEvidenceStore.add(evidenceRecord); result = { stored: true, - rawGvlSha256: rawEvidence.rawGvlSha256 + rawGvlSha256: evidenceRecord.rawGvlSha256 }; }; @@ -58,7 +65,7 @@ async function buildGvlSnapshotRecord( ) { const gvlJson = normalizeGvlSnapshotValueForMetadata(rawJson); - return { + return annotateGvlEvidenceRecordProvenance({ sha256: await calculateGvlSnapshotSha256(rawJson), rawGvlSha256: rawGvlSha256 ?? null, vendorListVersion: gvlJson?.vendorListVersion ?? null, @@ -72,7 +79,7 @@ async function buildGvlSnapshotRecord( // Existing GVL snapshots already use createdAt as the local mirror timestamp; // keep that field instead of duplicating it as recordedAt. createdAt: new Date().toISOString() - }; + }, "web"); } function storeGvlSnapshotIfNew(db, snapshot) { @@ -86,6 +93,9 @@ function storeGvlSnapshotIfNew(db, snapshot) { getRequest.onsuccess = () => { if (getRequest.result) { + snapshotsStore.put( + annotateGvlEvidenceRecordProvenance(getRequest.result, "web") + ); result = { stored: false, sha256: snapshot.sha256, diff --git a/src/core/gvl-evidence-json.js b/src/core/gvl-evidence-json.js index 7fef3a2..8a5ea83 100644 --- a/src/core/gvl-evidence-json.js +++ b/src/core/gvl-evidence-json.js @@ -7,6 +7,8 @@ const VENDORGET_GVL_REVISION_EVIDENCE_EXPORT_FORMAT = "vendorget-gvl-revision-evidence"; const VENDORGET_GVL_REVISION_EVIDENCE_EXPORT_FORMAT_VERSION = 1; const VENDORGET_GVL_REVISION_EVIDENCE_CONTENT_KIND = "iab-gvl-revision"; +const VENDORGET_GVL_PROVENANCE_WEB = "web"; +const VENDORGET_GVL_PROVENANCE_VAULT = "vault"; const VENDORGET_GVL_EVIDENCE_STORE_NAMES = [ VENDORGET_STORE_NAMES.gvlRawEvidence, @@ -159,12 +161,23 @@ async function verifyVendorGetGvlRevisionEvidenceJson(exportContainer) { errors.push("raw_body_sha256_mismatch"); } + if ( + exportContainer?.rawEvidence && + exportContainer.rawEvidence.rawGvlSha256 !== rawGvlSha256 + ) { + errors.push("raw_evidence_sha256_mismatch"); + } + const snapshotRecordSha256 = snapshot?.sha256 ?? snapshot?.snapshotSha256 ?? null; if (!snapshot || typeof snapshot !== "object" || Array.isArray(snapshot)) { errors.push("missing_snapshot"); } else if (snapshotRecordSha256 !== snapshotSha256) { errors.push("snapshot_sha256_mismatch"); + } else if (snapshot.rawGvlSha256 !== rawGvlSha256) { + errors.push("snapshot_raw_gvl_sha256_mismatch"); + } else if (snapshot.vendorListVersion !== metadata.vendorListVersion) { + errors.push("snapshot_vendor_list_version_mismatch"); } else if ( (await VendorGetGvlService.calculateGvlSnapshotSha256(snapshot.rawJson)) !== snapshotSha256 @@ -178,14 +191,54 @@ async function verifyVendorGetGvlRevisionEvidenceJson(exportContainer) { continue; } + const seenKeys = new Set(); + for (const record of normalized[storeName]) { if (record?.snapshotSha256 !== snapshotSha256) { errors.push(`normalized_record_snapshot_mismatch_${storeName}`); break; } + + if (record?.vendorListVersion !== metadata.vendorListVersion) { + errors.push(`normalized_record_vendor_list_version_mismatch_${storeName}`); + break; + } + + const recordKey = getGvlEvidenceRecordKeyByStoreName(storeName, record); + const recordKeySignature = JSON.stringify(recordKey); + + if (recordKey === null) { + errors.push(`normalized_record_missing_key_${storeName}`); + break; + } + + if (seenKeys.has(recordKeySignature)) { + errors.push(`normalized_record_duplicate_key_${storeName}`); + break; + } + + seenKeys.add(recordKeySignature); } } + if ( + getGvlEvidenceRecordKeyByStoreName( + VENDORGET_STORE_NAMES.gvlRawEvidence, + exportContainer?.rawEvidence + ) === null + ) { + errors.push("raw_evidence_missing_key"); + } + + if ( + getGvlEvidenceRecordKeyByStoreName( + VENDORGET_STORE_NAMES.gvlSnapshots, + snapshot + ) === null + ) { + errors.push("snapshot_missing_key"); + } + if (!metadata.exportPayloadSha256) { errors.push("missing_export_payload_sha256"); } else { @@ -208,6 +261,42 @@ async function verifyVendorGetGvlRevisionEvidenceJson(exportContainer) { }; } +async function importVendorGetGvlRevisionEvidenceJson(exportContainer) { + const verification = + await verifyVendorGetGvlRevisionEvidenceJson(exportContainer); + + if (!verification.valid) { + return { + imported: false, + verification, + counts: buildEmptyGvlRevisionEvidenceImportCounts() + }; + } + + const db = await openVendorGetDb(); + const counts = await importGvlRevisionEvidenceStores(db, exportContainer); + + return { + imported: true, + importedAt: new Date().toISOString(), + vendorListVersion: verification.vendorListVersion, + snapshotSha256: verification.snapshotSha256, + rawGvlSha256: verification.rawGvlSha256, + verification, + counts + }; +} + +async function markVendorGetGvlRevisionEvidenceVaultCopy(snapshotSha256) { + if (!snapshotSha256) { + throw new Error("missing_snapshot_sha256"); + } + + const db = await openVendorGetDb(); + + return markGvlRevisionEvidenceVaultCopyAvailable(db, snapshotSha256); +} + function getGvlEvidenceRecordByKey(db, storeName, key) { return new Promise((resolve, reject) => { const tx = db.transaction([storeName], "readonly"); @@ -284,6 +373,341 @@ function countGvlRevisionNormalizedRecords(normalized) { ); } +function importGvlRevisionEvidenceStores(db, exportContainer) { + return new Promise((resolve, reject) => { + const counts = buildEmptyGvlRevisionEvidenceImportCounts(); + const recordsByStoreName = + buildGvlRevisionEvidenceImportRecordsByStoreName(exportContainer); + const tx = db.transaction(VENDORGET_GVL_EVIDENCE_STORE_NAMES, "readwrite"); + + tx.onerror = () => reject(tx.error); + tx.onabort = () => reject(tx.error); + tx.oncomplete = () => resolve(counts); + + for (const storeName of VENDORGET_GVL_EVIDENCE_STORE_NAMES) { + importGvlRevisionEvidenceStoreRecords( + tx.objectStore(storeName), + recordsByStoreName[storeName] ?? [], + counts[storeName] + ); + } + }); +} + +function buildGvlRevisionEvidenceImportRecordsByStoreName(exportContainer) { + const rawEvidence = setGvlEvidenceRecordLocalProvenance( + exportContainer.rawEvidence, + VENDORGET_GVL_PROVENANCE_VAULT + ); + const snapshot = setGvlEvidenceRecordLocalProvenance( + exportContainer.snapshot, + VENDORGET_GVL_PROVENANCE_VAULT + ); + + return { + [VENDORGET_STORE_NAMES.gvlRawEvidence]: [rawEvidence], + [VENDORGET_STORE_NAMES.gvlSnapshots]: [snapshot], + ...Object.fromEntries( + VENDORGET_GVL_NORMALIZED_STORE_NAMES.map((storeName) => [ + storeName, + exportContainer.normalized?.[storeName] ?? [] + ]) + ) + }; +} + +function buildEmptyGvlRevisionEvidenceImportCounts() { + return Object.fromEntries( + VENDORGET_GVL_EVIDENCE_STORE_NAMES.map((storeName) => [ + storeName, + { + read: 0, + inserted: 0, + skippedExisting: 0, + skippedInvalid: 0 + } + ]) + ); +} + +function importGvlRevisionEvidenceStoreRecords(objectStore, records, counts) { + const seenKeys = new Set(); + + for (const record of records) { + counts.read += 1; + + const key = getGvlEvidenceRecordKey(objectStore, record); + const keySignature = JSON.stringify(key); + + if (key === null || seenKeys.has(keySignature)) { + counts.skippedInvalid += 1; + continue; + } + + seenKeys.add(keySignature); + + const getRequest = objectStore.get(key); + + getRequest.onsuccess = () => { + if (getRequest.result !== undefined) { + mergeExistingGvlRevisionEvidenceProvenance( + objectStore, + getRequest.result, + record + ); + counts.skippedExisting += 1; + return; + } + + const addRequest = objectStore.add(record); + + addRequest.onsuccess = () => { + counts.inserted += 1; + }; + }; + } +} + +function mergeExistingGvlRevisionEvidenceProvenance( + objectStore, + existingRecord, + importRecord +) { + if ( + objectStore.name !== VENDORGET_STORE_NAMES.gvlRawEvidence && + objectStore.name !== VENDORGET_STORE_NAMES.gvlSnapshots + ) { + return; + } + + const importedProvenance = normalizeGvlEvidenceProvenanceValues(importRecord); + let updatedRecord = existingRecord; + + for (const provenance of importedProvenance) { + updatedRecord = annotateGvlEvidenceRecordProvenance( + updatedRecord, + provenance + ); + } + + objectStore.put(updatedRecord); +} + +function mergeGvlEvidenceProvenance(record, provenance) { + const values = new Set(); + + for (const value of normalizeGvlEvidenceProvenanceValues(record)) { + values.add(value); + } + + if (provenance === VENDORGET_GVL_PROVENANCE_WEB) { + values.add(VENDORGET_GVL_PROVENANCE_WEB); + } + + if (provenance === VENDORGET_GVL_PROVENANCE_VAULT) { + values.add(VENDORGET_GVL_PROVENANCE_VAULT); + } + + return Array.from(values).sort(sortGvlEvidenceProvenanceValue); +} + +function annotateGvlEvidenceRecordProvenance(record, provenance) { + if (!record || typeof record !== "object" || Array.isArray(record)) { + return record; + } + + const provenanceValues = mergeGvlEvidenceProvenance(record, provenance); + const vaultCopyAvailable = + provenance === VENDORGET_GVL_PROVENANCE_VAULT || + provenanceValues.includes(VENDORGET_GVL_PROVENANCE_VAULT) || + record.vaultCopyAvailable === true; + + return { + ...record, + gvlEvidenceProvenance: provenanceValues, + vaultCopyAvailable, + evidenceWorkspaceDeleteAllowed: vaultCopyAvailable + }; +} + +function setGvlEvidenceRecordLocalProvenance(record, provenance) { + if (!record || typeof record !== "object" || Array.isArray(record)) { + return record; + } + + const provenanceValues = + provenance === VENDORGET_GVL_PROVENANCE_VAULT + ? [VENDORGET_GVL_PROVENANCE_VAULT] + : [VENDORGET_GVL_PROVENANCE_WEB]; + const vaultCopyAvailable = provenance === VENDORGET_GVL_PROVENANCE_VAULT; + + return { + ...record, + gvlEvidenceProvenance: provenanceValues, + vaultCopyAvailable, + evidenceWorkspaceDeleteAllowed: vaultCopyAvailable + }; +} + +function markGvlEvidenceRecordVaultCopyAvailable(record) { + if (!record || typeof record !== "object" || Array.isArray(record)) { + return record; + } + + return { + ...record, + vaultCopyAvailable: true, + evidenceWorkspaceDeleteAllowed: true + }; +} + +function getGvlEvidenceProvenanceState(record) { + const provenanceValues = normalizeGvlEvidenceProvenanceValues(record); + const containsWeb = provenanceValues.includes(VENDORGET_GVL_PROVENANCE_WEB); + const containsVault = provenanceValues.includes(VENDORGET_GVL_PROVENANCE_VAULT); + const vaultCopyAvailable = record?.vaultCopyAvailable === true || containsVault; + + return { + provenance: formatGvlEvidenceProvenance(provenanceValues), + containsWeb, + containsVault, + vaultCopyAvailable, + workspaceDeleteAllowed: vaultCopyAvailable, + workspaceDeleteProtected: containsWeb && !vaultCopyAvailable + }; +} + +function normalizeGvlEvidenceProvenanceValues(record) { + const values = new Set(); + const provenance = record?.gvlEvidenceProvenance ?? record?.provenance ?? null; + + if (Array.isArray(provenance)) { + provenance.forEach((value) => appendGvlEvidenceProvenanceValue(values, value)); + } else if (typeof provenance === "string") { + provenance + .split("+") + .forEach((value) => appendGvlEvidenceProvenanceValue(values, value)); + } + + if (!values.size && record?.sourceUrl) { + values.add(VENDORGET_GVL_PROVENANCE_WEB); + } + + return Array.from(values).sort(sortGvlEvidenceProvenanceValue); +} + +function appendGvlEvidenceProvenanceValue(values, value) { + if (value === VENDORGET_GVL_PROVENANCE_WEB) { + values.add(VENDORGET_GVL_PROVENANCE_WEB); + } + + if (value === VENDORGET_GVL_PROVENANCE_VAULT) { + values.add(VENDORGET_GVL_PROVENANCE_VAULT); + } +} + +function sortGvlEvidenceProvenanceValue(left, right) { + const order = { + [VENDORGET_GVL_PROVENANCE_WEB]: 0, + [VENDORGET_GVL_PROVENANCE_VAULT]: 1 + }; + + return (order[left] ?? 99) - (order[right] ?? 99); +} + +function formatGvlEvidenceProvenance(values) { + const normalizedValues = Array.isArray(values) ? values : []; + + if ( + normalizedValues.includes(VENDORGET_GVL_PROVENANCE_WEB) && + normalizedValues.includes(VENDORGET_GVL_PROVENANCE_VAULT) + ) { + return "web+vault"; + } + + if (normalizedValues.includes(VENDORGET_GVL_PROVENANCE_VAULT)) { + return "vault"; + } + + return "web"; +} + +function markGvlRevisionEvidenceVaultCopyAvailable(db, snapshotSha256) { + return updateGvlRevisionEvidenceRecords(db, snapshotSha256, (record) => + markGvlEvidenceRecordVaultCopyAvailable(record) + ); +} + +function markGvlRevisionEvidenceWebSource(db, snapshotSha256) { + return markGvlRevisionEvidenceProvenance( + db, + snapshotSha256, + VENDORGET_GVL_PROVENANCE_WEB + ); +} + +function markGvlRevisionEvidenceProvenance(db, snapshotSha256, provenance) { + return updateGvlRevisionEvidenceRecords(db, snapshotSha256, (record) => + annotateGvlEvidenceRecordProvenance(record, provenance) + ); +} + +function updateGvlRevisionEvidenceRecords(db, snapshotSha256, updateRecord) { + return new Promise((resolve, reject) => { + const tx = db.transaction( + [VENDORGET_STORE_NAMES.gvlSnapshots, VENDORGET_STORE_NAMES.gvlRawEvidence], + "readwrite" + ); + const snapshotsStore = tx.objectStore(VENDORGET_STORE_NAMES.gvlSnapshots); + const rawEvidenceStore = tx.objectStore(VENDORGET_STORE_NAMES.gvlRawEvidence); + const snapshotRequest = snapshotsStore.get(snapshotSha256); + let result = { + snapshotMarked: false, + rawEvidenceMarked: false, + snapshotSha256, + rawGvlSha256: null + }; + + snapshotRequest.onerror = () => reject(snapshotRequest.error); + snapshotRequest.onsuccess = () => { + const snapshot = snapshotRequest.result ?? null; + + if (!snapshot) { + return; + } + + const updatedSnapshot = updateRecord(snapshot); + + snapshotsStore.put(updatedSnapshot); + result.snapshotMarked = true; + result.rawGvlSha256 = updatedSnapshot.rawGvlSha256 ?? null; + + if (!updatedSnapshot.rawGvlSha256) { + return; + } + + const rawEvidenceRequest = rawEvidenceStore.get(updatedSnapshot.rawGvlSha256); + + rawEvidenceRequest.onsuccess = () => { + const rawEvidence = rawEvidenceRequest.result ?? null; + + if (!rawEvidence) { + return; + } + + rawEvidenceStore.put( + updateRecord(rawEvidence) + ); + result.rawEvidenceMarked = true; + }; + }; + + tx.onerror = () => reject(tx.error); + tx.onabort = () => reject(tx.error); + tx.oncomplete = () => resolve(result); + }); +} + function formatGvlEvidenceUtcCompact(date) { return [ date.getUTCFullYear(), @@ -444,15 +868,25 @@ function importGvlEvidenceStoreRecords(objectStore, records, counts, seenKeys) { } function getGvlEvidenceRecordKey(objectStore, record) { + return getGvlEvidenceRecordKeyByStoreName( + objectStore.name, + record, + objectStore.keyPath + ); +} + +function getGvlEvidenceRecordKeyByStoreName(storeName, record, keyPath = "id") { if (!record || typeof record !== "object" || Array.isArray(record)) { return null; } - if (objectStore.name === VENDORGET_STORE_NAMES.gvlSnapshots) { + if (storeName === VENDORGET_STORE_NAMES.gvlSnapshots) { return record.sha256 ?? record.snapshotSha256 ?? null; } - const keyPath = objectStore.keyPath; + if (storeName === VENDORGET_STORE_NAMES.gvlRawEvidence) { + return record.rawGvlSha256 ?? null; + } if (typeof keyPath !== "string") { return null; diff --git a/src/gvl-explorer/gvl-explorer.css b/src/gvl-explorer/gvl-explorer.css index 26daeb2..0cb541c 100644 --- a/src/gvl-explorer/gvl-explorer.css +++ b/src/gvl-explorer/gvl-explorer.css @@ -224,7 +224,7 @@ th { } .snapshot-list { - min-width: 820px; + min-width: 940px; border: 0; } diff --git a/src/gvl-explorer/gvl-explorer.html b/src/gvl-explorer/gvl-explorer.html index 1a618f6..fecfd4d 100644 --- a/src/gvl-explorer/gvl-explorer.html +++ b/src/gvl-explorer/gvl-explorer.html @@ -36,6 +36,15 @@ type="file" accept="application/json,.json" > + + Bereit @@ -54,6 +63,9 @@ Vendorlisten-Version + Herkunft + Vault + Schutz Abrufzeitpunkt SHA256 Quelle diff --git a/src/gvl-explorer/gvl-explorer.js b/src/gvl-explorer/gvl-explorer.js index fac95f8..a4a6cd1 100644 --- a/src/gvl-explorer/gvl-explorer.js +++ b/src/gvl-explorer/gvl-explorer.js @@ -15,6 +15,9 @@ const gvlRevisionEvidenceExportButton = document.getElementById( const gvlRevisionEvidenceVerifyInput = document.getElementById( "gvl-revision-evidence-verify-input" ); +const gvlRevisionEvidenceImportInput = document.getElementById( + "gvl-revision-evidence-import-input" +); const gvlEvidenceTransportStatus = document.getElementById( "gvl-evidence-transport-status" ); @@ -70,6 +73,10 @@ document.addEventListener("DOMContentLoaded", async () => { await verifyGvlRevisionEvidenceJsonFile(); }); + gvlRevisionEvidenceImportInput.addEventListener("change", async () => { + await importGvlRevisionEvidenceJsonFile(); + }); + gvlRebuildNormalizedButton.addEventListener("click", async () => { await rebuildSelectedGvlSnapshotNormalizedData(); }); @@ -91,6 +98,11 @@ async function fetchOfficialGvl() { type: "fetch_official_gvl" }); + if (result?.error === "gvl_revision_evidence_conflict") { + renderFetchStatus(buildGvlEvidenceConflictMessage(result)); + return; + } + if (!result?.success) { throw new Error(result?.error ?? "official_gvl_fetch_failed"); } @@ -107,6 +119,19 @@ async function fetchOfficialGvl() { } } +function buildGvlEvidenceConflictMessage(result) { + return [ + "GVL-Web-Abruf abgebrochen: lokale Vault-Evidence weicht vom Live-Web ab.", + `Revision ${formatNullable(result?.vendorListVersion)}.`, + `Lokal ${shortenSha256(result?.existingSnapshotSha256)} / ${shortenSha256( + result?.existingRawGvlSha256 + )}.`, + `Web ${shortenSha256(result?.fetchedSnapshotSha256)} / ${shortenSha256( + result?.fetchedRawGvlSha256 + )}.` + ].join(" "); +} + function buildGvlSyncStatusMessage(result) { if (result?.syncStatus === "new_gvl_revision_stored_and_normalized") { return "GVL aus Web geladen, neue Revision gespeichert und normalisiert."; @@ -190,6 +215,8 @@ async function exportSelectedGvlRevisionEvidenceJsonFile() { } downloadGvlRevisionEvidenceJsonExport(result.export); + await markGvlRevisionEvidenceVaultCopy(result.export); + await renderGvlSnapshots(); renderGvlEvidenceTransportStatus( [ "GVL-Revision exportiert und intern verifiziert.", @@ -244,6 +271,27 @@ function downloadGvlRevisionEvidenceJsonExport(exportContainer) { setTimeout(() => URL.revokeObjectURL(url), 0); } +async function markGvlRevisionEvidenceVaultCopy(exportContainer) { + const snapshotSha256 = exportContainer?.metadata?.snapshotSha256 ?? null; + + if (!snapshotSha256) { + return; + } + + const result = await browser.runtime.sendMessage({ + type: "mark_gvl_revision_evidence_vault_copy", + payload: { + snapshotSha256 + } + }); + + if (!result?.success) { + throw new Error( + result?.error ?? "mark_gvl_revision_evidence_vault_copy_failed" + ); + } +} + function buildGvlRevisionEvidenceExportSuccessMessage(exportContainer) { const metadata = exportContainer?.metadata ?? {}; const recordCount = getGvlRevisionEvidenceNormalizedRecordCount( @@ -315,6 +363,101 @@ function setGvlRevisionEvidenceVerifyDisabled(disabled) { importLabel?.classList.toggle("is-disabled", disabled); } +async function importGvlRevisionEvidenceJsonFile() { + const file = gvlRevisionEvidenceImportInput.files?.[0] ?? null; + + if (!file) { + return; + } + + setGvlRevisionEvidenceImportDisabled(true); + renderGvlEvidenceTransportStatus( + "GVL-Revision-Evidence wird verifiziert..." + ); + + try { + const exportContainer = JSON.parse(await file.text()); + const result = await browser.runtime.sendMessage({ + type: "import_gvl_revision_evidence_json", + payload: { + export: exportContainer + } + }); + + if (!result?.success) { + const verification = result?.verification ?? result?.import?.verification; + const message = verification + ? buildGvlRevisionEvidenceVerificationMessage(verification) + : `Fehler: ${result?.error ?? "import_gvl_revision_evidence_failed"}.`; + + renderGvlEvidenceTransportStatus( + `GVL-Revision-Evidence nicht valide. ${message}`, + "error" + ); + return; + } + + selectedSnapshotSha256 = result.import?.snapshotSha256 ?? null; + renderGvlEvidenceTransportStatus( + buildGvlRevisionEvidenceImportSuccessMessage(result.import), + "success" + ); + await renderGvlSnapshots(); + + if (gvlVendorIdInput.value) { + await renderGvlVendorDetail(); + } + } catch (error) { + renderGvlEvidenceTransportStatus( + "GVL-Revision-Evidence ist nicht valide.", + "error" + ); + console.warn("VG-Observe GVL revision evidence import failed", error); + } finally { + gvlRevisionEvidenceImportInput.value = ""; + setGvlRevisionEvidenceImportDisabled(false); + } +} + +function setGvlRevisionEvidenceImportDisabled(disabled) { + const importLabel = document.querySelector( + "label[for='gvl-revision-evidence-import-input']" + ); + + gvlRevisionEvidenceImportInput.disabled = disabled; + importLabel?.classList.toggle("is-disabled", disabled); +} + +function buildGvlRevisionEvidenceImportSuccessMessage(importResult) { + return [ + "GVL-Revision-Evidence erfolgreich importiert.", + `Vendorlisten-Version ${formatNullable(importResult?.vendorListVersion)}.`, + `Snapshot ${shortenSha256(importResult?.snapshotSha256)}.`, + `Raw-GVL ${shortenSha256(importResult?.rawGvlSha256)}.`, + formatGvlRevisionEvidenceImportCounts(importResult?.counts ?? {}) + ].join(" "); +} + +function formatGvlRevisionEvidenceImportCounts(counts) { + return [ + ["gvl_raw_evidence", counts.gvl_raw_evidence], + ["gvl_snapshots", counts.gvl_snapshots], + ["gvl_vendors", counts.gvl_vendors], + ["gvl_purposes", counts.gvl_purposes], + ["gvl_special_purposes", counts.gvl_special_purposes], + ["gvl_features", counts.gvl_features], + ["gvl_special_features", counts.gvl_special_features], + ["gvl_data_categories", counts.gvl_data_categories], + ["gvl_vendor_relationships", counts.gvl_vendor_relationships] + ] + .map(([storeName, storeCounts]) => { + return `${storeName}: importiert ${Number( + storeCounts?.inserted ?? 0 + )}, übersprungen ${Number(storeCounts?.skippedExisting ?? 0)}`; + }) + .join("; "); +} + function buildGvlRevisionEvidenceVerificationMessage(verification) { const validityLabel = verification.valid ? "valide" : "nicht valide"; const counts = verification.normalizedCounts ?? {}; @@ -921,6 +1064,9 @@ function renderGvlSnapshotList() { }); appendListCell(row, formatNullable(snapshot?.vendorListVersion), "numeric"); + appendListCell(row, formatGvlProvenanceMarker(snapshot?.provenance)); + appendListCell(row, formatGvlVaultMarker(snapshot?.vaultCopyAvailable)); + appendListCell(row, formatGvlProtectionMarker(snapshot)); appendListCell(row, formatNullable(snapshot?.fetchedAt)); appendListCell(row, shortenSha256(snapshot?.sha256), "sha-cell"); appendListCell(row, formatNullable(snapshot?.sourceUrl), "url-cell"); @@ -1118,6 +1264,9 @@ function renderSummaryTable(summary) { const body = document.createElement("tbody"); const rows = [ ["Vendorlisten-Version", formatNullable(summary.vendorListVersion)], + ["Herkunft", formatGvlProvenanceMarker(summary.provenance)], + ["Vault-Kopie", formatGvlVaultMarker(summary.vaultCopyAvailable)], + ["Workspace-Schutz", formatGvlProtectionMarker(summary)], ["Abrufzeitpunkt", formatNullable(summary.fetchedAt)], ["Quelle", formatNullable(summary.sourceUrl)], ["Anzahl Firmen/Vendoren", formatCount(summary.vendorCount)], @@ -1221,6 +1370,26 @@ function shortenSha256(value) { return `${String(value).slice(0, 12)}...`; } +function formatGvlProvenanceMarker(provenance) { + if (provenance === "web+vault") { + return "🌐💾"; + } + + if (provenance === "vault") { + return "💾"; + } + + return "🌐"; +} + +function formatGvlVaultMarker(vaultCopyAvailable) { + return vaultCopyAvailable ? "📦" : "❌"; +} + +function formatGvlProtectionMarker(snapshot) { + return snapshot?.workspaceDeleteAllowed ? "🔓" : "🔒"; +} + function formatExportTimestampUtcCompact(date) { const year = date.getUTCFullYear(); const month = padDatePart(date.getUTCMonth() + 1); diff --git a/src/popup/popup.js b/src/popup/popup.js index 7f25f9d..ed05e2a 100644 --- a/src/popup/popup.js +++ b/src/popup/popup.js @@ -194,7 +194,13 @@ async function purgeUnlockedEvidence() { function buildPurgeUnlockedSuccessMessage(result) { if (Number.isFinite(result.deletedCount)) { - return `Ungesperrte Evidence-Daten gelöscht: ${result.deletedCount} Records`; + const message = `Ungesperrte Evidence-Daten gelöscht: ${result.deletedCount} Records`; + + if (Number(result.keptGvlWorkspaceProtectedCount ?? 0) > 0) { + return `${message}. Diese GVL-Evidence wurde noch nicht in den Vault exportiert.`; + } + + return message; } return "Ungesperrte Evidence-Daten gelöscht";