Prepare GVL raw evidence schema

src/background/db/db-constants.js

@@ -1,12 +1,13 @@
 "use strict";
 
 const VENDORGET_DB_NAME = "vendorget_iv";
-const VENDORGET_DB_VERSION = 5;
+const VENDORGET_DB_VERSION = 6;
 
 const VENDORGET_STORE_NAMES = {
   consentStates: "consent_states",
   consentEvents: "consent_events",
   observedRequests: "observed_requests",
+  gvlRawEvidence: "gvl_raw_evidence",
   gvlSnapshots: "gvl_snapshots",
   gvlSnapshotEvents: "gvl_snapshot_events",
   gvlVendors: "gvl_vendors",

src/background/db/db-core.js

@@ -72,7 +72,7 @@ function openVendorGetDb() {
         );
       }
 
-      ensureGvlStores(db);
+      ensureGvlStores(db, event.target.transaction);
     };
 
     request.onsuccess = () => resolve(request.result);
@@ -110,18 +110,73 @@ async function deleteVendorGetDatabase() {
   });
 }
 
-function ensureGvlStores(db) {
+function ensureGvlStores(db, upgradeTransaction) {
+  if (!db.objectStoreNames.contains("gvl_raw_evidence")) {
+    const gvlRawEvidence = db.createObjectStore("gvl_raw_evidence", {
+      keyPath: "rawGvlSha256"
+    });
+
+    createIndexesIfMissing(gvlRawEvidence, [
+      "canonicalGvlSha256",
+      "vendorListVersion",
+      "fetchedAt",
+      "sourceUrl",
+      "ingestionSource",
+      "contentType",
+      "parseStatus"
+    ]);
+  }
+
+  const gvlRawEvidence = getUpgradeObjectStore(
+    db,
+    upgradeTransaction,
+    "gvl_raw_evidence"
+  );
+
+  if (gvlRawEvidence) {
+    createIndexesIfMissing(gvlRawEvidence, [
+      "canonicalGvlSha256",
+      "vendorListVersion",
+      "fetchedAt",
+      "sourceUrl",
+      "ingestionSource",
+      "contentType",
+      "parseStatus"
+    ]);
+  }
+
   if (!db.objectStoreNames.contains("gvl_snapshots")) {
     const gvlSnapshots = db.createObjectStore("gvl_snapshots", {
       keyPath: "sha256"
     });
 
-    gvlSnapshots.createIndex("gvlRevision", "gvlRevision", { unique: false });
-    gvlSnapshots.createIndex("vendorListVersion", "vendorListVersion", {
-      unique: false
-    });
-    gvlSnapshots.createIndex("fetchedAt", "fetchedAt", { unique: false });
-    gvlSnapshots.createIndex("sourceUrl", "sourceUrl", { unique: false });
+    createIndexesIfMissing(gvlSnapshots, [
+      "gvlRevision",
+      "vendorListVersion",
+      "fetchedAt",
+      "sourceUrl",
+      "rawGvlSha256",
+      "canonicalGvlSha256",
+      "snapshotSha256"
+    ]);
+  }
+
+  const gvlSnapshots = getUpgradeObjectStore(
+    db,
+    upgradeTransaction,
+    "gvl_snapshots"
+  );
+
+  if (gvlSnapshots) {
+    createIndexesIfMissing(gvlSnapshots, [
+      "gvlRevision",
+      "vendorListVersion",
+      "fetchedAt",
+      "sourceUrl",
+      "rawGvlSha256",
+      "canonicalGvlSha256",
+      "snapshotSha256"
+    ]);
   }
 
   if (!db.objectStoreNames.contains("gvl_snapshot_events")) {
@@ -130,22 +185,43 @@ function ensureGvlStores(db) {
       autoIncrement: true
     });
 
-    gvlSnapshotEvents.createIndex("eventType", "eventType", { unique: false });
-    gvlSnapshotEvents.createIndex("capturedAt", "capturedAt", { unique: false });
-    gvlSnapshotEvents.createIndex("gvlRevision", "gvlRevision", {
-      unique: false
-    });
-    gvlSnapshotEvents.createIndex("vendorListVersion", "vendorListVersion", {
-      unique: false
-    });
-    gvlSnapshotEvents.createIndex("sha256", "sha256", { unique: false });
-    gvlSnapshotEvents.createIndex("sourceUrl", "sourceUrl", { unique: false });
+    createIndexesIfMissing(gvlSnapshotEvents, [
+      "eventType",
+      "capturedAt",
+      "gvlRevision",
+      "vendorListVersion",
+      "sha256",
+      "sourceUrl",
+      "rawGvlSha256",
+      "canonicalGvlSha256",
+      "snapshotSha256"
+    ]);
   }
 
-  ensureGvlRelationshipStores(db);
+  const gvlSnapshotEvents = getUpgradeObjectStore(
+    db,
+    upgradeTransaction,
+    "gvl_snapshot_events"
+  );
+
+  if (gvlSnapshotEvents) {
+    createIndexesIfMissing(gvlSnapshotEvents, [
+      "eventType",
+      "capturedAt",
+      "gvlRevision",
+      "vendorListVersion",
+      "sha256",
+      "sourceUrl",
+      "rawGvlSha256",
+      "canonicalGvlSha256",
+      "snapshotSha256"
+    ]);
+  }
+
+  ensureGvlRelationshipStores(db, upgradeTransaction);
 }
 
-function ensureGvlRelationshipStores(db) {
+function ensureGvlRelationshipStores(db, upgradeTransaction) {
   const gvlRelationshipStoreDefinitions = [
     {
       name: "gvl_vendors",
@@ -154,28 +230,66 @@ function ensureGvlRelationshipStores(db) {
         "vendorId",
         "name",
         "policyUrl",
-        "deletedDate"
+        "deletedDate",
+        "rawGvlSha256",
+        "canonicalGvlSha256",
+        "snapshotSha256"
       ]
     },
     {
       name: "gvl_purposes",
-      indexes: ["vendorListVersion", "purposeId", "name"]
+      indexes: [
+        "vendorListVersion",
+        "purposeId",
+        "name",
+        "rawGvlSha256",
+        "canonicalGvlSha256",
+        "snapshotSha256"
+      ]
     },
     {
       name: "gvl_special_purposes",
-      indexes: ["vendorListVersion", "specialPurposeId", "name"]
+      indexes: [
+        "vendorListVersion",
+        "specialPurposeId",
+        "name",
+        "rawGvlSha256",
+        "canonicalGvlSha256",
+        "snapshotSha256"
+      ]
     },
     {
       name: "gvl_features",
-      indexes: ["vendorListVersion", "featureId", "name"]
+      indexes: [
+        "vendorListVersion",
+        "featureId",
+        "name",
+        "rawGvlSha256",
+        "canonicalGvlSha256",
+        "snapshotSha256"
+      ]
     },
     {
       name: "gvl_special_features",
-      indexes: ["vendorListVersion", "specialFeatureId", "name"]
+      indexes: [
+        "vendorListVersion",
+        "specialFeatureId",
+        "name",
+        "rawGvlSha256",
+        "canonicalGvlSha256",
+        "snapshotSha256"
+      ]
     },
     {
       name: "gvl_data_categories",
-      indexes: ["vendorListVersion", "dataCategoryId", "name"]
+      indexes: [
+        "vendorListVersion",
+        "dataCategoryId",
+        "name",
+        "rawGvlSha256",
+        "canonicalGvlSha256",
+        "snapshotSha256"
+      ]
     },
     {
       name: "gvl_vendor_relationships",
@@ -183,13 +297,26 @@ function ensureGvlRelationshipStores(db) {
         "vendorListVersion",
         "vendorId",
         "relationshipType",
-        "relatedId"
+        "relatedId",
+        "rawGvlSha256",
+        "canonicalGvlSha256",
+        "snapshotSha256"
       ]
     }
   ];
 
   gvlRelationshipStoreDefinitions.forEach((storeDefinition) => {
     if (db.objectStoreNames.contains(storeDefinition.name)) {
+      const objectStore = getUpgradeObjectStore(
+        db,
+        upgradeTransaction,
+        storeDefinition.name
+      );
+
+      if (objectStore) {
+        createIndexesIfMissing(objectStore, storeDefinition.indexes);
+      }
+
       return;
     }
 
@@ -197,8 +324,24 @@ function ensureGvlRelationshipStores(db) {
       keyPath: "id"
     });
 
-    storeDefinition.indexes.forEach((indexName) => {
-      objectStore.createIndex(indexName, indexName, { unique: false });
-    });
+    createIndexesIfMissing(objectStore, storeDefinition.indexes);
+  });
+}
+
+function getUpgradeObjectStore(db, upgradeTransaction, storeName) {
+  if (!upgradeTransaction || !db.objectStoreNames.contains(storeName)) {
+    return null;
+  }
+
+  return upgradeTransaction.objectStore(storeName);
+}
+
+function createIndexesIfMissing(objectStore, indexNames) {
+  indexNames.forEach((indexName) => {
+    if (objectStore.indexNames.contains(indexName)) {
+      return;
+    }
+
+    objectStore.createIndex(indexName, indexName, { unique: false });
   });
 }