Implement data portability Stage 2 verification

App.tsx

@@ -22,7 +22,7 @@ import {
   createSavedAdId,
   deleteSavedAd,
   getLatestTopicArchive,
-  initializeDurableDataMirrors,
+  verifyDurableDataMirrors,
   saveTopicArchive,
   touchSavedAdLastUsed,
   upsertSavedAd,
@@ -627,7 +627,9 @@ const App: React.FC = () => {
 
   // Check for API keys on mount; never show modal on load so user can see the app first
   useEffect(() => {
-    void initializeDurableDataMirrors();
+    // Stage 2: asynchronously shadow-read, compare, and reconcile both IDB
+    // mirrors. All user-facing reads remain localStorage-backed.
+    void verifyDurableDataMirrors();
 
     const checkApiKeys = async () => {
       setApiKeyCheckDone(true);

__tests__/tefArchiveStage1Mirror.test.ts

@@ -154,6 +154,7 @@ async function seedVersionTwo(params: {
 }
 
 async function seedVersionThree(params: {
+  ads?: TefSavedAd[];
   archives?: TefTopicArchive[];
   scenarios?: Scenario[];
 } = {}): Promise<void> {
@@ -165,7 +166,8 @@ async function seedVersionThree(params: {
     request.result.createObjectStore('migrationMetadata', { keyPath: 'name' });
   };
   const db = await requestResult(request);
-  const tx = db.transaction(['topicArchives', 'scenarios'], 'readwrite');
+  const tx = db.transaction(['savedAds', 'topicArchives', 'scenarios'], 'readwrite');
+  for (const record of params.ads ?? []) tx.objectStore('savedAds').put(record);
   for (const record of params.archives ?? []) tx.objectStore('topicArchives').put(record);
   for (const record of params.scenarios ?? []) tx.objectStore('scenarios').put(record);
   await transactionDone(tx);
@@ -176,6 +178,24 @@ async function openCurrentDatabase(): Promise<IDBDatabase> {
   return await requestResult(indexedDB.open('parle-tef'));
 }
 
+async function failReadWriteTransactionsForStore(storeName: string) {
+  const db = await openCurrentDatabase();
+  const databasePrototype = Object.getPrototypeOf(db) as IDBDatabase;
+  db.close();
+  const originalTransaction = databasePrototype.transaction;
+  return vi.spyOn(databasePrototype, 'transaction').mockImplementation(function (
+    this: IDBDatabase,
+    ...args: Parameters<IDBDatabase['transaction']>
+  ) {
+    const [storeNames, mode] = args;
+    const names = typeof storeNames === 'string' ? [storeNames] : Array.from(storeNames);
+    if (mode === 'readwrite' && names.includes(storeName)) {
+      throw new Error(`Forced ${storeName} mirror transaction failure`);
+    }
+    return originalTransaction.apply(this, args);
+  });
+}
+
 describe('Stage 1 durable exercise data IndexedDB mirrors', () => {
   beforeEach(() => {
     localStorage.clear();
@@ -465,3 +485,321 @@ describe('Stage 1 durable exercise data IndexedDB mirrors', () => {
     expect(mirrored.find((scenario) => scenario.id === current.id)).toEqual(current);
   });
 });
+
+describe('Stage 2 durable exercise data shadow verification', () => {
+  beforeEach(() => {
+    localStorage.clear();
+    vi.resetModules();
+    vi.stubGlobal('indexedDB', new IDBFactory());
+  });
+
+  it('detects same-count content mismatches and repairs both datasets independently', async () => {
+    const ads = [savedAd('ad-1'), savedAd('ad-2')];
+    const archives = [archive('archive-1', 'ad-1'), archive('archive-2', 'ad-2', 200)];
+    const scenarios = [legacyScenario(), currentScenario()];
+    const serializedArchives = JSON.stringify(archives);
+    const serializedScenarios = JSON.stringify(scenarios);
+    localStorage.setItem(TOPIC_ARCHIVES_KEY, serializedArchives);
+    localStorage.setItem(SCENARIOS_KEY, serializedScenarios);
+    await seedVersionThree({
+      ads,
+      archives: [
+        { ...archive('archive-1', 'missing-ad'), exerciseType: 'questioning' },
+        archive('extra-archive', 'ad-1'),
+      ],
+      scenarios: [
+        { ...legacyScenario(), name: 'Different legacy name' },
+        currentScenario('extra-scenario'),
+      ],
+    });
+    const service = await import('../services/tefArchiveService');
+
+    const result = await service.verifyDurableDataMirrors();
+
+    expect(result.topicArchives).toMatchObject({
+      operation: 'shadow-verify',
+      success: true,
+      sourceRecordCount: 2,
+      destinationRecordCount: 2,
+      comparison: {
+        missingIds: ['archive-2'],
+        extraIds: ['extra-archive'],
+        differingIds: ['archive-1'],
+        relationshipInvalidIds: ['archive-1'],
+        legacyShapeIds: [],
+      },
+      postRepairIntegrity: {
+        relationshipInvalidIds: [],
+        legacyShapeIds: [],
+      },
+      repairs: { inserted: 1, updated: 1, deleted: 1 },
+    });
+    expect(result.scenarios).toMatchObject({
+      operation: 'shadow-verify',
+      success: true,
+      comparison: {
+        missingIds: ['current-scenario'],
+        extraIds: ['extra-scenario'],
+        differingIds: ['legacy-scenario'],
+        legacyShapeIds: ['legacy-scenario'],
+      },
+      repairs: { inserted: 1, updated: 1, deleted: 1 },
+    });
+    expect(await service.getTopicArchiveMirrorSnapshot()).toEqual(expect.arrayContaining(archives));
+    expect(await service.getTopicArchiveMirrorSnapshot()).toHaveLength(2);
+    expect(await service.getScenarioMirrorSnapshot()).toEqual(expect.arrayContaining(scenarios));
+    expect(await service.getScenarioMirrorSnapshot()).toHaveLength(2);
+    expect(localStorage.getItem(TOPIC_ARCHIVES_KEY)).toBe(serializedArchives);
+    expect(localStorage.getItem(SCENARIOS_KEY)).toBe(serializedScenarios);
+    expect(await service.getTopicArchiveMigrationMetadata()).toMatchObject({
+      verificationStatus: 'verified',
+      mismatchCounts: { missing: 1, extra: 1, differing: 1 },
+      repairCounts: { inserted: 1, updated: 1, deleted: 1 },
+      preRepairIntegrityCounts: { relationshipInvalid: 1, legacyShape: 0 },
+      postRepairIntegrityCounts: { relationshipInvalid: 0, legacyShape: 0 },
+      relationshipInvalidRecordCount: 0,
+    });
+    expect(await service.getScenarioMigrationMetadata()).toMatchObject({
+      verificationStatus: 'verified',
+      mismatchCounts: { missing: 1, extra: 1, differing: 1 },
+      repairCounts: { inserted: 1, updated: 1, deleted: 1 },
+      legacyShapeRecordCount: 1,
+    });
+  });
+
+  it('reports invalid archive relationships without altering the authoritative archive', async () => {
+    const orphan = archive('orphan-archive', 'deleted-ad');
+    const serialized = JSON.stringify([orphan]);
+    localStorage.setItem(TOPIC_ARCHIVES_KEY, serialized);
+    const service = await import('../services/tefArchiveService');
+
+    const result = await service.verifyTopicArchiveMirror();
+
+    expect(result).toMatchObject({
+      success: true,
+      comparison: {
+        missingIds: ['orphan-archive'],
+        relationshipInvalidIds: ['orphan-archive'],
+      },
+      postRepairIntegrity: {
+        relationshipInvalidIds: ['orphan-archive'],
+      },
+    });
+    expect(await service.getTopicArchiveMirrorSnapshot()).toEqual([orphan]);
+    expect(localStorage.getItem(TOPIC_ARCHIVES_KEY)).toBe(serialized);
+    expect(await service.getTopicArchiveMigrationMetadata()).toMatchObject({
+      verificationStatus: 'verified',
+      preRepairIntegrityCounts: { relationshipInvalid: 1, legacyShape: 0 },
+      postRepairIntegrityCounts: { relationshipInvalid: 1, legacyShape: 0 },
+      relationshipInvalidRecordCount: 1,
+    });
+  });
+
+  it('persists an independent failed verification without mutating malformed localStorage', async () => {
+    const safeScenario = currentScenario();
+    const serializedScenarios = JSON.stringify([safeScenario]);
+    localStorage.setItem(TOPIC_ARCHIVES_KEY, '{malformed');
+    localStorage.setItem(SCENARIOS_KEY, serializedScenarios);
+    const consoleError = vi.spyOn(console, 'error').mockImplementation(() => undefined);
+    const service = await import('../services/tefArchiveService');
+
+    const result = await service.verifyDurableDataMirrors();
+
+    expect(result.topicArchives).toMatchObject({
+      success: false,
+      operation: 'shadow-verify',
+      error: 'Authoritative localStorage TEF topic archives are unreadable',
+    });
+    expect(result.scenarios.success).toBe(true);
+    expect(localStorage.getItem(TOPIC_ARCHIVES_KEY)).toBe('{malformed');
+    expect(localStorage.getItem(SCENARIOS_KEY)).toBe(serializedScenarios);
+    expect(await service.getTopicArchiveMigrationMetadata()).toMatchObject({
+      verificationStatus: 'failed',
+      verificationError: 'Authoritative localStorage TEF topic archives are unreadable',
+    });
+    expect(await service.getScenarioMigrationMetadata()).toMatchObject({
+      verificationStatus: 'verified',
+    });
+    consoleError.mockRestore();
+  });
+
+  it('already-matching archive and scenario stores verify independently with zero mismatches', async () => {
+    const ad = savedAd('ad-1');
+    const archives = [archive('matching-archive', ad.id)];
+    const scenarios = [currentScenario('matching-scenario')];
+    localStorage.setItem(TOPIC_ARCHIVES_KEY, JSON.stringify(archives));
+    localStorage.setItem(SCENARIOS_KEY, JSON.stringify(scenarios));
+    await seedVersionThree({ ads: [ad], archives, scenarios });
+    const service = await import('../services/tefArchiveService');
+
+    const archiveResult = await service.verifyTopicArchiveMirror();
+
+    expect(archiveResult).toMatchObject({
+      success: true,
+      comparison: {
+        missingIds: [],
+        extraIds: [],
+        differingIds: [],
+        relationshipInvalidIds: [],
+        legacyShapeIds: [],
+      },
+      postRepairIntegrity: {
+        relationshipInvalidIds: [],
+        legacyShapeIds: [],
+      },
+      repairs: { inserted: 0, updated: 0, deleted: 0 },
+    });
+    expect(await service.getScenarioMigrationMetadata()).toBeNull();
+
+    const scenarioResult = await service.verifyScenarioMirror();
+
+    expect(scenarioResult).toMatchObject({
+      success: true,
+      comparison: {
+        missingIds: [],
+        extraIds: [],
+        differingIds: [],
+        relationshipInvalidIds: [],
+        legacyShapeIds: [],
+      },
+      postRepairIntegrity: {
+        relationshipInvalidIds: [],
+        legacyShapeIds: [],
+      },
+      repairs: { inserted: 0, updated: 0, deleted: 0 },
+    });
+    expect(await service.getTopicArchiveMigrationMetadata()).toMatchObject({
+      verificationStatus: 'verified',
+      mismatchCounts: { missing: 0, extra: 0, differing: 0 },
+      repairCounts: { inserted: 0, updated: 0, deleted: 0 },
+      postRepairIntegrityCounts: { relationshipInvalid: 0, legacyShape: 0 },
+    });
+    expect(await service.getScenarioMigrationMetadata()).toMatchObject({
+      verificationStatus: 'verified',
+      mismatchCounts: { missing: 0, extra: 0, differing: 0 },
+      repairCounts: { inserted: 0, updated: 0, deleted: 0 },
+      postRepairIntegrityCounts: { relationshipInvalid: 0, legacyShape: 0 },
+    });
+  });
+
+  it('invalidates archive verification when a later save mirror fails, then re-verifies on repair', async () => {
+    const ad = savedAd('ad-1');
+    const initial = archive('initial-archive', ad.id);
+    localStorage.setItem(TOPIC_ARCHIVES_KEY, JSON.stringify([initial]));
+    await seedVersionThree({ ads: [ad], archives: [initial] });
+    const service = await import('../services/tefArchiveService');
+    await service.verifyTopicArchiveMirror();
+    expect(await service.getTopicArchiveMigrationMetadata()).toMatchObject({
+      verificationStatus: 'verified',
+    });
+
+    const transactionSpy = await failReadWriteTransactionsForStore('topicArchives');
+    service.saveTopicArchive({
+      adId: ad.id,
+      exerciseType: 'persuasion',
+      topicSuggestions: topics,
+    });
+    const failedMirror = await service.waitForTopicArchiveMirror();
+    transactionSpy.mockRestore();
+
+    expect(failedMirror).toMatchObject({
+      operation: 'save',
+      success: false,
+      error: 'Forced topicArchives mirror transaction failure',
+    });
+    expect(service.listTopicArchives()).toHaveLength(2);
+    expect(await service.getTopicArchiveMigrationMetadata()).toMatchObject({
+      verificationStatus: 'failed',
+      verificationError: 'Forced topicArchives mirror transaction failure',
+    });
+
+    expect((await service.verifyTopicArchiveMirror()).success).toBe(true);
+    expect(await service.getTopicArchiveMigrationMetadata()).toMatchObject({
+      verificationStatus: 'verified',
+      sourceRecordCount: 2,
+      destinationRecordCount: 2,
+    });
+  });
+
+  it('invalidates scenario verification when a later save mirror fails, then re-verifies on repair', async () => {
+    const initial = currentScenario('scenario-mutation-failure');
+    localStorage.setItem(SCENARIOS_KEY, JSON.stringify([initial]));
+    await seedVersionThree({ scenarios: [initial] });
+    const service = await import('../services/tefArchiveService');
+    const scenarioService = await import('../services/scenarioService');
+    await service.verifyScenarioMirror();
+    expect(await service.getScenarioMigrationMetadata()).toMatchObject({
+      verificationStatus: 'verified',
+    });
+
+    const transactionSpy = await failReadWriteTransactionsForStore('scenarios');
+    scenarioService.saveScenario({ ...initial, name: 'Saved locally only at first' });
+    const failedMirror = await service.waitForScenarioMirror();
+    transactionSpy.mockRestore();
+
+    expect(failedMirror).toMatchObject({
+      operation: 'save',
+      success: false,
+      error: 'Forced scenarios mirror transaction failure',
+    });
+    expect(scenarioService.loadScenarios()[0].name).toBe('Saved locally only at first');
+    expect(await service.getScenarioMigrationMetadata()).toMatchObject({
+      verificationStatus: 'failed',
+      verificationError: 'Forced scenarios mirror transaction failure',
+    });
+
+    expect((await service.verifyScenarioMirror()).success).toBe(true);
+    expect(await service.getScenarioMigrationMetadata()).toMatchObject({
+      verificationStatus: 'verified',
+      sourceRecordCount: 1,
+      destinationRecordCount: 1,
+    });
+  });
+
+  it('keeps both datasets dirty when IndexedDB cannot persist failure metadata', async () => {
+    const ad = savedAd('ad-1');
+    const initialArchive = archive('initial-offline-archive', ad.id);
+    const initialScenario = currentScenario('initial-offline-scenario');
+    localStorage.setItem(TOPIC_ARCHIVES_KEY, JSON.stringify([initialArchive]));
+    localStorage.setItem(SCENARIOS_KEY, JSON.stringify([initialScenario]));
+    await seedVersionThree({
+      ads: [ad],
+      archives: [initialArchive],
+      scenarios: [initialScenario],
+    });
+    const service = await import('../services/tefArchiveService');
+    const scenarioService = await import('../services/scenarioService');
+    await service.verifyDurableDataMirrors();
+
+    const availableIndexedDb = indexedDB;
+    vi.stubGlobal('indexedDB', undefined);
+    service.saveTopicArchive({
+      adId: ad.id,
+      exerciseType: 'persuasion',
+      topicSuggestions: topics,
+    });
+    scenarioService.saveScenario({ ...initialScenario, name: 'Changed while IDB unavailable' });
+    expect((await service.waitForTopicArchiveMirror())?.success).toBe(false);
+    expect((await service.waitForScenarioMirror())?.success).toBe(false);
+
+    vi.stubGlobal('indexedDB', availableIndexedDb);
+    expect(await service.getTopicArchiveMigrationMetadata()).toMatchObject({
+      verificationStatus: 'failed',
+      verificationError: 'Authoritative localStorage changed after the last verified reconciliation',
+    });
+    expect(await service.getScenarioMigrationMetadata()).toMatchObject({
+      verificationStatus: 'failed',
+      verificationError: 'Authoritative localStorage changed after the last verified reconciliation',
+    });
+
+    const repaired = await service.verifyDurableDataMirrors();
+    expect(repaired.topicArchives.success).toBe(true);
+    expect(repaired.scenarios.success).toBe(true);
+    expect(await service.getTopicArchiveMigrationMetadata()).toMatchObject({
+      verificationStatus: 'verified',
+    });
+    expect(await service.getScenarioMigrationMetadata()).toMatchObject({
+      verificationStatus: 'verified',
+    });
+  });
+});