feat: sync progress

.eslintrc

@@ -24,7 +24,7 @@
   ],
   "rules": {
     "max-len": ["error", {"code": 120, "ignoreComments": true, "ignoreTemplateLiterals": true}],
-    "no-console": "error",
+    "no-console": "warn",
     "semi": [ "error", "always" ],
     "@typescript-eslint/no-unsafe-member-access": "off",
     "@typescript-eslint/no-unsafe-assignment": "off",
@@ -36,7 +36,7 @@
     "@typescript-eslint/no-unsafe-return": "off",
     "@typescript-eslint/no-explicit-any": "off",
     "sort-imports": [
-      "error",
+      "off",
       {
         "ignoreCase": false,
         "ignoreDeclarationSort": false,

src/service-worker/command/api/store/api-store-sync-obj.command.ts

@@ -28,7 +28,7 @@ export interface ObjAddRequest {
 }
 
 export class ApiStoreSyncObjCommand extends ApiCallBase implements ICommand<Promise<void>> {
-  constructor(private obj: ObjDto, private initialHash: string) {
+  constructor(private obj: ObjDto, private hash: string, private tx: string) {
     super();
   }
   async execute(): Promise<void> {
@@ -42,7 +42,7 @@ export class ApiStoreSyncObjCommand extends ApiCallBase implements ICommand<Prom
         body: JSON.stringify({
           type: this.obj.type,
           localId: this.obj.id,
-          initialHash: this.initialHash
+          hash: this.hash
         })
       },
       this.refreshParams()

src/service-worker/command/sync/obj/sync-index.command.ts

@@ -32,6 +32,7 @@ import { SyncSetProgressCommand } from '../progress/sync-set-progress.command';
 import { SyncSnapshotCommand } from './sync-snapshot.command';
 import { fnConsoleLog } from '../../../../common/fn/fn-console';
 import { fnSleep } from '../../../../common/fn/fn-sleep';
+import { BeginTxResponse } from '../../api/store/api-store.model';
 
 export enum SyncObjectStatus {
   TX_LOCKED,
@@ -48,7 +49,7 @@ export interface SyncIndex extends ObjDateIndex {
 }
 
 export class SyncIndexCommand implements ICommand<Promise<SyncObjectStatus>> {
-  constructor(private progress: SyncProgress, private index?: ObjDateIndex) {}
+  constructor(private progress: SyncProgress, private tx: BeginTxResponse, private index?: ObjDateIndex) {}
 
   async execute(): Promise<SyncObjectStatus> {
     if (!this.index) {
@@ -64,8 +65,7 @@ export class SyncIndexCommand implements ICommand<Promise<SyncObjectStatus>> {
       case ObjTypeDto.PageSnapshot:
       case ObjTypeDto.PageElementSnapshot: {
         fnConsoleLog('SyncSnapshotCommand', obj.type, obj.id, 'index', this.index, 'obj', obj);
-        await new SyncSnapshotCommand(obj as ObjDto<ObjPageDto>, this.progress, this.index).execute();
+        return await new SyncSnapshotCommand(obj as ObjDto<ObjPageDto>, this.progress).execute();
-        break;
       }
       case ObjTypeDto.PageElementPin: {
         fnConsoleLog('SyncPinCommand', obj.type, obj.id, 'index', this.index, 'obj', obj);

src/service-worker/command/sync/obj/sync-note.command.ts

@@ -28,6 +28,6 @@ export class SyncNoteCommand implements ICommand<Promise<void>> {
   async execute(): Promise<void> {
     fnConsoleLog('SyncNoteCommand');
     const data = this.obj.data;
-    await new SyncObjectCommand(this.obj, data.hash, this.progress, this.index).execute();
+    await new SyncObjectCommand(this.obj, data.hash, this.progress).execute();
   }
 }

src/service-worker/command/sync/obj/sync-object.command.ts

@@ -19,24 +19,14 @@ import { ObjDateIndex } from '../../../../common/command/obj/index/obj-update-in
 import { ObjDto } from '../../../../common/model/obj/obj.dto';
 import { SyncProgress } from '../sync.model';
 import { fnConsoleLog } from '../../../../common/fn/fn-console';
+import { ApiStoreSyncObjCommand } from '../../api/store/api-store-sync-obj.command';
+import { BeginTxResponse } from '../../api/store/api-store.model';
 
 export class SyncObjectCommand implements ICommand<Promise<void>> {
-  constructor(
+  constructor(private obj: ObjDto, private hash: string, private tx: BeginTxResponse) {}
-    private obj: ObjDto,
-    private initialHash: string,
-    private progress: SyncProgress,
-    private index: ObjDateIndex
-  ) {}
   // eslint-disable-next-line @typescript-eslint/require-await
   async execute(): Promise<void> {
-    fnConsoleLog(
+    fnConsoleLog('SyncObjectDataCommand');
-      'SyncObjectDataCommand',
+    await new ApiStoreSyncObjCommand(this.obj, this.hash, this.tx.tx).execute();
-      this.obj.id,
-      this.obj.type,
-      this.obj.createdAt,
-      this.obj.updatedAt,
-      this.obj.version,
-      this.initialHash
-    );
   }
 }

src/service-worker/command/sync/obj/sync-page-note.command.ts

@@ -27,6 +27,6 @@ export class SyncPageNoteCommand implements ICommand<Promise<void>> {
   async execute(): Promise<void> {
     fnConsoleLog('SyncPageNoteCommand');
     const data = this.obj.data;
-    await new SyncObjectCommand(this.obj, data.hash, this.progress, this.index).execute();
+    await new SyncObjectCommand(this.obj, data.hash, this.progress).execute();
   }
 }

src/service-worker/command/sync/obj/sync-pdf.command.ts

@@ -27,6 +27,6 @@ export class SyncPdfCommand implements ICommand<Promise<void>> {
   async execute(): Promise<void> {
     fnConsoleLog('SyncPdfCommand');
     const data = this.obj.data;
-    await new SyncObjectCommand(this.obj, data.hash, this.progress, this.index).execute();
+    await new SyncObjectCommand(this.obj, data.hash, this.progress).execute();
   }
 }

src/service-worker/command/sync/obj/sync-pin.command.ts

@@ -21,12 +21,13 @@ import { ObjPinDto } from '../../../../common/model/obj/obj-pin.dto';
 import { SyncObjectCommand } from './sync-object.command';
 import { SyncProgress } from '../sync.model';
 import { fnConsoleLog } from '../../../../common/fn/fn-console';
+import { BeginTxResponse } from '../../api/store/api-store.model';
 
 export class SyncPinCommand implements ICommand<Promise<void>> {
-  constructor(private obj: ObjDto<ObjPinDto>, private progress: SyncProgress, private index: ObjDateIndex) {}
+  constructor(private obj: ObjDto<ObjPinDto>, private progress: SyncProgress, private tx: BeginTxResponse) {}
   async execute(): Promise<void> {
     fnConsoleLog('SyncPinCommand');
     const data = this.obj.data;
-    await new SyncObjectCommand(this.obj, data.data.hash, this.progress, this.index).execute();
+    await new SyncObjectCommand(this.obj, data.data.hash, this.progress).execute();
   }
 }

src/service-worker/command/sync/obj/sync-snapshot.command.ts

@@ -16,28 +16,28 @@
  */
 import { SegmentCss, SegmentImg, SegmentPage, SegmentType } from '@pinmenote/page-compute';
 import { ICommand } from '../../../../common/model/shared/common.dto';
-import { ObjDateIndex } from '../../../../common/command/obj/index/obj-update-index-add.command';
 import { ObjDto } from '../../../../common/model/obj/obj.dto';
 import { ObjPageDto } from '../../../../common/model/obj/obj-page.dto';
 import { PageSegmentGetCommand } from '../../../../common/command/snapshot/segment/page-segment-get.command';
 import { SyncObjectCommand } from './sync-object.command';
+import { SyncObjectStatus } from './sync-index.command';
 import { SyncProgress } from '../sync.model';
 import { fnConsoleLog } from '../../../../common/fn/fn-console';
 
-export class SyncSnapshotCommand implements ICommand<Promise<void>> {
+export class SyncSnapshotCommand implements ICommand<Promise<SyncObjectStatus>> {
-  constructor(private obj: ObjDto<ObjPageDto>, private progress: SyncProgress, private index: ObjDateIndex) {}
+  constructor(private obj: ObjDto<ObjPageDto>, private progress: SyncProgress) {}
-  async execute(): Promise<void> {
+  async execute(): Promise<SyncObjectStatus> {
     const snapshot = this.obj.data.snapshot;
-    // fnConsoleLog('SyncSnapshotCommand->comments', this.obj.data.comments);
+    if (!snapshot.segmentHash) {
-    // fnConsoleLog('SyncSnapshotCommand->snapshot', this.obj.id, 'index', this.index, 'obj', this.obj);
+      console.log('PROBLEM !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!', snapshot);
-    if (!snapshot.segmentHash) return;
+      return SyncObjectStatus.SERVER_ERROR;
-
+    }
-    await new SyncObjectCommand(this.obj, snapshot.segmentHash, this.progress, this.index).execute();
+    await new SyncObjectCommand(this.obj, snapshot.segmentHash, this.progress).execute();
-
+    return SyncObjectStatus.SERVER_ERROR;
-    await this.syncSegment(snapshot.segmentHash);
+    // await this.syncSegment(snapshot.segmentHash);
   }
 
-  private async syncSegment(hash: string, nested = false) {
+  private async syncSegment(hash: string) {
     const segment = await new PageSegmentGetCommand(hash).execute();
     if (!segment) return;
     switch (segment.type) {
@@ -64,15 +64,11 @@ export class SyncSnapshotCommand implements ICommand<Promise<void>> {
       }
       case SegmentType.IFRAME: {
         const content = segment.content as SegmentPage;
-        const matched = content.html.html.matchAll(/(data-pin-hash=")([a-z\d]+")/g);
-        const a = Array.from(matched);
-        // if (content.assets.length > 0) fnConsoleLog('IFRAME', a, content);
-        if (a.length > 0) fnConsoleLog('IFRAME', nested, content, 'MATCHED', a);
         for (const css of content.css) {
           await this.syncSegment(css);
         }
         for (const asset of content.assets) {
-          await this.syncSegment(asset, true);
+          await this.syncSegment(asset);
         }
         break;
       }

src/service-worker/command/sync/progress/sync-reset-progress.command.ts

@@ -19,6 +19,10 @@ import { ICommand } from '../../../../common/model/shared/common.dto';
 import { ObjectStoreKeys } from '../../../../common/keys/object.store.keys';
 import { SyncGetProgressCommand } from './sync-get-progress.command';
 import { SyncProgress } from '../sync.model';
+import { ObjDto } from '../../../../common/model/obj/obj.dto';
+import { fnDateKeyFormat } from '../../../../common/fn/fn-date-format';
+import { ObjDateIndex } from '../../../../common/command/obj/index/obj-update-index-add.command';
+import { fnConsoleLog } from '../../../../common/fn/fn-console';
 
 export class SyncResetProgressCommand implements ICommand<Promise<void>> {
   async execute(): Promise<void> {
@@ -28,5 +32,56 @@ export class SyncResetProgressCommand implements ICommand<Promise<void>> {
       timestamp: obj.createdAt,
       id: obj.id
     });
+    await this.resetObjects();
+  }
+
+  async resetObjects(): Promise<void> {
+    let listId = await BrowserStorage.get<number>(ObjectStoreKeys.OBJECT_LIST_ID);
+    const a = Date.now();
+    console.log('SyncResetProgressCommand->start !!!!', listId);
+
+    const toSortSet: Set<string> = new Set<string>();
+
+    while (listId > 0) {
+      const list = await BrowserStorage.get<number[]>(`${ObjectStoreKeys.OBJECT_LIST}:${listId}`);
+      for (const id of list) {
+        const obj = await BrowserStorage.get<ObjDto>(`${ObjectStoreKeys.OBJECT_ID}:${id}`);
+        const yearMonth = fnDateKeyFormat(new Date(obj.updatedAt));
+        toSortSet.add(yearMonth);
+
+        const updateList = await this.getList(yearMonth);
+        if (updateList.findIndex((o) => o.id === obj.id) > 0) continue;
+
+        updateList.push({ id: obj.id, dt: obj.updatedAt });
+
+        await this.setList(yearMonth, updateList);
+      }
+      listId -= 1;
+    }
+
+    // sort objects inside
+    for (const yearMonth of toSortSet) {
+      let list = await this.getList(yearMonth);
+      list = list.sort((a, b) => {
+        if (a.dt > b.dt) return 1;
+        if (a.dt < b.dt) return -1;
+        if (a.id > b.id) return 1;
+        if (a.id < b.id) return -1;
+        return 0;
+      });
+    }
+    fnConsoleLog('SyncResetProgressCommand->complete in ', Date.now() - a);
+  }
+
+  private async setList(yearMonth: string, list: ObjDateIndex[]): Promise<void> {
+    const updateListKey = `${ObjectStoreKeys.UPDATED_DT}:${yearMonth}`;
+    await BrowserStorage.set(updateListKey, list);
+  }
+
+  private async getList(yearMonth: string): Promise<ObjDateIndex[]> {
+    const updateListKey = `${ObjectStoreKeys.UPDATED_DT}:${yearMonth}`;
+    const updateList = await BrowserStorage.get<ObjDateIndex[] | undefined>(updateListKey);
+    if (!updateList) return [];
+    return updateList;
   }
 }

src/service-worker/command/sync/sync-month.command.ts

@@ -54,17 +54,19 @@ export class SyncMonthCommand implements ICommand<Promise<SyncIndex>> {
     const begin = await SyncTxHelper.begin();
     if (!begin) return { ...index, status: SyncObjectStatus.TX_LOCKED };
 
-    const newIndexList = [];
-
     for (let i = nextObjectIndex; i < indexList.length; i++) {
       index = { ...indexList[i], status: SyncObjectStatus.OK };
-      const status = await new SyncIndexCommand(this.progress, index).execute();
+      const status = await new SyncIndexCommand(this.progress, begin, index).execute();
-      if (![SyncObjectStatus.INDEX_NOT_EXISTS, SyncObjectStatus.OBJECT_NOT_EXISTS].includes(status)) {
-        newIndexList.push(index);
-      }
-    }
 
-    await SyncTxHelper.setList(indexListKey, newIndexList);
+      if ([SyncObjectStatus.SERVER_ERROR].includes(status)) {
+        index = { ...index, status: SyncObjectStatus.SERVER_ERROR };
+        await SyncTxHelper.commit();
+        return index;
+      }
+      if (![SyncObjectStatus.INDEX_NOT_EXISTS, SyncObjectStatus.OBJECT_NOT_EXISTS].includes(status)) {
+        console.log('PROBLEM !!!!!!!!!!!!!!!!!!!!!!!!', status);
+      }
+    }
 
     await SyncTxHelper.commit();
     return index;

src/service-worker/command/sync/sync-server.command.ts

@@ -19,6 +19,7 @@ import { ICommand } from '../../../common/model/shared/common.dto';
 import { SyncGetProgressCommand } from './progress/sync-get-progress.command';
 import { SyncMonthCommand } from './sync-month.command';
 import { SyncProgress } from './sync.model';
+import { SyncResetProgressCommand } from './progress/sync-reset-progress.command';
 import { SyncSetProgressCommand } from './progress/sync-set-progress.command';
 import { SyncTxHelper } from './sync-tx.helper';
 import { fnConsoleLog } from '../../../common/fn/fn-console';
@@ -31,7 +32,7 @@ export class SyncServerCommand implements ICommand<Promise<void>> {
     if (SyncServerCommand.isInSync) return;
     if (!(await SyncTxHelper.shouldSync())) return;
     try {
-      // await new SyncResetProgressCommand().execute();
+      await new SyncResetProgressCommand().execute();
 
       SyncServerCommand.isInSync = true;
 

src/service-worker/command/sync/sync-tx.helper.ts

@@ -66,8 +66,4 @@ export class SyncTxHelper {
     const value = await BrowserStorage.get<ObjDateIndex[] | undefined>(key);
     return value || [];
   }
-
-  static async setList(key: string, value: ObjDateIndex[]) {
-    await BrowserStorage.set(key, value);
-  }
 }