feat: add pako to compress segments

package-lock.json

@@ -28,6 +28,7 @@
         "marked": "^4.2.5",
         "nanoid": "^4.0.0",
         "openpgp": "^5.9.0",
+        "pako": "^2.1.0",
         "parse5": "^7.1.2",
         "pdfjs-dist": "^3.10.111",
         "prosemirror-commands": "^1.3.1",
@@ -54,6 +55,7 @@
         "@types/firefox-webext-browser": "^111.0.1",
         "@types/marked": "^4.0.8",
         "@types/node": "^18.16.18",
+        "@types/pako": "^2.0.0",
         "@types/react": "^18.0.15",
         "@types/react-dom": "^18.0.6",
         "@types/remove-markdown": "^0.3.1",
@@ -74,6 +76,7 @@
       }
     },
     "../browser-api": {
+      "name": "@pinmenote/browser-api",
       "version": "0.0.5",
       "license": "MIT",
       "devDependencies": {
@@ -17207,6 +17210,12 @@
       "integrity": "sha512-/aNaQZD0+iSBAGnvvN2Cx92HqE5sZCPZtx2TsK+4nvV23fFe09jVDvpArXr2j9DnYlzuU9WuoykDDc6wqvpNcw==",
       "dev": true
     },
+    "node_modules/@types/pako": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/@types/pako/-/pako-2.0.0.tgz",
+      "integrity": "sha512-10+iaz93qR5WYxTo+PMifD5TSxiOtdRaxBf7INGGXMQgTCu8Z/7GYWYFUOS3q/G0nE5boj1r4FEB+WSy7s5gbA==",
+      "dev": true
+    },
     "node_modules/@types/parse-json": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.0.tgz",
@@ -20670,6 +20679,11 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/pako": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/pako/-/pako-2.1.0.tgz",
+      "integrity": "sha512-w+eufiZ1WuJYgPXbV/PO3NCMEc3xqylkKHzp8bxp1uW4qaSNQUkwmLLEc3kKsfz8lpV1F8Ht3U1Cm+9Srog2ug=="
+    },
     "node_modules/parcel": {
       "version": "2.8.2",
       "resolved": "https://registry.npmjs.org/parcel/-/parcel-2.8.2.tgz",
@@ -33142,6 +33156,12 @@
       "integrity": "sha512-/aNaQZD0+iSBAGnvvN2Cx92HqE5sZCPZtx2TsK+4nvV23fFe09jVDvpArXr2j9DnYlzuU9WuoykDDc6wqvpNcw==",
       "dev": true
     },
+    "@types/pako": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/@types/pako/-/pako-2.0.0.tgz",
+      "integrity": "sha512-10+iaz93qR5WYxTo+PMifD5TSxiOtdRaxBf7INGGXMQgTCu8Z/7GYWYFUOS3q/G0nE5boj1r4FEB+WSy7s5gbA==",
+      "dev": true
+    },
     "@types/parse-json": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.0.tgz",
@@ -35585,6 +35605,11 @@
         "p-limit": "^3.0.2"
       }
     },
+    "pako": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/pako/-/pako-2.1.0.tgz",
+      "integrity": "sha512-w+eufiZ1WuJYgPXbV/PO3NCMEc3xqylkKHzp8bxp1uW4qaSNQUkwmLLEc3kKsfz8lpV1F8Ht3U1Cm+9Srog2ug=="
+    },
     "parcel": {
       "version": "2.8.2",
       "resolved": "https://registry.npmjs.org/parcel/-/parcel-2.8.2.tgz",

package.json

@@ -38,6 +38,7 @@
     "@types/firefox-webext-browser": "^111.0.1",
     "@types/marked": "^4.0.8",
     "@types/node": "^18.16.18",
+    "@types/pako": "^2.0.0",
     "@types/react": "^18.0.15",
     "@types/react-dom": "^18.0.6",
     "@types/remove-markdown": "^0.3.1",
@@ -76,6 +77,7 @@
     "marked": "^4.2.5",
     "nanoid": "^4.0.0",
     "openpgp": "^5.9.0",
+    "pako": "^2.1.0",
     "parse5": "^7.1.2",
     "pdfjs-dist": "^3.10.111",
     "prosemirror-commands": "^1.3.1",

src/common/fn/fn-byte-convert.ts

@@ -0,0 +1,9 @@
+export const fnByteToMb = (value?: number): number => {
+  if (!value) return 0;
+  return Math.floor(value / 10_000) / 100;
+};
+
+export const fnByteToGb = (value?: number): number => {
+  if (!value) return 0;
+  return Math.floor(value / 10_000_000) / 100;
+};

src/common/model/bus.model.ts

@@ -44,7 +44,7 @@ export enum BusMessageType {
   POPUP_PAGE_ELEMENT_SNAPSHOT_ADD = 'popup.page.element.snapshot.add',
   POPUP_PIN_START = 'popup.pin.start',
   POPUP_PAGE_ALTER_START = 'popup.page.alter.start',
-  POPUP_TAKE_SCREENSHOT = 'popup.take.screenshot',
+  POPUP_SERVER_QUOTA = 'popup.server.quota',
   // Content script
   CONTENT_DOWNLOAD_DATA = 'content.download',
   CONTENT_INVALIDATE = 'content.invalidate',

src/common/model/shared/token.dto.ts

@@ -22,6 +22,7 @@ export enum TokenTypeDto {
 export interface RefreshTokenDto {
   token: string;
   iat: number;
+  syncToken: string;
 }
 
 export interface AccessTokenDto {

src/common/model/sync-server.model.ts

@@ -14,15 +14,9 @@
  * You should have received a copy of the GNU General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-import { BrowserStorage } from '@pinmenote/browser-api';
+export interface ServerQuotaResponse {
-import { ICommand } from '../../model/shared/common.dto';
+  used: number;
-import { LogManager } from '../../popup/log.manager';
+  available: number;
-import { ObjectStoreKeys } from '../../keys/object.store.keys';
+  files: number;
-
+  documents: number;
-export class SyncClearServerCommand implements ICommand<Promise<void>> {
-  async execute(): Promise<void> {
-    // clear progress
-    await BrowserStorage.remove(ObjectStoreKeys.SYNC_PROGRESS);
-    LogManager.log(`SyncClearServerCommand->complete !!!`);
-  }
 }

src/default-popup/components/account/account-details.component.tsx

@@ -28,6 +28,8 @@ import { TokenDataDto } from '../../../common/model/shared/token.dto';
 import { TokenStorageRemoveCommand } from '../../../common/command/server/token/token-storage-remove.command';
 import Typography from '@mui/material/Typography';
 import jwtDecode from 'jwt-decode';
+import { ServerQuotaResponse } from '../../../common/model/sync-server.model';
+import { fnByteToGb } from '../../../common/fn/fn-byte-convert';
 
 interface Props {
   logoutSuccess: () => void;
@@ -35,22 +37,37 @@ interface Props {
 
 export const AccountDetailsComponent: FunctionComponent<Props> = ({ logoutSuccess }) => {
   const [tokenData, setTokenData] = useState<TokenDataDto | undefined>();
+  const [serverQuota, setServerQuota] = useState<ServerQuotaResponse>();
   const [responseError, setResponseError] = useState<ServerErrorDto | undefined>(undefined);
 
   useEffect(() => {
     LogManager.log(`AccountDetailsComponent init`);
+
+    const dispatcher = TinyDispatcher.getInstance();
+
     if (PopupTokenStore.token) {
       setTokenData(jwtDecode<TokenDataDto>(PopupTokenStore.token.access_token));
+      dispatcher.addListener<ServerQuotaResponse>(BusMessageType.POPUP_SERVER_QUOTA, (event, key, value) => {
+        LogManager.log(`${event} - ${JSON.stringify(value)}`);
+        dispatcher.removeListener(event, key);
+        setServerQuota(value);
+      });
+      BrowserApi.sendRuntimeMessage({ type: BusMessageType.POPUP_SERVER_QUOTA })
+        .then(() => {
+          /* */
+        })
+        .catch(() => {
+          /* */
+        });
     }
-    const loginSuccessKey = TinyDispatcher.getInstance().addListener(
+
-      BusMessageType.POPUP_LOGIN_SUCCESS,
+    const loginSuccessKey = dispatcher.addListener(BusMessageType.POPUP_LOGIN_SUCCESS, async (event, key) => {
-      async (event, key) => {
+      dispatcher.removeListener(event, key);
-        TinyDispatcher.getInstance().removeListener(event, key);
       await PopupTokenStore.init();
       if (PopupTokenStore.token) setTokenData(jwtDecode<TokenDataDto>(PopupTokenStore.token.access_token));
-      }
+    });
-    );
+
-    const logoutKey = TinyDispatcher.getInstance().addListener<FetchResponse<BoolDto | ServerErrorDto>>(
+    const logoutKey = dispatcher.addListener<FetchResponse<BoolDto | ServerErrorDto>>(
       BusMessageType.POPUP_LOGOUT,
       async (event, key, value) => {
         LogManager.log('POPUP_LOGOUT_RESPONSE');
@@ -63,8 +80,8 @@ export const AccountDetailsComponent: FunctionComponent<Props> = ({ logoutSucces
       }
     );
     return () => {
-      TinyDispatcher.getInstance().removeListener(BusMessageType.POPUP_LOGIN_SUCCESS, loginSuccessKey);
+      dispatcher.removeListener(BusMessageType.POPUP_LOGIN_SUCCESS, loginSuccessKey);
-      TinyDispatcher.getInstance().removeListener(BusMessageType.POPUP_LOGOUT, logoutKey);
+      dispatcher.removeListener(BusMessageType.POPUP_LOGOUT, logoutKey);
     };
   }, []);
 
@@ -76,10 +93,26 @@ export const AccountDetailsComponent: FunctionComponent<Props> = ({ logoutSucces
   };
 
   return (
+    <div>
+      <div>
         <div>
           <Typography align="center" fontSize="1.5em" fontWeight="bold">
             Welcome {tokenData?.data.username}
           </Typography>
+        </div>
+        <div style={{ margin: 10 }}>
+          <Typography fontSize="1.2em" fontWeight="bold">
+            Account statistics{' '}
+          </Typography>
+          <Typography style={{ marginTop: 5 }} fontSize="1.2em">
+            {fnByteToGb(serverQuota?.used)} GB of {fnByteToGb(serverQuota?.available)} GB disk space used
+          </Typography>
+          <Typography style={{ marginTop: 5 }} fontSize="1.2em">
+            {serverQuota?.files} files and {serverQuota?.documents} documents archived
+          </Typography>
+        </div>
+      </div>
+
       <div style={{ position: 'absolute', bottom: 0, width: 300 }}>
         <div style={{ margin: 10 }}>
           <Typography style={{ fontSize: '8pt', color: COLOR_DEFAULT_RED }}>

src/default-popup/components/account/login.component.tsx

@@ -105,7 +105,7 @@ export const LoginComponent: FunctionComponent<Props> = ({ loginSuccess }) => {
       </div>
       <Typography align="center" style={{ marginTop: 20 }}>
         <Link target="_blank" href={getWebsiteUrl('/register')}>
-          Buy <b>Premium</b> Account
+          Subscribe to <b>Premium</b> Account for 10$ / year
         </Link>
       </Typography>
       <div style={{ margin: 10 }}>

src/default-popup/components/tabs/logs-tab.component.tsx

@@ -19,7 +19,6 @@ import { BrowserApi } from '@pinmenote/browser-api';
 import { BusMessageType } from '../../../common/model/bus.model';
 import Button from '@mui/material/Button';
 import { LogManager } from '../../../common/popup/log.manager';
-import { SyncClearServerCommand } from '../../../common/command/sync/sync-clear-server.command';
 import { TinyDispatcher } from '@pinmenote/tiny-dispatcher';
 import Typography from '@mui/material/Typography';
 
@@ -46,10 +45,6 @@ export const LogsTabComponent: FunctionComponent = () => {
   const handleClearLogs = () => {
     LogManager.clear();
   };
-
-  const handleClearServer = async () => {
-    await new SyncClearServerCommand().execute();
-  };
   return (
     <div style={{ height: '100%', margin: 5 }}>
       <Typography fontSize="2em">Debug</Typography>
@@ -63,11 +58,6 @@ export const LogsTabComponent: FunctionComponent = () => {
           Clear logs
         </Button>
       </div>
-      <div style={{ margin: 10 }}>
-        <Button sx={{ width: '100%' }} variant="outlined" onClick={handleClearServer}>
-          Clear server
-        </Button>
-      </div>
       <Typography fontSize="1.5em" fontWeight="bold">
         Logs reverse
       </Typography>

src/default-popup/store/popup-token.store.ts

@@ -23,7 +23,6 @@ export class PopupTokenStore {
 
   static init = async () => {
     this.tokenValue = await new TokenStorageGetCommand().execute();
-    LogManager.log(`PopupTokenStore->init ${JSON.stringify(this.tokenValue)}`);
   };
 
   static get token(): AccessTokenDto | undefined {

src/manifest.ch.json

@@ -2,7 +2,7 @@
   "manifest_version": 3,
   "name": "pinmenote",
   "short_name": "pinmenote",
-  "description": "Pin note on website, modify, draw, comment and archive content.",
+  "description": "Pin note, modify, draw, comment and archive websites.",
   "version": "0.0.1",
   "icons": {
     "16": "assets/icon/16.png",

src/manifest.ff.json

@@ -7,7 +7,7 @@
   },
   "name": "pinmenote",
   "short_name": "pinmenote",
-  "description": "Pin note on website, modify, draw, comment and archive content.",
+  "description": "Pin note, modify, draw, comment and archive websites.",
   "homepage_url": "https://pinmenote.com",
   "version": "0.0.1",
   "icons": {

src/manifest.json

@@ -2,7 +2,7 @@
   "manifest_version": 3,
   "name": "pinmenote-dev",
   "short_name": "pinmenote-dev",
-  "description": "Pin note on website, modify, draw, comment and archive content.",
+  "description": "Pin note, modify, draw, comment and archive websites.",
   "version": "0.0.1",
   "icons": {
     "16": "assets/icon/16.png",

src/service-worker/command/api/store/segment/api-segment-add.command.ts

@@ -19,7 +19,7 @@ import { BeginTxResponse } from '../api-store.model';
 import { FetchService } from '@pinmenote/fetch-service';
 import { ICommand } from '../../../../../common/model/shared/common.dto';
 import { SyncHashType } from '../../../sync/sync.model';
-import { fnConsoleLog } from '../../../../../common/fn/fn-console';
+import { deflate } from 'pako';
 
 export interface FileDataDto {
   parent?: string;
@@ -57,9 +57,14 @@ export class ApiSegmentAddCommand extends ApiCallBase implements ICommand<Promis
 
   async addSegment(): Promise<boolean> {
     const formData = new FormData();
-    let fileData = this.file;
+    if (this.file instanceof Blob) {
-    if (!(this.file instanceof Blob)) fileData = new Blob([this.file], { type: 'application/json' });
+      formData.append('file', this.file);
-    formData.append('file', fileData);
+    } else {
+      const fileData = deflate(this.file);
+      formData.append('file', new Blob([fileData], { type: 'application/zip' }));
+      console.log(`compression ${Math.round((fileData.length / this.file.length) * 100)}%`);
+    }
+
     if (this.data.parent) formData.append('parent', this.data.parent);
     formData.append('key', this.data.key);
     formData.append('hash', this.data.hash);

src/service-worker/command/api/store/segment/api-segment-quota-get.command.ts

@@ -0,0 +1,38 @@
+/*
+ * This file is part of the pinmenote-extension distribution (https://github.com/pinmenote/pinmenote-extension).
+ * Copyright (c) 2023 Michal Szczepanski.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+import { ApiCallBase } from '../../api-call.base';
+import { FetchService } from '@pinmenote/fetch-service';
+import { ICommand } from '../../../../../common/model/shared/common.dto';
+import { ServerQuotaResponse } from '../../../../../common/model/sync-server.model';
+
+export class ApiSegmentQuotaGetCommand extends ApiCallBase implements ICommand<Promise<ServerQuotaResponse>> {
+  constructor() {
+    super();
+  }
+  async execute(): Promise<ServerQuotaResponse> {
+    await this.initTokenData();
+    const resp = await FetchService.fetch<ServerQuotaResponse>(
+      `${this.storeUrl!}/api/v1/segment/quota`,
+      {
+        type: 'JSON',
+        headers: this.getAuthHeaders(true)
+      },
+      this.refreshParams()
+    );
+    return resp.data;
+  }
+}

src/service-worker/command/popup/popup-server-quota.command.ts

@@ -14,20 +14,14 @@
  * You should have received a copy of the GNU General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
+import { ApiSegmentQuotaGetCommand, ServerQuotaResponse } from '../api/store/segment/api-segment-quota-get.command';
 import { BrowserApi } from '@pinmenote/browser-api';
-import { BrowserStorage } from '@pinmenote/browser-api';
 import { BusMessageType } from '../../../common/model/bus.model';
 import { ICommand } from '../../../common/model/shared/common.dto';
-import { ObjectStoreKeys } from '../../../common/keys/object.store.keys';
-import { SettingsConfig } from '../../../common/environment';
 
-export class PopupTakeScreenshotCommand implements ICommand<Promise<void>> {
+export class PopupServerQuotaCommand implements ICommand<Promise<void>> {
   async execute(): Promise<void> {
-    const settings = await BrowserStorage.get<SettingsConfig>(ObjectStoreKeys.CONTENT_SETTINGS_KEY);
+    const data = await new ApiSegmentQuotaGetCommand().execute();
-    const data = await BrowserApi.tabs.captureVisibleTab({
+    await BrowserApi.sendRuntimeMessage<ServerQuotaResponse>({ type: BusMessageType.POPUP_SERVER_QUOTA, data });
-      format: settings.screenshotFormat,
-      quality: settings.screenshotQuality
-    });
-    await BrowserApi.sendRuntimeMessage<string>({ type: BusMessageType.POPUP_TAKE_SCREENSHOT, data });
   }
 }

src/service-worker/command/sync/progress/sync-reset-progress.command.ts

@@ -37,8 +37,8 @@ export class SyncResetProgressCommand implements ICommand<Promise<void>> {
 
   async resetObjects(): Promise<void> {
     let listId = await BrowserStorage.get<number>(ObjectStoreKeys.OBJECT_LIST_ID);
-    const a = Date.now();
     console.log('SyncResetProgressCommand->start !!!!', listId);
+    const a = Date.now();
 
     const toSortSet: Set<string> = new Set<string>();
 

src/service-worker/command/sync/sync-index.command.ts

@@ -52,6 +52,9 @@ export class SyncIndexCommand implements ICommand<Promise<SyncObjectStatus>> {
       return SyncObjectStatus.OBJECT_NOT_EXISTS;
     }
     let status = SyncObjectStatus.OK;
+    // Skip for now for those with index
+    if (obj.server?.id) return status;
+
     switch (obj.type) {
       case ObjTypeDto.PageSnapshot:
       case ObjTypeDto.PageElementSnapshot: {

src/service-worker/command/sync/sync-tx.helper.ts

@@ -22,14 +22,31 @@ import { ObjDateIndex } from '../../../common/command/obj/index/obj-update-index
 import { ObjectStoreKeys } from '../../../common/keys/object.store.keys';
 import { TokenStorageGetCommand } from '../../../common/command/server/token/token-storage-get.command';
 import { fnConsoleLog } from '../../../common/fn/fn-console';
+import jwtDecode from 'jwt-decode';
+import { TokenDataDto } from '../../../common/model/shared/token.dto';
 
 export class SyncTxHelper {
   static async begin(): Promise<BeginTxResponse | undefined> {
     const tx = await BrowserStorage.get<BeginTxResponse | undefined>(ObjectStoreKeys.SYNC_TX);
     if (tx) return tx;
     const txResponse = await new ApiStoreBeginCommand().execute();
-    fnConsoleLog('locked', txResponse?.locked);
+    if (txResponse?.locked) {
-    if (txResponse?.locked) return undefined;
+      const token = await new TokenStorageGetCommand().execute();
+      if (!token) return undefined;
+      const tokenData = jwtDecode<TokenDataDto>(token.access_token);
+      fnConsoleLog(
+        'locked',
+        txResponse?.locked,
+        txResponse,
+        tokenData,
+        tokenData.refresh_token.syncToken === txResponse.lockedBy
+      );
+      if (tokenData.refresh_token.syncToken === txResponse.lockedBy) {
+        await BrowserStorage.set(ObjectStoreKeys.SYNC_TX, txResponse);
+        return txResponse;
+      }
+      return undefined;
+    }
     await BrowserStorage.set(ObjectStoreKeys.SYNC_TX, txResponse);
     return txResponse;
   }

src/service-worker/command/sync/sync.model.ts

@@ -21,8 +21,8 @@ export interface SyncProgress {
 }
 
 export enum SyncObjectStatus {
-  TX_LOCKED,
+  TX_LOCKED = -4,
-  SERVER_ERROR = -3,
+  SERVER_ERROR,
   INDEX_NOT_EXISTS,
   OBJECT_NOT_EXISTS,
   OK,

src/service-worker/service-worker.ts

@@ -30,7 +30,7 @@ import { PopupBugReportCommand } from './command/popup/popup-bug-report.command'
 import { PopupLoginCommand } from './command/popup/popup-login.command';
 import { PopupLoginSuccessCommand } from './command/popup/popup-login-success.command';
 import { PopupLogoutCommand } from './command/popup/popup-logout.command';
-import { PopupTakeScreenshotCommand } from './command/popup/popup-take-screenshot.command';
+import { PopupServerQuotaCommand } from './command/popup/popup-server-quota.command';
 import { PopupVerify2faCommand } from './command/popup/popup-verify-2fa.command';
 import { ScriptService } from './service/script.service';
 import { SwInitSettingsCommand } from './command/sw/sw-init-settings.command';
@@ -90,8 +90,8 @@ const handleMessage = async (
     case BusMessageType.POPUP_LOGOUT:
       await new PopupLogoutCommand().execute();
       break;
-    case BusMessageType.POPUP_TAKE_SCREENSHOT:
+    case BusMessageType.POPUP_SERVER_QUOTA:
-      await new PopupTakeScreenshotCommand().execute();
+      await new PopupServerQuotaCommand().execute();
       break;
     case BusMessageType.IFRAME_INDEX:
     case BusMessageType.IFRAME_INDEX_REGISTER: