pinmenote.com
/
pinmenote-extension
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Projects Releases Wiki Activity

feat: add pako to compress segments

This commit is contained in:
Michal Szczepanski 2023-09-21 00:46:17 +02:00
parent 71515b7995
commit df249679d0
21 changed files with 173 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
package-lock.json generated
View File

@ -28,6 +28,7 @@
"marked": "^4.2.5",
"nanoid": "^4.0.0",
"openpgp": "^5.9.0",
"pako": "^2.1.0",
"parse5": "^7.1.2",
"pdfjs-dist": "^3.10.111",
"prosemirror-commands": "^1.3.1",
@ -54,6 +55,7 @@
"@types/firefox-webext-browser": "^111.0.1",
"@types/marked": "^4.0.8",
"@types/node": "^18.16.18",
"@types/pako": "^2.0.0",
"@types/react": "^18.0.15",
"@types/react-dom": "^18.0.6",
"@types/remove-markdown": "^0.3.1",
@ -74,6 +76,7 @@
}
},
"../browser-api": {
"name": "@pinmenote/browser-api",
"version": "0.0.5",
"license": "MIT",
"devDependencies": {
@ -17207,6 +17210,12 @@
"integrity": "sha512-/aNaQZD0+iSBAGnvvN2Cx92HqE5sZCPZtx2TsK+4nvV23fFe09jVDvpArXr2j9DnYlzuU9WuoykDDc6wqvpNcw==",
"dev": true
},
"node_modules/@types/pako": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/@types/pako/-/pako-2.0.0.tgz",
"integrity": "sha512-10+iaz93qR5WYxTo+PMifD5TSxiOtdRaxBf7INGGXMQgTCu8Z/7GYWYFUOS3q/G0nE5boj1r4FEB+WSy7s5gbA==",
"dev": true
},
"node_modules/@types/parse-json": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.0.tgz",
@ -20670,6 +20679,11 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/pako": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/pako/-/pako-2.1.0.tgz",
"integrity": "sha512-w+eufiZ1WuJYgPXbV/PO3NCMEc3xqylkKHzp8bxp1uW4qaSNQUkwmLLEc3kKsfz8lpV1F8Ht3U1Cm+9Srog2ug=="
},
"node_modules/parcel": {
"version": "2.8.2",
"resolved": "https://registry.npmjs.org/parcel/-/parcel-2.8.2.tgz",
@ -33142,6 +33156,12 @@
"integrity": "sha512-/aNaQZD0+iSBAGnvvN2Cx92HqE5sZCPZtx2TsK+4nvV23fFe09jVDvpArXr2j9DnYlzuU9WuoykDDc6wqvpNcw==",
"dev": true
},
"@types/pako": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/@types/pako/-/pako-2.0.0.tgz",
"integrity": "sha512-10+iaz93qR5WYxTo+PMifD5TSxiOtdRaxBf7INGGXMQgTCu8Z/7GYWYFUOS3q/G0nE5boj1r4FEB+WSy7s5gbA==",
"dev": true
},
"@types/parse-json": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.0.tgz",
@ -35585,6 +35605,11 @@
"p-limit": "^3.0.2"
}
},
"pako": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/pako/-/pako-2.1.0.tgz",
"integrity": "sha512-w+eufiZ1WuJYgPXbV/PO3NCMEc3xqylkKHzp8bxp1uW4qaSNQUkwmLLEc3kKsfz8lpV1F8Ht3U1Cm+9Srog2ug=="
},
"parcel": {
"version": "2.8.2",
"resolved": "https://registry.npmjs.org/parcel/-/parcel-2.8.2.tgz",

2
package.json
View File

@ -38,6 +38,7 @@
"@types/firefox-webext-browser": "^111.0.1",
"@types/marked": "^4.0.8",
"@types/node": "^18.16.18",
"@types/pako": "^2.0.0",
"@types/react": "^18.0.15",
"@types/react-dom": "^18.0.6",
"@types/remove-markdown": "^0.3.1",
@ -76,6 +77,7 @@
"marked": "^4.2.5",
"nanoid": "^4.0.0",
"openpgp": "^5.9.0",
"pako": "^2.1.0",
"parse5": "^7.1.2",
"pdfjs-dist": "^3.10.111",
"prosemirror-commands": "^1.3.1",

9
src/common/fn/fn-byte-convert.ts Normal file
View File

@ -0,0 +1,9 @@
export const fnByteToMb = (value?: number): number => {
if (!value) return 0;
return Math.floor(value / 10_000) / 100;
};
export const fnByteToGb = (value?: number): number => {
if (!value) return 0;
return Math.floor(value / 10_000_000) / 100;
};

2
src/common/model/bus.model.ts
View File

@ -44,7 +44,7 @@ export enum BusMessageType {
POPUP_PAGE_ELEMENT_SNAPSHOT_ADD = 'popup.page.element.snapshot.add',
POPUP_PIN_START = 'popup.pin.start',
POPUP_PAGE_ALTER_START = 'popup.page.alter.start',
POPUP_TAKE_SCREENSHOT = 'popup.take.screenshot',
POPUP_SERVER_QUOTA = 'popup.server.quota',
// Content script
CONTENT_DOWNLOAD_DATA = 'content.download',
CONTENT_INVALIDATE = 'content.invalidate',

1
src/common/model/shared/token.dto.ts
View File

@ -22,6 +22,7 @@ export enum TokenTypeDto {
export interface RefreshTokenDto {
token: string;
iat: number;
syncToken: string;
}
export interface AccessTokenDto {

16
src/common/command/sync/sync-clear-server.command.ts → src/common/model/sync-server.model.ts
View File

@ -14,15 +14,9 @@
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { BrowserStorage } from '@pinmenote/browser-api';
import { ICommand } from '../../model/shared/common.dto';
import { LogManager } from '../../popup/log.manager';
import { ObjectStoreKeys } from '../../keys/object.store.keys';
export class SyncClearServerCommand implements ICommand<Promise<void>> {
async execute(): Promise<void> {
// clear progress
await BrowserStorage.remove(ObjectStoreKeys.SYNC_PROGRESS);
LogManager.log(`SyncClearServerCommand->complete !!!`);
}
export interface ServerQuotaResponse {
used: number;
available: number;
files: number;
documents: number;
}

61
src/default-popup/components/account/account-details.component.tsx
View File

@ -28,6 +28,8 @@ import { TokenDataDto } from '../../../common/model/shared/token.dto';
import { TokenStorageRemoveCommand } from '../../../common/command/server/token/token-storage-remove.command';
import Typography from '@mui/material/Typography';
import jwtDecode from 'jwt-decode';
import { ServerQuotaResponse } from '../../../common/model/sync-server.model';
import { fnByteToGb } from '../../../common/fn/fn-byte-convert';
interface Props {
logoutSuccess: () => void;
@ -35,22 +37,37 @@ interface Props {
export const AccountDetailsComponent: FunctionComponent<Props> = ({ logoutSuccess }) => {
const [tokenData, setTokenData] = useState<TokenDataDto | undefined>();
const [serverQuota, setServerQuota] = useState<ServerQuotaResponse>();
const [responseError, setResponseError] = useState<ServerErrorDto | undefined>(undefined);
useEffect(() => {
LogManager.log(`AccountDetailsComponent init`);
const dispatcher = TinyDispatcher.getInstance();
if (PopupTokenStore.token) {
setTokenData(jwtDecode<TokenDataDto>(PopupTokenStore.token.access_token));
dispatcher.addListener<ServerQuotaResponse>(BusMessageType.POPUP_SERVER_QUOTA, (event, key, value) => {
LogManager.log(`${event} - ${JSON.stringify(value)}`);
dispatcher.removeListener(event, key);
setServerQuota(value);
});
BrowserApi.sendRuntimeMessage({ type: BusMessageType.POPUP_SERVER_QUOTA })
.then(() => {
/* */
})
.catch(() => {
/* */
});
}
const loginSuccessKey = TinyDispatcher.getInstance().addListener(
BusMessageType.POPUP_LOGIN_SUCCESS,
async (event, key) => {
TinyDispatcher.getInstance().removeListener(event, key);
await PopupTokenStore.init();
if (PopupTokenStore.token) setTokenData(jwtDecode<TokenDataDto>(PopupTokenStore.token.access_token));
}
);
const logoutKey = TinyDispatcher.getInstance().addListener<FetchResponse<BoolDto | ServerErrorDto>>(
const loginSuccessKey = dispatcher.addListener(BusMessageType.POPUP_LOGIN_SUCCESS, async (event, key) => {
dispatcher.removeListener(event, key);
await PopupTokenStore.init();
if (PopupTokenStore.token) setTokenData(jwtDecode<TokenDataDto>(PopupTokenStore.token.access_token));
});
const logoutKey = dispatcher.addListener<FetchResponse<BoolDto | ServerErrorDto>>(
BusMessageType.POPUP_LOGOUT,
async (event, key, value) => {
LogManager.log('POPUP_LOGOUT_RESPONSE');
@ -63,8 +80,8 @@ export const AccountDetailsComponent: FunctionComponent<Props> = ({ logoutSucces
}
);
return () => {
TinyDispatcher.getInstance().removeListener(BusMessageType.POPUP_LOGIN_SUCCESS, loginSuccessKey);
TinyDispatcher.getInstance().removeListener(BusMessageType.POPUP_LOGOUT, logoutKey);
dispatcher.removeListener(BusMessageType.POPUP_LOGIN_SUCCESS, loginSuccessKey);
dispatcher.removeListener(BusMessageType.POPUP_LOGOUT, logoutKey);
};
}, []);
@ -77,9 +94,25 @@ export const AccountDetailsComponent: FunctionComponent<Props> = ({ logoutSucces
return (
<div>
<Typography align="center" fontSize="1.5em" fontWeight="bold">
Welcome {tokenData?.data.username}
</Typography>
<div>
<div>
<Typography align="center" fontSize="1.5em" fontWeight="bold">
Welcome {tokenData?.data.username}
</Typography>
</div>
<div style={{ margin: 10 }}>
<Typography fontSize="1.2em" fontWeight="bold">
Account statistics{' '}
</Typography>
<Typography style={{ marginTop: 5 }} fontSize="1.2em">
{fnByteToGb(serverQuota?.used)} GB of {fnByteToGb(serverQuota?.available)} GB disk space used
</Typography>
<Typography style={{ marginTop: 5 }} fontSize="1.2em">
{serverQuota?.files} files and {serverQuota?.documents} documents archived
</Typography>
</div>
</div>
<div style={{ position: 'absolute', bottom: 0, width: 300 }}>
<div style={{ margin: 10 }}>
<Typography style={{ fontSize: '8pt', color: COLOR_DEFAULT_RED }}>

2
src/default-popup/components/account/login.component.tsx
View File

@ -105,7 +105,7 @@ export const LoginComponent: FunctionComponent<Props> = ({ loginSuccess }) => {
</div>
<Typography align="center" style={{ marginTop: 20 }}>
<Link target="_blank" href={getWebsiteUrl('/register')}>
Buy <b>Premium</b> Account
Subscribe to <b>Premium</b> Account for 10$ / year
</Link>
</Typography>
<div style={{ margin: 10 }}>

10
src/default-popup/components/tabs/logs-tab.component.tsx
View File

@ -19,7 +19,6 @@ import { BrowserApi } from '@pinmenote/browser-api';
import { BusMessageType } from '../../../common/model/bus.model';
import Button from '@mui/material/Button';
import { LogManager } from '../../../common/popup/log.manager';
import { SyncClearServerCommand } from '../../../common/command/sync/sync-clear-server.command';
import { TinyDispatcher } from '@pinmenote/tiny-dispatcher';
import Typography from '@mui/material/Typography';
@ -46,10 +45,6 @@ export const LogsTabComponent: FunctionComponent = () => {
const handleClearLogs = () => {
LogManager.clear();
};
const handleClearServer = async () => {
await new SyncClearServerCommand().execute();
};
return (
<div style={{ height: '100%', margin: 5 }}>
<Typography fontSize="2em">Debug</Typography>
@ -63,11 +58,6 @@ export const LogsTabComponent: FunctionComponent = () => {
Clear logs
</Button>
</div>
<div style={{ margin: 10 }}>
<Button sx={{ width: '100%' }} variant="outlined" onClick={handleClearServer}>
Clear server
</Button>
</div>
<Typography fontSize="1.5em" fontWeight="bold">
Logs reverse
</Typography>

1
src/default-popup/store/popup-token.store.ts
View File

@ -23,7 +23,6 @@ export class PopupTokenStore {
static init = async () => {
this.tokenValue = await new TokenStorageGetCommand().execute();
LogManager.log(`PopupTokenStore->init ${JSON.stringify(this.tokenValue)}`);
};
static get token(): AccessTokenDto | undefined {

2
src/manifest.ch.json
View File

@ -2,7 +2,7 @@
"manifest_version": 3,
"name": "pinmenote",
"short_name": "pinmenote",
"description": "Pin note on website, modify, draw, comment and archive content.",
"description": "Pin note, modify, draw, comment and archive websites.",
"version": "0.0.1",
"icons": {
"16": "assets/icon/16.png",

2
src/manifest.ff.json
View File

@ -7,7 +7,7 @@
},
"name": "pinmenote",
"short_name": "pinmenote",
"description": "Pin note on website, modify, draw, comment and archive content.",
"description": "Pin note, modify, draw, comment and archive websites.",
"homepage_url": "https://pinmenote.com",
"version": "0.0.1",
"icons": {

2
src/manifest.json
View File

@ -2,7 +2,7 @@
"manifest_version": 3,
"name": "pinmenote-dev",
"short_name": "pinmenote-dev",
"description": "Pin note on website, modify, draw, comment and archive content.",
"description": "Pin note, modify, draw, comment and archive websites.",
"version": "0.0.1",
"icons": {
"16": "assets/icon/16.png",

13
src/service-worker/command/api/store/segment/api-segment-add.command.ts
View File

@ -19,7 +19,7 @@ import { BeginTxResponse } from '../api-store.model';
import { FetchService } from '@pinmenote/fetch-service';
import { ICommand } from '../../../../../common/model/shared/common.dto';
import { SyncHashType } from '../../../sync/sync.model';
import { fnConsoleLog } from '../../../../../common/fn/fn-console';
import { deflate } from 'pako';
export interface FileDataDto {
parent?: string;
@ -57,9 +57,14 @@ export class ApiSegmentAddCommand extends ApiCallBase implements ICommand<Promis
async addSegment(): Promise<boolean> {
const formData = new FormData();
let fileData = this.file;
if (!(this.file instanceof Blob)) fileData = new Blob([this.file], { type: 'application/json' });
formData.append('file', fileData);
if (this.file instanceof Blob) {
formData.append('file', this.file);
} else {
const fileData = deflate(this.file);
formData.append('file', new Blob([fileData], { type: 'application/zip' }));
console.log(`compression ${Math.round((fileData.length / this.file.length) * 100)}%`);
}
if (this.data.parent) formData.append('parent', this.data.parent);
formData.append('key', this.data.key);
formData.append('hash', this.data.hash);

38
src/service-worker/command/api/store/segment/api-segment-quota-get.command.ts Normal file
View File

@ -0,0 +1,38 @@
/*
* This file is part of the pinmenote-extension distribution (https://github.com/pinmenote/pinmenote-extension).
* Copyright (c) 2023 Michal Szczepanski.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { ApiCallBase } from '../../api-call.base';
import { FetchService } from '@pinmenote/fetch-service';
import { ICommand } from '../../../../../common/model/shared/common.dto';
import { ServerQuotaResponse } from '../../../../../common/model/sync-server.model';
export class ApiSegmentQuotaGetCommand extends ApiCallBase implements ICommand<Promise<ServerQuotaResponse>> {
constructor() {
super();
}
async execute(): Promise<ServerQuotaResponse> {
await this.initTokenData();
const resp = await FetchService.fetch<ServerQuotaResponse>(
`${this.storeUrl!}/api/v1/segment/quota`,
{
type: 'JSON',
headers: this.getAuthHeaders(true)
},
this.refreshParams()
);
return resp.data;
}
}

14
src/service-worker/command/popup/popup-take-screenshot.command.ts → src/service-worker/command/popup/popup-server-quota.command.ts
View File

@ -14,20 +14,14 @@
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { ApiSegmentQuotaGetCommand, ServerQuotaResponse } from '../api/store/segment/api-segment-quota-get.command';
import { BrowserApi } from '@pinmenote/browser-api';
import { BrowserStorage } from '@pinmenote/browser-api';
import { BusMessageType } from '../../../common/model/bus.model';
import { ICommand } from '../../../common/model/shared/common.dto';
import { ObjectStoreKeys } from '../../../common/keys/object.store.keys';
import { SettingsConfig } from '../../../common/environment';
export class PopupTakeScreenshotCommand implements ICommand<Promise<void>> {
export class PopupServerQuotaCommand implements ICommand<Promise<void>> {
async execute(): Promise<void> {
const settings = await BrowserStorage.get<SettingsConfig>(ObjectStoreKeys.CONTENT_SETTINGS_KEY);
const data = await BrowserApi.tabs.captureVisibleTab({
format: settings.screenshotFormat,
quality: settings.screenshotQuality
});
await BrowserApi.sendRuntimeMessage<string>({ type: BusMessageType.POPUP_TAKE_SCREENSHOT, data });
const data = await new ApiSegmentQuotaGetCommand().execute();
await BrowserApi.sendRuntimeMessage<ServerQuotaResponse>({ type: BusMessageType.POPUP_SERVER_QUOTA, data });
}
}

2
src/service-worker/command/sync/progress/sync-reset-progress.command.ts
View File

@ -37,8 +37,8 @@ export class SyncResetProgressCommand implements ICommand<Promise<void>> {
async resetObjects(): Promise<void> {
let listId = await BrowserStorage.get<number>(ObjectStoreKeys.OBJECT_LIST_ID);
const a = Date.now();
console.log('SyncResetProgressCommand->start !!!!', listId);
const a = Date.now();
const toSortSet: Set<string> = new Set<string>();

3
src/service-worker/command/sync/sync-index.command.ts
View File

@ -52,6 +52,9 @@ export class SyncIndexCommand implements ICommand<Promise<SyncObjectStatus>> {
return SyncObjectStatus.OBJECT_NOT_EXISTS;
}
let status = SyncObjectStatus.OK;
// Skip for now for those with index
if (obj.server?.id) return status;
switch (obj.type) {
case ObjTypeDto.PageSnapshot:
case ObjTypeDto.PageElementSnapshot: {

21
src/service-worker/command/sync/sync-tx.helper.ts
View File

@ -22,14 +22,31 @@ import { ObjDateIndex } from '../../../common/command/obj/index/obj-update-index
import { ObjectStoreKeys } from '../../../common/keys/object.store.keys';
import { TokenStorageGetCommand } from '../../../common/command/server/token/token-storage-get.command';
import { fnConsoleLog } from '../../../common/fn/fn-console';
import jwtDecode from 'jwt-decode';
import { TokenDataDto } from '../../../common/model/shared/token.dto';
export class SyncTxHelper {
static async begin(): Promise<BeginTxResponse | undefined> {
const tx = await BrowserStorage.get<BeginTxResponse | undefined>(ObjectStoreKeys.SYNC_TX);
if (tx) return tx;
const txResponse = await new ApiStoreBeginCommand().execute();
fnConsoleLog('locked', txResponse?.locked);
if (txResponse?.locked) return undefined;
if (txResponse?.locked) {
const token = await new TokenStorageGetCommand().execute();
if (!token) return undefined;
const tokenData = jwtDecode<TokenDataDto>(token.access_token);
fnConsoleLog(
'locked',
txResponse?.locked,
txResponse,
tokenData,
tokenData.refresh_token.syncToken === txResponse.lockedBy
);
if (tokenData.refresh_token.syncToken === txResponse.lockedBy) {
await BrowserStorage.set(ObjectStoreKeys.SYNC_TX, txResponse);
return txResponse;
}
return undefined;
}
await BrowserStorage.set(ObjectStoreKeys.SYNC_TX, txResponse);
return txResponse;
}

4
src/service-worker/command/sync/sync.model.ts
View File

@ -21,8 +21,8 @@ export interface SyncProgress {
}
export enum SyncObjectStatus {
TX_LOCKED,
SERVER_ERROR = -3,
TX_LOCKED = -4,
SERVER_ERROR,
INDEX_NOT_EXISTS,
OBJECT_NOT_EXISTS,
OK,

6
src/service-worker/service-worker.ts
View File

@ -30,7 +30,7 @@ import { PopupBugReportCommand } from './command/popup/popup-bug-report.command'
import { PopupLoginCommand } from './command/popup/popup-login.command';
import { PopupLoginSuccessCommand } from './command/popup/popup-login-success.command';
import { PopupLogoutCommand } from './command/popup/popup-logout.command';
import { PopupTakeScreenshotCommand } from './command/popup/popup-take-screenshot.command';
import { PopupServerQuotaCommand } from './command/popup/popup-server-quota.command';
import { PopupVerify2faCommand } from './command/popup/popup-verify-2fa.command';
import { ScriptService } from './service/script.service';
import { SwInitSettingsCommand } from './command/sw/sw-init-settings.command';
@ -90,8 +90,8 @@ const handleMessage = async (
case BusMessageType.POPUP_LOGOUT:
await new PopupLogoutCommand().execute();
break;
case BusMessageType.POPUP_TAKE_SCREENSHOT:
await new PopupTakeScreenshotCommand().execute();
case BusMessageType.POPUP_SERVER_QUOTA:
await new PopupServerQuotaCommand().execute();
break;
case BusMessageType.IFRAME_INDEX:
case BusMessageType.IFRAME_INDEX_REGISTER: