From 2710720461aa77916cf63b03f6afa7c97bf759d6 Mon Sep 17 00:00:00 2001 From: ascariandrea Date: Fri, 16 Sep 2022 18:37:23 +0200 Subject: [PATCH] fix(shared): added metadata logger box to extension --- packages/shared/package.json | 1 + .../shared/src/arbitraries/Supporter.arb.ts | 14 + .../contributions/ContributionEvent.arb.ts | 32 ++ packages/shared/src/extension/app.ts | 27 +- .../shared/src/extension/tooltip/index.tsx | 23 - .../ui/components/MetadataLogger.tsx | 477 ++++++++++++++++++ .../{tooltip => ui}/components/tooltip.tsx | 0 packages/shared/src/extension/ui/index.tsx | 69 +++ platforms/storybook/.storybook/main.js | 1 + .../src/stories/MetadataLogger.stories.tsx | 93 ++++ platforms/storybook/tsconfig.json | 12 +- platforms/tktrex/extension/src/app/index.ts | 4 + .../shared/src/parser/metadata-logger.ts | 64 +++ .../tktrex/shared/src/parser/parsers/index.ts | 5 +- platforms/yttrex/extension/src/app/index.ts | 4 + platforms/yttrex/extension/tsconfig.json | 2 +- .../shared/src/parser/metadata-logger.ts | 66 +++ platforms/yttrex/shared/tsconfig.json | 1 + yarn.lock | 1 + 19 files changed, 860 insertions(+), 36 deletions(-) create mode 100644 packages/shared/src/arbitraries/Supporter.arb.ts create mode 100644 packages/shared/src/arbitraries/contributions/ContributionEvent.arb.ts delete mode 100644 packages/shared/src/extension/tooltip/index.tsx create mode 100644 packages/shared/src/extension/ui/components/MetadataLogger.tsx rename packages/shared/src/extension/{tooltip => ui}/components/tooltip.tsx (100%) create mode 100644 packages/shared/src/extension/ui/index.tsx create mode 100644 platforms/storybook/src/stories/MetadataLogger.stories.tsx create mode 100644 platforms/tktrex/shared/src/parser/metadata-logger.ts create mode 100644 platforms/yttrex/shared/src/parser/metadata-logger.ts diff --git a/packages/shared/package.json b/packages/shared/package.json index 8d1c048e6..da3f0a7ed 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -30,6 +30,7 @@ "react": "^17.0.2", "react-dom": "^17.0.2", "react-i18next": "^11.18.0", + "react-json-view": "^1.21.3", "ts-endpoint": "^2.0.0", "ts-endpoint-express": "^2.0.0", "ts-io-error": "^2.0.0", diff --git a/packages/shared/src/arbitraries/Supporter.arb.ts b/packages/shared/src/arbitraries/Supporter.arb.ts new file mode 100644 index 000000000..721073b88 --- /dev/null +++ b/packages/shared/src/arbitraries/Supporter.arb.ts @@ -0,0 +1,14 @@ +import { getArbitrary } from 'fast-check-io-ts'; +import { Supporter } from '../models/Supporter'; +import * as t from 'io-ts'; + +const { creationTime, lastActivity, ...supporterProps } = Supporter.type.props; +export const SupporterArb = getArbitrary( + t.strict({ + ...supporterProps, + }) +).map((cc) => ({ + ...cc, + creationTime: new Date(), + lastActivity: new Date(), +})); diff --git a/packages/shared/src/arbitraries/contributions/ContributionEvent.arb.ts b/packages/shared/src/arbitraries/contributions/ContributionEvent.arb.ts new file mode 100644 index 000000000..e0ae46744 --- /dev/null +++ b/packages/shared/src/arbitraries/contributions/ContributionEvent.arb.ts @@ -0,0 +1,32 @@ +import { getArbitrary } from 'fast-check-io-ts'; +import { + ADVContributionEvent, + ContributionEvent, + VideoContributionEvent, +} from '../../models/ContributionEvent'; +import * as t from 'io-ts'; +import fc from 'fast-check'; + +const { ...videoContributionEventProps } = VideoContributionEvent.type.props; +export const VideoContributionEventArb = getArbitrary( + t.strict({ + ...videoContributionEventProps, + }) +).map((cc): any => ({ + ...cc, + clientTime: new Date(), +})); + +const { ...advContributionEventProps } = ADVContributionEvent.type.props; +export const ADVContributionEventArb = getArbitrary( + t.strict({ + ...advContributionEventProps, + }) +).map((cc) => ({ + ...cc, +})); + +export const ContributionEventArb: fc.Arbitrary = fc.oneof( + VideoContributionEventArb, + ADVContributionEventArb +); diff --git a/packages/shared/src/extension/app.ts b/packages/shared/src/extension/app.ts index 90c97fcac..3dea5f7b5 100644 --- a/packages/shared/src/extension/app.ts +++ b/packages/shared/src/extension/app.ts @@ -1,4 +1,5 @@ import _ from 'lodash'; +import { ParserConfiguration, ParserFn } from '../providers/parser.provider'; import { HandshakeResponse } from '../models/HandshakeBody'; import { clearCache } from '../providers/dataDonation.provider'; import { FIXED_USER_NAME, initializeKey } from './background/account'; @@ -16,6 +17,7 @@ import log from './logger'; import HubEvent from './models/HubEvent'; import { ServerLookup } from './models/Message'; import UserSettings from './models/UserSettings'; +import { renderUI, RenderUIProps } from './ui'; import { bo } from './utils/browser.utils'; // instantiate a proper logger @@ -67,7 +69,12 @@ interface SetupObserverOpts { onLocationChange: (oldLocation: string, newLocation: string) => void; } -export interface BootOpts { +export interface BootOpts< + S = any, + M = any, + C extends ParserConfiguration = ParserConfiguration, + PP extends Record> = any +> { payload: ServerLookup['payload']; mapLocalConfig: ( c: UserSettings, @@ -79,6 +86,7 @@ export interface BootOpts { onRegister: (h: Hub, config: UserSettings) => void; }; onAuthenticated: (res: any) => void; + ui?: Omit, 'hub'>; } /** @@ -232,7 +240,12 @@ const serverHandshakeP = ( let loading = false; let app: App | undefined; -export async function boot(opts: BootOpts): Promise { +export async function boot< + S = any, + M = any, + C extends ParserConfiguration = ParserConfiguration, + PP extends Record> = any +>(opts: BootOpts): Promise { if (app) { appLog.debug('App already booted!'); return app; @@ -321,12 +334,12 @@ export async function boot(opts: BootOpts): Promise { // register platform specific event handlers opts.hub.onRegister(opts.hub.hub, config); - // emergency button should be used when a supported with - // UX hack in place didn't see any UX change, so they - // can report the problem and we can handle it. - // initializeEmergencyButton(); + // render shared ui if configuration is given + if (opts.ui) { + renderUI({ hub: opts.hub.hub, ...opts.ui }); + } - // because the URL has been for sure reloaded, be sure to also + // because the URL has been for sure reloaded, be sure to also clear cache clearCache(); // send the configuration to the server to register the extension diff --git a/packages/shared/src/extension/tooltip/index.tsx b/packages/shared/src/extension/tooltip/index.tsx deleted file mode 100644 index a460f19de..000000000 --- a/packages/shared/src/extension/tooltip/index.tsx +++ /dev/null @@ -1,23 +0,0 @@ -import React from 'react'; -import ReactDOM from 'react-dom'; - -import { createTheme, ThemeProvider } from '@material-ui/core/styles'; - -import Tooltip from './components/tooltip'; - -const theme = createTheme({ - typography: { - fontFamily: 'Trex-Regular', - }, -}); - -function main(): void { - ReactDOM.render( - - - , - document.getElementById('yttrex--tooltip') - ); -} - -main(); diff --git a/packages/shared/src/extension/ui/components/MetadataLogger.tsx b/packages/shared/src/extension/ui/components/MetadataLogger.tsx new file mode 100644 index 000000000..cc0242acf --- /dev/null +++ b/packages/shared/src/extension/ui/components/MetadataLogger.tsx @@ -0,0 +1,477 @@ +import { + Accordion, + AccordionDetails, + AccordionSummary, + Box, + Typography, + TypographyProps, +} from '@material-ui/core'; +import * as E from 'fp-ts/lib/Either'; +import { pipe } from 'fp-ts/lib/function'; +import * as M from 'fp-ts/lib/Map'; +import * as O from 'fp-ts/lib/Option'; +import * as S from 'fp-ts/lib/string'; +import * as t from 'io-ts'; +import { PathReporter } from 'io-ts/lib/PathReporter'; +import * as React from 'react'; +import ReactJSON from 'react-json-view'; +import { v4 as uuid } from 'uuid'; +import { + executionLoop, + ParserConfiguration, + ParserContext, + ParserFn, +} from '../../../providers/parser.provider'; +import { Hub } from '../../hub'; +import trexLogger from '../../logger'; + +const log = trexLogger.extend('metadata-logger'); + +export interface MetadataLoggerProps< + S, + M, + C extends ParserConfiguration, + PP extends Record> +> { + hub: Hub; + mapEvent: (id: string, e: unknown) => S | null; + decode: t.Decode; + parser: Omit< + ParserContext, + | 'db' + | 'codecs' + | 'getLastContributions' + | 'getMetadata' + | 'saveResults' + | 'getContributions' + >; +} + +interface MetadataLoggerEntry { + source: S; + metadata?: M; + errors: { + decode: string[]; + parser: string[]; + }; +} + +type MetadataLoggerState = Map>; + +const ParserErrorsHeader: React.FC<{ count: number } & TypographyProps> = ({ + count, + ...props +}) => ( + + Parser {count} + +); + +const MetadataHeader: React.FC<{ count: number } & TypographyProps> = ({ + count, + ...props +}) => ( + + M {count} + +); + +const DecodeErrorsHeader: React.FC<{ count: number } & TypographyProps> = ({ + count, + ...props +}) => ( + + Decode {count} + +); + +export const MetadataLogger = < + S extends any, + M extends any, + C extends ParserConfiguration, + PP extends Record> +>({ + hub, + mapEvent, + decode, + parser, +}: MetadataLoggerProps): JSX.Element => { + const [contributions, setContributions] = React.useState< + MetadataLoggerState + >(new Map()); + + const contributionEntries = pipe(contributions, M.toArray(S.Ord)); + + const getContributions = React.useCallback( + (f, s, a) => { + const sources = contributionEntries.map(([key, c]) => c.source); + log.debug('Get last sources %O', sources); + return Promise.resolve({ sources, errors: 0, overflow: false }); + }, + [contributionEntries] + ); + + const getMetadata = React.useCallback( + async (e: any): Promise => { + log.debug('Store %O', contributions); + const entryId = parser.getEntryId(e); + + const m = pipe( + contributions, + M.lookup(S.Eq)(entryId), + O.filterMap((m) => (m.metadata ? O.some(m.metadata) : O.none)), + O.toNullable + ); + + log.debug('Get metadata from entry id %s => %O', entryId, m); + + return m; + }, + [contributionEntries] + ); + + const saveResults = React.useCallback( + (s: any, m: any) => { + log.debug('Save results: source %O, metadata %O', s, m); + const metadataUpsert = pipe( + contributions, + M.lookup(S.Eq)(s.id), + O.fold( + (): MetadataLoggerEntry => ({ + errors: { + parser: [], + decode: [], + }, + source: s, + metadata: m, + }), + (c) => ({ + ...c, + metadata: m, + }) + ) + ); + log.debug('Old contributions %O', contributions); + pipe( + contributions, + M.upsertAt(S.Eq)(parser.getEntryId(s), metadataUpsert), + (cc) => { + log.debug('New contributions %s', cc); + setContributions(cc); + } + ); + + return Promise.resolve({ + metadata: m, + source: s, + count: { metadata: 1 }, + }); + }, + [contributionEntries] + ); + + const parserCtx: ParserContext = { + ...parser, + log, + db: { api: {} as any, read: {} as any, write: {} as any }, + codecs: { metadata: t.any, contribution: t.any }, + getMetadata, + getContributions, + saveResults, + }; + + React.useEffect(() => { + log.debug('Start parsing %O', M.toArray(S.Ord)(contributions)); + const unprocessedContributions = pipe( + contributions, + // once the parser has finished the entry should have either `metadata` or some `errors.parser` + M.filter((s) => s.metadata === undefined && s.errors.parser.length === 0) + ); + + log.debug( + 'Unprocessed contributions %O', + M.toArray(S.Ord)(unprocessedContributions) + ); + + if (unprocessedContributions.size >= 1) { + void executionLoop(parserCtx)({ + singleUse: true, + htmlAmount: 1, + stop: 1, + }).then((r) => { + log.debug('Parser output %O', r); + if (r.type === 'Success') { + const outputContributions = r.payload.reduce((acc, p) => { + const sourceId = parser.getEntryId(p.source); + + let parserErrors: any[] = []; + if (p.failures) { + parserErrors = Object.entries(p.failures).reduce( + (acc, [key, value]) => { + if (value instanceof Error) { + return acc.concat({ + [key]: value.message, + }); + } + + return acc.concat({ + [key]: value, + }); + }, + [] + ); + } + + const entry = pipe( + acc, + M.lookup(S.Eq)(sourceId), + O.fold( + (): MetadataLoggerEntry => ({ + source: p.source, + metadata: undefined, + errors: { + decode: [], + parser: parserErrors, + }, + }), + (m) => ({ + ...m, + metadata: p.metadata, + errors: { + ...m.errors, + parser: parserErrors, + }, + }) + ) + ); + + return pipe(acc, M.upsertAt(S.Eq)(sourceId, entry)); + }, contributions); + + log.debug('Update contributions %O', outputContributions); + setContributions(outputContributions); + } + }); + } + }, [contributionEntries]); + + const onAnyEvent = (event: any): void => { + // log.debug('event received', event); + const eventId = uuid(); + const datum = mapEvent(eventId, event); + + if (datum === null) { + log.debug('Avoid parsing event type %s', event.type); + return; + } + + log.debug('Decoding event %O', datum); + // log.debug('datum mapped %O', datum); + const contribution = decode(datum); + // when the received event decode fails + // we update the `decodeErrors` map + // to render the errors + + if (E.isLeft(contribution)) { + const errors = PathReporter.report(contribution); + + log.debug('Decode errors %O', errors); + + const decodeErrorUpsert = pipe( + contributions, + M.lookup(S.Eq)(eventId), + O.fold( + (): MetadataLoggerEntry => ({ + errors: { parser: [], decode: errors }, + metadata: undefined, + source: datum, + }), + (c) => ({ + ...c, + errors: { + ...c.errors, + decode: errors, + }, + }) + ) + ); + + pipe( + contributions, + M.upsertAt(S.Eq)(eventId, decodeErrorUpsert), + setContributions + ); + } else { + log.debug('Contribution id %s', eventId); + + pipe( + contributions, + M.lookup(S.Eq)(eventId), + O.fold( + () => ({ + metadata: undefined, + source: contribution.right, + errors: { + decode: [], + parser: [], + }, + }), + (c) => ({ ...c, source: contribution.right }) + ), + (c) => { + log.debug('Contribution %O', c); + pipe(contributions, M.upsertAt(S.Eq)(eventId, c), setContributions); + } + ); + } + }; + + React.useEffect(() => { + hub.onAnyEvent(onAnyEvent); + }, []); + + log.debug('contributions', contributions); + + const { summary, details } = React.useMemo(() => { + const metadata = pipe( + contributions, + M.filter((c) => !!c.metadata) + ); + + const decodeErrors = pipe( + contributions, + M.filter((c) => c.errors.decode.length > 0) + ); + + const parserErrors = pipe( + contributions, + M.filter((c) => c.errors.parser.length > 0) + ); + + return { + summary: ( +
+ + + +
+ ), + details: ( +
+
    + {pipe( + contributions, + M.mapWithIndex((key, data) => ( +
  • +
    + + {(data.metadata as any)?.type}:{' '} + {parser.getEntryId(data.source as any)} + + + + +
    + {data.errors.parser.length > 0 ? ( +
    + + {data.errors.parser.map((l, i) => ( + + {JSON.stringify(l)} + + ))} +
    + ) : null} + + {data.errors.decode.length > 0 ? ( +
    + + + + + {data.errors.decode.map((l, i) => ( + + {l} + + ))} +
    + ) : null} +
    • + )), + M.toArray(S.Ord), + (entries) => entries.map(([k, e]) => e) + )} +
+
+ ), + }; + }, [contributionEntries]); + + return ( + + + {summary} + {details} + + + ); +}; diff --git a/packages/shared/src/extension/tooltip/components/tooltip.tsx b/packages/shared/src/extension/ui/components/tooltip.tsx similarity index 100% rename from packages/shared/src/extension/tooltip/components/tooltip.tsx rename to packages/shared/src/extension/ui/components/tooltip.tsx diff --git a/packages/shared/src/extension/ui/index.tsx b/packages/shared/src/extension/ui/index.tsx new file mode 100644 index 000000000..3cb66badc --- /dev/null +++ b/packages/shared/src/extension/ui/index.tsx @@ -0,0 +1,69 @@ +import { createTheme, ThemeProvider } from '@material-ui/core/styles'; +import React from 'react'; +import ReactDOM from 'react-dom'; +// import Tooltip from './components/tooltip'; +import { ErrorBoundary } from '../../components/Error/ErrorBoundary'; +import { ParserConfiguration, ParserFn } from '../../providers/parser.provider'; +import { Hub } from '../hub'; +import { + MetadataLogger, + MetadataLoggerProps +} from './components/MetadataLogger'; + +const theme = createTheme({ + typography: { + fontFamily: 'Trex-Regular', + }, +}); + +// const TOOLTIP_ID = 'trex--tooltip'; +const METADATA_LOGGER_ID = 'trex--metadata-logger'; + +export interface RenderUIProps< + S, + M, + C extends ParserConfiguration, + PP extends Record> +> { + hub: Hub; + metadataLogger: Omit, 'hub'>; +} + +export function renderUI< + S, + M, + C extends ParserConfiguration, + PP extends Record> +>({ hub, metadataLogger }: RenderUIProps): void { + // const tooltipNode = document.getElementById(TOOLTIP_ID); + // if (!tooltipNode) { + // const node = document.createElement('div'); + + // node.id = TOOLTIP_ID; + // document.body.append(node); + // ReactDOM.render( + // + // + // , + // node + // ); + // } + + const metadataLoggerNode = document.getElementById(METADATA_LOGGER_ID); + if (!metadataLoggerNode) { + const node = document.createElement('div'); + + node.id = METADATA_LOGGER_ID; + document.body.append(node); + ReactDOM.render( + + + + + + + , + node + ); + } +} diff --git a/platforms/storybook/.storybook/main.js b/platforms/storybook/.storybook/main.js index 991c7910d..249b6d650 100644 --- a/platforms/storybook/.storybook/main.js +++ b/platforms/storybook/.storybook/main.js @@ -54,6 +54,7 @@ module.exports = { }, env: (config) => ({ ...config, + DEBUG: '*', BUILD: 'BUILD env variable', BUILD_DATE: new Date().toISOString(), DEVELOPMENT: true, diff --git a/platforms/storybook/src/stories/MetadataLogger.stories.tsx b/platforms/storybook/src/stories/MetadataLogger.stories.tsx new file mode 100644 index 000000000..c2179475a --- /dev/null +++ b/platforms/storybook/src/stories/MetadataLogger.stories.tsx @@ -0,0 +1,93 @@ +import { VideoContributionEventArb } from '@shared/arbitraries/contributions/ContributionEvent.arb'; +import { + MetadataLogger, + MetadataLoggerProps, +} from '@shared/extension/ui/components/MetadataLogger'; +import { trexLogger } from '@shared/logger'; +import { ParserConfiguration } from '@shared/providers/parser.provider'; +import { Meta, Story } from '@storybook/react'; +import tkHub from '@tktrex/extension/src/handlers/hub'; +import { toMetadata as toTKMetadata } from '@tktrex/shared/parser/metadata'; +import { HTMLSource as TKHTMLSource } from '@tktrex/shared/parser/source'; +import ytHub from '@yttrex/extension/src/handlers/hub'; +import { metadataLoggerParserProps } from '@yttrex/shared/parser/metadata-logger'; +import fc from 'fast-check'; +import * as React from 'react'; +import { v4 as uuid } from 'uuid'; + +const meta: Meta> = { + title: 'Example/MetadataLogger', + component: MetadataLogger, +}; + +export default meta; + +const Template: Story< + MetadataLoggerProps & { + platform: 'yttrex' | 'tktrex'; + } +> = ({ platform, ...args }) => { + const onNewVideoContributionClick = React.useCallback(() => { + // eslint-disable-next-line @typescript-eslint/no-var-requires + const fixture = require(`../../../${platform}/backend/__tests__/fixtures/htmls/video/004fdccdb8bddf7a2588fcbf09916015ff898238.json`); + + const { html, savingTime, ...source } = fixture.sources[0]; + const newVideoContribution = fc + .sample(VideoContributionEventArb, 1) + .map((v) => ({ + type: 'NewVideo', + payload: { + ...v, + ...source, + metadataId: uuid(), + publicKey: '', + element: html, + clientTime: new Date(), + savingTime: new Date(), + }, + }))[0]; + + args.hub.dispatch(newVideoContribution); + }, []); + + return ( +
+
+ + +
+ +
+ ); +}; + +export const YoutubeBasic = Template.bind({}); +YoutubeBasic.args = { + platform: 'yttrex', + hub: ytHub, + ...metadataLoggerParserProps, +}; + +export const TikTokBasic = Template.bind({}); +TikTokBasic.args = { + platform: 'tktrex', + hub: tkHub, + parser: { + name: 'tk-metadata-logger', + log: trexLogger.extend('metadata'), + parsers: {}, + getEntryId: (e) => e.html.id, + getEntryDate: (e) => e.html.savingTime, + getEntryNatureType: (e) => e.html.nature.type, + buildMetadata: toTKMetadata, + config: {}, + addDom: (s) => ({ + ...s, + dom: new DOMParser().parseFromString(s.html.html, 'text/html'), + }), + }, + decode: TKHTMLSource.decode, + mapEvent(id, e) { + return {}; + }, +}; diff --git a/platforms/storybook/tsconfig.json b/platforms/storybook/tsconfig.json index 76cca4abb..317515d3a 100644 --- a/platforms/storybook/tsconfig.json +++ b/platforms/storybook/tsconfig.json @@ -22,14 +22,20 @@ "moduleResolution": "node", "isolatedModules": true, "paths": { - "@shared/*": [ - "../../../packages/shared/src/*" - ] + "@shared/*": ["../../../packages/shared/src/*"], + "@yttrex/shared/*": ["../../yttrex/shared/src/*"], + "@tktrex/shared/*": ["../../tktrex/shared/src/*"] } }, "references": [ { "path": "../../packages/shared" + }, + { + "path": "../yttrex/shared" + }, + { + "path": "../tktrex/shared" } ], "include": ["./src"], diff --git a/platforms/tktrex/extension/src/app/index.ts b/platforms/tktrex/extension/src/app/index.ts index e8b85b3da..6ad9b48cf 100644 --- a/platforms/tktrex/extension/src/app/index.ts +++ b/platforms/tktrex/extension/src/app/index.ts @@ -2,6 +2,7 @@ import { boot } from '@shared/extension/app'; import { tiktokDomainRegExp } from '@tktrex/parser/v2/constant'; import { registerTkHandlers } from './handlers'; import { feedId, onLocationChange, tkHandlers, tkTrexActions } from './app'; +import { metadataLoggerProps } from '@tktrex/parser/metadata-logger'; import tkHub from '../handlers/hub'; // Boot the app script. This is the first function called. @@ -23,4 +24,7 @@ void boot({ onLocationChange, }, onAuthenticated: tkTrexActions, + ui: { + metadataLogger: metadataLoggerProps, + }, }); diff --git a/platforms/tktrex/shared/src/parser/metadata-logger.ts b/platforms/tktrex/shared/src/parser/metadata-logger.ts new file mode 100644 index 000000000..ba7922796 --- /dev/null +++ b/platforms/tktrex/shared/src/parser/metadata-logger.ts @@ -0,0 +1,64 @@ +import { MetadataLoggerProps } from '@shared/extension/ui/components/MetadataLogger'; +import { trexLogger } from '@shared/logger'; +import { ParserConfiguration } from '@shared/providers/parser.provider'; +import { v4 as uuid } from 'uuid'; +import { TKMetadata } from '../models/Metadata'; +import { toMetadata } from './metadata'; +import { parsers } from './parsers'; +import { HTMLSource } from './source'; + +const metadataLogger = trexLogger.extend('tk:parser'); + +export const metadataLoggerProps: Omit< + MetadataLoggerProps, + 'hub' +> = { + parser: { + name: 'tk:parser', + log: metadataLogger, + parsers, + addDom: (h) => ({ + ...h, + jsdom: new DOMParser().parseFromString(h.html.html, 'text/html'), + }), + getEntryId: (e) => e.html.id, + getEntryDate: (e) => e.html.savingTime, + getEntryNatureType: (e) => e.html.nature.type, + buildMetadata: toMetadata, + config: {}, + }, + decode: HTMLSource.decode, + mapEvent(id, e) { + metadataLogger.debug('Map event %O', e); + const ev: any = e; + + if (ev.type === 'NewVideo') { + return { + html: { + ...ev.payload, + id, + metadataId: uuid(), + blang: '', + counters: [], + clientTime: new Date(), + savingTime: new Date(), + publicKey: '', + type: 'video', + videoId: '', + authorId: '', + nature: { type: 'video', videoId: '', authorId: '' }, + processed: false, + }, + supporter: { + publicKey: '', + version: '', + lastActivity: new Date(), + creationTime: new Date(), + p: '', + }, + jsdom: {}, + }; + } + return null; + }, +}; diff --git a/platforms/tktrex/shared/src/parser/parsers/index.ts b/platforms/tktrex/shared/src/parser/parsers/index.ts index e6542a1f7..f063158c2 100644 --- a/platforms/tktrex/shared/src/parser/parsers/index.ts +++ b/platforms/tktrex/shared/src/parser/parsers/index.ts @@ -8,7 +8,7 @@ import author from './author'; import search from './search'; import profile from './profile'; import native from './native'; -import downloader from './downloader'; +// import downloader from './downloader'; export const parsers = { nature, @@ -21,7 +21,8 @@ export const parsers = { search, profile, native, - downloader, + downloader: () => Promise.resolve({}), + // downloader, }; export type TKParsers = typeof parsers; diff --git a/platforms/yttrex/extension/src/app/index.ts b/platforms/yttrex/extension/src/app/index.ts index 453c6d5b0..5dd02b498 100644 --- a/platforms/yttrex/extension/src/app/index.ts +++ b/platforms/yttrex/extension/src/app/index.ts @@ -3,6 +3,7 @@ import { boot } from '@shared/extension/app'; import { bo } from '@shared/extension/utils/browser.utils'; +import { metadataLoggerParserProps } from '@yttrex/shared/parser/metadata-logger'; import { youtubeDomainRegExp } from '@yttrex/shared/parser/selectors'; import * as hubHandlers from '../handlers/events'; import ytHub from '../handlers/hub'; @@ -36,6 +37,9 @@ bo.runtime.sendMessage({ type: 'chromeConfig' }, (config) => { }, }, onAuthenticated: ytTrexActions, + ui: { + metadataLogger: metadataLoggerParserProps, + }, }); } catch (e) { // eslint-disable-next-line diff --git a/platforms/yttrex/extension/tsconfig.json b/platforms/yttrex/extension/tsconfig.json index e1fb8c8d7..5e393a57b 100644 --- a/platforms/yttrex/extension/tsconfig.json +++ b/platforms/yttrex/extension/tsconfig.json @@ -29,7 +29,7 @@ "path": "../../../packages/shared" } ], - "include": ["./"], + "include": ["./src"], "exclude": [ "./jest.config.js", "./jest.setup.ts", diff --git a/platforms/yttrex/shared/src/parser/metadata-logger.ts b/platforms/yttrex/shared/src/parser/metadata-logger.ts new file mode 100644 index 000000000..7fe199b70 --- /dev/null +++ b/platforms/yttrex/shared/src/parser/metadata-logger.ts @@ -0,0 +1,66 @@ +import { MetadataLoggerProps } from '@shared/extension/ui/components/MetadataLogger'; +import { trexLogger } from '@shared/logger'; +import { ParserConfiguration } from '@shared/providers/parser.provider'; +import { v4 as uuid } from 'uuid'; +import { Metadata } from '../models/Metadata'; +import { toMetadata } from './metadata'; +import nature, { getNatureFromURL } from './parsers/nature'; +import search from './parsers/searches'; +import { HTMLSource } from './source'; + +const metadataParserLogger = trexLogger.extend('metadata-logger'); + +export const metadataLoggerParserProps: Omit< + MetadataLoggerProps, + 'hub' +> = { + parser: { + name: 'yt-metadata', + log: metadataParserLogger, + addDom: (h) => ({ + ...h, + jsdom: new DOMParser().parseFromString(h.html.html, 'text/html'), + }), + buildMetadata: toMetadata as any, + parsers: { + nature, + search, + }, + getEntryDate: (e) => e.html.clientTime, + getEntryNatureType: (e) => e.html.nature.type, + getEntryId: (e) => e.html.id, + config: {}, + }, + decode: HTMLSource.decode, + mapEvent: (id, e) => { + const ev: any = e; + if (ev.type === 'NewVideo') { + const { element, ...payload } = ev.payload; + return { + html: { + id, + metadataId: uuid(), + blang: '', + publicKey: '', + clientTime: new Date(), + savingTime: new Date(), + nature: getNatureFromURL(payload.href), + counters: [], + ...payload, + html: element, + processed: false, + }, + supporter: { + publicKey: '', + lastActivity: new Date(), + version: '', + creationTime: new Date(), + p: '', + }, + jsdom: {}, + }; + } + + return null; + }, +}; diff --git a/platforms/yttrex/shared/tsconfig.json b/platforms/yttrex/shared/tsconfig.json index 7b848aad8..7cd624e89 100644 --- a/platforms/yttrex/shared/tsconfig.json +++ b/platforms/yttrex/shared/tsconfig.json @@ -8,6 +8,7 @@ "baseUrl": "./src", "rootDir": "./", "noEmit": false, + "jsx": "react", "esModuleInterop": true, "skipLibCheck": true, "isolatedModules": true, diff --git a/yarn.lock b/yarn.lock index 863fb3b94..7f36b5ec7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6571,6 +6571,7 @@ __metadata: react: ^17.0.2 react-dom: ^17.0.2 react-i18next: ^11.18.0 + react-json-view: ^1.21.3 react-refresh: ^0.12.0 react-refresh-typescript: ^2.0.7 sass-loader: ^12.6.0