feat(shared): defined puppeteer hook for type

packages/shared/src/models/Step.ts

@@ -43,6 +43,19 @@ export const ClickStep = t.strict(
 
 export type ClickStep = t.TypeOf<typeof ClickStep>;
 
+export const TypeType = t.literal('type');
+export const TypeStep = t.strict(
+  {
+    type: TypeType,
+    selector: t.string,
+    text: t.string,
+    delay: t.union([t.number, t.undefined]),
+  },
+  'TypeStep'
+);
+
+export type TypeStep = t.TypeOf<typeof TypeStep>;
+
 export const CustomStepType = t.literal('custom');
 export type CustomStepType = t.TypeOf<typeof CustomStepType>;
 
@@ -81,6 +94,7 @@ export const Step = t.union(
     CustomStep,
     KeypressStep,
     ClickStep,
+    TypeStep,
     // since `openURL` step is the default and the `type` can be `undefined`
     // the `OpenURLStep` codec needs to be the last value of the union
     OpenURLStep,

packages/shared/src/providers/parser.provider.ts

@@ -311,7 +311,10 @@ const pipeline =
   >(
     ctx: ParserContext<T, M, PP>
   ) =>
-  async (e: T, parsers: PP): Promise<PipelineInput<T, PP>> => {
+  async (
+    e: t.TypeOf<T>,
+    parsers: PP
+  ): Promise<PipelineInput<t.TypeOf<T>, PP>> => {
     let results: PipelineInput<T, PP> = {
       failures: {},
       source: e,
@@ -401,7 +404,7 @@ export const parseContributions =
 
     const results: Array<PipelineOutput<t.TypeOf<T>, t.TypeOf<M>, PP>> = [];
 
-    ctx.log.debug('Sources %O', envelops.sources);
+    ctx.log.debug('Sources %O', envelops.sources.map(ctx.getEntryId));
 
     for (const entry of envelops.sources) {
       ctx.log.debug('Parsing entry %O', ctx.getEntryId(entry));
@@ -513,7 +516,10 @@ export const executionLoop =
         } else {
           ctx.log.debug('Data to process %d', envelops.sources.length);
           const currentResult = await parseContributions(ctx)(envelops);
-          ctx.log.debug('Processed envelops %O', currentResult);
+          ctx.log.debug(
+            'Processed sources %O',
+            currentResult.map((r) => ctx.getEntryId(r.source))
+          );
           results.push(...currentResult);
         }
 

packages/shared/src/providers/puppeteer/puppeteer.provider.ts

@@ -10,6 +10,7 @@ import {
   ScrollStepType,
   KeyPressType,
   ClickType,
+  TypeType,
 } from '../../models/Step';
 // import { throwTE } from '../../utils/task.utils';
 import { StepHooks } from './StepHooks';
@@ -18,6 +19,7 @@ import { GetScrollFor } from './steps/scroll';
 import StealthPlugin from 'puppeteer-extra-plugin-stealth';
 import { GetKeypressStep } from './steps/keyPress';
 import { GetClickStep } from './steps/click';
+import { GetTypeStep } from './steps/type';
 
 export type LaunchOptions = puppeteer.LaunchOptions &
   puppeteer.BrowserLaunchArgumentOptions &
@@ -64,6 +66,9 @@ const operateTab =
       case ClickType.value: {
         return GetClickStep(ctx)(page, h);
       }
+      case TypeType.value: {
+        return GetTypeStep(ctx)(page, h);
+      }
       case CustomStepType.value:
         return TE.tryCatch(() => {
           ctx.logger.debug('Custom handler %s', h.handler);

packages/shared/src/providers/puppeteer/steps/type.ts

@@ -0,0 +1,54 @@
+import * as E from 'fp-ts/lib/Either';
+import * as TE from 'fp-ts/lib/TaskEither';
+import * as puppeteer from 'puppeteer-core';
+import { AppError, toAppError } from '../../../errors/AppError';
+import { TypeStep } from '../../../models/Step';
+import { sleep } from '../../../utils/promise.utils';
+import { StepContext } from './types';
+
+/**
+ * Click command REgExp
+ *
+ * click('video')
+ */
+export const TYPE_COMMAND_REGEXP =
+  // eslint-disable-next-line no-useless-escape
+  /type\(([#|\.]?[\w|:|\s|\.|\-]+);\"([\w|\s|\d]+)\"\)/gm;
+
+export const parseTypeCommand = (
+  cmd: string
+): E.Either<AppError, { selector: string; text: string; delay: number }> => {
+  const match = TYPE_COMMAND_REGEXP.exec(cmd);
+  // console.log(match);
+  if (match?.[1] && match[2]) {
+    const selector: any = match[1];
+    const text = match[2];
+    const delay = parseInt(match[3], 10);
+    return E.right({ selector, text, delay });
+  }
+  return E.left(
+    new AppError('TypeStepError', `Cannot parse command: ${cmd}`, [])
+  );
+};
+/**
+ * Type step
+ *
+ * Type into an element by the given selector
+ *
+ */
+
+export const GetTypeStep =
+  (ctx: StepContext) =>
+  (
+    page: puppeteer.Page,
+    { selector, text, delay: _delay }: TypeStep
+  ): TE.TaskEither<AppError, void> => {
+    const delay = _delay ?? 0;
+
+    return TE.tryCatch(async () => {
+      ctx.logger.debug('Type %s with delay %d', selector, delay);
+      await sleep(delay);
+      const el = await page.waitForSelector(selector, { timeout: 0 });
+      await el?.type(text);
+    }, toAppError);
+  };

packages/shared/src/test/utils/parser.utils.ts

@@ -2,38 +2,52 @@ import * as E from 'fp-ts/lib/Either';
 import * as fs from 'fs';
 import * as t from 'io-ts';
 import { PathReporter } from 'io-ts/lib/PathReporter';
-import * as path from 'path';
+import path from 'path';
 import { Logger } from '../../logger';
 import {
-  parseContributions, ParserFn, ParserProviderContext
+  parseContributions,
+  ParserFn,
+  ParserProviderContext,
 } from '../../providers/parser.provider';
 
 /**
- * Read the parsed metadata history from '../fixtures/${nature}'
+ * Read fixtures file from path
+ *
+ * @param fixtureDir
+ * @returns string
  */
-export const readHistoryResults = (
-  fixtureDir: string,
-  publicKey: string
-): any[] => {
+export const readFixtureJSONPaths = (fixtureDir: string): any[] => {
   return fs
     .readdirSync(fixtureDir, 'utf-8')
     .filter((file) => file.includes('.json'))
-    .map((file) => {
-      const filePath = path.resolve(fixtureDir, file);
-      // eslint-disable-next-line
-      console.log('reading content from ', filePath);
-      const content = fs.readFileSync(filePath, 'utf-8');
-      const contentJSON = JSON.parse(content);
-      return contentJSON;
-    })
-    .map((mt) => ({
-      ...mt,
-      sources: mt.sources.map((h: any) => ({
-        ...h,
-        publicKey,
-      })),
-      metadata: { ...mt.metadata, publicKey },
-    }));
+    .map((fp) => path.resolve(fixtureDir, fp));
+};
+
+/**
+ * Read the parsed metadata history from '../fixtures/${nature}'
+ *
+ * @param fixtureFilePath
+ * @param publicKey Supporter public key
+ */
+export const readFixtureJSON = (
+  fixtureFilePath: string,
+  publicKey: string
+): { sources: any[]; metadata: any } => {
+  // eslint-disable-next-line
+  console.log(
+    'reading content from ',
+    path.relative(process.cwd(), fixtureFilePath)
+  );
+  const content = fs.readFileSync(fixtureFilePath, 'utf-8');
+  const mt = JSON.parse(content);
+  return {
+    ...mt,
+    sources: mt.sources.map((h: any) => ({
+      ...h,
+      publicKey,
+    })),
+    metadata: { ...mt.metadata, publicKey },
+  };
 };
 
 // type MetadataResult<T, S> = T & { sources: S[]; _id: string };
@@ -62,7 +76,7 @@ export const runParserTest =
     expectSources: (s: Array<t.TypeOf<S>>) => void;
   } & ParserProviderContext<S, M, PP>) =>
   async ({ sources, metadata }: any) => {
-    opts.log.debug('Sources %d', sources.length);
+    // opts.log.debug('Sources %d', sources.length);
 
     // insert the sources in the db
     await opts.db.api.insertMany(

platforms/guardoni/src/guardoni/experiment.ts

@@ -10,13 +10,15 @@ import {
   OpenURLStepType,
   ScrollStepType,
   Step,
+  TypeType,
 } from '@shared/models/Step';
 import * as E from 'fp-ts/lib/Either';
 import { pipe } from 'fp-ts/lib/function';
 import _ from 'lodash';
 // import * as NEA from 'fp-ts/lib/NonEmptyArray';
 import { parseClickCommand } from '@shared/providers/puppeteer/steps/click';
 import { parseKeypressCommand } from '@shared/providers/puppeteer/steps/keyPress';
+import { parseTypeCommand } from '@shared/providers/puppeteer/steps/type';
 import { NonEmptyArray } from 'fp-ts/lib/NonEmptyArray';
 import * as TE from 'fp-ts/lib/TaskEither';
 import * as fs from 'fs';
@@ -145,6 +147,29 @@ export const readCSVAndParse =
                           )
                         );
                       }
+
+                      if (c.startsWith('type')) {
+                        pipe(
+                          parseTypeCommand(c),
+                          E.fold(
+                            (e) => {
+                              logger.warn(e.name, e.message);
+                            },
+                            (opts) => {
+                              logger.debug(
+                                'Type command %s parsed %O',
+                                c,
+                                opts
+                              );
+                              const typeStep = {
+                                type: TypeType.value,
+                                ...opts,
+                              };
+                              acc.push(typeStep);
+                            }
+                          )
+                        );
+                      }
                       return acc;
                     },
                     []

platforms/tktrex/backend/parsers/__tests__/native.e2e.ts

@@ -1,5 +1,6 @@
 import {
-  readHistoryResults,
+  readFixtureJSON,
+  readFixtureJSONPaths,
   runParserTest,
 } from '@shared/test/utils/parser.utils';
 import { sanitizeHTML } from '@shared/utils/html.utils';
@@ -48,14 +49,17 @@ describe('Parser: "native"', () => {
   describe('Native', () => {
     jest.setTimeout(20 * 1000);
 
-    const history = readHistoryResults(
-      path.resolve(__dirname, 'fixtures/native'),
-      publicKey
+    const history = readFixtureJSONPaths(
+      path.resolve(__dirname, 'fixtures/native')
     );
 
     test.each(history)(
       'Should correctly parse "native" contribution',
-      async ({ sources: _sources, metadata }) => {
+      async (fixturePath) => {
+        const { sources: _sources, metadata } = readFixtureJSON(
+          fixturePath,
+          publicKey
+        );
         const sources = _sources.map((s: any) => ({
           html: {
             ...s,

platforms/yttrex/backend/__tests__/parser/html/parseHome.e2e.ts

@@ -1,5 +1,6 @@
 import {
-  readHistoryResults,
+  readFixtureJSON,
+  readFixtureJSONPaths,
   runParserTest,
 } from '@shared/test/utils/parser.utils';
 import { sanitizeHTML } from '@shared/utils/html.utils';
@@ -18,7 +19,7 @@ import {
 import processHome from '../../../parsers/home';
 import { GetTest, Test } from '../../../tests/Test';
 
-describe('Parserv', () => {
+describe('Parser: home', () => {
   let appTest: Test;
   const newKeypair = nacl.sign.keyPair();
   const publicKey = base58.encode(newKeypair.publicKey);
@@ -57,14 +58,18 @@ describe('Parserv', () => {
     );
   });
 
-  const history = readHistoryResults(
-    path.resolve(__dirname, '../../fixtures/home'),
-    publicKey
-  ).filter((v, i) => ![5, 9].includes(i));
+  const history = readFixtureJSONPaths(
+    path.resolve(__dirname, '../../fixtures/home')
+  )
+  // .filter((v, i) => ![5, 9].includes(i));
 
   test.each([history[0]])(
     'Should correctly parse home contributions',
-    async ({ sources: _sources, metadata }) => {
+    async (filePath) => {
+      const { sources: _sources, metadata } = readFixtureJSON(
+        filePath,
+        publicKey
+      );
       const sources = _sources.map((h: any) => ({
         html: {
           ...h,
@@ -76,7 +81,7 @@ describe('Parserv', () => {
         jsdom: new JSDOM(sanitizeHTML(h.html)).window.document,
       }));
 
-      const result = await runParserTest({
+      await runParserTest({
         log: appTest.logger,
         sourceSchema: appTest.config.get('schema').htmls,
         metadataSchema: appTest.config.get('schema').metadata,