Html comments

.github/ubiquibot-config.yml

@@ -36,4 +36,14 @@ auto-pay-mode: true
 comment-incentives: true
 max-concurrent-bounties: 2
 promotion-comment: "\n<h6>If you enjoy the DevPool experience, please follow <a href='https://github.com/ubiquity'>Ubiquity on GitHub</a> and star <a href='https://github.com/ubiquity/devpool-directory'>this repo</a> to show your support. It helps a lot!</h6>"
-
+incentives:
+  comment:
+    elements:
+      code: 5
+      img: 5
+      h1: 1
+      li: 0.5
+      a: 0.5
+      blockquote: 0
+    totals:
+      word: 0.1

package.json

@@ -34,7 +34,6 @@
     "@probot/adapter-github-actions": "^3.1.3",
     "@sinclair/typebox": "^0.25.9",
     "@supabase/supabase-js": "^2.4.0",
-    "@types/mdast": "^3.0.11",
     "@types/ms": "^0.7.31",
     "@typescript-eslint/eslint-plugin": "^5.59.11",
     "@typescript-eslint/parser": "^5.59.11",
@@ -49,13 +48,11 @@
     "js-yaml": "^4.1.0",
     "libsodium-wrappers": "^0.7.11",
     "lint-staged": "^13.1.0",
-    "mdast-util-from-markdown": "^1.3.0",
-    "mdast-util-gfm": "^2.0.2",
-    "micromark-extension-gfm": "^2.0.3",
     "ms": "^2.1.3",
     "node-html-parser": "^6.1.5",
     "node-html-to-image": "^3.3.0",
     "nodemon": "^2.0.19",
+    "parse5": "^7.1.2",
     "prettier": "^2.7.1",
     "probot": "^12.2.4",
     "telegraf": "^4.11.2",

src/bindings/config.ts

@@ -13,7 +13,7 @@ export const loadConfig = async (context: Context): Promise<BotConfig> => {
     baseMultiplier,
     timeLabels,
     priorityLabels,
-    commentElementPricing,
+    incentives,
     autoPayMode,
     disableAnalytics,
     bountyHunterMax,
@@ -37,7 +37,7 @@ export const loadConfig = async (context: Context): Promise<BotConfig> => {
       issueCreatorMultiplier,
       timeLabels,
       priorityLabels,
-      commentElementPricing,
+      incentives,
       defaultLabels,
     },
     comments: {

src/configs/price.ts

@@ -1,5 +1,4 @@
 import { PriceConfig } from "../types";
-import { MarkdownItem } from "../types/markdown";
 
 export const DefaultPriceConfig: PriceConfig = {
   baseMultiplier: 1000,
@@ -53,12 +52,15 @@ export const DefaultPriceConfig: PriceConfig = {
       weight: 5,
     },
   ],
-  commentElementPricing: {
-    [MarkdownItem.Text]: 0.1,
-    [MarkdownItem.Link]: 0.5,
-    [MarkdownItem.List]: 0.5,
-    [MarkdownItem.Code]: 5,
-    [MarkdownItem.Image]: 5,
+  incentives: {
+    comment: {
+      elements: {
+        li: 1,
+      },
+      totals: {
+        word: 0.1,
+      },
+    },
   },
   defaultLabels: [],
 };

src/handlers/payout/post.ts

@@ -1,9 +1,9 @@
 import { getWalletAddress } from "../../adapters/supabase";
 import { getBotConfig, getBotContext, getLogger } from "../../bindings";
 import { addCommentToIssue, generatePermit2Signature, getAllIssueComments, getIssueDescription, getTokenSymbol, parseComments } from "../../helpers";
-import { MarkdownItem, Payload, UserType, CommentElementPricing } from "../../types";
+import { Incentives, Payload, UserType } from "../../types";
 
-const ItemsToExclude: string[] = [MarkdownItem.BlockQuote];
+const ItemsToExclude: string[] = ["blockquote"];
 /**
  * Incentivize the contributors based on their contribution.
  * The default formula has been defined in https://github.com/ubiquity/ubiquibot/issues/272
@@ -12,7 +12,7 @@ export const incentivizeComments = async () => {
   const logger = getLogger();
   const {
     mode: { incentiveMode },
-    price: { baseMultiplier, commentElementPricing },
+    price: { baseMultiplier, incentives },
     payout: { paymentToken, rpc },
   } = getBotConfig();
   if (!incentiveMode) {
@@ -34,13 +34,13 @@ export const incentivizeComments = async () => {
     return;
   }
 
-  const issueComments = await getAllIssueComments(issue.number);
+  const issueComments = await getAllIssueComments(issue.number, "html");
   logger.info(`Getting the issue comments done. comments: ${JSON.stringify(issueComments)}`);
   const issueCommentsByUser: Record<string, string[]> = {};
   for (const issueComment of issueComments) {
     const user = issueComment.user;
     if (user.type == UserType.Bot || user.login == assignee) continue;
-    issueCommentsByUser[user.login].push(issueComment.body);
+    issueCommentsByUser[user.login].push(issueComment.body_html);
   }
   const tokenSymbol = await getTokenSymbol(paymentToken, rpc);
   logger.info(`Filtering by the user type done. commentsByUser: ${JSON.stringify(issueCommentsByUser)}`);
@@ -54,7 +54,7 @@ export const incentivizeComments = async () => {
   for (const user of Object.keys(issueCommentsByUser)) {
     const comments = issueCommentsByUser[user];
     const commentsByNode = await parseComments(comments, ItemsToExclude);
-    const rewardValue = calculateRewardValue(commentsByNode, commentElementPricing);
+    const rewardValue = calculateRewardValue(commentsByNode, incentives);
     logger.debug(`Comment parsed for the user: ${user}. comments: ${JSON.stringify(commentsByNode)}, sum: ${rewardValue}`);
     const account = await getWalletAddress(user);
     const amountInETH = ((rewardValue * baseMultiplier) / 1000).toString();
@@ -77,7 +77,7 @@ export const incentivizeCreatorComment = async () => {
   const logger = getLogger();
   const {
     mode: { incentiveMode },
-    price: { commentElementPricing, issueCreatorMultiplier },
+    price: { incentives, issueCreatorMultiplier },
     payout: { paymentToken, rpc },
   } = getBotConfig();
   if (!incentiveMode) {
@@ -99,7 +99,7 @@ export const incentivizeCreatorComment = async () => {
     return;
   }
 
-  const description = await getIssueDescription(issue.number);
+  const description = await getIssueDescription(issue.number, "html");
   logger.info(`Getting the issue description done. description: ${description}`);
   const creator = issue.user;
   if (creator?.type === UserType.Bot || creator?.login === issue?.assignee) {
@@ -108,7 +108,7 @@ export const incentivizeCreatorComment = async () => {
   }
 
   const tokenSymbol = await getTokenSymbol(paymentToken, rpc);
-  const result = await generatePermitForComments(creator?.login, [description], issueCreatorMultiplier, commentElementPricing, tokenSymbol, issue.node_id);
+  const result = await generatePermitForComments(creator?.login, [description], issueCreatorMultiplier, incentives, tokenSymbol, issue.node_id);
 
   if (result.payoutUrl) {
     logger.info(`Permit url generated for creator. reward: ${result.payoutUrl}`);
@@ -123,13 +123,13 @@ const generatePermitForComments = async (
   user: string,
   comments: string[],
   multiplier: number,
-  commentElementPricing: Record<string, number>,
+  incentives: Incentives,
   tokenSymbol: string,
   node_id: string
 ): Promise<{ comment: string; payoutUrl?: string; amountInETH?: string }> => {
   const logger = getLogger();
   const commentsByNode = await parseComments(comments, ItemsToExclude);
-  const rewardValue = calculateRewardValue(commentsByNode, commentElementPricing);
+  const rewardValue = calculateRewardValue(commentsByNode, incentives);
   logger.debug(`Comment parsed for the user: ${user}. comments: ${JSON.stringify(commentsByNode)}, sum: ${rewardValue}`);
   const account = await getWalletAddress(user);
   const amountInETH = ((rewardValue * multiplier) / 1000).toString();
@@ -146,17 +146,26 @@ const generatePermitForComments = async (
  * @dev Calculates the reward values for a given comments. We'll improve the formula whenever we get the better one.
  *
  * @param comments - The comments to calculate the reward for
- * @param commentElementPricing - The basic price table for reward calculation
+ * @param incentives - The basic price table for reward calculation
  * @returns - The reward value
  */
-const calculateRewardValue = (comments: Record<string, string[]>, commentElementPricing: CommentElementPricing): number => {
+const calculateRewardValue = (comments: Record<string, string[]>, incentives: Incentives): number => {
   let sum = 0;
   for (const key of Object.keys(comments)) {
-    const rewardValue = commentElementPricing[key];
     const value = comments[key];
-    if (key == MarkdownItem.Text || key == MarkdownItem.Paragraph) {
-      sum += value.length * rewardValue;
+
+    // if it's a text node calculate word count and multiply with the reward value
+    if (key == "#text") {
+      const wordReward = incentives.comment.totals.word;
+      if (!wordReward) {
+        continue;
+      }
+      sum += value.map((str) => str.trim().split(" ").length).reduce((totalWords, wordCount) => totalWords + wordCount, 0) * wordReward;
     } else {
+      const rewardValue = incentives.comment.elements[key];
+      if (!rewardValue) {
+        continue;
+      }
       sum += rewardValue;
     }
   }

src/helpers/comment.ts

@@ -1,44 +1,36 @@
-type MdastNode = {
-  type: string;
-  value: string;
-  children: MdastNode[];
+import parse5 from "parse5";
+
+type Node = {
+  nodeName: string;
+  tagName?: string;
+  value?: string;
+  childNodes?: Node[];
 };
-const cachedResult: Record<string, string[]> = {};
-const traverse = (node: MdastNode, itemsToExclude: string[]): Record<string, string[]> => {
-  if (!cachedResult[node.type]) {
-    cachedResult[node.type] = [];
+
+const traverse = (result: Record<string, string[]>, node: Node, itemsToExclude: string[]): Record<string, string[]> => {
+  if (itemsToExclude.includes(node.nodeName)) {
+    return result;
   }
 
-  if (!itemsToExclude.includes(node.type)) {
-    // skip pushing if the node type has been excluded
-    cachedResult[node.type].push(node.value);
-  } else if (node.children.length > 0) {
-    node.children.forEach((child) => traverse(child, itemsToExclude));
+  if (!result[node.nodeName]) {
+    result[node.nodeName] = [];
   }
 
-  return cachedResult;
-};
+  result[node.nodeName].push(node.value ?? "");
+
+  if (node.childNodes && node.childNodes.length > 0) {
+    node.childNodes.forEach((child) => traverse(result, child, itemsToExclude));
+  }
 
-export const parseComments = async (comments: string[], itemsToExclude: string[]): Promise<Record<string, string[]>> => {
-  const { fromMarkdown } = await import("mdast-util-from-markdown");
-  const { gfmFromMarkdown } = await import("mdast-util-gfm");
-  const { gfm } = await import("micromark-extension-gfm");
+  return result;
+};
 
+export const parseComments = (comments: string[], itemsToExclude: string[]): Record<string, string[]> => {
   const result: Record<string, string[]> = {};
+
   for (const comment of comments) {
-    const tree = fromMarkdown(comment, {
-      extensions: [gfm()],
-      mdastExtensions: [gfmFromMarkdown()],
-    });
-
-    const parsedContent = traverse(tree as MdastNode, itemsToExclude);
-    for (const key of Object.keys(parsedContent)) {
-      if (Object.keys(result).includes(key)) {
-        result[key].push(...parsedContent[key]);
-      } else {
-        result[key] = parsedContent[key];
-      }
-    }
+    const fragment = parse5.parseFragment(comment);
+    traverse(result, fragment as Node, itemsToExclude);
   }
 
   return result;

src/helpers/issue.ts

@@ -175,7 +175,7 @@ export const getCommentsOfIssue = async (issue_number: number): Promise<Comment[
   return result;
 };
 
-export const getIssueDescription = async (issue_number: number): Promise<string> => {
+export const getIssueDescription = async (issue_number: number, format: "raw" | "html" | "text" = "raw"): Promise<string> => {
   const context = getBotContext();
   const logger = getLogger();
   const payload = context.payload as Payload;
@@ -186,17 +186,32 @@ export const getIssueDescription = async (issue_number: number): Promise<string>
       owner: payload.repository.owner.login,
       repo: payload.repository.name,
       issue_number: issue_number,
+      mediaType: {
+        format,
+      },
     });
 
     await checkRateLimitGit(response?.headers);
-    if (response.data.body) result = response.data.body;
+    if (response.data.body) {
+      switch (format) {
+        case "raw":
+          result = response.data.body;
+          break;
+        case "html":
+          result = response.data.body_html ?? "";
+          break;
+        case "text":
+          result = response.data.body_text ?? "";
+          break;
+      }
+    }
   } catch (e: unknown) {
     logger.debug(`Getting issue description failed!, reason: ${e}`);
   }
   return result;
 };
 
-export const getAllIssueComments = async (issue_number: number): Promise<Comment[]> => {
+export const getAllIssueComments = async (issue_number: number, format: "raw" | "html" | "text" | "full" = "raw"): Promise<Comment[]> => {
   const context = getBotContext();
   const payload = context.payload as Payload;
 
@@ -211,6 +226,9 @@ export const getAllIssueComments = async (issue_number: number): Promise<Comment
         issue_number: issue_number,
         per_page: 100,
         page: page_number,
+        mediaType: {
+          format,
+        },
       });
 
       await checkRateLimitGit(response?.headers);

src/types/config.ts

@@ -7,15 +7,25 @@ const LabelItemSchema = Type.Object({
 });
 export type LabelItem = Static<typeof LabelItemSchema>;
 
-const CommentElementPricingSchema = Type.Record(Type.String(), Type.Number());
-export type CommentElementPricing = Static<typeof CommentElementPricingSchema>;
+const CommentIncentivesSchema = Type.Object({
+  elements: Type.Record(Type.String(), Type.Number()),
+  totals: Type.Object({
+    word: Type.Number(),
+  }),
+});
+export type CommentIncentives = Static<typeof CommentIncentivesSchema>;
+
+const IncentivesSchema = Type.Object({
+  comment: CommentIncentivesSchema,
+});
+export type Incentives = Static<typeof IncentivesSchema>;
 
 export const PriceConfigSchema = Type.Object({
   baseMultiplier: Type.Number(),
   issueCreatorMultiplier: Type.Number(),
   timeLabels: Type.Array(LabelItemSchema),
   priorityLabels: Type.Array(LabelItemSchema),
-  commentElementPricing: CommentElementPricingSchema,
+  incentives: IncentivesSchema,
   defaultLabels: Type.Array(Type.String()),
 });
 export type PriceConfig = Static<typeof PriceConfigSchema>;

src/types/payload.ts

@@ -234,6 +234,8 @@ export const CommentSchema = Type.Object({
   updated_at: Type.String({ format: "date-time" }),
   author_association: Type.String(),
   body: Type.String(),
+  body_html: Type.String(),
+  body_text: Type.String(),
 });
 
 export type Comment = Static<typeof CommentSchema>;

src/utils/helpers.ts

@@ -1,6 +1,5 @@
 import { DEFAULT_NETWORK_ID, DefaultPriceConfig } from "../configs";
-import { CommentElementPricing } from "../types";
-import { WideLabel, WideOrgConfig, WideRepoConfig } from "./private";
+import { Incentives, WideLabel, WideOrgConfig, WideRepoConfig } from "./private";
 
 export const getNetworkId = (parsedRepo: WideRepoConfig | undefined, parsedOrg: WideOrgConfig | undefined): number => {
   if (parsedRepo && parsedRepo["evm-network-id"] && !Number.isNaN(Number(parsedRepo["evm-network-id"]))) {
@@ -52,13 +51,13 @@ export const getPriorityLabels = (parsedRepo: WideRepoConfig | undefined, parsed
   }
 };
 
-export const getCommentItemPrice = (parsedRepo: WideRepoConfig | undefined, parsedOrg: WideOrgConfig | undefined): CommentElementPricing => {
-  if (parsedRepo && parsedRepo["comment-element-pricing"]) {
-    return parsedRepo["comment-element-pricing"];
-  } else if (parsedOrg && parsedOrg["comment-element-pricing"]) {
-    return parsedOrg["comment-element-pricing"];
+export const getIncentives = (parsedRepo: WideRepoConfig | undefined, parsedOrg: WideOrgConfig | undefined): Incentives => {
+  if (parsedRepo && parsedRepo["incentives"]) {
+    return parsedRepo["incentives"];
+  } else if (parsedOrg && parsedOrg["incentives"]) {
+    return parsedOrg["incentives"];
   } else {
-    return DefaultPriceConfig["commentElementPricing"];
+    return DefaultPriceConfig["incentives"];
   }
 };