feat: graphql pr fetch

.cspell.json

@@ -1,7 +1,7 @@
 {
   "$schema": "https://raw.githubusercontent.com/streetsidesoftware/cspell/main/cspell.schema.json",
   "version": "0.2",
-  "ignorePaths": ["**/*.json", "**/*.css", "node_modules", "**/*.log", "**/tests/__mocks__"],
+  "ignorePaths": ["**/*.json", "**/*.css", "node_modules", "**/*.log", "**/tests/**"],
   "useGitignore": true,
   "language": "en",
   "words": ["dataurl", "devpool", "outdir", "servedir", "ubiquibot", "tiktoken", "typebox", "supabase", "wxdai", "noopener", "knip", "hellip", "mswjs"],

.gitignore

@@ -10,3 +10,4 @@ static/dist
 .env
 junit.xml
 coverage
+test-dashboard.md

graphql.config.yml

@@ -0,0 +1,6 @@
+schema:
+  - https://api.github.com/graphql:
+      headers:
+        Authorization: Bearer ${GITHUB_TOKEN}
+documents: src/*
+projects: {}

jest.config.ts

@@ -11,6 +11,7 @@ const cfg: Config = {
   reporters: ["default", "jest-junit", "jest-md-dashboard"],
   coverageDirectory: "coverage",
   testTimeout: 10000,
+  roots: ["<rootDir>", "tests"],
 };
 
 export default cfg;

package.json

@@ -27,14 +27,16 @@
   "dependencies": {
     "@actions/core": "1.10.1",
     "@actions/github": "6.0.0",
+    "@octokit/graphql-schema": "15.25.0",
+    "@octokit/plugin-paginate-graphql": "5.2.2",
     "@octokit/plugin-retry": "6.0.1",
     "@octokit/rest": "20.1.0",
     "@octokit/webhooks": "13.2.7",
     "@sinclair/typebox": "0.32.23",
     "@supabase/supabase-js": "2.42.0",
     "@ubiquibot/permit-generation": "1.3.1",
-    "@ubiquity-dao/rpc-handler": "1.2.3",
-    "@ubiquity-dao/ubiquibot-logger": "1.3.0",
+    "@ubiquity-dao/rpc-handler": "1.3.0",
+    "@ubiquity-dao/ubiquibot-logger": "1.3.1",
     "decimal.js": "10.4.3",
     "dotenv": "16.4.5",
     "ethers": "^6.13.0",

src/data-collection/collect-linked-pulls.ts

@@ -1,77 +1,34 @@
-import { GitHubLinkEvent, isGitHubLinkEvent } from "../github-types";
-import { IssueParams, getAllTimelineEvents, parseGitHubUrl } from "../start";
+import { PullRequest, Repository, User } from "@octokit/graphql-schema";
+import { getOctokitInstance } from "../octokit";
+import { IssueParams } from "../start";
+import { LINKED_PULL_REQUESTS } from "../types/requests";
 
-export async function collectLinkedMergedPulls(issue: IssueParams) {
-  // normally we should only use this one to calculate incentives, because this specifies that the pull requests are merged (accepted)
-  // and that are also related to the current issue, no just mentioned by
-  const onlyPullRequests = await collectLinkedPulls(issue);
-  return onlyPullRequests.filter((event) => {
-    if (!event.source.issue.body) {
-      return false;
-    }
-    // Matches all keywords according to the docs:
-    // https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword
-    // Works on multiple linked issues, and matches #<number> or URL patterns
-    const linkedIssueRegex =
-      /\b(?:Close(?:s|d)?|Fix(?:es|ed)?|Resolve(?:s|d)?):?\s+(?:#(\d+)|https?:\/\/(?:www\.)?github\.com\/(?:[^/\s]+\/[^/\s]+\/(?:issues|pull)\/(\d+)))\b/gi;
-    // We remove the comments as they should not be part of the linked pull requests
-    const linkedPrUrls = event.source.issue.body.replace(/<!--[\s\S]+-->/, "").match(linkedIssueRegex);
-    if (!linkedPrUrls) {
-      return false;
-    }
-    let isClosingPr = false;
-    for (const linkedPrUrl of linkedPrUrls) {
-      const idx = linkedPrUrl.indexOf("#");
-      if (idx !== -1) {
-        isClosingPr = Number(linkedPrUrl.slice(idx + 1)) === issue.issue_number;
-      } else {
-        const url = linkedPrUrl.match(/https.+/)?.[0];
-        if (url) {
-          const linkedRepo = parseGitHubUrl(url);
-          isClosingPr =
-            linkedRepo.issue_number === issue.issue_number &&
-            linkedRepo.repo === issue.repo &&
-            linkedRepo.owner === issue.owner;
-        }
-      }
-      if (isClosingPr) break;
-    }
-    return isGitHubLinkEvent(event) && event.source.issue.pull_request?.merged_at && isClosingPr;
-  });
-}
-export async function collectLinkedPulls(issue: IssueParams) {
-  // this one was created to help with tests, but probably should not be used in the main code
-  const issueLinkEvents = await getLinkedEvents(issue);
-  const onlyConnected = eliminateDisconnects(issueLinkEvents);
-  return onlyConnected.filter((event) => isGitHubLinkEvent(event) && event.source.issue.pull_request);
-}
+type ClosedByPullRequestsReferences = {
+  node: Pick<PullRequest, "url" | "title" | "number" | "state" | "body"> & {
+    author: Pick<User, "login" | "id">;
+    repository: Pick<Repository, "owner" | "name">;
+  };
+};
 
-function eliminateDisconnects(issueLinkEvents: GitHubLinkEvent[]) {
-  // Track connections and disconnections
-  const connections = new Map<number, GitHubLinkEvent>(); // Use issue/pr number as key for easy access
-  const disconnections = new Map<number, GitHubLinkEvent>(); // Track disconnections
+type IssueWithClosedByPRs = {
+  repository: {
+    issue: {
+      closedByPullRequestsReferences: {
+        edges: ClosedByPullRequestsReferences[];
+      };
+    };
+  };
+};
 
-  issueLinkEvents.forEach((issueEvent: GitHubLinkEvent) => {
-    const issueNumber = issueEvent.source.issue.number as number;
+export async function collectLinkedMergedPull(issue: IssueParams) {
+  const octokit = getOctokitInstance();
+  const { owner, repo, issue_number } = issue;
 
-    if (issueEvent.event === "connected" || issueEvent.event === "cross-referenced") {
-      // Only add to connections if there is no corresponding disconnected event
-      if (!disconnections.has(issueNumber)) {
-        connections.set(issueNumber, issueEvent);
-      }
-    } else if (issueEvent.event === "disconnected") {
-      disconnections.set(issueNumber, issueEvent);
-      // If a disconnected event is found, remove the corresponding connected event
-      if (connections.has(issueNumber)) {
-        connections.delete(issueNumber);
-      }
-    }
+  const result = await octokit.graphql.paginate<IssueWithClosedByPRs>(LINKED_PULL_REQUESTS, {
+    owner,
+    repo,
+    issue_number,
   });
 
-  return Array.from(connections.values());
-}
-
-async function getLinkedEvents(params: IssueParams): Promise<GitHubLinkEvent[]> {
-  const issueEvents = await getAllTimelineEvents(params);
-  return issueEvents.filter(isGitHubLinkEvent);
+  return result.repository.issue.closedByPullRequestsReferences.edges.map((edge) => edge.node).slice(-1);
 }

src/github-types.ts

@@ -3,32 +3,10 @@ import { RestEndpointMethodTypes } from "@octokit/rest";
 export type GitHubIssue = RestEndpointMethodTypes["issues"]["get"]["response"]["data"];
 export type GitHubPullRequest = RestEndpointMethodTypes["pulls"]["get"]["response"]["data"];
 export type GitHubIssueComment = RestEndpointMethodTypes["issues"]["listComments"]["response"]["data"][0];
-// eslint-disable-next-line @typescript-eslint/no-unused-vars
-type GitHubLabel = RestEndpointMethodTypes["issues"]["listLabelsOnIssue"]["response"]["data"][0];
 export type GitHubIssueEvent = RestEndpointMethodTypes["issues"]["listEvents"]["response"]["data"][0];
 export type GitHubTimelineEvent = RestEndpointMethodTypes["issues"]["listEventsForTimeline"]["response"]["data"][0];
 export type GitHubRepository = RestEndpointMethodTypes["repos"]["get"]["response"]["data"];
 export type GitHubUser = RestEndpointMethodTypes["users"]["getByUsername"]["response"]["data"];
 export type GitHubPullRequestReviewState = RestEndpointMethodTypes["pulls"]["listReviews"]["response"]["data"][0];
 export type GitHubPullRequestReviewComment =
   RestEndpointMethodTypes["pulls"]["listReviewComments"]["response"]["data"][0];
-
-type LinkPullRequestDetail = {
-  url: "https://api.github.com/repos/ubiquibot/comment-incentives/pulls/25";
-  html_url: "https://github.com/ubiquibot/comment-incentives/pull/25";
-  diff_url: "https://github.com/ubiquibot/comment-incentives/pull/25.diff";
-  patch_url: "https://github.com/ubiquibot/comment-incentives/pull/25.patch";
-  merged_at: "2024-02-16T19:22:01Z";
-};
-
-type SourceIssueWithPullRequest =
-  | GitHubIssue
-  | ((GitHubPullRequest & { pull_request: LinkPullRequestDetail }) & { repository: GitHubRepository });
-
-export type GitHubLinkEvent = RestEndpointMethodTypes["issues"]["listEventsForTimeline"]["response"]["data"][0] & {
-  event: "connected" | "disconnected" | "cross-referenced";
-  source: { issue: SourceIssueWithPullRequest };
-};
-export function isGitHubLinkEvent(event: GitHubTimelineEvent): event is GitHubLinkEvent {
-  return "source" in event;
-}

src/issue-activity.ts

@@ -1,7 +1,7 @@
 import { CommentAssociation, CommentKind } from "./configuration/comment-types";
 import configuration from "./configuration/config-reader";
 import { DataCollectionConfiguration } from "./configuration/data-collection-config";
-import { collectLinkedMergedPulls } from "./data-collection/collect-linked-pulls";
+import { collectLinkedMergedPull } from "./data-collection/collect-linked-pulls";
 import {
   GitHubIssue,
   GitHubIssueComment,
@@ -46,19 +46,21 @@ export class IssueActivity {
   }
 
   private async _getLinkedReviews(): Promise<Review[]> {
-    const pulls = await collectLinkedMergedPulls(this._issueParams);
+    logger.debug("Trying to fetch linked pull-requests for", this._issueParams);
+    const pulls = await collectLinkedMergedPull(this._issueParams);
+    logger.debug("Collected linked pull-requests", { pulls });
     const promises = pulls
       .map(async (pull) => {
-        const repository = pull.source.issue.repository;
+        const repository = pull.repository;
 
         if (!repository) {
-          console.error(`No repository found for [${pull.source.issue.repository}]`);
+          logger.error(`No repository found for`, { ...pull.repository });
           return null;
         } else {
           const pullParams = {
             owner: repository.owner.login,
             repo: repository.name,
-            pull_number: pull.source.issue.number,
+            pull_number: pull.number,
           };
           const review = new Review(pullParams);
           await review.init();

src/octokit.ts

@@ -2,12 +2,13 @@ import { Octokit } from "@octokit/rest";
 import { retry } from "@octokit/plugin-retry";
 import program from "./parser/command-line";
 import configuration from "./configuration/config-reader";
+import { paginateGraphQL, paginateGraphQLInterface } from "@octokit/plugin-paginate-graphql";
 
-const customOctokit = Octokit.plugin(retry);
+const customOctokit = Octokit.plugin(retry, paginateGraphQL);
 
-let octokitInstance: Octokit | null = null;
+let octokitInstance: (Octokit & paginateGraphQLInterface) | null = null;
 
-function getOctokitInstance(): Octokit {
+function getOctokitInstance() {
   if (!octokitInstance) {
     octokitInstance = new customOctokit({
       auth: program.authToken,

src/types/requests.ts

@@ -0,0 +1,35 @@
+export const LINKED_PULL_REQUESTS = /* GraphQL */ `
+  query collectLinkedPullRequests($owner: String!, $repo: String!, $issue_number: Int!, $cursor: String) {
+    repository(owner: $owner, name: $repo) {
+      issue(number: $issue_number) {
+        id
+        closedByPullRequestsReferences(first: 10, includeClosedPrs: false, after: $cursor) {
+          edges {
+            node {
+              id
+              title
+              number
+              url
+              author {
+                login
+                ... on User {
+                  id: databaseId
+                }
+              }
+              repository {
+                owner {
+                  login
+                }
+                name
+              }
+            }
+          }
+          pageInfo {
+            hasNextPage
+            endCursor
+          }
+        }
+      }
+    }
+  }
+`;

tests/__mocks__/@octokit/plugin-paginate-graphql.js

@@ -0,0 +1,19 @@
+module.exports = {
+  paginateGraphQL() {
+    return {
+      graphql: {
+        paginate(query, args) {
+          return {
+            repository: {
+              issue: {
+                closedByPullRequestsReferences: {
+                  edges: [],
+                },
+              },
+            },
+          };
+        },
+      },
+    };
+  },
+};

tests/__mocks__/handlers.ts

@@ -10,8 +10,10 @@ import issueEvents2Get from "./routes/issue-events-2-get.json";
 import issueEventsGet from "./routes/issue-events-get.json";
 import issueTimelineGet from "./routes/issue-timeline-get.json";
 import issue69TimelineGet from "./routes/issue-69-timeline-get.json";
+import issue70CommentsGet from "./routes/issue-70-comments-get.json";
 import pullsCommentsGet from "./routes/pulls-comments-get.json";
 import pullsGet from "./routes/pulls-get.json";
+import pulls70Get from "./routes/issue-70-get.json";
 import pullsReviewsGet from "./routes/pulls-reviews-get.json";
 
 /**
@@ -57,6 +59,18 @@ export const handlers = [
   http.get("https://api.github.com/repos/ubiquibot/comment-incentives/pulls/25/comments", () => {
     return HttpResponse.json(pullsCommentsGet);
   }),
+  http.get("https://api.github.com/repos/ubiquity/work.ubq.fi/pulls/70", () => {
+    return HttpResponse.json(pulls70Get);
+  }),
+  http.get("https://api.github.com/repos/ubiquity/work.ubq.fi/pulls/70/reviews", () => {
+    return HttpResponse.json(pullsReviewsGet);
+  }),
+  http.get("https://api.github.com/repos/ubiquity/work.ubq.fi/pulls/70/comments", () => {
+    return HttpResponse.json([]);
+  }),
+  http.get("https://api.github.com/repos/ubiquity/work.ubq.fi/issues/70/comments", () => {
+    return HttpResponse.json(issue70CommentsGet);
+  }),
   http.get("https://api.github.com/users/:login", ({ params: { login } }) => {
     const user = db.users.findFirst({
       where: {