From d540cc80e66928c38e0f0521a85b5384f43cb746 Mon Sep 17 00:00:00 2001 From: Tim Wekken Date: Mon, 13 Jan 2025 09:46:26 -0800 Subject: [PATCH 1/3] Added functionality for viewing and running previously uploaded CSV tests from Github repo, Added functionality for adding/removing previously uploaded CSV tests to/from Github repo, Added functionality for running a bunch of different CSV test files at once, Added Github API support for saving CSV test files to the repo, Updated processing of CSV to return if all tests passed successfully or not, Separated adding new CSV file modal into its own file --- app/components/RuleManager/RuleManager.tsx | 3 + .../ScenarioCSV/NewScenarioCSV.module.css | 58 ++++ .../ScenarioCSV/NewScenarioCSV.tsx | 109 ++++++ .../ScenarioCSV/ScenarioCSV.module.css | 60 +--- .../ScenarioCSV/ScenarioCSV.tsx | 328 ++++++++++++++---- .../ScenariosManager/ScenariosManager.tsx | 14 +- app/types/csv.d.ts | 12 + app/utils/api.ts | 5 +- app/utils/githubApi.ts | 157 +++++++-- 9 files changed, 601 insertions(+), 145 deletions(-) create mode 100644 app/components/ScenariosManager/ScenarioCSV/NewScenarioCSV.module.css create mode 100644 app/components/ScenariosManager/ScenarioCSV/NewScenarioCSV.tsx create mode 100644 app/types/csv.d.ts diff --git a/app/components/RuleManager/RuleManager.tsx b/app/components/RuleManager/RuleManager.tsx index 9f38921..7f92e39 100644 --- a/app/components/RuleManager/RuleManager.tsx +++ b/app/components/RuleManager/RuleManager.tsx @@ -200,6 +200,9 @@ export default function RuleManager({ simulationContext={simulationContext} runSimulation={runSimulation} resultsOfSimulation={resultsOfSimulation} + branchName={ruleInfo.reviewBranch} + pathName={ruleInfo.filepath} + isInReviewMode={editing === RULE_VERSION.inReview} /> )} diff --git a/app/components/ScenariosManager/ScenarioCSV/NewScenarioCSV.module.css b/app/components/ScenariosManager/ScenarioCSV/NewScenarioCSV.module.css new file mode 100644 index 0000000..bd658e5 --- /dev/null +++ b/app/components/ScenariosManager/ScenarioCSV/NewScenarioCSV.module.css @@ -0,0 +1,58 @@ +.instructionsList { + list-style: none; + padding: 20px; + margin: 20px 0; + background-color: #f4f4f9; + border-radius: var(--border-radius); + border: 1px solid #eaeaea; +} + +.instructionsList li { + margin-bottom: 15px; + padding: 10px; + background: #fff; + border-radius: 6px; + box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1); + font-size: 16px; + line-height: 1.5; + position: relative; +} + +.instructionsList li::before { + content: counter(li); + counter-increment: li; + position: absolute; + left: -30px; + top: 50%; + transform: translateY(-50%); + background: var(--color-link-focus); + color: white; + border-radius: 50%; + width: 24px; + height: 24px; + display: flex; + align-items: center; + justify-content: center; +} + +.instructionsList a { + color: var(--color-link-focus); + text-decoration: none; +} + +.instructionsList a:hover { + text-decoration: underline; +} + +.instructionsList { + counter-reset: li; + max-width: calc(100% - 48px); +} + +.upload { + margin-right: 10px; +} + +.runButton { + margin-left: 10px; +} \ No newline at end of file diff --git a/app/components/ScenariosManager/ScenarioCSV/NewScenarioCSV.tsx b/app/components/ScenariosManager/ScenarioCSV/NewScenarioCSV.tsx new file mode 100644 index 0000000..eaf4027 --- /dev/null +++ b/app/components/ScenariosManager/ScenarioCSV/NewScenarioCSV.tsx @@ -0,0 +1,109 @@ +import { useState, useEffect } from "react"; +import { App, Modal, Button, Flex, Upload } from "antd"; +import { UploadOutlined } from "@ant-design/icons"; +import { DecisionGraphType } from "@gorules/jdm-editor"; +import { logError } from "@/app/utils/logger"; +import { getCSVForRuleRun } from "@/app/utils/api"; +import styles from "./NewScenarioCSV.module.css"; + +interface NewScenarioCSVProps { + openNewCSVModal: boolean; + jsonFile: string; + ruleContent?: DecisionGraphType; + confirmAddingNewCSVFile: (file: File) => void; + cancelAddingCSVFile: () => void; + runCSVScenarios: (fileToRun: File | null, filename: string) => void; +} + +export default function NewScenarioCSV({ + openNewCSVModal, + jsonFile, + ruleContent, + confirmAddingNewCSVFile, + cancelAddingCSVFile, + runCSVScenarios, +}: NewScenarioCSVProps) { + const { message } = App.useApp(); + + const [file, setFile] = useState(null); + const [uploadedFile, setUploadedFile] = useState(false); + + const handleDownloadScenarios = async () => { + try { + const csvContent = await getCSVForRuleRun(jsonFile, ruleContent); + message.success(`Scenario Testing Template: ${csvContent}`); + } catch (error: unknown) { + message.error("Error downloading scenarios."); + logError("Error downloading scenarios:", error instanceof Error ? error : "Unknown error occurred."); + } + }; + + const deleteCurrentCSV = () => { + setFile(null); + setUploadedFile(false); + }; + + useEffect(() => { + if (openNewCSVModal) { + deleteCurrentCSV(); + } + }, [openNewCSVModal]); + + return ( + file && confirmAddingNewCSVFile(file)} + onCancel={() => cancelAddingCSVFile()} + > + +
    +
  1. + Download a template CSV file:{" "} + +
    2. +
  2. Add additional scenarios to the CSV file
    3. +
  3. + Upload your edited CSV file with scenarios:{" "} + +
    4. +
  4. + Run the scenarios against the GO Rules JSON file:{" "} + +
    5. +
  5. Receive a csv file with the results! 🎉
    6. +
+ + + ); +} diff --git a/app/components/ScenariosManager/ScenarioCSV/ScenarioCSV.module.css b/app/components/ScenariosManager/ScenarioCSV/ScenarioCSV.module.css index 39c5340..7207e2c 100644 --- a/app/components/ScenariosManager/ScenarioCSV/ScenarioCSV.module.css +++ b/app/components/ScenariosManager/ScenarioCSV/ScenarioCSV.module.css @@ -1,57 +1,9 @@ -.instructionsList { - list-style: none; - padding: 20px; - margin: 20px 0; - background-color: #f4f4f9; - border-radius: var(--border-radius); - border: 1px solid #eaeaea; +.filenameColumn { + min-width: 400px; + word-break: break-all; } -.instructionsList li { - margin-bottom: 15px; - padding: 10px; - background: #fff; - border-radius: 6px; - box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1); - font-size: 16px; - line-height: 1.5; - position: relative; -} - -.instructionsList li::before { - content: counter(li); - counter-increment: li; - position: absolute; - left: -30px; - top: 50%; - transform: translateY(-50%); - background: var(--color-link-focus); - color: white; - border-radius: 50%; - width: 24px; - height: 24px; - display: flex; - align-items: center; - justify-content: center; -} - -.instructionsList a { - color: var(--color-link-focus); - text-decoration: none; -} - -.instructionsList a:hover { - text-decoration: underline; -} - -.instructionsList { - counter-reset: li; -} - -.upload { - margin-right: 10px; -} - -.runButton { - margin-left: 10px; +.runResultIcon { + display: block !important; + font-size: 20px; } \ No newline at end of file diff --git a/app/components/ScenariosManager/ScenarioCSV/ScenarioCSV.tsx b/app/components/ScenariosManager/ScenarioCSV/ScenarioCSV.tsx index 27f2d58..db0cade 100644 --- a/app/components/ScenariosManager/ScenarioCSV/ScenarioCSV.tsx +++ b/app/components/ScenariosManager/ScenarioCSV/ScenarioCSV.tsx @@ -1,97 +1,289 @@ -import { useState } from "react"; -import { Button, Flex, Upload, message } from "antd"; -import { UploadOutlined } from "@ant-design/icons"; +import { useState, useEffect } from "react"; +import dayjs from "dayjs"; +import axios from "axios"; +import { App, Button, Flex, Spin, Table, TableProps } from "antd"; +import { CheckCircleFilled, CloseCircleFilled } from "@ant-design/icons"; import { DecisionGraphType } from "@gorules/jdm-editor"; import { logError } from "@/app/utils/logger"; -import { uploadCSVAndProcess, getCSVForRuleRun } from "@/app/utils/api"; +import { CSVRow, CSVRowData } from "@/app/types/csv"; +import { uploadCSVAndProcess } from "@/app/utils/api"; +import { getCSVTestFilesFromBranch, addCSVTestFileToReview, removeCSVTestFileFromReview } from "@/app/utils/githubApi"; +import NewScenarioCSV from "./NewScenarioCSV"; import styles from "./ScenarioCSV.module.css"; interface ScenarioCSVProps { jsonFile: string; ruleContent?: DecisionGraphType; + branchName?: string; + pathName: string; + isInReviewMode: boolean; } -export default function ScenarioCSV({ jsonFile, ruleContent }: ScenarioCSVProps) { - const [file, setFile] = useState(null); - const [uploadedFile, setUploadedFile] = useState(false); +export default function ScenarioCSV({ jsonFile, ruleContent, branchName, pathName, isInReviewMode }: ScenarioCSVProps) { + const { message } = App.useApp(); + const [openNewCSVModal, setOpenNewCSVModal] = useState(false); + const [isLoadingInTestFiles, setIsLoadingInTestFiles] = useState(true); + const [githubCSVTestsData, setGithubCSVTestsData] = useState([]); + const [localTestFiles, setLocalTestFiles] = useState([]); + const [csvTableData, setCSVTableData] = useState([]); + const [scenarioRunResults, setScenarioRunResults] = useState>({}); + const [currentlySelectedRows, setCurrentlySelectedRows] = useState([]); - const handleRunUploadScenarios = async () => { - if (!file) { + /** + * Create a new row for the table + */ + const createRow = (fileRowData: CSVRowData, isLocal: boolean = false): CSVRow => { + const { filename, downloadFile, lastUpdated, updatedBy } = fileRowData; + return { + key: filename, + filename, + downloadFile, + lastUpdated, + updatedBy, + actions: ( + + + {isInReviewMode && + (isLocal ? ( + + ) : ( + + ))} + {isLocal && ( + + )} + + ), + }; + }; + + /** + * Select multiple rows to run bulk operations + */ + const rowSelection: TableProps["rowSelection"] = { + onChange: (selectedRowKeys: React.Key[], selectedRows: CSVRow[]) => { + console.log("Selected rows: ", selectedRows); + setCurrentlySelectedRows(selectedRows); + }, + }; + + /** + * Run CSV Scenarios for a test file + */ + const runCSVScenarios = async (fileToRun: File | string | null, filename: string) => { + if (!fileToRun) { message.error("No file uploaded."); return; } try { - const csvContent = await uploadCSVAndProcess(file, jsonFile, ruleContent); - message.success(`Scenarios Test: ${csvContent}`); + if (typeof fileToRun === "string") { + const response = await axios.get(fileToRun as string, { + responseType: "blob", // Ensure the response is a Blob + }); + const blob = response.data; + fileToRun = new File([blob], filename, { type: blob.type }); + } + if (!fileToRun || typeof fileToRun === "string") { + throw new Error("Cannot get valid file"); + } + // Process it + const { successMessage, allTestsPassed } = await uploadCSVAndProcess(fileToRun, jsonFile, ruleContent); + // Update run result value in the table + setScenarioRunResults((prevValues) => { + const updatedValues = { ...prevValues }; + updatedValues[filename] = allTestsPassed; + return updatedValues; + }); + message.success(`Scenarios Test: ${successMessage}`); + } catch (error: any) { + message.error("Error processing scenarios"); + logError("Error processing scenarios:", error); + } + }; + + /** + * Run all scenarios for selected CSV test files + */ + const runAllSelectedCSVScenarios = () => { + currentlySelectedRows.forEach(({ downloadFile, filename }) => { + runCSVScenarios(downloadFile, filename); + }); + }; + + /** + * Upload to GitHub review + */ + const addCSVToReview = (fileRowData: CSVRowData) => { + try { + if (!branchName) { + throw new Error("No branch name exists"); + } + const { filename, downloadFile } = fileRowData; + if (!(downloadFile instanceof File)) { + throw new Error("No local file to add to review"); + } + const csvPathName = pathName.replace(/[^/]+$/, filename || ""); // Get csv path name from json file name + const reader = new FileReader(); // Use FileReader to encode file to base64 + reader.onload = async () => { + const base64Content = reader.result?.toString().split(",")[1]; + if (base64Content) { + await addCSVTestFileToReview(base64Content, branchName, csvPathName); + setGithubCSVTestsData([...githubCSVTestsData, fileRowData]); + deleteLocalCSV(filename); + } else { + throw new Error("Failed to encode file to base64"); + } + }; + reader.onerror = () => { + throw new Error("Error reading file"); + }; + if (downloadFile) { + reader.readAsDataURL(downloadFile); + } } catch (error: any) { - message.error("Error processing scenarios."); - logError("Error:", error); + message.error("Error adding CSV to review"); + console.error("Error adding CSV to review:", error); } }; - const handleDownloadScenarios = async () => { + /** + * Remove from GitHub review + */ + const removeCSVFromReview = async (filenameToRemove: string) => { try { - const csvContent = await getCSVForRuleRun(jsonFile, ruleContent); - message.success(`Scenario Testing Template: ${csvContent}`); + if (!branchName) { + throw new Error("No branch name exists"); + } + const csvPathName = pathName.replace(/[^/]+$/, filenameToRemove); + await removeCSVTestFileFromReview(branchName, csvPathName); + const githubFilesWithoutRemovedOne = githubCSVTestsData.filter(({ filename }) => filenameToRemove != filename); + setGithubCSVTestsData(githubFilesWithoutRemovedOne); + message.success("CSV removed from review"); } catch (error: any) { - message.error("Error downloading scenarios."); - logError("Error:", error); + message.error("Error removing CSV from review"); + console.error("Error removing CSV from review:", error); } }; + /** + * Delete CSV test file or remove it from review + */ + const deleteLocalCSV = (filenameToRemove: string) => { + const localFilesWithoutRemovedOne = localTestFiles.filter(({ name }) => name != filenameToRemove); + setLocalTestFiles(localFilesWithoutRemovedOne); + }; + + /** + * Add test file from New modal + */ + const confirmAddingNewCSVFile = (file: File) => { + setLocalTestFiles([...localTestFiles, file]); + setOpenNewCSVModal(false); + }; + + /** + * Gets scenario test files from GitHub + */ + useEffect(() => { + const getGithubCSVTestFiles = async () => { + try { + const testFiles: CSVRowData[] = await getCSVTestFilesFromBranch(branchName || "main", "/tests/util"); + setGithubCSVTestsData(testFiles); + setIsLoadingInTestFiles(false); + } catch (error: any) { + message.error("Error getting CSV test files from Github"); + console.error("Error getting CSV test files from Github:", error); + } + }; + getGithubCSVTestFiles(); + }, []); + + /** + * Generates all the rows of data for the table of scenario test files + */ + useEffect(() => { + if (githubCSVTestsData == null) return; + // Convert github test file data to rows in the table + const updatedCSVTableData: CSVRow[] = githubCSVTestsData.map((testFileData) => createRow(testFileData)); + // Adds any locally updated scenario test file to the table + localTestFiles.forEach((file) => + updatedCSVTableData.push( + createRow( + { + filename: file.name, + lastUpdated: file?.lastModified, + updatedBy: "You", + downloadFile: file, + }, + true + ) + ) + ); + // Add results of previous runs to the row + const updatedCSVTableDataWithRunResults: CSVRow[] = updatedCSVTableData.map((row: CSVRow) => ({ + ...row, + runResult: + scenarioRunResults[row.filename] == null ? null : scenarioRunResults[row.filename] ? ( + + ) : ( + + ), + })); + // Sets the updated data + setCSVTableData(updatedCSVTableDataWithRunResults); + }, [localTestFiles, githubCSVTestsData, scenarioRunResults]); + + const handleCancelAddingCSVFile = () => { + setOpenNewCSVModal(false); + }; + + if (isLoadingInTestFiles) { + return ( + +
+ + ); + } + return (
- -
    -
  1. - Download a template CSV file:{" "} - -
    2. -
  2. Add additional scenarios to the CSV file
    3. -
  3. - Upload your edited CSV file with scenarios:{" "} - -
    4. -
  4. - Run the scenarios against the GO Rules JSON file:{" "} - -
    5. -
  5. Receive a csv file with the results! 🎉
    6. -
+ + +
{text}
} + /> + dayjs(timestamp).format("MM/DD/YYYY")} + /> + + + +
+ + + + +
); } diff --git a/app/components/ScenariosManager/ScenariosManager.tsx b/app/components/ScenariosManager/ScenariosManager.tsx index 7a8a250..00e64e7 100644 --- a/app/components/ScenariosManager/ScenariosManager.tsx +++ b/app/components/ScenariosManager/ScenariosManager.tsx @@ -26,6 +26,9 @@ interface ScenariosManagerProps { setSimulationContext: (newContext: Record) => void; runSimulation: (newContext?: Record) => void; resultsOfSimulation?: Record | null; + branchName?: string; + pathName: string; + isInReviewMode: boolean; } export enum ScenariosManagerTabs { @@ -50,6 +53,9 @@ export default function ScenariosManager({ setSimulationContext, runSimulation, resultsOfSimulation, + branchName, + pathName, + isInReviewMode, }: ScenariosManagerProps) { const [resetTrigger, setResetTrigger] = useState(false); const [activeTabKey, setActiveTabKey] = useState( @@ -127,7 +133,13 @@ export default function ScenariosManager({ const csvTab = ( - + ); diff --git a/app/types/csv.d.ts b/app/types/csv.d.ts new file mode 100644 index 0000000..b5c4202 --- /dev/null +++ b/app/types/csv.d.ts @@ -0,0 +1,12 @@ +export interface CSVRowData { + filename: string; + downloadFile: string | File; + lastUpdated: number; + updatedBy: string; +} + +export interface CSVRow extends CSVRowData { + key: string; + actions: JSX.Element; + runResult?: JSX.Element | null; +} diff --git a/app/utils/api.ts b/app/utils/api.ts index 55d0f70..57baf6f 100644 --- a/app/utils/api.ts +++ b/app/utils/api.ts @@ -346,7 +346,7 @@ export const uploadCSVAndProcess = async ( file: File, filepath: string, ruleContent?: DecisionGraphType -): Promise => { +): Promise<{ successMessage: string; allTestsPassed: boolean }> => { try { const response = await axiosAPIInstance.post( `/scenario/evaluation/upload/`, @@ -363,11 +363,12 @@ export const uploadCSVAndProcess = async ( } ); + const allTestsPassed = response.headers["x-all-tests-passed"] === "true"; const timestamp = new Date().toISOString().replace(/:/g, "-").replace(/\.\d+/, ""); const filename = `${filepath.replace(".json", "")}_testing_${file.name.replace(".csv", "")}_${timestamp}.csv`; downloadFileBlob(response.data, "text/csv", filename); - return "File processed successfully"; + return { successMessage: "File processed successfully", allTestsPassed }; } catch (error) { logError(`Error processing CSV file: ${error}`); throw new Error("Error processing CSV file"); diff --git a/app/utils/githubApi.ts b/app/utils/githubApi.ts index c82dc9d..c55500d 100644 --- a/app/utils/githubApi.ts +++ b/app/utils/githubApi.ts @@ -1,8 +1,10 @@ import axios, { AxiosInstance } from "axios"; import { logError } from "./logger"; +import { CSVRowData } from "@/app/types/csv"; import { getShortFilenameOnly } from "./utils"; -const GITHUB_REPO_URL = "https://api.github.com/repos/bcgov/brms-rules"; +// TODO: CHANGE BACK +const GITHUB_REPO_URL = "https://api.github.com/repos/timwekkenbc/brms-rules-dev"; const GITHUB_REPO_OWNER = "bcgov"; const GITHUB_BASE_BRANCH = "dev"; @@ -43,11 +45,11 @@ export const isGithubAuthTokenValid = async (oauthToken?: string): Promise<{ val } initializeGithubAxiosInstance(oauthToken, githubAuthUsername); // Check that the user has authorized the organization (bcgov) properly - try { - await axiosGithubInstance.get(`${GITHUB_REPO_URL}/git/ref/heads/${GITHUB_BASE_BRANCH}`); - } catch (error) { - return { valid: false, reason: AuthFailureReasons.NO_ORG_ACCESS }; - } + // try { + // await axiosGithubInstance.get(`${GITHUB_REPO_URL}/git/ref/heads/${GITHUB_BASE_BRANCH}`); + // } catch (error) { + // return { valid: false, reason: AuthFailureReasons.NO_ORG_ACCESS }; + // } return { valid: true }; }; @@ -106,10 +108,15 @@ const createNewBranch = async (branchName: string, sha: string) => { } }; -// Attempt to fetch the file to get its SHA (if it exists) +/** + * Attempt to fetch the file to get its SHA (if it exists) + * @param branchName + * @param filePath + * @returns + */ const getFileIfAlreadyExists = async (branchName: string, filePath: string) => { try { - const contentsUrl = `${GITHUB_REPO_URL}/contents/rules/${encodeURIComponent(filePath)}`; + const contentsUrl = `${GITHUB_REPO_URL}/contents/${filePath}`; const getFileResponse = await axiosGithubInstance.get(contentsUrl, { params: { ref: branchName }, // Ensure we're checking the correct branch }); @@ -123,11 +130,22 @@ const getFileIfAlreadyExists = async (branchName: string, filePath: string) => { } }; -// Commit file changes to branch (or add new file entirely) -const commitFileToBranch = async (branchName: string, filePath: string, ruleContent: object, commitMessage: string) => { +// +/** + * Commit file changes to branch (or add new file entirely) + * @param branchName + * @param filePath + * @param contentBase64 + * @param commitMessage + * @returns + */ +const _commitFileToBranch = async ( + branchName: string, + filePath: string, + contentBase64: string, + commitMessage: string +) => { try { - // Encode JSON object to Base64 for GitHub API - const contentBase64 = Buffer.from(JSON.stringify(ruleContent, null, 2)).toString("base64"); // If the file already exists, get its sha const file = await getFileIfAlreadyExists(branchName, filePath); // Prepare the request body, including the SHA if the file exists @@ -140,7 +158,7 @@ const commitFileToBranch = async (branchName: string, filePath: string, ruleCont requestBody["sha"] = file?.sha; // Include the SHA to update the existing file } // Create or update the file - const contentsUrl = `${GITHUB_REPO_URL}/contents/rules/${filePath}`; + const contentsUrl = `${GITHUB_REPO_URL}/contents/${filePath}`; await axiosGithubInstance.put(contentsUrl, requestBody); console.log("File updated"); return file?.sha; @@ -205,7 +223,31 @@ export const getFileAsJsonIfAlreadyExists = async (branchName: string, filePath: } }; -// Do whole process of adding file to a branch and sending it off for review +/** + * Ensure that the branch exists before trying to use it + * @param branchName + */ +const _ensureBranchExists = async (branchName: string) => { + try { + const baseSha = await getShaOfLatestCommit(); + console.log("Base SHA:", baseSha); + const branchExists = await doesBranchExist(branchName); + if (!branchExists) { + await createNewBranch(branchName, baseSha); + } + } catch (error: any) { + logError("Error creating branch or committing file:", error); + throw error; + } +}; + +/** + * Do whole process of adding file to a branch and sending it off for review + * @param ruleContent + * @param branchName + * @param filePath + * @param reviewDescription + */ export const sendRuleForReview = async ( ruleContent: object, branchName: string, @@ -213,13 +255,10 @@ export const sendRuleForReview = async ( reviewDescription: string ) => { try { - const baseSha = await getShaOfLatestCommit(); - const branchExists = await doesBranchExist(branchName); - if (!branchExists) { - await createNewBranch(branchName, baseSha); - } + _ensureBranchExists(branchName); const commitMessage = generateCommitMessage(true, filePath); - await commitFileToBranch(branchName, filePath, ruleContent, commitMessage); + const contentBase64 = Buffer.from(JSON.stringify(ruleContent, null, 2)).toString("base64"); + _commitFileToBranch(branchName, `rules/${filePath}`, contentBase64, commitMessage); const prExists = await doesPRExist(branchName); if (!prExists) { await createPR(branchName, commitMessage, reviewDescription); @@ -229,3 +268,81 @@ export const sendRuleForReview = async ( throw error; } }; + +/** + * Get the CSV test files from a branch and path name + * @param branchName + * @param filePath + * @returns + */ +export const getCSVTestFilesFromBranch = async (branchName: string, filePath: string): Promise => { + try { + // Get the list of files + const url = `${GITHUB_REPO_URL}/contents/${filePath}`; + const filesResponse = await axiosGithubInstance.get(url, { + params: { ref: branchName }, // Ensure we're checking the correct branch + }); + const files = filesResponse.data; + // For each file, get the latest commit details to get update info + const fileDetails = await Promise.all( + files.map(async (file: any) => { + const commitsUrl = `${GITHUB_REPO_URL}/commits?path=${file.path}&per_page=1&sha=${branchName}`; + const commitsResponse = await axiosGithubInstance.get(commitsUrl); + console.log(file, commitsResponse); + const commits = commitsResponse.data; + return { + filename: file.name, + downloadFile: file.download_url, + lastUpdated: commits[0]?.commit?.author?.date, + updatedBy: commits[0]?.author?.login, + }; + }) + ); + return fileDetails; + } catch (error: any) { + logError(`Error getting ${filePath} as JSON for ${branchName}`, error); + throw error; + } +}; + +/** + * Add a new CSV test file to a review + * @param csvTests + * @param branchName + * @param filePath + */ +export const addCSVTestFileToReview = async (csvTests: any, branchName: string, filePath: string) => { + try { + _ensureBranchExists(branchName); + const commitMessage = `Adding tests ${filePath}`; + await _commitFileToBranch(branchName, `tests/${filePath}`, csvTests, commitMessage); + } catch { + throw new Error(`Failed to add test file ${filePath}`); + } +}; + +/** + * Remove CSV test file from a review + * @param branchName + * @param filePath + */ +export const removeCSVTestFileFromReview = async (branchName: string, filePath: string) => { + try { + _ensureBranchExists(branchName); + // If the file already exists, get its sha + const file = await getFileIfAlreadyExists(branchName, `tests/${filePath}`); + if (!file) { + throw new Error(`No file to remove: ${filePath}`); + } + // Delete the file + await axiosGithubInstance.delete(`${GITHUB_REPO_URL}/contents/tests/${filePath}`, { + data: { + message: `Removing tests ${filePath}`, + branch: branchName, + sha: file.sha, + }, + }); + } catch { + throw new Error(`Failed to remove test file ${filePath}`); + } +}; From 57133ee48080d160f716920533eac526b7ed1f5f Mon Sep 17 00:00:00 2001 From: Tim Wekken Date: Mon, 13 Jan 2025 11:42:10 -0800 Subject: [PATCH 2/3] Updated "editing" to "version", Updated local documentation to be in line with new changes to CSV tests --- app/components/RuleManager/RuleManager.tsx | 23 ++++---- .../subcomponents/LinkRuleComponent.tsx | 2 +- .../ScenarioCSV/NewScenarioCSV.tsx | 20 ++++++- .../ScenarioCSV/ScenarioCSV.tsx | 29 +++++++---- .../ScenarioHelper/ScenarioHelper.tsx | 52 ++++++++++++++----- .../ScenariosManager/ScenariosManager.tsx | 22 ++++---- app/rule/[ruleId]/embedded/page.tsx | 2 +- app/rule/[ruleId]/page.tsx | 2 +- app/rule/new/NewRule.tsx | 2 +- app/utils/githubApi.ts | 11 ++-- 10 files changed, 107 insertions(+), 58 deletions(-) diff --git a/app/components/RuleManager/RuleManager.tsx b/app/components/RuleManager/RuleManager.tsx index 7f92e39..9d5abfc 100644 --- a/app/components/RuleManager/RuleManager.tsx +++ b/app/components/RuleManager/RuleManager.tsx @@ -23,14 +23,14 @@ const RuleViewerEditor = dynamic(() => import("../RuleViewerEditor"), { ssr: fal interface RuleManagerProps { ruleInfo: RuleInfo; initialRuleContent?: DecisionGraphType; - editing?: string | boolean; + version: RULE_VERSION | boolean; showAllScenarioTabs?: boolean; } export default function RuleManager({ ruleInfo, initialRuleContent = DEFAULT_RULE_CONTENT, - editing = false, + version, showAllScenarioTabs = true, }: RuleManagerProps) { const { _id: ruleId, filepath: jsonFile } = ruleInfo; @@ -56,8 +56,8 @@ export default function RuleManager({ const [simulationContext, setSimulationContext] = useState>(); const [resultsOfSimulation, setResultsOfSimulation] = useState | null>(); const { setHasUnsavedChanges } = useLeaveScreenPopup(); - const canEditGraph = editing === RULE_VERSION.draft || editing === true; - const canEditScenarios = editing === RULE_VERSION.draft || editing === RULE_VERSION.inReview || editing === true; + const canEditGraph = version === RULE_VERSION.draft || version === true; + const canEditScenarios = version === RULE_VERSION.draft || version === RULE_VERSION.inReview || version === true; const updateRuleContent = (updatedRuleContent: DecisionGraphType) => { if (ruleContent !== updatedRuleContent) { @@ -146,21 +146,21 @@ export default function RuleManager({ ); } - const versionColour = getVersionColor(editing.toString()); + const versionColour = getVersionColor(version.toString()); return (
- {editing !== false && ( + {version !== false && ( - + setHasUnsavedChanges(false)} /> @@ -188,6 +188,7 @@ export default function RuleManager({ {scenarios && rulemap && ( )} diff --git a/app/components/RuleViewerEditor/subcomponents/LinkRuleComponent.tsx b/app/components/RuleViewerEditor/subcomponents/LinkRuleComponent.tsx index 8f8d61c..b5f7e5e 100644 --- a/app/components/RuleViewerEditor/subcomponents/LinkRuleComponent.tsx +++ b/app/components/RuleViewerEditor/subcomponents/LinkRuleComponent.tsx @@ -92,7 +92,7 @@ export default function LinkRuleComponent({ specification, id, isSelected, name, )} diff --git a/app/components/ScenariosManager/ScenarioCSV/NewScenarioCSV.tsx b/app/components/ScenariosManager/ScenarioCSV/NewScenarioCSV.tsx index eaf4027..7183086 100644 --- a/app/components/ScenariosManager/ScenarioCSV/NewScenarioCSV.tsx +++ b/app/components/ScenariosManager/ScenarioCSV/NewScenarioCSV.tsx @@ -43,6 +43,14 @@ export default function NewScenarioCSV({ setUploadedFile(false); }; + const handleOk = () => { + file && confirmAddingNewCSVFile(file); + }; + + const handleCancel = () => { + cancelAddingCSVFile(); + }; + useEffect(() => { if (openNewCSVModal) { deleteCurrentCSV(); @@ -53,8 +61,16 @@ export default function NewScenarioCSV({ file && confirmAddingNewCSVFile(file)} - onCancel={() => cancelAddingCSVFile()} + onOk={handleOk} + onCancel={handleCancel} + footer={[ + , + , + ]} >
    diff --git a/app/components/ScenariosManager/ScenarioCSV/ScenarioCSV.tsx b/app/components/ScenariosManager/ScenarioCSV/ScenarioCSV.tsx index db0cade..0a66e11 100644 --- a/app/components/ScenariosManager/ScenarioCSV/ScenarioCSV.tsx +++ b/app/components/ScenariosManager/ScenarioCSV/ScenarioCSV.tsx @@ -5,21 +5,22 @@ import { App, Button, Flex, Spin, Table, TableProps } from "antd"; import { CheckCircleFilled, CloseCircleFilled } from "@ant-design/icons"; import { DecisionGraphType } from "@gorules/jdm-editor"; import { logError } from "@/app/utils/logger"; +import { RuleInfo } from "@/app/types/ruleInfo"; import { CSVRow, CSVRowData } from "@/app/types/csv"; +import { RULE_VERSION } from "@/app/constants/ruleVersion"; import { uploadCSVAndProcess } from "@/app/utils/api"; import { getCSVTestFilesFromBranch, addCSVTestFileToReview, removeCSVTestFileFromReview } from "@/app/utils/githubApi"; import NewScenarioCSV from "./NewScenarioCSV"; import styles from "./ScenarioCSV.module.css"; interface ScenarioCSVProps { + ruleInfo: RuleInfo; jsonFile: string; ruleContent?: DecisionGraphType; - branchName?: string; - pathName: string; - isInReviewMode: boolean; + version: RULE_VERSION | boolean; } -export default function ScenarioCSV({ jsonFile, ruleContent, branchName, pathName, isInReviewMode }: ScenarioCSVProps) { +export default function ScenarioCSV({ ruleInfo, jsonFile, ruleContent, version }: ScenarioCSVProps) { const { message } = App.useApp(); const [openNewCSVModal, setOpenNewCSVModal] = useState(false); const [isLoadingInTestFiles, setIsLoadingInTestFiles] = useState(true); @@ -29,6 +30,9 @@ export default function ScenarioCSV({ jsonFile, ruleContent, branchName, pathNam const [scenarioRunResults, setScenarioRunResults] = useState>({}); const [currentlySelectedRows, setCurrentlySelectedRows] = useState([]); + const { filepath, reviewBranch } = ruleInfo; + const branchName = version === RULE_VERSION.inReview ? reviewBranch : version === RULE_VERSION.inDev ? "dev" : "main"; + /** * Create a new row for the table */ @@ -43,7 +47,7 @@ export default function ScenarioCSV({ jsonFile, ruleContent, branchName, pathNam actions: ( - {isInReviewMode && + {version === RULE_VERSION.inReview && (isLocal ? ( ) : ( @@ -126,7 +130,7 @@ export default function ScenarioCSV({ jsonFile, ruleContent, branchName, pathNam if (!(downloadFile instanceof File)) { throw new Error("No local file to add to review"); } - const csvPathName = pathName.replace(/[^/]+$/, filename || ""); // Get csv path name from json file name + const csvPathName = filepath.replace(/[^/]+$/, filename || ""); // Get csv path name from json file name const reader = new FileReader(); // Use FileReader to encode file to base64 reader.onload = async () => { const base64Content = reader.result?.toString().split(",")[1]; @@ -158,7 +162,7 @@ export default function ScenarioCSV({ jsonFile, ruleContent, branchName, pathNam if (!branchName) { throw new Error("No branch name exists"); } - const csvPathName = pathName.replace(/[^/]+$/, filenameToRemove); + const csvPathName = filepath.replace(/[^/]+$/, filenameToRemove); await removeCSVTestFileFromReview(branchName, csvPathName); const githubFilesWithoutRemovedOne = githubCSVTestsData.filter(({ filename }) => filenameToRemove != filename); setGithubCSVTestsData(githubFilesWithoutRemovedOne); @@ -191,7 +195,7 @@ export default function ScenarioCSV({ jsonFile, ruleContent, branchName, pathNam useEffect(() => { const getGithubCSVTestFiles = async () => { try { - const testFiles: CSVRowData[] = await getCSVTestFilesFromBranch(branchName || "main", "/tests/util"); + const testFiles: CSVRowData[] = await getCSVTestFilesFromBranch(branchName || "main", "tests/util"); setGithubCSVTestsData(testFiles); setIsLoadingInTestFiles(false); } catch (error: any) { @@ -200,6 +204,7 @@ export default function ScenarioCSV({ jsonFile, ruleContent, branchName, pathNam } }; getGithubCSVTestFiles(); + // eslint-disable-next-line react-hooks/exhaustive-deps }, []); /** @@ -235,6 +240,7 @@ export default function ScenarioCSV({ jsonFile, ruleContent, branchName, pathNam })); // Sets the updated data setCSVTableData(updatedCSVTableDataWithRunResults); + // eslint-disable-next-line react-hooks/exhaustive-deps }, [localTestFiles, githubCSVTestsData, scenarioRunResults]); const handleCancelAddingCSVFile = () => { @@ -252,7 +258,12 @@ export default function ScenarioCSV({ jsonFile, ruleContent, branchName, pathNam return (
    - +

    - The scenario's saved inputs for the rule are shown in the 'Inputs' window, and the final results - will be processed in real time through the current visible version of the rule, and returned in the - 'Decision' window. + The scenario's saved inputs for the rule are shown in the 'Inputs' window, and the + final results will be processed in real time through the current visible version of the rule, and + returned in the 'Decision' window.

    By running rules individually, you can track their progress as the scenario is processed by the @@ -102,8 +102,8 @@ export default function ScenariosHelper({ section }: ScenariosHelperProps) { scenario inputs, or delete a scenario.

    - Clicking on the edit button will redirect you to the 'Simulate manual inputs' tab to update the - inputs and expected results for a scenario. + Clicking on the edit button will redirect you to the 'Simulate manual inputs' tab to + update the inputs and expected results for a scenario.

    @@ -212,8 +212,8 @@ export default function ScenariosHelper({ section }: ScenariosHelperProps) {

    The simulate button runs your current inputs through the rule to generate results.

    - After a successful simulation, you'll have the option to save these inputs as a new scenario. This - includes a field to name your scenario and a save button to store it for future use. + After a successful simulation, you'll have the option to save these inputs as a new scenario. + This includes a field to name your scenario and a save button to store it for future use.

    @@ -296,8 +296,8 @@ export default function ScenariosHelper({ section }: ScenariosHelperProps) {

    1. Re-Run Scenarios

    - The 'Re-Run Scenarios' button allows you to run all saved scenarios against the current version of - the rule in view. + The 'Re-Run Scenarios' button allows you to run all saved scenarios against the current + version of the rule in view.

    This is particularly useful when making changes to a rule, as it allows you to verify that all @@ -318,7 +318,9 @@ export default function ScenariosHelper({ section }: ScenariosHelperProps) {

  1. No expected results were specified (default pass state)
    2. -
  2. A red X (✗) indicates the scenario's actual results don't match the expected results
    3. +
  3. + A red X (✗) indicates the scenario's actual results don't match the expected results +
    4. @@ -340,15 +342,22 @@ export default function ScenariosHelper({ section }: ScenariosHelperProps) {

    The table provides several controls for managing the view of your scenarios:

      -
    • 'Show Error Scenarios' - Instantly filters the list to display only scenarios with errors
      • -
    • 'Clear Filters and Sorters' - Resets all active filters and sorting to their default state
      • +
    • + 'Show Error Scenarios' - Instantly filters the list to display only scenarios with + errors +
      • +
    • + 'Clear Filters and Sorters' - Resets all active filters and sorting to their default + state +

    Additional filtering and sorting options:

    • Each column can be filtered based on its content
    • Columns can be sorted in ascending or descending order
    • - All filtering and sorting is visual only and doesn't affect the underlying scenarios or results + All filtering and sorting is visual only and doesn't affect the underlying scenarios or + results
    @@ -359,7 +368,22 @@ export default function ScenariosHelper({ section }: ScenariosHelperProps) { case ScenariosManagerTabs.CSVTab: return (
    -

    Follow the instructions within this tab to utilize CSV tests.

    +

    + Scenario CSV test files will show up in the table below. It will list both test files that are stored on + GitHub as well as locally uploaded ones. +

    +

    + You can run the scenarios for those test files by clicking the 'Run Scenarios' button in the + table. +

    +

    + To add a new CSV test file, click the '+ Create new CSV test file' button and follow the + instructions there. +

    +

    + When you are doing a review, the 'Add to Review' button will appear in the table and you can use + that to add your local test files to the review and have them stored in GitHub. +

    Specific details about the CSV Tests tab are available on The Hive:{" "} ) => void; runSimulation: (newContext?: Record) => void; resultsOfSimulation?: Record | null; - branchName?: string; - pathName: string; - isInReviewMode: boolean; + version: RULE_VERSION | boolean; } export enum ScenariosManagerTabs { @@ -41,6 +42,7 @@ export enum ScenariosManagerTabs { export default function ScenariosManager({ ruleId, + ruleInfo, jsonFile, ruleContent, rulemap, @@ -53,9 +55,7 @@ export default function ScenariosManager({ setSimulationContext, runSimulation, resultsOfSimulation, - branchName, - pathName, - isInReviewMode, + version, }: ScenariosManagerProps) { const [resetTrigger, setResetTrigger] = useState(false); const [activeTabKey, setActiveTabKey] = useState( @@ -133,13 +133,7 @@ export default function ScenariosManager({ const csvTab = ( - + ); @@ -199,6 +193,7 @@ export default function ScenariosManager({ scenarios, rulemap, ruleId, + ruleInfo, jsonFile, setSimulationContext, resultsOfSimulation, @@ -213,6 +208,7 @@ export default function ScenariosManager({ handleReset, scenarioName, setScenarios, + version, ]); return ( diff --git a/app/rule/[ruleId]/embedded/page.tsx b/app/rule/[ruleId]/embedded/page.tsx index a0f0660..b0ba615 100644 --- a/app/rule/[ruleId]/embedded/page.tsx +++ b/app/rule/[ruleId]/embedded/page.tsx @@ -11,6 +11,6 @@ export default async function Rule({ params: { ruleId } }: { params: { ruleId: s } return ( - + ); } diff --git a/app/rule/[ruleId]/page.tsx b/app/rule/[ruleId]/page.tsx index 5409059..cfdf3ad 100644 --- a/app/rule/[ruleId]/page.tsx +++ b/app/rule/[ruleId]/page.tsx @@ -45,7 +45,7 @@ export default async function Rule({ params: { ruleId }, searchParams }: Props)

    diff --git a/app/rule/new/NewRule.tsx b/app/rule/new/NewRule.tsx index d21da26..5b08a07 100644 --- a/app/rule/new/NewRule.tsx +++ b/app/rule/new/NewRule.tsx @@ -81,7 +81,7 @@ export default function NewRule() {
    {ruleInfo.filepath && ( - + )}
    diff --git a/app/utils/githubApi.ts b/app/utils/githubApi.ts index c55500d..85816b7 100644 --- a/app/utils/githubApi.ts +++ b/app/utils/githubApi.ts @@ -3,8 +3,7 @@ import { logError } from "./logger"; import { CSVRowData } from "@/app/types/csv"; import { getShortFilenameOnly } from "./utils"; -// TODO: CHANGE BACK -const GITHUB_REPO_URL = "https://api.github.com/repos/timwekkenbc/brms-rules-dev"; +const GITHUB_REPO_URL = "https://api.github.com/repos/bcgov/brms-rules"; const GITHUB_REPO_OWNER = "bcgov"; const GITHUB_BASE_BRANCH = "dev"; @@ -300,8 +299,12 @@ export const getCSVTestFilesFromBranch = async (branchName: string, filePath: st ); return fileDetails; } catch (error: any) { - logError(`Error getting ${filePath} as JSON for ${branchName}`, error); - throw error; + if (error.status == 404) { + return []; + } else { + logError(`Error getting ${filePath} as CSV for ${branchName}`, error); + throw error; + } } }; From 93af5610b770f09a31b6ea4b0a75f7a765fa0a4e Mon Sep 17 00:00:00 2001 From: Tim Wekken Date: Mon, 13 Jan 2025 16:50:01 -0800 Subject: [PATCH 3/3] Minor improvements to CSV testing including checking for name duplication --- .../ScenarioCSV/NewScenarioCSV.tsx | 23 ++++++++++++------- .../ScenarioCSV/ScenarioCSV.tsx | 2 +- .../ScenarioHelper/ScenarioHelper.tsx | 1 + app/utils/githubApi.ts | 10 ++++---- 4 files changed, 22 insertions(+), 14 deletions(-) diff --git a/app/components/ScenariosManager/ScenarioCSV/NewScenarioCSV.tsx b/app/components/ScenariosManager/ScenarioCSV/NewScenarioCSV.tsx index 7183086..5e2101b 100644 --- a/app/components/ScenariosManager/ScenarioCSV/NewScenarioCSV.tsx +++ b/app/components/ScenariosManager/ScenarioCSV/NewScenarioCSV.tsx @@ -13,6 +13,7 @@ interface NewScenarioCSVProps { confirmAddingNewCSVFile: (file: File) => void; cancelAddingCSVFile: () => void; runCSVScenarios: (fileToRun: File | null, filename: string) => void; + existingFilenames: string[]; } export default function NewScenarioCSV({ @@ -22,6 +23,7 @@ export default function NewScenarioCSV({ confirmAddingNewCSVFile, cancelAddingCSVFile, runCSVScenarios, + existingFilenames, }: NewScenarioCSVProps) { const { message } = App.useApp(); @@ -67,7 +69,7 @@ export default function NewScenarioCSV({ , - , ]} @@ -88,11 +90,17 @@ export default function NewScenarioCSV({ accept=".csv" multiple={false} maxCount={1} - customRequest={({ file, onSuccess }) => { - setFile(file as File); - message.success(`${(file as File).name} file uploaded successfully.`); - onSuccess && onSuccess("ok"); - setUploadedFile(true); + customRequest={({ file, onSuccess, onError }) => { + const fileName = (file as File).name; + if (!existingFilenames.includes(fileName)) { + setFile(file as File); + message.success(`${fileName} file uploaded successfully.`); + onSuccess && onSuccess("ok"); + setUploadedFile(true); + } else { + message.error("File name already exists"); + onError && onError(new Error("File name already exists")); + } }} onRemove={deleteCurrentCSV} showUploadList={true} @@ -106,7 +114,7 @@ export default function NewScenarioCSV({
  4. - Run the scenarios against the GO Rules JSON file:{" "} + Run the uploaded scenarios against the rule (Optional):{" "}
    5. -
  5. Receive a csv file with the results! 🎉
    6. diff --git a/app/components/ScenariosManager/ScenarioCSV/ScenarioCSV.tsx b/app/components/ScenariosManager/ScenarioCSV/ScenarioCSV.tsx index 0a66e11..cdb25b1 100644 --- a/app/components/ScenariosManager/ScenarioCSV/ScenarioCSV.tsx +++ b/app/components/ScenariosManager/ScenarioCSV/ScenarioCSV.tsx @@ -70,7 +70,6 @@ export default function ScenarioCSV({ ruleInfo, jsonFile, ruleContent, version } */ const rowSelection: TableProps["rowSelection"] = { onChange: (selectedRowKeys: React.Key[], selectedRows: CSVRow[]) => { - console.log("Selected rows: ", selectedRows); setCurrentlySelectedRows(selectedRows); }, }; @@ -294,6 +293,7 @@ export default function ScenarioCSV({ ruleInfo, jsonFile, ruleContent, version } confirmAddingNewCSVFile={confirmAddingNewCSVFile} cancelAddingCSVFile={handleCancelAddingCSVFile} runCSVScenarios={runCSVScenarios} + existingFilenames={csvTableData.map(({ filename }) => filename)} /> ); diff --git a/app/components/ScenariosManager/ScenarioHelper/ScenarioHelper.tsx b/app/components/ScenariosManager/ScenarioHelper/ScenarioHelper.tsx index 2af286c..1d747c8 100644 --- a/app/components/ScenariosManager/ScenarioHelper/ScenarioHelper.tsx +++ b/app/components/ScenariosManager/ScenarioHelper/ScenarioHelper.tsx @@ -376,6 +376,7 @@ export default function ScenariosHelper({ section }: ScenariosHelperProps) { You can run the scenarios for those test files by clicking the 'Run Scenarios' button in the table.

    +

    Any time you run scenarios you'll recieve a CSV file with the results.

    To add a new CSV test file, click the '+ Create new CSV test file' button and follow the instructions there. diff --git a/app/utils/githubApi.ts b/app/utils/githubApi.ts index 85816b7..3f894fa 100644 --- a/app/utils/githubApi.ts +++ b/app/utils/githubApi.ts @@ -44,11 +44,11 @@ export const isGithubAuthTokenValid = async (oauthToken?: string): Promise<{ val } initializeGithubAxiosInstance(oauthToken, githubAuthUsername); // Check that the user has authorized the organization (bcgov) properly - // try { - // await axiosGithubInstance.get(`${GITHUB_REPO_URL}/git/ref/heads/${GITHUB_BASE_BRANCH}`); - // } catch (error) { - // return { valid: false, reason: AuthFailureReasons.NO_ORG_ACCESS }; - // } + try { + await axiosGithubInstance.get(`${GITHUB_REPO_URL}/git/ref/heads/${GITHUB_BASE_BRANCH}`); + } catch (error) { + return { valid: false, reason: AuthFailureReasons.NO_ORG_ACCESS }; + } return { valid: true }; };