From 24f171e82d88608278d0ff54231bcb7a0d025b05 Mon Sep 17 00:00:00 2001 From: Jagger <634750802@qq.com> Date: Sat, 12 Oct 2024 12:08:41 +0800 Subject: [PATCH] feat(frontend): support datasource filtering (#326) Implement UI part for #325 --- frontend/app/src/api/documents.ts | 39 +++- frontend/app/src/api/rag.ts | 18 +- .../app/src/components/data-table-remote.tsx | 4 +- .../documents/documents-table-filters.tsx | 178 ++++++++++++++++++ .../components/documents/documents-table.tsx | 34 +--- frontend/app/src/lib/request/params.ts | 12 +- 6 files changed, 249 insertions(+), 36 deletions(-) create mode 100644 frontend/app/src/components/documents/documents-table-filters.tsx diff --git a/frontend/app/src/api/documents.ts b/frontend/app/src/api/documents.ts index 7a2e441c8..4cfe5ee5e 100644 --- a/frontend/app/src/api/documents.ts +++ b/frontend/app/src/api/documents.ts @@ -1,7 +1,18 @@ +import { indexStatuses } from '@/api/rag'; import { authenticationHeaders, handleResponse, type Page, type PageParams, requestUrl, zodPage } from '@/lib/request'; import { zodJsonDate } from '@/lib/zod'; import { z, type ZodType } from 'zod'; +export const mimeTypes = [ + { name: 'Text', value: 'text/plain' }, + { name: 'Markdown', value: 'text/markdown' }, + { name: 'Pdf', value: 'application/pdf' }, + { name: 'Microsoft Word (docx)', value: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' }, + { name: 'Microsoft PowerPoint (pptx)', value: 'application/vnd.openxmlformats-officedocument.presentationml.presentation' }, +] as const satisfies MimeType[]; + +const mimeValues: (typeof mimeTypes)[number]['value'] = mimeTypes.map(m => m.value) as never; + export interface Document { id: number, name: string, @@ -34,9 +45,33 @@ const documentSchema = z.object({ data_source_id: z.number(), }) satisfies ZodType; -export async function listDocuments ({ page = 1, size = 10, query, data_source_id }: PageParams & { data_source_id?: number, query?: string } = {}): Promise> { - return await fetch(requestUrl('/api/v1/admin/documents', { page, size, data_source_id, query }), { +const zDate = z.coerce.date().or(z.literal('').transform(() => undefined)).optional(); + +export const listDocumentsFiltersSchema = z.object({ + name: z.string().optional(), + source_uri: z.string().optional(), + data_source_id: z.coerce.number().optional(), + created_at_start: zDate, + created_at_end: zDate, + updated_at_start: zDate, + updated_at_end: zDate, + last_modified_at_start: zDate, + last_modified_at_end: zDate, + mime_type: z.enum(mimeValues).optional(), + index_status: z.enum(indexStatuses).optional(), +}); + +export type ListDocumentsTableFilters = z.infer; + +export async function listDocuments ({ page = 1, size = 10, ...filters }: PageParams & ListDocumentsTableFilters = {}): Promise> { + return await fetch(requestUrl('/api/v1/admin/documents', { page, size, ...filters }), { headers: await authenticationHeaders(), }) .then(handleResponse(zodPage(documentSchema))); } + +export interface MimeType { + name: string; + value: string; +} + diff --git a/frontend/app/src/api/rag.ts b/frontend/app/src/api/rag.ts index 4b914331f..0a8ff0e57 100644 --- a/frontend/app/src/api/rag.ts +++ b/frontend/app/src/api/rag.ts @@ -1,13 +1,17 @@ import { authenticationHeaders, handleResponse, requestUrl } from '@/lib/request'; import { z, type ZodType } from 'zod'; -export type IndexProgress = { - not_started?: number - pending?: number - running?: number - completed?: number - failed?: number -} +export const indexStatuses = [ + 'not_started', + 'pending', + 'running', + 'completed', + 'failed', +] as const; + +export type IndexStatus = typeof indexStatuses[number]; + +export type IndexProgress = Partial> export type IndexTotalStats = { total: number diff --git a/frontend/app/src/components/data-table-remote.tsx b/frontend/app/src/components/data-table-remote.tsx index 7bc447684..d0e4f59aa 100644 --- a/frontend/app/src/components/data-table-remote.tsx +++ b/frontend/app/src/components/data-table-remote.tsx @@ -22,6 +22,7 @@ interface DataTableRemoteProps { idColumn: keyof TData; apiKey: string; api: (page: PageParams, options: PageApiOptions) => Promise>; + apiDeps?: unknown[]; columns: ColumnDef[]; selectable?: boolean; batchOperations?: (rows: string[], revalidate: () => void) => ReactNode; @@ -38,6 +39,7 @@ export function DataTableRemote ({ api, apiKey, columns, + apiDeps = [], selectable = false, batchOperations, refreshInterval, @@ -70,7 +72,7 @@ export function DataTableRemote ({ useEffect(() => { void mutate(); - }, [pagination.pageSize, pagination.pageIndex, globalFilter]); + }, [pagination.pageSize, pagination.pageIndex, globalFilter, ...apiDeps]); // Column definitions. columns = useMemo(() => { diff --git a/frontend/app/src/components/documents/documents-table-filters.tsx b/frontend/app/src/components/documents/documents-table-filters.tsx new file mode 100644 index 000000000..bf2aa1b51 --- /dev/null +++ b/frontend/app/src/components/documents/documents-table-filters.tsx @@ -0,0 +1,178 @@ +import { type Document, listDocumentsFiltersSchema, type ListDocumentsTableFilters, mimeTypes } from '@/api/documents'; +import { indexStatuses } from '@/api/rag'; +import { Button } from '@/components/ui/button'; +import { Collapsible, CollapsibleContent, CollapsibleTrigger } from '@/components/ui/collapsible'; +import { Form, FormControl, FormField, FormItem, FormLabel, FormMessage } from '@/components/ui/form'; +import { Input } from '@/components/ui/input'; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select'; +import { zodResolver } from '@hookform/resolvers/zod'; +import { Table as ReactTable } from '@tanstack/react-table'; +import { capitalCase } from 'change-case-all'; +import { ChevronDownIcon } from 'lucide-react'; +import { useForm } from 'react-hook-form'; + +export function DocumentsTableFilters ({ onFilterChange }: { table: ReactTable, onFilterChange: (data: ListDocumentsTableFilters) => void }) { + const form = useForm({ + resolver: zodResolver(listDocumentsFiltersSchema), + }); + + const onSubmit = form.handleSubmit((data) => { + onFilterChange?.(data); + }); + + return ( +
+ + ( + + + + + + + )} + /> + + + + Advanced Filters + + + ( + + + + + + + )} + /> +
+ ( + + + + + + + )} + /> + ( + + + + + + + )} + /> + ( + + Created After + + + + + + )} + /> + ( + + Created Before + + + + + + )} + /> + ( + + Updated After + + + + + + )} + /> + ( + + Updated Before + + + + + + )} + /> + ( + + Last Modified After + + + + + + )} + /> + ( + + Last Modified Before + + + + + + )} + /> +
+ + + + + + ); +} \ No newline at end of file diff --git a/frontend/app/src/components/documents/documents-table.tsx b/frontend/app/src/components/documents/documents-table.tsx index 639d868d0..3a7a2db27 100644 --- a/frontend/app/src/components/documents/documents-table.tsx +++ b/frontend/app/src/components/documents/documents-table.tsx @@ -1,17 +1,15 @@ 'use client'; -import { type Document, listDocuments } from '@/api/documents'; +import { type Document, listDocuments, type ListDocumentsTableFilters } from '@/api/documents'; import { datetime } from '@/components/cells/datetime'; import { mono } from '@/components/cells/mono'; import { DatasourceCell } from '@/components/cells/reference'; import { DataTableRemote } from '@/components/data-table-remote'; import { DocumentPreviewDialog } from '@/components/document-viewer'; -import { Button } from '@/components/ui/button'; -import { Input } from '@/components/ui/input'; +import { DocumentsTableFilters } from '@/components/documents/documents-table-filters'; import type { CellContext, ColumnDef } from '@tanstack/react-table'; import { createColumnHelper } from '@tanstack/table-core'; -import isHotkey from 'is-hotkey'; -import { useRef } from 'react'; +import { useState } from 'react'; const helper = createColumnHelper(); @@ -33,30 +31,18 @@ const columnsWithoutDatasource = [...columns]; columnsWithoutDatasource.splice(1, 1); export function DocumentsTable ({ datasourceId }: { datasourceId?: number }) { - const searchRef = useRef(null); + const [filters, setFilters] = useState({}); + + console.log(filters); return ( ( -
- { - if (isHotkey('Enter', event)) { - setGlobalFilter(searchRef.current?.value ?? ''); - } - }} - /> - -
+ toolbar={((table) => ( + ))} columns={datasourceId != null ? columnsWithoutDatasource : columns} apiKey={datasourceId != null ? `api.datasource.${datasourceId}.documents` : 'api.documents.list'} - api={(params, { globalFilter }) => listDocuments({ ...params, data_source_id: datasourceId, query: globalFilter })} + api={(params) => listDocuments({ ...params, ...filters, data_source_id: datasourceId })} + apiDeps={[filters]} idColumn="id" /> ); diff --git a/frontend/app/src/lib/request/params.ts b/frontend/app/src/lib/request/params.ts index 015b3ba42..3136f97ed 100644 --- a/frontend/app/src/lib/request/params.ts +++ b/frontend/app/src/lib/request/params.ts @@ -10,12 +10,20 @@ export function buildUrlParams (object: object) { if (value instanceof Array) { for (let item of value) { - usp.append(key, String(value)); + usp.append(key, stringify(item)); } } else { - usp.append(key, String(value)); + usp.append(key, stringify(value)); } } return usp; } + +function stringify (item: any) { + if (item instanceof Date) { + return item.toISOString(); + } else { + return String(item); + } +}