feat(frontend): support datasource filtering (#326)

Implement UI part for #325

frontend/app/src/api/documents.ts

@@ -1,7 +1,18 @@
+import { indexStatuses } from '@/api/rag';
 import { authenticationHeaders, handleResponse, type Page, type PageParams, requestUrl, zodPage } from '@/lib/request';
 import { zodJsonDate } from '@/lib/zod';
 import { z, type ZodType } from 'zod';
 
+export const mimeTypes = [
+  { name: 'Text', value: 'text/plain' },
+  { name: 'Markdown', value: 'text/markdown' },
+  { name: 'Pdf', value: 'application/pdf' },
+  { name: 'Microsoft Word (docx)', value: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' },
+  { name: 'Microsoft PowerPoint (pptx)', value: 'application/vnd.openxmlformats-officedocument.presentationml.presentation' },
+] as const satisfies MimeType[];
+
+const mimeValues: (typeof mimeTypes)[number]['value'] = mimeTypes.map(m => m.value) as never;
+
 export interface Document {
   id: number,
   name: string,
@@ -34,9 +45,33 @@ const documentSchema = z.object({
   data_source_id: z.number(),
 }) satisfies ZodType<Document, any, any>;
 
-export async function listDocuments ({ page = 1, size = 10, query, data_source_id }: PageParams & { data_source_id?: number, query?: string } = {}): Promise<Page<Document>> {
-  return await fetch(requestUrl('/api/v1/admin/documents', { page, size, data_source_id, query }), {
+const zDate = z.coerce.date().or(z.literal('').transform(() => undefined)).optional();
+
+export const listDocumentsFiltersSchema = z.object({
+  name: z.string().optional(),
+  source_uri: z.string().optional(),
+  data_source_id: z.coerce.number().optional(),
+  created_at_start: zDate,
+  created_at_end: zDate,
+  updated_at_start: zDate,
+  updated_at_end: zDate,
+  last_modified_at_start: zDate,
+  last_modified_at_end: zDate,
+  mime_type: z.enum(mimeValues).optional(),
+  index_status: z.enum(indexStatuses).optional(),
+});
+
+export type ListDocumentsTableFilters = z.infer<typeof listDocumentsFiltersSchema>;
+
+export async function listDocuments ({ page = 1, size = 10, ...filters }: PageParams & ListDocumentsTableFilters = {}): Promise<Page<Document>> {
+  return await fetch(requestUrl('/api/v1/admin/documents', { page, size, ...filters }), {
     headers: await authenticationHeaders(),
   })
     .then(handleResponse(zodPage(documentSchema)));
 }
+
+export interface MimeType {
+  name: string;
+  value: string;
+}
+

frontend/app/src/api/rag.ts

@@ -1,13 +1,17 @@
 import { authenticationHeaders, handleResponse, requestUrl } from '@/lib/request';
 import { z, type ZodType } from 'zod';
 
-export type IndexProgress = {
-  not_started?: number
-  pending?: number
-  running?: number
-  completed?: number
-  failed?: number
-}
+export const indexStatuses = [
+  'not_started',
+  'pending',
+  'running',
+  'completed',
+  'failed',
+] as const;
+
+export type IndexStatus = typeof indexStatuses[number];
+
+export type IndexProgress = Partial<Record<IndexStatus, number>>
 
 export type IndexTotalStats = {
   total: number

frontend/app/src/components/data-table-remote.tsx

@@ -22,6 +22,7 @@ interface DataTableRemoteProps<TData, TValue> {
   idColumn: keyof TData;
   apiKey: string;
   api: (page: PageParams, options: PageApiOptions) => Promise<Page<TData>>;
+  apiDeps?: unknown[];
   columns: ColumnDef<TData, TValue>[];
   selectable?: boolean;
   batchOperations?: (rows: string[], revalidate: () => void) => ReactNode;
@@ -38,6 +39,7 @@ export function DataTableRemote<TData, TValue> ({
   api,
   apiKey,
   columns,
+  apiDeps = [],
   selectable = false,
   batchOperations,
   refreshInterval,
@@ -70,7 +72,7 @@ export function DataTableRemote<TData, TValue> ({
 
   useEffect(() => {
     void mutate();
-  }, [pagination.pageSize, pagination.pageIndex, globalFilter]);
+  }, [pagination.pageSize, pagination.pageIndex, globalFilter, ...apiDeps]);
 
   // Column definitions.
   columns = useMemo(() => {

frontend/app/src/components/documents/documents-table-filters.tsx

@@ -0,0 +1,178 @@
+import { type Document, listDocumentsFiltersSchema, type ListDocumentsTableFilters, mimeTypes } from '@/api/documents';
+import { indexStatuses } from '@/api/rag';
+import { Button } from '@/components/ui/button';
+import { Collapsible, CollapsibleContent, CollapsibleTrigger } from '@/components/ui/collapsible';
+import { Form, FormControl, FormField, FormItem, FormLabel, FormMessage } from '@/components/ui/form';
+import { Input } from '@/components/ui/input';
+import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select';
+import { zodResolver } from '@hookform/resolvers/zod';
+import { Table as ReactTable } from '@tanstack/react-table';
+import { capitalCase } from 'change-case-all';
+import { ChevronDownIcon } from 'lucide-react';
+import { useForm } from 'react-hook-form';
+
+export function DocumentsTableFilters ({ onFilterChange }: { table: ReactTable<Document>, onFilterChange: (data: ListDocumentsTableFilters) => void }) {
+  const form = useForm({
+    resolver: zodResolver(listDocumentsFiltersSchema),
+  });
+
+  const onSubmit = form.handleSubmit((data) => {
+    onFilterChange?.(data);
+  });
+
+  return (
+    <Form {...form}>
+      <form className="space-y-4" onSubmit={onSubmit}>
+        <FormField
+          name="name"
+          render={({ field }) => (
+            <FormItem>
+              <FormControl>
+                <Input {...field} value={field.value ?? ''} placeholder="Search..." />
+              </FormControl>
+              <FormMessage />
+            </FormItem>
+          )}
+        />
+        <Collapsible>
+          <CollapsibleTrigger className="group text-sm flex items-center py-1.5 hover:underline focus:underline outline-none">
+            <ChevronDownIcon className="size-4 mr-1 transition-transform group-data-[state=open]:rotate-180" />
+            Advanced Filters
+          </CollapsibleTrigger>
+          <CollapsibleContent className="py-2 space-y-4">
+            <FormField
+              name="source_uri"
+              render={({ field }) => (
+                <FormItem>
+                  <FormControl>
+                    <Input {...field} value={field.value ?? ''} placeholder="Search Source URI..." />
+                  </FormControl>
+                  <FormMessage />
+                </FormItem>
+              )}
+            />
+            <div className="grid grid-cols-2 gap-4">
+              <FormField
+                name="mime_type"
+                render={({ field: { value, onChange, name, disabled, ...field } }) => (
+                  <FormItem>
+                    <FormControl>
+                      <Select value={value ?? ''} name={name} disabled={disabled} onValueChange={onChange}>
+                        <SelectTrigger {...field}>
+                          <SelectValue placeholder={<span className='text-muted-foreground'>Select Document Type...</span>} />
+                        </SelectTrigger>
+                        <SelectContent>
+                          {mimeTypes.map(mime => (
+                            <SelectItem key={mime.value} value={mime.value}>
+                              {mime.name}
+                            </SelectItem>
+                          ))}
+                        </SelectContent>
+                      </Select>
+                    </FormControl>
+                    <FormMessage />
+                  </FormItem>
+                )}
+              />
+              <FormField
+                name="index_status"
+                render={({ field: { value, onChange, name, disabled, ...field } }) => (
+                  <FormItem>
+                    <FormControl>
+                      <Select value={value ?? ''} name={name} disabled={disabled} onValueChange={onChange}>
+                        <SelectTrigger {...field}>
+                          <SelectValue placeholder={<span className='text-muted-foreground'>Select Document Type...</span>} />
+                        </SelectTrigger>
+                        <SelectContent>
+                          {indexStatuses.map(indexStatus => (
+                            <SelectItem key={indexStatus} value={indexStatus}>
+                              {capitalCase(indexStatus)}
+                            </SelectItem>
+                          ))}
+                        </SelectContent>
+                      </Select>
+                    </FormControl>
+                    <FormMessage />
+                  </FormItem>
+                )}
+              />
+              <FormField
+                name="created_at_start"
+                render={({ field }) => (
+                  <FormItem>
+                    <FormLabel>Created After</FormLabel>
+                    <FormControl>
+                      <Input {...field} value={field.value ?? ''} type="datetime-local" />
+                    </FormControl>
+                    <FormMessage />
+                  </FormItem>
+                )}
+              />
+              <FormField
+                name="created_at_end"
+                render={({ field }) => (
+                  <FormItem>
+                    <FormLabel>Created Before</FormLabel>
+                    <FormControl>
+                      <Input {...field} value={field.value ?? ''} type="datetime-local" />
+                    </FormControl>
+                    <FormMessage />
+                  </FormItem>
+                )}
+              />
+              <FormField
+                name="updated_at_start"
+                render={({ field }) => (
+                  <FormItem>
+                    <FormLabel>Updated After</FormLabel>
+                    <FormControl>
+                      <Input {...field} value={field.value ?? ''} type="datetime-local" />
+                    </FormControl>
+                    <FormMessage />
+                  </FormItem>
+                )}
+              />
+              <FormField
+                name="updated_at_end"
+                render={({ field }) => (
+                  <FormItem>
+                    <FormLabel>Updated Before</FormLabel>
+                    <FormControl>
+                      <Input {...field} value={field.value ?? ''} type="datetime-local" />
+                    </FormControl>
+                    <FormMessage />
+                  </FormItem>
+                )}
+              />
+              <FormField
+                name="last_modified_at_start"
+                render={({ field }) => (
+                  <FormItem>
+                    <FormLabel>Last Modified After</FormLabel>
+                    <FormControl>
+                      <Input {...field} value={field.value ?? ''} type="datetime-local" />
+                    </FormControl>
+                    <FormMessage />
+                  </FormItem>
+                )}
+              />
+              <FormField
+                name="last_modified_at_end"
+                render={({ field }) => (
+                  <FormItem>
+                    <FormLabel>Last Modified Before</FormLabel>
+                    <FormControl>
+                      <Input {...field} value={field.value ?? ''} type="datetime-local" />
+                    </FormControl>
+                    <FormMessage />
+                  </FormItem>
+                )}
+              />
+            </div>
+          </CollapsibleContent>
+        </Collapsible>
+        <Button type="submit">Search</Button>
+      </form>
+    </Form>
+  );
+}

frontend/app/src/components/documents/documents-table.tsx

@@ -1,17 +1,15 @@
 'use client';
 
-import { type Document, listDocuments } from '@/api/documents';
+import { type Document, listDocuments, type ListDocumentsTableFilters } from '@/api/documents';
 import { datetime } from '@/components/cells/datetime';
 import { mono } from '@/components/cells/mono';
 import { DatasourceCell } from '@/components/cells/reference';
 import { DataTableRemote } from '@/components/data-table-remote';
 import { DocumentPreviewDialog } from '@/components/document-viewer';
-import { Button } from '@/components/ui/button';
-import { Input } from '@/components/ui/input';
+import { DocumentsTableFilters } from '@/components/documents/documents-table-filters';
 import type { CellContext, ColumnDef } from '@tanstack/react-table';
 import { createColumnHelper } from '@tanstack/table-core';
-import isHotkey from 'is-hotkey';
-import { useRef } from 'react';
+import { useState } from 'react';
 
 const helper = createColumnHelper<Document>();
 
@@ -33,30 +31,18 @@ const columnsWithoutDatasource = [...columns];
 columnsWithoutDatasource.splice(1, 1);
 
 export function DocumentsTable ({ datasourceId }: { datasourceId?: number }) {
-  const searchRef = useRef<HTMLInputElement>(null);
+  const [filters, setFilters] = useState<ListDocumentsTableFilters>({});
+
+  console.log(filters);
   return (
     <DataTableRemote
-      toolbar={(({ setGlobalFilter }) => (
-        <div className="flex gap-2 items-center">
-          <Input
-            placeholder="Search URL..."
-            ref={searchRef}
-            onKeyDown={event => {
-              if (isHotkey('Enter', event)) {
-                setGlobalFilter(searchRef.current?.value ?? '');
-              }
-            }}
-          />
-          <Button onClick={() => {
-            setGlobalFilter(searchRef.current?.value ?? '');
-          }}>
-            Search
-          </Button>
-        </div>
+      toolbar={((table) => (
+        <DocumentsTableFilters table={table} onFilterChange={setFilters} />
       ))}
       columns={datasourceId != null ? columnsWithoutDatasource : columns}
       apiKey={datasourceId != null ? `api.datasource.${datasourceId}.documents` : 'api.documents.list'}
-      api={(params, { globalFilter }) => listDocuments({ ...params, data_source_id: datasourceId, query: globalFilter })}
+      api={(params) => listDocuments({ ...params, ...filters, data_source_id: datasourceId })}
+      apiDeps={[filters]}
       idColumn="id"
     />
   );

frontend/app/src/lib/request/params.ts

@@ -10,12 +10,20 @@ export function buildUrlParams (object: object) {
 
     if (value instanceof Array) {
       for (let item of value) {
-        usp.append(key, String(value));
+        usp.append(key, stringify(item));
       }
     } else {
-      usp.append(key, String(value));
+      usp.append(key, stringify(value));
     }
   }
 
   return usp;
 }
+
+function stringify (item: any) {
+  if (item instanceof Date) {
+    return item.toISOString();
+  } else {
+    return String(item);
+  }
+}