From b850883fbf29ab334907ca801b02540bce685a5b Mon Sep 17 00:00:00 2001 From: marknguyen1302 Date: Wed, 14 Aug 2024 04:18:38 +0700 Subject: [PATCH 1/7] feat: add how to integrate remote engine page --- docs/integrate-remote-engine | 87 ++++++++++++++++++++++++++++++++++++ sidebars.ts | 1 + 2 files changed, 88 insertions(+) create mode 100644 docs/integrate-remote-engine diff --git a/docs/integrate-remote-engine b/docs/integrate-remote-engine new file mode 100644 index 0000000..0b11968 --- /dev/null +++ b/docs/integrate-remote-engine @@ -0,0 +1,87 @@ +--- +title: Integrate Remote Engine +description: How to integrate remote engine +--- + +# Adding a Remote Engine Model to Cortex + +This document provides a step-by-step guide to adding a new engine to the Cortex codebase, similar to the `OpenAIEngineExtension`. + +## Steps to Add a New Engine + +### 1. Create a New Engine Extension File + +- Navigate to the `cortex-js/src/extensions` directory. +- Create a new file named `.engine.ts` (replace `` with the name of your engine). + +### 2. Implement the New Engine Extension Class + +Use the following template to implement your new engine extension class. Replace placeholders with appropriate values for your engine. + +```typescript +class EngineExtension extends OAIEngineExtension { + apiUrl = 'https://api..com/v1/chat/completions'; + name = ''; + productName = ' Inference Engine'; + description = 'This extension enables chat completion API calls'; + version = '0.0.1'; + apiKey?: string; +} +``` +### 3. Register the New Engine + +1. **Open** `cortex-js/src/extensions/extensions.module.ts`. + +2. **Register your new engine in the provider array:** + +```typescript +[ + new OpenAIEngineExtension(httpService, configUsecases, eventEmitter), + //... other remote engines + new EngineExtension(httpService, configUsecases, eventEmitter), +] +``` + +## Explanation of Key Properties and Methods + +### `apiUrl` +This is the URL endpoint for the new engine's API. It is used to make chat completion requests. + +### `name` +This is a unique identifier for the engine. It is used internally to reference the engine. + +### `productName` +This is a human-readable name for the engine. It is used for display purposes. + +### `description` +This provides a brief description of what the engine does. It is used for documentation and display purposes. + +### `version` +This indicates the version of the engine extension. It is used for version control and display purposes. + +### `eventEmmitter.on('config.updated')` +This is an event listener that listens for configuration updates. When the configuration for the engine is updated, this listener updates the `apiKey` and the engine's status. + +### `onLoad` +This method is called when the engine extension is loaded. It retrieves the engine's configuration (such as the `apiKey`) and sets the engine's status based on whether the `apiKey` is available. + +## Advanced: Transforming Payloads and Responses + +Some engines require custom transformations for the payload sent to the API and the response received from the API. This is achieved using the `transformPayload` and `transformResponse` methods. These methods allow you to modify the data structure to match the specific requirements of the engine. + +### `transformPayload` + +The `transformPayload` method is used to transform the data before sending it to the engine's API. This method takes the original payload and modifies it as needed. + +**Example: Anthropic Engine** + +In the Anthropic Engine, the `transformPayload` method extracts the system message and other messages, and includes additional parameters like `model`, `stream`, and `max_tokens`. + +### `transformResponse` + +The `transformResponse` method is used to transform the data received from the engine's API. This method processes the response and converts it into a format that the application can use. + +**Example: Anthropic Engine** + +In the Anthropic Engine, the `transformResponse` method handles both stream and non-stream responses. It processes the response data and converts it into a standardized format. + diff --git a/sidebars.ts b/sidebars.ts index 2396e7d..7517526 100644 --- a/sidebars.ts +++ b/sidebars.ts @@ -90,6 +90,7 @@ const sidebars: SidebarsConfig = { { type: "doc", id: "hub/nvidia-ngc", label: "Nvidia Catalog (Coming Soon)" }, ], }, + { type: "doc", id: "integrate-remote-engine", label: "How to Integrate Remote Engine" }, // BASIC USAGE // { // type: "html", From 584070aa4475228b8785d27aafa98b107b13c3f3 Mon Sep 17 00:00:00 2001 From: marknguyen1302 Date: Wed, 14 Aug 2024 04:27:23 +0700 Subject: [PATCH 2/7] update doc --- docs/integrate-remote-engine | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/integrate-remote-engine b/docs/integrate-remote-engine index 0b11968..66c3fa4 100644 --- a/docs/integrate-remote-engine +++ b/docs/integrate-remote-engine @@ -1,11 +1,19 @@ --- title: Integrate Remote Engine description: How to integrate remote engine +slug: "integrate-remote-engine" --- -# Adding a Remote Engine Model to Cortex +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; +:::warning +🚧 Cortex is under construction. +::: + +:::info This document provides a step-by-step guide to adding a new engine to the Cortex codebase, similar to the `OpenAIEngineExtension`. +::: ## Steps to Add a New Engine From 39f7721d9d76edcc2d930e5b647ee497bc2a6422 Mon Sep 17 00:00:00 2001 From: marknguyen1302 Date: Wed, 14 Aug 2024 04:30:17 +0700 Subject: [PATCH 3/7] format --- docs/integrate-remote-engine | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/integrate-remote-engine b/docs/integrate-remote-engine index 66c3fa4..564c9c1 100644 --- a/docs/integrate-remote-engine +++ b/docs/integrate-remote-engine @@ -36,6 +36,7 @@ class EngineExtension extends OAIEngineExtension { apiKey?: string; } ``` + ### 3. Register the New Engine 1. **Open** `cortex-js/src/extensions/extensions.module.ts`. From 9601524f1c07cbfdf58940a9cca8df0390c9e623 Mon Sep 17 00:00:00 2001 From: marknguyen1302 Date: Wed, 14 Aug 2024 09:59:10 +0700 Subject: [PATCH 4/7] change format --- docs/{integrate-remote-engine => integrate-remote-engine.mdx} | 1 - 1 file changed, 1 deletion(-) rename docs/{integrate-remote-engine => integrate-remote-engine.mdx} (99%) diff --git a/docs/integrate-remote-engine b/docs/integrate-remote-engine.mdx similarity index 99% rename from docs/integrate-remote-engine rename to docs/integrate-remote-engine.mdx index 564c9c1..8cab736 100644 --- a/docs/integrate-remote-engine +++ b/docs/integrate-remote-engine.mdx @@ -1,7 +1,6 @@ --- title: Integrate Remote Engine description: How to integrate remote engine -slug: "integrate-remote-engine" --- import Tabs from "@theme/Tabs"; From 78f164fa6ae7e8371219b71e8c537ed1d1abe028 Mon Sep 17 00:00:00 2001 From: irfanpena Date: Wed, 14 Aug 2024 11:53:50 +0700 Subject: [PATCH 5/7] Reformat the doc --- docs/integrate-remote-engine.mdx | 60 +++++++++++++------------------- sidebars.ts | 2 +- 2 files changed, 25 insertions(+), 37 deletions(-) diff --git a/docs/integrate-remote-engine.mdx b/docs/integrate-remote-engine.mdx index 8cab736..e57256c 100644 --- a/docs/integrate-remote-engine.mdx +++ b/docs/integrate-remote-engine.mdx @@ -1,6 +1,6 @@ --- title: Integrate Remote Engine -description: How to integrate remote engine +description: How to integrate remote engine into Cortex. --- import Tabs from "@theme/Tabs"; @@ -10,20 +10,17 @@ import TabItem from "@theme/TabItem"; 🚧 Cortex is under construction. ::: -:::info -This document provides a step-by-step guide to adding a new engine to the Cortex codebase, similar to the `OpenAIEngineExtension`. -::: -## Steps to Add a New Engine +This document provides a step-by-step guide to adding a new engine to the Cortex codebase, similar to the `OpenAIEngineExtension`. -### 1. Create a New Engine Extension File -- Navigate to the `cortex-js/src/extensions` directory. -- Create a new file named `.engine.ts` (replace `` with the name of your engine). +## Integrate a New Remote Engine -### 2. Implement the New Engine Extension Class +### Step 1: Create the New Engine Extension -Use the following template to implement your new engine extension class. Replace placeholders with appropriate values for your engine. +1. Navigate to the `cortex-js/src/extensions` directory. +2. Create a new file named `.engine.ts` (replace `` with the name of your engine). +3. Implement your new engine extension class using the following template: ```typescript class EngineExtension extends OAIEngineExtension { @@ -35,12 +32,26 @@ class EngineExtension extends OAIEngineExtension { apiKey?: string; } ``` +Each parameter in the above is defined as follows: +| **Value** | **Description** | +|------------------------------------|--------------------------------------------------------------------------------------------------| +| `apiUrl` | This is the URL endpoint for the new engine's API. It is used to make chat completion requests. | +| `name` | This is a unique identifier for the engine. It is used internally to reference the engine. | +| `productName` | This is a human-readable name for the engine. It is used for display purposes. | +| `description` | This provides a brief description of what the engine does. It is used for documentation and display purposes. | +| `version` | This indicates the version of the engine extension. It is used for version control and display purposes. | +| `eventEmmitter.on('config.updated')` | This is an event listener that listens for configuration updates. When the configuration for the engine is updated, this listener updates the `apiKey` and the engine's status. | +| `onLoad` | This method is called when the engine extension is loaded. It retrieves the engine's configuration (such as the `apiKey`) and sets the engine's status based on whether the `apiKey` is available. | -### 3. Register the New Engine +:::info +Be sure to replace all placeholders with the appropriate values for your engine. +::: -1. **Open** `cortex-js/src/extensions/extensions.module.ts`. +### Step 2: Register the New Engine -2. **Register your new engine in the provider array:** +1. Open the `extensions.module.ts` located at `cortex-js/src/extensions/`. + +2. Register your new engine in the provider array using the following code: ```typescript [ @@ -50,29 +61,6 @@ class EngineExtension extends OAIEngineExtension { ] ``` -## Explanation of Key Properties and Methods - -### `apiUrl` -This is the URL endpoint for the new engine's API. It is used to make chat completion requests. - -### `name` -This is a unique identifier for the engine. It is used internally to reference the engine. - -### `productName` -This is a human-readable name for the engine. It is used for display purposes. - -### `description` -This provides a brief description of what the engine does. It is used for documentation and display purposes. - -### `version` -This indicates the version of the engine extension. It is used for version control and display purposes. - -### `eventEmmitter.on('config.updated')` -This is an event listener that listens for configuration updates. When the configuration for the engine is updated, this listener updates the `apiKey` and the engine's status. - -### `onLoad` -This method is called when the engine extension is loaded. It retrieves the engine's configuration (such as the `apiKey`) and sets the engine's status based on whether the `apiKey` is available. - ## Advanced: Transforming Payloads and Responses Some engines require custom transformations for the payload sent to the API and the response received from the API. This is achieved using the `transformPayload` and `transformResponse` methods. These methods allow you to modify the data structure to match the specific requirements of the engine. diff --git a/sidebars.ts b/sidebars.ts index 57959ca..fe1a005 100644 --- a/sidebars.ts +++ b/sidebars.ts @@ -90,7 +90,7 @@ const sidebars: SidebarsConfig = { { type: "doc", id: "hub/nvidia-ngc", label: "Nvidia Catalog (Coming Soon)" }, ], }, - { type: "doc", id: "integrate-remote-engine", label: "How to Integrate Remote Engine" }, + { type: "doc", id: "integrate-remote-engine", label: "Integrate Remote Engine" }, // BASIC USAGE // { // type: "html", From f8cfef07a05c803054b1e8aaebfc55e8a7db9135 Mon Sep 17 00:00:00 2001 From: irfanpena Date: Wed, 14 Aug 2024 12:02:59 +0700 Subject: [PATCH 6/7] nits --- docs/integrate-remote-engine.mdx | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/docs/integrate-remote-engine.mdx b/docs/integrate-remote-engine.mdx index e57256c..b519cbc 100644 --- a/docs/integrate-remote-engine.mdx +++ b/docs/integrate-remote-engine.mdx @@ -32,16 +32,6 @@ class EngineExtension extends OAIEngineExtension { apiKey?: string; } ``` -Each parameter in the above is defined as follows: -| **Value** | **Description** | -|------------------------------------|--------------------------------------------------------------------------------------------------| -| `apiUrl` | This is the URL endpoint for the new engine's API. It is used to make chat completion requests. | -| `name` | This is a unique identifier for the engine. It is used internally to reference the engine. | -| `productName` | This is a human-readable name for the engine. It is used for display purposes. | -| `description` | This provides a brief description of what the engine does. It is used for documentation and display purposes. | -| `version` | This indicates the version of the engine extension. It is used for version control and display purposes. | -| `eventEmmitter.on('config.updated')` | This is an event listener that listens for configuration updates. When the configuration for the engine is updated, this listener updates the `apiKey` and the engine's status. | -| `onLoad` | This method is called when the engine extension is loaded. It retrieves the engine's configuration (such as the `apiKey`) and sets the engine's status based on whether the `apiKey` is available. | :::info Be sure to replace all placeholders with the appropriate values for your engine. @@ -61,6 +51,17 @@ Be sure to replace all placeholders with the appropriate values for your engine. ] ``` +## Explanation of Key Properties and Methods +| **Value** | **Description** | +|------------------------------------|--------------------------------------------------------------------------------------------------| +| `apiUrl` | This is the URL endpoint for the new engine's API. It is used to make chat completion requests. | +| `name` | This is a unique identifier for the engine. It is used internally to reference the engine. | +| `productName` | This is a human-readable name for the engine. It is used for display purposes. | +| `description` | This provides a brief description of what the engine does. It is used for documentation and display purposes. | +| `version` | This indicates the version of the engine extension. It is used for version control and display purposes. | +| `eventEmmitter.on('config.updated')` | This is an event listener that listens for configuration updates. When the configuration for the engine is updated, this listener updates the `apiKey` and the engine's status. | +| `onLoad` | This method is called when the engine extension is loaded. It retrieves the engine's configuration (such as the `apiKey`) and sets the engine's status based on whether the `apiKey` is available. | + ## Advanced: Transforming Payloads and Responses Some engines require custom transformations for the payload sent to the API and the response received from the API. This is achieved using the `transformPayload` and `transformResponse` methods. These methods allow you to modify the data structure to match the specific requirements of the engine. From 609015295dfd9a0e745089aa9737fb825b64d164 Mon Sep 17 00:00:00 2001 From: irfanpena Date: Wed, 14 Aug 2024 12:34:33 +0700 Subject: [PATCH 7/7] Move the page to under Engines --- sidebars.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sidebars.ts b/sidebars.ts index fe1a005..45f10e1 100644 --- a/sidebars.ts +++ b/sidebars.ts @@ -90,7 +90,6 @@ const sidebars: SidebarsConfig = { { type: "doc", id: "hub/nvidia-ngc", label: "Nvidia Catalog (Coming Soon)" }, ], }, - { type: "doc", id: "integrate-remote-engine", label: "Integrate Remote Engine" }, // BASIC USAGE // { // type: "html", @@ -128,6 +127,7 @@ const sidebars: SidebarsConfig = { { type: "doc", id: "cortex-llamacpp", label: "llama.cpp" }, { type: "doc", id: "cortex-tensorrt-llm", label: "TensorRT-LLM" }, { type: "doc", id: "cortex-onnx", label: "ONNX" }, + { type: "doc", id: "integrate-remote-engine", label: "Integrate Remote Engine" }, ], }, {