v0 of finic copilot complete

python_library/finic_py/copilot.py

@@ -209,17 +209,16 @@ async def handle_process_node(
         finic_api_key: str, 
         task_name: str, 
         task_file_path: str,
+        selector_file_path: str,
         intent: str, 
         selected_node: List[NodeDetails], 
         cdp_session: CDPSession, 
         page: Page, 
         dom_snapshot: List[Dict[str, Any]]
     ) -> None:
     # Generate a selector for the selected node
-    selectors = await generate_selectors(selected_node[0], page, cdp_session, dom_snapshot)
-    import pdb; pdb.set_trace()
-    return
-    selected_node[0].selector = selector
+    valid_selectors = await generate_selectors(selected_node[0], page, cdp_session, dom_snapshot)
+    selected_node[0].selector = valid_selectors[0]
     generated_code = None
 
     with open(task_file_path, 'r') as f:
@@ -232,12 +231,13 @@ async def handle_process_node(
         }
         body = {
             "intent": intent,
-            "element": selected_node[0],
+            "element": selected_node[0].dict(),
             "existing_code": existing_code
         }
-        response = requests.post("https://api.finic.ai/copilot", headers=headers, json=body)
+        response = requests.get("http://0.0.0.0:8080/copilot", headers=headers, json=body)
         response.raise_for_status()  # Raise an exception for bad status codes
         generated_code = response.json()["code"]
+        element_identifier = response.json()["elementIdentifier"]
         # You can process the copilot_data here as needed
     except requests.RequestException as e:
         print(f"Error making request to Finic AI Copilot API: {e}")
@@ -246,6 +246,12 @@ async def handle_process_node(
     with open(task_file_path, 'w') as f:
         f.write(existing_code + generated_code)
 
+    with open(selector_file_path, 'a') as f:
+        f.write(f"\n{element_identifier}: {selected_node[0].selector}")
+
+    print(f"\nCode generated and written to {task_file_path}")
+    print(f"Selector written to {selector_file_path}")
+
 
 def print_element_to_terminal(element: NodeDetails):
     terminal_width, _ = shutil.get_terminal_size()
@@ -431,11 +437,11 @@ def handle_console_event(event: Dict[str, Any]):
             }
         });
     """)
-    print("\nSelect an element in the browser. Enter 'mode' to toggle between selection and interaction mode, or 'quit' to exit: ", end="", flush=True)
 
     dom_snapshot = await cdp_session.send("DOMSnapshot.captureSnapshot", {"computedStyles": []})
 
     while True:
+        print("\nSelect an element in the browser. Enter 'mode' to toggle between selection and interaction mode, or 'quit' to exit: ", end="", flush=True)
         user_input = await asyncio.get_event_loop().run_in_executor(None, input)
         if user_input.lower() in ['quit', 'q']:
             browser.close()
@@ -449,12 +455,6 @@ def handle_console_event(event: Dict[str, Any]):
                 await cdp_session.send('Overlay.setInspectMode', {'mode': 'searchForNode', 'highlightConfig': {'showInfo': True, 'showExtensionLines': True, 'contentColor': {'r': 255, 'g': 81, 'b': 6, 'a': 0.2}}})
                 inspection_mode = True
                 print("Switched to selection mode.")
-        elif user_input.lower() in ['help', 'h']:
-            print("Click on eleements in the browser. Confirm your selection in the box above. Use 'a|add' to queue an element for generation. Use 'g|generate' to generate selectors for queued elements. Use 'l|list' to view queued elements.")
-            print("\n\033[1mCommands:\033[0m")
-            print("  • \033[1m'm'|'mode'\033[0m    - Change between interaction and selection mode")
-            print("  • \033[1m'quit'|'q'\033[0m    - Quit the program")
-            print("\n\033[1m\033[38;2;255;165;0mEnter command:\033[0m ", end="", flush=True)
         else:
             if selected_node[0] is None:
                 print("Invalid input. Please select an element in the browser first or enter a command.")
@@ -465,7 +465,6 @@ def handle_console_event(event: Dict[str, Any]):
                 time.sleep(3)
                 print("\033[A\033[K", end="")
             else:
-                pass
-                # await handle_process_node(finic_api_key, task_name, task_file_path, user_input, selected_node, cdp_session, page, dom_snapshot)
+                await handle_process_node(finic_api_key, task_name, task_file_path, selectors_path, user_input, selected_node, cdp_session, page, dom_snapshot)
 
     browser.close()

python_library/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "finic-py"
-version = "0.1.29"
+version = "0.1.30"
 description = "Finic.ai is a platform for building and deploying browser automations. This is the Python client for Finic"
 authors = ["Ayan Bandyopadhyay <[email protected]>", "jasonwcfan <[email protected]>"]
 readme = "README.md"

server/baml_client/__init__.py

@@ -0,0 +1,32 @@
+###############################################################################
+#
+#  Welcome to Baml! To use this generated code, please run the following:
+#
+#  $ pip install baml
+#
+###############################################################################
+
+# This file was generated by BAML: please do not edit it. Instead, edit the
+# BAML files and re-generate this code.
+#
+# ruff: noqa: E501,F401
+# flake8: noqa: E501,F401
+# pylint: disable=unused-import,line-too-long
+# fmt: off
+from . import types
+from . import tracing
+from . import partial_types
+from .globals import reset_baml_env_vars
+
+
+from .sync_client import b
+
+
+
+__all__ = [
+  "b",
+  "partial_types",
+  "tracing",
+  "types",
+  "reset_baml_env_vars",
+]

server/baml_client/async_client.py

@@ -0,0 +1,135 @@
+###############################################################################
+#
+#  Welcome to Baml! To use this generated code, please run the following:
+#
+#  $ pip install baml
+#
+###############################################################################
+
+# This file was generated by BAML: please do not edit it. Instead, edit the
+# BAML files and re-generate this code.
+#
+# ruff: noqa: E501,F401
+# flake8: noqa: E501,F401
+# pylint: disable=unused-import,line-too-long
+# fmt: off
+from typing import Any, Dict, List, Optional, TypeVar, Union, TypedDict, Type
+from typing_extensions import NotRequired
+import pprint
+
+import baml_py
+from pydantic import BaseModel, ValidationError, create_model
+
+from . import partial_types, types
+from .type_builder import TypeBuilder
+from .globals import DO_NOT_USE_DIRECTLY_UNLESS_YOU_KNOW_WHAT_YOURE_DOING_CTX, DO_NOT_USE_DIRECTLY_UNLESS_YOU_KNOW_WHAT_YOURE_DOING_RUNTIME
+
+
+OutputType = TypeVar('OutputType')
+
+def coerce(cls: Type[BaseModel], parsed: Any) -> Any:
+  try:
+    return cls.model_validate({"inner": parsed}).inner # type: ignore
+  except ValidationError as e:
+    raise TypeError(
+      "Internal BAML error while casting output to {}\n{}".format(
+        cls.__name__,
+        pprint.pformat(parsed)
+      )
+    ) from e
+
+# Define the TypedDict with optional parameters having default values
+class BamlCallOptions(TypedDict, total=False):
+    tb: NotRequired[TypeBuilder]
+    client_registry: NotRequired[baml_py.baml_py.ClientRegistry]
+
+class BamlAsyncClient:
+    __runtime: baml_py.BamlRuntime
+    __ctx_manager: baml_py.BamlCtxManager
+    __stream_client: "BamlStreamClient"
+
+    def __init__(self, runtime: baml_py.BamlRuntime, ctx_manager: baml_py.BamlCtxManager):
+      self.__runtime = runtime
+      self.__ctx_manager = ctx_manager
+      self.__stream_client = BamlStreamClient(self.__runtime, self.__ctx_manager)
+
+    @property
+    def stream(self):
+      return self.__stream_client
+
+
+
+    async def GeneratePlaywrightCode(
+        self,
+        intent: str,element: types.Element,existing_code: str,
+        baml_options: BamlCallOptions = {},
+    ) -> types.ResponseFormat:
+      __tb__ = baml_options.get("tb", None)
+      if __tb__ is not None:
+        tb = __tb__._tb
+      else:
+        tb = None
+      __cr__ = baml_options.get("client_registry", None)
+
+      raw = await self.__runtime.call_function(
+        "GeneratePlaywrightCode",
+        {
+          "intent": intent,"element": element,"existing_code": existing_code,
+        },
+        self.__ctx_manager.get(),
+        tb,
+        __cr__,
+      )
+      mdl = create_model("GeneratePlaywrightCodeReturnType", inner=(types.ResponseFormat, ...))
+      return coerce(mdl, raw.parsed())
+
+
+
+class BamlStreamClient:
+    __runtime: baml_py.BamlRuntime
+    __ctx_manager: baml_py.BamlCtxManager
+
+    def __init__(self, runtime: baml_py.BamlRuntime, ctx_manager: baml_py.BamlCtxManager):
+      self.__runtime = runtime
+      self.__ctx_manager = ctx_manager
+
+
+    def GeneratePlaywrightCode(
+        self,
+        intent: str,element: types.Element,existing_code: str,
+        baml_options: BamlCallOptions = {},
+    ) -> baml_py.BamlStream[partial_types.ResponseFormat, types.ResponseFormat]:
+      __tb__ = baml_options.get("tb", None)
+      if __tb__ is not None:
+        tb = __tb__._tb
+      else:
+        tb = None
+      __cr__ = baml_options.get("client_registry", None)
+
+      raw = self.__runtime.stream_function(
+        "GeneratePlaywrightCode",
+        {
+          "intent": intent,
+          "element": element,
+          "existing_code": existing_code,
+        },
+        None,
+        self.__ctx_manager.get(),
+        tb,
+        __cr__,
+      )
+
+      mdl = create_model("GeneratePlaywrightCodeReturnType", inner=(types.ResponseFormat, ...))
+      partial_mdl = create_model("GeneratePlaywrightCodePartialReturnType", inner=(partial_types.ResponseFormat, ...))
+
+      return baml_py.BamlStream[partial_types.ResponseFormat, types.ResponseFormat](
+        raw,
+        lambda x: coerce(partial_mdl, x),
+        lambda x: coerce(mdl, x),
+        self.__ctx_manager.get(),
+      )
+
+
+b = BamlAsyncClient(DO_NOT_USE_DIRECTLY_UNLESS_YOU_KNOW_WHAT_YOURE_DOING_RUNTIME, DO_NOT_USE_DIRECTLY_UNLESS_YOU_KNOW_WHAT_YOURE_DOING_CTX)
+
+__all__ = ["b"]

server/baml_client/globals.py

@@ -0,0 +1,41 @@
+###############################################################################
+#
+#  Welcome to Baml! To use this generated code, please run the following:
+#
+#  $ pip install baml
+#
+###############################################################################
+
+# This file was generated by BAML: please do not edit it. Instead, edit the
+# BAML files and re-generate this code.
+#
+# ruff: noqa: E501,F401
+# flake8: noqa: E501,F401
+# pylint: disable=unused-import,line-too-long
+# fmt: off
+import os
+
+from baml_py import BamlCtxManager, BamlRuntime
+from baml_py.baml_py import BamlError
+from .inlinedbaml import get_baml_files
+from typing import Dict
+
+DO_NOT_USE_DIRECTLY_UNLESS_YOU_KNOW_WHAT_YOURE_DOING_RUNTIME = BamlRuntime.from_files(
+  "baml_src",
+  get_baml_files(),
+  os.environ.copy()
+)
+DO_NOT_USE_DIRECTLY_UNLESS_YOU_KNOW_WHAT_YOURE_DOING_CTX = BamlCtxManager(DO_NOT_USE_DIRECTLY_UNLESS_YOU_KNOW_WHAT_YOURE_DOING_RUNTIME)
+
+def reset_baml_env_vars(env_vars: Dict[str, str]):
+  if DO_NOT_USE_DIRECTLY_UNLESS_YOU_KNOW_WHAT_YOURE_DOING_CTX.allow_reset():
+    DO_NOT_USE_DIRECTLY_UNLESS_YOU_KNOW_WHAT_YOURE_DOING_RUNTIME.reset(
+      "baml_src",
+      get_baml_files(),
+      env_vars
+    )
+    DO_NOT_USE_DIRECTLY_UNLESS_YOU_KNOW_WHAT_YOURE_DOING_CTX.reset()
+  else:
+    raise BamlError("Cannot reset BAML environment variables while there are active BAML contexts.")
+
+__all__ = []

server/baml_client/inlinedbaml.py

@@ -0,0 +1,25 @@
+###############################################################################
+#
+#  Welcome to Baml! To use this generated code, please run the following:
+#
+#  $ pip install baml
+#
+###############################################################################
+
+# This file was generated by BAML: please do not edit it. Instead, edit the
+# BAML files and re-generate this code.
+#
+# ruff: noqa: E501,F401
+# flake8: noqa: E501,F401
+# pylint: disable=unused-import,line-too-long
+# fmt: off
+
+file_map = {
+
+    "clients.baml": "// Learn more about clients at https://docs.boundaryml.com/docs/snippets/clients/overview",
+    "copilot.baml": "class Attribute {\n  name string\n  value string\n}\n\nclass Element {\n  tagName string\n  attributes Attribute[]\n  textContent string\n  outerHTML string\n}\n\nenum InputDataType {\n  TEXT\n  NUMBER\n}\n\nclass ResponseFormat {\n  elementIdentifier string @description(\"A descriptive identifier for this element, for example: 'login-button'\")\n  code string\n}\n\nfunction GeneratePlaywrightCode(intent: string, element: Element, existing_code: string) -> ResponseFormat {\n  client \"anthropic/claude-3-5-sonnet-20240620\"\n  prompt #\"\n    Given a the user's intent and info about an element on a web page, generate python code that uses Playwright to interact with the page according to the user's intent. \n    Use `finic.selectors.get(<element_identifier>)` to get the selector for the element, using the elementIdentifier from the response format.\n    \n    The existing code is provided. You should generate code that starts on the line exactly after the last line of the existing code.\n\n    User's intent: {{ intent }}\n\n    Element info:\n    {{ element }}\n\n    Existing code:\n    ```python\n    {{ existing_code }}\n    ```\n\n    Make sure to provide the code response as a markdown code block and start your response as follows:\n    ```python\n\n    {{ ctx.output_format }}\n  \"#\n}",
+    "generators.baml": "// This helps use auto generate libraries you can use in the language of\n// your choice. You can have multiple generators if you use multiple languages.\n// Just ensure that the output_dir is different for each generator.\ngenerator target {\n    // Valid values: \"python/pydantic\", \"typescript\", \"ruby/sorbet\", \"rest/openapi\"\n    output_type \"python/pydantic\"\n\n    // Where the generated code will be saved (relative to baml_src/)\n    output_dir \"../\"\n\n    // The version of the BAML package you have installed (e.g. same version as your baml-py or @boundaryml/baml).\n    // The BAML VSCode extension version should also match this version.\n    version \"0.60.0\"\n\n    // Valid values: \"sync\", \"async\"\n    // This controls what `b.FunctionName()` will be (sync or async).\n    default_client_mode sync\n}\n",
+}
+
+def get_baml_files():
+    return file_map

server/baml_client/partial_types.py

@@ -0,0 +1,49 @@
+###############################################################################
+#
+#  Welcome to Baml! To use this generated code, please run the following:
+#
+#  $ pip install baml
+#
+###############################################################################
+
+# This file was generated by BAML: please do not edit it. Instead, edit the
+# BAML files and re-generate this code.
+#
+# ruff: noqa: E501,F401
+# flake8: noqa: E501,F401
+# pylint: disable=unused-import,line-too-long
+# fmt: off
+import baml_py
+from enum import Enum
+from pydantic import BaseModel, ConfigDict
+from typing import Dict, List, Optional, Union
+
+from . import types
+
+###############################################################################
+#
+#  These types are used for streaming, for when an instance of a type
+#  is still being built up and any of its fields is not yet fully available.
+#
+###############################################################################
+
+
+class Attribute(BaseModel):
+
+
+    name: Optional[str] = None
+    value: Optional[str] = None
+
+class Element(BaseModel):
+
+
+    tagName: Optional[str] = None
+    attributes: List["Attribute"]
+    textContent: Optional[str] = None
+    outerHTML: Optional[str] = None
+
+class ResponseFormat(BaseModel):
+
+
+    elementIdentifier: Optional[str] = None
+    code: Optional[str] = None