diff --git a/modules/genai-ecosystem/pages/langchain-js.adoc b/modules/genai-ecosystem/pages/langchain-js.adoc index cb5c7a5..a968786 100644 --- a/modules/genai-ecosystem/pages/langchain-js.adoc +++ b/modules/genai-ecosystem/pages/langchain-js.adoc @@ -135,6 +135,12 @@ const res = await chain.run("Who acted in Pulp Fiction?"); // Bruce Willis acted in Pulp Fiction. ---- +=== Knowledge Graph Construction + +Creating a Knowledge Graph from unstructured data like PDF documents used to be a complex and time-consuming task that required training and using dedicated, large NLP models. + +The https://js.langchain.com/docs/use_cases/graph/construction[Graph Transformers^] are tools that allows you to extract structured data from unstructured documents and transform it into a Knowledge Graph. + == Documentation * https://js.langchain.com/docs/modules/data_connection/experimental/graph_databases/neo4j[Neo4jGraph^] diff --git a/modules/genai-ecosystem/pages/langchain.adoc b/modules/genai-ecosystem/pages/langchain.adoc index 599709e..adc054c 100644 --- a/modules/genai-ecosystem/pages/langchain.adoc +++ b/modules/genai-ecosystem/pages/langchain.adoc @@ -42,6 +42,7 @@ The Neo4j Vector integration supports a number of operations * query vector with additional graph retrieval Cypher query * construct vector instance from existing graph data * hybrid search +* metadata filtering // todo include [source,python] @@ -106,6 +107,25 @@ query = "What did the president say about Ketanji Brown Jackson" docs_with_score = db.similarity_search_with_score(query, k=2) ---- +==== Metadata filtering + +Metadata filtering enhances vector search by allowing searches to be refined based on specific node properties. +This integrated approach ensures more precise and relevant search results by leveraging both the vector similarities and the contextual attributes of the nodes. + +[source,python] +---- +db = Neo4jVector.from_documents( + docs, + OpenAIEmbeddings(), + url=url, username=username, password=password +) + +query = "What did the president say about Ketanji Brown Jackson" +filter = {"name": {"$eq": "adam"}} + +docs = db.similarity_search(query, filter=filter) +---- + === Neo4j Graph The Neo4j Graph integration is a wrapper for the Neo4j Python driver. diff --git a/modules/genai-ecosystem/pages/llamaindex.adoc b/modules/genai-ecosystem/pages/llamaindex.adoc index a2ab497..94ec089 100644 --- a/modules/genai-ecosystem/pages/llamaindex.adoc +++ b/modules/genai-ecosystem/pages/llamaindex.adoc @@ -16,7 +16,107 @@ LlamaIndex provides a lot of detailed examples for GenAI application development The Neo4j integration covers both the vector store as well as query generation from natural language and knowledge graph construction. -== Example Usage +== Functionality Includes + +=== Neo4jVector + +The Neo4j Vector integration supports a number of operations + +* create vector from langchain documents +* query vector +* query vector with additional graph retrieval Cypher query +* construct vector instance from existing graph data +* hybrid search +* metadata filtering + +[source,python] +---- +%pip install llama-index-llms-openai +%pip install llama-index-vector-stores-neo4jvector +%pip install llama-index-embeddings-openai +%pip install neo4j + +import os +import openai +from llama_index.vector_stores.neo4jvector import Neo4jVectorStore +from llama_index.core import VectorStoreIndex, SimpleDirectoryReader + +os.environ["OPENAI_API_KEY"] = "OPENAI_API_KEY" +openai.api_key = os.environ["OPENAI_API_KEY"] +username = "neo4j" +password = "pleaseletmein" +url = "bolt://localhost:7687" +embed_dim = 1536 + +neo4j_vector = Neo4jVectorStore(username, password, url, embed_dim) +# load documents +documents = SimpleDirectoryReader("./data/paul_graham").load_data() +from llama_index.core import StorageContext + +storage_context = StorageContext.from_defaults(vector_store=neo4j_vector) +index = VectorStoreIndex.from_documents( + documents, storage_context=storage_context +) + +query_engine = index.as_query_engine() +response = query_engine.query("What happened at interleaf?") +---- + +==== Hybrid search + +Hybrid search combines vector search with fulltext search with re-ranking and de-duplication of the results. + +[source,python] +---- +neo4j_vector_hybrid = Neo4jVectorStore( + username, password, url, embed_dim, hybrid_search=True +) + +storage_context = StorageContext.from_defaults( + vector_store=neo4j_vector_hybrid +) +index = VectorStoreIndex.from_documents( + documents, storage_context=storage_context +) +query_engine = index.as_query_engine() +response = query_engine.query("What happened at interleaf?") +---- + +==== Metadata filtering + +Metadata filtering enhances vector search by allowing searches to be refined based on specific node properties. +This integrated approach ensures more precise and relevant search results by leveraging both the vector similarities and the contextual attributes of the nodes. + +[source,python] +---- +from llama_index.core.vector_stores import ( + MetadataFilter, + MetadataFilters, + FilterOperator, +) + +filters = MetadataFilters( + filters=[ + MetadataFilter( + key="theme", operator=FilterOperator.EQ, value="Fiction" + ), + ] +) + +retriever = index.as_retriever(filters=filters) +retriever.retrieve("What is inception about?") +---- + +=== Neo4jGraphStore + +The Neo4j Graph Store integration is a wrapper for the Neo4j Python driver. +It allows querying and updating the Neo4j database in a simplified manner from LlamaIndex. +Many integrations allow you to use the Neo4j Graph Store as a source of data for LlamaIndex. + +==== Knowledge graph index + +Knowledge graph index can be used to extract graph representation of information from text and use it to construct a knowledge graph. +The graph information can then be retrieved in a RAG application for more accurate responses. [source,python] ---- @@ -68,6 +168,41 @@ response = query_engine.query( ) ---- +==== Knowledge graph query engine + +The Knowledge Graph Query Engine generated Cypher statements based on natural language input to retrieve information from the knowledge graph. + +[source,python] +---- +%pip install llama-index-llms-openai +%pip install llama-index-graph-stores-neo4j +%pip install llama-index-embeddings-openai +%pip install neo4j + +from llama_index.llms.openai import OpenAI +from llama_index.embeddings.openai import OpenAIEmbedding +from llama_index.query_engine import KnowledgeGraphQueryEngine +from llama_index.graph_stores.neo4j import Neo4jGraphStore + +llm=OpenAI(model_name="gpt-3.5-turbo") +service_context = ServiceContext.from_defaults(llm=llm, chunk_size=256) +graph_store = Neo4jGraphStore(username=username,password=password, + url=url,database=database) + +storage_context = StorageContext.from_defaults(graph_store=graph_store) +query_engine = KnowledgeGraphQueryEngine( + storage_context=storage_context, + service_context=service_context, + llm=llm, + verbose=True, + refresh_schema=True +) + +response = query_engine.query( + "Tell me more about what the author worked on at Interleaf", +) +---- + == Documentation * https://docs.llamaindex.ai/en/latest/examples/index_structs/knowledge_graph/Neo4jKGIndexDemo.html[Neo4jKGIndexDemo^] @@ -115,3 +250,5 @@ This https://llamahub.ai/l/llama_packs-neo4j_query_engine[Neo4j Query Engine Lla * https://blog.llamaindex.ai/enriching-llamaindex-models-from-graphql-and-graph-databases-bcaecec262d7[Enriching LlamaIndex Models from GraphQL and Graph Databases^] * https://levelup.gitconnected.com/a-simpler-way-to-query-neo4j-knowledge-graphs-99c0a8bbf1d7[A Simpler Way to Query Neo4j Knowledge Graphs^] + +* https://medium.com/@yu-joshua/using-llamaparse-for-knowledge-graph-creation-from-documents-3bd1e1849754[Using LlamaParse for Knowledge Graph Creation from Documents^]