1from pathlib import Path
2from shutil import rmtree
3import httpx
4from kern.agent import Agent
5from kern.knowledge.knowledge import Knowledge
6from kern.vectordb.llamaindex import LlamaIndexVectorDb
7from llama_index.core import (
8 SimpleDirectoryReader,
9 StorageContext,
10 VectorStoreIndex,
11)
12from llama_index.core.node_parser import SentenceSplitter
13from llama_index.core.retrievers import VectorIndexRetriever
14
15data_dir = Path(__file__).parent.parent.parent.joinpath("wip", "data", "paul_graham")
16if data_dir.is_dir():
17 rmtree(path=data_dir, ignore_errors=True)
18data_dir.mkdir(parents=True, exist_ok=True)
19
20url = "https://raw.githubusercontent.com/run-llama/llama_index/main/docs/docs/examples/data/paul_graham/paul_graham_essay.txt"
21file_path = data_dir.joinpath("paul_graham_essay.txt")
22response = httpx.get(url)
23if response.status_code == 200:
24 with open(file_path, "wb") as file:
25 file.write(response.content)
26 print(f"File downloaded and saved as {file_path}")
27else:
28 print("Failed to download the file")
29
30documents = SimpleDirectoryReader(str(data_dir)).load_data()
31
32splitter = SentenceSplitter(chunk_size=1024)
33
34nodes = splitter.get_nodes_from_documents(documents)
35
36storage_context = StorageContext.from_defaults()
37
38index = VectorStoreIndex(nodes=nodes, storage_context=storage_context)
39
40knowledge_retriever = VectorIndexRetriever(index)
41
42knowledge = Knowledge(
43 vector_db=LlamaIndexVectorDb(knowledge_retriever=knowledge_retriever)
44)
45
46agent = Agent(
47 knowledge=knowledge,
48 search_knowledge=True,
49 debug_mode=True,
50)
51
52agent.print_response(
53 "Explain what this text means: low end eats the high end", markdown=True
54)
