-
Notifications
You must be signed in to change notification settings - Fork 0
/
ingestion.py
36 lines (28 loc) · 1.39 KB
/
ingestion.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
import os
from langchain_community.document_loaders import ReadTheDocsLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain_community.vectorstores import Pinecone as PineconeLangChain
from pinecone import Pinecone
from langchain_openai import OpenAIEmbeddings
from dotenv import load_dotenv
from consts import INDEX_NAME
load_dotenv()
pc = Pinecone(api_key=os.environ.get("PINECONE_API_KEY"))
def ingest_docs() -> None:
loader = ReadTheDocsLoader(path="langchain-docs/langchain.readthedocs.io/en/latest")
raw_documents = loader.load()
print(f"loaded {len(raw_documents)}documents")
text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100,
separators=["\n\n", "\n", " ", ""])
documents = text_splitter.split_documents(documents=raw_documents)
print(f"Splitted into {len(documents)} chunks")
for doc in documents:
old_path = doc.metadata["source"]
new_url = old_path.replace("langchain-docs", "https:/")
doc.metadata.update({"source": new_url})
print(f"Going to insert {len(documents)} to Pinecone")
embeddings = OpenAIEmbeddings()
PineconeLangChain.from_documents(documents=documents, embedding=embeddings, index_name=INDEX_NAME)
print("-------------Added vector to Pinecone---------------")
if __name__ == '__main__':
ingest_docs()