added ingestion

Pipfile

@@ -18,6 +18,7 @@ jinja2 = "*"
 uvicorn = "*"
 streamlit = "*"
 streamlit-chat = "*"
+tqdm = "*"
 
 [dev-packages]
 

ingestion.py

@@ -0,0 +1,36 @@
+import os
+
+from langchain.document_loaders import ReadTheDocsLoader
+from langchain.embeddings import OpenAIEmbeddings
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.vectorstores import Pinecone
+import pinecone
+
+pinecone.init(
+    api_key=os.environ["PINECONE_API_KEY"],
+    environment=os.environ["PINECONE_ENVIRONMENT_REGION"],
+)
+INDEX_NAME = "langchain-doc-index"
+
+
+def ingest_docs():
+    loader = ReadTheDocsLoader("langchain-docs/langchain.readthedocs.io/en/latest")
+    raw_documents = loader.load()
+    print(f"loaded {len(raw_documents)} documents")
+    text_splitter = RecursiveCharacterTextSplitter(
+        chunk_size=2000, chunk_overlap=200, separators=["\n\n", "\n", " ", ""]
+    )
+    documents = text_splitter.split_documents(raw_documents)
+    for doc in documents:
+        new_url = doc.metadata["source"]
+        new_url = new_url.replace("langchain-docs", "https:/")
+        doc.metadata.update({"source": new_url})
+
+    embeddings = OpenAIEmbeddings()
+    print(f"Going to add {len(documents)} to Pinecone")
+    Pinecone.from_documents(documents, embeddings, index_name=INDEX_NAME)
+    print("****Loading to vectorestore done ***")
+
+
+if __name__ == "__main__":
+    ingest_docs()