Project Completed

.gitignore

@@ -1,5 +1,8 @@
 # Byte-compiled / optimized / DLL files
 __pycache__/
+research/test_repo
+.env
+llm
 *.py[cod]
 *$py.class
 

app.py

@@ -0,0 +1,72 @@
+from langchain.vectorstores import Chroma
+from src.helper import load_embedding
+from dotenv import load_dotenv
+import os
+from src.helper import repo_ingestion
+from flask import Flask, render_template, jsonify, request
+from langchain.chat_models import ChatOpenAI
+from langchain.memory import ConversationSummaryMemory
+from langchain.chains import ConversationalRetrievalChain
+
+
+app = Flask(__name__)
+
+
+load_dotenv()
+
+OPENAI_API_KEY = os.environ.get('OPENAI_API_KEY')
+os.environ["OPENAI_API_KEY"] = OPENAI_API_KEY
+
+
+embeddings = load_embedding()
+persist_directory = "db"
+# Now we can load the persisted database from disk, and use it as normal.
+vectordb = Chroma(persist_directory=persist_directory,
+                  embedding_function=embeddings)
+
+
+
+llm = ChatOpenAI()
+memory = ConversationSummaryMemory(llm=llm, memory_key = "chat_history", return_messages=True)
+qa = ConversationalRetrievalChain.from_llm(llm, retriever=vectordb.as_retriever(search_type="mmr", search_kwargs={"k":8}), memory=memory)
+
+
+
+
+@app.route('/', methods=["GET", "POST"])
+def index():
+    return render_template('index.html')
+
+
+
+@app.route('/chatbot', methods=["GET", "POST"])
+def gitRepo():
+
+    if request.method == 'POST':
+        user_input = request.form['question']
+        repo_ingestion(user_input)
+        os.system("python store_index.py")
+
+    return jsonify({"response": str(user_input) })
+
+
+
+@app.route("/get", methods=["GET", "POST"])
+def chat():
+    msg = request.form["msg"]
+    input = msg
+    print(input)
+
+    if input == "clear":
+        os.system("rm -rf repo")
+
+    result = qa(input)
+    print(result['answer'])
+    return str(result["answer"])
+
+
+
+
+
+if __name__ == '__main__':
+    app.run(host='0.0.0.0', port=8080, debug=True)

requirements.txt

@@ -0,0 +1,7 @@
+openai==0.28
+tiktoken
+chromadb==0.4.4
+langchain==0.0.249
+flask
+GitPython
+python-dotenv

setup.py

@@ -0,0 +1,11 @@
+from setuptools import find_packages, setup
+
+setup(
+    name = 'Generative AI Project',
+    version= '0.0.0',
+    author= 'Abdul Qadir',
+    author_email= 'abdulkadir9929@gmail.com',
+    packages= find_packages(),
+    install_requires = []
+
+)

src/helper.py

@@ -0,0 +1,47 @@
+import os
+from langchain.text_splitter import Language
+from langchain.document_loaders.generic import GenericLoader
+from langchain.document_loaders.parsers import LanguageParser
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain.vectorstores import Chroma
+from langchain.chat_models import ChatOpenAI
+from langchain.memory import ConversationSummaryMemory
+from langchain.chains import ConversationalRetrievalChain
+
+#clone any github repositories
+def repo_ingestion(repo_url):
+    os.makedirs("repo",exist_ok=True)
+    repo_path='repo/'
+    Repo.clone_from(repo_url,to_path=repo_path)
+
+
+def load_repo(repo_path):
+    loader=GenericLoader.from_filesystem(repo_path,
+                                         glob="**/*",
+                                         suffixes=[".py"],
+                                         parser=LanguageParser(language=Language.PYTHON,parser_threshold=500))
+
+    documents=loader.load()
+    return documents
+
+
+def text_splitter(documents):
+    documents_splitter=RecursiveCharacterTextSplitter.from_language(language=Language.PYTHON,
+                                                                    chunk_size=2000,
+                                                                    chunk_overlap=200)
+    text_chunks=documents_splitter.split_documents(documents)
+    return text_chunks
+
+
+#Loading embeddings model
+
+def load_embedding():
+    embeddings=OpenAIEmbeddings(disallowed_special=())
+    return embeddings
+
+
+
+
+
+

static/style.css

@@ -0,0 +1,223 @@
+body,html{
+	height: 100%;
+	margin: 0;
+	background: rgb(44, 47, 59);
+   background: -webkit-linear-gradient(to right, rgb(40, 59, 34), rgb(54, 60, 70), rgb(32, 32, 43));
+	background: linear-gradient(to right, rgb(38, 51, 61), rgb(50, 55, 65), rgb(33, 33, 78));
+}
+
+.chat{
+	margin-top: auto;
+	margin-bottom: auto;
+}
+.card{
+	height: 500px;
+	border-radius: 15px !important;
+	background-color: rgba(0,0,0,0.4) !important;
+}
+.contacts_body{
+	padding:  0.75rem 0 !important;
+	overflow-y: auto;
+	white-space: nowrap;
+}
+.msg_card_body{
+	overflow-y: auto;
+}
+.card-header{
+	border-radius: 15px 15px 0 0 !important;
+	border-bottom: 0 !important;
+}
+.card-footer{
+border-radius: 0 0 15px 15px !important;
+	border-top: 0 !important;
+}
+.container{
+	align-content: center;
+}
+.search{
+	border-radius: 15px 0 0 15px !important;
+	background-color: rgba(0,0,0,0.3) !important;
+	border:0 !important;
+	color:white !important;
+}
+.search:focus{
+	 box-shadow:none !important;
+   outline:0px !important;
+}
+.type_msg{
+	background-color: rgba(0,0,0,0.3) !important;
+	border:0 !important;
+	color:white !important;
+	height: 60px !important;
+	overflow-y: auto;
+}
+	.type_msg:focus{
+	 box-shadow:none !important;
+   outline:0px !important;
+}
+.attach_btn{
+	border-radius: 15px 0 0 15px !important;
+	background-color: rgba(0,0,0,0.3) !important;
+	border:0 !important;
+	color: white !important;
+	cursor: pointer;
+}
+.send_btn{
+	border-radius: 0 15px 15px 0 !important;
+	background-color: rgba(0,0,0,0.3) !important;
+	border:0 !important;
+	color: white !important;
+	cursor: pointer;
+}
+.search_btn{
+	border-radius: 0 15px 15px 0 !important;
+	background-color: rgba(0,0,0,0.3) !important;
+	border:0 !important;
+	color: white !important;
+	cursor: pointer;
+}
+.contacts{
+	list-style: none;
+	padding: 0;
+}
+.contacts li{
+	width: 100% !important;
+	padding: 5px 10px;
+	margin-bottom: 15px !important;
+}
+.active{
+	background-color: rgba(0,0,0,0.3);
+}
+.user_img{
+	height: 70px;
+	width: 70px;
+	border:1.5px solid #f5f6fa;
+
+}
+.user_img_msg{
+	height: 40px;
+	width: 40px;
+	border:1.5px solid #f5f6fa;
+
+}
+.img_cont{
+	position: relative;
+	height: 70px;
+	width: 70px;
+}
+.img_cont_msg{
+	height: 40px;
+	width: 40px;
+}
+.online_icon{
+	position: absolute;
+	height: 15px;
+	width:15px;
+	background-color: #4cd137;
+	border-radius: 50%;
+	bottom: 0.2em;
+	right: 0.4em;
+	border:1.5px solid white;
+}
+.offline{
+	background-color: #c23616 !important;
+}
+.user_info{
+	margin-top: auto;
+	margin-bottom: auto;
+	margin-left: 15px;
+}
+.user_info span{
+	font-size: 20px;
+	color: white;
+}
+.user_info p{
+	font-size: 10px;
+	color: rgba(255,255,255,0.6);
+}
+.video_cam{
+	margin-left: 50px;
+	margin-top: 5px;
+}
+.video_cam span{
+	color: white;
+	font-size: 20px;
+	cursor: pointer;
+	margin-right: 20px;
+}
+.msg_cotainer{
+	margin-top: auto;
+	margin-bottom: auto;
+	margin-left: 10px;
+	border-radius: 25px;
+	background-color: rgb(82, 172, 255);
+	padding: 10px;
+	position: relative;
+}
+.msg_cotainer_send{
+	margin-top: auto;
+	margin-bottom: auto;
+	margin-right: 10px;
+	border-radius: 25px;
+	background-color: #58cc71;
+	padding: 10px;
+	position: relative;
+}
+.msg_time{
+	position: absolute;
+	left: 0;
+	bottom: -15px;
+	color: rgba(255,255,255,0.5);
+	font-size: 10px;
+}
+.msg_time_send{
+	position: absolute;
+	right:0;
+	bottom: -15px;
+	color: rgba(255,255,255,0.5);
+	font-size: 10px;
+}
+.msg_head{
+	position: relative;
+}
+#action_menu_btn{
+	position: absolute;
+	right: 10px;
+	top: 10px;
+	color: white;
+	cursor: pointer;
+	font-size: 20px;
+}
+.action_menu{
+	z-index: 1;
+	position: absolute;
+	padding: 15px 0;
+	background-color: rgba(0,0,0,0.5);
+	color: white;
+	border-radius: 15px;
+	top: 30px;
+	right: 15px;
+	display: none;
+}
+.action_menu ul{
+	list-style: none;
+	padding: 0;
+	margin: 0;
+}
+.action_menu ul li{
+	width: 100%;
+	padding: 10px 15px;
+	margin-bottom: 5px;
+}
+.action_menu ul li i{
+	padding-right: 10px;
+}
+.action_menu ul li:hover{
+	cursor: pointer;
+	background-color: rgba(0,0,0,0.2);
+}
+@media(max-width: 576px){
+	.contacts_card{
+	margin-bottom: 15px !important;
+}
+}

store_index.py

@@ -0,0 +1,17 @@
+from src.helper import repo_ingestion,text_splitter,load_embedding,load_repo
+from dotenv import load_dotenv
+from langchain.vectorstores import Chroma
+
+load_dotenv()
+
+OPENAI_API_KEY=os.environ.get('OPENAI_API_KEY')
+os.environ["OPENAI_API_KEY"]="OPENAI_API_KEY"
+
+documents=load_repo("repo/")
+text_chunks=text_splitter(documents)
+embeddings=load_embedding()
+
+#storing vector in chromadb
+
+vectordb=Chroma.from_documents(text_chunks,embedding=embeddings,persist_directory='./db')
+vectordb.persist()