proxy-rotation

Cloud Database/Database - Firebase/firebase-admin-key.json

@@ -0,0 +1,12 @@
+{
+  "type": "service_account",
+  "project_id": "<project-id>",
+  "private_key_id": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
+  "private_key": "-----BEGIN PRIVATE KEY-----\nxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\n-----END PRIVATE KEY-----\n",
+  "client_email": "firebase-adminsdk-ipcf5@<project-id>.iam.gserviceaccount.com",
+  "client_id": "xxxxxxxxxxx",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "https://oauth2.googleapis.com/token",
+  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/firebase-adminsdk-ipcf5%40<project-id>.iam.gserviceaccount.com"
+}

Cloud Database/Storage - Firebase/README.md

@@ -0,0 +1,53 @@
+#Firebase Imports
+import firebase_admin
+from firebase_admin import credentials
+from firebase_admin import db
+
+cred = credentials.Certificate("/firebase-admin-key.json")
+
+
+print("initilising firebase app")
+
+#Initilising Database
+firebase_admin.initialize_app(cred, {
+    'databaseURL' : 'https://<project-id>-<database-instance>.firebaseio.com/'
+})
+
+# Initilising Database 2
+app2 = firebase_admin.initialize_app(cred, {
+    'databaseURL': 'https://<project-id>-<database-instance>.firebaseio.com/'
+}, name='app2')
+
+
+
+# Fetch Data from firebase
+def fetch_data_firebase(path):
+	ref=db.reference('Users',app2).child(path)
+	data=ref.get()
+	print(data)
+	print("Succesfully fetched data from firebase ")
+
+# Update Data from firebase
+def update_data_firebase(path,data):
+	ref=db.reference('Users',app2).child(path)
+	ref.set(data)
+	print("Succesfully updated the data in firebase")
+
+# Delete Data from firebase
+def delete_data_firebase(path):
+	ref=db.reference('Users',app2).child(path)
+	ref.delete()
+	print("Succesfully deleted the data from firebase")
+
+# Main Function
+def main():
+	fetch_data_firebase("Registered")
+
+	data_to_update={"my-key": "value"}
+	update_data_firebase("Updated",data_to_update)
+
+	delete_data_firebase("UnRegistered")
+
+#Main program
+if __name__ == '__main__':
+	main()

Cloud Database/Storage - Firebase/firebase-admin-key.json

@@ -0,0 +1,12 @@
+{
+  "type": "service_account",
+  "project_id": "<project-id>",
+  "private_key_id": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
+  "private_key": "-----BEGIN PRIVATE KEY-----\nxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\n-----END PRIVATE KEY-----\n",
+  "client_email": "firebase-adminsdk-ipcf5@<project-id>.iam.gserviceaccount.com",
+  "client_id": "xxxxxxxxxxx",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "https://oauth2.googleapis.com/token",
+  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/firebase-adminsdk-ipcf5%40<project-id>.iam.gserviceaccount.com"
+}

Cloud Database/Storage - Firebase/sample.py

@@ -0,0 +1,53 @@
+#Firebase Imports
+import firebase_admin
+from firebase_admin import credentials
+from firebase_admin import db
+
+cred = credentials.Certificate("/firebase-admin-key.json")
+
+
+print("initilising firebase app")
+
+#Initilising Database
+firebase_admin.initialize_app(cred, {
+    'databaseURL' : 'https://<project-id>-<database-instance>.firebaseio.com/'
+})
+
+# Initilising Database 2
+app2 = firebase_admin.initialize_app(cred, {
+    'databaseURL': 'https://<project-id>-<database-instance>.firebaseio.com/'
+}, name='app2')
+
+
+
+# Fetch Data from firebase
+def fetch_data_firebase(path):
+	ref=db.reference('Users',app2).child(path)
+	data=ref.get()
+	print(data)
+	print("Succesfully fetched data from firebase ")
+
+# Update Data from firebase
+def update_data_firebase(path,data):
+	ref=db.reference('Users',app2).child(path)
+	ref.set(data)
+	print("Succesfully updated the data in firebase")
+
+# Delete Data from firebase
+def delete_data_firebase(path):
+	ref=db.reference('Users',app2).child(path)
+	ref.delete()
+	print("Succesfully deleted the data from firebase")
+
+# Main Function
+def main():
+	fetch_data_firebase("Registered")
+
+	data_to_update={"my-key": "value"}
+	update_data_firebase("Updated",data_to_update)
+
+	delete_data_firebase("UnRegistered")
+
+#Main program
+if __name__ == '__main__':
+	main()

Data Extraction/Data Extraction - HTML + Proxy/README.md

@@ -0,0 +1,113 @@
+# Proxy Rotation 
+- Proxy are used to send requests from `different IP`
+- Proxy enables you to acess `location specific` contents
+- Proxy enables you for `data extraction`, as `IP gets blocked` for too many requests
+- `Dedicated Proxy` is better than `FREE proxies`
+
+
+### Sample for testing single proxy
+
+```python
+import requests
+
+def check_proxy(proxy):
+	try:
+		testurl="https://httpbin.org/ip"
+		print("Testing the IP Proxy ",proxy)
+
+		test_resp=requests.get(testurl,proxies={'http':proxy,'https':proxy},timeout=3)
+		print(test_resp.status_code)
+		print(test_resp.json())
+		print("Working")
+		return proxy
+
+	except:
+		print("Proxy failed")
+		pass
+```
+
+### Sample for testing multiple proxy in loop
+
+```python
+import csv
+
+# Reading proxylist from csv file
+proxylist=[]
+with open('proxy-list.csv','r') as f:
+	reader= csv.reader(f)
+	for row in reader:
+		proxylist.append(row[0])
+
+print("Number of proxies in the CSV: ",len(proxylist))
+
+#Checking if the Proxy IP is working or NOT
+for proxy in proxylist:
+	proxy_resp=check_proxy(proxy)
+	if proxy_resp:
+		print("The Proxy ",proxy," is working...")
+		print("This proxy might not be stable or reliable like dedicated proxy, but works")
+		break
+```
+
+### Download Proxy List from Internet
+```python
+from requests_html import HTMLSession
+
+def collect_proxies(url):
+	#NSE INDIA DERIVATIVE STOCK WATCH
+	r=session.get(url)
+	print("waiting to render...")
+	r.html.render(sleep=1)
+	tabledata = r.html.find("table#tbl_proxy_list",first=True)
+
+	proxy_list=[]
+	for tbody in tabledata.find("tbody"):
+		for item in tbody.find("tr"):
+			count=0
+
+			ip=""
+			port=""
+			country=""
+
+			for td in item.find("td"):
+				# print(td.text)
+				count=count+1
+
+				if(count==1):
+					ip=td.text
+					# print(ip)
+
+					if "adsbygoogle = window.adsbygoogle" in ip:
+						break
+					else:
+						if ")))" in ip:
+							ip=ip.split(")))")[1]
+							# print("Ip: ",ip)
+
+						if "))" in ip:
+							ip=ip.split("))")[1]
+							# print("Ip: ",ip)
+
+				if(count==2):
+					port=td.text
+					# print("Port: ",port)
+				if(count==6):
+					country=td.text
+					# print("Country: ",country)
+
+			if ip is not "":
+				proxy=str(ip)+":"+str(port)
+				print(proxy)
+				proxy_list.append(proxy)
+
+	return proxy_list
+
+# Reading Proxy from free proxy list from internet
+session = HTMLSession()
+print("Requesting Session")
+proxy_url="https://www.proxynova.com/proxy-server-list/elite-proxies"
+proxylist=collect_proxies(proxy_url)    
+
+print("Number of proxies from internet: ",len(proxylist))
+
+```

Data Extraction/Data Extraction - HTML + Proxy/proxy-rotation.py

@@ -0,0 +1,97 @@
+import os
+import requests
+import csv
+
+from requests_html import HTMLSession
+from bs4 import BeautifulSoup
+import json
+
+def check_proxy(proxy):
+    try:
+        testurl="https://httpbin.org/ip"
+        print("Testing the IP Proxy ",proxy)
+
+        test_resp=requests.get(testurl,proxies={'http':proxy,'https':proxy},timeout=3)
+        print(test_resp.status_code)
+        print(test_resp.json())
+        print("Working")
+        return proxy
+
+    except:
+        # print("Proxy failed")
+        pass
+
+
+def collect_proxies(url):
+    r=session.get(url)
+    print("waiting to render...")
+    r.html.render(sleep=1)
+    tabledata = r.html.find("table#tbl_proxy_list",first=True)
+
+    proxy_list=[]
+    for tbody in tabledata.find("tbody"):
+        for item in tbody.find("tr"):
+            count=0
+
+            ip=""
+            port=""
+            country=""
+
+            for td in item.find("td"):
+                # print(td.text)
+                count=count+1
+
+                if(count==1):
+                    ip=td.text
+                    # print(ip)
+
+                    if "adsbygoogle = window.adsbygoogle" in ip:
+                        break
+                    else:
+                        if ")))" in ip:
+                            ip=ip.split(")))")[1]
+                            # print("Ip: ",ip)
+
+                        if "))" in ip:
+                            ip=ip.split("))")[1]
+                            # print("Ip: ",ip)
+
+                if(count==2):
+                    port=td.text
+                    # print("Port: ",port)
+                if(count==6):
+                    country=td.text
+                    # print("Country: ",country)
+
+            if ip is not "":
+                proxy=str(ip)+":"+str(port)
+                print(proxy)
+                proxy_list.append(proxy)
+
+    return proxy_list
+
+
+# Reading Proxy from free proxy list from internet
+session = HTMLSession()
+print("Requesting Session")
+proxy_url="https://www.proxynova.com/proxy-server-list/elite-proxies"
+
+proxylist=collect_proxies(proxy_url)    
+
+
+# Reading proxy from csv file
+# proxylist=[]
+# with open('proxy-list.csv','r') as f:
+#     reader= csv.reader(f)
+#     for row in reader:
+#         proxylist.append(row[0])
+
+print("Number of proxies: ",len(proxylist))
+
+#Checking if the Proxy IP is working or NOT
+for proxy in proxylist:
+    proxy_resp=check_proxy(proxy)
+    if proxy_resp:
+        print("The Proxy ",proxy," is working...")
+        print("This proxy might not be stable or reliable like dedicated proxy, but works")
+        break