Make the insertions to mongodb atlas as bulk insert

SiddharthaAnand · Dec 1, 2018 · f5c5042 · f5c5042
1 parent 1c54f3c
commit f5c5042
Showing 1 changed file with 11 additions and 2 deletions.
diff --git a/coauthornetwork/pipelines.py b/coauthornetwork/pipelines.py
@@ -18,6 +18,7 @@ def __init__(self, mongo_uri, mongo_db):
         self.mongo_db = mongo_db
         self.db = None
         self.client = None
+        self.items = []
 
     @classmethod
     def from_crawler(cls, crawler):
@@ -33,10 +34,18 @@ def open_spider(self, spider):
 
     def process_item(self, item, spider):
         try:
-            self.db[self.collection_name].insert_one(dict(item))
+            self.items.append(item)
+            if len(self.items) % 10 == 0:
+                print "length: ", len(self.items)
+                self.db[self.collection_name].insert_many(self.items)
+                print "-----------------------BULK INSERT COMPLETE--------------------"
+                self.items = []
+            else:
+                print "False condition"
             return item
         except Exception:
             raise DropItem()
 
     def close_spider(self, item, spider):
-        self.client.close()
+        self.client.close()
+