Update to version v2.2.2

andyoknen · Sep 15, 2019 · 79e0f36 · 79e0f36
1 parent 3fcd04c
commit 79e0f36
Show file tree

Hide file tree

Showing 56 changed files with 1,842 additions and 455 deletions.
diff --git a/changelog.md b/changelog.md
@@ -1,3 +1,17 @@
+# Version 2.2.2 (15.09.2019)
+
+# Core 
+
+- [fea] More effective usage of btree index for GT/LT and sort in concurent read write operations
+- [fix] Potential crash on index update or deletion
+- [fea] Timeout of background indexes optimization can be changed from #config namespace
+
+# Reindexer server
+
+- [fea] User list moved from users.json to users.yml
+- [fea] Hash is used insead of plain password in users.yml file
+- [fix] Pass operation timeout from cproto client to core
+
 # Version 2.2.1 (07.09.2019)
 
 # Core 

diff --git a/cpp_src/CMakeLists.txt b/cpp_src/CMakeLists.txt
@@ -14,7 +14,7 @@ option (ENABLE_LIBUNWIND "Enable libunwind" ON)
 option (ENABLE_TCMALLOC "Enable tcmalloc extensions" ON)
 option (ENABLE_JEMALLOC "Enable jemalloc extensions" ON)
 option (ENABLE_ROCKSDB "Enable rocksdb storage" ON)
-set (REINDEXER_VERSION_DEFAULT "2.2.0")
+set (REINDEXER_VERSION_DEFAULT "2.2.2")
 
 if(NOT CMAKE_BUILD_TYPE)
   set(CMAKE_BUILD_TYPE "RelWithDebInfo")
@@ -433,5 +433,5 @@ add_custom_target(collect_coverage
   COMMAND lcov --directory . --capture -o coverage.info
   COMMAND lcov --remove coverage.info '/usr/*' '/Library/*' '${PROJECT_SOURCE_DIR}/vendor/*' -o coverage_filtered.info
   COMMAND genhtml coverage_filtered.info -o coverage_output
-  COMMENT "Coollecting Reindexer coverage"
+  COMMENT "Collecting Reindexer coverage"
 )
diff --git a/cpp_src/cmd/reindexer_server/contrib/config.yml.in b/cpp_src/cmd/reindexer_server/contrib/config.yml.in
@@ -9,6 +9,7 @@ net:
   httpaddr: 0.0.0.0:9088
   rpcaddr: 0.0.0.0:6534
   webroot: ${REINDEXER_INSTALL_PREFIX}/share/reindexer/web
+  # Enables authorization via login/password (requires users.yml file in db directory)
   security: false
 
 # Logger configuration
@@ -28,3 +29,9 @@ debug:
   pprof: false
   allocs: false
 
+# Metrics configuration
+metrics:
+  prometheus: false
+  # Metrics collect period in millisecods
+  collect_period: 1000
+
diff --git a/cpp_src/cmd/reindexer_server/test/__main__.py b/cpp_src/cmd/reindexer_server/test/__main__.py
@@ -6,7 +6,10 @@
 
 
 if __name__ == '__main__':
-    suite = unittest.TestLoader().loadTestsFromModule(specs)
+    if len(sys.argv) > 1:
+        suite = unittest.TestLoader().loadTestsFromTestCase(eval('specs.' + sys.argv[1]))
+    else:
+        suite = unittest.TestLoader().loadTestsFromModule(specs)
 
     ret_code = not runner.ColourTextTestRunner(
         verbosity=2).run(suite).wasSuccessful()

diff --git a/cpp_src/cmd/reindexer_server/test/end_to_end_test.sh b/cpp_src/cmd/reindexer_server/test/end_to_end_test.sh
@@ -28,7 +28,11 @@ while ((1==1)); do
   attempt=`expr $attempt + 1`
 done
 
-python3 .
+if [ -z "$1" ]; then
+  python3 .
+else
+  python3 . "$1"
+fi
 test_exit_code=$?
 
 pycleanup --cache --pyc --egg > /dev/null

diff --git a/cpp_src/cmd/reindexer_server/test/mocks/users.yml b/cpp_src/cmd/reindexer_server/test/mocks/users.yml
@@ -0,0 +1,16 @@
+user_data_read:
+  hash: $1$salt1$sxceK5uCFboYZNAuscl7U.
+  roles:
+    *: data_read
+user_data_write:
+  hash: $1$salt2$mVTUg8rz3JZbWZdu8BmgB.
+  roles:
+    *: data_write
+user_db_admin:
+  hash: $1$salt3$vX3bUgz2B6qUtO8/Pp8..1
+  roles:
+    *: db_admin
+user_owner:
+  hash: $1$salt4$FP6vfMCasfDJIl7o3YzvC.
+  roles:
+    *: owner
diff --git a/cpp_src/core/cbinding/reindexer_c.cc b/cpp_src/core/cbinding/reindexer_c.cc
@@ -76,6 +76,7 @@ static QueryResultsWrapper* new_results() {
 	}
 	alloced_res_count++;
 	if (res_pool.empty()) {
+		lck.unlock();
 		return new QueryResultsWrapper;
 	} else {
 		auto res = res_pool.back().release();

diff --git a/cpp_src/core/dbconfig.cc b/cpp_src/core/dbconfig.cc
@@ -46,6 +46,7 @@ Error DBConfigProvider::FromJSON(const gason::JsonNode &root) {
 				data.cacheMode = str2cacheMode(nsNode["join_cache_mode"].As<string>("off"));
 				data.startCopyPoliticsCount = nsNode["start_copy_politics_count"].As<int>(data.startCopyPoliticsCount);
 				data.mergeLimitCount = nsNode["merge_limit_count"].As<int>(data.mergeLimitCount);
+				data.optimizationTimeout = nsNode["optimization_timeout_ms"].As<int>(data.optimizationTimeout);
 				namespacesData_.emplace(nsNode["namespace"].As<string>(), std::move(data));
 			}
 			auto it = handlers_.find(NamespaceDataConf);
@@ -77,6 +78,7 @@ ProfilingConfigData DBConfigProvider::GetProfilingConfig() {
 	smart_lock<shared_timed_mutex> lk(mtx_, false);
 	return profilingData_;
 }
+
 ReplicationConfigData DBConfigProvider::GetReplicationConfig() {
 	smart_lock<shared_timed_mutex> lk(mtx_, false);
 	return replicationData_;

diff --git a/cpp_src/core/dbconfig.h b/cpp_src/core/dbconfig.h
@@ -36,6 +36,7 @@ struct NamespaceConfigData {
 	CacheMode cacheMode = CacheModeOff;
 	int startCopyPoliticsCount = 20000;
 	int mergeLimitCount = 30000;
+	int optimizationTimeout = 800;
 };
 
 enum ReplicationRole { ReplicationNone, ReplicationMaster, ReplicationSlave };

diff --git a/cpp_src/core/ft/ftsetcashe.h b/cpp_src/core/ft/ftsetcashe.h
@@ -5,7 +5,7 @@
 namespace reindexer {
 
 struct FtIdSetCacheVal {
-	FtIdSetCacheVal() : ids(std::make_shared<IdSet>()) {}
+	FtIdSetCacheVal() : ids(make_intrusive<intrusive_atomic_rc_wrapper<IdSet>>()) {}
 	FtIdSetCacheVal(const IdSet::Ptr &i) : ids(i) {}
 	FtIdSetCacheVal(const IdSet::Ptr &i, FtCtx::Data::Ptr c) : ids(i), ctx(c) {}
 

diff --git a/cpp_src/core/idset.h b/cpp_src/core/idset.h
@@ -6,6 +6,7 @@
 #include <string>
 #include "cpp-btree/btree_set.h"
 #include "estl/h_vector.h"
+#include "estl/intrusive_ptr.h"
 #include "estl/span.h"
 
 namespace reindexer {
@@ -29,9 +30,12 @@ class IdSetPlain : protected base_idset {
 	using base_idset::shrink_to_fit;
 	using base_idset::back;
 	using base_idset::heap_size;
-
-	iterator begin() { return base_idset::begin(); }
-	iterator end() { return base_idset::end(); }
+	using base_idset::begin;
+	using base_idset::end;
+	using base_idset::rbegin;
+	using base_idset::rend;
+	using base_idset::const_reverse_iterator;
+	using base_idset::const_iterator;
 
 	enum EditMode {
 		Ordered,   // Keep idset ordered, and ready to select (insert is slow O(logN)+O(N))
@@ -72,7 +76,7 @@ class IdSet : public IdSetPlain {
 	friend class SingleSelectKeyResult;
 
 public:
-	typedef shared_ptr<IdSet> Ptr;
+	using Ptr = intrusive_ptr<intrusive_atomic_rc_wrapper<IdSet>>;
 	IdSet() : usingBtree_(false) {}
 	IdSet(const IdSet &other)
 		: IdSetPlain(other), set_(!other.set_ ? nullptr : new base_idsetset(*other.set_)), usingBtree_(other.usingBtree_.load()) {}
@@ -159,6 +163,11 @@ class IdSet : public IdSetPlain {
 	void ReserveForSorted(int sortedIdxCount) { reserve(((set_ ? set_->size() : size())) * (sortedIdxCount + 1)); }
 
 protected:
+	template <typename>
+	friend class BtreeIndexForwardIteratorImpl;
+	template <typename>
+	friend class BtreeIndexReverseIteratorImpl;
+
 	std::unique_ptr<base_idsetset> set_;
 	std::atomic<bool> usingBtree_;
 };

diff --git a/cpp_src/core/index/index.h b/cpp_src/core/index/index.h
@@ -11,6 +11,7 @@
 #include "core/perfstatcounter.h"
 #include "core/selectfunc/ctx/basefunctionctx.h"
 #include "core/selectkeyresult.h"
+#include "indexiterator.h"
 
 namespace reindexer {
 
@@ -22,10 +23,11 @@ class RdxContext;
 class Index {
 public:
 	struct SelectOpts {
-		SelectOpts() : distinct(0), disableIdSetCache(0), forceComparator(0) {}
+		SelectOpts() : distinct(0), disableIdSetCache(0), forceComparator(0), unbuiltSortOrders(0) {}
 		unsigned distinct : 1;
 		unsigned disableIdSetCache : 1;
 		unsigned forceComparator : 1;
+		unsigned unbuiltSortOrders : 1;
 	};
 	using KeyEntry = reindexer::KeyEntry<IdSet>;
 	using KeyEntryPlain = reindexer::KeyEntry<IdSetPlain>;
@@ -48,6 +50,7 @@ class Index {
 	virtual bool IsOrdered() const { return false; }
 	virtual IndexMemStat GetMemStat() = 0;
 	virtual int64_t GetTTLValue() const { return 0; }
+	virtual IndexIterator::Ptr CreateIterator() const { return nullptr; }
 
 	const PayloadType& GetPayloadType() const { return payloadType_; }
 	void UpdatePayloadType(const PayloadType payloadType) { payloadType_ = payloadType; }

diff --git a/cpp_src/core/index/indexiterator.h b/cpp_src/core/index/indexiterator.h
@@ -0,0 +1,24 @@
+#pragma once
+
+#include "core/type_consts.h"
+#include "estl/intrusive_ptr.h"
+
+namespace reindexer {
+
+class IndexIteratorBase {
+public:
+	virtual ~IndexIteratorBase() = default;
+	virtual void Start(bool reverse) = 0;
+	virtual IdType Value() const = 0;
+	virtual bool Next() = 0;
+	virtual void ExcludeLastSet() = 0;
+	virtual size_t GetMaxIterations(size_t limitIters) = 0;
+	virtual void SetMaxIterations(size_t iters) = 0;
+};
+
+class IndexIterator : public intrusive_atomic_rc_wrapper<IndexIteratorBase> {
+public:
+	using Ptr = intrusive_ptr<IndexIterator>;
+};
+
+}  // namespace reindexer
diff --git a/cpp_src/core/index/indexordered.cc b/cpp_src/core/index/indexordered.cc
@@ -1,5 +1,6 @@
 
 #include "indexordered.h"
+#include "core/nsselecter/btreeindexiterator.h"
 #include "core/rdxcontext.h"
 #include "tools/errors.h"
 #include "tools/logger.h"
@@ -85,11 +86,15 @@ SelectKeyResults IndexOrdered<T>::SelectKey(const VariantArray &keys, CondType c
 			throw Error(errParams, "Unknown query type %d", condition);
 	}
 
-	if (endIt == startIt || startIt == this->idx_map.end() || endIt == this->idx_map.begin())
+	if (endIt == startIt || startIt == this->idx_map.end() || endIt == this->idx_map.begin()) {
 		// Empty result
 		return SelectKeyResults(res);
+	}
 
-	if (sortId && this->sortId_ == sortId && !opts.distinct) {
+	if (opts.unbuiltSortOrders) {
+		IndexIterator::Ptr btreeIt(make_intrusive<BtreeIndexIterator<T>>(this->idx_map, startIt, endIt));
+		res.push_back(SingleSelectKeyResult(btreeIt));
+	} else if (sortId && this->sortId_ == sortId && !opts.distinct) {
 		assert(startIt->second.Sorted(this->sortId_).size());
 		IdType idFirst = startIt->second.Sorted(this->sortId_).front();
 
@@ -183,6 +188,11 @@ bool IndexOrdered<T>::IsOrdered() const {
 	return true;
 }
 
+template <typename T>
+IndexIterator::Ptr IndexOrdered<T>::CreateIterator() const {
+	return make_intrusive<BtreeIndexIterator<T>>(this->idx_map);
+}
+
 template <typename KeyEntryT>
 static Index *IndexOrdered_New(const IndexDef &idef, const PayloadType payloadType, const FieldsSet &fields) {
 	switch (idef.Type()) {

diff --git a/cpp_src/core/index/indexordered.h b/cpp_src/core/index/indexordered.h
@@ -16,6 +16,7 @@ class IndexOrdered : public IndexUnordered<T> {
 							   BaseFunctionCtx::Ptr ctx, const RdxContext &) override;
 	Variant Upsert(const Variant &key, IdType id) override;
 	void MakeSortOrders(UpdateSortedContext &ctx) override;
+	IndexIterator::Ptr CreateIterator() const override;
 	Index *Clone() override;
 	bool IsOrdered() const override;
 };

diff --git a/cpp_src/core/index/indextext/fastindextext.cc b/cpp_src/core/index/indextext/fastindextext.cc
@@ -100,7 +100,7 @@ IdSet::Ptr FastIndexText<T>::Select(FtCtx::Ptr fctx, FtDSLQuery &dsl) {
 
 	auto merdeInfo = Selecter(this->holder_, this->fields_.size(), fctx->NeedArea()).Process(dsl);
 	// convert vids(uniq documents id) to ids (real ids)
-	IdSet::Ptr mergedIds = std::make_shared<IdSet>();
+	IdSet::Ptr mergedIds = make_intrusive<intrusive_atomic_rc_wrapper<IdSet>>();
 	auto &holder = this->holder_;
 	auto &vdocs = holder.vdocs_;
 

diff --git a/cpp_src/core/index/indextext/fuzzyindextext.cc b/cpp_src/core/index/indextext/fuzzyindextext.cc
@@ -18,7 +18,7 @@ template <typename T>
 IdSet::Ptr FuzzyIndexText<T>::Select(FtCtx::Ptr fctx, FtDSLQuery& dsl) {
 	auto result = engine_.Search(dsl);
 
-	auto mergedIds = make_shared<IdSet>();
+	auto mergedIds = make_intrusive<intrusive_atomic_rc_wrapper<IdSet>>();
 
 	mergedIds->reserve(result.data_->size() * 2);
 	fctx->Reserve(result.data_->size() * 2);