Update version to 2.1.1

bindings.go

@@ -92,7 +92,7 @@ func packItem(ns *reindexerNamespace, item interface{}, json []byte, ser *cjson.
 		if t.Kind() == reflect.Ptr {
 			t = t.Elem()
 		}
-		if ns.rtype.Name() != t.Name() {
+		if ns.rtype.Name() != t.Name() || ns.rtype.PkgPath() != t.PkgPath() {
 			panic(ErrWrongType)
 		}
 
@@ -409,7 +409,9 @@ func (db *reindexerImpl) updateQuery(ctx context.Context, q *Query) *Iterator {
 
 	ser := newSerializer(result.GetBuf())
 	// skip total count
-	rawQueryParams := ser.readRawQueryParams()
+	rawQueryParams := ser.readRawQueryParams(func(nsid int) {
+		ns.cjsonState.ReadPayloadType(&ser.Serializer)
+	})
 
 	ns.cacheLock.Lock()
 	for i := 0; i < rawQueryParams.count; i++ {
@@ -428,6 +430,7 @@ func (db *reindexerImpl) updateQuery(ctx context.Context, q *Query) *Iterator {
 		panic("Internal error: data after end of update query result")
 	}
 
+	q.nsArray = append(q.nsArray, nsArrayEntry{ns, ns.cjsonState.Copy()})
 	return newIterator(ctx, q, result, q.nsArray, nil, nil, nil)
 }
 

bindings/builtin/builtin_posix.go

@@ -1,5 +1,5 @@
 // +build !windows
-//go:generate sh -c "cd ../.. && mkdir -p build && cd build && cmake .. && make reindexer -j4"
+//go:generate sh -c "cd ../.. && mkdir -p build && cd build && cmake -DENABLE_LIBUNWIND=Off .. && make reindexer -j4"
 
 package builtin
 

bindings/builtinserver/builtinserver_posix.go

@@ -1,5 +1,5 @@
 // +build !windows
-//go:generate sh -c "cd ../.. && mkdir -p build && cd build && cmake -DLINK_RESOURCES=On .. && make reindexer_server_library -j4"
+//go:generate sh -c "cd ../.. && mkdir -p build && cd build && cmake -DENABLE_LIBUNWIND=Off -DLINK_RESOURCES=On .. && make reindexer_server_library reindexer -j4"
 
 package builtinserver
 

changelog.md

@@ -1,3 +1,21 @@
+# Version 2.1.1 (12.06.2019)
+
+## Core 
+
+- [fix] sparse index crashes on type conflicts
+- [fix] fixed memory stats for dataSize of string hash indexes 
+
+# Reindexer server
+
+- [fix] possible crash on http meta update
+- [fix] return back item data via cproto  on UpdateQuery operation 
+
+# go connector
+
+- [fix] Build builinserver with libunwind conflict fixed
+- [fix] Query.Update panic fixed
+- [fix] Stronger check of namespace's item's type (PkgPath is included to check)
+
 # Version 2.1.0 (08.06.2019)
 
 ## Core

cpp_src/CMakeLists.txt

@@ -10,9 +10,10 @@ set(CMAKE_DISABLE_IN_SOURCE_BUILD ON)
 option (WITH_ASAN "Enable AddressSanitized build" OFF)
 option (WITH_TSAN "Enable ThreadSanitized build" OFF)
 option (WITH_GCOV "Enable instrumented code coverage build" OFF)
+option (ENABLE_LIBUNWIND "Enable libunwind" ON)
 option (ENABLE_TCMALLOC "Enable tcmalloc extensions" ON)
 option (ENABLE_JEMALLOC "Enable jemalloc extensions" ON)
-set (REINDEXER_VERSION_DEFAULT "2.1.0")
+set (REINDEXER_VERSION_DEFAULT "2.1.1")
 
 if(NOT CMAKE_BUILD_TYPE)
   set(CMAKE_BUILD_TYPE "RelWithDebInfo")
@@ -199,17 +200,18 @@ if (HAVE_BACKTRACE AND HAVE_GETIPINFO)
 endif()
 
 # libunwind
-find_library(LIBUNWIND unwind)
-if(LIBUNWIND)
-list (APPEND REINDEXER_LIBRARIES ${LIBUNWIND} )
-  find_path(LIBUNWIND_INCLUDE_PATH libunwind.h)
-  if (LIBUNWIND_INCLUDE_PATH)
-    add_definitions(-DREINDEX_WITH_LIBUNWIND=1)
-    message ("-- Found Libunwind: ${LIBUNWIND} ${LIBUNWIND_INCLUDE_PATH}") 
+if (ENABLE_LIBUNWIND)
+  find_library(LIBUNWIND unwind)
+  if(LIBUNWIND)
+  list (APPEND REINDEXER_LIBRARIES ${LIBUNWIND} )
+    find_path(LIBUNWIND_INCLUDE_PATH libunwind.h)
+    if (LIBUNWIND_INCLUDE_PATH)
+      add_definitions(-DREINDEX_WITH_LIBUNWIND=1)
+      message ("-- Found Libunwind: ${LIBUNWIND} ${LIBUNWIND_INCLUDE_PATH}") 
+    endif()
+  else ()
   endif()
-else ()
-endif()
-
+endif ()
 
 if (APPLE)
   add_definitions (-DREINDEX_WITH_APPLE_SYMBOLICATION=1)

cpp_src/core/cbinding/reindexer_c.cc

@@ -403,9 +403,10 @@ reindexer_ret reindexer_update_query(uintptr_t rx, reindexer_buffer in) {
 		if (!result) return ret2c(err_too_many_queries, out);
 		res = db->Update(q, *result);
 		if (q.debugLevel >= LogError && res.code() != errOK) logPrintf(LogError, "Query error %s", res.what());
-		if (res.ok())
-			results2c(result, &out);
-		else
+		if (res.ok()) {
+			int32_t ptVers = -1;
+			results2c(result, &out, 0, &ptVers, 1);
+		} else
 			put_results_to_pool(result);
 	}
 	return ret2c(res, out);

cpp_src/core/ft/ft_fast/dataprocessor.h

@@ -11,7 +11,7 @@ using std::vector;
 class DataProcessor {
 public:
 	using words_map = fast_hash_map<string, WordEntry>;
-	DataProcessor(DataHolder& holder, size_t fieldSize) : holder_(holder), fieldSize_(fieldSize) {}
+	DataProcessor(DataHolder& holder, size_t fieldSize) : holder_(holder), multithread_(false), fieldSize_(fieldSize) {}
 
 	void Process(bool multithread);
 

cpp_src/core/ft/ftdsl.h

@@ -46,7 +46,6 @@ class FtDSLQuery : public h_vector<FtDSLEntry> {
 protected:
 	void parseFields(wstring &utf16str, wstring::iterator &it, h_vector<float, 8> &fieldsBoost);
 
-	int numFields_;
 	std::function<int(const string &)> resolver_;
 
 	const fast_hash_map<string, int> &fields_;

cpp_src/core/index/index.h

@@ -21,9 +21,9 @@ class Index {
 public:
 	struct SelectOpts {
 		SelectOpts() : distinct(0), disableIdSetCache(0), forceComparator(0) {}
-		int distinct : 1;
-		int disableIdSetCache : 1;
-		int forceComparator : 1;
+		unsigned distinct : 1;
+		unsigned disableIdSetCache : 1;
+		unsigned forceComparator : 1;
 	};
 	using KeyEntry = reindexer::KeyEntry<IdSet>;
 	using KeyEntryPlain = reindexer::KeyEntry<IdSetPlain>;

cpp_src/core/index/indexstore.cc

@@ -43,7 +43,7 @@ Variant IndexStore<PayloadValue>::Upsert(const Variant &key, IdType /*id*/) {
 
 template <typename T>
 Variant IndexStore<T>::Upsert(const Variant &key, IdType id) {
-	if (!opts_.IsArray() && !opts_.IsDense()) {
+	if (!opts_.IsArray() && !opts_.IsDense() && !opts_.IsSparse() && key.Type() != KeyValueNull) {
 		idx_data.resize(std::max(id + 1, int(idx_data.size())));
 		idx_data[id] = static_cast<T>(key);
 	}

cpp_src/core/index/indexunordered.cc

@@ -21,11 +21,14 @@ IndexUnordered<T>::IndexUnordered(const IndexUnordered &other)
 	  tracker_(other.tracker_) {}
 
 template <typename key_type>
-static inline size_t heap_size(key_type & /*kt*/) {
+size_t heap_size(const key_type & /*kt*/) {
 	return 0;
 }
 
-static inline size_t heap_size(const key_string &kt) { return kt->heap_size() + sizeof(*kt.get()); }
+template <>
+size_t heap_size<key_string>(const key_string &kt) {
+	return kt->heap_size() + sizeof(*kt.get());
+}
 
 template <typename T>
 void IndexUnordered<T>::addMemStat(typename T::iterator it) {
@@ -138,10 +141,6 @@ SelectKeyResults IndexUnordered<T>::SelectKey(const VariantArray &keys, CondType
 				throw Error(errParams, "The 'is NULL' condition is suported only by 'sparse' or 'array' indexes");
 			res.push_back(SingleSelectKeyResult(this->empty_ids_, sortId));
 			break;
-		case CondAny:
-			// Get set of any keys
-			return IndexStore<typename T::key_type>::SelectKey(keys, condition, sortId, opts, ctx);
-			break;
 		// Get set of keys or single key
 		case CondEq:
 		case CondSet:
@@ -188,6 +187,7 @@ SelectKeyResults IndexUnordered<T>::SelectKey(const VariantArray &keys, CondType
 			return rslts;
 		}
 
+		case CondAny:  // Get set of any keys
 		case CondGe:
 		case CondLe:
 		case CondRange:

cpp_src/core/namespace.cc

@@ -53,7 +53,7 @@ void Namespace::IndexesStorage::MoveBase(IndexesStorage &&src) { Base::operator=
 
 // private implementation and NOT THREADSAFE of copy CTOR
 // use 'Namespace::Clone(Namespace& ns)'
-Namespace::Namespace(const Namespace &src) : indexes_(*this), observers_(src.observers_) { CopyContentsFrom(src); }
+Namespace::Namespace(const Namespace &src) : indexes_(*this), observers_(src.observers_), lastSelectTime_(0) { CopyContentsFrom(src); }
 
 Namespace::Namespace(const string &name, UpdatesObservers &observers)
 	: indexes_(*this),
@@ -67,6 +67,7 @@ Namespace::Namespace(const string &name, UpdatesObservers &observers)
 	  enablePerfCounters_(false),
 	  observers_(observers),
 	  storageLoaded_(false),
+	  lastSelectTime_(0),
 	  cancelCommit_(false),
 	  lastUpdateTime_(0),
 	  itemsCount_(0) {
@@ -907,7 +908,11 @@ void Namespace::doUpsert(ItemImpl *ritem, IdType id, bool doUpdate) {
 
 		if (isIndexSparse) {
 			assert(index.Fields().getTagsPathsLength() > 0);
-			plNew.GetByJsonPath(index.Fields().getTagsPath(0), skrefs, index.KeyType());
+			try {
+				plNew.GetByJsonPath(index.Fields().getTagsPath(0), skrefs, index.KeyType());
+			} catch (const Error &err) {
+				skrefs.resize(0);
+			}
 		} else {
 			plNew.Get(field, skrefs);
 		}
@@ -918,7 +923,11 @@ void Namespace::doUpsert(ItemImpl *ritem, IdType id, bool doUpdate) {
 		// Check for update
 		if (doUpdate) {
 			if (isIndexSparse) {
-				pl.GetByJsonPath(index.Fields().getTagsPath(0), krefs, index.KeyType());
+				try {
+					pl.GetByJsonPath(index.Fields().getTagsPath(0), krefs, index.KeyType());
+				} catch (const Error &err) {
+					krefs.resize(0);
+				}
 			} else {
 				pl.Get(field, krefs, index.Opts().IsArray());
 			}
@@ -930,10 +939,12 @@ void Namespace::doUpsert(ItemImpl *ritem, IdType id, bool doUpdate) {
 		krefs.reserve(skrefs.size());
 		for (auto key : skrefs) krefs.push_back(index.Upsert(key, id));
 
-		// Put value to payload
-		if (!isIndexSparse) pl.Set(field, krefs);
-		// If no krefs doUpsert empty value to index
-		if (!skrefs.size()) index.Upsert(Variant(), id);
+		if (!isIndexSparse) {
+			// Put value to payload
+			pl.Set(field, krefs);
+			// If no krefs doUpsert empty value to index
+			if (!skrefs.size()) index.Upsert(Variant(), id);
+		}
 	} while (++field != borderIdx);
 
 	// Upsert to composite indexes
@@ -948,7 +959,7 @@ void Namespace::ReplaceTagsMatcher(const TagsMatcher &tm) {
 	assert(!items_.size() && repl_.slaveMode);
 	cancelCommit_ = true;
 	WLock lck(mtx_);
-	cancelCommit_ = false;
+	cancelCommit_ = false;  // -V519
 	tagsMatcher_ = tm;
 	tagsMatcher_.UpdatePayloadType(payloadType_);
 }
@@ -1334,7 +1345,7 @@ NamespaceMemStat Namespace::GetMemStat() {
 	ret.indexes.reserve(indexes_.size());
 	for (auto &idx : indexes_) {
 		auto istat = idx->GetMemStat();
-		istat.sortOrdersSize = idx->IsOrdered() ? items_.size() : 0;
+		istat.sortOrdersSize = idx->IsOrdered() ? (items_.size() * sizeof(IdType)) : 0;
 		ret.Total.indexesSize += istat.idsetPlainSize + istat.idsetBTreeSize + istat.sortOrdersSize + istat.fulltextSize + istat.columnSize;
 		ret.Total.dataSize += istat.dataSize;
 		ret.Total.cacheSize += istat.idsetCache.totalSize;

cpp_src/core/namespacecloner.cc

@@ -156,7 +156,7 @@ ClonableNamespace::~ClonableNamespace() {
 	// if we work in single nm mode
 	if (!clonedNs_) return;
 	// check if it last in order try to close
-	if (clonedNs_ && clonedNs_.use_count() == 2) {
+	if (clonedNs_.use_count() == 2) {
 		atomic_exchange_sptr(&clonedNs_, {});
 		nsCloner_->CloseClonedNs();
 		return;

cpp_src/core/namespacecloner.h

@@ -59,7 +59,7 @@ class NamespaceCloner {
 class ClonableNamespace {
 public:
 	ClonableNamespace(size_t size, NamespaceCloner::Ptr nsWarpper);
-	ClonableNamespace() {}
+	ClonableNamespace() : size_(0) {}
 
 	~ClonableNamespace();
 	Namespace *operator->() const noexcept;

cpp_src/core/nsselecter/selectiterator.h

@@ -125,7 +125,7 @@ class SelectIterator : public SelectKeyResult {
 	const char *TypeName() const;
 	string Dump() const;
 
-	bool distinct;
+	bool distinct = false;
 	string name;
 
 protected:
@@ -142,8 +142,8 @@ class SelectIterator : public SelectKeyResult {
 
 	bool isUnsorted = false;
 	bool isReverse_ = false;
-	bool forcedFirst_;
-	int type_;
+	bool forcedFirst_ = false;
+	int type_ = 0;
 	IdType lastVal_ = INT_MIN;
 	iterator lastIt_ = nullptr;
 	IdType end_ = 0;

cpp_src/core/query/query.cc

@@ -164,10 +164,10 @@ void Query::deserialize(Serializer &ser) {
 				selectFilter_.push_back(string(ser.GetVString()));
 				break;
 			case QueryEqualPosition: {
-				const unsigned start = ser.GetVarUint();
+				const unsigned strt = ser.GetVarUint();
 				vector<string> ep(ser.GetVarUint());
 				for (size_t i = 0; i < ep.size(); ++i) ep[i] = string(ser.GetVString());
-				equalPositions_.insert({start, entries.DetermineEqualPositionIndexes(start, ep)});
+				equalPositions_.insert({strt, entries.DetermineEqualPositionIndexes(strt, ep)});
 				break;
 			}
 			case QueryExplain:
@@ -605,7 +605,7 @@ int Query::selectParse(tokenizer &parser, SqlParsingCtx &ctx) {
 				tok = parser.next_token();
 				for (tok = parser.peek_token(); tok.text() == ","_sv; tok = parser.peek_token()) {
 					tok = parser.next_token();
-					tok = peekSqlToken(parser, ctx, SingleSelectFieldSqlToken);
+					tok = peekSqlToken(parser, ctx, SingleSelectFieldSqlToken);  // -V519
 					entry.fields_.push_back(string(tok.text()));
 					tok = parser.next_token();
 				}

cpp_src/core/query/querytree.h

@@ -44,22 +44,40 @@ class QueryTree {
 
 		/// @param it - points to this object
 		/// @return iterator points to the first child
-		virtual iterator begin(iterator it) { (void)it, throw std::runtime_error("It is not subtree"); }
+		virtual iterator begin(iterator it) {
+			(void)it;
+			throw std::runtime_error("It is not subtree");
+		}
 		/// @param it - points to this object
 		/// @return iterator points to the first child
-		virtual const_iterator begin(const_iterator it) const { (void)it, throw std::runtime_error("It is not subtree"); }
+		virtual const_iterator begin(const_iterator it) const {
+			(void)it;
+			throw std::runtime_error("It is not subtree");
+		}
 		/// @param it - points to this object
 		/// @return iterator points to the first child
-		virtual const_iterator cbegin(const_iterator it) const { (void)it, throw std::runtime_error("It is not subtree"); }
+		virtual const_iterator cbegin(const_iterator it) const {
+			(void)it;
+			throw std::runtime_error("It is not subtree");
+		}
 		/// @param it - points to this object
 		/// @return iterator points to the node after the last child
-		virtual iterator end(iterator it) { (void)it, throw std::runtime_error("It is not subtree"); }
+		virtual iterator end(iterator it) {  // -V524
+			(void)it;
+			throw std::runtime_error("It is not subtree");
+		}
 		/// @param it - points to this object
 		/// @return iterator points to the node after the last child
-		virtual const_iterator end(const_iterator it) const { (void)it, throw std::runtime_error("It is not subtree"); }
+		virtual const_iterator end(const_iterator it) const {  // -V524
+			(void)it;
+			throw std::runtime_error("It is not subtree");
+		}
 		/// @param it - points to this object
 		/// @return iterator points to the node after the last child
-		virtual const_iterator cend(const_iterator it) const { (void)it, throw std::runtime_error("It is not subtree"); }
+		virtual const_iterator cend(const_iterator it) const {  // -V524
+			(void)it;
+			throw std::runtime_error("It is not subtree");
+		}
 
 		OpType Op = OpAnd;
 	};

cpp_src/core/query/querywhere.cc

@@ -143,7 +143,7 @@ void QueryEntries::writeSQL(const_iterator from, const_iterator to, WrSerializer
 	for (const_iterator it = from; it != to; ++it) {
 		if (it != from) {
 			ser << ' ';
-			if (unsigned(it->Op) < sizeof(opNames) / sizeof(opNames[0])) ser << opNames[it->Op] << ' ';
+			if (unsigned(it->Op) < sizeof(opNames) / sizeof(opNames[0])) ser << opNames[it->Op] << ' ';  // -V547
 		} else if (it->Op == OpNot) {
 			ser << "NOT ";
 		}