[perf] upgrade realm and improve schemas/queries

- upgrade to realm 2.28 to benefit from perf improvements - remove user-statuses and replace by seen and outgoing-status fields to get rid of a lot of bloat queries and computations - remove unused seen message, bottom-infos - remove unused fields in transport schema - use objectForPrimaryKey whenever possible instead of get by field Signed-off-by: yenda <eric@status.im>
status-im · Jun 4, 2019 · 1ae42ea · 1ae42ea
1 parent b3df9d4
commit 1ae42ea
Show file tree

Hide file tree

Showing 28 changed files with 388 additions and 578 deletions.
diff --git a/mobile_files/package.json.orig b/mobile_files/package.json.orig
@@ -65,7 +65,7 @@
     "react-native-webview": "^5.2.1",
     "react-native-webview-bridge": "git+https://github.com/status-im/react-native-webview-bridge.git#fix/classnames-colision",
     "react-navigation": "^2.12.1",
-    "realm": "^2.26.1",
+    "realm": "2.28.0",
     "rn-snoopy": "git+https://github.com/status-im/rn-snoopy.git#v2.0.2-status",
     "string_decoder": "0.10.31",
     "text-encoding": "^0.6.4",

diff --git a/mobile_files/yarn.lock b/mobile_files/yarn.lock
diff --git a/src/status_im/chat/core.cljs b/src/status_im/chat/core.cljs
diff --git a/src/status_im/chat/db.cljs b/src/status_im/chat/db.cljs
@@ -1,13 +1,13 @@
 (ns status-im.chat.db
   (:require [clojure.set :as clojure.set]
             [clojure.string :as string]
-            [status-im.contact.db :as contact.db]
             [status-im.chat.commands.core :as commands]
             [status-im.chat.commands.input :as commands.input]
+            [status-im.contact.db :as contact.db]
             [status-im.group-chats.db :as group-chats.db]
-            [status-im.utils.gfycat.core :as gfycat]
+            [status-im.mailserver.core :as mailserver]
             [status-im.transport.partitioned-topic :as topic]
-            [status-im.mailserver.core :as mailserver]))
+            [status-im.utils.gfycat.core :as gfycat]))
 
 (defn group-chat-name
   [{:keys [public? name]}]
@@ -93,7 +93,7 @@
   (= type :gap))
 
 (defn transform-message
-  [messages message-statuses referenced-messages]
+  [messages referenced-messages]
   (fn [{:keys [message-id timestamp-str] :as reference}]
     (if (or (datemark? reference)
             (gap? reference))
@@ -104,8 +104,7 @@
                           (quoted-message-data messages referenced-messages))]
         (cond-> (-> message
                     (update :content dissoc :response-to :response-to-v2)
-                    (assoc :timestamp-str timestamp-str
-                           :user-statuses (get message-statuses message-id)))
+                    (assoc :timestamp-str timestamp-str))
           ;; quoted message reference
           quote
           (assoc-in [:content :response-to] quote))))))
@@ -145,15 +144,15 @@
          :value (clojure.string/join (:ids gaps))
          :gaps gaps}))
 
-(defn messages-with-datemarks-and-statuses
+(defn messages-with-datemarks
   "Converts message groups into sequence of messages interspersed with datemarks,
   with correct user statuses associated into message"
-  [message-groups messages message-statuses referenced-messages messages-gaps
+  [message-groups messages referenced-messages messages-gaps
    {:keys [highest-request-to lowest-request-from]} all-loaded? public?]
   (transduce
    (comp
     (mapcat add-datemark)
-    (map (transform-message messages message-statuses referenced-messages)))
+    (map (transform-message messages referenced-messages)))
    (fn
      ([]
       (let [acc {:messages         (list)

diff --git a/src/status_im/chat/models.cljs b/src/status_im/chat/models.cljs
@@ -5,26 +5,22 @@
             [status-im.contact.core :as contact.core]
             [status-im.data-store.chats :as chats-store]
             [status-im.data-store.messages :as messages-store]
-            [status-im.data-store.user-statuses :as user-statuses-store]
             [status-im.i18n :as i18n]
             [status-im.mailserver.core :as mailserver]
             [status-im.transport.chat.core :as transport.chat]
-            [status-im.transport.message.protocol :as protocol]
             [status-im.transport.message.public-chat :as public-chat]
             [status-im.tribute-to-talk.core :as tribute-to-talk]
             [status-im.ui.components.colors :as colors]
             [status-im.ui.components.desktop.events :as desktop.events]
             [status-im.ui.components.react :as react]
             [status-im.ui.screens.navigation :as navigation]
             [status-im.utils.clocks :as utils.clocks]
+            [status-im.utils.config :as config]
             [status-im.utils.fx :as fx]
             [status-im.utils.gfycat.core :as gfycat]
             [status-im.utils.platform :as platform]
             [status-im.utils.priority-map :refer [empty-message-map]]
             [status-im.utils.utils :as utils]
-            [status-im.utils.config :as config]
-            [status-im.mailserver.core :as mailserver]
-            [status-im.transport.partitioned-topic :as transport.topic]
             [taoensso.timbre :as log]))
 
 (defn- get-chat [cofx chat-id]
@@ -191,11 +187,6 @@
                  (contact-code/stop-listening % chat-id))
               (navigation/navigate-to-cofx :home {}))))
 
-(fx/defn send-messages-seen
-  [{:keys [db] :as cofx} chat-id message-ids]
-  (when (not (get-in db [:chats chat-id :group-chat]))
-    (protocol/send (protocol/map->MessagesSeen {:message-ids message-ids}) chat-id cofx)))
-
 (defn- unread-messages-number [chats]
   (apply + (map :unviewed-messages-count chats)))
 
@@ -238,26 +229,17 @@
   "Marks all unviewed loaded messages as seen in particular chat"
   [{:keys [db] :as cofx} chat-id]
   (let [public-key          (accounts.db/current-public-key cofx)
-        loaded-unviewed-ids (get-in db [:chats chat-id :loaded-unviewed-messages-ids])
-        updated-statuses    (map (fn [message-id]
-                                   {:chat-id    chat-id
-                                    :message-id message-id
-                                    :status-id  (str chat-id "-" message-id)
-                                    :public-key public-key
-                                    :status     :seen})
-                                 loaded-unviewed-ids)]
+        loaded-unviewed-ids (get-in db [:chats chat-id :loaded-unviewed-messages-ids])]
     (when (seq loaded-unviewed-ids)
       (fx/merge cofx
-                {:db            (reduce (fn [acc {:keys [message-id status]}]
-                                          (assoc-in acc [:chats chat-id :message-statuses
-                                                         message-id public-key :status]
-                                                    status))
+                {:db            (reduce (fn [acc message-id]
+                                          (assoc-in acc [:chats chat-id :messages
+                                                         message-id :seen]
+                                                    true))
                                         db
-                                        updated-statuses)
-                 :data-store/tx [(user-statuses-store/save-statuses-tx updated-statuses)]}
+                                        loaded-unviewed-ids)
+                 :data-store/tx [(messages-store/mark-messages-seen-tx loaded-unviewed-ids)]}
                 (update-chats-unviewed-messages-count {:chat-id chat-id})
-                ;;TODO(rasom): uncomment when seen messages will be revisited
-                #_(send-messages-seen chat-id loaded-unviewed-ids)
                 (when platform/desktop?
                   (update-dock-badge-label))))))
 

diff --git a/src/status_im/chat/models/loading.cljs b/src/status_im/chat/models/loading.cljs
@@ -53,16 +53,6 @@
               (remove #(some message-id->messages (vals %))))
         (vals message-id->messages)))
 
-(defn get-unviewed-messages-ids
-  [statuses public-key]
-  (keep
-   (fn [[message-id statuses]]
-     (let [{:keys [status]}
-           (get statuses public-key)]
-       (when (= (keyword status) :received)
-         message-id)))
-   statuses))
-
 (fx/defn update-chats-in-app-db
   {:events [:chats-list/load-success]}
   [{:keys [db] :as cofx} chats]
@@ -118,8 +108,8 @@
   "Loads more messages for current chat"
   [{{:keys [current-chat-id] :as db} :db
     get-stored-messages :get-stored-messages
-    get-stored-user-statuses :get-stored-user-statuses
-    get-referenced-messages :get-referenced-messages :as cofx}]
+    get-referenced-messages :get-referenced-messages
+    get-unviewed-message-ids :get-unviewed-message-ids :as cofx}]
   ;; TODO: re-implement functionality for status-go protocol
   (when-not (or config/use-status-go-protocol?
                 (get-in db [:chats current-chat-id :all-loaded?]))
@@ -135,14 +125,12 @@
           referenced-messages        (into empty-message-map
                                            (get-referenced-messages (get-referenced-ids indexed-messages)))
           new-message-ids            (keys indexed-messages)
-          new-statuses               (get-stored-user-statuses current-chat-id new-message-ids)
           public-key                 (accounts.db/current-public-key cofx)
-          loaded-unviewed-messages   (get-unviewed-messages-ids new-statuses public-key)]
+          loaded-unviewed-messages   (get-unviewed-message-ids)]
       (fx/merge cofx
                 {:db (-> db
                          (assoc-in [:chats current-chat-id :messages-initialized?] true)
                          (update-in [:chats current-chat-id :messages] merge indexed-messages)
-                         (update-in [:chats current-chat-id :message-statuses] merge new-statuses)
                          (update-in [:chats current-chat-id :referenced-messages]
                                     #(into (apply dissoc % new-message-ids) referenced-messages))
                          (assoc-in [:chats current-chat-id :pagination-info] pagination-info)

diff --git a/src/status_im/chat/models/message.cljs b/src/status_im/chat/models/message.cljs
@@ -9,7 +9,6 @@
             [status-im.constants :as constants]
             [status-im.contact.db :as contact.db]
             [status-im.data-store.messages :as messages-store]
-            [status-im.data-store.user-statuses :as user-statuses-store]
             [status-im.mailserver.core :as mailserver]
             [status-im.native-module.core :as status]
             [status-im.notifications.core :as notifications]
@@ -39,9 +38,9 @@
   [{:keys [content content-type] :as message} chat-id current-chat?]
   (let [emoji? (message-content/emoji-only-content? content)]
     ;; TODO janherich: enable the animations again once we can do them more efficiently
-    (cond-> (assoc message :appearing? true)
-      (not current-chat?)
-      (assoc :appearing? false)
+    (cond-> message
+      current-chat?
+      (assoc :seen true)
 
       emoji?
       (assoc :content-type constants/content-type-emoji)
@@ -73,21 +72,12 @@
                                              (assoc groups new-datemark message-refs))))
                              {}))}))
 
-(fx/defn add-own-status
-  [{:keys [db] :as cofx} chat-id message-id status]
-  (let [me     (accounts.db/current-public-key cofx)
-        status {:chat-id          chat-id
-                :message-id       message-id
-                :public-key       me
-                :status           status}]
-    {:db            (assoc-in db
-                              [:chats chat-id :message-statuses message-id me]
-                              status)
-     :data-store/tx [(user-statuses-store/save-status-tx status)]}))
-
-(defn add-outgoing-status [{:keys [from] :as message} current-public-key]
-  (assoc message :outgoing (and (= from current-public-key)
-                                (not (system-message? message)))))
+(defn add-outgoing-status
+  [{:keys [from] :as message} current-public-key]
+  (if (and (= from current-public-key)
+           (not (system-message? message)))
+    (assoc message :outgoing true)
+    message))
 
 (defn build-desktop-notification
   [{:keys [db] :as cofx} {:keys [chat-id timestamp content from] :as message}]
@@ -145,11 +135,6 @@
               (when-not batch?
                 (chat-loading/group-chat-messages chat-id [message])))))
 
-(fx/defn send-message-seen
-  [cofx chat-id message-id send-seen?]
-  (when send-seen?
-    (protocol/send (protocol/map->MessagesSeen {:message-ids #{message-id}}) chat-id cofx)))
-
 (defn ensure-clock-value [{:keys [clock-value] :as message} {:keys [last-clock-value]}]
   (if clock-value
     message
@@ -170,35 +155,24 @@
    old-id->message
    {:keys [from message-id chat-id js-obj content dedup-id] :as raw-message}]
   (let [{:keys [web3 current-chat-id view-id]} db
-        current-public-key            (accounts.db/current-public-key cofx)
-        current-chat?                 (and (or (= :chat view-id)
-                                               (= :chat-modal view-id))
-                                           (= current-chat-id chat-id))
-        {:keys [group-chat] :as chat} (get-in db [:chats chat-id])
-        message                       (-> raw-message
-                                          (commands-receiving/enhance-receive-parameters cofx)
-                                          (ensure-clock-value chat)
-                                          (check-response-to old-id->message)
-                                          ;; TODO (cammellos): Refactor so it's not computed twice
-                                          (add-outgoing-status current-public-key))]
+        current-public-key             (accounts.db/current-public-key cofx)
+        current-chat?                  (and (or (= :chat view-id)
+                                                (= :chat-modal view-id))
+                                            (= current-chat-id chat-id))
+        {:keys [group-chat] :as chat}  (get-in db [:chats chat-id])
+        {:keys [outgoing] :as message} (-> raw-message
+                                           (commands-receiving/enhance-receive-parameters cofx)
+                                           (ensure-clock-value chat)
+                                           (check-response-to old-id->message)
+                                           ;; TODO (cammellos): Refactor so it's not computed twice
+                                           (add-outgoing-status current-public-key))]
     (fx/merge cofx
               (add-message {:batch?       true
                             :message      message
                             :dedup-id     dedup-id
                             :current-chat current-chat?
                             :raw-message  js-obj})
-              ;; Checking :outgoing here only works for now as we don't have a :seen
-              ;; status for public chats, if we add processing of our own messages
-              ;; for 1-to-1 care needs to be taken not to override the :seen status
-              (add-own-status chat-id message-id (cond (:outgoing message) :sent
-                                                       current-chat? :seen
-                                                       :else :received))
-              (commands-receiving/receive message)
-              ;;TODO(rasom): uncomment when seen messages will be revisited
-              #_(send-message-seen chat-id message-id (and (not group-chat)
-                                                           current-chat?
-                                                           (not (= constants/system from))
-                                                           (not (:outgoing message)))))))
+              (commands-receiving/receive message))))
 
 (fx/defn update-group-messages [cofx chat->message chat-id]
   (fx/merge cofx
@@ -374,7 +348,8 @@
     (get-in message [:content :params :coin :icon :source])
     (update-in [:content :params :coin] dissoc :icon)))
 
-(fx/defn upsert-and-send [{:keys [now] :as cofx} {:keys [chat-id from] :as message}]
+(fx/defn upsert-and-send
+  [{:keys [now] :as cofx} {:keys [chat-id from] :as message}]
   (let [message         (remove-icon message)
         send-record     (protocol/map->Message (select-keys message transport-keys))
         old-message-id  (transport.utils/old-message-id send-record)
@@ -384,6 +359,7 @@
         raw-payload     (transport.utils/from-utf8 (transit/serialize wrapped-record))
         message-id      (transport.utils/message-id from raw-payload)
         message-with-id (assoc message
+                               :outgoing-status :sending
                                :message-id message-id
                                :old-message-id old-message-id
                                :raw-payload-hash (transport.utils/sha3 raw-payload))]
@@ -398,10 +374,10 @@
               (add-message {:batch?           false
                             :message          message-with-id
                             :current-chat?    true})
-              (add-own-status chat-id message-id :sending)
               (send chat-id message-id wrapped-record))))
 
-(fx/defn send-push-notification [cofx chat-id message-id fcm-tokens status]
+(fx/defn send-push-notification
+  [cofx chat-id message-id fcm-tokens status]
   (log/debug "#6772 - send-push-notification" message-id fcm-tokens)
   (when (and (seq fcm-tokens) (= status :sent))
     (let [payload {:from (accounts.db/current-public-key cofx)
@@ -410,17 +386,15 @@
       {:send-notification {:data-payload (notifications/encode-notification-payload payload)
                            :tokens       fcm-tokens}})))
 
-(fx/defn update-message-status [{:keys [db]} chat-id message-id status]
-  (let [from           (get-in db [:chats chat-id :messages message-id :from])
-        updated-status (-> db
-                           (get-in [:chats chat-id :message-statuses message-id from])
-                           (assoc :status status))]
-    {:db            (assoc-in db
-                              [:chats chat-id :message-statuses message-id from]
-                              updated-status)
-     :data-store/tx [(user-statuses-store/save-status-tx updated-status)]}))
-
-(fx/defn resend-message [cofx chat-id message-id]
+(fx/defn update-message-status
+  [{:keys [db]} chat-id message-id status]
+  {:db (assoc-in db
+                 [:chats chat-id :messages message-id :outgoing-status]
+                 status)
+   :data-store/tx [(messages-store/update-outgoing-status-tx message-id status)]})
+
+(fx/defn resend-message
+  [cofx chat-id message-id]
   (let [message (get-in cofx [:db :chats chat-id :messages message-id])
         send-record (-> message
                         (select-keys transport-keys)

diff --git a/src/status_im/chat/specs.cljs b/src/status_im/chat/specs.cljs
@@ -15,7 +15,6 @@
 (s/def :chat/public-group-topic-error (s/nilable string?))
 (s/def :chat/messages (s/nilable map?))                           ; messages indexed by message-id
 (s/def :chat/message-groups (s/nilable map?))                     ; grouped/sorted messages
-(s/def :chat/message-statuses (s/nilable map?))                   ; message/user statuses indexed by two level index
 (s/def :chat/referenced-messages (s/nilable map?))                ; map of messages indexed by message-id which are not displayed directly, but referenced by other messages
 (s/def :chat/last-clock-value (s/nilable number?))                ; last logical clock value of messages in chat
 (s/def :chat/loaded-chats (s/nilable seq?))