forked from chromium/chromium
-
Notifications
You must be signed in to change notification settings - Fork 0
/
connection.cc
2002 lines (1697 loc) · 66.8 KB
/
connection.cc
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// Copyright (c) 2012 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "sql/connection.h"
#include <string.h>
#include "base/bind.h"
#include "base/debug/dump_without_crashing.h"
#include "base/files/file_path.h"
#include "base/files/file_util.h"
#include "base/format_macros.h"
#include "base/json/json_file_value_serializer.h"
#include "base/lazy_instance.h"
#include "base/logging.h"
#include "base/message_loop/message_loop.h"
#include "base/metrics/histogram.h"
#include "base/metrics/sparse_histogram.h"
#include "base/strings/string_split.h"
#include "base/strings/string_util.h"
#include "base/strings/stringprintf.h"
#include "base/strings/utf_string_conversions.h"
#include "base/synchronization/lock.h"
#include "base/trace_event/memory_dump_manager.h"
#include "base/trace_event/process_memory_dump.h"
#include "sql/statement.h"
#include "third_party/sqlite/sqlite3.h"
#if defined(OS_IOS) && defined(USE_SYSTEM_SQLITE)
#include "third_party/sqlite/src/ext/icu/sqliteicu.h"
#endif
namespace {
// Spin for up to a second waiting for the lock to clear when setting
// up the database.
// TODO(shess): Better story on this. http://crbug.com/56559
const int kBusyTimeoutSeconds = 1;
class ScopedBusyTimeout {
public:
explicit ScopedBusyTimeout(sqlite3* db)
: db_(db) {
}
~ScopedBusyTimeout() {
sqlite3_busy_timeout(db_, 0);
}
int SetTimeout(base::TimeDelta timeout) {
DCHECK_LT(timeout.InMilliseconds(), INT_MAX);
return sqlite3_busy_timeout(db_,
static_cast<int>(timeout.InMilliseconds()));
}
private:
sqlite3* db_;
};
// Helper to "safely" enable writable_schema. No error checking
// because it is reasonable to just forge ahead in case of an error.
// If turning it on fails, then most likely nothing will work, whereas
// if turning it off fails, it only matters if some code attempts to
// continue working with the database and tries to modify the
// sqlite_master table (none of our code does this).
class ScopedWritableSchema {
public:
explicit ScopedWritableSchema(sqlite3* db)
: db_(db) {
sqlite3_exec(db_, "PRAGMA writable_schema=1", NULL, NULL, NULL);
}
~ScopedWritableSchema() {
sqlite3_exec(db_, "PRAGMA writable_schema=0", NULL, NULL, NULL);
}
private:
sqlite3* db_;
};
// Helper to wrap the sqlite3_backup_*() step of Raze(). Return
// SQLite error code from running the backup step.
int BackupDatabase(sqlite3* src, sqlite3* dst, const char* db_name) {
DCHECK_NE(src, dst);
sqlite3_backup* backup = sqlite3_backup_init(dst, db_name, src, db_name);
if (!backup) {
// Since this call only sets things up, this indicates a gross
// error in SQLite.
DLOG(FATAL) << "Unable to start sqlite3_backup(): " << sqlite3_errmsg(dst);
return sqlite3_errcode(dst);
}
// -1 backs up the entire database.
int rc = sqlite3_backup_step(backup, -1);
int pages = sqlite3_backup_pagecount(backup);
sqlite3_backup_finish(backup);
// If successful, exactly one page should have been backed up. If
// this breaks, check this function to make sure assumptions aren't
// being broken.
if (rc == SQLITE_DONE)
DCHECK_EQ(pages, 1);
return rc;
}
// Be very strict on attachment point. SQLite can handle a much wider
// character set with appropriate quoting, but Chromium code should
// just use clean names to start with.
bool ValidAttachmentPoint(const char* attachment_point) {
for (size_t i = 0; attachment_point[i]; ++i) {
if (!((attachment_point[i] >= '0' && attachment_point[i] <= '9') ||
(attachment_point[i] >= 'a' && attachment_point[i] <= 'z') ||
(attachment_point[i] >= 'A' && attachment_point[i] <= 'Z') ||
attachment_point[i] == '_')) {
return false;
}
}
return true;
}
void RecordSqliteMemory10Min() {
const int64 used = sqlite3_memory_used();
UMA_HISTOGRAM_COUNTS("Sqlite.MemoryKB.TenMinutes", used / 1024);
}
void RecordSqliteMemoryHour() {
const int64 used = sqlite3_memory_used();
UMA_HISTOGRAM_COUNTS("Sqlite.MemoryKB.OneHour", used / 1024);
}
void RecordSqliteMemoryDay() {
const int64 used = sqlite3_memory_used();
UMA_HISTOGRAM_COUNTS("Sqlite.MemoryKB.OneDay", used / 1024);
}
void RecordSqliteMemoryWeek() {
const int64 used = sqlite3_memory_used();
UMA_HISTOGRAM_COUNTS("Sqlite.MemoryKB.OneWeek", used / 1024);
}
// SQLite automatically calls sqlite3_initialize() lazily, but
// sqlite3_initialize() uses double-checked locking and thus can have
// data races.
//
// TODO(shess): Another alternative would be to have
// sqlite3_initialize() called as part of process bring-up. If this
// is changed, remove the dynamic_annotations dependency in sql.gyp.
base::LazyInstance<base::Lock>::Leaky
g_sqlite_init_lock = LAZY_INSTANCE_INITIALIZER;
void InitializeSqlite() {
base::AutoLock lock(g_sqlite_init_lock.Get());
static bool first_call = true;
if (first_call) {
sqlite3_initialize();
// Schedule callback to record memory footprint histograms at 10m, 1h, and
// 1d. There may not be a message loop in tests.
if (base::MessageLoop::current()) {
base::MessageLoop::current()->PostDelayedTask(
FROM_HERE, base::Bind(&RecordSqliteMemory10Min),
base::TimeDelta::FromMinutes(10));
base::MessageLoop::current()->PostDelayedTask(
FROM_HERE, base::Bind(&RecordSqliteMemoryHour),
base::TimeDelta::FromHours(1));
base::MessageLoop::current()->PostDelayedTask(
FROM_HERE, base::Bind(&RecordSqliteMemoryDay),
base::TimeDelta::FromDays(1));
base::MessageLoop::current()->PostDelayedTask(
FROM_HERE, base::Bind(&RecordSqliteMemoryWeek),
base::TimeDelta::FromDays(7));
}
first_call = false;
}
}
// Helper to get the sqlite3_file* associated with the "main" database.
int GetSqlite3File(sqlite3* db, sqlite3_file** file) {
*file = NULL;
int rc = sqlite3_file_control(db, NULL, SQLITE_FCNTL_FILE_POINTER, file);
if (rc != SQLITE_OK)
return rc;
// TODO(shess): NULL in file->pMethods has been observed on android_dbg
// content_unittests, even though it should not be possible.
// http://crbug.com/329982
if (!*file || !(*file)->pMethods)
return SQLITE_ERROR;
return rc;
}
// Convenience to get the sqlite3_file* and the size for the "main" database.
int GetSqlite3FileAndSize(sqlite3* db,
sqlite3_file** file, sqlite3_int64* db_size) {
int rc = GetSqlite3File(db, file);
if (rc != SQLITE_OK)
return rc;
return (*file)->pMethods->xFileSize(*file, db_size);
}
// This should match UMA_HISTOGRAM_MEDIUM_TIMES().
base::HistogramBase* GetMediumTimeHistogram(const std::string& name) {
return base::Histogram::FactoryTimeGet(
name,
base::TimeDelta::FromMilliseconds(10),
base::TimeDelta::FromMinutes(3),
50,
base::HistogramBase::kUmaTargetedHistogramFlag);
}
std::string AsUTF8ForSQL(const base::FilePath& path) {
#if defined(OS_WIN)
return base::WideToUTF8(path.value());
#elif defined(OS_POSIX)
return path.value();
#endif
}
} // namespace
namespace sql {
// static
Connection::ErrorIgnorerCallback* Connection::current_ignorer_cb_ = NULL;
// static
bool Connection::ShouldIgnoreSqliteError(int error) {
if (!current_ignorer_cb_)
return false;
return current_ignorer_cb_->Run(error);
}
// static
bool Connection::ShouldIgnoreSqliteCompileError(int error) {
// Put this first in case tests need to see that the check happened.
if (ShouldIgnoreSqliteError(error))
return true;
// Trim extended error codes.
int basic_error = error & 0xff;
// These errors relate more to the runtime context of the system than to
// errors with a SQL statement or with the schema, so they aren't generally
// interesting to flag. This list is not comprehensive.
return basic_error == SQLITE_BUSY ||
basic_error == SQLITE_NOTADB ||
basic_error == SQLITE_CORRUPT;
}
bool Connection::OnMemoryDump(const base::trace_event::MemoryDumpArgs& args,
base::trace_event::ProcessMemoryDump* pmd) {
if (args.level_of_detail ==
base::trace_event::MemoryDumpLevelOfDetail::LIGHT ||
!db_) {
return true;
}
// The high water mark is not tracked for the following usages.
int cache_size, dummy_int;
sqlite3_db_status(db_, SQLITE_DBSTATUS_CACHE_USED, &cache_size, &dummy_int,
0 /* resetFlag */);
int schema_size;
sqlite3_db_status(db_, SQLITE_DBSTATUS_SCHEMA_USED, &schema_size, &dummy_int,
0 /* resetFlag */);
int statement_size;
sqlite3_db_status(db_, SQLITE_DBSTATUS_STMT_USED, &statement_size, &dummy_int,
0 /* resetFlag */);
std::string name = base::StringPrintf(
"sqlite/%s_connection/%p",
histogram_tag_.empty() ? "Unknown" : histogram_tag_.c_str(), this);
base::trace_event::MemoryAllocatorDump* dump = pmd->CreateAllocatorDump(name);
dump->AddScalar(base::trace_event::MemoryAllocatorDump::kNameSize,
base::trace_event::MemoryAllocatorDump::kUnitsBytes,
cache_size + schema_size + statement_size);
dump->AddScalar("cache_size",
base::trace_event::MemoryAllocatorDump::kUnitsBytes,
cache_size);
dump->AddScalar("schema_size",
base::trace_event::MemoryAllocatorDump::kUnitsBytes,
schema_size);
dump->AddScalar("statement_size",
base::trace_event::MemoryAllocatorDump::kUnitsBytes,
statement_size);
return true;
}
void Connection::ReportDiagnosticInfo(int extended_error, Statement* stmt) {
AssertIOAllowed();
std::string debug_info;
const int error = (extended_error & 0xFF);
if (error == SQLITE_CORRUPT) {
debug_info = CollectCorruptionInfo();
} else {
debug_info = CollectErrorInfo(extended_error, stmt);
}
if (!debug_info.empty() && RegisterIntentToUpload()) {
char debug_buf[2000];
base::strlcpy(debug_buf, debug_info.c_str(), arraysize(debug_buf));
base::debug::Alias(&debug_buf);
base::debug::DumpWithoutCrashing();
}
}
// static
void Connection::SetErrorIgnorer(Connection::ErrorIgnorerCallback* cb) {
CHECK(current_ignorer_cb_ == NULL);
current_ignorer_cb_ = cb;
}
// static
void Connection::ResetErrorIgnorer() {
CHECK(current_ignorer_cb_);
current_ignorer_cb_ = NULL;
}
bool StatementID::operator<(const StatementID& other) const {
if (number_ != other.number_)
return number_ < other.number_;
return strcmp(str_, other.str_) < 0;
}
Connection::StatementRef::StatementRef(Connection* connection,
sqlite3_stmt* stmt,
bool was_valid)
: connection_(connection),
stmt_(stmt),
was_valid_(was_valid) {
if (connection)
connection_->StatementRefCreated(this);
}
Connection::StatementRef::~StatementRef() {
if (connection_)
connection_->StatementRefDeleted(this);
Close(false);
}
void Connection::StatementRef::Close(bool forced) {
if (stmt_) {
// Call to AssertIOAllowed() cannot go at the beginning of the function
// because Close() is called unconditionally from destructor to clean
// connection_. And if this is inactive statement this won't cause any
// disk access and destructor most probably will be called on thread
// not allowing disk access.
// TODO(paivanof@gmail.com): This should move to the beginning
// of the function. http://crbug.com/136655.
AssertIOAllowed();
sqlite3_finalize(stmt_);
stmt_ = NULL;
}
connection_ = NULL; // The connection may be getting deleted.
// Forced close is expected to happen from a statement error
// handler. In that case maintain the sense of |was_valid_| which
// previously held for this ref.
was_valid_ = was_valid_ && forced;
}
Connection::Connection()
: db_(NULL),
page_size_(0),
cache_size_(0),
exclusive_locking_(false),
restrict_to_user_(false),
transaction_nesting_(0),
needs_rollback_(false),
in_memory_(false),
poisoned_(false),
mmap_disabled_(false),
mmap_enabled_(false),
total_changes_at_last_release_(0),
stats_histogram_(NULL),
commit_time_histogram_(NULL),
autocommit_time_histogram_(NULL),
update_time_histogram_(NULL),
query_time_histogram_(NULL),
clock_(new TimeSource()) {
base::trace_event::MemoryDumpManager::GetInstance()->RegisterDumpProvider(
this, "sql::Connection", nullptr);
}
Connection::~Connection() {
base::trace_event::MemoryDumpManager::GetInstance()->UnregisterDumpProvider(
this);
Close();
}
void Connection::RecordEvent(Events event, size_t count) {
for (size_t i = 0; i < count; ++i) {
UMA_HISTOGRAM_ENUMERATION("Sqlite.Stats", event, EVENT_MAX_VALUE);
}
if (stats_histogram_) {
for (size_t i = 0; i < count; ++i) {
stats_histogram_->Add(event);
}
}
}
void Connection::RecordCommitTime(const base::TimeDelta& delta) {
RecordUpdateTime(delta);
UMA_HISTOGRAM_MEDIUM_TIMES("Sqlite.CommitTime", delta);
if (commit_time_histogram_)
commit_time_histogram_->AddTime(delta);
}
void Connection::RecordAutoCommitTime(const base::TimeDelta& delta) {
RecordUpdateTime(delta);
UMA_HISTOGRAM_MEDIUM_TIMES("Sqlite.AutoCommitTime", delta);
if (autocommit_time_histogram_)
autocommit_time_histogram_->AddTime(delta);
}
void Connection::RecordUpdateTime(const base::TimeDelta& delta) {
RecordQueryTime(delta);
UMA_HISTOGRAM_MEDIUM_TIMES("Sqlite.UpdateTime", delta);
if (update_time_histogram_)
update_time_histogram_->AddTime(delta);
}
void Connection::RecordQueryTime(const base::TimeDelta& delta) {
UMA_HISTOGRAM_MEDIUM_TIMES("Sqlite.QueryTime", delta);
if (query_time_histogram_)
query_time_histogram_->AddTime(delta);
}
void Connection::RecordTimeAndChanges(
const base::TimeDelta& delta, bool read_only) {
if (read_only) {
RecordQueryTime(delta);
} else {
const int changes = sqlite3_changes(db_);
if (sqlite3_get_autocommit(db_)) {
RecordAutoCommitTime(delta);
RecordEvent(EVENT_CHANGES_AUTOCOMMIT, changes);
} else {
RecordUpdateTime(delta);
RecordEvent(EVENT_CHANGES, changes);
}
}
}
bool Connection::Open(const base::FilePath& path) {
if (!histogram_tag_.empty()) {
int64_t size_64 = 0;
if (base::GetFileSize(path, &size_64)) {
size_t sample = static_cast<size_t>(size_64 / 1024);
std::string full_histogram_name = "Sqlite.SizeKB." + histogram_tag_;
base::HistogramBase* histogram =
base::Histogram::FactoryGet(
full_histogram_name, 1, 1000000, 50,
base::HistogramBase::kUmaTargetedHistogramFlag);
if (histogram)
histogram->Add(sample);
}
}
return OpenInternal(AsUTF8ForSQL(path), RETRY_ON_POISON);
}
bool Connection::OpenInMemory() {
in_memory_ = true;
return OpenInternal(":memory:", NO_RETRY);
}
bool Connection::OpenTemporary() {
return OpenInternal("", NO_RETRY);
}
void Connection::CloseInternal(bool forced) {
// TODO(shess): Calling "PRAGMA journal_mode = DELETE" at this point
// will delete the -journal file. For ChromiumOS or other more
// embedded systems, this is probably not appropriate, whereas on
// desktop it might make some sense.
// sqlite3_close() needs all prepared statements to be finalized.
// Release cached statements.
statement_cache_.clear();
// With cached statements released, in-use statements will remain.
// Closing the database while statements are in use is an API
// violation, except for forced close (which happens from within a
// statement's error handler).
DCHECK(forced || open_statements_.empty());
// Deactivate any outstanding statements so sqlite3_close() works.
for (StatementRefSet::iterator i = open_statements_.begin();
i != open_statements_.end(); ++i)
(*i)->Close(forced);
open_statements_.clear();
if (db_) {
// Call to AssertIOAllowed() cannot go at the beginning of the function
// because Close() must be called from destructor to clean
// statement_cache_, it won't cause any disk access and it most probably
// will happen on thread not allowing disk access.
// TODO(paivanof@gmail.com): This should move to the beginning
// of the function. http://crbug.com/136655.
AssertIOAllowed();
int rc = sqlite3_close(db_);
if (rc != SQLITE_OK) {
UMA_HISTOGRAM_SPARSE_SLOWLY("Sqlite.CloseFailure", rc);
DLOG(FATAL) << "sqlite3_close failed: " << GetErrorMessage();
}
}
db_ = NULL;
}
void Connection::Close() {
// If the database was already closed by RazeAndClose(), then no
// need to close again. Clear the |poisoned_| bit so that incorrect
// API calls are caught.
if (poisoned_) {
poisoned_ = false;
return;
}
CloseInternal(false);
}
void Connection::Preload() {
AssertIOAllowed();
if (!db_) {
DLOG_IF(FATAL, !poisoned_) << "Cannot preload null db";
return;
}
// Use local settings if provided, otherwise use documented defaults. The
// actual results could be fetching via PRAGMA calls.
const int page_size = page_size_ ? page_size_ : 1024;
sqlite3_int64 preload_size = page_size * (cache_size_ ? cache_size_ : 2000);
if (preload_size < 1)
return;
sqlite3_file* file = NULL;
sqlite3_int64 file_size = 0;
int rc = GetSqlite3FileAndSize(db_, &file, &file_size);
if (rc != SQLITE_OK)
return;
// Don't preload more than the file contains.
if (preload_size > file_size)
preload_size = file_size;
scoped_ptr<char[]> buf(new char[page_size]);
for (sqlite3_int64 pos = 0; pos < preload_size; pos += page_size) {
rc = file->pMethods->xRead(file, buf.get(), page_size, pos);
// TODO(shess): Consider calling OnSqliteError().
if (rc != SQLITE_OK)
return;
}
}
// SQLite keeps unused pages associated with a connection in a cache. It asks
// the cache for pages by an id, and if the page is present and the database is
// unchanged, it considers the content of the page valid and doesn't read it
// from disk. When memory-mapped I/O is enabled, on read SQLite uses page
// structures created from the memory map data before consulting the cache. On
// write SQLite creates a new in-memory page structure, copies the data from the
// memory map, and later writes it, releasing the updated page back to the
// cache.
//
// This means that in memory-mapped mode, the contents of the cached pages are
// not re-used for reads, but they are re-used for writes if the re-written page
// is still in the cache. The implementation of sqlite3_db_release_memory() as
// of SQLite 3.8.7.4 frees all pages from pcaches associated with the
// connection, so it should free these pages.
//
// Unfortunately, the zero page is also freed. That page is never accessed
// using memory-mapped I/O, and the cached copy can be re-used after verifying
// the file change counter on disk. Also, fresh pages from cache receive some
// pager-level initialization before they can be used. Since the information
// involved will immediately be accessed in various ways, it is unclear if the
// additional overhead is material, or just moving processor cache effects
// around.
//
// TODO(shess): It would be better to release the pages immediately when they
// are no longer needed. This would basically happen after SQLite commits a
// transaction. I had implemented a pcache wrapper to do this, but it involved
// layering violations, and it had to be setup before any other sqlite call,
// which was brittle. Also, for large files it would actually make sense to
// maintain the existing pcache behavior for blocks past the memory-mapped
// segment. I think drh would accept a reasonable implementation of the overall
// concept for upstreaming to SQLite core.
//
// TODO(shess): Another possibility would be to set the cache size small, which
// would keep the zero page around, plus some pre-initialized pages, and SQLite
// can manage things. The downside is that updates larger than the cache would
// spill to the journal. That could be compensated by setting cache_spill to
// false. The downside then is that it allows open-ended use of memory for
// large transactions.
//
// TODO(shess): The TrimMemory() trick of bouncing the cache size would also
// work. There could be two prepared statements, one for cache_size=1 one for
// cache_size=goal.
void Connection::ReleaseCacheMemoryIfNeeded(bool implicit_change_performed) {
DCHECK(is_open());
// If memory-mapping is not enabled, the page cache helps performance.
if (!mmap_enabled_)
return;
// On caller request, force the change comparison to fail. Done before the
// transaction-nesting test so that the signal can carry to transaction
// commit.
if (implicit_change_performed)
--total_changes_at_last_release_;
// Cached pages may be re-used within the same transaction.
if (transaction_nesting())
return;
// If no changes have been made, skip flushing. This allows the first page of
// the database to remain in cache across multiple reads.
const int total_changes = sqlite3_total_changes(db_);
if (total_changes == total_changes_at_last_release_)
return;
total_changes_at_last_release_ = total_changes;
sqlite3_db_release_memory(db_);
}
base::FilePath Connection::DbPath() const {
if (!is_open())
return base::FilePath();
const char* path = sqlite3_db_filename(db_, "main");
const base::StringPiece db_path(path);
#if defined(OS_WIN)
return base::FilePath(base::UTF8ToWide(db_path));
#elif defined(OS_POSIX)
return base::FilePath(db_path);
#else
NOTREACHED();
return base::FilePath();
#endif
}
// Data is persisted in a file shared between databases in the same directory.
// The "sqlite-diag" file contains a dictionary with the version number, and an
// array of histogram tags for databases which have been dumped.
bool Connection::RegisterIntentToUpload() const {
static const char* kVersionKey = "version";
static const char* kDiagnosticDumpsKey = "DiagnosticDumps";
static int kVersion = 1;
AssertIOAllowed();
if (histogram_tag_.empty())
return false;
if (!is_open())
return false;
if (in_memory_)
return false;
const base::FilePath db_path = DbPath();
if (db_path.empty())
return false;
// Put the collection of diagnostic data next to the databases. In most
// cases, this is the profile directory, but safe-browsing stores a Cookies
// file in the directory above the profile directory.
base::FilePath breadcrumb_path(
db_path.DirName().Append(FILE_PATH_LITERAL("sqlite-diag")));
// Lock against multiple updates to the diagnostics file. This code should
// seldom be called in the first place, and when called it should seldom be
// called for multiple databases, and when called for multiple databases there
// is _probably_ something systemic wrong with the user's system. So the lock
// should never be contended, but when it is the database experience is
// already bad.
base::AutoLock lock(g_sqlite_init_lock.Get());
scoped_ptr<base::Value> root;
if (!base::PathExists(breadcrumb_path)) {
scoped_ptr<base::DictionaryValue> root_dict(new base::DictionaryValue());
root_dict->SetInteger(kVersionKey, kVersion);
scoped_ptr<base::ListValue> dumps(new base::ListValue);
dumps->AppendString(histogram_tag_);
root_dict->Set(kDiagnosticDumpsKey, dumps.Pass());
root = root_dict.Pass();
} else {
// Failure to read a valid dictionary implies that something is going wrong
// on the system.
JSONFileValueDeserializer deserializer(breadcrumb_path);
scoped_ptr<base::Value> read_root(
deserializer.Deserialize(nullptr, nullptr));
if (!read_root.get())
return false;
scoped_ptr<base::DictionaryValue> root_dict =
base::DictionaryValue::From(read_root.Pass());
if (!root_dict)
return false;
// Don't upload if the version is missing or newer.
int version = 0;
if (!root_dict->GetInteger(kVersionKey, &version) || version > kVersion)
return false;
base::ListValue* dumps = nullptr;
if (!root_dict->GetList(kDiagnosticDumpsKey, &dumps))
return false;
const size_t size = dumps->GetSize();
for (size_t i = 0; i < size; ++i) {
std::string s;
// Don't upload if the value isn't a string, or indicates a prior upload.
if (!dumps->GetString(i, &s) || s == histogram_tag_)
return false;
}
// Record intention to proceed with upload.
dumps->AppendString(histogram_tag_);
root = root_dict.Pass();
}
const base::FilePath breadcrumb_new =
breadcrumb_path.AddExtension(FILE_PATH_LITERAL("new"));
base::DeleteFile(breadcrumb_new, false);
// No upload if the breadcrumb file cannot be updated.
// TODO(shess): Consider ImportantFileWriter::WriteFileAtomically() to land
// the data on disk. For now, losing the data is not a big problem, so the
// sync overhead would probably not be worth it.
JSONFileValueSerializer serializer(breadcrumb_new);
if (!serializer.Serialize(*root))
return false;
if (!base::PathExists(breadcrumb_new))
return false;
if (!base::ReplaceFile(breadcrumb_new, breadcrumb_path, nullptr)) {
base::DeleteFile(breadcrumb_new, false);
return false;
}
return true;
}
std::string Connection::CollectErrorInfo(int error, Statement* stmt) const {
// Buffer for accumulating debugging info about the error. Place
// more-relevant information earlier, in case things overflow the
// fixed-size reporting buffer.
std::string debug_info;
// The error message from the failed operation.
base::StringAppendF(&debug_info, "db error: %d/%s\n",
GetErrorCode(), GetErrorMessage());
// TODO(shess): |error| and |GetErrorCode()| should always be the same, but
// reading code does not entirely convince me. Remove if they turn out to be
// the same.
if (error != GetErrorCode())
base::StringAppendF(&debug_info, "reported error: %d\n", error);
// System error information. Interpretation of Windows errors is different
// from posix.
#if defined(OS_WIN)
base::StringAppendF(&debug_info, "LastError: %d\n", GetLastErrno());
#elif defined(OS_POSIX)
base::StringAppendF(&debug_info, "errno: %d\n", GetLastErrno());
#else
NOTREACHED(); // Add appropriate log info.
#endif
if (stmt) {
base::StringAppendF(&debug_info, "statement: %s\n",
stmt->GetSQLStatement());
} else {
base::StringAppendF(&debug_info, "statement: NULL\n");
}
// SQLITE_ERROR often indicates some sort of mismatch between the statement
// and the schema, possibly due to a failed schema migration.
if (error == SQLITE_ERROR) {
const char* kVersionSql = "SELECT value FROM meta WHERE key = 'version'";
sqlite3_stmt* s;
int rc = sqlite3_prepare_v2(db_, kVersionSql, -1, &s, nullptr);
if (rc == SQLITE_OK) {
rc = sqlite3_step(s);
if (rc == SQLITE_ROW) {
base::StringAppendF(&debug_info, "version: %d\n",
sqlite3_column_int(s, 0));
} else if (rc == SQLITE_DONE) {
debug_info += "version: none\n";
} else {
base::StringAppendF(&debug_info, "version: error %d\n", rc);
}
sqlite3_finalize(s);
} else {
base::StringAppendF(&debug_info, "version: prepare error %d\n", rc);
}
debug_info += "schema:\n";
// sqlite_master has columns:
// type - "index" or "table".
// name - name of created element.
// tbl_name - name of element, or target table in case of index.
// rootpage - root page of the element in database file.
// sql - SQL to create the element.
// In general, the |sql| column is sufficient to derive the other columns.
// |rootpage| is not interesting for debugging, without the contents of the
// database. The COALESCE is because certain automatic elements will have a
// |name| but no |sql|,
const char* kSchemaSql = "SELECT COALESCE(sql, name) FROM sqlite_master";
rc = sqlite3_prepare_v2(db_, kSchemaSql, -1, &s, nullptr);
if (rc == SQLITE_OK) {
while ((rc = sqlite3_step(s)) == SQLITE_ROW) {
base::StringAppendF(&debug_info, "%s\n", sqlite3_column_text(s, 0));
}
if (rc != SQLITE_DONE)
base::StringAppendF(&debug_info, "error %d\n", rc);
sqlite3_finalize(s);
} else {
base::StringAppendF(&debug_info, "prepare error %d\n", rc);
}
}
return debug_info;
}
// TODO(shess): Since this is only called in an error situation, it might be
// prudent to rewrite in terms of SQLite API calls, and mark the function const.
std::string Connection::CollectCorruptionInfo() {
AssertIOAllowed();
// If the file cannot be accessed it is unlikely that an integrity check will
// turn up actionable information.
const base::FilePath db_path = DbPath();
int64 db_size = -1;
if (!base::GetFileSize(db_path, &db_size) || db_size < 0)
return std::string();
// Buffer for accumulating debugging info about the error. Place
// more-relevant information earlier, in case things overflow the
// fixed-size reporting buffer.
std::string debug_info;
base::StringAppendF(&debug_info, "SQLITE_CORRUPT, db size %" PRId64 "\n",
db_size);
// Only check files up to 8M to keep things from blocking too long.
const int64 kMaxIntegrityCheckSize = 8192 * 1024;
if (db_size > kMaxIntegrityCheckSize) {
debug_info += "integrity_check skipped due to size\n";
} else {
std::vector<std::string> messages;
// TODO(shess): FullIntegrityCheck() splits into a vector while this joins
// into a string. Probably should be refactored.
const base::TimeTicks before = base::TimeTicks::Now();
FullIntegrityCheck(&messages);
base::StringAppendF(
&debug_info,
"integrity_check %" PRId64 " ms, %" PRIuS " records:\n",
(base::TimeTicks::Now() - before).InMilliseconds(),
messages.size());
// SQLite returns up to 100 messages by default, trim deeper to
// keep close to the 2000-character size limit for dumping.
const size_t kMaxMessages = 20;
for (size_t i = 0; i < kMaxMessages && i < messages.size(); ++i) {
base::StringAppendF(&debug_info, "%s\n", messages[i].c_str());
}
}
return debug_info;
}
size_t Connection::GetAppropriateMmapSize() {
AssertIOAllowed();
// TODO(shess): Using sql::MetaTable seems indicated, but mixing
// sql::MetaTable and direct access seems error-prone. It might make sense to
// simply integrate sql::MetaTable functionality into sql::Connection.
#if defined(OS_IOS)
// iOS SQLite does not support memory mapping.
return 0;
#endif
// If the database doesn't have a place to track progress, assume the worst.
// This will happen when new databases are created.
if (!DoesTableExist("meta")) {
RecordOneEvent(EVENT_MMAP_META_MISSING);
return 0;
}
// Key into meta table to get status from a previous run. The value
// represents how much data in bytes has successfully been read from the
// database. |kMmapFailure| indicates that there was a read error and the
// database should not be memory-mapped, while |kMmapSuccess| indicates that
// the entire file was read at some point and can be memory-mapped without
// constraint.
const char* kMmapStatusKey = "mmap_status";
static const sqlite3_int64 kMmapFailure = -2;
static const sqlite3_int64 kMmapSuccess = -1;
// Start reading from 0 unless status is found in meta table.
sqlite3_int64 mmap_ofs = 0;
// Retrieve the current status. It is fine for the status to be missing
// entirely, but any error prevents memory-mapping.
{
const char* kMmapStatusSql = "SELECT value FROM meta WHERE key = ?";
Statement s(GetUniqueStatement(kMmapStatusSql));
s.BindString(0, kMmapStatusKey);
if (s.Step()) {
mmap_ofs = s.ColumnInt64(0);
} else if (!s.Succeeded()) {
RecordOneEvent(EVENT_MMAP_META_FAILURE_READ);
return 0;
}
}
// Database read failed in the past, don't memory map.
if (mmap_ofs == kMmapFailure) {
RecordOneEvent(EVENT_MMAP_FAILED);
return 0;
} else if (mmap_ofs != kMmapSuccess) {
// Continue reading from previous offset.
DCHECK_GE(mmap_ofs, 0);
// TODO(shess): Could this reading code be shared with Preload()? It would
// require locking twice (this code wouldn't be able to access |db_size| so
// the helper would have to return amount read).
// Read more of the database looking for errors. The VFS interface is used
// to assure that the reads are valid for SQLite. |g_reads_allowed| is used
// to limit checking to 20MB per run of Chromium.
sqlite3_file* file = NULL;
sqlite3_int64 db_size = 0;
if (SQLITE_OK != GetSqlite3FileAndSize(db_, &file, &db_size)) {
RecordOneEvent(EVENT_MMAP_VFS_FAILURE);
return 0;
}
// Read the data left, or |g_reads_allowed|, whichever is smaller.
// |g_reads_allowed| limits the total amount of I/O to spend verifying data
// in a single Chromium run.
sqlite3_int64 amount = db_size - mmap_ofs;
if (amount < 0)
amount = 0;
if (amount > 0) {
base::AutoLock lock(g_sqlite_init_lock.Get());
static sqlite3_int64 g_reads_allowed = 20 * 1024 * 1024;
if (g_reads_allowed < amount)
amount = g_reads_allowed;
g_reads_allowed -= amount;
}
// |amount| can be <= 0 if |g_reads_allowed| ran out of quota, or if the
// database was truncated after a previous pass.
if (amount <= 0 && mmap_ofs < db_size) {
DCHECK_EQ(0, amount);
RecordOneEvent(EVENT_MMAP_SUCCESS_NO_PROGRESS);
} else {
static const int kPageSize = 4096;
char buf[kPageSize];
while (amount > 0) {
int rc = file->pMethods->xRead(file, buf, sizeof(buf), mmap_ofs);
if (rc == SQLITE_OK) {
mmap_ofs += sizeof(buf);
amount -= sizeof(buf);
} else if (rc == SQLITE_IOERR_SHORT_READ) {
// Reached EOF for a database with page size < |kPageSize|.
mmap_ofs = db_size;
break;
} else {
// TODO(shess): Consider calling OnSqliteError().
mmap_ofs = kMmapFailure;
break;
}
}
// Log these events after update to distinguish meta update failure.
Events event;
if (mmap_ofs >= db_size) {
mmap_ofs = kMmapSuccess;
event = EVENT_MMAP_SUCCESS_NEW;
} else if (mmap_ofs > 0) {
event = EVENT_MMAP_SUCCESS_PARTIAL;
} else {
DCHECK_EQ(kMmapFailure, mmap_ofs);
event = EVENT_MMAP_FAILED_NEW;
}