Skip to content

Commit

Permalink
Merge branch 'RavenDB-42' of https://github.com/arekais/ravendb
Browse files Browse the repository at this point in the history
  • Loading branch information
@ayende
ayende committed Feb 5, 2013
2 parents 2cff72f + aeaac08 commit b7aaa4d
Show file tree
Hide file tree
Showing 11 changed files with 495 additions and 114 deletions.
2 changes: 1 addition & 1 deletion Raven.Database/Config/ConfigOptionDocs.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ internal class ConfigOptionDocs : IEnumerable<string>
{"Raven/TempIndexInMemoryMaxMB", "int", "1 - 1024 MB", "The max size in MB of a temporary index held in memory. When a temporary dynamic index exceeds that value, it will be using on disk indexing, rather then RAM indexing. Default: 25 MB."},
{"Raven/CreateTemporaryIndexesForAdHocQueriesIfNeeded", "bool", "true", "Whatever we allow creation of temporary indexes on dynamic queries"},
// Memory
{"Raven/MemoryCacheLimitMegabytes", "int", null, "The max size in MB for the internal document cache inside RavenDB server, default is half of the machine available RAM minus the size of the esent cache."},
{"Raven/IntegerSetting", "int", null, "The max size in MB for the internal document cache inside RavenDB server, default is half of the machine available RAM minus the size of the esent cache."},
{"Raven/MemoryCacheLimitPercentage","int", "0-99", "The percentage of memory that the internal document cache inside RavenDB server will use, default: auto."},
{"Raven/MemoryCacheLimitCheckInterval", "TimeSpan", "HH:MM:SS", "The internal for checking that the internal document cache inside RavenDB server will be cleaned."},
{"Raven/MemoryCacheExpiration", "int", null, "The expiration value for documents in the internal document cache. Value is in seconds. Default: 5 minutes"},
Expand Down
177 changes: 64 additions & 113 deletions Raven.Database/Config/InMemoryRavenConfiguration.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,7 @@
using System.ComponentModel.Composition.Primitives;
using System.IO;
using System.Linq;
using System.Runtime.Caching;
using System.Security.Cryptography;
using System.Security.Cryptography.X509Certificates;
using System.Threading;
using System.Threading.Tasks;
using System.Web;
using Raven.Abstractions.Data;
Expand All @@ -39,20 +36,11 @@ public InMemoryRavenConfiguration()
{
Settings = new NameValueCollection(StringComparer.InvariantCultureIgnoreCase);

MaxNumberOfItemsToIndexInSingleBatch = Environment.Is64BitProcess ? 128 * 1024 : 64 * 1024;
MaxNumberOfItemsToReduceInSingleBatch = MaxNumberOfItemsToIndexInSingleBatch / 2;
InitialNumberOfItemsToIndexInSingleBatch = Environment.Is64BitProcess ? 512 : 256;
InitialNumberOfItemsToReduceInSingleBatch = InitialNumberOfItemsToIndexInSingleBatch / 2;
NumberOfItemsToExecuteReduceInSingleStep = 1024;
MaxIndexingRunLatency = TimeSpan.FromMinutes(5);

CreateTemporaryIndexesForAdHocQueriesIfNeeded = true;

CreatePluginsDirectoryIfNotExisting = true;
CreateAnalyzersDirectoryIfNotExisting = true;

AvailableMemoryForRaisingIndexBatchSizeLimit = Math.Min(768, MemoryStatistics.TotalPhysicalMemory / 2);
MaxNumberOfParallelIndexTasks = 8;

IndexingScheduler = new FairIndexingSchedulerWithNewIndexesBias();

Expand All @@ -74,67 +62,58 @@ public void PostInit()

public void Initialize()
{
int defaultMaxNumberOfItemsToIndexInSingleBatch = Environment.Is64BitProcess ? 128 * 1024 : 64 * 1024;
int defaultInitialNumberOfItemsToIndexInSingleBatch = Environment.Is64BitProcess ? 512 : 256;

var ravenSettings = new StronglyTypedRavenSettings(Settings);
ravenSettings.Setup(defaultMaxNumberOfItemsToIndexInSingleBatch, defaultInitialNumberOfItemsToIndexInSingleBatch);

// Core settings
var maxPageSizeStr = Settings["Raven/MaxPageSize"];
MaxPageSize = maxPageSizeStr != null ? int.Parse(maxPageSizeStr) : 1024;
MaxPageSize = Math.Max(MaxPageSize, 10);

var cacheMemoryLimitMegabytes = Settings["Raven/MemoryCacheLimitMegabytes"];
MemoryCacheLimitMegabytes = cacheMemoryLimitMegabytes == null
? GetDefaultMemoryCacheLimitMegabytes()
: int.Parse(cacheMemoryLimitMegabytes);

var memoryCacheExpiration = Settings["Raven/MemoryCacheExpiration"];
MemoryCacheExpiration = memoryCacheExpiration == null
? TimeSpan.FromMinutes(5)
: TimeSpan.FromSeconds(int.Parse(memoryCacheExpiration));

var memoryCacheLimitPercentage = Settings["Raven/MemoryCacheLimitPercentage"];
MemoryCacheLimitPercentage = memoryCacheLimitPercentage == null
? 0 // auto-size
: int.Parse(memoryCacheLimitPercentage);
var memoryCacheLimitCheckInterval = Settings["Raven/MemoryCacheLimitCheckInterval"];
MemoryCacheLimitCheckInterval = memoryCacheLimitCheckInterval == null
? MemoryCache.Default.PollingInterval
: TimeSpan.Parse(memoryCacheLimitCheckInterval);
MaxPageSize = ravenSettings.MaxPageSize.Value;

MemoryCacheLimitMegabytes = ravenSettings.MemoryCacheLimitMegabytes.Value;

MemoryCacheExpiration = ravenSettings.MemoryCacheExpiration.Value;

MemoryCacheLimitPercentage = ravenSettings.MemoryCacheLimitPercentage.Value;

MemoryCacheLimitCheckInterval = ravenSettings.MemoryCacheLimitCheckInterval.Value;

// Index settings
var maxIndexingRunLatencyStr = Settings["Raven/MaxIndexingRunLatency"];
if (maxIndexingRunLatencyStr != null)
{
MaxIndexingRunLatency = TimeSpan.Parse(maxIndexingRunLatencyStr);
}
MaxIndexingRunLatency = ravenSettings.MaxIndexingRunLatency.Value;

var maxNumberOfItemsToIndexInSingleBatch = Settings["Raven/MaxNumberOfItemsToIndexInSingleBatch"];
if (maxNumberOfItemsToIndexInSingleBatch != null)
MaxNumberOfItemsToIndexInSingleBatch = ravenSettings.MaxNumberOfItemsToIndexInSingleBatch.Value;

if (MaxNumberOfItemsToIndexInSingleBatch == ravenSettings.MaxNumberOfItemsToIndexInSingleBatch.Default)
{
MaxNumberOfItemsToIndexInSingleBatch = Math.Max(int.Parse(maxNumberOfItemsToIndexInSingleBatch), 128);
InitialNumberOfItemsToIndexInSingleBatch = Math.Min(MaxNumberOfItemsToIndexInSingleBatch,
InitialNumberOfItemsToIndexInSingleBatch);
InitialNumberOfItemsToIndexInSingleBatch = defaultInitialNumberOfItemsToIndexInSingleBatch;
}
var availableMemoryForRaisingIndexBatchSizeLimit = Settings["Raven/AvailableMemoryForRaisingIndexBatchSizeLimit"];
if (availableMemoryForRaisingIndexBatchSizeLimit != null)
else
{
AvailableMemoryForRaisingIndexBatchSizeLimit = int.Parse(availableMemoryForRaisingIndexBatchSizeLimit);
InitialNumberOfItemsToIndexInSingleBatch = Math.Min(MaxNumberOfItemsToIndexInSingleBatch, InitialNumberOfItemsToIndexInSingleBatch);
}

AvailableMemoryForRaisingIndexBatchSizeLimit = ravenSettings.AvailableMemoryForRaisingIndexBatchSizeLimit.Value;

var initialNumberOfItemsToIndexInSingleBatch = Settings["Raven/InitialNumberOfItemsToIndexInSingleBatch"];
if (initialNumberOfItemsToIndexInSingleBatch != null)
{
InitialNumberOfItemsToIndexInSingleBatch = Math.Min(int.Parse(initialNumberOfItemsToIndexInSingleBatch),
MaxNumberOfItemsToIndexInSingleBatch);
}
var maxNumberOfItemsToReduceInSingleBatch = Settings["Raven/MaxNumberOfItemsToReduceInSingleBatch"];
if (maxNumberOfItemsToReduceInSingleBatch != null)

MaxNumberOfItemsToReduceInSingleBatch = ravenSettings.MaxNumberOfItemsToReduceInSingleBatch.Value;
if (MaxNumberOfItemsToReduceInSingleBatch == ravenSettings.MaxNumberOfItemsToReduceInSingleBatch.Default)
{
InitialNumberOfItemsToReduceInSingleBatch = defaultInitialNumberOfItemsToIndexInSingleBatch / 2;
}
else
{
MaxNumberOfItemsToReduceInSingleBatch = Math.Max(int.Parse(maxNumberOfItemsToReduceInSingleBatch), 128);
InitialNumberOfItemsToReduceInSingleBatch = Math.Min(MaxNumberOfItemsToReduceInSingleBatch,
InitialNumberOfItemsToReduceInSingleBatch);
}
var numberOfItemsToExecuteReduceInSingleStep = Settings["Raven/NumberOfItemsToExecuteReduceInSingleStep"];
if (numberOfItemsToExecuteReduceInSingleStep != null)
{
NumberOfItemsToExecuteReduceInSingleStep = int.Parse(numberOfItemsToExecuteReduceInSingleStep);
}

NumberOfItemsToExecuteReduceInSingleStep = ravenSettings.NumberOfItemsToExecuteReduceInSingleStep.Value;

var initialNumberOfItemsToReduceInSingleBatch = Settings["Raven/InitialNumberOfItemsToReduceInSingleBatch"];
if (initialNumberOfItemsToReduceInSingleBatch != null)
Expand All @@ -143,81 +122,73 @@ public void Initialize()
MaxNumberOfItemsToReduceInSingleBatch);
}

var maxNumberOfParallelIndexTasks = Settings["Raven/MaxNumberOfParallelIndexTasks"];
MaxNumberOfParallelIndexTasks = maxNumberOfParallelIndexTasks != null ? int.Parse(maxNumberOfParallelIndexTasks) : Environment.ProcessorCount;
MaxNumberOfParallelIndexTasks = Math.Max(1, MaxNumberOfParallelIndexTasks);
MaxNumberOfParallelIndexTasks = ravenSettings.MaxNumberOfParallelIndexTasks.Value;

var minimumQueryCount = Settings["Raven/TempIndexPromotionMinimumQueryCount"];
TempIndexPromotionMinimumQueryCount = minimumQueryCount != null ? int.Parse(minimumQueryCount) : 100;
TempIndexPromotionMinimumQueryCount = ravenSettings.TempIndexPromotionMinimumQueryCount.Value;

var queryThreshold = Settings["Raven/TempIndexPromotionThreshold"];
TempIndexPromotionThreshold = queryThreshold != null ? int.Parse(queryThreshold) : 60000; // once a minute
TempIndexPromotionThreshold = ravenSettings.TempIndexPromotionThreshold.Value;

var cleanupPeriod = Settings["Raven/TempIndexCleanupPeriod"];
TempIndexCleanupPeriod = cleanupPeriod != null ? TimeSpan.FromSeconds(int.Parse(cleanupPeriod)) : TimeSpan.FromMinutes(10);
TempIndexCleanupPeriod = ravenSettings.TempIndexCleanupPeriod.Value;

var cleanupThreshold = Settings["Raven/TempIndexCleanupThreshold"];
TempIndexCleanupThreshold = cleanupThreshold != null ? TimeSpan.FromSeconds(int.Parse(cleanupThreshold)) : TimeSpan.FromMinutes(20);
TempIndexCleanupThreshold = ravenSettings.TempIndexCleanupThreshold.Value;

var tempMemoryMaxMb = Settings["Raven/TempIndexInMemoryMaxMB"];
TempIndexInMemoryMaxBytes = tempMemoryMaxMb != null ? int.Parse(tempMemoryMaxMb) * 1024 * 1024 : 26214400;
TempIndexInMemoryMaxBytes = Math.Max(1024 * 1024, TempIndexInMemoryMaxBytes);
TempIndexInMemoryMaxBytes = ravenSettings.TempIndexInMemoryMaxMb.Value;

// Data settings
RunInMemory = GetConfigurationValue<bool>("Raven/RunInMemory") ?? false;
RunInMemory = ravenSettings.RunInMemory.Value;

if (string.IsNullOrEmpty(DefaultStorageTypeName))
{
DefaultStorageTypeName = Settings["Raven/StorageTypeName"] ?? Settings["Raven/StorageEngine"] ?? "esent";
}

CreateTemporaryIndexesForAdHocQueriesIfNeeded =
GetConfigurationValue<bool>("Raven/CreateTemporaryIndexesForAdHocQueriesIfNeeded") ?? true;
CreateTemporaryIndexesForAdHocQueriesIfNeeded = ravenSettings.CreateTemporaryIndexesForAdHocQueriesIfNeeded.Value;

ResetIndexOnUncleanShutdown = GetConfigurationValue<bool>("Raven/ResetIndexOnUncleanShutdown") ?? false;
ResetIndexOnUncleanShutdown = ravenSettings.ResetIndexOnUncleanShutdown.Value;

SetupTransactionMode();

DataDirectory = Settings["Raven/DataDir"] ?? @"~\Data";
DataDirectory = ravenSettings.DataDir.Value;

if (string.IsNullOrEmpty(Settings["Raven/IndexStoragePath"]) == false)
var indexStoragePathSettingValue = ravenSettings.IndexStoragePath.Value;
if (string.IsNullOrEmpty(indexStoragePathSettingValue) == false)
{
IndexStoragePath = Settings["Raven/IndexStoragePath"];
IndexStoragePath = indexStoragePathSettingValue;
}

// HTTP settings
HostName = Settings["Raven/HostName"];
HostName = ravenSettings.HostName.Value;

if (string.IsNullOrEmpty(DatabaseName)) // we only use this for root database
Port = PortUtil.GetPort(Settings["Raven/Port"]);
Port = PortUtil.GetPort(ravenSettings.Port.Value);
SetVirtualDirectory();

bool httpCompressionTemp;
if (bool.TryParse(Settings["Raven/HttpCompression"], out httpCompressionTemp) == false)
httpCompressionTemp = true;
HttpCompression = httpCompressionTemp;
HttpCompression = ravenSettings.HttpCompression.Value;

AccessControlAllowOrigin = Settings["Raven/AccessControlAllowOrigin"];
AccessControlMaxAge = Settings["Raven/AccessControlMaxAge"] ?? "1728000"; // 20 days
AccessControlAllowMethods = Settings["Raven/AccessControlAllowMethods"] ?? "PUT,PATCH,GET,DELETE,POST";
AccessControlRequestHeaders = Settings["Raven/AccessControlRequestHeaders"];
AccessControlAllowOrigin = ravenSettings.AccessControlAllowOrigin.Value;
AccessControlMaxAge = ravenSettings.AccessControlMaxAge.Value;
AccessControlAllowMethods = ravenSettings.AccessControlAllowMethods.Value;
AccessControlRequestHeaders = ravenSettings.AccessControlRequestHeaders.Value;

AnonymousUserAccessMode = GetAnonymousUserAccessMode();

RedirectStudioUrl = Settings["Raven/RedirectStudioUrl"];
RedirectStudioUrl = ravenSettings.RedirectStudioUrl.Value;

DisableDocumentPreFetchingForIndexing = GetConfigurationValue<bool>("Raven/DisableDocumentPreFetchingForIndexing") ??
false;
DisableDocumentPreFetchingForIndexing = ravenSettings.DisableDocumentPreFetchingForIndexing.Value;

// Misc settings
WebDir = Settings["Raven/WebDir"] ?? GetDefaultWebDir();
WebDir = ravenSettings.WebDir.Value;

PluginsDirectory = (Settings["Raven/PluginsDirectory"] ?? @"~\Plugins").ToFullPath();
PluginsDirectory = ravenSettings.PluginsDirectory.Value.ToFullPath();

var taskSchedulerType = Settings["Raven/TaskScheduler"];
var taskSchedulerType = ravenSettings.TaskScheduler.Value;
if (taskSchedulerType != null)
{
var type = Type.GetType(taskSchedulerType);
CustomTaskScheduler = (TaskScheduler)Activator.CreateInstance(type);
}

AllowLocalAccessWithoutAuthorization = GetConfigurationValue<bool>("Raven/AllowLocalAccessWithoutAuthorization") ?? false;
AllowLocalAccessWithoutAuthorization = ravenSettings.AllowLocalAccessWithoutAuthorization.Value;

PostInit();
}
Expand Down Expand Up @@ -315,21 +286,6 @@ private byte[] GetOAuthKey()
return defaultOauthKey.Value; // ensure we only create this once per process
}


private int GetDefaultMemoryCacheLimitMegabytes()
{
// we need to leave ( a lot ) of room for other things as well, so we limit the cache size

var val = (MemoryStatistics.TotalPhysicalMemory / 2) -
// reduce the unmanaged cache size from the default limit
(GetConfigurationValue<int>("Raven/Esent/CacheSizeMax") ?? 1024);

if (val < 0)
return 128; // if machine has less than 1024 MB, then only use 128 MB

return val;
}

public NameValueCollection Settings { get; set; }

public string ServerUrl
Expand Down Expand Up @@ -763,11 +719,6 @@ protected AnonymousUserAccessMode GetAnonymousUserAccessMode()
return AnonymousUserAccessMode.Get;
}

private static string GetDefaultWebDir()
{
return Path.Combine(AppDomain.CurrentDomain.BaseDirectory, @"Raven/WebUI");
}

public string GetFullUrl(string baseUrl)
{
if (baseUrl.StartsWith("/"))
Expand Down
22 changes: 22 additions & 0 deletions Raven.Database/Config/Settings/BooleanSetting.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
// -----------------------------------------------------------------------
// <copyright file="BooleanSetting.cs" company="Hibernating Rhinos LTD">
// Copyright (c) Hibernating Rhinos LTD. All rights reserved.
// </copyright>
// -----------------------------------------------------------------------
namespace Raven.Database.Config.Settings
{
public class BooleanSetting : Setting<bool>
{
public BooleanSetting(string value, bool defaultValue) : base(value, defaultValue)
{
}

public override bool Value
{
get
{
return string.IsNullOrEmpty(value) == false ? bool.Parse(value) : defaultValue;
}
}
}
}
31 changes: 31 additions & 0 deletions Raven.Database/Config/Settings/IntegerSetting.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
// -----------------------------------------------------------------------
// <copyright file="IntegerSetting.cs" company="Hibernating Rhinos LTD">
// Copyright (c) Hibernating Rhinos LTD. All rights reserved.
// </copyright>
// -----------------------------------------------------------------------
using System;

namespace Raven.Database.Config.Settings
{
public class IntegerSetting : Setting<int>
{
public IntegerSetting(string value, int defaultValue)
: base(value, defaultValue)
{
}

public IntegerSetting(string value, Func<int> getDefaultValue) : base(value, getDefaultValue)
{
}

public override int Value
{
get
{
return string.IsNullOrEmpty(value) == false
? int.Parse(value)
: (getDefaultValue != null ? getDefaultValue() : defaultValue);
}
}
}
}
33 changes: 33 additions & 0 deletions Raven.Database/Config/Settings/IntegerSettingWithMin.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
// -----------------------------------------------------------------------
// <copyright file="IntegerSettingWithMin.cs" company="Hibernating Rhinos LTD">
// Copyright (c) Hibernating Rhinos LTD. All rights reserved.
// </copyright>
// -----------------------------------------------------------------------
using System;

namespace Raven.Database.Config.Settings
{
public class IntegerSettingWithMin : Setting<int>
{
private readonly int min;

public IntegerSettingWithMin(string value, int defaultValue, int min)
: base(value, defaultValue)
{
this.min = min;
}

public override int Value
{
get
{
return string.IsNullOrEmpty(value) == false ? Math.Max(int.Parse(value), min) : defaultValue;
}
}

public int Default
{
get { return defaultValue; }
}
}
}
Loading

0 comments on commit b7aaa4d

Please sign in to comment.