Permalink
Browse files

Merge branch 'master' of github.com:ayende/ravendb

  • Loading branch information...
2 parents f550e85 + c1d21c0 commit 08aa244b3599059348cf7db94859b35ede25c56b @fitzchak fitzchak committed Jan 7, 2013
Showing with 1,376 additions and 390 deletions.
  1. +0 −1 Bundles/Raven.Bundles.Tests/Expiration/WithCascade.cs
  2. +0 −1 Bundles/Raven.Bundles.Tests/IndexReplication/ReplicateToSql.cs
  3. +1 −1 Imports/Newtonsoft.Json/Src/Newtonsoft.Json/JsonReader.cs
  4. +1 −1 Raven.Abstractions/Connection/HttpRavenRequest.cs
  5. +1 −1 Raven.Abstractions/Data/ApiKeyDefinition.cs
  6. +1 −0 Raven.Abstractions/Default.cs
  7. +3 −1 Raven.Abstractions/Json/Linq/DictionaryWithParentSnapshot.cs
  8. +4 −0 Raven.Abstractions/Smuggler/SmugglerApiBase.cs
  9. +36 −0 Raven.Abstractions/Smuggler/SmugglerOptions.cs
  10. +0 −5 Raven.Client.Embedded/EmbeddedAsyncServerClient.cs
  11. +2 −20 Raven.Client.Lightweight/Connection/Async/AsyncServerClient.cs
  12. +0 −5 Raven.Client.Lightweight/Connection/Async/IAsyncDatabaseCommands.cs
  13. +9 −2 Raven.Client.Lightweight/Connection/HttpJsonRequest.cs
  14. +7 −1 Raven.Client.Lightweight/Connection/ObservableLineStream.cs
  15. +0 −5 Raven.Client.Lightweight/Connection/RavenUrlExtensions.cs
  16. +18 −0 Raven.Client.Lightweight/Connection/ServerClient.cs
  17. +26 −1 Raven.Client.Lightweight/Document/RemoteBulkInsertOperation.cs
  18. +8 −3 Raven.Client.Lightweight/Indexes/ExpressionStringBuilder.cs
  19. +10 −3 Raven.Client.Lightweight/Linq/LinqPathProvider.cs
  20. +5 −0 Raven.Client.Silverlight/Connection/HttpJsonRequest.cs
  21. +11 −3 Raven.Database/Bundles/Expiration/ExpirationReadTrigger.cs
  22. +1 −1 Raven.Database/Bundles/Expiration/ExpiredDocumentsCleaner.cs
  23. +1 −2 Raven.Database/Config/ConfigOptionDocs.cs
  24. +62 −3 Raven.Database/DocumentDatabase.cs
  25. +19 −2 Raven.Database/Extensions/HttpExtensions.cs
  26. +4 −4 Raven.Database/Indexing/Index.cs
  27. +5 −2 Raven.Database/Indexing/LuceneIntegration/TermsMatchQuery.cs
  28. +6 −74 Raven.Database/Indexing/MapReduceIndex.cs
  29. +1 −1 Raven.Database/Indexing/PrefetchingBehavior.cs
  30. +47 −15 Raven.Database/Indexing/QueryBuilder.cs
  31. +69 −0 Raven.Database/Indexing/RavenPerFieldAnalyzerWrapper.cs
  32. +8 −6 Raven.Database/Indexing/ReducingExecuter.cs
  33. +1 −1 Raven.Database/Indexing/SimpleQueryParser.cs
  34. +8 −3 Raven.Database/Linq/DynamicViewCompiler.cs
  35. +1 −1 Raven.Database/Queries/MoreLikeThisQueryRunner.cs
  36. +3 −0 Raven.Database/Raven.Database.csproj
  37. +15 −1 Raven.Database/Server/HttpServer.cs
  38. +75 −19 Raven.Database/Server/Responders/Admin/AdminRestore.cs
  39. +48 −18 Raven.Database/Server/Responders/BulkInsert.cs
  40. +45 −0 Raven.Database/Server/Responders/OperationStatus.cs
  41. +1 −1 Raven.Database/Server/Security/MixedModeRequestAuthorizer.cs
  42. +2 −1 Raven.Database/Server/Security/Windows/WindowsAuthConfigureHttpListener.cs
  43. +29 −12 Raven.Database/Storage/Esent/Backup/RestoreOperation.cs
  44. +60 −0 Raven.Database/Storage/Esent/StorageActions/EsentUtil.cs
  45. +20 −0 Raven.Database/Storage/Esent/StorageActions/General.cs
  46. +1 −7 Raven.Database/Storage/Esent/StorageActions/Indexing.cs
  47. +8 −17 Raven.Database/Storage/Esent/StorageActions/MappedResults.cs
  48. +5 −0 Raven.Database/Storage/Esent/StorageActions/OptimizedIndexReader.cs
  49. +16 −10 Raven.Database/Storage/Esent/TransactionalStorage.cs
  50. +9 −2 Raven.Database/Storage/Esent/TransactionalStorageConfigurator.cs
  51. +1 −1 Raven.Database/Storage/IGeneralStorageActions.cs
  52. +3 −3 Raven.Database/Storage/IMappedResultsStorageAction.cs
  53. +2 −2 Raven.Database/Storage/Managed/GeneralStorageActions.cs
  54. +5 −15 Raven.Database/Storage/Managed/MappedResultsStorageAction.cs
  55. +20 −24 Raven.Database/Storage/Managed/TransactionStorageActions.cs
  56. +29 −0 Raven.Database/Util/PerformanceCountersUtils.cs
  57. +2 −4 Raven.Server/Program.cs
  58. +1 −0 Raven.Smuggler/Program.cs
  59. +3 −1 Raven.Smuggler/SmugglerApi.cs
  60. +1 −0 Raven.Studio/Commands/CreateDatabaseCommand.cs
  61. +14 −0 Raven.Studio/Impl/AsyncDatabaseCommandExtensions.cs
  62. +1 −1 Raven.Studio/Infrastructure/Converters/GetApiKeyDirectLink.cs
  63. +4 −3 Raven.Studio/Models/ServerModel.cs
  64. +1 −0 Raven.Studio/Raven.Studio.csproj
  65. +3 −3 Raven.Tests.Helpers/RavenTestBase.cs
  66. +46 −0 Raven.Tests/Bugs/PrefetchingBug.cs
  67. +7 −4 Raven.Tests/Bundles/Expiration/Expiration.cs
  68. +1 −1 Raven.Tests/Indexes/MapReduceIndexOnLargeDataSet.cs
  69. +20 −20 Raven.Tests/Indexes/UsingQueryBuilder.cs
  70. +1 −1 Raven.Tests/Issues/RavenDB_626.cs
  71. +2 −2 Raven.Tests/Issues/RavenDB_766.cs
  72. +6 −0 Raven.Tests/MailingList/DtcIssue.cs
  73. +38 −38 Raven.Tests/MailingList/IisQueryLengthIssues.cs
  74. +15 −5 Raven.Tests/MailingList/LazyStats.cs
  75. +223 −0 Raven.Tests/MailingList/Oguzhntopcu.cs
  76. +61 −0 Raven.Tests/MailingList/SmugglerTests.cs
  77. +119 −0 Raven.Tests/MailingList/WhereInTests.cs
  78. +4 −0 Raven.Tests/Raven.Tests.csproj
  79. +16 −0 Raven.Tests/Views/MapReduce.cs
  80. +4 −3 Raven.Tryouts/Program.cs
  81. +10 −2 default.ps1
@@ -44,7 +44,6 @@ public WithCascade()
};
ravenConfiguration.PostInit();
ravenDbServer = new RavenDbServer(ravenConfiguration);
- database::Raven.Bundles.Expiration.ExpirationReadTrigger.GetCurrentUtcDate = () => DateTime.UtcNow;
documentStore = new DocumentStore
{
Url = "http://localhost:8079"
@@ -49,7 +49,6 @@ public CanReplicateToSql()
}
};
documentStore.Initialize();
- database::Raven.Bundles.Expiration.ExpirationReadTrigger.GetCurrentUtcDate = () => DateTime.UtcNow;
documentStore.DatabaseCommands.PutIndex(
"Questions/Votes",
@@ -254,7 +254,7 @@ protected JsonReader()
_currentState = State.Start;
_stack = new List<JsonPosition>(4);
_dateTimeZoneHandling = DateTimeZoneHandling.RoundtripKind;
- _dateParseHandling = DateParseHandling.DateTime;
+ _dateParseHandling = DateParseHandling.None;
CloseInput = true;
}
@@ -138,7 +138,7 @@ public T ExecuteRequest<T>()
return result;
}
- public void ExecuteRequest(Action<StreamReader> action)
+ public void ExecuteRequest(Action<TextReader> action)
{
SendRequestToServer(response =>
{
@@ -55,7 +55,7 @@ public string ConnectionString
if (string.IsNullOrWhiteSpace(Name) || string.IsNullOrWhiteSpace(Secret))
return null;
- return string.Format(@"ApiKey = {0}, Database = {1}", FullApiKey, DbName);
+ return string.Format(@"ApiKey = {0}; Database = {1}", FullApiKey, DbName);
}
}
@@ -7,6 +7,7 @@ public static class Default
{
public static readonly string[] OnlyDateTimeFormat = new[]
{
+ "yyyy'-'MM'-'dd'T'HH':'mm':'ss",
"yyyy'-'MM'-'dd'T'HH':'mm':'ss.fffffff",
"yyyy'-'MM'-'dd'T'HH':'mm':'ss.fffffff'Z'"
};
@@ -92,7 +92,9 @@ public ICollection<string> Keys
foreach (var key in LocalChanges.Keys)
{
- if (LocalChanges[key] == DeletedMarker)
+ RavenJToken value;
+ if (LocalChanges.TryGetValue(key, out value) == false ||
+ value == DeletedMarker)
continue;
ret.Add(key);
++counter;
@@ -184,6 +184,10 @@ private Guid ExportDocuments(SmugglerOptions options, JsonTextWriter jsonWriter,
ModifyBatchSize(options, currentProcessingTime);
var final = documents.Where(options.MatchFilters).ToList();
+
+ if (options.ShouldExcludeExpired)
+ final = documents.Where(options.ExcludeExpired).ToList();
+
final.ForEach(item => item.WriteTo(jsonWriter));
totalCount += final.Count;
@@ -21,6 +21,7 @@ public SmugglerOptions()
Timeout = 30 * 1000; // 30 seconds
BatchSize = 1024;
LastAttachmentEtag = LastDocsEtag = Guid.Empty;
+ ShouldExcludeExpired = false;
}
/// <summary>
@@ -79,8 +80,43 @@ public virtual bool MatchFilters(RavenJToken item)
}
return true;
}
+
+ /// <summary>
+ /// Should we exclude any documents which have already expired by checking the expiration meta property created by the expiration bundle
+ /// </summary>
+ public bool ShouldExcludeExpired { get; set; }
+
+ public virtual bool ExcludeExpired(RavenJToken item)
+ {
+ var metadata= item.Value<RavenJObject>("@metadata");
+
+ const string RavenExpirationDate = "Raven-Expiration-Date";
+
+ // check for expired documents and exclude them if expired
+ if (metadata == null)
+ {
+ return false;
+ }
+ var property = metadata[RavenExpirationDate];
+ if (property == null)
+ return false;
+
+ DateTime dateTime;
+ try
+ {
+ dateTime = property.Value<DateTime>();
+ }
+ catch (FormatException)
+ {
+ return false;
+ }
+
+ return dateTime >= SystemTime.UtcNow;
+ }
}
+
+
[Flags]
public enum ItemType
{
@@ -172,11 +172,6 @@ public Task<string[]> GetTermsAsync(string index, string field, string fromValue
return new CompletedTask<string[]>(databaseCommands.GetTerms(index, field, fromValue, pageSize).ToArray());
}
- public Task EnsureSilverlightStartUpAsync()
- {
- throw new NotSupportedException("Method to be removed from IAsyncDatabaseCommands (RavenDB-761)");
- }
-
public IDisposable DisableAllCaching()
{
return databaseCommands.DisableAllCaching();
@@ -700,7 +700,8 @@ public Task StartRestoreAsync(string restoreLocation, string databaseLocation, s
return request.ExecuteWriteAsync(new RavenJObject
{
{"RestoreLocation", restoreLocation},
- {"DatabaseLocation", databaseLocation}
+ {"DatabaseLocation", databaseLocation},
+ {"DatabaseName", name}
}.ToString(Formatting.None))
.ContinueWith(task =>
{
@@ -1201,25 +1202,6 @@ public IDisposable DisableAllCaching()
return jsonRequestFactory.DisableAllCaching();
}
- /// <summary>
- /// Ensures that the silverlight startup tasks have run
- /// </summary>
- public Task EnsureSilverlightStartUpAsync()
- {
-#if !SILVERLIGHT
- throw new NotSupportedException("Only applicable in silverlight");
-#else
- return ExecuteWithReplication("GET", url =>
- {
- return url
- .SilverlightEnsuresStartup()
- .NoCache()
- .ToJsonRequest(this, credentials, convention)
- .ReadResponseBytesAsync();
- });
-#endif
- }
-
///<summary>
/// Get the possible terms for the specified field in the index asynchronously
/// You can page through the results by use fromValue parameter as the
@@ -203,11 +203,6 @@ public interface IAsyncDatabaseCommands : IDisposable, IHoldProfilingInformation
Task<string[]> GetTermsAsync(string index, string field, string fromValue, int pageSize);
/// <summary>
- /// Ensures that the silverlight startup tasks have run
- /// </summary>
- Task EnsureSilverlightStartUpAsync();
-
- /// <summary>
/// Disable all caching within the given scope
/// </summary>
IDisposable DisableAllCaching();
@@ -546,6 +546,13 @@ public string ContentType
get { return webRequest.ContentType; }
set { webRequest.ContentType = value; }
}
+ public TimeSpan Timeout
+ {
+ set
+ {
+ webRequest.Timeout = (int)value.TotalMilliseconds;
+ }
+ }
private void WriteMetadata(RavenJObject metadata)
{
@@ -806,11 +813,11 @@ public Task<Stream> GetRawRequestStream()
return Task.Factory.FromAsync<Stream>(webRequest.BeginGetRequestStream, webRequest.EndGetRequestStream, null);
}
- public void RawExecuteRequest()
+ public WebResponse RawExecuteRequest()
{
try
{
- webRequest.GetResponse().Close();
+ return webRequest.GetResponse();
}
catch (WebException we)
{
@@ -6,6 +6,7 @@
using System;
using System.Collections.Generic;
using System.IO;
+using System.Net;
using System.Text;
using System.Threading.Tasks;
using Raven.Abstractions.Extensions;
@@ -103,8 +104,13 @@ public void Start()
// explicitly ignoring this
}
var aggregateException = task.Exception;
- if (aggregateException.ExtractSingleInnerException() is ObjectDisposedException)
+ var exception = aggregateException.ExtractSingleInnerException();
+ if (exception is ObjectDisposedException)
return; // this isn't an error
+ var we = exception as WebException;
+ if (we != null && we.Status == WebExceptionStatus.RequestCanceled)
+ return; // not an error, actually
+
foreach (var subscriber in subscribers)
{
subscriber.OnError(aggregateException);
@@ -43,11 +43,6 @@ public static string Databases(this string url, int pageSize, int start)
return start > 0 ? databases + "&start=" + start : databases;
}
- public static string SilverlightEnsuresStartup(this string url)
- {
- return url + "/silverlight/ensureStartup";
- }
-
public static string Terms(this string url, string index, string field, string fromValue, int pageSize)
{
return url + "/terms/" + index + "?field=" + field + "&fromValue=" + fromValue + "&pageSize=" + pageSize;
@@ -1710,6 +1710,24 @@ public void Dispose()
{
Dispose();
}
+
+ public RavenJToken GetOperationStatus(long id)
+ {
+ var request = jsonRequestFactory.CreateHttpJsonRequest(
+ new CreateHttpJsonRequestParams(this, url + "/operation/status?id" + id, "GET", credentials, convention)
+ .AddOperationHeaders(OperationsHeaders));
+ try
+ {
+ return request.ReadResponseJson();
+ }
+ catch (WebException e)
+ {
+ var httpWebResponse = e.Response as HttpWebResponse;
+ if (httpWebResponse == null || httpWebResponse.StatusCode != HttpStatusCode.NotFound)
+ throw;
+ return null;
+ }
+ }
}
}
#endif
@@ -3,10 +3,12 @@
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
+using System.Threading;
using System.Threading.Tasks;
using Raven.Abstractions.Data;
using Raven.Client.Connection;
using Raven.Client.Extensions;
+using Raven.Imports.Newtonsoft.Json;
using Raven.Imports.Newtonsoft.Json.Bson;
using Raven.Json.Linq;
@@ -24,6 +26,7 @@ public interface ILowLevelBulkInsertOperation : IDisposable
public class RemoteBulkInsertOperation : ILowLevelBulkInsertOperation
{
+ private readonly ServerClient client;
private readonly MemoryStream bufferedStream = new MemoryStream();
private readonly HttpJsonRequest httpJsonRequest;
private readonly BlockingCollection<RavenJObject> items;
@@ -32,6 +35,7 @@ public class RemoteBulkInsertOperation : ILowLevelBulkInsertOperation
public RemoteBulkInsertOperation(BulkInsertOptions options, ServerClient client)
{
+ this.client = client;
items = new BlockingCollection<RavenJObject>(options.BatchSize*8);
string requestUrl = "/bulkInsert?";
if (options.CheckForUpdates)
@@ -46,6 +50,8 @@ public RemoteBulkInsertOperation(BulkInsertOptions options, ServerClient client)
httpJsonRequest = client.CreateRequest("POST", requestUrl, disableRequestCompression: true);
+ // the request may take a long time to process, so we need to set a large timeout value
+ httpJsonRequest.Timeout = TimeSpan.FromHours(6);
nextTask = httpJsonRequest.GetRawRequestStream()
.ContinueWith(task =>
{
@@ -97,7 +103,26 @@ public void Dispose()
{
report("Finished writing all results to server");
}
- httpJsonRequest.RawExecuteRequest();
+ long id;
+
+ using (var response = httpJsonRequest.RawExecuteRequest())
+ using(var stream = response.GetResponseStream())
+ using(var streamReader = new StreamReader(stream))
+ {
+ var result = RavenJObject.Load(new JsonTextReader(streamReader));
+ id = result.Value<long>("OperationId");
+ }
+
+ while (true)
+ {
+ var status = client.GetOperationStatus(id);
+ if (status == null)
+ break;
+ if (status.Value<bool>("Completed"))
+ break;
+ Thread.Sleep(500);
+ }
+
if (report != null)
{
report("Done writing to server");
@@ -780,9 +780,14 @@ protected override Expression VisitConstant(ConstantExpression node)
Out(s);
return node;
}
- Out('"');
- Out(node.Value.ToString());
- Out('"');
+ if (convention.SaveEnumsAsIntegers)
+ Out((Convert.ToInt32(node.Value)).ToString());
+ else
+ {
+ Out('"');
+ Out(node.Value.ToString());
+ Out('"');
+ }
return node;
}
if (node.Value is decimal)
@@ -107,10 +107,17 @@ public object GetValueFromExpression(Expression expression, Type type)
object value;
if (GetValueFromExpressionWithoutConversion(expression, out value))
{
- if (type.IsEnum && (value is IEnumerable == false) && // skip arrays, lists
- conventions.SaveEnumsAsIntegers == false)
+ if (value is IEnumerable)
+ return value;
+
+ var nonNullableType = Nullable.GetUnderlyingType(type) ?? type;
+ if (value is Enum || nonNullableType.IsEnum)
{
- return Enum.GetName(type, value);
+ if (value == null)
+ return null;
+ if (conventions.SaveEnumsAsIntegers == false)
+ return Enum.GetName(nonNullableType, value);
+ return Convert.ToInt32(value);
}
return value;
}
@@ -73,6 +73,11 @@ private Task RecreateWebRequest(Action<HttpWebRequest> result)
private static Task noopWaitForTask = new CompletedTask();
+
+ public TimeSpan Timeout
+ {
+ set { } // can't set timeout in Silverlight
+ }
/// <summary>
/// Gets or sets the response headers.
/// </summary>
Oops, something went wrong.

0 comments on commit 08aa244

Please sign in to comment.