Skip to content

Commit

Permalink
Merge pull request #709 from quant1729/master-sharpcompress-fix
Browse files Browse the repository at this point in the history 
Removed reference to System.IO.Compression zip library
  • Loading branch information
@jaredbroad
jaredbroad committed Jan 21, 2017
2 parents 3e3cf7d + 4382821 commit a28ae77
Show file tree
Hide file tree
Showing 5 changed files with 28 additions and 39 deletions.
17 changes: 6 additions & 11 deletions Compression/Compression.cs
View file
Expand Up @@ -474,7 +474,7 @@ public static void ZipFiles(string destination, IEnumerable<string> files)
/// <param name="filename">Location of the original zip file</param>
/// <param name="zip">The ZipFile instance to be returned to the caller</param>
/// <returns>Stream reader of the first file contents in the zip file</returns>
public static StreamReader Unzip(string filename, out ZipArchive zip)
public static StreamReader Unzip(string filename, out ZipFile zip)
{
return Unzip(filename, null, out zip);
}
Expand All @@ -485,9 +485,9 @@ public static StreamReader Unzip(string filename, out ZipArchive zip)
/// </summary>
/// <param name="filename">Location of the original zip file</param>
/// <param name="zipEntryName">The zip entry name to open a reader for. Specify null to access the first entry</param>
/// <param name="zip">The ZipArchive instance to be returned to the caller</param>
/// <param name="zip">The ZipFile instance to be returned to the caller</param>
/// <returns>Stream reader of the first file contents in the zip file</returns>
public static StreamReader Unzip(string filename, string zipEntryName, out ZipArchive zip)
public static StreamReader Unzip(string filename, string zipEntryName, out ZipFile zip)
{
StreamReader reader = null;
zip = null;
Expand All @@ -498,20 +498,15 @@ public static StreamReader Unzip(string filename, string zipEntryName, out ZipAr
{
try
{
var file = File.OpenRead(filename);
// reading up first two bytes
// http://george.chiramattel.com/blog/2007/09/deflatestream-block-length-does-not-match.html
file.ReadByte(); file.ReadByte();

zip = new ZipArchive(file);
var entry = zip.Entries.FirstOrDefault(x => zipEntryName == null || string.Compare(x.FullName, zipEntryName, StringComparison.OrdinalIgnoreCase) == 0);
zip = new ZipFile(filename);
var entry = zip.FirstOrDefault(x => zipEntryName == null || string.Compare(x.FileName, zipEntryName, StringComparison.OrdinalIgnoreCase) == 0);
if (entry == null)
{
// Unable to locate zip entry
return null;
}

reader = new StreamReader(entry.Open());
reader = new StreamReader(entry.OpenReader());
}
catch (Exception err)
{
Expand Down
23 changes: 9 additions & 14 deletions Engine/DataFeeds/DataFileCacheProvider.cs
View file
Expand Up @@ -18,14 +18,14 @@
using System.IO;
using QuantConnect.Lean.Engine.DataFeeds.Transport;
using System.Collections.Concurrent;
using System.IO.Compression;
using QuantConnect.Logging;
using System.Linq;
using Ionic.Zip;

namespace QuantConnect.Lean.Engine.DataFeeds
{
// Cache entry tuple contains: date of the cache entry, and reference to .net zip archive
using CacheEntry = Tuple<DateTime, ZipArchive>;
using CacheEntry = Tuple<DateTime, ZipFile>;

/// <summary>
/// File provider implements optimized zip archives caching facility. Cache is thread safe.
Expand All @@ -35,7 +35,7 @@ public class DataFileCacheProvider : IDisposable
const int _cachePeriodBars = 10;

// ZipArchive cache used by the class
private ConcurrentDictionary<string, Lazy<CacheEntry>> _zipArchiveCache = new ConcurrentDictionary<string, Lazy<CacheEntry>>();
private ConcurrentDictionary<string, Lazy<CacheEntry>> _zipFileCache = new ConcurrentDictionary<string, Lazy<CacheEntry>>();
private DateTime lastDate = DateTime.MinValue;

/// <summary>
Expand Down Expand Up @@ -68,28 +68,23 @@ public IStreamReader Fetch(Symbol symbol, SubscriptionDataSource source, DateTim
if (lastDate == DateTime.MinValue || lastDate < date.Date)
{
// clean all items that that are older than _cachePeriodBars bars than the current date
foreach (var zip in _zipArchiveCache.Where(x => x.Value.Value.Item1 < date.Date.AddDays(-_cachePeriodBars)))
foreach (var zip in _zipFileCache.Where(x => x.Value.Value.Item1 < date.Date.AddDays(-_cachePeriodBars)))
{
// disposing zip archive
zip.Value.Value.Item2.Dispose();

// removing it from the cache
Lazy<CacheEntry> removed;
_zipArchiveCache.TryRemove(zip.Key, out removed);
_zipFileCache.TryRemove(zip.Key, out removed);
}

lastDate = date.Date;
}

_zipArchiveCache.AddOrUpdate(filename,
_zipFileCache.AddOrUpdate(filename,
x =>
{
var file = File.OpenRead(filename);
// reading up first two bytes
// http://george.chiramattel.com/blog/2007/09/deflatestream-block-length-does-not-match.html
file.ReadByte(); file.ReadByte();
var newItem = Tuple.Create(date.Date, new ZipArchive(file));
var newItem = Tuple.Create(date.Date, new ZipFile(filename));
reader = new LocalFileSubscriptionStreamReader(newItem.Item2, entryName);
return newItem;
},
Expand Down Expand Up @@ -121,12 +116,12 @@ public IStreamReader Fetch(Symbol symbol, SubscriptionDataSource source, DateTim
/// <filterpriority>2</filterpriority>
public void Dispose()
{
foreach (var zip in _zipArchiveCache)
foreach (var zip in _zipFileCache)
{
zip.Value.Value.Item2.Dispose();
}

_zipArchiveCache.Clear();
_zipFileCache.Clear();
}
}
}
18 changes: 9 additions & 9 deletions Engine/DataFeeds/Transport/LocalFileSubscriptionStreamReader.cs
View file
Expand Up @@ -30,7 +30,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Transport
public class LocalFileSubscriptionStreamReader : IStreamReader
{
private StreamReader _streamReader;
private readonly ZipArchive _zipArchive;
private readonly ZipFile _zipFile;

/// <summary>
/// Initializes a new instance of the <see cref="LocalFileSubscriptionStreamReader"/> class.
Expand All @@ -42,7 +42,7 @@ public LocalFileSubscriptionStreamReader(string source, string entryName = null)
{
// unzip if necessary
_streamReader = source.GetExtension() == ".zip"
? Compression.Unzip(source, entryName, out _zipArchive)
? Compression.Unzip(source, entryName, out _zipFile)
: new StreamReader(source);
}

Expand All @@ -57,7 +57,7 @@ public LocalFileSubscriptionStreamReader(string source, string entryName, long s
{
// unzip if necessary
_streamReader = source.GetExtension() == ".zip"
? Compression.Unzip(source, entryName, out _zipArchive)
? Compression.Unzip(source, entryName, out _zipFile)
: new StreamReader(source);

if (startingPosition != 0)
Expand All @@ -70,17 +70,17 @@ public LocalFileSubscriptionStreamReader(string source, string entryName, long s
/// <summary>
/// Initializes a new instance of the <see cref="LocalFileSubscriptionStreamReader"/> class.
/// </summary>
/// <param name="zipArchive">The local zip archive to be read</param>
/// <param name="zipFile">The local zip archive to be read</param>
/// <param name="entryName">Specifies the zip entry to be opened. Leave null if not applicable,
/// or to open the first zip entry found regardless of name</param>
public LocalFileSubscriptionStreamReader(ZipArchive zipArchive, string entryName = null)
public LocalFileSubscriptionStreamReader(ZipFile zipFile, string entryName = null)
{
_zipArchive = zipArchive;
var entry = _zipArchive.Entries.FirstOrDefault(x => entryName == null || string.Compare(x.FullName, entryName, StringComparison.OrdinalIgnoreCase) == 0);
_zipFile = zipFile;
var entry = _zipFile.Entries.FirstOrDefault(x => entryName == null || string.Compare(x.FileName, entryName, StringComparison.OrdinalIgnoreCase) == 0);
if (entry != null)
{
var stream = new MemoryStream();
entry.Open().CopyTo(stream);
entry.OpenReader().CopyTo(stream);
stream.Position = 0;
_streamReader = new StreamReader(stream);
}
Expand All @@ -93,7 +93,7 @@ public IEnumerable<string> EntryFileNames
{
get
{
return _zipArchive != null ? _zipArchive.Entries.Select(x => x.FullName).ToList() : Enumerable.Empty<string>();
return _zipFile != null ? _zipFile.Entries.Select(x => x.FileName).ToList() : Enumerable.Empty<string>();
}
}

Expand Down
6 changes: 3 additions & 3 deletions Tests/Compression/CompressionTests.cs
View file
Expand Up @@ -55,9 +55,9 @@ public void ZipBytes()
public void ExtractsZipEntryByName()
{
var zip = Path.Combine("TestData", "multizip.zip");
ZipArchive zipArchive;
using (var entryStream = QuantConnect.Compression.Unzip(zip, "multizip/two.txt", out zipArchive))
using (zipArchive)
ZipFile zipFile;
using (var entryStream = QuantConnect.Compression.Unzip(zip, "multizip/two.txt", out zipFile))
using (zipFile)
{
var text = entryStream.ReadToEnd();
Assert.AreEqual("2", text);
Expand Down
3 changes: 1 addition & 2 deletions ToolBox/CoarseUniverseGenerator/Program.cs
View file
Expand Up @@ -24,7 +24,6 @@
using QuantConnect.Data.Auxiliary;
using QuantConnect.Util;
using Log = QuantConnect.Logging.Log;
using System.IO.Compression;

namespace QuantConnect.ToolBox.CoarseUniverseGenerator
{
Expand Down Expand Up @@ -203,7 +202,7 @@ public static ICollection<string> ProcessDailyFolder(string dailyFolder, string
}
}

ZipArchive zip;
ZipFile zip;
using (var reader = Compression.Unzip(file, out zip))
{
// 30 period EMA constant
Expand Down

0 comments on commit a28ae77

Please sign in to comment.