Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

CORE: Blob support #1460

Closed
wants to merge 23 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
6fcb140
feat(core): scaffolding blob uploads (WIP)
didimitrie Jul 27, 2022
4773533
feat(core): blobs scaffolding
didimitrie Jul 27, 2022
3e98224
feat(core): blobs wip receiving
didimitrie Jul 28, 2022
b9505b5
feat(core): blobs receiving first pass ✅
didimitrie Jul 28, 2022
928b16d
feat(core): blob local cache check pre-download
didimitrie Jul 29, 2022
f6729ef
feat(core): blobs wip
didimitrie Jul 30, 2022
98d9b93
feat(core): more blobs wip (pre send diffing)
didimitrie Jul 31, 2022
09991bc
feat(blobs): adds noise to obj and removes cache bypass hack
didimitrie Jul 31, 2022
da58802
Merge branch 'main' into dim/blobs
didimitrie Oct 10, 2022
54ab5bd
Merge branch 'main' of github.com:specklesystems/speckle-sharp into d…
gjedlicska Nov 1, 2022
555b5df
move example app to net6
gjedlicska Nov 9, 2022
83a6d44
Merge branch 'main' of github.com:specklesystems/speckle-sharp into d…
gjedlicska Nov 9, 2022
56cbb75
remove Dim folder dependency from blob tests
gjedlicska Nov 10, 2022
653d324
Merge branch 'release/2.10' of github.com:specklesystems/speckle-shar…
gjedlicska Nov 10, 2022
0569bfc
fix integration test setup
gjedlicska Nov 11, 2022
81db4db
reformat with default dotnet formatter
gjedlicska Nov 15, 2022
cd445bd
refactor path helpers into SpecklePathProvider
gjedlicska Nov 15, 2022
fe1d7de
Obsolete old path helper methods
gjedlicska Nov 15, 2022
da1e20d
core internals visible for unittesting
gjedlicska Nov 15, 2022
02f3a3e
sqlite transport is now an IBlobCapableTransport
gjedlicska Nov 15, 2022
2a018c1
add original path to Blobs, to be able to copy data into cache on send
gjedlicska Nov 15, 2022
17bfb72
add tests for blob send and receive
gjedlicska Nov 15, 2022
4a884e4
fix failing test cases
gjedlicska Nov 15, 2022
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Expand Up @@ -10,6 +10,7 @@
*.userosscache
*.sln.docstates
*.tsep
*.tool-versions

# Intellij Crap
.idea
Expand Down
Expand Up @@ -38,7 +38,6 @@
<ProjectReference Include="..\..\Core\Transports\DiskTransport\DiskTransport.csproj" />
<ProjectReference Include="..\ConnectorGrasshopperUtils\ConnectorGrasshopperUtils.csproj" />
</ItemGroup>

<ItemGroup>
<None Update="SchemaBuilder\SchemaBuilderGen.tt">
<Generator>TextTemplatingFileGenerator</Generator>
Expand Down
25 changes: 13 additions & 12 deletions ConnectorRhino/ConnectorRhino/ConnectorRhinoShared/Entry/Plugin.cs
Expand Up @@ -2,18 +2,19 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text;
using Avalonia;
using Avalonia.ReactiveUI;
using ConnectorRhinoShared;
using DesktopUI2;
using DesktopUI2.ViewModels;
using Rhino;
using Rhino.PlugIns;
using Rhino.Runtime;
using Speckle.Core.Api;
using Speckle.Core.Models.Extensions;

[assembly: Guid("8dd5f30b-a13d-4a24-abdc-3e05c8c87143")]

namespace SpeckleRhino
{
public class SpeckleRhinoConnectorPlugin : PlugIn
Expand Down Expand Up @@ -63,17 +64,17 @@ public void Init()

public static void InitAvaloniaMac()
{
var rhinoMenuPtr = MacOSHelpers.MainMenu;
var rhinoDelegate = MacOSHelpers.AppDelegate;
var titlePtr = MacOSHelpers.MenuItemGetTitle(MacOSHelpers.MenuItemGetSubmenu(MacOSHelpers.MenuItemAt(rhinoMenuPtr, 0)));
var rhinoMenuPtr = MacOSHelpers.MainMenu;
var rhinoDelegate = MacOSHelpers.AppDelegate;
var titlePtr = MacOSHelpers.MenuItemGetTitle(MacOSHelpers.MenuItemGetSubmenu(MacOSHelpers.MenuItemAt(rhinoMenuPtr, 0)));

appBuilder = BuildAvaloniaApp().SetupWithoutStarting();
appBuilder = BuildAvaloniaApp().SetupWithoutStarting();

// don't use Avalonia's AppDelegate.. not sure what consequences this might have to Avalonia functionality
MacOSHelpers.AppDelegate = rhinoDelegate;
MacOSHelpers.MainMenu = rhinoMenuPtr;
MacOSHelpers.MenuItemSetTitle(MacOSHelpers.MenuItemGetSubmenu(MacOSHelpers.MenuItemAt(rhinoMenuPtr, 0)), MacOSHelpers.NewObject("NSString"));
MacOSHelpers.MenuItemSetTitle(MacOSHelpers.MenuItemGetSubmenu(MacOSHelpers.MenuItemAt(rhinoMenuPtr, 0)), titlePtr);
// don't use Avalonia's AppDelegate.. not sure what consequences this might have to Avalonia functionality
MacOSHelpers.AppDelegate = rhinoDelegate;
MacOSHelpers.MainMenu = rhinoMenuPtr;
MacOSHelpers.MenuItemSetTitle(MacOSHelpers.MenuItemGetSubmenu(MacOSHelpers.MenuItemAt(rhinoMenuPtr, 0)), MacOSHelpers.NewObject("NSString"));
MacOSHelpers.MenuItemSetTitle(MacOSHelpers.MenuItemGetSubmenu(MacOSHelpers.MenuItemAt(rhinoMenuPtr, 0)), titlePtr);

}

Expand Down Expand Up @@ -155,7 +156,7 @@ protected override LoadReturnCode OnLoad(ref string errorMessage)
Init();

#if !MAC
System.Type panelType = typeof(Panel);
System.Type panelType = typeof(Panel);
Rhino.UI.Panels.RegisterPanel(this, panelType, "Speckle", Resources.icon);
#endif
// Get the version number of our plugin, that was last used, from our settings file.
Expand Down
7 changes: 2 additions & 5 deletions ConnectorRhino/ConnectorRhino6/ConnectorRhino6.csproj
Expand Up @@ -6,14 +6,15 @@
<Title>SpeckleRhino</Title>
<Description>Description of SpeckleRhino</Description>
<TargetExt>.rhp</TargetExt>
<Product>ConnectorRhino6</Product>
<RhinoMacLauncher>/Applications/Rhinoceros.app</RhinoMacLauncher>
<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
<AppendTargetFrameworkToOutputPath>false</AppendTargetFrameworkToOutputPath>
<AssemblyName>SpeckleConnectorRhino</AssemblyName>
<Configurations>Debug;Release;Debug Mac;Release Mac</Configurations>
<RootNamespace>SpeckleRhino</RootNamespace>
<PlatformTarget>x64</PlatformTarget>
<RuntimeIdentifier>win-x64</RuntimeIdentifier>
<Version>2.0.0-dev</Version>
</PropertyGroup>

<ItemGroup>
Expand All @@ -24,10 +25,6 @@
<PackageReference Include="Avalonia" Version="0.10.18" />
<PackageReference Include="Avalonia.Desktop" Version="0.10.18" />
<PackageReference Include="Avalonia.ReactiveUI" Version="0.10.18" />
<PackageReference Include="MSBuild.AssemblyVersion" Version="1.3.0">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
<PackageReference Include="Newtonsoft.Json" Version="13.0.1">
</PackageReference>
<PackageReference Include="RhinoCommon" Version="6.28.20199.17141">
Expand Down
37 changes: 0 additions & 37 deletions ConnectorRhino/ConnectorRhino6/Properties/AssemblyInfo.cs

This file was deleted.

7 changes: 2 additions & 5 deletions ConnectorRhino/ConnectorRhino7/ConnectorRhino7.csproj
Expand Up @@ -6,12 +6,13 @@
<Title>SpeckleRhino</Title>
<Description>Description of SpeckleRhino</Description>
<TargetExt>.rhp</TargetExt>
<Product>ConnectorRhino7</Product>
<RhinoMacLauncher>/Applications/Rhino 7.app</RhinoMacLauncher>
<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
<AppendTargetFrameworkToOutputPath>false</AppendTargetFrameworkToOutputPath>
<AssemblyName>SpeckleConnectorRhino</AssemblyName>
<Configurations>Debug;Release;Debug Mac;Release Mac</Configurations>
<RootNamespace>SpeckleRhino</RootNamespace>
<Version>2.0.0-dev</Version>
<!--NOTE-->
<!--
Since Rhino uses mono we build for win-x64 also on mac
Expand All @@ -31,10 +32,6 @@
<PackageReference Include="Avalonia" Version="0.10.18" />
<PackageReference Include="Avalonia.Desktop" Version="0.10.18" />
<PackageReference Include="Avalonia.ReactiveUI" Version="0.10.18" />
<PackageReference Include="MSBuild.AssemblyVersion" Version="1.3.0">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
<PackageReference Include="Newtonsoft.Json" Version="13.0.1">
</PackageReference>
<PackageReference Include="RhinoCommon" Version="7.4.21078.1001">
Expand Down
36 changes: 0 additions & 36 deletions ConnectorRhino/ConnectorRhino7/Properties/AssemblyInfo.cs

This file was deleted.

6 changes: 1 addition & 5 deletions Core/Core/Api/GraphQL/Client.Subscriptions.Branch.cs
@@ -1,11 +1,7 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using GraphQL;
using Sentry;
using Speckle.Core.Logging;
using Speckle.Core.Api.SubscriptionModels;
using Speckle.Core.Logging;

namespace Speckle.Core.Api
{
Expand Down
6 changes: 1 addition & 5 deletions Core/Core/Api/GraphQL/Client.Subscriptions.Commit.cs
@@ -1,11 +1,7 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using GraphQL;
using Sentry;
using Speckle.Core.Logging;
using Speckle.Core.Api.SubscriptionModels;
using Speckle.Core.Logging;

namespace Speckle.Core.Api
{
Expand Down
4 changes: 4 additions & 0 deletions Core/Core/Api/Helpers.cs
Expand Up @@ -243,19 +243,22 @@ public static string PluralS(int num)
/// Returns the correct location of the Speckle installation folder. Usually this would be the user's %appdata%/Speckle folder, unless the install was made for all users.
/// </summary>
/// <returns>The location of the Speckle installation folder</returns>
[Obsolete("Please use Helpers/SpecklePathProvider.InstallSpeckleFolderPath", true)]
public static string InstallSpeckleFolderPath => Path.Combine(InstallApplicationDataPath, "Speckle");

/// <summary>
/// Returns the correct location of the Speckle folder for the current user. Usually this would be the user's %appdata%/Speckle folder.
/// </summary>
/// <returns>The location of the Speckle installation folder</returns>
[Obsolete("Please use Helpers/SpecklePathProvider.UserSpeckleFolderPath", true)]
public static string UserSpeckleFolderPath => Path.Combine(UserApplicationDataPath, "Speckle");


/// <summary>
/// Returns the correct location of the AppData folder where Speckle is installed. Usually this would be the user's %appdata% folder, unless the install was made for all users.
/// </summary>
/// <returns>The location of the AppData folder where Speckle is installed</returns>
[Obsolete("Please use Helpers/SpecklePathProvider.InstallApplicationDataPath ", true)]
public static string InstallApplicationDataPath =>

Assembly.GetAssembly(typeof(Helpers)).Location.Contains("ProgramData")
Expand All @@ -267,6 +270,7 @@ public static string PluralS(int num)
/// Returns the correct location for `Environment.SpecialFolder.ApplicationData` for the current roaming user.
/// </summary>
/// <returns>The location of the user's `%appdata%` folder.</returns>
[Obsolete("Please use Helpers/SpecklePathProvider.UserApplicationDataPath", true)]
public static string UserApplicationDataPath
// We combine our own path to the %appdata% folder due to solve issues with network account management in windows,
// where the normal `SpecialFolder.ApplicationData` would point to the `Default` user instead of the active one.
Expand Down
12 changes: 8 additions & 4 deletions Core/Core/Api/Operations/Operations.Receive.cs
Expand Up @@ -92,6 +92,10 @@ public static async Task<Base> Receive(string objectId, CancellationToken cancel
serializerV2.OnProgressAction = internalProgressAction;
serializerV2.OnErrorAction = onErrorAction;
serializerV2.CancellationToken = cancellationToken;
if (remoteTransport is IBlobCapableTransport t)
{
serializerV2.BlobStorageFolder = t.BlobStorageFolder;
}
}

// First we try and get the object from the local transport. If it's there, we assume all its children are there, and proceed with deserialisation.
Expand All @@ -114,9 +118,9 @@ public static async Task<Base> Receive(string objectId, CancellationToken cancel
{
localRes = serializerV2.Deserialize(objString);
}
catch ( Exception e )
catch (Exception e)
{
if ( serializerV2.OnErrorAction == null ) throw;
if (serializerV2.OnErrorAction == null) throw;
serializerV2.OnErrorAction.Invoke($"A deserialization error has occurred: {e.Message}", new SpeckleException(
$"A deserialization error has occurred: {e.Message}", e));
localRes = null;
Expand Down Expand Up @@ -156,9 +160,9 @@ public static async Task<Base> Receive(string objectId, CancellationToken cancel
{
res = serializerV2.Deserialize(objString);
}
catch ( Exception e )
catch (Exception e)
{
if ( serializerV2.OnErrorAction == null ) throw;
if (serializerV2.OnErrorAction == null) throw;
serializerV2.OnErrorAction.Invoke($"A deserialization error has occurred: {e.Message}", e);
res = null;
}
Expand Down
7 changes: 3 additions & 4 deletions Core/Core/Api/Operations/Operations.Send.cs
Expand Up @@ -5,7 +5,6 @@
using System.Threading;
using System.Threading.Tasks;
using Sentry;
using Sentry;
using Speckle.Core.Logging;
using Speckle.Core.Models;
using Speckle.Core.Serialisation;
Expand Down Expand Up @@ -63,7 +62,7 @@ public static async Task<string> Send(Base @object, CancellationToken cancellati

if (transports.Count == 0 && useDefaultCache == false)
{
throw new SpeckleException($"You need to provide at least one transport: cannot send with an empty transport list and no default cache.", level : SentryLevel.Error);
throw new SpeckleException($"You need to provide at least one transport: cannot send with an empty transport list and no default cache.", level: SentryLevel.Error);
}

if (useDefaultCache)
Expand Down Expand Up @@ -121,7 +120,7 @@ public static async Task<string> Send(Base @object, CancellationToken cancellati
transportAwaits = serializerV2.WriteTransports.Select(t => t.WriteComplete()).ToList();
}

if (cancellationToken.IsCancellationRequested)return null;
if (cancellationToken.IsCancellationRequested) return null;

await Task.WhenAll(transportAwaits).ConfigureAwait(false);

Expand All @@ -132,7 +131,7 @@ public static async Task<string> Send(Base @object, CancellationToken cancellati
if (disposeTransports && t is IDisposable disp) disp.Dispose();
}

if (cancellationToken.IsCancellationRequested)return null;
if (cancellationToken.IsCancellationRequested) return null;

var hash = JObject.Parse(obj).GetValue("id").ToString();
return hash;
Expand Down
4 changes: 2 additions & 2 deletions Core/Core/Api/Operations/Operations.Serialize.cs
@@ -1,9 +1,9 @@
using Speckle.Newtonsoft.Json;
using System;
using System;
using System.Collections.Generic;
using System.Threading;
using Speckle.Core.Models;
using Speckle.Core.Serialisation;
using Speckle.Newtonsoft.Json;

namespace Speckle.Core.Api
{
Expand Down
6 changes: 3 additions & 3 deletions Core/Core/Api/Operations/Operations.cs
@@ -1,9 +1,9 @@
using Speckle.Newtonsoft.Json;
using Speckle.Newtonsoft.Json.Serialization;
using System;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using Speckle.Core.Serialisation;
using Speckle.Newtonsoft.Json;
using Speckle.Newtonsoft.Json.Serialization;

namespace Speckle.Core.Api
{
Expand Down
4 changes: 4 additions & 0 deletions Core/Core/Core.csproj
Expand Up @@ -26,6 +26,10 @@
<GenerateDocumentationFile>true</GenerateDocumentationFile>
</PropertyGroup>

<ItemGroup>
<InternalsVisibleTo Include="TestsUnit"/>
</ItemGroup>

<!--Do not publish a Core nuget when publishing Objects-->
<PropertyGroup Condition="'$(SolutionName)'=='Objects' Or '$(SolutionName)'=='DesktopUI'">
<IsPackable>false</IsPackable>
Expand Down
6 changes: 3 additions & 3 deletions Core/Core/Credentials/AccountManager.cs
Expand Up @@ -6,14 +6,14 @@
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Reflection;
using System.Security.Cryptography;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using GraphQL;
using GraphQL.Client.Http;
using Speckle.Core.Api;
using Speckle.Core.Api.GraphQL.Serializer;
using Speckle.Core.Helpers;
using Speckle.Core.Logging;
using Speckle.Core.Transports;
using Speckle.Newtonsoft.Json;
Expand Down Expand Up @@ -131,7 +131,7 @@ public static string GetDefaultServerUrl()
{
var defaultServerUrl = "https://speckle.xyz";

var customServerFile = Path.Combine(Helpers.UserSpeckleFolderPath, "server");
var customServerFile = Path.Combine(SpecklePathProvider.UserSpeckleFolderPath, "server");
if (File.Exists(customServerFile))
{
var customUrl = File.ReadAllText(customServerFile);
Expand Down Expand Up @@ -206,7 +206,7 @@ public static IEnumerable<Account> GetAccounts()
private static IEnumerable<Account> GetLocalAccounts()
{
var accounts = new List<Account>();
var accountsDir = Path.Combine(Helpers.UserSpeckleFolderPath, "Accounts");
var accountsDir = SpecklePathProvider.AccountsFolderPath;
if (!Directory.Exists(accountsDir))
{
return accounts;
Expand Down