From 97db1ef4f9096ff69a06fe6e22959e9d99f4f4b0 Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Fri, 21 Nov 2025 15:52:30 +0100 Subject: [PATCH 1/4] Skip tests using nuget from signed builds (network isolation need for signed builds) --- .../DependencyManagerInteractiveTests.fs | 10 ++--- .../FSharpScriptTests.fs | 22 +++++----- .../ProjectAnalysisTests.fs | 2 +- tests/FSharp.Test.Utilities/Utilities.fs | 24 +++++++++++ tests/fsharp/tests.fs | 42 +++++++++---------- 5 files changed, 62 insertions(+), 38 deletions(-) diff --git a/tests/FSharp.Compiler.Private.Scripting.UnitTests/DependencyManagerInteractiveTests.fs b/tests/FSharp.Compiler.Private.Scripting.UnitTests/DependencyManagerInteractiveTests.fs index a2d102b40a6..bee85322a2c 100644 --- a/tests/FSharp.Compiler.Private.Scripting.UnitTests/DependencyManagerInteractiveTests.fs +++ b/tests/FSharp.Compiler.Private.Scripting.UnitTests/DependencyManagerInteractiveTests.fs @@ -39,7 +39,7 @@ type DependencyManagerInteractiveTests() = let getErrors ((_value: Result), (errors: FSharpDiagnostic[])) = errors - [] + [] member _.``SmokeTest - #r nuget``() = let text = """ #r @"nuget:Newtonsoft.Json, Version=9.0.1" @@ -50,7 +50,7 @@ type DependencyManagerInteractiveTests() = Assert.Equal(typeof, value.ReflectionType) Assert.Equal(0, value.ReflectionValue :?> int) - [] + [] member _.``SmokeTest - #r nuget package not found``() = let text = """ #r @"nuget:System.Collections.Immutable.DoesNotExist, version=1.5.0" @@ -267,7 +267,7 @@ TorchSharp.Tensor.LongTensor.From([| 0L .. 100L |]).Device () - [] + [] member _.``Use Dependency Manager to restore packages with native dependencies, build and run script that depends on the results``() = // Skip test on arm64, because there is not an arm64 native library if RuntimeInformation.ProcessArchitecture = Architecture.Arm64 then @@ -366,7 +366,7 @@ printfn ""%A"" result let value = opt.Value Assert.Equal(123, value.ReflectionValue :?> int32) - [] + [] member _.``Use NativeResolver to resolve native dlls.``() = // Skip test on arm64, because there is not an arm64 native library if RuntimeInformation.ProcessArchitecture = Architecture.Arm64 then @@ -450,7 +450,7 @@ printfn ""%A"" result let value = opt.Value Assert.Equal(123, value.ReflectionValue :?> int32) - [] + [] member _.``Use AssemblyResolver to resolve assemblies``() = // Skip test on arm64, because there is not an arm64 native library if RuntimeInformation.ProcessArchitecture = Architecture.Arm64 then diff --git a/tests/FSharp.Compiler.Private.Scripting.UnitTests/FSharpScriptTests.fs b/tests/FSharp.Compiler.Private.Scripting.UnitTests/FSharpScriptTests.fs index e53dc45f877..34faed6114d 100644 --- a/tests/FSharp.Compiler.Private.Scripting.UnitTests/FSharpScriptTests.fs +++ b/tests/FSharp.Compiler.Private.Scripting.UnitTests/FSharpScriptTests.fs @@ -168,7 +168,7 @@ stacktype.Name = "Stack" | Error(ex) -> Assert.IsAssignableFrom(typeof, ex) - [] + [] member _.``Script using System.Configuration succeeds``() = use script = new FSharpScript() let result, errors = script.Eval(""" @@ -246,7 +246,7 @@ System.Configuration.ConfigurationManager.AppSettings.Item "Environment" <- "LOC /// Native dll resolution is not implemented on desktop #if NETSTANDARD - [] + [] member _.``ML - use assembly with native dependencies``() = // Skip test on arm64, because there is not an arm64 native library if RuntimeInformation.ProcessArchitecture = Architecture.Arm64 then @@ -304,7 +304,7 @@ printfn "{@"%A"}" result | Ok(_) -> Assert.False(true, "expected a failure") | Error(ex) -> Assert.IsAssignableFrom(typeof, ex) - [] + [] member _.``Eval script with invalid PackageName should fail immediately``() = use capture = new TestConsole.ExecutionCapture() use script = new FSharpScript(additionalArgs=[| |]) @@ -316,7 +316,7 @@ printfn "{@"%A"}" result Assert.True( errors |> Seq.exists (fun error -> error.Message.Contains("error NU1101:")), "Expect to error containing 'error NU1101:'") Assert.True( errors |> Seq.exists (fun error -> error.Message.Contains("FSharp.Really.Not.A.Package")), "Expect to error containing 'FSharp.Really.Not.A.Package'") - [] + [] member _.``Eval script with invalid PackageName should fail immediately and resolve one time only``() = use capture = new TestConsole.ExecutionCapture() use script = new FSharpScript(additionalArgs=[| |]) @@ -330,7 +330,7 @@ printfn "{@"%A"}" result Assert.Equal(1, (errors |> Seq.filter (fun error -> error.Message.Contains("FSharp.Really.Not.A.Package")) |> Seq.length)) Assert.Equal(1, (errors |> Seq.filter (fun error -> error.Message.Contains("FSharp.Really.Not.Another.Package")) |> Seq.length)) - [] + [] member _.``FsharpPlus - report errors``() = let code = """ #i "nuget:https://api.nuget.org/v3/index.json" @@ -366,7 +366,7 @@ printTable [{|Age = 15; Weight = 88; Name = "Blahboolahboogaloo"|}] let value = opt.Value Assert.Equal(1, downcast value.ReflectionValue) - [] + [] member _.``ML - use assembly with ref dependencies``() = let code = """ #r "nuget:Microsoft.ML.OnnxTransformer,1.4.0" @@ -383,7 +383,7 @@ tInput.Length let value = opt.Value Assert.Equal(4L, downcast value.ReflectionValue) - [] // usessdkrefs is not a valid option for desktop compiler + [] // usessdkrefs is not a valid option for desktop compiler member _.``ML - use assembly with ref dependencies and without refing SMemory``() = let code = """ #r "nuget:Microsoft.ML.OnnxTransformer,1.4.0" @@ -399,7 +399,7 @@ tInput.Length let value = opt.Value Assert.Equal(4L, downcast value.ReflectionValue) - [] + [] member _.``System.Device.Gpio - Ensure we reference the runtime version of the assembly``() = let code = """ #r "nuget:System.Device.Gpio, 1.0.0" @@ -512,7 +512,7 @@ let x = Assert.False(foundInner) - [] + [] member _.``Script with nuget package that yields out of order dependencies works correctly``() = // regression test for: https://github.com/dotnet/fsharp/issues/9217 let code = """ @@ -535,7 +535,7 @@ test pfloat "1.234" let value = opt.Value Assert.True(true = downcast value.ReflectionValue) - [] + [] member _.``Nuget package with method duplicates differing only in generic arity``() = // regression test for: https://github.com/dotnet/fsharp/issues/17796 // Was an internal error @@ -549,7 +549,7 @@ let add (col:IServiceCollection) = let _value,diag = script.Eval(code) Assert.Empty(diag) - [] + [] [] [] [] diff --git a/tests/FSharp.Compiler.Service.Tests/ProjectAnalysisTests.fs b/tests/FSharp.Compiler.Service.Tests/ProjectAnalysisTests.fs index 2ecdc4fefb7..50258738045 100644 --- a/tests/FSharp.Compiler.Service.Tests/ProjectAnalysisTests.fs +++ b/tests/FSharp.Compiler.Service.Tests/ProjectAnalysisTests.fs @@ -5807,7 +5807,7 @@ let checkContentAsScript content = [] module ScriptClosureCacheUse = - [] + [] let ``References from #r nuget are included in script project options`` () = let checkResults = checkContentAsScript """ #i "nuget:https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json" diff --git a/tests/FSharp.Test.Utilities/Utilities.fs b/tests/FSharp.Test.Utilities/Utilities.fs index 92b8c1dffa9..f6adfe3b9fb 100644 --- a/tests/FSharp.Test.Utilities/Utilities.fs +++ b/tests/FSharp.Test.Utilities/Utilities.fs @@ -39,6 +39,30 @@ type FactForDESKTOPAttribute() = do base.Skip <- "NETCOREAPP is not supported runtime for this kind of test, it is intended for DESKTOP only" #endif +module SignedBuildSkip = + let isSignedBuild = System.Environment.GetEnvironmentVariable("SIGNTYPE") = "Real" + let skipMessage = "Test skipped on signed builds due to NuGet package restore restrictions" + + let skipIfSigned (attr: #FactAttribute) = + if isSignedBuild then + attr.Skip <- skipMessage + +type FactSkipOnSignedBuildAttribute() = + inherit FactAttribute() + do SignedBuildSkip.skipIfSigned this + +type TheorySkipOnSignedBuildAttribute() = + inherit TheoryAttribute() + do SignedBuildSkip.skipIfSigned this + +type FactForNETCOREAPPSkipOnSignedBuildAttribute() = + inherit FactAttribute() + #if !NETCOREAPP + do base.Skip <- "Only NETCOREAPP is supported runtime for this kind of test." + #else + do SignedBuildSkip.skipIfSigned this + #endif + // This file mimics how Roslyn handles their compilation references for compilation testing module Utilities = diff --git a/tests/fsharp/tests.fs b/tests/fsharp/tests.fs index 764481fc469..b187ba0728c 100644 --- a/tests/fsharp/tests.fs +++ b/tests/fsharp/tests.fs @@ -627,46 +627,46 @@ module CoreTests = | "" -> () | diffs -> failwithf "'%s' and '%s' differ; %A" diffFileErr expectedFileErr diffs - [] + [] let ``printing`` () = runPrintingTest "--multiemit- --debug+" "output" // F# 5.0 changed some things printing output - [] + [] let ``printing-langversion47`` () = runPrintingTest "--langversion:4.7" "output.47" // Output should not change with optimization off - [] + [] let ``printing-optimizeoff`` () = runPrintingTest "--multiemit- --debug+ --optimize-" "output" // Legacy one-dynamic-assembly emit is the default for .NET Framework, which these tests are using // Turning that off enables multi-assembly-emit. The printing test is useful for testing multi-assembly-emit // as it feeds in many incremental fragments into stdin of the FSI process. - [] + [] let ``printing-multiemit`` () = runPrintingTest "--multiemit+ --debug+" "output.multiemit" // Multi-assembly-emit establishes some slightly different rules regarding internals, and this // needs to be tested with optimizations off. The output should not change. - [] + [] let ``printing-multiemit-optimizeoff`` () = runPrintingTest "--multiemit+ --debug+ --optimize-" "output.multiemit" - [] + [] let ``printing-width-1000`` () = runPrintingTest "--use:preludePrintSize1000.fsx" "output.1000" - [] + [] let ``printing-width-200`` () = runPrintingTest "--use:preludePrintSize200.fsx" "output.200" - [] + [] let ``printing-off`` () = runPrintingTest "--use:preludeShowDeclarationValuesFalse.fsx" "output.off" - [] + [] let ``printing-quiet`` () = runPrintingTest "--quiet" "output.quiet" @@ -1645,7 +1645,7 @@ module RegressionTests = [] let ``12383-FSC_OPTIMIZED`` () = singleTestBuildAndRun "regression/12383" FSC_OPTIMIZED - [] + [] let ``13219-bug-FSI`` () = singleTestBuildAndRun "regression/13219" FSI [] @@ -1653,69 +1653,69 @@ module RegressionTests = let cfg = testConfig "regression/4715" fsc cfg "%s -o:test.exe --optimize+" cfg.fsc_flags ["date.fs"; "env.fs"; "main.fs"] - [] + [] let ``multi-package-type-provider-test-FSI`` () = singleTestBuildAndRun "regression/13710" FSI #if NETCOREAPP - [] + [] let ``Large inputs 12322 fsc.dll 64-bit fsc.dll .NET SDK generating optimized code`` () = let cfg = testConfig "regression/12322" let cfg = { cfg with fsc_flags = cfg.fsc_flags + " --debug:portable --define:PORTABLE_PDB" } singleTestBuildAndRunAux cfg (FSC_BUILDONLY true) - [] + [] let ``Large inputs 12322 fsc.dll 64-bit .NET SDK generating debug code`` () = let cfg = testConfig "regression/12322" let cfg = { cfg with fsc_flags = cfg.fsc_flags + " --debug:portable --define:PORTABLE_PDB" } singleTestBuildAndRunAux cfg (FSC_BUILDONLY false) #else - [] + [] let ``Large inputs 12322 fsc.exe 32-bit .NET Framework generating optimized code, portable PDB`` () = let cfg = testConfig "regression/12322" let cfg = { cfg with fsc_flags = cfg.fsc_flags + " --debug:portable --define:PORTABLE_PDB" } singleTestBuildAndRunAux cfg (FSC_BUILDONLY true) - [] + [] let ``Large inputs 12322 fsc.exe 32-bit .NET Framework generating optimized code, full PDB`` () = let cfg = testConfig "regression/12322" let cfg = { cfg with fsc_flags = cfg.fsc_flags + " --debug:full" } singleTestBuildAndRunAux cfg (FSC_BUILDONLY true) - [] + [] let ``Large inputs 12322 fsc.exe 32-bit .NET Framework generating debug code portable PDB`` () = let cfg = testConfig "regression/12322" let cfg = { cfg with fsc_flags = cfg.fsc_flags + " --debug:portable --define:PORTABLE_PDB" } singleTestBuildAndRunAux cfg (FSC_BUILDONLY false) - [] + [] let ``Large inputs 12322 fsc.exe 32-bit .NET Framework generating debug code, full PDB`` () = let cfg = testConfig "regression/12322" let cfg = { cfg with fsc_flags = cfg.fsc_flags + " --debug:full" } singleTestBuildAndRunAux cfg (FSC_BUILDONLY false) - [] + [] let ``Large inputs 12322 fscAnyCpu.exe 64-bit .NET Framework generating optimized code, portable PDB`` () = let cfg = testConfig "regression/12322" let cfg = { cfg with FSC = cfg.FSCANYCPU } let cfg = { cfg with fsc_flags = cfg.fsc_flags + " --debug:portable --define:PORTABLE_PDB" } singleTestBuildAndRunAux cfg (FSC_BUILDONLY true) - [] + [] let ``Large inputs 12322 fscAnyCpu.exe 64-bit .NET Framework generating optimized code, full PDB`` () = let cfg = testConfig "regression/12322" let cfg = { cfg with FSC = cfg.FSCANYCPU } let cfg = { cfg with fsc_flags = cfg.fsc_flags + " --debug:full " } singleTestBuildAndRunAux cfg (FSC_BUILDONLY true) - [] + [] let ``12322 fscAnyCpu.exe 64-bit .NET Framework generating debug code, portable PDB`` () = let cfg = testConfig "regression/12322" let cfg = { cfg with FSC = cfg.FSCANYCPU } let cfg = { cfg with fsc_flags = cfg.fsc_flags + " --debug:portable --define:PORTABLE_PDB" } singleTestBuildAndRunAux cfg (FSC_BUILDONLY false) - [] + [] let ``12322 fscAnyCpu.exe 64-bit .NET Framework generating debug code, full PDB`` () = let cfg = testConfig "regression/12322" let cfg = { cfg with FSC = cfg.FSCANYCPU } From b8312e6aee3dd7c4c13b294d6b660052d8cd8249 Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Mon, 24 Nov 2025 12:55:40 +0100 Subject: [PATCH 2/4] Apply suggestions from code review --- tests/FSharp.Test.Utilities/Utilities.fs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/FSharp.Test.Utilities/Utilities.fs b/tests/FSharp.Test.Utilities/Utilities.fs index f6adfe3b9fb..149ee182ab4 100644 --- a/tests/FSharp.Test.Utilities/Utilities.fs +++ b/tests/FSharp.Test.Utilities/Utilities.fs @@ -47,15 +47,15 @@ module SignedBuildSkip = if isSignedBuild then attr.Skip <- skipMessage -type FactSkipOnSignedBuildAttribute() = +type FactSkipOnSignedBuildAttribute() as this = inherit FactAttribute() do SignedBuildSkip.skipIfSigned this -type TheorySkipOnSignedBuildAttribute() = +type TheorySkipOnSignedBuildAttribute() as this = inherit TheoryAttribute() do SignedBuildSkip.skipIfSigned this -type FactForNETCOREAPPSkipOnSignedBuildAttribute() = +type FactForNETCOREAPPSkipOnSignedBuildAttribute() as this = inherit FactAttribute() #if !NETCOREAPP do base.Skip <- "Only NETCOREAPP is supported runtime for this kind of test." From 052808a1eb99e9bfc32e21464602f16fb4589f7b Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Mon, 24 Nov 2025 18:18:42 +0100 Subject: [PATCH 3/4] Apply suggestion from @T-Gro --- tests/FSharp.Test.Utilities/Utilities.fs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/FSharp.Test.Utilities/Utilities.fs b/tests/FSharp.Test.Utilities/Utilities.fs index 149ee182ab4..f3b1b7c892e 100644 --- a/tests/FSharp.Test.Utilities/Utilities.fs +++ b/tests/FSharp.Test.Utilities/Utilities.fs @@ -57,10 +57,9 @@ type TheorySkipOnSignedBuildAttribute() as this = type FactForNETCOREAPPSkipOnSignedBuildAttribute() as this = inherit FactAttribute() + do SignedBuildSkip.skipIfSigned this #if !NETCOREAPP do base.Skip <- "Only NETCOREAPP is supported runtime for this kind of test." - #else - do SignedBuildSkip.skipIfSigned this #endif // This file mimics how Roslyn handles their compilation references for compilation testing From 7740e22f40bcb433960db0e8da64d62953e2133d Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Tue, 25 Nov 2025 13:45:05 +0100 Subject: [PATCH 4/4] Apply suggestion from @T-Gro --- tests/FSharp.Test.Utilities/Utilities.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/FSharp.Test.Utilities/Utilities.fs b/tests/FSharp.Test.Utilities/Utilities.fs index f3b1b7c892e..3a8a7ae487e 100644 --- a/tests/FSharp.Test.Utilities/Utilities.fs +++ b/tests/FSharp.Test.Utilities/Utilities.fs @@ -59,7 +59,7 @@ type FactForNETCOREAPPSkipOnSignedBuildAttribute() as this = inherit FactAttribute() do SignedBuildSkip.skipIfSigned this #if !NETCOREAPP - do base.Skip <- "Only NETCOREAPP is supported runtime for this kind of test." + do base.Skip <- "Only NETCOREAPP is supported runtime for this kind of test." #endif // This file mimics how Roslyn handles their compilation references for compilation testing