From 39ee252b078de9f20da6fc438ce3388745a726b0 Mon Sep 17 00:00:00 2001 From: Alex Earl Date: Fri, 29 Jun 2018 18:03:01 -0700 Subject: [PATCH 1/4] Small updates --- CurrentVersion.props | 4 ++-- Package/choco/IronPython.nuspec | 1 + Package/choco/tools/VERIFICATION.txt | 3 +++ Package/msi/IronPython.Installer.wixproj | 4 ++++ 4 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 Package/choco/tools/VERIFICATION.txt diff --git a/CurrentVersion.props b/CurrentVersion.props index 878d8b73d..193ad3813 100644 --- a/CurrentVersion.props +++ b/CurrentVersion.props @@ -3,9 +3,9 @@ 2 7 - 8 + 9 - final + candidate 1 0 diff --git a/Package/choco/IronPython.nuspec b/Package/choco/IronPython.nuspec index 2b087b2a6..bfe278b28 100644 --- a/Package/choco/IronPython.nuspec +++ b/Package/choco/IronPython.nuspec @@ -23,5 +23,6 @@ + diff --git a/Package/choco/tools/VERIFICATION.txt b/Package/choco/tools/VERIFICATION.txt new file mode 100644 index 000000000..9e3c5e78b --- /dev/null +++ b/Package/choco/tools/VERIFICATION.txt @@ -0,0 +1,3 @@ +VERIFICATION Verification is intended to assist the Chocolatey moderators and community in verifying that this package's contents are trustworthy. + +This package is published by the IronPython Project itself. The binaries are identical to other package types published by the project, in particular the IronPython nuget package. diff --git a/Package/msi/IronPython.Installer.wixproj b/Package/msi/IronPython.Installer.wixproj index 112576e68..a0b452186 100644 --- a/Package/msi/IronPython.Installer.wixproj +++ b/Package/msi/IronPython.Installer.wixproj @@ -79,6 +79,10 @@ $(WixExtDir)\WixNetFxExtension.dll WixNetFxExtension + + $(WixExtDir)\WixUtilExtension.dll + WixUtilExtension + From 2219b070c1e199096cc57a50e48e4b8fe1c97f03 Mon Sep 17 00:00:00 2001 From: Alex Earl Date: Sat, 14 Jul 2018 13:05:02 -0700 Subject: [PATCH 2/4] Fixes #145, fixes #133 Adds additional command line parameters and ability to add DLL's when in standalone mode. --- Src/IronPython/Runtime/ClrModule.cs | 17 ++--- Src/IronPythonCompiler/Config.cs | 104 ++++++++++++++++++++++------ Src/IronPythonCompiler/Program.cs | 61 +++++++++++----- Src/IronPythonCompiler/README.md | 17 +++-- 4 files changed, 145 insertions(+), 54 deletions(-) diff --git a/Src/IronPython/Runtime/ClrModule.cs b/Src/IronPython/Runtime/ClrModule.cs index 145742730..2651ae6d6 100644 --- a/Src/IronPython/Runtime/ClrModule.cs +++ b/Src/IronPython/Runtime/ClrModule.cs @@ -848,7 +848,7 @@ public static void CompileModules(CodeContext/*!*/ context, string/*!*/ assembly List code = new List(); foreach (string filename in filenames) { if (!pc.DomainManager.Platform.FileExists(filename)) { - throw PythonOps.IOError("Couldn't find file for compilation: {0}", filename); + throw PythonOps.IOError($"Couldn't find file for compilation: {filename}"); } ScriptCode sc; @@ -868,11 +868,10 @@ public static void CompileModules(CodeContext/*!*/ context, string/*!*/ assembly } else { modName = Path.GetFileNameWithoutExtension(filename); } - + // see if we have a parent package, if so incorporate it into // our name - string parentPackage; - if (packageMap.TryGetValue(dname, out parentPackage)) { + if (packageMap.TryGetValue(dname, out string parentPackage)) { modName = parentPackage + "." + modName; } @@ -893,20 +892,18 @@ public static void CompileModules(CodeContext/*!*/ context, string/*!*/ assembly code.Add((SavableScriptCode)sc); } - object mainModule; - if (kwArgs != null && kwArgs.TryGetValue("mainModule", out mainModule)) { - string strModule = mainModule as string; - if (strModule != null) { + if (kwArgs != null && kwArgs.TryGetValue("mainModule", out object mainModule)) { + if (mainModule is string strModule) { if (!pc.DomainManager.Platform.FileExists(strModule)) { throw PythonOps.IOError("Couldn't find main file for compilation: {0}", strModule); } - + SourceUnit su = pc.CreateFileUnit(strModule, pc.DefaultEncoding, SourceCodeKind.File); code.Add((SavableScriptCode)context.LanguageContext.GetScriptCode(su, "__main__", ModuleOptions.Initialize, Compiler.CompilationMode.ToDisk)); } } - SavableScriptCode.SaveToAssembly(assemblyName, code.ToArray()); + SavableScriptCode.SaveToAssembly(assemblyName, kwArgs, code.ToArray()); } #endif diff --git a/Src/IronPythonCompiler/Config.cs b/Src/IronPythonCompiler/Config.cs index 966a8bc07..72770348b 100644 --- a/Src/IronPythonCompiler/Config.cs +++ b/Src/IronPythonCompiler/Config.cs @@ -17,16 +17,21 @@ public Config() { Embed = false; Files = new List(); Platform = IKVM.Reflection.PortableExecutableKinds.ILOnly; - Machine = IKVM.Reflection.ImageFileMachine.I386; + Machine = IKVM.Reflection.ImageFileMachine.AMD64; Standalone = false; Target = PEFileKinds.Dll; UseMta = false; MainName = Main = string.Empty; Output = string.Empty; + OutputPath = string.Empty; Win32Icon = string.Empty; - Version = string.Empty; + FileVersion = string.Empty; + ProductName = string.Empty; + Copyright = string.Empty; + ProductVersion = string.Empty; ErrorMessageFormat = "Error occurred: {0}"; PythonOptions = new Dictionary(); + DLLs = new List(); } public string ErrorMessageFormat { @@ -34,7 +39,7 @@ public string ErrorMessageFormat { private set; } - public string Version { + public string FileVersion { get; private set; } @@ -44,11 +49,31 @@ public string Win32Icon { private set; } + public string ProductName { + get; + private set; + } + + public string Copyright { + get; + private set; + } + + public string ProductVersion { + get; + private set; + } + public string Output { get; private set; } + public string OutputPath { + get; + private set; + } + public string Main { get; private set; @@ -79,6 +104,11 @@ public List Files { private set; } + public List DLLs { + get; + private set; + } + public IDictionary PythonOptions { get; private set; @@ -142,13 +172,19 @@ public void ParseArgs(IEnumerable args, List respFiles = null) { break; default: Platform = IKVM.Reflection.PortableExecutableKinds.ILOnly; - Machine = IKVM.Reflection.ImageFileMachine.I386; + Machine = IKVM.Reflection.ImageFileMachine.AMD64; break; } } else if (arg.StartsWith("/win32icon:")) { Win32Icon = arg.Substring(11).Trim('"'); - } else if (arg.StartsWith("/version:")) { - Version = arg.Substring(9).Trim('"'); + } else if (arg.StartsWith("/fileversion:")) { + FileVersion = arg.Substring(13).Trim('"'); + } else if (arg.StartsWith("/productversion:")) { + ProductVersion = arg.Substring(16).Trim('"'); + } else if(arg.StartsWith("/productname:")) { + ProductName = arg.Substring(13).Trim('"'); + } else if(arg.StartsWith("/copyright:")) { + Copyright = arg.Substring(11).Trim('"'); } else if (arg.StartsWith("/errfmt:")) { ErrorMessageFormat = arg.Substring(8); } else if (arg.StartsWith("/embed")) { @@ -157,6 +193,14 @@ public void ParseArgs(IEnumerable args, List respFiles = null) { Standalone = true; } else if (arg.StartsWith("/mta")) { UseMta = true; + } else if(arg.StartsWith("/recurse:")) { + string pattern = arg.Substring(9); + if(string.IsNullOrWhiteSpace(pattern)) { + ConsoleOps.Error(true, "Missing pattern for /recurse option"); + } + foreach(var f in Directory.EnumerateFiles(Environment.CurrentDirectory, pattern)) { + Files.Add(Path.GetFullPath(f)); + } } else if (Array.IndexOf(helpStrings, arg) >= 0) { ConsoleOps.Usage(true); } else if(arg.StartsWith("/py:")) { @@ -177,7 +221,7 @@ public void ParseArgs(IEnumerable args, List respFiles = null) { case "-X:GCStress": int gcStress; if (!int.TryParse(pyargs[1], out gcStress) || (gcStress < 0 || gcStress > GC.MaxGeneration)) { - ConsoleOps.Error(true, "The argument for the {0} option must be between 0 and {1}.", pyargs[1], GC.MaxGeneration); + ConsoleOps.Error(true, $"The argument for the {pyargs[1]} option must be between 0 and {GC.MaxGeneration}."); } PythonOptions["GCStress"] = gcStress; @@ -187,7 +231,7 @@ public void ParseArgs(IEnumerable args, List respFiles = null) { // we need about 6 frames for starting up, so 10 is a nice round number. int limit; if (!int.TryParse(pyargs[1], out limit) || limit < 10) { - ConsoleOps.Error(true, "The argument for the {0} option must be an integer >= 10.", pyargs[1]); + ConsoleOps.Error(true, $"The argument for the {pyargs[1]} option must be an integer >= 10."); } PythonOptions["RecursionLimit"] = limit; @@ -216,20 +260,28 @@ public void ParseArgs(IEnumerable args, List respFiles = null) { respFiles.Add(respFile); ParseArgs(File.ReadAllLines(respFile), respFiles); } else { - ConsoleOps.Warning("Already parsed response file '{0}'", arg.Substring(1)); + ConsoleOps.Warning($"Already parsed response file '{arg.Substring(1)}'"); } } else { - Files.Add(arg); + if (arg.ToLower().EndsWith(".dll")) { + DLLs.Add(arg); + } else { + Files.Add(arg); + } } } } } public bool Validate() { - if (Files.Count == 1 && string.IsNullOrEmpty(MainName)) { + if (Files.Count == 1 && string.IsNullOrWhiteSpace(MainName)) { MainName = Files[0]; } + if(Files.Count == 0 && !string.IsNullOrWhiteSpace(MainName)) { + Files.Add(MainName); + } + if (Files == null || Files.Count == 0 || string.IsNullOrEmpty(MainName)) { ConsoleOps.Error("No files or main defined"); return false; @@ -240,17 +292,24 @@ public bool Validate() { return false; } - if (string.IsNullOrEmpty(Output) && !string.IsNullOrEmpty(MainName)) { + if(DLLs.Count > 0 && !Standalone) { + ConsoleOps.Error("DLLs can only be used in standalone mode"); + return false; + } + + if (string.IsNullOrWhiteSpace(Output) && !string.IsNullOrWhiteSpace(MainName)) { Output = Path.GetFileNameWithoutExtension(MainName); - } else if (string.IsNullOrEmpty(Output) && Files != null && Files.Count > 0) { + OutputPath = Path.GetDirectoryName(MainName); + } else if (string.IsNullOrWhiteSpace(Output) && Files != null && Files.Count > 0) { Output = Path.GetFileNameWithoutExtension(Files[0]); + OutputPath = Path.GetDirectoryName(Files[0]); } - if (!string.IsNullOrEmpty(Win32Icon) && Target == PEFileKinds.Dll) { + if (!string.IsNullOrWhiteSpace(Win32Icon) && Target == PEFileKinds.Dll) { ConsoleOps.Error("DLLs may not have a win32icon"); return false; - } else if (!string.IsNullOrEmpty(Win32Icon) && !File.Exists(Win32Icon)) { - ConsoleOps.Error("win32icon '{0}' does not exist", Win32Icon); + } else if (!string.IsNullOrWhiteSpace(Win32Icon) && !File.Exists(Win32Icon)) { + ConsoleOps.Error($"win32icon '{Win32Icon}' does not exist"); return false; } @@ -260,12 +319,13 @@ public bool Validate() { public override string ToString() { StringBuilder res = new StringBuilder("Input Files:\n"); foreach (var file in Files) { - res.AppendFormat("\t{0}\n", file); + res.AppendLine($"\t{file}"); } - res.AppendFormat("Output:\n\t{0}\n", Output); - res.AppendFormat("Target:\n\t{0}\n", Target); - res.AppendFormat("Platform:\n\t{0}\n", Machine); + res.AppendLine($"Output:\n\t{Output}"); + res.AppendLine($"OutputPath:\n\t{OutputPath}"); + res.AppendLine($"Target:\n\t{Target}"); + res.AppendLine($"Platform:\n\t{Machine}"); if (Target == PEFileKinds.WindowApplication) { res.AppendLine("Threading:"); if (UseMta) { @@ -276,9 +336,9 @@ public override string ToString() { } if (PythonOptions.Count > 0) { - res.AppendFormat("\nIronPython Context Options:\n"); + res.AppendLine("\nIronPython Context Options:"); foreach(var option in PythonOptions) { - res.AppendFormat("\t{0} = {1}\n", option.Key, option.Value); + res.AppendLine($"\t{option.Key} = {option.Value}"); } } diff --git a/Src/IronPythonCompiler/Program.cs b/Src/IronPythonCompiler/Program.cs index 02db8b047..21c9339b2 100644 --- a/Src/IronPythonCompiler/Program.cs +++ b/Src/IronPythonCompiler/Program.cs @@ -22,19 +22,18 @@ using Microsoft.Scripting.Hosting; using Microsoft.Scripting.Hosting.Providers; - namespace IronPythonCompiler { - class Program { + public class Program { /// /// Generates the stub .exe file for starting the app /// /// - static void GenerateExe(Config config) { + private static void GenerateExe(Config config) { var u = new Universe(); var aName = new AssemblyName(Path.GetFileNameWithoutExtension(new FileInfo(config.Output).Name)); - var ab = u.DefineDynamicAssembly(aName, AssemblyBuilderAccess.Save, Path.GetDirectoryName(config.Output)); + var ab = u.DefineDynamicAssembly(aName, AssemblyBuilderAccess.Save, config.OutputPath); var mb = ab.DefineDynamicModule(config.Output, aName.Name + (aName.Name.EndsWith(".exe") ? string.Empty : ".exe")); var tb = mb.DefineType("PythonMain", IKVM.Reflection.TypeAttributes.Public); @@ -42,6 +41,22 @@ static void GenerateExe(Config config) { ab.__DefineIconResource(File.ReadAllBytes(config.Win32Icon)); } + var attributes = new List> { + Tuple.Create(config.FileVersion, u.Import(typeof(System.Reflection.AssemblyFileVersionAttribute))), + Tuple.Create(config.ProductName, u.Import(typeof(System.Reflection.AssemblyProductAttribute))), + Tuple.Create(config.ProductVersion, u.Import(typeof(System.Reflection.AssemblyInformationalVersionAttribute))), + Tuple.Create(config.Copyright, u.Import(typeof(System.Reflection.AssemblyCopyrightAttribute))) + }; + + foreach(var attr in attributes) { + if (!string.IsNullOrWhiteSpace(config.FileVersion)) { + CustomAttributeBuilder builder = new CustomAttributeBuilder(attr.Item2.GetConstructor(new[] { u.Import(typeof(string)) }), new object[] { attr.Item1 }); + ab.SetCustomAttribute(builder); + } + } + + ab.DefineVersionInfoResource(); + MethodBuilder assemblyResolveMethod = null; ILGenerator gen = null; @@ -52,12 +67,19 @@ static void GenerateExe(Config config) { foreach (var a in System.AppDomain.CurrentDomain.GetAssemblies()) { var n = new AssemblyName(a.FullName); if (!a.IsDynamic && a.EntryPoint == null && (n.Name.StartsWith("IronPython") || n.Name == "Microsoft.Dynamic" || n.Name == "Microsoft.Scripting")) { - ConsoleOps.Info("\tEmbedded {0} {1}", n.Name, n.Version); + ConsoleOps.Info($"\tEmbedded {n.Name} {n.Version}"); var f = new FileStream(a.Location, FileMode.Open, FileAccess.Read); mb.DefineManifestResource("Dll." + n.Name, f, IKVM.Reflection.ResourceAttributes.Public); } } + foreach(var dll in config.DLLs) { + var name = Path.GetFileNameWithoutExtension(dll); + ConsoleOps.Info($"\tEmbedded {name}"); + var f = new FileStream(dll, FileMode.Open, FileAccess.Read); + mb.DefineManifestResource("Dll." + name, f, IKVM.Reflection.ResourceAttributes.Public); + } + // we currently do no error checking on what is passed in to the assemblyresolve event handler assemblyResolveMethod = tb.DefineMethod("AssemblyResolve", MethodAttributes.Public | MethodAttributes.Static, u.Import(typeof(System.Reflection.Assembly)), new IKVM.Reflection.Type[] { u.Import(typeof(System.Object)), u.Import(typeof(System.ResolveEventArgs)) }); gen = assemblyResolveMethod.GetILGenerator(); @@ -128,8 +150,7 @@ static void GenerateExe(Config config) { foreach (var option in config.PythonOptions) { gen.Emit(OpCodes.Ldloc, dictVar); gen.Emit(OpCodes.Ldstr, option.Key); - if (option.Value is int) { - int val = (int)option.Value; + if (option.Value is int val) { if (val >= -128 && val <= 127) gen.Emit(OpCodes.Ldc_I4_S, val); // this is more optimized else @@ -137,7 +158,7 @@ static void GenerateExe(Config config) { gen.Emit(OpCodes.Box, u.Import(typeof(System.Int32))); } else if (option.Value.Equals(ScriptingRuntimeHelpers.True)) { gen.Emit(OpCodes.Ldsfld, True); - } else if(option.Value.Equals(ScriptingRuntimeHelpers.False)) { + } else if (option.Value.Equals(ScriptingRuntimeHelpers.False)) { gen.Emit(OpCodes.Ldsfld, False); } gen.EmitCall(OpCodes.Callvirt, u.Import(typeof(Dictionary)).GetMethod("Add", new IKVM.Reflection.Type[] { u.Import(typeof(string)), u.Import(typeof(object)) }), Type.EmptyTypes); @@ -151,11 +172,11 @@ static void GenerateExe(Config config) { // put the generated DLL into the resources for the stub exe var mem = new MemoryStream(); var rw = new ResourceWriter(mem); - rw.AddResource("IPDll." + Path.GetFileNameWithoutExtension(config.Output) + ".dll", File.ReadAllBytes(config.Output + ".dll")); + rw.AddResource("IPDll." + Path.GetFileNameWithoutExtension(config.Output) + ".dll", File.ReadAllBytes(Path.Combine(config.OutputPath, config.Output) + ".dll")); rw.Generate(); mem.Position = 0; mb.DefineManifestResource("IPDll.resources", mem, ResourceAttributes.Public); - File.Delete(config.Output + ".dll"); + File.Delete(Path.Combine(config.OutputPath, config.Output) + ".dll"); // generate code to load the resource gen.Emit(OpCodes.Ldstr, "IPDll"); @@ -240,7 +261,7 @@ static void GenerateExe(Config config) { ab.Save(fileName, config.Platform, config.Machine); } - static int Main(string[] args) { + public static int Main(string[] args) { var files = new List(); var config = new Config(); config.ParseArgs(args); @@ -251,27 +272,31 @@ static int Main(string[] args) { // we don't use the engine, but we create it so we can have a default context. ScriptEngine engine = Python.CreateEngine(config.PythonOptions); - ConsoleOps.Info("IronPython Compiler for {0} ({1})", engine.Setup.DisplayName, engine.LanguageVersion); - ConsoleOps.Info("{0}", config); + ConsoleOps.Info($"IronPython Compiler for {engine.Setup.DisplayName} ({engine.LanguageVersion})"); + ConsoleOps.Info($"{config}"); ConsoleOps.Info("compiling..."); var compileOptions = new Dictionary() { - { "mainModule", config.MainName } + { "mainModule", config.MainName }, + { "assemblyFileVersion", config.FileVersion }, + { "copyright", config.Copyright }, + { "productName", config.ProductName }, + { "productVersion", config.ProductVersion }, }; try { ClrModule.CompileModules(DefaultContext.DefaultCLS, - Path.ChangeExtension(config.Output, ".dll"), + Path.Combine(config.OutputPath, Path.ChangeExtension(config.Output, ".dll")), compileOptions, config.Files.ToArray()); - var outputfilename = Path.ChangeExtension(config.Output, ".dll"); + var outputfilename = Path.Combine(config.OutputPath, Path.ChangeExtension(config.Output, ".dll")); if (config.Target != PEFileKinds.Dll) { - outputfilename = Path.ChangeExtension(config.Output, ".exe"); + outputfilename = Path.Combine(config.OutputPath, Path.ChangeExtension(config.Output, ".exe")); GenerateExe(config); } - ConsoleOps.Info("Saved to {0}", outputfilename); + ConsoleOps.Info($"Saved to {outputfilename}"); } catch (Exception e) { diff --git a/Src/IronPythonCompiler/README.md b/Src/IronPythonCompiler/README.md index fcb5bfc49..f8333629e 100644 --- a/Src/IronPythonCompiler/README.md +++ b/Src/IronPythonCompiler/README.md @@ -1,7 +1,7 @@ IronPython Compiler ===== -Usage: ipc.exe [options] file [file ...] +Usage: ipyc.exe [options] file [file ...] Options: @@ -12,10 +12,18 @@ Options: /target:exe Generate CONSOLE executable stub for startup in addition to dll. /target:winexe Generate WINDOWS executable stub for startup in addition to dll. + + /fileversion: Sets the file version attribute for the generated assembly + + /copyright: Sets the copyright message for the generated assembly + + /productname: Sets the product name attribute for the generated assembly + + /productversion: Sets the product version attribute for the generated assembly @ Specifies a response file to be parsed for input files and command line options (one per line) - - /? /h This message + + /? /h This message EXE/WinEXE specific options: @@ -36,6 +44,7 @@ EXE/WinEXE specific options: /win32icon:file.ico Sets file.ico as the icon for the executable + Example: - ipc.exe /main:Program.py Form.py /target:winexe + ipyc.exe /main:Program.py Form.py /target:winexe From ef64be84ec5205c4e1d7188b979e501391782c13 Mon Sep 17 00:00:00 2001 From: Alex Earl Date: Sat, 14 Jul 2018 19:26:43 -0700 Subject: [PATCH 3/4] Update DLR --- Src/DLR | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Src/DLR b/Src/DLR index 7d0e268a0..dbbd8813a 160000 --- a/Src/DLR +++ b/Src/DLR @@ -1 +1 @@ -Subproject commit 7d0e268a0419274d37653164a41c3d924ac9026c +Subproject commit dbbd8813a66e8ba071d6710dd4ccf3cc5a7eed16 From 411ef91b5fd23edde168018d981efff89e010857 Mon Sep 17 00:00:00 2001 From: slozier Date: Sun, 15 Jul 2018 10:22:25 -0400 Subject: [PATCH 4/4] Fix test_cgcheck --- Src/Scripts/generate_comdispatch.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/Src/Scripts/generate_comdispatch.py b/Src/Scripts/generate_comdispatch.py index fc949fdda..eb39ded29 100644 --- a/Src/Scripts/generate_comdispatch.py +++ b/Src/Scripts/generate_comdispatch.py @@ -184,9 +184,8 @@ def gen_exposed_code_security(cw): getStatements=[ "if (_typeUnion._unionTypes._bstr != IntPtr.Zero) {", " return Marshal.PtrToStringBSTR(_typeUnion._unionTypes._bstr);", - "} else {", - " return null;", - "}" + "}", + "return null;" ], setStatements=[ "if (value != null) {", @@ -200,9 +199,8 @@ def gen_exposed_code_security(cw): getStatements=[ "if (_typeUnion._unionTypes._dispatch != IntPtr.Zero) {", " return Marshal.GetObjectForIUnknown(_typeUnion._unionTypes._unknown);", - "} else {", - " return null;", - "}" + "}", + "return null;" ], setStatements=[ "if (value != null) {", @@ -216,9 +214,8 @@ def gen_exposed_code_security(cw): getStatements=[ "if (_typeUnion._unionTypes._dispatch != IntPtr.Zero) {", " return Marshal.GetObjectForIUnknown(_typeUnion._unionTypes._dispatch);", - "} else {", - " return null;", - "}" + "}", + "return null;" ], setStatements=[ "if (value != null) {",