MSVC add support for preview/prerelease versions #4048
Comments
|
Code snippets for vswhere to json and processing json output in #3988 (comment). Sample vswhere to json functions: # logical placement is in vc.py after msvc_find_vswhere
def vswhere_query_json(vswhere_args, env=None):
""" Find MSVC instances using the vswhere program.
Args:
vswhere_args: query arguments passed to vswhere
env: optional to look up VSWHERE variable
Returns:
json output or None
"""
if env is None or not env.get('VSWHERE'):
vswhere_path = msvc_find_vswhere()
else:
vswhere_path = env.subst('$VSWHERE')
if vswhere_path is None:
debug("vswhere path not found")
return None
debug('VSWHERE = %s' % vswhere_path)
vswhere_cmd = [vswhere_path] + vswhere_args + ['-format', 'json']
debug("running: %s" % vswhere_cmd)
#cp = subprocess.run(vswhere_cmd, capture_output=True) # 3.7+ only
cp = subprocess.run(vswhere_cmd, stdout=PIPE, stderr=PIPE)
if not cp.stdout:
debug("no vswhere information returned")
return None
vswhere_output = cp.stdout.decode("mbcs")
if not vswhere_output:
debug("no vswhere information output")
return None
try:
vswhere_json = json.loads(vswhere_output)
except json.decoder.JSONDecodeError:
debug("json decode exception loading vswhere output")
vswhere_json = None
return vswhere_json
# alternate version from WIP stand-alone script that used utf-8 encoding
# that takes as input a list of queries and returns a list of json output
@classmethod
def vswhere_query_json(cls, vswhere_queries):
vswhere_exe = cls._vswhere_executable()
if not vswhere_exe:
debug("vswhere path not found")
return None
vswhere_output = []
for vswhere_args in vswhere_queries:
vswhere_cmd = [vswhere_exe] + vswhere_args + ['-format', 'json', '-utf8']
vswhere_out = _VSDetectEnvironment.subprocess_run(vswhere_cmd, encoding='utf8', errors='strict')
if not vswhere_out:
debug("no vswhere information output")
continue
try:
vswhere_json = json.loads(vswhere_out)
except json.decoder.JSONDecodeError:
# JCB internal error?
debug("json decode exception loading vswhere output")
continue
vswhere_output.extend(vswhere_json)
return vswhere_outputfragment: vswhere_json.txt |
|
Two fragments for processing json output: # version 1
def _register_msvc_instances_in_vswhere_json(vswhere_output):
msvc_instances = []
for instance in vswhere_output:
productId = instance.get('productId','')
if not productId:
debug('productId not found in vswhere output')
continue
installationPath = instance.get('installationPath','')
if not installationPath:
debug('installationPath not found in vswhere output')
continue
installationVersion = instance.get('installationVersion','')
if not installationVersion:
debug('installationVersion not found in vswhere output')
continue
major = installationVersion.split('.')[0]
try:
msvc_vernum = _MSVC_CONFIG.MSVS_MAJORVERSION_TO_VCVER[major]
except KeyError:
debug("Unknown version of msvs: %s" % installationVersion)
# TODO|JCB new exception type: MSVSProductUnknown?
raise UnsupportedVersion("Unknown msvs version %s" % installationVersion)
componentId = productId.split('.')[-1]
if componentId not in _MSVC_PRODUCTVERSION_COMPONENTIDS[msvc_vernum]['Products']:
debug('ignore componentId:%s' % componentId)
continue
component_rank = _MSVC_COMPONENTID_SELECTION_RANKING.get(componentId,0)
if component_rank == 0:
debug('unknown componentId:%s' % componentId)
continue
msvc_version = msvc_vernum
msvc_product = msvc_version + _MSVC_COMPONENTID_VERSIONSUFFIX[componentId]
if msvc_product in _VCVER:
msvc_version = msvc_product
if _MSVC_CONFIG.TESTING_IGNORE_INSTALLED_PRODUCT:
if msvc_product in _MSVC_CONFIG.TESTING_IGNORE_INSTALLED_PRODUCTS:
debug('ignoring product: ' + msvc_product)
continue
d = {
'msvc_version' : msvc_version,
'msvc_version_numeric' : msvc_vernum,
'msvc_product' : msvc_product,
'productId' : productId,
'installationPath': installationPath,
'componentId' : componentId,
'component_rank' : component_rank,
}
debug('found msvc instance:(%s,%s)' % (productId, msvc_version))
_MSVC_CONFIG.N_PRODUCTS += 1
_MSVC_CONFIG.MSVC_VSWHERE_INSTANCES_UNIQUE.setdefault(msvc_version,[]).append(d)
_MSVC_CONFIG.MSVC_VSWHERE_INSTANCES.setdefault(msvc_version,[]).append(d)
if msvc_product != msvc_version:
# synthetic version number (e.g., 14.2Ent, 14.2Pro, 14.2Com, 14.2BT) needed when looking up vc_dir
debug('found specific msvc instance:(%s,%s)' % (productId, msvc_product))
_MSVC_CONFIG.MSVC_VSWHERE_INSTANCES.setdefault(msvc_product,[]).append(d)
else:
# Known products (e.g., 14.1Exp) are not entered in the "bare" version number list
# due to vc_dir lookup when walking _VCVER list.
# Extended version support will use the "fully qualified" product type list.
pass
msvc_instances.append(d)
for version, instances in _MSVC_CONFIG.MSVC_VSWHERE_INSTANCES_UNIQUE.items():
if len(instances):
# sort multiple instances remaining based on productId priority: largest rank to smallest rank
_MSVC_CONFIG.MSVC_VSWHERE_INSTANCES_UNIQUE[version] = sorted(instances, key = lambda x: x['component_rank'], reverse=True)
for version, instances in _MSVC_CONFIG.MSVC_VSWHERE_INSTANCES.items():
if len(instances):
# sort multiple instances remaining based on productId priority: largest rank to smallest rank
_MSVC_CONFIG.MSVC_VSWHERE_INSTANCES[version] = sorted(instances, key = lambda x: x['component_rank'], reverse=True)
return msvc_instances
# version 2 including prerelease flag
@classmethod
def vs_process_json_output(cls, vs_rel, vswhere_output, vs_cfg):
msvs_instances = []
for instance in vswhere_output:
#print(json.dumps(instance, indent=4, sort_keys=True))
productId = instance.get('productId','')
if not productId:
debug('productId not found in vswhere output')
continue
installationPath = instance.get('installationPath','')
if not installationPath:
debug('installationPath not found in vswhere output')
continue
installationVersion = instance.get('installationVersion','')
if not installationVersion:
debug('installationVersion not found in vswhere output')
continue
major = installationVersion.split('.')[0]
if major != vs_cfg.vs_major:
raise InternalError('Internal error: msvs major version mis-match: %s, %s' % (major, vs_cfg.vs_major))
component_id = productId.split('.')[-1]
if component_id not in vs_cfg.vs_products:
debug('ignore component_id:%s' % component_id)
continue
component_rank = _VSDetectConst.MSVC_COMPONENTID_SELECTION_RANKING.get(component_id,0)
if component_rank == 0:
# JCB|TODO: exception and/or stderr?
debug('unknown component_id:%s' % component_id)
continue
isPrerelease = 1 if instance.get('isPrerelease', False) else 0
isRelease = 0 if isPrerelease else 1
vs_root = _VSDetectUtil.process_path(installationPath)
if vs_root in vs_rel.msvs_roots:
continue
vs_rel.msvs_roots.add(vs_root)
msvs_base = MSVSBase._factory(
title = vs_cfg.vs_title,
root_dir = vs_root,
root_version = installationVersion,
vc_release = isRelease,
vc_version = vs_cfg.vc_version,
vc_runtime = vs_cfg.vc_runtime,
vc_component_id = component_id,
vc_component_rank = component_rank,
)
msvs_instance = MSVSInstance._factory(
msvs_base = msvs_base,
vs_dir = vs_root,
vs_version = installationVersion,
vs_executable = _VSDetectUtil.vs_executable(vs_root, vs_cfg.vs_executable),
vs_script = _VSDetectUtil.vs_batchfile(vs_root, [vs_cfg.vs_batchfile]),
vs_script_errs = vs_cfg.vs_script_errs,
vs_environ_vars = vs_cfg.vs_environ_vars,
vs_preserve_vars = vs_cfg.vs_preserve_vars,
)
msvs_instances.append(msvs_instance)
msvs_instances = sorted(msvs_instances, key=cmp_to_key(MSVSInstance._default_order))
return msvs_instancesfragment: vswhere_json_processing.txt Version 1: old branch supporting specific version selection |
|
The WIP stand-alone script for VS/VC detection departs from the current SCons implementation in subtle but important ways. The script attempts to ferret out all installed versions of MSVS/MSVC. Prerelease versions are currently enabled via the command line. I will probably have to rethink the current sorting/ranking. Not implemented yet is SDK support. Windows only. If you're interested in the beginning and ending system environments for msvc batch file invocations, then this is the tool for you. There are no dependencies outside of standard python. I have been developing using embedded python 3.6 (32-bit and 64-bit). There may be some ideas in here worth pursuing. It is my intention that this script is reduced to a couple callable functions that return minimal arguments to an SCons Environment. I also have a need for older MSVC detection outside of SCons. This probably is not the place for it, but the environment passed to subprocess when invoking some of the newer MSVC batch files in the current SCons tree are too aggressively minimal in some cases. The vsdetect.py script writes voluminous output to stdout and minimal information to stderr. A typical invocation redirecting output to a text file is: python vsdetect.py --noverify 1>vsdetect-stdout.txt
To actually run all of the batch files for each tool and redirecting output to a text file is: python vsdetect.py 1>vsdetect-full-stdout.txtOn my computer it takes 154 seconds to run all 171 batch files for each tool (toolset). This is alpha quality and currently outside of any version control system. vsdetect.py zip: vsdetect-20211105.zip For your amusement and/or horror. |
|
Documentation for future reference. Candidate vswhere command line: vswhere.exe -all -products * -prerelease -format json -utf8Notes:
Visual Studio workload and component IDs documentation:
Product IDs of interest (productId):
Product IDs that can likely be ignored (productId):
|
That's pretty ingrained in the SCons philosophy, though I'm well aware it causes problems.
wonder if that will cause problems with subprocess... there are plenty of notes about it blocking when there's two much output from the subprocess, I have no idea if more recent Pythons are less prone to that. Just a question... |
|
Isn't there also a |
The vsdetect.py script above is writing a ton of effectively debug data to stdout after the subprocess calls. This script is mostly for detecting all MSVS instances and toolsets outside of SCons and verifying that there are no errors when running the batch files. It also serves as a testbed to see what environment variables should be added and possible preserved. Currently the output is to determine if all instances and toolsets are detected correctly and the data structures are built appropriately.
One thing that became apparent was that newer versions of MSVS (2017+) invoke a number of batch files that may make use "standard" windows environment variables that are not present in the environment when the batch file is invoked. For example:
Locally, vcvarsall.bat in one 2017 instance actually reports an error due to typescript not being found. This passes in SCons because cl.exe is on the path but the error message is not detected. This was a surprise and is what led to expanding the environment passed to the subprocess calls. The implementation is a bit tricky as python and/or the windows shell can be 32-bit on a 64-bit host. On a 64-bit host, a new 64-bit command shell is started to execute the vc batch file. The vsdetect script has a switch to produce "scons-like" environments and "extended" environments and then comparing the output. For the most part the resulting output is the same with the a handful of minor exceptions. In addition to the one mentioned above, the HTML help compiler is added to the path for some instances of MSVS. This morning the vswhere queries in vsdetect.py were reworked from one+ queries per version (e.g., 14.3) to a single query for all installations as suggested above. The runtime improvement was fairly significant. From approximately 200ms to 90ms. Note: the vsdetect script is hard on both the registry and filesystem as it finds all toolsets available. @mwichmann - Perhaps a new discussion for MSVS/MSVS detection (similar to the discussion for QT) would be a useful location to document some of the existing behavior and possible improvements/remedies? |
The legacy flag will return instances of VS2010 (10.0) to VS2015 (14.0) in addition to VS2017+ (14.1+) versions. The legacy instance fields are limited to installationPath and installationVersion. Locally, I only have one instance each of VS2010-VS2015 installed so I can't be sure what vswhere returns when there is more than one instance of a given product (e.g., VS2015 Community and VS2015 Express). For the following installed products:
The following invocation: Returns the following with VS2017+ entries manually removed: As SCons treats Express as an independent version specification (e.g., '11.0Exp'), some additional work would need to be done to determine if the installation path is an express version when using vswhere. I have the installers for VS2015 Express and VS2013 Express so it would be possible to see what is returned when there are multiple instances of the same product. VS2010 SDK Notes:
|
|
Visual Studio versions as of 2021-11-10
|
|
An SCons branch with preliminary support for preview/prerelease versions of VS 2017-2022 is in my GitHub repository at: If there is any interest, a PR could be issued as a basis for discussion. It turns out that a single vswhere json query is the easy part. The harder parts were related to find_vc_pdir and find_vc_pdir_vswhere being used for both finding all installed versions (setup/initialization) and for finding the appropriate batch file when processing a user environment (runtime). When finding all installed versions there is a need to bypass checking the environment (if it even exists) for the preview flag and force traversal for all versions and all preview versions (14.1-14.3). This was implemented by adding an optional argument to a number of functions that force the release/preview indicator to be used rather than querying the environment key. Also, there are now two installed_versions lists: one for release versions and one for preview versions. The cached installed_versions list was changed to a dictionary of two lists with the keys for release (True) and preview/prerelease (False). The changes to find_vc_pdir, find_vc_pdir_vswhere, get_installed_vcs, and msvc_exists have larger implications for the test suite. Two minimal changes were made to the existing tests for the additional parameter added. However, adding new tests for the preview versions will be more difficult than actually supporting preview versions. The following tests pass or are skipped:
The following SConstruct environments were tested with a single c file: The environment variable The MSVC_PREVIEW environment variable is silently ignored for VS versions 14.0 and earlier. From the list above, I believe numbers 1, 3, and 4 may have been accomplished. Intentionally, the registry components were not touched. I am not sure what will happen if there is only one installed instance of VS and it is a preview version. It should work but it has not been tested yet. Note: manually disabling the preview environment lookup would result in the current scons behavior (release versions only) but with a single vswhere query with results cached. |
|
|
|
Completely agree. Updated code and comment above. |
|
So where are we on this one? What will it take to get it over the line? |
|
I need to make sure the proposed code is consistent with any/all changes since Nov 2021. I haven't looked at the proposed code in the last two months. Unfortunately, I had to reinstall windows (twice) since then which meant reinstalling all versions of MSVS again (currently at 19 instances and 231 toolsets for testing purposes) which was a bit of a time sink. There are degenerate/corner cases that probably still need to be investigated (e.g., there is on one instance installed and it is a preview version) which probably are easiest to test in a virtual machine. The use of MSVC_Preview and a boolean value precludes the use of "Any" (i.e., release if one exists otherwise preview in a deterministic search order when they are are multiple instances installed). In addition, it may be harder to expand in the future should any other "channel-like" options be added to vswhere. For testing in my standalone script, I adopted "MSVC_Channel" which can have the values; "Release", "Preview", or "Any". Any makes more sense when specific buildtools (v143) or toolsets (14.3) are supported as well. For example, build with the v143 toolset and use any release instance before using a preview instance that contains a v143 toolset. The default value when not specified would be "Release". If nothing else, switching to "MSVC_Channel" and the values "Release" and "Preview" (i.e., not supporting "Any") could be mapped to the proposed implementation and could be considered more descriptive. Any thoughts on "MSVC_Preview" versus "MSVC_Channel"? It is advisable for someone other than me to review the proposed code as well. The current architecture/implementation of the msvc detection implementation imposes a number of constraints that had to be worked around. |
|
Wow that's a lot to install! Once we have the output from the single vswhere run, we can filter it for things other than the specific requested version, including prerelease. Another thing probably worth considering is do we want a way to say both of:
|
|
Visual Studio instances:
Also attached is jcb-vsdetect.zip containing jcb-vsdetect.txt which is a 1.65mb text file containing a json dump of the internal data structures for my standalone msvs detection tool. The tool detects all 19 instances above, 38 toolsets (note: this is inflated due to v140 tools), and 231 tools (toolset and host/target). If nothing else, thus far it is a useful diagnostic tool. On my computer, it took about 0.110s (64-bit) and 0.136s (32-bit) to detect the instances and construct the data structure. Currently, many tables/dictionaries are pre-computed with a variety of keys offering a number of ways to query the data structure.
Short answer is yes. Long answer is more difficult as the current MSVC_VERSION is really a Visual Studio product and not the toolset/tool version. There is no requirement that the v143 tools are actually installed in VS2022 while the v142 tools and lower may be installed. I have not tested this yet. The preview toolset versions (14.31.30919) are larger than the release toolset versions (14.30.30705) for VS2022. Possible independent ordering keys:
In the sample data structure output attached above, if you search for "edition_toolset" you will find sorted lists by toolset version first and channel value second with release ordered before preview for the same toolset version for a given key. |
|
Agree the triple-versioning of things is a problem, but it's what Microsoft gives us. Is there any way we can do better than what we're doing in how this is exposed to the programmer? We already have |
|
I apologize in advance for the lack of a more polished and thoughtful response. Time limited today... In a perfect world, adding MSVC_PRODUCT, MSVC_CHANNEL, MSVC_COMPONENT, and MSVC_TOOLSET as user-facing options would satisfy most of the outstanding issues/enhancements. Also, MSVC_RUNTIME is useful as a request for any vc runtime compatible product/buildtools/toolset installed. Backward compatibility when MSVC_VERSION is specified is more difficult due to validation across an expanded set of inputs. It would be better if the existing MSVC_VERSION were derived internally from the toolset that is selected. Other than the test suite, MSVC_VERSION seems to only appear once outside of the detection code (Tool\msvc.py). This would require non-trivial expansion to the underlying detection code and to the internal data structures. The standalone script I'm using to test these concepts supports the additions above and a handful more that aren't relevant to this particular discussion. MSVC_PRODUCT:
MSVC_CHANNEL:
MSVC_COMPONENT:
Note: components for 2015 and earlier are limited at present. MSVC_TOOLSET [buildtools or proper toolset version prefix] examples:
MSVC_RUNTIME:
The fragments below show one way all of the microsoft version numbers can be tied together:
|
|
@jcbrill - Your "unpolished" responses exceed most other users' polished responses. Thanks for your continued efforts and attention to detail on this issue! |
For discussion purposes, all new variables are introduced so there is no conceptual baggage attached to re-use of existing variables. Ignore MSVC_VERSION for the moment. For modern versions of visual studio detected via vswhere (2015 and later), a visual studio instance can be uniquely defined by:
Despite what was presented above, the more appropriate names for these elements are:
An installed visual studio instance can be uniquely identified by the triple: MSVS_PRODUCT, MSVS_CHANNEL, MSVS_COMPONENT. For visual studio 2015 and later, multiple vc buildtools/toolsets can be installed. MSVC specific variables could include:
The outstanding issue is what is the exact semantic definition of MSVC_VERSION when defined by a user, if allowed at all, in the new system of definitions?
The current SCONS implementation is effectively method 1 (product interpretation). A better approach for backwards compatibility is likely method 2 (product and toolset interpetation) to avoid any ambiguity. A modern interpretation of "find the newest product that supports my desired toolset" is best served with method 3 (toolset interpretation). Methods 2 and 3 require the buildtools/toolset versions to be recorded during detection. There may be a degenerate case with method 1 that leads to undesirable results. This degenerate case may exist in the current SCONS implementation. The implied assumption is that the "latest" buildtools are always installed (e.g., buildtools v142 are installed in VS2019). This is not guaranteed. In the degenerate case above, method 1 appears to violate the Law of Least Astonishment. The crux of the problem for backwards compatibility is the semantic interpretation of MSVC_VERSION when specified. The semantic interpretation has consistency implications when MSVC_VERSION is defined and MSVS_PRODUCT and/or MSVC_TOOLSET are defined at the same time as well. Moving forward, it would probably be better if MSVC_VERSION was derived from the selected toolset/instance rather than given as input by a user. Validating the set of user-provided arguments is significantly less difficult if MSVC_VERSION and MSVS_VERSION are removed from consideration. I should probably actually test the degenerate case example above via a virtual machine as I may be wrong about the current SCONS implementaiton. Unfortunately, motivation is low. Any thoughts? |
|
I have a thought: my head hurts. |
|
Longing to see this in a release. Is there anything that can be done to help? |
Due to recent PRs accepted and a handful that are under consideration for the msvc bug scrub (https://github.com/SCons/scons/projects/13), the code in the preview branch needs to be reworked to accommodate recent changes. Specifically, #4131 accepted two days ago, refactored parts of the existing code and affected a number of my other branches. Currently, there are 3 more outstanding PRs in and around the msvc code: #4117 and #4125 have been reviewed and are complete, #4132 needs to be reviewed further and documented. #4132 was updated yesterday for the #4131 changes and could experience a lengthy discussion period. These outstanding PRs may affect the code in the preview branch as well. I don't expect it will take very long to bring the preview branch back up-to-date with the current main branch and pending PRs. This was a known problem with the recent proliferation of PRs all in and around the msvc code for other enhancements and the msvc bug scrub. Once it is up-to-date, a proper PR is probably in order for a more thorough discussion of capabilities and needs. Additional feedback and testing is always welcome. |
Yes you could. It has always been possible to use a preview version via For example, VS2022 Community Preview 64-bit target on a 64-bit host: or Edit: the difference between the two approaches is that the above will still invoke the batch file and capture the environment where #4125 bypasses the detection and the environment is the input. I like to think of the utility of #4125 as the result of some sort of automation/caching on the user-side. |
|
Any change this can be added at some point? If there would just be a way to provide additional args to vswhere so we can provide the |
|
We've been talking about it. Whether it's as simple as |
This. There would likely be an unsupported version exception raised. A new product version, release or preview, has to be configured internally. However, when preview versions have "first-class" support, then the internal configuration for a new product can be done upon the availability of the preview edition since no release versions would be detected anyway. The "best" way to proceed depends on what happens with #4409 as the vswhere query implementation is changed significantly. One of the changes made in the PR is to replace all the individual vswhere queries with a single vswhere query for all instances. This becomes more efficient as there is a single query for all versions versus (possibly) multiple queries for each version (e.g., detection of BuildTools and Express require separate queries with different arguments). There are preview versions of the BuildTools as well as the "full" products. A single query is easier to work with as the versions, workloads, and "channel" (release/preview) can be picked out of the output as desired. This allows implementation of a deterministic ordering of editions within each product (e.g., Enterprise, Professional, Community, BuildTools, Express). Additionally, this allows for "channel" specific lists to be constructed and queried: release, preview, and any/either (subject to a predetermined ordering within a product version) which is the desire here. In the PR, the prerelease argument was not included simply due to the fact that it was not going to be used immediately. Some provisions have been made for supporting preview versions. The source code in the PR for the vswhere query handling was taken from the branch posted above in one of the earlier comments. Due to the age of the branch above and the evolution of the msvc code, some minor refactoring is necessary to support preview in a more straightforward manner should it be adopted. There are other ways to accomplish the same goal, but a single vswhere query is a net gain in simplicity, flexibility, and ease of future expansion in the current code base. The acceptance or rejection of 4409 will have an impact on how to more forward with "first-class" support for preview versions. |
|
It is possible to use a preview installation via the msvc batch files directly using scons. This bypasses the scons auto-detection. The following example is for a 2022 Community Preview build targeting In this case, the This is only "portable" (i.e., for other build environments) if the build computer is using the default msvs installation paths. Otherwise, the path to The following example is for a 2022 Community Preview build targeting While this may not be desirable, it is possible. Edit: change to toolset version text for example. |
Describe the Feature
This is a continuation of discussion in issue #3988
Currently SCons MSVC logic will not detect installed prerelease versions.
To see these versions it needs to pass
-prereleaseflag to it's invocation ofvswhereAdditionally it may be possible to vastly speed up MSVC detect if we do the following
vswhere -all -prerelease -json(fix if incorrect invocation) and process the generated JSON file.The text was updated successfully, but these errors were encountered: