diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..44a4210 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: +- package-ecosystem: Cargo + directory: "/" + schedule: + interval: "daily" diff --git a/.github/workflows/binaries.yml b/.github/workflows/binaries.yml new file mode 100644 index 0000000..2ee7f51 --- /dev/null +++ b/.github/workflows/binaries.yml @@ -0,0 +1,68 @@ +name: Build Binaries + +on: + workflow_call: + inputs: + binary_name: + description: "Expected name of binary" + required: false + type: string + default: "dls" + output_binary_name: + description: "Name of uploaded binary artifact" + type: string + default: "dls" + log-dir: + description: "Folder to put cargo logs in" + required: false + type: string + os: + description: "Machine to run on" + required: true + type: string +env: + CARGO_TERM_COLOR: always +jobs: + build: + runs-on: ${{ inputs.os }} + steps: + - uses: actions/checkout@v4 + - name: versions + run: | + rustup --version + cargo --version + rustc --version + - name: Prepare log dir + run: mkdir -p ${{ inputs.log-dir }} + - name: Build + shell: bash + run: cargo build --release --verbose 2>&1 | tee -a ${{ inputs.log-dir }}/build.log + - name: Test + shell: bash + run: | + set -o pipefail + cargo test --verbose 2>&1 | tee -a ${{ inputs.log-dir }}/test.log + - name: Clippy + shell: bash + if: ${{ success() || failure() }} + run: | + set -o pipefail + rustup component add clippy + cargo clippy --version + cargo clippy --all-targets -- --deny warnings 2>&1 | tee -a ${{ inputs.log-dir }}/clippy.log + - name: Upload logs + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v4 + with: + name: cargo-logs + overwrite: true + path: ${{ inputs.log-dir }} + if-no-files-found: error + - name: Upload binary + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.output_binary_name }} + overwrite: true + path: target/release/${{ inputs.binary_name }} + if-no-files-found: error diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml new file mode 100644 index 0000000..1baa29c --- /dev/null +++ b/.github/workflows/rust.yml @@ -0,0 +1,42 @@ +name: Archive Binary + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + schedule: + - cron: "0 0 * * *" +jobs: + build-package: + strategy: + matrix: + os: + - ubuntu-latest + - windows-latest + include: + - os: ubuntu-latest + binary: dls + - os: windows-latest + binary: dls.exe + uses: ./.github/workflows/binaries.yml + with: + binary_name: ${{ matrix.binary }} + output_binary_name: dml-server-${{ matrix.os }} + log-dir: ${{ matrix.binary }}-logs + os: ${{ matrix.os }} + check-package: + uses: ./.github/workflows/scans.yml + with: + os: ubuntu-latest + log-dir: checking-logs + merge-package: + runs-on: + - ubuntu-latest + needs: build-package + steps: + - name: Merge Artifacts + uses: actions/upload-artifact/merge@v4 + with: + name: dml-server + pattern: dml-server-* diff --git a/.github/workflows/scans.yml b/.github/workflows/scans.yml new file mode 100644 index 0000000..3b6d94a --- /dev/null +++ b/.github/workflows/scans.yml @@ -0,0 +1,51 @@ +name: Cargo Check + +on: + workflow_call: + inputs: + os: + description: "Machine to run on" + required: true + type: string + log-dir: + description: "Folder to put cargo logs in" + required: false + type: string + +env: + CARGO_TERM_COLOR: always +jobs: + audit: + runs-on: ${{ inputs.os }} + steps: + - uses: actions/checkout@v4 + - name: versions + run: | + rustup --version + cargo --version + rustc --version + - name: Prepare log dir + run: mkdir -p ${{ inputs.log-dir }} + - name: Audit Deny + shell: bash + if: ${{ success() || failure() }} + run: | + cargo install cargo-deny + cargo deny --version + set -o pipefail + cargo deny check 2>&1 | tee -a ${{ inputs.log-dir }}/deny-log + - name: Audit Outdated + shell: bash + if: ${{ success() || failure() }} + run: | + cargo install cargo-outdated + cargo outdated --version + set -o pipefail + cargo outdated --exit-code 1 2>&1 | tee -a ${{ inputs.log-dir }}/outdated-log + - name: Upload logs + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v4 + with: + name: cargo-scan-logs + overwrite: true + path: ${{ inputs.log-dir }} diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..0898090 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,8 @@ + +# Change Log + +## 1.0.0 +- Initial open-source release diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f682f4e..b7ca799 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,7 +2,7 @@ ### License - is licensed under the terms in [LICENSE]. By contributing to the project, you agree to the license and copyright terms therein and release your contribution under these terms. +DML Language Server is dual-licensed under the terms in [APACHE 2.0](./LICENSE-APACHE) and [MIT](./LICENSE-MIT). By contributing to the project, you agree to the license and copyright terms therein and release your contribution under these terms. ### Sign your work @@ -55,3 +55,152 @@ Use your real name (sorry, no pseudonyms or anonymous contributions.) If you set your `user.name` and `user.email` git configs, you can sign your commit automatically with `git commit -s`. + +## How-To + +This section provides information for developers who want to contribute to the +DLS or run it in a heavily customised configuration. + +Testing, reporting issues, writing documentation, writing tests, +writing code, and implementing clients are all extremely valuable. + +To get help with this, either leave an issue on the repo or contact the primary +developer directly. + +If you want to implement DLS support in an editor, see [clients.md](clients.md). + +## Building + +``` +git clone https://github.com/intel/dml-language-server.git +cd dml-language-server +cargo build --release +``` + +## Running and testing + +There are three main ways to run the DLS, you can run it directly with: + +``` +cargo run --bin dls +``` +Which starts up the DLS in a server-mode, so its of narrow applicability unless +you are planning on directly sending LSP communication into it. By passing the +'--cli' option you start up the server in command-line mode, see [CLI](#cli). + +You can run the Direct-File-Analysis (DFA) binary with: +``` +cargo run --bin dfa [options ...] [files ...] +``` +This will analyze the specified 'files' using the 'dls-binary' server binary +as-if they had been opened in a language client, this is usefull to quickly +test and debug initial file analysis and error reporting without advanced +client-interaction. + +Most commonly, you'll use an IDE plugin to invoke the DLS binary for you +(see [README.md](README.md) for details). + +Test the crate using `cargo test`. + +Testing is unfortunately minimal. There is support for some regression tests, +but not many actual tests exists yet. There is significant work to do +before we have a comprehensive testing story. + +### CLI + +You can run DLS in the command line mode which is useful for debugging and +testing, especially to narrow down a bug to either the DLS or a client. + +You need to run it with the +`--cli` flag, e.g., `cargo run --bin dls -- --cli`. This should initialize the +DLS and then give you a `>` prompt. +Look for the final message that will signal the end of the +initialization phase which will look something like: +``` +{"jsonrpc":"2.0","method":"window/showMessage","params":{"message": "DML Language Server Started", "type": 3}} +``` + +Type `help` (or just `h`) to see the [commands available][CLI_COMMANDS]. Note +that the [positions][LSP_POSITION] in the requests and the responses are +_zero-based_ (contrary to what you'll normally see in the IDE line numbers). + +[LSP_POSITION]: https://github.com/Microsoft/language-server-protocol/blob/gh-pages/specification.md#position + +[CLI_COMMANDS]: dls/src/cmd.rs#L508-L542 + +## Implementation overview + +The idea behind the DLS is to grant fast and mostly-accurate feedback for DML +development, of either specific devices or DML common-code. Due to complexities +in the DML language, and the high level of customization available when running +DMLC, this information will not be perfectly accurate or holistic at all times. +However the goal is to give a best-effort analysis that minimizes noisy +incorrect feedback and maximizes the amount of valuable info a developer can +extract, as well as providing basic standard-issue IDE functionality. + +### Analysis + +The DLS tracks changes to files, and keeps the changed file in memory (i.e., the +DLS does not need the IDE to save a file before providing data). These changed +files are tracked by the 'Virtual File System'. + +Analysis is divided into two phases. +- The 'isolated' analysis analyses each +DML file individually, and does parsing and some post-parse processing. This +analysis perform syntactic analysis and is enough to report basic syntax errors, +- The 'device' analysis analyses from the perspective of a particular DML file with +a device declaration, pulling in information from the various isolated +analysises. This analysis performs semantic analysis, and is what +powers reference-matching, object resolution, etc. + +Isolated analysis is started as soon as the user opens or changes a file +(although this can be configured in the settings to be only on save). +In addition the DLS will use information about import paths and workspaces to +try to resolve file imports, and recursively analyse any file imported from +an opened one. See the section on [include paths](#include-paths) for more +information on them. Device analysis is started as soon as it is determined that +it could be, or needs to be re-done. Commonly this is when an isolated analysis +involved in the import tree of a device-file is updated, if all files that the +device depends on have had an isolated analysis done. There are corner-case +exceptions to this (allowing for a partially correct device analysis when an +imported file is missing) but for simplicity that is not described here. + +### Communicating with IDEs + +The DLS communicates with IDEs via +the [Language Server protocol](https://github.com/Microsoft/language-server-protocol/blob/master/protocol.md). + +The LS protocol uses JSON sent over stdin/stdout. The JSON is rather dynamic - +we can't make structs to easily map to many of the protocol objects. The client +sends commands and notifications to the DLS. Commands must get a reply, +notifications do not. Usually the structure of the reply is dictated by the +protocol spec. The DLS can also send notifications to the client. So for a long +running task (such as a build), the DLS will reply quickly to acknowledge the +request, then send a message later with the result of the task. + +Associating requests with replies is done using an id which must be handled by +the DLS. + +### Extensions to the Language Server Protocol + +The DLS uses some custom extensions to the Language Server Protocol. +These are all sent from the DLS to an LSP client and are only used to +improve the user experience by showing progress indicators. + +* `window/progress`: notification, `title: "Analysing", value: WorkDoneProgressBegin`. Sent when the first analysis starts +* ... standard LSP `publishDiagnostics` +* `window/progress`: notification, `title: "Analysing", value: WorkDoneProgressEnd`. Sent when the last analysis finishes + +### Include Paths +In order to support fairly generic and complicating import configurations, the +DLS uses a context-aware methodology in order to resolve import paths. + +Sidenote: be aware that due to past DMLC 1.2/1.4 interoperability functionality, +the DLS will correctly search the "./" and "./1.4/" of every path it searches. + +When importing a file "A.dml" from some file "B", the DLS will search, in no +particular order: +- The folder of B +- The root folder of the workspace +- All other workspace roots +- Every include path specified by the [DML Compile commands](README.md#dml-compile-commands) under the related module diff --git a/COPYRIGHT b/COPYRIGHT new file mode 100644 index 0000000..91ce3cc --- /dev/null +++ b/COPYRIGHT @@ -0,0 +1,6 @@ +Source code is a modified version of the code available at +https://github.com/rust-lang/rls, which is licensed under Apache 2.0 +and MIT. + +This code is similarly dual-licensed under Apache 2.0 and MIT, +see the corresponding license files for more information. diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..8de7151 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,53 @@ +[package] +name = "dls" +version = "0.9.5" +edition = "2018" +authors = [] +description = "Simics DML language server" +license = "Apache-2.0/MIT" +categories = ["development-tools"] + +[profile.release] +debug = 1 + +[lib] +name = "dls" +doctest = false +path = "src/lib.rs" + +[[bin]] +name = "dls" +test = false +path = "src/main.rs" + +[[bin]] +name = "dfa" +test = false +path = "src/dfa/main.rs" + +[dependencies] +anyhow = "1.0" +clap = { version = "4.2", features = ["cargo", "derive"] } +crossbeam = "0.8" +crossbeam-deque = "0.8.1" +crossbeam-utils = "0.8.7" +env_logger = "0.11" +itertools = "0.13" +jsonrpc = "0.18" +lsp-types = { version = "0.97" } +lazy_static = "1.4" +log = "0.4" +logos = "0.14" +rayon = "1" +regex = "1.5.5" +serde = "1.0" +serde_ignored = "0.1" +serde_json = "1.0" +slotmap = "1.0" +store-interval-tree = "0.4" +strum = { version = "0.26", features = ["derive"] } +subprocess = "0.2" +thiserror = "2.0" +utf8-read = "0.4" +walkdir = "2" +heck = "0.5" diff --git a/LICENSE b/LICENSE deleted file mode 100644 index cf75ceb..0000000 --- a/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2024 Intel Corporation - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/LICENSE-APACHE b/LICENSE-APACHE new file mode 100644 index 0000000..16fe87b --- /dev/null +++ b/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/LICENSE-MIT b/LICENSE-MIT new file mode 100644 index 0000000..40b8817 --- /dev/null +++ b/LICENSE-MIT @@ -0,0 +1,25 @@ +Copyright (c) 2016 The Rust Project Developers + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md index 331258c..463aee6 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,56 @@ -# The DML language server + +# DML Language Server (DLS) -This repository is a place holder repository for a [Device Modeling Languge](https://github.com/intel/device-modeling-language) [Language Server](https://langserver.org/). As that is a mouthful, it'll most likely be called DLS at some point. +The DLS provides a server that runs in the background, providing IDEs, +editors, and other tools with information about DML device and common code. +It currently supports basic syntax error reporting, symbol search, +'goto-definition', 'goto-implementation', 'goto-reference', and 'goto-base'. +It also has some basic configurable linting support, in the form of warning +messages. -The language server will be implemented in Rust and our intent is to provide pre-built binaries for Linux and Windows. Any editor that supports the [LSP protocol](https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/) shall be able to use it, but we are also implementing a [VS-code Simics Modelling Extension](https://github.com/intel/vscode-simics) (also still in intial development) in which it is to be integrated. +Future planned features are extended semantic and type analysis, basic +refactoring patterns, improved language construct templates, renaming +support, and more. -The current expectation is that we'll publish a work-in-progress version around end of 24Q3. +Do note that the DLS only supports DML 1.4 code, and there are no plans to +extend this functionality to support DML 1.2 code. It can only perform +analysis on files declared as using DML 1.4 version. -Please stay tuned! +## Building -/The DLS developers. +Simply run "cargo build --release" in the checkout directory. +## Running + +The DLS is built to work with the Language Server Protocol, and as such it in +theory supports many IDEs and editors. However currently the only implemented +language client is the Simics Modeling Extension for Visual Studio Code, which +is not yet publicly available. + +See [clients.md](clients.md) for information about how to implement +your own language client compatible with the DML language server. + +## DML Compile Commands +The DML compile commands file is used by the language server in order to obtain +per-module information used to resolve imports and obtain relevant command-line +DMLC flags for specific devices. + +Commonly this will be auto-generated by the CMake indexing in your Simics +project (you may need to specify the environment variable "CMAKE_EXPORT_COMPILE_COMMANDS=1" +before invoking cmake), but it is possible to construct such a file by hand if absolutely +neccessary. It is a json file with the following format: +``` +{ + : { + "includes": [], + "dmlc_flags: [] + }, + ... +} +``` + +This will add the include folders specified when analysing files included, +directly or indirectly, from the specified device file. diff --git a/clients.md b/clients.md new file mode 100644 index 0000000..ebae67e --- /dev/null +++ b/clients.md @@ -0,0 +1,98 @@ + +# Implementing clients + +A short guide to implementing DLS support in your favourite editor. + +Typically you will need to implement an extension or plugin in whatever format +and language your editor requires. + +If your editor has a LSP-supporting library, then this will be much more straightforward. Implementing a client without the support of a library is outside the scope of this document. + +## Preliminaries + +Check with the maintainers of this repo to see if support already exists or is in development. If you start developing support for some editor, it is good to inform us so that we can track this. + +If you encounter bugs or unexpected behaviour while implementing a client, please file an issue or create a PR. + +## Where there is existing LSP support + +If your editor has LSP support, then getting up and running is relatively easy. You +need a way to run the DLS and point the editor's LSP client at it. Hopefully +that is only a few lines of code. The next step is to ensure that the DLS gets +re-started after a crash - the LSP client may or may not do this automatically +(VSCode will do this five times before stopping). + +Once you have this basic support in place, the hard work begins: + +* Implement [extensions to the protocol]contributing.md#extensions-to-the-language-server-protocol) +* Client-side configuration. + - You'll need to send the `workspace/didChangeConfiguration` notification when + configuration changes. + - For the config options, see [config.rs](../dls/src/config.rs#L99-L111) +* Check for and install the DLS + - Download the latest [released binary](https://github.com/intel/dml-language-server.git/releases), you should regularly check for newly released binaries and update accordingly. +* Client-side features + - e.g., code snippets, build tasks, syntax highlighting +* Testing +* Ensure alignment with existing DML semantics + - e.g., syntax highlighting +* 'Marketing' + - because we want people to actually use the extension + - documentation - users need to know how to install and use the extension + - keep us informed about status so we can advertise it appropriately + - keep the DLS website updated + - submit the extension to the editor package manager or marketplace + +## Where there is no LSP support + +If your editor has no existing LSP support, you'll need to do all the above plus +implement (parts of) the LSP. This is a fair amount of work, but probably not as +bad as it sounds. The LSP is a fairly simple JSON over stdio protocol. The +interesting bit is tying the client end of the protocol to functionality in your +editor. + +### Required message support + +The DLS currently requires support for the following messages. Note that we +often don't use anywhere near all the options, so even with this subset, you +don't need to implement everything. + +Notifications: + +* `exit` +* `initialized` +* `textDocument/didOpen` +* `textDocument/didChange` +* `textDocument/didSave` +* `workspace/didChangeConfiguration` +* `workspace/didChangeWatchedFiles` +* `cancel` + +Requests: + +* `shutdown` +* `initialize` +* `textDocument/definition` +* `textDocument/references` +* `textDocument/documentSymbol` +* `workspace/symbol` + +From Server to client: + +* `client/registerCapability` +* `client/unregisterCapability` + +The DLS also uses some [custom messages](contributing.md#extensions-to-the-language-server-protocol). + +## Resources + +* [LSP spec](https://microsoft.github.io/language-server-protocol/specification) +* [contributing.md](contributing.md) - overview of the DLS and how to build, test, etc. + +## Getting help + +We're happy to help however we can. The best way to get help is either to +leave a comment on an issue in this repo, or to send me (jonatan.waern@intel.com) an email. diff --git a/deny.toml b/deny.toml new file mode 100644 index 0000000..0f5ed68 --- /dev/null +++ b/deny.toml @@ -0,0 +1,238 @@ +# This template contains all of the possible sections and their default values + +# Note that all fields that take a lint level have these possible values: +# * deny - An error will be produced and the check will fail +# * warn - A warning will be produced, but the check will not fail +# * allow - No warning or error will be produced, though in some cases a note +# will be + +# The values provided in this template are the default values that will be used +# when any section or field is not specified in your own configuration + +# Root options + +# The graph table configures how the dependency graph is constructed and thus +# which crates the checks are performed against +[graph] +# If 1 or more target triples (and optionally, target_features) are specified, +# only the specified targets will be checked when running `cargo deny check`. +# This means, if a particular package is only ever used as a target specific +# dependency, such as, for example, the `nix` crate only being used via the +# `target_family = "unix"` configuration, that only having windows targets in +# this list would mean the nix crate, as well as any of its exclusive +# dependencies not shared by any other crates, would be ignored, as the target +# list here is effectively saying which targets you are building for. +targets = [ + # The triple can be any string, but only the target triples built in to + # rustc (as of 1.40) can be checked against actual config expressions + #"x86_64-unknown-linux-musl", + # You can also specify which target_features you promise are enabled for a + # particular target. target_features are currently not validated against + # the actual valid features supported by the target architecture. + #{ triple = "wasm32-unknown-unknown", features = ["atomics"] }, +] +# When creating the dependency graph used as the source of truth when checks are +# executed, this field can be used to prune crates from the graph, removing them +# from the view of cargo-deny. This is an extremely heavy hammer, as if a crate +# is pruned from the graph, all of its dependencies will also be pruned unless +# they are connected to another crate in the graph that hasn't been pruned, +# so it should be used with care. The identifiers are [Package ID Specifications] +# (https://doc.rust-lang.org/cargo/reference/pkgid-spec.html) +#exclude = [] +# If true, metadata will be collected with `--all-features`. Note that this can't +# be toggled off if true, if you want to conditionally enable `--all-features` it +# is recommended to pass `--all-features` on the cmd line instead +all-features = false +# If true, metadata will be collected with `--no-default-features`. The same +# caveat with `all-features` applies +no-default-features = false +# If set, these feature will be enabled when collecting metadata. If `--features` +# is specified on the cmd line they will take precedence over this option. +#features = [] + +# The output table provides options for how/if diagnostics are outputted +[output] +# When outputting inclusion graphs in diagnostics that include features, this +# option can be used to specify the depth at which feature edges will be added. +# This option is included since the graphs can be quite large and the addition +# of features from the crate(s) to all of the graph roots can be far too verbose. +# This option can be overridden via `--feature-depth` on the cmd line +feature-depth = 1 + +# This section is considered when running `cargo deny check advisories` +# More documentation for the advisories section can be found here: +# https://embarkstudios.github.io/cargo-deny/checks/advisories/cfg.html +[advisories] +# The path where the advisory databases are cloned/fetched into +#db-path = "$CARGO_HOME/advisory-dbs" +# The url(s) of the advisory databases to use +#db-urls = ["https://github.com/rustsec/advisory-db"] +# A list of advisory IDs to ignore. Note that ignored advisories will still +# output a note when they are encountered. +ignore = [ + #"RUSTSEC-0000-0000", + #{ id = "RUSTSEC-0000-0000", reason = "you can specify a reason the advisory is ignored" }, + #"a-crate-that-is-yanked@0.1.1", # you can also ignore yanked crate versions if you wish + #{ crate = "a-crate-that-is-yanked@0.1.1", reason = "you can specify why you are ignoring the yanked crate" }, +] +# If this is true, then cargo deny will use the git executable to fetch advisory database. +# If this is false, then it uses a built-in git library. +# Setting this to true can be helpful if you have special authentication requirements that cargo-deny does not support. +# See Git Authentication for more information about setting up git authentication. +#git-fetch-with-cli = true + +# This section is considered when running `cargo deny check licenses` +# More documentation for the licenses section can be found here: +# https://embarkstudios.github.io/cargo-deny/checks/licenses/cfg.html +[licenses] +# List of explicitly allowed licenses +# See https://spdx.org/licenses/ for list of possible licenses +# [possible values: any SPDX 3.11 short identifier (+ optional exception)]. +allow = [ + "MIT", + "Apache-2.0", + "Zlib", + "Unicode-DFS-2016", + "CC0-1.0", +] +# The confidence threshold for detecting a license from license text. +# The higher the value, the more closely the license text must be to the +# canonical license text of a valid SPDX license file. +# [possible values: any between 0.0 and 1.0]. +confidence-threshold = 0.8 +# Allow 1 or more licenses on a per-crate basis, so that particular licenses +# aren't accepted for every possible crate as with the normal allow list +exceptions = [ + # Each entry is the crate and version constraint, and its specific allow + # list + #{ allow = ["Zlib"], crate = "adler32" }, +] + +# Some crates don't have (easily) machine readable licensing information, +# adding a clarification entry for it allows you to manually specify the +# licensing information +#[[licenses.clarify]] +# The package spec the clarification applies to +#crate = "ring" +# The SPDX expression for the license requirements of the crate +#expression = "MIT AND ISC AND OpenSSL" +# One or more files in the crate's source used as the "source of truth" for +# the license expression. If the contents match, the clarification will be used +# when running the license check, otherwise the clarification will be ignored +# and the crate will be checked normally, which may produce warnings or errors +# depending on the rest of your configuration +#license-files = [ +# Each entry is a crate relative path, and the (opaque) hash of its contents +#{ path = "LICENSE", hash = 0xbd0eed23 } +#] + +[licenses.private] +# If true, ignores workspace crates that aren't published, or are only +# published to private registries. +# To see how to mark a crate as unpublished (to the official registry), +# visit https://doc.rust-lang.org/cargo/reference/manifest.html#the-publish-field. +ignore = false +# One or more private registries that you might publish crates to, if a crate +# is only published to private registries, and ignore is true, the crate will +# not have its license(s) checked +registries = [ + #"https://sekretz.com/registry +] + +# This section is considered when running `cargo deny check bans`. +# More documentation about the 'bans' section can be found here: +# https://embarkstudios.github.io/cargo-deny/checks/bans/cfg.html +[bans] +# Lint level for when multiple versions of the same crate are detected +multiple-versions = "warn" +# Lint level for when a crate version requirement is `*` +wildcards = "allow" +# The graph highlighting used when creating dotgraphs for crates +# with multiple versions +# * lowest-version - The path to the lowest versioned duplicate is highlighted +# * simplest-path - The path to the version with the fewest edges is highlighted +# * all - Both lowest-version and simplest-path are used +highlight = "all" +# The default lint level for `default` features for crates that are members of +# the workspace that is being checked. This can be overridden by allowing/denying +# `default` on a crate-by-crate basis if desired. +workspace-default-features = "allow" +# The default lint level for `default` features for external crates that are not +# members of the workspace. This can be overridden by allowing/denying `default` +# on a crate-by-crate basis if desired. +external-default-features = "allow" +# List of crates that are allowed. Use with care! +allow = [ + #"ansi_term@0.11.0", + #{ crate = "ansi_term@0.11.0", reason = "you can specify a reason it is allowed" }, +] +# List of crates to deny +deny = [ + #"ansi_term@0.11.0", + #{ crate = "ansi_term@0.11.0", reason = "you can specify a reason it is banned" }, + # Wrapper crates can optionally be specified to allow the crate when it + # is a direct dependency of the otherwise banned crate + #{ crate = "ansi_term@0.11.0", wrappers = ["this-crate-directly-depends-on-ansi_term"] }, +] + +# List of features to allow/deny +# Each entry the name of a crate and a version range. If version is +# not specified, all versions will be matched. +#[[bans.features]] +#crate = "reqwest" +# Features to not allow +#deny = ["json"] +# Features to allow +#allow = [ +# "rustls", +# "__rustls", +# "__tls", +# "hyper-rustls", +# "rustls", +# "rustls-pemfile", +# "rustls-tls-webpki-roots", +# "tokio-rustls", +# "webpki-roots", +#] +# If true, the allowed features must exactly match the enabled feature set. If +# this is set there is no point setting `deny` +#exact = true + +# Certain crates/versions that will be skipped when doing duplicate detection. +skip = [ + "windows-sys", + #"ansi_term@0.11.0", + #{ crate = "ansi_term@0.11.0", reason = "you can specify a reason why it can't be updated/removed" }, +] +# Similarly to `skip` allows you to skip certain crates during duplicate +# detection. Unlike skip, it also includes the entire tree of transitive +# dependencies starting at the specified crate, up to a certain depth, which is +# by default infinite. +skip-tree = [ + #"ansi_term@0.11.0", # will be skipped along with _all_ of its direct and transitive dependencies + #{ crate = "ansi_term@0.11.0", depth = 20 }, +] + +# This section is considered when running `cargo deny check sources`. +# More documentation about the 'sources' section can be found here: +# https://embarkstudios.github.io/cargo-deny/checks/sources/cfg.html +[sources] +# Lint level for what to happen when a crate from a crate registry that is not +# in the allow list is encountered +unknown-registry = "warn" +# Lint level for what to happen when a crate from a git repository that is not +# in the allow list is encountered +unknown-git = "warn" +# List of URLs for allowed crate registries. Defaults to the crates.io index +# if not specified. If it is specified but empty, no registries are allowed. +allow-registry = ["https://github.com/rust-lang/crates.io-index"] +# List of URLs for allowed Git repositories +allow-git = [] + +[sources.allow-org] +# github.com organizations to allow git sources for +github = [] +# gitlab.com organizations to allow git sources for +gitlab = [] +# bitbucket.org organizations to allow git sources for +bitbucket = [] diff --git a/example_files/example_lint_cfg.README b/example_files/example_lint_cfg.README new file mode 100644 index 0000000..2289e2b --- /dev/null +++ b/example_files/example_lint_cfg.README @@ -0,0 +1,22 @@ +// This is an example and explanation of a configuration file for +// the DLS linting module +// NOTE: This file CANNOT be used as-is, since it contains in-line comments. +// If you simply want some good default parameters for your linting, it is +// better to _not specify a file at all_ and instead use the defaults provided +// in the server +// The 'example_lint_cfg.json' file contains a configuration corresponding to +// these defaults, which is a suitable place to start +// your modifications + +{ + // if set to 'false', will lint indirectly imported files as well + "direct_only": true, + // linting rules with no arguments are enabled by assigning an empty struct + "sp_brace": {}, + // linting rules with arguments are straightforward + "long_lines": { "max_length": 80 }, + // to disable a rule, either do not set it + // "nsp_unary": {} + // or assign it to null + "nsp_unary": null +} diff --git a/example_files/example_lint_cfg.json b/example_files/example_lint_cfg.json new file mode 100644 index 0000000..cfb5fee --- /dev/null +++ b/example_files/example_lint_cfg.json @@ -0,0 +1,10 @@ +{ + "direct_only": true, + "sp_brace": {}, + "sp_punct": {}, + "nsp_funpar": {}, + "nsp_inparen": {}, + "nsp_unary": {}, + "nsp_trailing": {}, + "long_lines": { "max_length": 80 } +} diff --git a/src/actions/analysis_queue.rs b/src/actions/analysis_queue.rs new file mode 100644 index 0000000..23716b5 --- /dev/null +++ b/src/actions/analysis_queue.rs @@ -0,0 +1,508 @@ +// © 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 and MIT +//! Queue of analysis to perform + +#![allow(missing_docs)] + +use std::collections::hash_map::DefaultHasher; +use std::collections::{HashMap, HashSet}; +use std::hash::{Hash, Hasher}; +use std::panic::RefUnwindSafe; +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; +use std::thread::{self, Thread}; +use std::time::SystemTime; +use crate::lint::LintCfg; +use crate::lint::LinterAnalysis; + +use itertools::{Either, Itertools}; + +use crate::actions::analysis_storage::{AnalysisStorage, ResultChannel, + TimestampedStorage, timestamp_is_newer}; +use crate::analysis::{DeviceAnalysis, IsolatedAnalysis}; +use crate::analysis::structure::objects::Import; + +use crate::concurrency::JobToken; +use crate::file_management::CanonPath; +use crate::vfs::{TextFile, Vfs}; +use crate::server::ServerToHandle; + +use log::{info, debug, trace, error}; +use crossbeam::channel; + +// Maps in-process device jobs the timestamps of their dependencies +type InFlightDeviceJobTracker = HashMap>; +type InFlightIsolatedJobTracker = HashSet; +// Queue up analysis tasks and execute them on the same thread (this is slower +// than executing in parallel, but allows us to skip indexing tasks). +pub struct AnalysisQueue { + queue: Arc>>, + // Need this so we do not queue up redundant jobs + // TODO: Allow us to cancel in-flight jobs if they are going to be + // made redundant by a later job + device_tracker: Arc>, + isolated_tracker: Arc>, + // Handle to the worker thread where we handle analysis tasks. + worker_thread: Thread, +} + +impl AnalysisQueue { + // Create a new queue and start the worker thread. + pub fn init() -> AnalysisQueue { + let queue = Arc::default(); + let device_tracker = Arc::default(); + let isolated_tracker = Arc::default(); + let worker_thread = thread::spawn({ + let queue = Arc::clone(&queue); + let device_tracker = Arc::clone(&device_tracker); + let isolated_tracker = Arc::clone(&isolated_tracker); + || AnalysisQueue::run_worker_thread(queue, + device_tracker, + isolated_tracker) + }) + .thread() + .clone(); + + AnalysisQueue { queue, worker_thread, device_tracker, isolated_tracker} + } + + pub fn enqueue_linter_job(&self, + storage: &mut AnalysisStorage, + cfg: LintCfg, + vfs: &Vfs, + file: CanonPath, + tracking_token: JobToken) { + match LinterJob::new(tracking_token, storage, cfg, vfs, file) { + Ok(newjob) => { + self.enqueue(QueuedJob::FileLinterJob(newjob)); + }, + Err(desc) => { + error!("Failed to enqueue Linter job: {}", desc); + } + } + } + + pub fn enqueue_isolated_job(&self, + storage: &mut AnalysisStorage, + vfs: &Vfs, + context: Option, + path: CanonPath, + client_path: PathBuf, + tracking_token: JobToken) { + match IsolatedAnalysisJob::new(tracking_token, + storage, + context, + vfs, + path, + client_path) { + // NOTE: An enqueued isolated job is always considered newer + // than an ongoing one, so we always queue + Ok(newjob) => { + debug!("Enqueued isolated analysis job of {}", + newjob.path.as_str()); + self.enqueue(QueuedJob::IsolatedAnalysisJob(newjob)) + }, + Err(desc) => { + error!("Failed to enqueue isolated job: {}", desc); + } + } + } + + pub fn enqueue_device_job(&self, + storage: &mut AnalysisStorage, + device: &CanonPath, + bases: HashSet, + tracking_token: JobToken) { + match DeviceAnalysisJob::new(tracking_token, storage, bases, device) { + Ok(newjob) => { + if let Some(previous_bases) = self.device_tracker + .lock().unwrap() + .get(&newjob.hash) { + let mut newer_bases = false; + for base in &newjob.bases { + // If any base is missing or newer, + // the job is ok to go + if let Some(old_base_timestamp) = + previous_bases.get(&base.stored.path) { + if !timestamp_is_newer(base.timestamp, + *old_base_timestamp) { + continue; + } + } + newer_bases = true; + break; + } + if !newer_bases { + debug!("Skipped enqueueing device analysis job of \ + {:?}, no new dependencies", device); + return; + } + } + debug!("Enqueued device analysis job of {:?}", device); + self.enqueue(QueuedJob::DeviceAnalysisJob(newjob)) + }, + Err(desc) => { + error!("Failed to create device analysis job; {}", desc); + } + } + } + + fn enqueue(&self, queuedjob: QueuedJob) { + trace!("enqueue job"); + + { + let mut queue = self.queue.lock().unwrap(); + // Remove any analysis jobs which this job obsoletes. + debug!("Pre-prune queue len: {}", queue.len()); + queue.retain(|j|j.hash() != queuedjob.hash()); + debug!("Post-prune queue len: {}", queue.len()); + queue.push(queuedjob); + } + + self.worker_thread.unpark(); + } + + fn run_worker_thread(queue: Arc>>, + device_tracker: Arc>, + isolated_tracker: Arc>) { + loop { + let job = { + let mut queue = queue.lock().unwrap(); + if queue.is_empty() { + trace!("Worker thread: Queue empty"); + None + } else { + // We peek first here, so we can make sure the job is always + // either in the queue or the flight tracker, and as such + // we avoid double job queueing + match queue.first() { + Some(QueuedJob::DeviceAnalysisJob(job)) => { + device_tracker.lock().unwrap().insert( + job.hash, + job.bases.iter().map( + |base|(base.stored.path.clone(), + base.timestamp)).collect()); + }, + Some(QueuedJob::IsolatedAnalysisJob(job)) => { + isolated_tracker.lock().unwrap().insert(job.hash); + }, + _ => (), + } + + let job = queue.remove(0); + match &job { + QueuedJob::IsolatedAnalysisJob(_) | + QueuedJob::DeviceAnalysisJob(_) | + QueuedJob::FileLinterJob(_) => + queue.push(QueuedJob::Sentinel), + _ => (), + } + + Some(job) + } + }; + + match job { + Some(QueuedJob::Terminate) => return, + Some(QueuedJob::IsolatedAnalysisJob(job)) => { + thread::spawn({ + let iso_tracker = Arc::clone(&isolated_tracker); + let hash = job.hash; + move ||{ + job.process(); + iso_tracker.lock().unwrap().remove(&hash); + }}); + }, + Some(QueuedJob::FileLinterJob(job)) => { + job.process() + }, + Some(QueuedJob::DeviceAnalysisJob(job)) => { + thread::spawn({ + let dev_tracker = Arc::clone(&device_tracker); + let hash = job.hash; + move ||{ + job.process(); + dev_tracker.lock().unwrap().remove(&hash); + }}); + }, + Some(QueuedJob::Sentinel) => { + trace!("Consumed sentinel"); + }, + None => thread::park(), + } + } + } + + pub fn has_work(&self) -> bool { + let queue_lock = self.queue.lock().unwrap(); + let device_lock = self.device_tracker.lock().unwrap(); + let isolated_lock = self.isolated_tracker.lock().unwrap(); + let has_work = !queue_lock.is_empty() + || !device_lock.is_empty() + || !isolated_lock.is_empty(); + if has_work { + debug!("Queue still has work ({:?}, {:?}, {:?})", + queue_lock, device_lock, isolated_lock); + } + has_work + } +} + +impl RefUnwindSafe for AnalysisQueue {} + +impl Drop for AnalysisQueue { + fn drop(&mut self) { + if let Ok(mut queue) = self.queue.lock() { + queue.push(QueuedJob::Terminate); + } + } +} + +#[allow(clippy::large_enum_variant)] +#[derive(Debug)] +enum QueuedJob { + IsolatedAnalysisJob(IsolatedAnalysisJob), + FileLinterJob(LinterJob), + DeviceAnalysisJob(DeviceAnalysisJob), + Sentinel, + Terminate, +} + +impl QueuedJob { + fn hash(&self) -> Option { + match self { + QueuedJob::IsolatedAnalysisJob(job) => Some(job.hash), + QueuedJob::DeviceAnalysisJob(job) => Some(job.hash), + QueuedJob::Sentinel => Some(0), + QueuedJob::FileLinterJob(job) => { Some(job.hash) }, + QueuedJob::Terminate => None, + } + } +} + +// An analysis task to be queued and executed by `AnalysisQueue`. +#[derive(Debug)] +pub struct IsolatedAnalysisJob { + path: CanonPath, + client_path: PathBuf, + timestamp: SystemTime, + report: ResultChannel, + notify: channel::Sender, + content: TextFile, + context: Option, + hash: u64, + _token: JobToken, +} + +impl IsolatedAnalysisJob { + fn new(token: JobToken, + analysis: &mut AnalysisStorage, + context: Option, + vfs: &Vfs, + path: CanonPath, + client_path: PathBuf) + -> Result { + + // TODO: Use some sort of timestamp from VFS instead of systemtime + let timestamp = SystemTime::now(); + // We make a hash from the device path + let mut hasher = DefaultHasher::new(); + Hash::hash(&path, &mut hasher); + let hash = hasher.finish(); + // TODO: error handling + let content = vfs.snapshot_file(&path)?; + Ok(IsolatedAnalysisJob { + path, + client_path, + timestamp, + report: analysis.report.clone(), + notify: analysis.notify.clone(), + hash, + context, + content, + _token: token, + }) + } + + fn process(self) { + info!("Started work on isolated analysis of {}", self.path.as_str()); + match IsolatedAnalysis::new(&self.path, + &self.client_path, + self.content) { + Ok(analysis) => { + let new_context = if analysis.is_device_file() { + Some(self.path.clone()) + } else { + self.context.clone() + }; + self.notify.send(ServerToHandle::IsolatedAnalysisDone( + self.path.clone(), + new_context, + analysis.get_import_names() + )).ok(); + self.report.send(TimestampedStorage::make_isolated_result( + self.timestamp, + analysis)).ok(); + }, + Err(e) => { + trace!("Failed to create isolated analysis: {}", e); + // TODO: perhaps collect this for reporting to server + } + } + } +} + +// An analysis task to be queued and executed by `AnalysisQueue`. +#[derive(Debug)] +pub struct DeviceAnalysisJob { + bases: Vec>, + root: IsolatedAnalysis, + import_sources: HashMap, + timestamp: SystemTime, + report: ResultChannel, + notify: channel::Sender, + hash: u64, + _token: JobToken, +} + +impl DeviceAnalysisJob { + fn new(token: JobToken, + analysis: &mut AnalysisStorage, + bases: HashSet, + root: &CanonPath) + -> Result { + info!("Creating a device analysis job of {:?}", root); + // TODO: Use some sort of timestamp from VFS instead of systemtime + let timestamp = SystemTime::now(); + // We make a hash from the sorted paths of the bases + // NOTE/TODO: Might be a need to consider compiler options too, when + // they are properly supported + let mut hasher = DefaultHasher::new(); + + let mut sorted_paths: Vec<_> = bases.iter().collect(); + sorted_paths.sort(); + for path in &sorted_paths { + Hash::hash(path, &mut hasher); + } + let hash = hasher.finish(); + + trace!("Bases are {:?}", bases.iter().collect::>()); + let root_analysis = analysis.get_isolated_analysis(root) + .ok_or_else( + ||"Failed to get root isolated analysis".to_string())?.clone(); + let (bases, missing) : (Vec>, + HashSet) = + bases.iter().map(|p|(p, analysis.isolated_analysis.get(p).cloned())) + .partition_map(|(p, a)| + match a { + Some(ia) => Either::Left(ia), + None => Either::Right((*p).clone()), + }); + trace!("Missing bases are {:?}", missing.iter() + .map(|a|a.as_str()).collect::>()); + let mut import_sources: HashMap = HashMap::default(); + trace!("Dependency map is {:?}", analysis.dependencies); + trace!("Root is {:?}", root); + for base in &bases { + trace!("wants dependencies of {:?}", base.stored.path); + let import_maps_of_path_at_context = + analysis.import_map + .get(&base.stored.path).unwrap() + .get(&Some(root.clone())).unwrap(); + import_sources.extend(import_maps_of_path_at_context.clone()); + } + + Ok(DeviceAnalysisJob { + bases, + root: root_analysis, + timestamp, + import_sources, + report: analysis.report.clone(), + notify: analysis.notify.clone(), + hash, + _token: token, + }) + } + + fn process(self) { + info!("Started work on deviceanalysis of {:?}, depending on {:?}", + self.root.path, + self.bases.iter().map(|i|&i.stored.path) + .collect::>()); + match DeviceAnalysis::new(self.root, self.bases, self.import_sources) { + Ok(analysis) => { + info!("Finished device analysis of {:?}", analysis.name); + self.notify.send(ServerToHandle::DeviceAnalysisDone( + analysis.path.clone())).ok(); + self.report.send(TimestampedStorage::make_device_result( + self.timestamp, + analysis)).ok(); + }, + // In general, an analysis shouldn't fail to be created + Err(e) => { + trace!("Failed to create device analysis: {}", e); + // TODO: perhaps collect this for reporting to server + } + } + } +} + +#[derive(Debug)] +pub struct LinterJob { + file: CanonPath, + timestamp: SystemTime, + report: ResultChannel, + notify: channel::Sender, + content: TextFile, + hash: u64, + ast: IsolatedAnalysis, + cfg: LintCfg, + _token: JobToken, +} + +impl LinterJob { + fn new(token: JobToken, + analysis: &mut AnalysisStorage, + cfg: LintCfg, + vfs: &Vfs, + device: CanonPath) + -> Result { + + // TODO: Use some sort of timestamp from VFS instead of systemtime + let timestamp = SystemTime::now(); + let mut hasher = DefaultHasher::new(); + Hash::hash(&device, &mut hasher); + let hash = hasher.finish(); + if let Some(isolated_analysis) = analysis.get_isolated_analysis(&device) { + Ok(LinterJob { + file: device.to_owned(), + timestamp, + report: analysis.report.clone(), + notify: analysis.notify.clone(), + hash, + ast: isolated_analysis.to_owned(), + cfg, + _token: token, + content: vfs.snapshot_file(&device)?, + }) + } else { + Err("Failed to get isolated analysis to trigger LinterJob".to_string()) + } + } + + fn process(self) { + debug!("Started work on isolated linting of {:?}", self.file); + match LinterAnalysis::new(&self.file, self.content, self.cfg, self.ast) { + Ok(analysis) => { + self.report.send(TimestampedStorage::make_linter_result( + self.timestamp, + analysis)).ok(); + self.notify.send(ServerToHandle::LinterDone( + self.file.clone())).ok(); + }, + Err(e) => { + trace!("Failed to create isolated linter analysis: {}", e); + } + } + } +} \ No newline at end of file diff --git a/src/actions/analysis_storage.rs b/src/actions/analysis_storage.rs new file mode 100644 index 0000000..3b9320d --- /dev/null +++ b/src/actions/analysis_storage.rs @@ -0,0 +1,741 @@ +// © 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 and MIT +//! Stores currently completed analysis. + +use log::{debug, error, trace, info}; + +use crossbeam::channel; + +use std::cmp::Ordering; +use std::collections::{HashMap, HashSet}; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; +use std::time::{Duration, SystemTime}; + +use crate::actions::progress::{DiagnosticsNotifier, + AnalysisDiagnosticsNotifier}; +use crate::analysis::scope::ContextedSymbol; +use crate::analysis::structure::objects::Import; +use crate::analysis::{IsolatedAnalysis, DeviceAnalysis, DMLError}; + +use crate::lsp_data::*; +use crate::analysis::parsing::tree::{ZeroSpan, ZeroFilePosition}; +use crate::analysis::reference::Reference; +use crate::server::{Output, ServerToHandle}; +use crate::Span; + +use crate::lint::LinterAnalysis; + +use crate::file_management::{PathResolver, CanonPath}; + +#[allow(clippy::large_enum_variant)] +#[derive(Debug)] +pub enum AnalysisResult { + Isolated(IsolatedAnalysis), + Linter(LinterAnalysis), + Device(DeviceAnalysis), +} + +impl AnalysisResult { + pub fn path(&self) -> &Path { + match self { + AnalysisResult::Isolated(analysis) => &analysis.path, + AnalysisResult::Device(analysis) => &analysis.path, + AnalysisResult::Linter(analysis) => &analysis.path, + } + } +} + +pub type ResultChannel = channel::Sender>; + +#[derive(Debug, Clone)] +pub struct TimestampedStorage { + pub timestamp: SystemTime, + pub stored: T, +} + +impl TimestampedStorage { + pub fn make_isolated_result(timestamp: SystemTime, + analysis: IsolatedAnalysis) + -> TimestampedStorage{ + TimestampedStorage { + timestamp, + stored : AnalysisResult::Isolated(analysis), + } + } + pub fn make_device_result(timestamp: SystemTime, + analysis: DeviceAnalysis) + -> TimestampedStorage{ + TimestampedStorage { + timestamp, + stored : AnalysisResult::Device(analysis), + } + } + pub fn make_linter_result(timestamp: SystemTime, + analysis: LinterAnalysis) + -> TimestampedStorage { + TimestampedStorage { + timestamp, + stored: AnalysisResult::Linter(analysis), + } + } +} + +// Maps file paths to maps of +// contexts to file paths that files at the path under that +// context might directly import +// {File : { Context : ImportedFiles}} +type AnalysisDirectDependencies = + HashMap, HashSet>>; + +// Maps paths -> contexts -> imports -> string name of resolved path +// Keeps track of how specific imports were resolved under +// specific contexts. The path indirection is used for easier +// clearing when recalculation is needed +type AnalysisImportMap = + HashMap, + HashMap>>; + +// General note, all functions on AnalysisStorage assume that incoming PathBufs +// are canonicalized +#[derive(Debug)] +pub struct AnalysisStorage { + pub notify: channel::Sender, + + pub results: channel::Receiver>, + pub report: ResultChannel, + + last_use: HashMap>, + invalidators: HashMap, + pub isolated_analysis: HashMap< + CanonPath, TimestampedStorage>, + pub device_analysis: HashMap< + CanonPath, TimestampedStorage>, + pub lint_analysis: HashMap< + CanonPath, TimestampedStorage>, + // Maps file paths to device paths that depend on them + pub device_triggers: HashMap>, + + pub dependencies: AnalysisDirectDependencies, + pub import_map: AnalysisImportMap, + + // Maps files that need to be analyzed to contexts in which + // they have been imported from + pub unresolved_dependency: HashMap>>, +} + +pub fn timestamp_is_newer(later: SystemTime, previous: SystemTime) -> bool { + previous.duration_since(later).is_err() +} + +impl AnalysisStorage { + pub fn manipulate_isolated_analysises(&mut self) -> + HashMap<&CanonPath, &mut IsolatedAnalysis> { + self.isolated_analysis.iter_mut().map( + |(p, tss)|(p, &mut tss.stored)).collect() + } + + pub fn all_isolated_analysises(&self) -> + HashMap<&CanonPath, &IsolatedAnalysis> { + self.isolated_analysis.iter().map( + |(p, tss)|(p, &tss.stored)).collect() + } + + pub fn all_device_analysises_containing_file( + &self, path: &CanonPath) -> Vec<&DeviceAnalysis>{ + self.device_triggers.get(path).map( + |triggers|triggers.iter().filter_map(|p|self.get_device_analysis(p)) + .collect()) + .unwrap_or_else(||vec![]) + } + + pub fn init(notify: channel::Sender) -> Self { + let (report, results) = channel::unbounded(); + AnalysisStorage { + notify, + results, + report, + lint_analysis: HashMap::default(), + isolated_analysis: HashMap::default(), + device_analysis: HashMap::default(), + device_triggers: HashMap::default(), + dependencies: HashMap::default(), + import_map: HashMap::default(), + unresolved_dependency: HashMap::default(), + invalidators: HashMap::default(), + last_use: HashMap::default(), + } + } + + pub fn context_symbol_at_pos<'t>(&'t self, pos: &ZeroFilePosition) + -> Option> { + let analysis = self.get_isolated_analysis( + &CanonPath::from_path_buf(pos.path())?)?; + let mut context = analysis.lookup_context_symbol(pos); + // Patch out leading 'device' context, unneeded + if let Some(ref mut ic) = context { + ic.remove_head_context(); + } + context + } + + pub fn reference_at_pos(&self, pos: &ZeroFilePosition) + -> Option<&Reference> { + let analysis = self.get_isolated_analysis( + &CanonPath::from_path_buf(pos.path())?)?; + analysis.lookup_reference(pos) + } + + pub fn has_client_file(&self, path: &Path) -> bool { + self.isolated_analysis.keys().any( + |cp|cp.as_path().ends_with(path)) + } + + pub fn has_isolated_analysis(&self, path: &CanonPath) -> bool { + self.isolated_analysis.contains_key(path) + } + + pub fn all_dependencies(&self, + path: &CanonPath, + context: Option<&CanonPath>) + -> HashSet { + let mut queue = vec![path.clone()]; + + let mut to_return = HashSet::default(); + while let Some(next) = queue.pop() { + trace!("Next to handle is {:?}", next); + if to_return.contains(&next) { + trace!("Already handled, skip"); + continue; + } + to_return.insert(next.clone()); + + if let Some(dependencies) = self.dependencies + .get(&next).and_then(|cm|cm.get(&context.cloned())) { + trace!("Extended with {:?} through {:?} at {:?}", + dependencies.iter().collect::>(), + next, context); + queue.extend( + dependencies.iter().cloned()); + } + } + debug!("Full dependencies of {} under {:?} are {:?}", + path.as_str(), context, to_return); + to_return + } + + pub fn get_file_contexts(&self, path: &CanonPath) + -> HashSet> { + if let Some(deps) = self.dependencies.get(path) { + return deps.keys().cloned().collect() + } else { + vec![None].into_iter().collect() + } + } + + /// Update all dependency info involving a specific canon path + pub fn update_dependencies(&mut self, + path: &CanonPath, + resolver: &PathResolver) { + debug!("Updating dependencies of {}", path.as_str()); + + let mut contexts: HashSet> = + if let Some(dependency_map) = + self.dependencies.get(path) { + dependency_map.keys().cloned().collect() + } else { + HashSet::default() + }; + contexts.insert(None); + + debug!("Full contexts for {:?} are {:?}", path, contexts); + + if self.get_isolated_analysis(path).map_or( + false, |a|a.is_device_file()) { + contexts.insert(Some(path.clone())); + } + + if let Some(previously_failed) = + self.unresolved_dependency.remove(path) { + contexts.extend(previously_failed); + } + + trace!("Will update for contexts {:?}", contexts); + + for context in &contexts { + // This is needed because circular dependencies might occur here + let mut updated = HashSet::default(); + self.update_dependencies_aux(path, context.as_ref(), resolver, + &mut updated); + + } + + + // Set up device triggers + let paths: HashSet = contexts.iter() + .flat_map(|c|self.all_dependencies(path, c.as_ref()) + .into_iter()) + .collect(); + + let mut target_devices: Vec = vec![]; + + if self.get_isolated_analysis(path) + .map_or(false, |a|a.is_device_file()) { + target_devices.push(path.clone()); + } else { + // Remove ourselves from the trigee list of any file + // we depend on + for trigger_path in &paths { + self.device_triggers.get_mut(trigger_path) + .map(|e|e.remove(path)); + if self.device_triggers.get(trigger_path) + .map_or(false, |e|e.is_empty()) { + self.device_triggers.remove(trigger_path); + } + } + } + + if let Some(trigger_paths) = self.device_triggers.get(path) { + target_devices.extend(trigger_paths.iter().cloned()); + } + + // If we trigger some device, things that depend on us should + // also trigger that device + + for trigger_path in &paths { + for device in &target_devices { + let entry = self.device_triggers + .entry(trigger_path.clone()).or_default(); + entry.insert((*device).clone()); + } + } + } + + fn update_dependencies_aux(&mut self, + path: &CanonPath, + context: Option<&CanonPath>, + resolver: &PathResolver, + updated: &mut HashSet) { + // TODO: This is all painfully un-optimized. We resolve many + // paths several times and we do a lot of copying of paths around + if !updated.contains(path) { + updated.insert(path.clone()); + } else { + return; + } + debug!("Updating dependencies of {} under {:?}", + path.as_str(), context); + + if !self.isolated_analysis.contains_key(path) { + trace!("Supposedly real file {} did not have an analysis", + path.as_str()); + // Clear out previous dependencies + self.dependencies.remove(path); + self.import_map.remove(path); + return; + } + + let mut next_to_recurse: HashSet = HashSet::default(); + + { + let analysis = &mut self.isolated_analysis.get_mut(path) + .unwrap().stored; + // Dependencies for this path + let all_dependencies = self.dependencies.entry(path.clone()) + .or_default(); + // Dependencies for this context + let context_dependencies = + all_dependencies.entry(context.cloned()).or_default(); + context_dependencies.clear(); + context_dependencies.insert(path.clone()); + + let (direct_dependencies, missing) = + analysis.resolve_imports(resolver, context); + + // Similarly, import map for this path and context + let all_import_maps = + self.import_map.entry(path.clone()).or_default(); + let context_import_maps = + all_import_maps.entry(context.cloned()).or_default(); + + debug!("While updating dependencies, these were not resolved {:?}", + missing); + + for (dependency, import) in direct_dependencies { + context_dependencies.insert(dependency.clone()); + context_import_maps.insert( + import, dependency.as_str().to_string()); + + // We need to request an analysis here, because this dependency + // might have just been found due to changed context + if self.isolated_analysis.contains_key(&dependency) { + next_to_recurse.insert(dependency); + } else { + debug!("-> server; analyze {}", dependency.as_str()); + self.notify.send(ServerToHandle::AnalysisRequest( + dependency.clone().to_path_buf(), + context.cloned())).ok(); + self.unresolved_dependency + .entry(dependency.clone()) + .or_default().insert(context.cloned()); + } + } + + trace!("Updated dependencies of {} under {:?} to: {:?}", + path.as_str(), context, context_dependencies); + } + + // Recurse so that analysises we depend on also have their + // dependencies correct + for dependency in &next_to_recurse { + self.update_dependencies_aux(dependency, context, + resolver, updated); + } + } + + // update dependencies based on each device file context, and None + pub fn update_all_context_dependencies(&mut self, resolver: PathResolver) { + trace!("Updating all dependencies"); + // Technically, we're overkilling here. We will double-update + // dependenies for paths which have new analysises + self.update_analysis(&resolver); + let dependencies_to_update: HashSet = + self.isolated_analysis.keys().cloned().collect(); + for file in dependencies_to_update { + self.update_dependencies(&file, &resolver); + } + } + + pub fn discard_dependant_device_analysis(&mut self, path: &Path) { + // There is probably a more rustic way to do this + let device_trigger_holder = self.device_triggers.clone(); + self.device_analysis.retain( + |k, _| !device_trigger_holder.get(k) + .unwrap().contains(&path.to_path_buf().into())); + } + + pub fn update_analysis(&mut self, resolver: &PathResolver) { + let mut device_analysis = vec![]; + let mut results_holder = vec![]; + debug!("Updating stored analysises"); + for r in self.results.try_iter() { + results_holder.push(r); + } + + // Do this in two passes, once to get new analysis, and once to + // update the dependencies + let mut dependencies_to_update: HashSet = HashSet::default(); + + for result in results_holder { + let timestamp = result.timestamp; + match result.stored { + AnalysisResult::Isolated(analysis) => { + let canon_path = analysis.path.clone(); + trace!("Handling isolated analysis on {}", + canon_path.as_str()); + if self.isolated_analysis.get(&canon_path).map_or( + true, + |prev| timestamp_is_newer(timestamp, prev.timestamp)) { + trace!("invalidators are {:?}", self.invalidators); + if self.invalidators.get(&canon_path).map_or( + true, |invalid| timestamp_is_newer(timestamp, + *invalid)) { + trace!("was new, or fresh compared to previous"); + dependencies_to_update.insert(canon_path.clone()); + self.isolated_analysis.insert(canon_path.clone(), + TimestampedStorage { + timestamp, + stored: analysis, + }); + self.last_use.insert(canon_path.clone(), + Mutex::new(SystemTime::now())); + self.update_last_use(&canon_path); + self.discard_dependant_device_analysis(&canon_path); + self.invalidators.remove(&canon_path); + } else { + trace!("was pre-emptively invalidated"); + } + } + }, + AnalysisResult::Device(_) => { + device_analysis.push(result) + }, + AnalysisResult::Linter(analysis) => { + let canon_path = analysis.path.clone(); + self.lint_analysis.insert(canon_path.clone(), + TimestampedStorage { + timestamp, + stored: analysis, + }); + }, + } + } + + for path in dependencies_to_update { + self.update_dependencies(&path, resolver); + } + + for analysisresult in device_analysis { + let timestamp = analysisresult.timestamp; + if let AnalysisResult::Device(analysis) = analysisresult.stored { + let canon_path = analysis.path.clone(); + trace!("Handling device analysis on {}", canon_path.as_str()); + if self.device_analysis.get(&canon_path).map_or( + true, |prev| timestamp_is_newer(timestamp, + prev.timestamp)) { + trace!("was new, or fresh compared to previous"); + // This should be guaranteed + let invalidators = self.all_dependencies( + &canon_path, Some(&canon_path)); + if !invalidators.iter().any( + // NOTE: This is where last-use gets updated for + // dependee analysises + |p|self.isolated_analysis.get(p).map( + |i|!timestamp_is_newer(timestamp, + i.timestamp)) + .unwrap_or(false)) { + debug!("was not invalidated by recent \ + isolated analysis"); + self.device_analysis.insert(canon_path, + TimestampedStorage { + timestamp, + stored: analysis, + }); + } + } + } else { + unreachable!("Enum variant should be device analysis"); + } + } + + trace!("Now knows about these isolated analysises: {:?}", + self.isolated_analysis.keys().collect::>()); + trace!("Now knows about these device analysises: {:?}", + self.device_analysis.keys().collect::>()); + } + + pub fn get_linter_analysis<'a>(&'a mut self, path: &Path) + -> Option<&'a LinterAnalysis> { + trace!("Looking for linter analysis of {:?}", path); + let analysis = self.lint_analysis.get( + &CanonPath::from_path_buf(path.to_path_buf())?).map( + |storage|&storage.stored); + if analysis.is_none() { + trace!("Failed to find linter analysis"); + } + analysis + } + pub fn discard_overly_old_analysis(&mut self, max_age: Duration) { + let now = SystemTime::now(); + for path in self.last_use.keys().cloned().collect::>() { + if now.duration_since(*self.last_use.get(&path) + .unwrap().lock().unwrap()) + .map_or(false, |duration|duration > max_age) { + info!("Discarded analysis of {} due to it being \ + unused for too long.", path.as_str()); + self.mark_file_dirty(&path); + } + } + } + + pub fn get_isolated_analysis<'a>(&'a self, path: &CanonPath) + -> Option<&'a IsolatedAnalysis> { + trace!("Looking for isolated analysis of {}", path.as_str()); + let analysis = self.isolated_analysis.get(path).map( + |storage|&storage.stored); + if analysis.is_none() { + trace!("Failed to find isolated analysis"); + } else { + self.update_last_use(path); + } + analysis + } + + fn update_last_use(&self, path: &CanonPath) { + if let Some(mut_lock) = self.last_use.get(path) { + let now = SystemTime::now(); + debug!("Updated last-use of {} to {:?}", path.as_str(), now); + *mut_lock.lock().unwrap() = now; + } + } + + pub fn get_device_analysis<'a>(&'a self, path: &CanonPath) + -> Option<&'a DeviceAnalysis> { + trace!("Looking for device analysis of {:?}", path); + let analysis = self.device_analysis.get(path) + .map(|storage|&storage.stored); + if analysis.is_none() { + trace!("Failed to find device analysis"); + } else { + for p in self.dependencies.get(path) + .map_or_else(||vec![], + |map|map.values().collect()) + .into_iter() + .flat_map(|set|set.iter()) + { + self.update_last_use(p); + } + } + analysis + } + + pub fn mark_file_dirty(&mut self, path: &CanonPath) { + trace!("Marked {} as dirty", path.as_str()); + self.isolated_analysis.remove(path); + self.lint_analysis.remove(path); + self.discard_dependant_device_analysis(path.as_path()); + self.invalidators.insert(path.clone(), SystemTime::now()); + self.last_use.remove(path); + } + + pub fn has_dependencies(&mut self, path: &CanonPath) -> bool { + for path in self.all_dependencies(path, Some(path)) { + if !self.isolated_analysis.contains_key(&path) { + return false; + } + } + true + } + + pub fn device_newer_than_dependencies(&self, path: &CanonPath) -> bool { + debug!("Checking if {:?} needs a newer analysis", path); + if let Some(device_timestamp) = self.device_analysis.get(path) + .map(|device|device.timestamp) + { + debug!("Timestamp is {:?}", device_timestamp); + for dependee_path in self.all_dependencies(path, Some(path)) { + // NOTE: This means that calling this function with a missing + // isolated analysis will not tell you the device needs to be + // remade, which is correct (later adding the missing analysis + // will then have a newer timestamp) + if let Some(dependee_timestamp) = self.isolated_analysis + .get(&dependee_path) + .map(|dependee|dependee.timestamp) { + if timestamp_is_newer(dependee_timestamp, + device_timestamp) { + debug!("Outdated by {:?}", dependee_timestamp); + return false; + } + } + } + true + } else { + debug!("No analysis, needs analysis"); + false + } + } + + pub fn report_errors(&mut self, path: &CanonPath, output: &O) { + debug!("Reporting all errors for {:?}", path); + // By this being a hashset, we will not double-report any errors + let mut dmlerrors:HashMap> + = HashMap::default(); + let all_files: HashSet = + self.get_file_contexts(path).iter().flat_map( + |c|self.all_dependencies(path, c.as_ref()) + .into_iter()).collect(); + for file in all_files { + if let Some((file, errors)) + = self.gather_local_errors(&file) { + dmlerrors.entry(file) + .or_default() + .extend(errors.into_iter()); + } + dmlerrors.entry(file.to_path_buf()) + .or_default() + .extend(self + .gather_linter_errors(&file).into_iter()); + } + for (file, errors) in self.gather_device_errors(path) { + dmlerrors.entry(file.clone()) + .or_default() + .extend(errors.into_iter()); + if !self.has_client_file(&PathBuf::from("dml-builtins.dml")) { + dmlerrors.get_mut(&file).unwrap().insert( + DMLError { + span: ZeroSpan::invalid(&file), + description: "Could not find required builtin \ + file 'dml-builtins.dml'".to_string(), + related: vec![], + severity: Some(DiagnosticSeverity::ERROR), + }); + } + } + + let notifier = AnalysisDiagnosticsNotifier::new("indexing".to_string(), + output.clone()); + notifier.notify_begin_diagnostics(); + for (file, errors) in dmlerrors { + debug!("Reporting errors for {:?}", file); + let mut sorted_errors: Vec = errors.into_iter().collect(); + // Sort by line + sorted_errors.sort_unstable_by( + |e1, e2|if e1.span.range > e2.span.range { + Ordering::Greater + } else { + Ordering::Less + }); + match parse_uri(file.to_str().unwrap()) { + Ok(url) => notifier.notify_publish_diagnostics( + PublishDiagnosticsParams::new( + url, + sorted_errors.iter() + .map(DMLError::to_diagnostic).collect(), + None)), + // The Url crate does not report interesting errors + Err(_) => error!("Could not convert {:?} to Url", file), + } + } + notifier.notify_end_diagnostics(); + } + + pub fn gather_linter_errors(&mut self, + path: &CanonPath) -> Vec { + if let Some(linter_analysis) = self.get_linter_analysis(path) { + linter_analysis.errors.clone() + } else { + vec![] + } + } + + pub fn gather_local_errors(&self, path: &CanonPath) + -> Option<(PathBuf, Vec)> { + + self.get_isolated_analysis(path) + .map(|a|(a.clientpath.clone(), a.errors.clone())) + } + + pub fn gather_device_errors(&self, path: &CanonPath) + -> HashMap> { + self.get_device_analysis(path).map_or(HashMap::default(), + |a|a.errors.clone()) + } + + pub fn errors(&mut self, span: &Span) -> Vec { + trace!("Reporting errors at {:?} for {:?}", span.range, span.file); + let real_file = if let Some(file) = CanonPath::from_path_buf( + span.path()) { + file + } else { + error!("Could not resolve {:?} to point to a real file", span); + return vec![]; + }; + if let Some(isolated_analysis) = + self.get_isolated_analysis(&real_file) { + // Obtain any error which at least partially overlaps our span + let mut errors = vec![]; + for error in &isolated_analysis.errors { + if error.span.range.overlaps(span.range) { + errors.push(error.clone()) + } + } + errors + } else { + trace!("lacked analysis"); + vec![] + } + } +} diff --git a/src/actions/hover.rs b/src/actions/hover.rs new file mode 100644 index 0000000..f1a27fc --- /dev/null +++ b/src/actions/hover.rs @@ -0,0 +1,29 @@ +// © 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 and MIT +use log::*; + +use crate::span::{Range, ZeroIndexed}; +use serde::{Deserialize, Serialize}; + +use crate::actions::InitActionContext; +use crate::lsp_data::*; +use crate::server::ResponseError; + +#[derive(Debug, Deserialize, Serialize, PartialEq, Eq)] +pub struct Tooltip { + pub contents: Vec, + pub range: Range, +} + +// Build a hover tooltip +pub fn tooltip( + ctx: &mut InitActionContext, + params: &TextDocumentPositionParams, +) -> Result { + let hover_file_path = parse_file_path!(¶ms.text_document.uri, "hover")?; + let hover_span = ctx.convert_pos_to_span(hover_file_path, params.position); + // TODO: sort out how to handle hovers, and what info they should provide + let contents = vec![]; + debug!("tooltip: contents.len: {}", contents.len()); + Ok(Tooltip { contents, range: hover_span.range }) +} diff --git a/src/actions/mod.rs b/src/actions/mod.rs new file mode 100644 index 0000000..d6a4a27 --- /dev/null +++ b/src/actions/mod.rs @@ -0,0 +1,759 @@ +// © 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 and MIT +//! Actions that the DLS can perform: responding to requests, watching files, +//! etc. + +use log::{debug, trace, error, warn}; +use thiserror::Error; +use crossbeam::channel; +use serde::Deserialize; +use serde_json::json; + +use std::collections::{HashMap, HashSet}; +use std::path::{Path, PathBuf}; +use std::fs; +use std::sync::atomic::AtomicBool; +use std::sync::{Arc, Mutex}; + +use crate::actions::analysis_storage::AnalysisStorage; +use crate::actions::analysis_queue::AnalysisQueue; +use crate::actions::progress::{AnalysisProgressNotifier, ProgressNotifier}; +use crate::analysis::structure::expressions::Expression; +use crate::concurrency::{Jobs, ConcurrentJob}; +use crate::config::Config; +use crate::file_management::{PathResolver, CanonPath}; +use crate::lint::{LintCfg, maybe_parse_lint_cfg}; +use crate::lsp_data; +use crate::lsp_data::*; +use crate::server::{Output, ServerToHandle, error_message}; +use crate::Span; +use crate::span; +use crate::span::{ZeroIndexed, FilePosition}; +use crate::vfs::Vfs; + +// Define macros before submodules +macro_rules! parse_file_path { + ($uri: expr, $log_name: expr) => { + ignore_non_file_uri!(parse_file_path($uri), $uri, $log_name) + }; +} + +// TODO: Support non-`file` URI schemes in VFS. We're currently ignoring them because +// we don't want to crash the DLS in case a client opens a file under different URI scheme +// like with git:/ or perforce:/ (Probably even http:/? We currently don't support remote schemes). +macro_rules! ignore_non_file_uri { + ($expr: expr, $uri: expr, $log_name: expr) => { + $expr.map_err(|_| { + trace!("{}: Non-`file` URI scheme, ignoring: {:?}", $log_name, $uri); + }) + }; +} + +pub mod analysis_storage; +pub mod analysis_queue; +pub mod hover; +pub mod notifications; +pub mod requests; +pub mod progress; +pub mod work_pool; + +/// Persistent context shared across all requests and notifications. +pub enum ActionContext { + /// Context after server initialization. + Init(InitActionContext), + /// Context before initialization. + Uninit(UninitActionContext), +} + +#[derive(Error, Debug)] +#[error("Initialization error")] +pub struct InitError; + +impl From<()> for InitError { + fn from(_: ()) -> InitError { + InitError {} + } +} + +impl ActionContext { + /// Construct a new, uninitialized context. + pub fn new( + vfs: Arc, + config: Arc>, + notify: channel::Sender, + ) -> ActionContext { + ActionContext::Uninit(UninitActionContext::new( + Arc::new(Mutex::new(AnalysisStorage::init(notify))), + vfs, config)) + } + + /// Initialize this context, returns `Err(())` if it has already been initialized. + pub fn init( + &mut self, + init_options: InitializationOptions, + client_capabilities: lsp_data::ClientCapabilities, + out: O, + ) -> Result<(), InitError> { + let ctx = match *self { + ActionContext::Uninit(ref uninit) => { + // This means other references to the config will mismatch if + // we update it, but I am fairly sure they do not exist + let new_config = init_options.settings.as_ref() + .map(|settings|Arc::new(Mutex::new(settings.dml.clone()))) + .unwrap_or(Arc::clone(&uninit.config)); + + let mut ctx = InitActionContext::new( + Arc::clone(&uninit.analysis), + Arc::clone(&uninit.vfs), + new_config, + client_capabilities, + uninit.pid, + init_options.cmd_run, + ); + ctx.init(init_options, out); + ctx + } + ActionContext::Init(_) => return Err(().into()), + }; + trace!("Inited context has {:?} as config", + ctx.config.lock().unwrap()); + *self = ActionContext::Init(ctx); + + Ok(()) + } + + /// Returns an initialiased wrapped context, + /// or `Err(())` if not initialised. + pub fn inited(&self) -> Result { + match *self { + ActionContext::Uninit(_) => Err(().into()), + ActionContext::Init(ref ctx) => Ok(ctx.clone()), + } + } + + pub fn pid(&self) -> u32 { + match self { + ActionContext::Uninit(ctx) => ctx.pid, + ActionContext::Init(ctx) => ctx.pid, + } + } +} + +#[derive(Clone, Debug)] +pub struct CompilationDefine { + pub name: String, + pub expression: Expression, +} + + +#[derive(Clone, Debug, Default)] +pub struct CompilationInfo { + pub extra_defines: Vec, + pub include_paths: HashSet, +} + +pub type CompilationInfoStorage = HashMap; + +/// Persistent context shared across all requests and actions after the DLS has +/// been initialized. +// NOTE: This is sometimes cloned before being passed to a handler +// (not concurrent), so make sure shared info is behind Arcs, and that no overly +// large data structures are stored. +#[derive(Clone)] +pub struct InitActionContext { + pub analysis: Arc>, + vfs: Arc, + // Queues analysis jobs so that we don't over-use the CPU. + analysis_queue: Arc, + current_notifier: Arc>>, + + // Set to true when a potentially mutating request is received. Set to false + // if a change arrives. We can thus tell if the DLS has been quiescent while + // waiting to mutate the client state. + pub quiescent: Arc, + + // the root workspaces + pub workspace_roots: Arc>>, + + // directly opened files + pub direct_opens: Arc>>, + + pub compilation_info: Arc>, + + prev_changes: Arc>>, + + pub config: Arc>, + pub lint_config: Arc>, + jobs: Arc>, + pub client_capabilities: Arc, + pub has_notified_missing_builtins: bool, + /// Whether the server is performing cleanup (after having received + /// 'shutdown' request), just before final 'exit' request. + pub shut_down: Arc, + pub pid: u32, +} + +/// Persistent context shared across all requests and actions before the DLS has +/// been initialized. +pub struct UninitActionContext { + analysis: Arc>, + vfs: Arc, + config: Arc>, + pid: u32, +} + +impl UninitActionContext { + fn new( + analysis: Arc>, + vfs: Arc, + config: Arc>, + ) -> UninitActionContext { + UninitActionContext { analysis, vfs, config, pid: ::std::process::id() } + } +} + +impl InitActionContext { + fn new( + analysis: Arc>, + vfs: Arc, + config: Arc>, + client_capabilities: lsp_data::ClientCapabilities, + pid: u32, + _client_supports_cmd_run: bool, + ) -> InitActionContext { + let lint_config = Arc::new(Mutex::new( + config.lock().unwrap().lint_cfg_path.clone() + .and_then(maybe_parse_lint_cfg) + .unwrap_or_default())); + InitActionContext { + vfs, + analysis, + analysis_queue: Arc::new(AnalysisQueue::init()), + current_notifier: Arc::default(), + config, + lint_config, + jobs: Arc::default(), + direct_opens: Arc::default(), + quiescent: Arc::new(AtomicBool::new(false)), + prev_changes: Arc::default(), + client_capabilities: Arc::new(client_capabilities), + has_notified_missing_builtins: false, + //client_supports_cmd_run, + shut_down: Arc::new(AtomicBool::new(false)), + pid, + workspace_roots: Arc::default(), + compilation_info: Arc::default(), + } + } + + fn add_direct_open(&mut self, path: PathBuf) { + let canon_path: CanonPath = path.into(); + self.direct_opens.lock().unwrap().insert(canon_path); + } + + fn remove_direct_open(&mut self, path: PathBuf) { + let canon_path: CanonPath = path.into(); + if !self.direct_opens.lock().unwrap().remove(&canon_path) { + debug!("Tried to remove a directly opened file ({:?}) \ + that wasnt tracked", canon_path); + } + } + + fn init(&mut self, + _init_options: InitializationOptions, + out: O) { + self.update_compilation_info(&out) + } + + pub fn update_workspaces(&mut self, + mut add: Vec, + remove: Vec) { + if let Ok(mut workspaces) = self.workspace_roots.lock() { + workspaces.retain(|workspace| + remove.iter().all(|rem|rem != workspace)); + workspaces.append(&mut add); + } + } + + fn update_linter_config(&mut self, _out: &O) { + trace!("Updating linter config"); + if let Ok(config) = self.config.lock() { + *self.lint_config.lock().unwrap() = + config.lint_cfg_path.clone() + .and_then(maybe_parse_lint_cfg) + .unwrap_or_default(); + } + } + + pub fn update_compilation_info(&mut self, out: &O) { + trace!("Updating compile info"); + if let Ok(config) = self.config.lock() { + if let Some(compile_info) = &config.compile_info_path { + if let Some(canon_path) = CanonPath::from_path_buf( + compile_info.clone()) { + let workspaces = self.workspace_roots.lock().unwrap(); + if !workspaces.is_empty() && + workspaces.iter().any( + |root|parse_file_path!(&root.uri, "workspace") + .map_or(false, |p|canon_path.as_path() + .starts_with(p))) { + crate::server::warning_message( + out, + "Compilation info file is not under \ + any workspace root, might be configured \ + for a different workspace.".to_string()); + } + } + match self.compilation_info_from_file(compile_info) { + Ok(compilation_info) => { + trace!("Updated to {:?}", compilation_info); + { + let mut ci = self.compilation_info.lock().unwrap(); + *ci = compilation_info; + } + self.analysis.lock().unwrap() + .update_all_context_dependencies( + self.construct_resolver()); + }, + Err(e) => { + error!("Failed to update compilation info: {}", e); + error_message( + out, + format!("Could not update compilation info: {}", + e)); + } + } + } else { + trace!("No compile info path"); + } + } else { + trace!("Failed to lock config"); + } + } + + pub fn compilation_info_from_file(&self, path: &PathBuf) -> + Result { + debug!("Reading compilation info from {:?}", + path); + let file_content = fs::read_to_string(path).map_err( + |e|e.to_string())?; + trace!("Content is {:?}", file_content); + #[allow(dead_code)] + #[derive(Deserialize)] + struct FileInfo { + dmlc_flags: Vec, + includes: Vec, + } + type CompileCommands = HashMap; + let json_structure: CompileCommands = + serde_json::from_str(&file_content).map_err(|e|e.to_string())?; + let mut new_compinfo = CompilationInfoStorage::default(); + for (file, file_info) in json_structure { + // This is sanity, by design all files in this file should be + // .dml + if let Some(extension) = file.extension() { + if extension == "dml" { + let FileInfo { + includes, .. + } = file_info; + if let Some(canon_path) = CanonPath::from_path_buf(file) + { + let compentry = new_compinfo.entry( + canon_path).or_insert(CompilationInfo { + extra_defines: vec![], + include_paths : HashSet::default(), + }); + // TODO: For now, ignore flags since we have no + // means to pass them to device analysis anyway + compentry.include_paths + .extend(includes.into_iter()); + } + } else { + warn!( + "File in compile information file is not .dml; \ + {:?}", + file + ); + } + } + } + Ok(new_compinfo) + } + + pub fn update_analysis(&mut self) { + self.analysis.lock().unwrap() + .update_analysis(&self.construct_resolver()); + } + + pub fn trigger_device_analysis(&mut self, + file: &Path, + out: &O) { + let canon_path: CanonPath = file.to_path_buf().into(); + debug!("triggering devices dependant on {}", canon_path.as_str()); + self.update_analysis(); + let maybe_triggers = self.analysis.lock().unwrap().device_triggers + .get(&canon_path).cloned(); + trace!("should trigger: {:?}", maybe_triggers); + if let Some(triggers) = maybe_triggers { + for trigger in triggers { + debug!("Wants to trigger {}", trigger.as_str()); + let ready = { + let mut analysis = self.analysis.lock().unwrap(); + let has_dependencies = analysis.has_dependencies(&trigger) + && analysis.get_isolated_analysis(&trigger).unwrap() + .is_device_file(); + // Skip triggering if the device cannot be outdated, + // i.e. it's newer than all it's dependencies + let not_outdated = analysis.device_newer_than_dependencies( + &trigger); + has_dependencies && !not_outdated + }; + if ready { + debug!("Triggered device analysis {}", trigger.as_str()); + self.device_analyze(&trigger, out); + } + } + } + } + + // Called when config might have changed, re-update include paths + // and similar + pub fn maybe_changed_config(&mut self, out: &O) { + trace!("Compilation info might have changed"); + self.update_compilation_info(out); + self.update_linter_config(out); + } + + // Call before adding new analysis + pub fn maybe_start_progress(&mut self, out: &O) { + + let mut notifier = self.current_notifier.lock().unwrap(); + + if notifier.is_none() { + debug!("started progress status"); + let new_notifier = AnalysisProgressNotifier::new( + "Analysing".to_string(), out.clone()); + *notifier = Some(new_notifier.id()); + new_notifier.notify_begin_progress(); + } + } + pub fn maybe_end_progress(&mut self, out: &O) { + if !self.analysis_queue.has_work() { + // Need the scope here to succesfully drop the guard lock before + // going into maybe_warn_missing_builtins below + let lock_id = { self.current_notifier.lock().unwrap().clone() }; + if let Some(id) = lock_id { + debug!("ended progress status"); + let notifier = AnalysisProgressNotifier::new_with_id( + id, + "Analysing".to_string(), + out.clone()); + notifier.notify_end_progress(); + self.maybe_warn_missing_builtins(out); + } + } + } + + fn maybe_warn_missing_builtins(&mut self, out: &O) { + if !self.has_notified_missing_builtins && + !self.analysis.lock().unwrap().has_client_file( + &PathBuf::from("dml-builtins.dml")) { + self.has_notified_missing_builtins = true; + crate::server::warning_message( + out, + "Unable to find dml-builtins, various essential \ + built-in templates, methods, and paramters will \ + be unavailable and semantic analysis is likely \ + to produce errors as a result".to_string()); + } + } + + pub fn construct_resolver(&self) -> PathResolver { + trace!("About to construct resolver"); + let mut toret: PathResolver = + self.client_capabilities.root.clone().into(); + toret.add_paths(self.workspace_roots.lock().unwrap() + .iter().map(|w|parse_file_path!(&w.uri, "workspace") + .unwrap())); + toret.set_include_paths(&self.compilation_info.lock().unwrap().iter() + .map(|(r, info)|(r.clone(), + info.include_paths.clone() + .into_iter().collect())) + .collect()); + trace!("Constructed resolver: {:?}", toret); + toret + } + + pub fn isolated_analyze(&mut self, + client_path: &Path, + context: Option, + out: &O) { + debug!("Wants isolated analysis of {:?}{}", + client_path, + context.as_ref().map(|s|format!(" under context {}", s.as_str())) + .unwrap_or_default()); + let path = if let Some(p) = + self.construct_resolver() + .resolve_with_maybe_context(client_path, context.as_ref()) { + p + } else { + debug!("Could not canonicalize client path {:?}", client_path); + return; + }; + if self.analysis.lock().unwrap().has_isolated_analysis(&path) { + debug!("Was already analyzed"); + return; + } + self.maybe_start_progress(out); + let (job, token) = ConcurrentJob::new(); + self.add_job(job); + + self.analysis_queue.enqueue_isolated_job( + &mut self.analysis.lock().unwrap(), + &self.vfs, context, path, client_path.to_path_buf(), token); + } + + fn device_analyze(&mut self, device: &CanonPath, out: &O) { + debug!("Wants device analysis of {:?}", device); + self.maybe_start_progress(out); + let (job, token) = ConcurrentJob::new(); + self.add_job(job); + let locked_analysis = &mut self.analysis.lock().unwrap(); + let dependencies = locked_analysis.all_dependencies(device, + Some(device)); + self.analysis_queue.enqueue_device_job( + locked_analysis, + device, + dependencies, + token); + } + + pub fn maybe_trigger_lint_analysis(&mut self, + file: &Path, + out: &O) { + if !self.config.lock().unwrap().linting_enabled { + return; + } + let lint_config = self.lint_config.lock().unwrap().to_owned(); + if lint_config.direct_only { + let canon_path: CanonPath = file.to_path_buf().into(); + if !self.direct_opens.lock().unwrap().contains(&canon_path) { + return; + } + } + debug!("Triggering linting analysis of {:?}", file); + self.lint_analyze(file, + None, + lint_config, + out); + } + + fn lint_analyze(&mut self, + file: &Path, + context: Option, + cfg: LintCfg, + out: &O) { + debug!("Wants to lint {:?}", file); + self.maybe_start_progress(out); + let path = if let Some(p) = + self.construct_resolver() + .resolve_with_maybe_context(file, context.as_ref()) { + p + } else { + debug!("Could not canonicalize client path {:?}", file); + return; + }; + let (job, token) = ConcurrentJob::new(); + self.add_job(job); + + self.analysis_queue.enqueue_linter_job( + &mut self.analysis.lock().unwrap(), + cfg, + &self.vfs, path, token); + } + + pub fn add_job(&self, job: ConcurrentJob) { + self.jobs.lock().unwrap().add(job); + } + + pub fn wait_for_concurrent_jobs(&self) { + self.jobs.lock().unwrap().wait_for_all(); + } + + /// See docs on VersionOrdering + fn check_change_version(&self, file_path: &Path, + version_num: i32) -> VersionOrdering { + let file_path = file_path.to_owned(); + let mut prev_changes = self.prev_changes.lock().unwrap(); + + if prev_changes.contains_key(&file_path) { + let prev_version = prev_changes[&file_path]; + if version_num <= prev_version { + debug!( + "Out of order or duplicate change {:?}, prev: {}, current: {}", + file_path, prev_version, version_num, + ); + + if version_num == prev_version { + return VersionOrdering::Duplicate; + } else { + return VersionOrdering::OutOfOrder; + } + } + } + + prev_changes.insert(file_path, version_num); + VersionOrdering::Ok + } + + fn reset_change_version(&self, file_path: &Path) { + let file_path = file_path.to_owned(); + let mut prev_changes = self.prev_changes.lock().unwrap(); + prev_changes.remove(&file_path); + } + + fn text_doc_pos_to_pos(&self, + params: &TextDocumentPositionParams, + context: &str) + -> Option> { + let file_path = parse_file_path!( + ¶ms.text_document.uri, context) + .ok()?; + // run this through pos_to_span once to get the word range, then return + // the front of it + Some(self.convert_pos_to_span(file_path, params.position) + .start_position()) + } + + fn convert_pos_to_span(&self, file_path: PathBuf, pos: Position) -> Span { + trace!("convert_pos_to_span: {:?} {:?}", file_path, pos); + + let pos = ls_util::position_to_dls(pos); + let line = self.vfs.load_line(&file_path, pos.row).unwrap(); + trace!("line: `{}`", line); + + let (start, end) = find_word_at_pos(&line, pos.col); + trace!("start: {}, end: {}", start.0, end.0); + + Span::from_positions( + span::Position::new(pos.row, start), + span::Position::new(pos.row, end), + file_path, + ) + } +} + +/// Some notifications come with sequence numbers, we check that these are in +/// order. However, clients might be buggy about sequence numbers so we do cope +/// with them being wrong. +/// +/// This enum defines the state of sequence numbers. +#[derive(Eq, PartialEq, Debug, Clone, Copy)] +pub enum VersionOrdering { + /// Sequence number is in order (note that we don't currently check that + /// sequence numbers are sequential, but we probably should). + Ok, + /// This indicates the client sent us multiple copies of the same notification + /// and some should be ignored. + Duplicate, + /// Just plain wrong sequence number. No obvious way for us to recover. + OutOfOrder, +} + +/// Represents a text cursor between characters, pointing at the next character +/// in the buffer. +type Column = span::Column; + +/// Returns a text cursor range for a found word inside `line` at which `pos` +/// text cursor points to. Resulting type represents a (`start`, `end`) range +/// between `start` and `end` cursors. +/// For example (4, 4) means an empty selection starting after first 4 characters. +fn find_word_at_pos(line: &str, pos: Column) -> (Column, Column) { + let col = pos.0 as usize; + let is_ident_char = |c: char| c.is_alphanumeric() || c == '_'; + + let start = line + .chars() + .enumerate() + .take(col) + .filter(|&(_, c)| !is_ident_char(c)) + .last() + .map(|(i, _)| i + 1) + .unwrap_or(0) as u32; + + #[allow(clippy::filter_next)] + let end = line + .chars() + .enumerate() + .skip(col) + .filter(|&(_, c)| !is_ident_char(c)) + .next() + .map(|(i, _)| i) + .unwrap_or(col) as u32; + + (span::Column::new_zero_indexed(start), span::Column::new_zero_indexed(end)) +} + +// /// Client file-watching request / filtering logic +pub struct FileWatch { + file_path: PathBuf, +} + +impl FileWatch { + /// Construct a new `FileWatch`. + pub fn new(ctx: &InitActionContext) -> Option { + match ctx.config.lock() { + Ok(config) => { + config.compile_info_path.as_ref().map( + |c| FileWatch { + file_path: c.clone() + }) + }, + Err(e) => { + error!("Unable to access configuration: {:?}", e); + None + } + } + } + + /// Returns if a file change is relevant to the files we + /// actually wanted to watch + /// Implementation note: This is expected to be called a + /// large number of times in a loop so should be fast / avoid allocation. + #[inline] + fn relevant_change_kind(&self, change_uri: &Uri, + _kind: FileChangeType) -> bool { + let path = change_uri.as_str(); + self.file_path.to_str().map_or(false, |fp|fp == path) + } + + #[inline] + pub fn is_relevant(&self, change: &FileEvent) -> bool { + self.relevant_change_kind(&change.uri, change.typ) + } + + #[inline] + pub fn is_relevant_save_doc(&self, did_save: &DidSaveTextDocumentParams) + -> bool { + self.relevant_change_kind(&did_save.text_document.uri, + FileChangeType::CHANGED) + } + + /// Returns json config for desired file watches + pub fn watchers_config(&self) -> serde_json::Value { + fn watcher(pat: String) -> FileSystemWatcher { + FileSystemWatcher { glob_pattern: GlobPattern::String(pat), + kind: None } + } + fn _watcher_with_kind(pat: String, kind: WatchKind) + -> FileSystemWatcher { + FileSystemWatcher { glob_pattern: GlobPattern::String(pat), + kind: Some(kind) } + } + + let watchers = vec![watcher( + self.file_path.to_string_lossy().to_string())]; + + json!({ "watchers": watchers }) + } +} diff --git a/src/actions/notifications.rs b/src/actions/notifications.rs new file mode 100644 index 0000000..96656bf --- /dev/null +++ b/src/actions/notifications.rs @@ -0,0 +1,238 @@ +// © 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 and MIT +//! One-way notifications that the DLS receives from the client. + +use crate::actions::{FileWatch, InitActionContext, VersionOrdering}; +use crate::server::message::Request; +use crate::span::{Span}; +use crate::vfs::{Change, VfsSpan}; +use crate::lsp_data::*; + +use log::{trace, warn}; + +use lsp_types::notification::ShowMessage; +use lsp_types::request::RegisterCapability; + +use std::sync::atomic::Ordering; +use std::thread; + +pub use crate::lsp_data::notification::{ + Cancel, DidChangeConfiguration, + DidChangeTextDocument, DidChangeWatchedFiles, + DidChangeWorkspaceFolders, + DidOpenTextDocument, DidCloseTextDocument, DidSaveTextDocument, Initialized, +}; + +use crate::server::{BlockingNotificationAction, Notification, + Output, ResponseError}; + +impl BlockingNotificationAction for Initialized { + // Respond to the `initialized` notification. + fn handle( + _params: Self::Params, + ctx: &mut InitActionContext, + out: O, + ) -> Result<(), ResponseError> { + // TODO: register any dynamic capabilities + + // Register files we watch for changes based on config + const WATCH_ID: &str = "dls-watch"; + let id = out.provide_id(); + let reg_params = RegistrationParams { + registrations: vec![Registration { + id: WATCH_ID.to_owned(), + method: + ::METHOD.to_owned(), + register_options: FileWatch::new(ctx).map( + |fw|fw.watchers_config()), + }], + }; + + let request = Request::::new(id, reg_params); + out.request(request); + Ok(()) + } +} + +impl BlockingNotificationAction for DidOpenTextDocument { + fn handle( + params: Self::Params, + ctx: &mut InitActionContext, + out: O, + ) -> Result<(), ResponseError> { + trace!("on_open: {:?}", params.text_document.uri); + let file_path = parse_file_path!(¶ms.text_document.uri, "on_open")?; + ctx.reset_change_version(&file_path); + ctx.vfs.set_file(&file_path, ¶ms.text_document.text); + ctx.add_direct_open(file_path.to_path_buf()); + if !ctx.config.lock().unwrap().analyse_on_save { + ctx.isolated_analyze(&file_path, None, &out); + } + Ok(()) + } +} + +impl BlockingNotificationAction for DidCloseTextDocument { + fn handle( + params: Self::Params, + ctx: &mut InitActionContext, + _out: O, + ) -> Result<(), ResponseError> { + trace!("on_close: {:?}", params.text_document.uri); + let file_path = parse_file_path!(¶ms.text_document.uri, "on_close")?; + ctx.remove_direct_open(file_path.to_path_buf()); + Ok(()) + } +} + +impl BlockingNotificationAction for DidChangeTextDocument { + fn handle( + params: Self::Params, + ctx: &mut InitActionContext, + out: O, + ) -> Result<(), ResponseError> { + trace!("on_change: {:?}, thread: {:?}", params, thread::current().id()); + if params.content_changes.is_empty() { + return Ok(()); + } + + ctx.quiescent.store(false, Ordering::SeqCst); + let file_path = parse_file_path!( + ¶ms.text_document.uri, "on_change")?; + let version_num = params.text_document.version; + + match ctx.check_change_version(&file_path, version_num) { + VersionOrdering::Ok => {} + VersionOrdering::Duplicate => return Ok(()), + VersionOrdering::OutOfOrder => { + out.notify(Notification::::new(ShowMessageParams { + typ: MessageType::WARNING, + message: format!("Out of order change in {:?}", file_path), + })); + return Ok(()); + } + } + + let changes: Vec = params + .content_changes + .iter() + .map(|i| { + if let Some(range) = i.range { + let range = ls_util::range_to_dls(range); + Change::ReplaceText { + // LSP sends UTF-16 code units based offsets and length + span: VfsSpan::from_utf16( + Span::from_range(range, file_path.clone()), + i.range_length.map(u64::from), + ), + text: i.text.clone(), + } + } else { + Change::AddFile { file: file_path.clone(), text: i.text.clone() } + } + }) + .collect(); + ctx.vfs.on_changes(&changes).expect("error committing to VFS"); + ctx.analysis.lock().unwrap() + .mark_file_dirty(&file_path.to_path_buf().into()); + + if !ctx.config.lock().unwrap().analyse_on_save { + ctx.isolated_analyze(&file_path, None, &out); + } + Ok(()) + } +} + +impl BlockingNotificationAction for Cancel { + fn handle( + _params: CancelParams, + _ctx: &mut InitActionContext, + _out: O, + ) -> Result<(), ResponseError> { + // Nothing to do. + Ok(()) + } +} + +impl BlockingNotificationAction for DidChangeConfiguration { + fn handle( + params: DidChangeConfigurationParams, + ctx: &mut InitActionContext, + out: O, + ) -> Result<(), ResponseError> { + trace!("config change: {:?}", params.settings); + use std::collections::HashMap; + let mut dups = HashMap::new(); + let mut unknowns = vec![]; + let mut deprecated = vec![]; + let settings = ChangeConfigSettings::try_deserialize( + ¶ms.settings, + &mut dups, + &mut unknowns, + &mut deprecated, + ); + crate::server::maybe_notify_unknown_configs(&out, &unknowns); + crate::server::maybe_notify_deprecated_configs(&out, &deprecated); + crate::server::maybe_notify_duplicated_configs(&out, &dups); + + let new_config = match settings { + Ok(value) => value.dml, + Err(err) => { + warn!("Received unactionable config: {:?} (error: {:?})", params.settings, err); + return Err(().into()); + } + }; + + ctx.config.lock().unwrap().update(new_config); + ctx.maybe_changed_config(&out); + + Ok(()) + } +} + +impl BlockingNotificationAction for DidSaveTextDocument { + fn handle( + params: DidSaveTextDocumentParams, + ctx: &mut InitActionContext, + out: O, + ) -> Result<(), ResponseError> { + let file_path = parse_file_path!(¶ms.text_document.uri, "on_save")?; + + ctx.vfs.file_saved(&file_path).unwrap(); + + if ctx.config.lock().unwrap().analyse_on_save { + ctx.isolated_analyze(&file_path, None, &out); + } + + Ok(()) + } +} + +impl BlockingNotificationAction for DidChangeWatchedFiles { + fn handle( + params: DidChangeWatchedFilesParams, + ctx: &mut InitActionContext, + out: O, + ) -> Result<(), ResponseError> { + if let Some(file_watch) = FileWatch::new(ctx) { + if params.changes.iter().any(|c| file_watch.is_relevant(c)) { + ctx.maybe_changed_config(&out); + } + } + Ok(()) + } +} + +impl BlockingNotificationAction for DidChangeWorkspaceFolders { + // Respond to the `initialized` notification. + fn handle( + params: DidChangeWorkspaceFoldersParams, + ctx: &mut InitActionContext, + _out: O, + ) -> Result<(), ResponseError> { + let added = params.event.added; + let removed = params.event.removed; + ctx.update_workspaces(added, removed); + Ok(()) + } +} diff --git a/src/actions/progress.rs b/src/actions/progress.rs new file mode 100644 index 0000000..07bc8d6 --- /dev/null +++ b/src/actions/progress.rs @@ -0,0 +1,189 @@ +// © 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 and MIT +use std::sync::atomic::{AtomicUsize, Ordering}; + +use crate::server::{Notification, Output}; +use lazy_static::lazy_static; +use lsp_types::notification::{Progress, PublishDiagnostics, ShowMessage}; +use lsp_types::{MessageType, + ProgressParams, ProgressParamsValue, ProgressToken, + PublishDiagnosticsParams, ShowMessageParams, + WorkDoneProgress, + WorkDoneProgressBegin, + WorkDoneProgressReport, + WorkDoneProgressEnd}; + + +/// Communication of build progress back to the client. +pub trait ProgressNotifier { + fn id(&self) -> String; + fn notify_begin_progress(&self); + fn update(&mut self, update: ProgressUpdate); + fn notify_progress(&mut self, update: ProgressUpdate); + fn notify_end_progress(&self); +} + +/// Kinds of progress updates. +pub enum ProgressUpdate { + Message(String), + Percentage(u32), + Cancellable(bool), +} + +/// Trait for communication of diagnostics (i.e., analysis results) +/// back to the rest of the DLS (and on to the client). +// This trait only really exists to work around the object safety rules (Output +// is not object-safe). +pub trait DiagnosticsNotifier { + fn id(&self) -> String; + fn notify_begin_diagnostics(&self); + fn update(&mut self, update: ProgressUpdate); + fn notify_progress(&mut self, update: ProgressUpdate); + fn notify_publish_diagnostics(&self, _: PublishDiagnosticsParams); + fn notify_error_diagnostics(&self, msg: String); + fn notify_end_diagnostics(&self); +} + +/// Notifier of progress for the analysis (window/progress notifications). +/// the same instance is used for the entirety of one single build. +pub struct AnalysisProgressNotifier { + out: O, + id: String, + title: String, + cancellable: bool, + message: Option, + percentage: Option, +} + +impl AnalysisProgressNotifier { + pub fn new_with_id(id: String, title: String, out: O) + -> AnalysisProgressNotifier { + AnalysisProgressNotifier { + out, + id , + title, + cancellable: false, + message: None, + percentage: None, + } + } + pub fn new(title: String, out: O) -> AnalysisProgressNotifier { + // Counter to generate unique IDs for each chain-of-progress + // notification. + lazy_static! { + static ref PROGRESS_ID_COUNTER: AtomicUsize = AtomicUsize::new(0); + }; + Self::new_with_id( + format!("progress_{}", + PROGRESS_ID_COUNTER.fetch_add(1, Ordering::SeqCst)), + title, out) + } + + fn begin_params(&self) -> ProgressParams { + ProgressParams { + token: ProgressToken::String(self.id.clone()), + value: ProgressParamsValue::WorkDone( + WorkDoneProgress::Begin( + WorkDoneProgressBegin { + title: self.title.clone(), + cancellable: Some(self.cancellable), + message: self.message.clone(), + percentage: self.percentage, + }) + ), + } + } + + fn progress_params(&self) -> ProgressParams { + ProgressParams { + token: ProgressToken::String(self.id.clone()), + value: ProgressParamsValue::WorkDone( + WorkDoneProgress::Report( + WorkDoneProgressReport { + cancellable: Some(self.cancellable), + message: self.message.clone(), + percentage: self.percentage, + }) + ), + } + } + + fn end_params(&self) -> ProgressParams { + ProgressParams { + token: ProgressToken::String(self.id.clone()), + value: ProgressParamsValue::WorkDone( + WorkDoneProgress::End( + WorkDoneProgressEnd { + message: self.message.clone(), + }) + ), + } + } +} + +impl ProgressNotifier for AnalysisProgressNotifier { + fn id(&self) -> String { + self.id.clone() + } + fn update(&mut self, update: ProgressUpdate) { + match update { + ProgressUpdate::Message(s) => self.message = Some(s), + // TODO: We could sanity check that percentage remains + // non-decreasing here + ProgressUpdate::Percentage(p) => self.percentage = Some(p), + ProgressUpdate::Cancellable(b) => self.cancellable = b, + } + } + fn notify_begin_progress(&self) { + self.out.notify(Notification::::new(self.begin_params())); + } + fn notify_progress(&mut self, update: ProgressUpdate) { + self.update(update); + self.out.notify(Notification::::new(self.progress_params())); + } + fn notify_end_progress(&self) { + self.out.notify(Notification::::new(self.end_params())); + } +} + +/// Notifier of diagnostics after analysis has completed +pub struct AnalysisDiagnosticsNotifier { + sub_notifier: AnalysisProgressNotifier, +} + +impl AnalysisDiagnosticsNotifier { + pub fn new(title: String, out: O) -> AnalysisDiagnosticsNotifier { + AnalysisDiagnosticsNotifier { + sub_notifier: AnalysisProgressNotifier::new(title, out), + } + } +} + +impl DiagnosticsNotifier for AnalysisDiagnosticsNotifier { + fn id(&self) -> String { + self.sub_notifier.id() + } + fn notify_begin_diagnostics(&self) { + self.sub_notifier.notify_begin_progress(); + } + fn notify_publish_diagnostics(&self, params: PublishDiagnosticsParams) { + self.sub_notifier.out.notify( + Notification::::new(params)); + } + fn notify_error_diagnostics(&self, message: String) { + self.sub_notifier.out.notify( + Notification::::new(ShowMessageParams { + typ: MessageType::ERROR, + message, + })); + } + fn update(&mut self, update: ProgressUpdate) { + self.sub_notifier.update(update); + } + fn notify_progress(&mut self, update: ProgressUpdate) { + self.sub_notifier.notify_progress(update); + } + fn notify_end_diagnostics(&self) { + self.sub_notifier.notify_end_progress(); + } +} diff --git a/src/actions/requests.rs b/src/actions/requests.rs new file mode 100644 index 0000000..4b5c55a --- /dev/null +++ b/src/actions/requests.rs @@ -0,0 +1,703 @@ +// © 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 and MIT +//! Requests that the DLS can respond to. + +use jsonrpc::error::{StandardError, standard_error}; +use log::{info, debug, error, trace}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::collections::HashSet; +use std::sync::Arc; + +use crate::actions::hover; +use crate::actions::InitActionContext; +use crate::analysis::{ZeroSpan, ZeroFilePosition, SymbolRef}; +use crate::analysis::symbols::SimpleSymbol; + +pub use crate::lsp_data::request::{ + ApplyWorkspaceEdit, + CodeActionRequest, + CodeLensRequest, + Completion, + DocumentHighlightRequest, + DocumentSymbolRequest, + ExecuteCommand, + Formatting, + GotoDeclaration, GotoDeclarationResponse, + GotoDefinition, + GotoImplementation, GotoImplementationResponse, + HoverRequest, + RangeFormatting, + References, + Rename, + ResolveCompletionItem as ResolveCompletion, + WorkspaceSymbolRequest, +}; + +pub use crate::lsp_data::{self as lsp_data, *}; +use crate::analysis::{Named, DeclarationSpan, LocationSpan}; +use crate::analysis::structure::objects::CompObjectKind; + +use crate::analysis::scope::{SymbolContext, SubSymbol, ContextKey, Scope}; +use crate::analysis::symbols::{DMLSymbolKind, StructureSymbol}; +use crate::file_management::CanonPath; +use crate::server; +use crate::server::{Ack, Output, Request, RequestAction, + ResponseError, ResponseWithMessage}; + +fn fp_to_symbol_refs(fp: &ZeroFilePosition, ctx: &InitActionContext) + -> Option> { + let analysis = ctx.analysis.lock().unwrap(); + // This step-by-step approach could be folded into analysis_storage, + // but I keep it as separate here so that we could, perhaps, + // returns different information for "no symbols found" and + // "no info at pos" + debug!("Looking up symbols/references at {:?}", fp); + let (context_sym, reference) = (analysis.context_symbol_at_pos(fp), + analysis.reference_at_pos(fp)); + debug!("Got {:?} and {:?}", context_sym, reference); + + let mut definitions = vec![]; + match (context_sym, reference) { + (None, None) => { + debug!("No symbol or reference at point"); + return None; + }, + (Some(sym), refer) => { + if refer.is_some() { + error!("Obtained both symbol and reference at {:?}\ + (reference is {:?}), defaulted to symbol", + &fp, refer); + } + for device in analysis.all_device_analysises_containing_file( + &CanonPath::from_path_buf(fp.path()).unwrap()) { + definitions.extend( + device.lookup_symbols_by_contexted_symbol(&sym) + .into_iter()); + } + }, + (None, Some(refr)) => { + debug!("Mapping {:?} to symbols", refr.loc_span()); + for device in analysis.all_device_analysises_containing_file( + &CanonPath::from_path_buf(fp.path()).unwrap()) { + debug!("reference info is {:?}", device.reference_info.keys()); + if let Some(defs) = device.reference_info.get( + refr.loc_span()) { + for def in defs { + definitions.push(Arc::clone(def)); + } + } + } + }, + } + Some(definitions) +} + +fn handle_default_remapping(ctx: &InitActionContext, + symbols: Vec, + fp: &ZeroFilePosition) -> HashSet { + let analysis = ctx.analysis.lock().unwrap(); + let refr_opt = analysis.reference_at_pos(fp); + if let Some(refr) = refr_opt { + if refr.to_string().as_str() == "default" { + // If we are at a defaut reference, + // remap symbol references to methods + // to the correct decl site, leave others as-is + return symbols.into_iter() + .flat_map(|d|'sym: { + let sym = d.lock().unwrap(); + if sym.kind == DMLSymbolKind::Method { + if let Some(loc) = sym.default_mappings + .get(refr.loc_span()) { + break 'sym vec![*loc]; + } + } + sym.declarations.clone() + }).collect(); + } + } + symbols.into_iter() + .flat_map(|d|d.lock().unwrap().declarations.clone()) + .collect() +} + +/// The result of a deglob action for a single wildcard import. +/// +/// The `location` is the position of the wildcard. +/// `new_text` is the text which should replace the wildcard. +#[derive(Debug, Deserialize, Serialize)] +pub struct DeglobResult { + /// The `Location` of the "*" character in a wildcard import. + pub location: Location, + /// The replacement text. + pub new_text: String, +} + +// DML Structure kinds to not map 1-1 with the kinds supported by +// the lsp protocol +// TODO: Some of these should, maybe, be re-thought into different terms +fn context_to_symbolkind(context: &SymbolContext) -> lsp_types::SymbolKind { + match &context.context { + ContextKey::Structure(sym) | + ContextKey::Method(sym) | + ContextKey::Template(sym) => structure_to_symbolkind(sym.kind()), + ContextKey::AllWithTemplate(_, _) => SymbolKind::NAMESPACE, + } +} + +fn structure_to_symbolkind(kind: DMLSymbolKind) + -> lsp_types::SymbolKind { + match kind { + DMLSymbolKind::Parameter | + DMLSymbolKind::Constant | + DMLSymbolKind::Loggroup + => SymbolKind::CONSTANT, + DMLSymbolKind::Extern | + DMLSymbolKind::Saved | + DMLSymbolKind::Session | + DMLSymbolKind::Local | + DMLSymbolKind::MethodArg => + SymbolKind::VARIABLE, + DMLSymbolKind::Hook | + DMLSymbolKind::Method => + SymbolKind::FUNCTION, + DMLSymbolKind::Template => + SymbolKind::CLASS, + // TODO: There is no typedef kind? + DMLSymbolKind::Typedef => + SymbolKind::CONSTANT, + DMLSymbolKind::CompObject(kind) => match kind { + CompObjectKind::Interface => SymbolKind::INTERFACE, + CompObjectKind::Implement => SymbolKind::STRUCT, + // Generic comp objects most easily map to namespaces, I think? + _ => SymbolKind::NAMESPACE, + }, + } +} + +fn subsymbol_to_document_symbol(sub: &SubSymbol) -> DocumentSymbol { + match sub { + SubSymbol::Context(con) => context_to_document_symbol(con), + SubSymbol::Simple(simple) => { + #[allow(deprecated)] + DocumentSymbol { + name: simple.get_name(), + detail: None, + kind: structure_to_symbolkind(simple.kind()), + tags: None, + deprecated: None, + range: ls_util::dls_to_range( + simple.loc_span().range), + selection_range: ls_util::dls_to_range( + simple.loc_span().range), + children: None, + } + }, + } +} + +fn simplesymbol_to_workspace_symbol(parent_name: &str, + simple: &SimpleSymbol) -> WorkspaceSymbol { + WorkspaceSymbol { + name: simple.get_name(), + container_name: Some(parent_name.to_string()), + kind: structure_to_symbolkind(simple.kind()), + tags: None, + location: OneOf::Left(ls_util::dls_to_location( + simple.loc_span())), + data: None, + } +} + +fn context_to_document_symbol(context: &SymbolContext) -> DocumentSymbol { + // Note: This is probably slightly inefficient for simple contexts, + // but is unlikely to be a large problem + let name = context.get_name(); + let span = context.span(); + let loc = context.loc_span(); + + #[allow(deprecated)] + DocumentSymbol { + name, + detail: None, + kind: context_to_symbolkind(context), + tags: None, + deprecated: None, + range: ls_util::dls_to_range(span.range), + selection_range: ls_util::dls_to_range(loc.range), + children: Some(context.subsymbols.iter() + .map(subsymbol_to_document_symbol) + .collect()), + } +} + +fn context_to_workspace_symbols_aux(context: &SymbolContext, + parent_name: Option<&str>, + symbols: &mut Vec) { + // Note: This is probably slightly inefficient for simple contexts, + // but is unlikely to be a large problem + let name = context.get_name(); + let loc = context.loc_span(); + + let full_name = if let Some(pname) = parent_name { + format!("{}.{}", pname, name) + } else { + name.clone() + }; + + symbols.push(WorkspaceSymbol { + name, + container_name: parent_name.map(|name|name.to_string()), + kind: context_to_symbolkind(context), + tags: None, + location: OneOf::Left(ls_util::dls_to_location(loc)), + data: None, + }); + + for child in &context.subsymbols { + match child { + SubSymbol::Context(con) => context_to_workspace_symbols_aux( + con, Some(&full_name), symbols), + SubSymbol::Simple(simple) => symbols.push( + simplesymbol_to_workspace_symbol(&full_name, simple)), + }; + } +} + +fn context_to_workspace_symbols(context: &SymbolContext, + symbols: &mut Vec) { + context_to_workspace_symbols_aux(context, None, symbols); +} + +impl RequestAction for WorkspaceSymbolRequest { + type Response = Option; + + fn fallback_response() -> Result { + Ok(None) + } + + fn handle( + ctx: InitActionContext, + params: Self::Params, + ) -> Result { + let analysis = ctx.analysis.lock().unwrap(); + let mut workspace_symbols = vec![]; + for context in analysis.all_isolated_analysises().values() + .map(|i|&i.top_context) { + context_to_workspace_symbols(context, + &mut workspace_symbols); + } + Ok(Some(WorkspaceSymbolResponse::Nested( + workspace_symbols.into_iter() + .filter(|sym|sym.name.contains(¶ms.query)).collect() + ))) + } +} + +impl RequestAction for DocumentSymbolRequest { + type Response = Option; + + fn fallback_response() -> Result { + Ok(None) + } + + fn handle( + ctx: InitActionContext, + params: Self::Params, + ) -> Result { + info!("Handing doc symbol request {:?}", params); + let parse_canon_path = parse_file_path!( + ¶ms.text_document.uri, "document symbols") + .map(CanonPath::from_path_buf); + + if let Ok(Some(canon_path)) = parse_canon_path { + ctx.analysis.lock().unwrap() + .get_isolated_analysis(&canon_path) + .map(|isolated|{ + let context = isolated.toplevel.to_context(); + // Fold out the toplevel context + let symbols = context.subsymbols.iter().map( + subsymbol_to_document_symbol).collect(); + Some(DocumentSymbolResponse::Nested(symbols)) + }) + .map_or_else(Self::fallback_response, |r|Ok(r)) + } else { + Self::fallback_response() + } + } +} + +impl RequestAction for HoverRequest { + type Response = lsp_data::Hover; + + fn fallback_response() -> Result { + Ok(lsp_data::Hover { contents: HoverContents::Array(vec![]), + range: None }) + } + + fn handle(mut ctx: InitActionContext, + params: Self::Params, + ) -> Result { + trace!("handling hover ({:?})", params); + let tooltip = hover::tooltip(&mut ctx, + ¶ms.text_document_position_params)?; + + Ok(lsp_data::Hover { + contents: HoverContents::Array(tooltip.contents), + range: Some(ls_util::dls_to_range(tooltip.range)), + }) + } +} + +impl RequestAction for GotoImplementation { + type Response = Option; + + fn fallback_response() -> Result { + Ok(None) + } + + fn handle( + ctx: InitActionContext, + params: Self::Params, + ) -> Result { + info!("Requesting implementations with params {:?}", params); + let fp = { + let maybe_fp = ctx.text_doc_pos_to_pos( + ¶ms.text_document_position_params, + "goto_impl"); + if maybe_fp.is_none() { + return Self::fallback_response(); + } + maybe_fp.unwrap() + }; + + if let Some(symbols) = fp_to_symbol_refs(&fp, &ctx) { + if symbols.is_empty() { + info!("No symbols found"); + Ok(None) + } else { + let mut unique_locations: HashSet + = HashSet::default(); + for symbol in symbols { + for implementation in &symbol.lock().unwrap().implementations { + unique_locations.insert(*implementation); + } + } + let lsp_locations = unique_locations.into_iter() + .map(|l|ls_util::dls_to_location(&l)) + .collect(); + info!("Requested implementations are {:?}", lsp_locations); + Ok(Some(GotoImplementationResponse::Array(lsp_locations))) + } + } else { + Self::fallback_response() + } + } +} + +impl RequestAction for GotoDeclaration { + type Response = Option; + + fn fallback_response() -> Result { + Ok(None) + } + + fn handle( + ctx: InitActionContext, + params: Self::Params, + ) -> Result { + info!("Requesting declarations with params {:?}", params); + let fp = { + let maybe_fp = ctx.text_doc_pos_to_pos( + ¶ms.text_document_position_params, + "goto_decl"); + if maybe_fp.is_none() { + return Self::fallback_response(); + } + maybe_fp.unwrap() + }; + if let Some(symbols) = fp_to_symbol_refs(&fp, &ctx) { + if symbols.is_empty() { + info!("No symbols found"); + Ok(None) + } else { + let unique_locations = handle_default_remapping(&ctx, + symbols, + &fp); + let lsp_locations = unique_locations.into_iter() + .map(|l|ls_util::dls_to_location(&l)) + .collect(); + info!("Requested declarations are {:?}", lsp_locations); + Ok(Some(GotoDefinitionResponse::Array(lsp_locations))) + } + } else { + Self::fallback_response() + } + } +} + +impl RequestAction for GotoDefinition { + type Response = Option; + + fn fallback_response() -> Result { + Ok(None) + } + + fn handle( + ctx: InitActionContext, + params: Self::Params, + ) -> Result { + info!("Requesting definitions with params {:?}", params); + let fp = { + let maybe_fp = ctx.text_doc_pos_to_pos( + ¶ms.text_document_position_params, + "goto_def"); + if maybe_fp.is_none() { + return Self::fallback_response(); + } + maybe_fp.unwrap() + }; + + if let Some(symbols) = fp_to_symbol_refs(&fp, &ctx) { + if symbols.is_empty() { + info!("No symbols found"); + Ok(None) + } else { + let unique_locations: HashSet = + symbols.into_iter() + .flat_map(|d|d.lock().unwrap().definitions.clone()).collect(); + let lsp_locations = unique_locations.into_iter() + .map(|l|ls_util::dls_to_location(&l)) + .collect(); + info!("Requested definitions are {:?}", lsp_locations); + Ok(Some(GotoDefinitionResponse::Array(lsp_locations))) + } + } else { + Self::fallback_response() + } + } +} + +impl RequestAction for References { + type Response = Vec; + + fn fallback_response() -> Result { + Ok(vec![]) + } + + fn handle( + ctx: InitActionContext, + params: Self::Params, + ) -> Result { + info!("Requesting references with params {:?}", params); + let fp = { + let maybe_fp = ctx.text_doc_pos_to_pos( + ¶ms.text_document_position, + "find_refs"); + if maybe_fp.is_none() { + return Self::fallback_response(); + } + maybe_fp.unwrap() + }; + if let Some(symbols) = fp_to_symbol_refs(&fp, &ctx) { + if symbols.is_empty() { + info!("No symbols found"); + Ok(vec![]) + } else { + let unique_locations: HashSet = + symbols.into_iter() + .flat_map(|d|d.lock().unwrap().references.clone()).collect(); + let lsp_locations = unique_locations.into_iter() + .map(|l|ls_util::dls_to_location(&l)) + .collect(); + info!("Requested references are {:?}", lsp_locations); + Ok(lsp_locations) + } + } else { + Self::fallback_response() + } + } +} + +impl RequestAction for Completion { + type Response = Vec; + + fn fallback_response() -> Result { + Ok(vec![]) + } + + fn handle( + _ctx: InitActionContext, + _params: Self::Params, + ) -> Result { + // TODO: Acquire completions for location + Self::fallback_response() + } +} + +impl RequestAction for DocumentHighlightRequest { + type Response = Vec; + + fn fallback_response() -> Result { + Ok(vec![]) + } + + fn handle( + _ctx: InitActionContext, + _params: Self::Params, + ) -> Result { + // TODO: Acquire highlighting info for file and span + Self::fallback_response() + } +} + +impl RequestAction for Rename { + type Response = ResponseWithMessage; + + fn fallback_response() -> Result { + Ok(ResponseWithMessage::Response( + WorkspaceEdit { changes: None, + document_changes: None, + change_annotations: None })) + } + + fn handle( + _ctx: InitActionContext, + _params: Self::Params, + ) -> Result { + // TODO: Perform a rename + Self::fallback_response() + } +} + +#[derive(Debug)] +pub enum ExecuteCommandResponse { + /// Response/client request containing workspace edits. + ApplyEdit(ApplyWorkspaceEditParams), +} + +impl server::Response for ExecuteCommandResponse { + fn send(self, id: server::RequestId, out: &O) { + // FIXME should handle the client's responses + match self { + ExecuteCommandResponse::ApplyEdit(ref params) => { + let id = out.provide_id(); + let params = ApplyWorkspaceEditParams { + label: None, + edit: params.edit.clone() }; + + let request = Request::::new(id, params); + out.request(request); + } + } + + // The formal request response is a simple ACK, though the objective + // is the preceding client requests. + Ack.send(id, out); + } +} + +impl RequestAction for ExecuteCommand { + type Response = ExecuteCommandResponse; + + fn fallback_response() -> Result { + Err(ResponseError::Empty) + } + + /// Currently, no support for this + fn handle( + _ctx: InitActionContext, + _params: ExecuteCommandParams, + ) -> Result { + // TODO: handle specialized commands. or if no such commands, remove + Self::fallback_response() + } +} + +fn rpc_error_code(code: StandardError) -> Value { + Value::from(standard_error(code, None).code) +} + +impl RequestAction for CodeActionRequest { + type Response = Vec; + + fn fallback_response() -> Result { + Ok(vec![]) + } + + fn handle( + _ctx: InitActionContext, + _params: Self::Params, + ) -> Result { + // TODO: figure out if we want to use this + // note: a "code action" is like a command tied to a code position, I think + Self::fallback_response() + } +} + +impl RequestAction for Formatting { + type Response = Vec; + + fn fallback_response() -> Result { + Err(ResponseError::Message( + rpc_error_code(StandardError::InternalError), + "Reformat failed to complete successfully".into(), + )) + } + + fn handle( + _ctx: InitActionContext, + _params: Self::Params, + ) -> Result { + // TODO: format document + Self::fallback_response() + } +} + +impl RequestAction for RangeFormatting { + type Response = Vec; + + fn fallback_response() -> Result { + Err(ResponseError::Message( + rpc_error_code(StandardError::InternalError), + "Reformat failed to complete successfully".into(), + )) + } + + fn handle( + _ctx: InitActionContext, + _params: Self::Params, + ) -> Result { + // TODO: format range + Self::fallback_response() + } +} + +impl RequestAction for ResolveCompletion { + type Response = CompletionItem; + + fn fallback_response() -> Result { + Err(ResponseError::Empty) + } + + fn handle(_: InitActionContext, _params: Self::Params) -> Result { + // TODO: figure out if we want to use this + Self::fallback_response() + } +} + +impl RequestAction for CodeLensRequest { + type Response = Vec; + + fn fallback_response() -> Result { + Err(ResponseError::Empty) + } + + fn handle( + _ctx: InitActionContext, + _params: Self::Params, + ) -> Result { + // TODO: figure out if we want to use this + Self::fallback_response() + } +} diff --git a/src/actions/work_pool.rs b/src/actions/work_pool.rs new file mode 100644 index 0000000..f4a7863 --- /dev/null +++ b/src/actions/work_pool.rs @@ -0,0 +1,103 @@ +// © 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 and MIT +use crate::server::DEFAULT_REQUEST_TIMEOUT; +use lazy_static::lazy_static; +use log::{info, warn}; +use std::sync::{mpsc, Mutex}; +use std::time::{Duration, Instant}; +use std::{fmt, panic}; + +/// Description of work on the request work pool. Equality implies two pieces of work are the same +/// kind of thing. The `str` should be human readable for logging (e.g., the language server +/// protocol request message name or similar). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct WorkDescription(pub &'static str); + +impl fmt::Display for WorkDescription { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +lazy_static! { + /// Maximum total concurrent working tasks + // TODO: unsure what to do about this + static ref NUM_THREADS: usize = 4;//::num_cpus::get(); + + /// Duration of work after which we should warn something is taking a long time + static ref WARN_TASK_DURATION: Duration = DEFAULT_REQUEST_TIMEOUT * 5; + + /// Current work descriptions active on the work pool + static ref WORK: Mutex> = Mutex::new(vec![]); + + /// Thread pool for request execution allowing concurrent request processing. + static ref WORK_POOL: rayon::ThreadPool = rayon::ThreadPoolBuilder::new() + .num_threads(*NUM_THREADS) + .thread_name(|num| format!("request-worker-{}", num)) + .build() + .unwrap(); +} + +/// Maximum concurrent working tasks of the same type (equal `WorkDescription`) +/// Note: `2` allows a single task to run immediately after a similar task has timed out. +/// Once multiple similar tasks have timed out but remain running we start refusing to start new +/// ones. +const MAX_SIMILAR_CONCURRENT_WORK: usize = 2; + +/// Runs work in a new thread on the `WORK_POOL` returning a result `Receiver` +/// +/// Panicking work will receive `Err(RecvError)` / `Err(RecvTimeoutError::Disconnected)` +/// +/// If too many tasks are already running the work will not be done and the receiver will +/// immediately return `Err(RecvTimeoutError::Disconnected)` +pub fn receive_from_thread(work_fn: F, description: WorkDescription) -> mpsc::Receiver +where + T: Send + 'static, + F: FnOnce() -> T + Send + panic::UnwindSafe + 'static, +{ + let (sender, receiver) = mpsc::channel(); + + { + let mut work = WORK.lock().unwrap(); + if work.len() >= *NUM_THREADS { + // there are already N ongoing tasks, that may or may not have timed out + // don't add yet more to the queue fail fast to allow the work pool to recover + warn!("Could not start `{}` as at work capacity, {:?} in progress", description, *work,); + return receiver; + } + if work.iter().filter(|desc| *desc == &description).count() >= MAX_SIMILAR_CONCURRENT_WORK { + // this type of work is already filling max proportion of the work pool, so fail + // new requests of this kind until some/all the ongoing work finishes + info!( + "Could not start `{}` as same work-type is filling half capacity, {:?} in progress", + description, *work, + ); + return receiver; + } + work.push(description); + } + + WORK_POOL.spawn(move || { + let start = Instant::now(); + + // panic details will be on stderr, otherwise ignore the work panic as it + // will already cause a mpsc disconnect-error & there isn't anything else to log + if let Ok(work_result) = panic::catch_unwind(work_fn) { + // an error here simply means the work took too long and the receiver has been dropped + let _ = sender.send(work_result); + } + + let mut work = WORK.lock().unwrap(); + if let Some(index) = work.iter().position(|desc| desc == &description) { + work.swap_remove(index); + } + + let elapsed = start.elapsed(); + if elapsed >= *WARN_TASK_DURATION { + let secs = + elapsed.as_secs() as f64 + f64::from(elapsed.subsec_nanos()) / 1_000_000_000_f64; + warn!("`{}` took {:.1}s", description, secs); + } + }); + receiver +} diff --git a/src/analysis/mod.rs b/src/analysis/mod.rs new file mode 100644 index 0000000..d4d54de --- /dev/null +++ b/src/analysis/mod.rs @@ -0,0 +1,1795 @@ +// © 2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 and MIT +// Load parser and tree first to ensure existance of macros +#[macro_use] +pub mod parsing; +#[macro_use] +pub mod symbols; +pub mod provisionals; +pub mod scope; +pub mod reference; +pub mod structure; +pub mod templating; + +use std::collections::{HashMap, HashSet}; +use std::fmt; +use std::fmt::Write; +use std::hash::{Hash, Hasher}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::sync::Mutex; + +use itertools::Itertools; + +use lsp_types::{Diagnostic, DiagnosticRelatedInformation, DiagnosticSeverity}; +use logos::Logos; +use log::{debug, error, info, trace}; +use rayon::prelude::*; + +use crate::actions::analysis_storage::TimestampedStorage; +use crate::analysis::symbols::{SimpleSymbol, DMLSymbolKind, Symbol, + SymbolSource}; +use crate::analysis::reference::{Reference, + GlobalReference, VariableReference, + ReferenceKind, NodeRef}; +use crate::analysis::scope::{Scope, SymbolContext, + ContextKey, ContextedSymbol}; +use crate::analysis::parsing::parser::{FileInfo, FileParser}; +use crate::analysis::parsing::lexer::TokenKind; +use crate::analysis::provisionals::ProvisionalsManager; + +pub use crate::analysis::parsing::tree:: +{ZeroRange, ZeroSpan, ZeroPosition, ZeroFilePosition}; + +use crate::analysis::parsing::tree::{MissingToken, MissingContent, TreeElement}; +use crate::analysis::structure::objects::{Template, Import, CompObjectKind, + ParamValue}; +use crate::analysis::structure::toplevel::{ObjectDecl, TopLevel}; +use crate::analysis::structure::expressions::{Expression, ExpressionKind, + DMLString}; +use crate::analysis::templating::objects::{make_device, DMLObject, + DMLResolvedObject, + DMLShallowObjectVariant, + DMLCompositeObject, + DMLShallowObject, + DMLHierarchyMember, + DMLNamedMember, + StructureContainer, StructureKey}; +use crate::analysis::templating::topology::{RankMaker, + rank_templates, + create_templates_traits}; +use crate::analysis::templating::methods::{MethodDeclaration, DMLMethodRef, + DMLMethodArg}; +use crate::analysis::templating::traits::{DMLTemplate, + TemplateTraitInfo}; +use crate::analysis::templating::types::DMLResolvedType; + +use crate::file_management::{PathResolver, CanonPath}; + +use crate::vfs::{TextFile, Error}; +use crate::lsp_data::ls_util::{dls_to_range, dls_to_location}; + +#[derive(Clone, Copy)] +pub struct FileSpec<'a> { + pub path: &'a Path, + pub file: &'a TextFile, +} + +pub const IMPLICIT_IMPORTS: [&str; 2] = ["dml-builtins.dml", + "simics/device-api.dml"]; + +fn collapse_referencematches(matches: T) -> ReferenceMatch +where T : IntoIterator { + matches.into_iter().fold( + ReferenceMatch::NotFound(vec![]), + |acc, rf|match (acc, rf) { + (ReferenceMatch::Found(mut f1), ReferenceMatch::Found(mut f2)) => { + f1.append(&mut f2); + ReferenceMatch::Found(f1) + }, + (ReferenceMatch::NotFound(mut f1), + ReferenceMatch::NotFound(mut f2)) => { + f1.append(&mut f2); + ReferenceMatch::NotFound(f1) + }, + (f @ ReferenceMatch::Found(_), _) => f, + (_, f @ ReferenceMatch::Found(_)) => f, + (f @ ReferenceMatch::WrongType(_), _) => f, + (_, f) => f, + }) +} + +// For things whose names are in one spot as a dmlstring +pub trait DMLNamed { + fn name(&self) -> &DMLString; +} + +impl LocationSpan for T { + fn loc_span(&self) -> &ZeroSpan { + &self.name().span + } +} + +// serves as the catch-all when we _might_ need something whose +// name is not present as a literal in analyzed source +pub trait Named { + fn get_name(&self) -> String; +} + +impl Named for T { + fn get_name(&self) -> String { + self.name().val.clone() + } +} + +// These traits relate to things knowing the parts of the file +// that their declaration covers +pub trait DeclarationRange { + fn range(&self) -> &ZeroRange; +} + +pub trait DeclarationFile { + fn file(&self) -> PathBuf; +} + +pub trait DeclarationSpan { + fn span(&self) -> &ZeroSpan; +} + +impl DeclarationRange for T { + fn range(&self) -> &ZeroRange { + &self.span().range + } +} + +impl DeclarationFile for T { + fn file(&self) -> PathBuf { + self.span().path() + } +} + +// For things that know a more specific range where their named declaration +// is, these traits apply +pub trait LocationRange { + fn loc_range(&self) -> &ZeroRange; +} + +pub trait LocationFile { + fn loc_file(&self) -> PathBuf; +} + +pub trait LocationSpan { + fn loc_span(&self) -> &ZeroSpan; +} + +fn combine_vec_of_decls(vec: &[T]) + -> ZeroSpan { + // We do not have an 'invalid' file value, + // so doing this for empty vecs is not allowed + let file = &vec.first().unwrap().file(); + assert!(!vec.iter().any(|f|&f.file() != file)); + ZeroSpan::combine(*vec.first().unwrap().span(), + *vec.last().unwrap().span()) +} + +impl LocationRange for T { + fn loc_range(&self) -> &ZeroRange { + &self.loc_span().range + } +} + +impl LocationFile for T { + fn loc_file(&self) -> PathBuf { + self.loc_span().path() + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DMLError { + pub span: ZeroSpan, + pub description: String, + pub severity: Option, + pub related: Vec<(ZeroSpan, String)>, +} + +impl Hash for DMLError { + fn hash(&self, state: &mut H) { + self.span.hash(state); + self.description.hash(state); + match self.severity { + Some(DiagnosticSeverity::ERROR) => 1.hash(state), + Some(DiagnosticSeverity::WARNING) => 2.hash(state), + Some(DiagnosticSeverity::INFORMATION) => 3.hash(state), + Some(DiagnosticSeverity::HINT) => 4.hash(state), + _ => panic!(), + } + self.related.hash(state); + } +} + +impl DMLError { + pub fn to_diagnostic(&self) -> Diagnostic { + Diagnostic::new( + dls_to_range(self.span.range), + self.severity, None, None, + self.description.clone(), + Some( + self.related.iter().map( + |(span, desc)|DiagnosticRelatedInformation { + location: dls_to_location(span), + message: desc.clone(), + }).collect()), + None + ) + } +} + +#[derive(Debug, Clone)] +pub struct LocalDMLError { + pub range: ZeroRange, + pub description: String, +} + +impl LocalDMLError { + fn with_file>(self, file: F) -> DMLError { + DMLError { + span: ZeroSpan::from_range(self.range, file), + description: self.description, + severity: Some(DiagnosticSeverity::ERROR), + related: vec![], + } + } + pub fn to_diagnostic(&self) -> Diagnostic { + Diagnostic::new_simple(dls_to_range(self.range), + self.description.clone()) + } + pub fn warning_with_file>(self, file: F) -> DMLError { + DMLError { + span: ZeroSpan::from_range(self.range, file), + description: self.description, + related: vec![], + severity: Some(DiagnosticSeverity::WARNING), + } + } +} + +impl From<&MissingToken> for LocalDMLError { + fn from(miss: &MissingToken) -> Self { + LocalDMLError { + range: ZeroRange::from_positions(miss.position, miss.position), + description: format!("Expected {}, got {}", miss.description, + match miss.ended_by { + Some(endtok) => endtok.kind.description(), + None => "EOF", + }), + } + } +} + +pub fn make_error_from_missing_content( + range: ZeroRange, content: &MissingContent) -> LocalDMLError { + LocalDMLError { + range, + description: format!("Expected {}, got {}", content.description, + match content.ended_by { + Some(endtok) => endtok.kind.description(), + None => "EOF", + }), + } +} + +// Analysis from the perspective of a particular DML file +#[derive(Debug, Clone)] +pub struct IsolatedAnalysis { + // Toplevel ast of file + pub ast: parsing::structure::TopAst, + + // Toplevel structure of file + pub toplevel: TopLevel, + + // Cached structural contexts of file + pub top_context: SymbolContext, + + // File info + pub path: CanonPath, + // This is the path the client has the file open as, + // which is relevant so that we report errors correctly + // NOTE: This might not be a canonpath, but it still needs + // to be an absolute path or we'll run into + // trouble reporting errors + pub clientpath: PathBuf, + + // Errors are used as input for various responses/requests to the client + pub errors: Vec, +} + +// Mas symbol decl locations to symbols, +pub type SymbolRef = Arc>; +#[derive(Debug, Clone, Default)] +pub struct SymbolStorage { + pub template_symbols: HashMap, + // Because some implicit parameters are defined in the same + // place, we need to disambiguate this by name + pub param_symbols: HashMap<(ZeroSpan, String), + HashMap>, + pub object_symbols: HashMap, + pub method_symbols: HashMap, + // constants, sessions, saveds, hooks, method args + pub variable_symbols: HashMap, +} + +// This maps non-auth symbol decls to auth decl +// and references to the symbol decl they ref +type ReferenceStorage = HashMap>; + +// Analysis from the perspective of a particular DML device +#[derive(Debug, Clone)] +pub struct DeviceAnalysis { + // Device name + pub name: String, + pub errors: HashMap>, + pub objects: StructureContainer, + pub device_obj: DMLObject, + pub templates: TemplateTraitInfo, + pub symbol_info: SymbolStorage, + pub reference_info: ReferenceStorage, + pub template_object_implementation_map: HashMap>, + pub path: CanonPath, + pub dependant_files: Vec, + pub clientpath: PathBuf, +} + +#[derive(Debug, Clone)] +pub enum ReferenceMatch { + Found(Vec), + WrongType(SymbolRef), + NotFound(Vec), +} + +/// TODO: Consider usage and variants of type hints +pub type TypeHint = DMLResolvedType; + +// We replicate some of the structures from scope and reference here, because +// we need to _discard_ the location information for the caching to work + +// agnostic context key +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +enum AgnConKey { + Object(String), + Template(String), + AllWithTemplate(Vec), +} + +// Agnostic reference +type AgnRef = Vec; + +type ReferenceCacheKey = (Vec, AgnRef); +#[derive(Default)] +struct ReferenceCache { + underlying_cache: HashMap, +} + +impl ReferenceCache { + fn flatten_ref(refr: &NodeRef, agn: &mut Vec) { + match refr { + NodeRef::Simple(dmlstring) => agn.push(dmlstring.val.clone()), + NodeRef::Sub(sub, dmlstring, _) => { + Self::flatten_ref(sub.as_ref(), agn); + agn.push(dmlstring.val.clone()); + }, + } + } + fn convert_to_key(key: (Vec, VariableReference)) + -> ReferenceCacheKey { + let (contexts, refr) = key; + let agnostic_context = contexts.into_iter().map( + |con|match con { + ContextKey::Structure(sym) | + ContextKey::Method(sym) => AgnConKey::Object(sym.get_name()), + ContextKey::Template(sym) => AgnConKey::Template( + sym.get_name()), + ContextKey::AllWithTemplate(_, names) => + AgnConKey::AllWithTemplate(names.clone()), + }).collect(); + let mut agnostic_reference = vec![]; + Self::flatten_ref(&refr.reference, &mut agnostic_reference); + (agnostic_context, agnostic_reference) + } + + pub fn get(&self, key: (Vec, + VariableReference)) + -> Option<&ReferenceMatch> { + let agn_key = Self::convert_to_key(key); + self.underlying_cache.get(&agn_key) + } + + pub fn insert(&mut self, + key: (Vec, + VariableReference), + val: ReferenceMatch) + { + let agn_key = Self::convert_to_key(key); + self.underlying_cache.insert(agn_key, val); + } +} + +fn all_scopes<'c>(bases: &'c Vec) + -> Vec> { + let mut scopes = vec![]; + for base in bases { + gather_scopes(vec![&base.toplevel as &dyn Scope], + vec![], + &mut scopes); + } + scopes +} + +fn gather_scopes<'c>(next_scopes: Vec<&'c dyn Scope>, + scope_chain: Vec<&'c dyn Scope>, + collected_scopes: &mut Vec>) { + for scope in next_scopes { + let mut new_chain = scope_chain.clone(); + new_chain.push(scope); + collected_scopes.push(new_chain.clone()); + gather_scopes(scope.defined_scopes(), + new_chain, + collected_scopes); + } +} + +impl DeviceAnalysis { + pub fn get_device_obj(&self) -> &DMLObject { + &self.device_obj + } + + pub fn get_device_symbol(&self) -> SymbolRef { + Arc::clone( + self.symbol_info.object_symbols + .get(&self.get_device_obj_key()).unwrap() + ) + } + + pub fn get_device_obj_key(&self) -> StructureKey { + if let DMLObject::CompObject(key) = &self.device_obj { + *key + } else { + panic!("Internal Error: DeviceAnalysis device object was \ + not a composite object"); + } + } + + pub fn get_device_comp_obj(&self) -> &DMLCompositeObject { + self.objects.get(self.get_device_obj_key()).unwrap() + } + + // TODO: Currently these functions do a lot of dumb cast-to-comp-then-back + // which is the result of us wanting to return DMLObject _references. + // A DMLObject is (relatively) small so we could consider returning + // values, allowing us to re-construct DMLObjects and having + // get_objs_matching_templates operate on composite objects as roots + fn get_objs_matching_templates_aux(&self, + root: &DMLCompositeObject, + spec: &[&Arc], + aux: &mut Vec) { + // TODO: Remind yourself of how dmlc in-each works, does a match + // blocks further recursion? + let mut all_match = true; + for templ in spec { + if !root.templates.contains_key(&templ.name) { + all_match = false; + } + } + if all_match { + aux.push(DMLObject::CompObject(root.key)); + } else { + for obj in root.components.values() { + if let DMLResolvedObject::CompObject(robj) + = obj.resolve(&self.objects) { + self.get_objs_matching_templates_aux( + robj, spec, aux); + } + } + } + } + + fn get_objs_matching_templates(&self, + root: &DMLCompositeObject, + spec: &[&str]) + -> Vec { + // TODO: Are we guaranteed that each-ins handled here specify + // _only_ existing template names? I suspect they do not + let unique_names: HashSet<&str> = spec.iter().cloned().collect(); + let set: Vec<&Arc> = unique_names.into_iter() + .map(|name|self.templates.templates.get(name).unwrap()) + .collect(); + let mut result = vec![]; + for obj in root.components.values() { + if let DMLResolvedObject::CompObject(robj) + = obj.resolve(&self.objects) { + self.get_objs_matching_templates_aux(robj, &set, + &mut result); + } + } + result + } + + + // TODO: Reconsider function signature + // currently returning whole DMLObject so we can faux + // them from structurekeys obtained through + fn contexts_to_objs(&self, + curr_objs: Vec, + context_chain: &[ContextKey]) + -> Option> { + if context_chain.is_empty() { + Some(curr_objs) + } else { + self.contexts_to_objs_aux( + curr_objs.into_iter().filter_map( + |obj|match obj.resolve(&self.objects) { + DMLResolvedObject::CompObject(robj) => Some(robj), + DMLResolvedObject::ShallowObject(sobj) => { + error!("Internal Error: \ + Wanted to find context {:?} in {:?} \ + which is not \ + a composite object", sobj, context_chain); + None + } + }).collect(), + context_chain) + } + } + + fn contexts_to_objs_aux(&self, + curr_objs: Vec<&DMLCompositeObject>, + context_chain: &[ContextKey]) + -> Option> { + let result: Vec = curr_objs.into_iter() + .filter_map(|o|self.context_to_objs(o, context_chain)) + .flatten().collect(); + if result.is_empty() { + None + } else { + Some(result) + } + } + + fn context_to_objs(&self, + curr_obj: &DMLCompositeObject, + context_chain: &[ContextKey]) + -> Option> { + // Should be guaranteed by caller responsibility + if context_chain.is_empty() { + error!("Internal Error: context chain invariant broken at {:?}", + curr_obj.identity()) + } + + let (first, rest) = context_chain.split_first().unwrap(); + let next_objs = match first { + ContextKey::Structure(sym) => + curr_obj.get_object(sym.name_ref()).cloned() + .into_iter().collect(), + ContextKey::Method(sym) => { + if let Some(found_obj) + = curr_obj.get_object(sym.name_ref()) { + if found_obj.resolve(&self.objects).as_shallow() + .map_or(false, |s|matches!( + &s.variant, + DMLShallowObjectVariant::Method(_))) { + vec![found_obj.clone()] + } + else { + error!("Internal Error: Context chain \ + suggested {:?} should be a method, \ + but it wasn't", + found_obj.resolve(&self.objects)); + vec![] + } + } + else { + // If it is an error to not find an result, handle + // it higher up in stack + return None; + } + }, + ContextKey::Template(sym) => + if let Some(templ) = self.templates.templates.get(sym.name_ref()) { + if let Some(templ_impls) = + templ.location.as_ref().and_then( + |loc|self + .template_object_implementation_map + .get(loc)) + { + return self.contexts_to_objs( + templ_impls + .iter() + .map(|key|DMLObject::CompObject(*key)) + .collect(), + rest); + } + else { + error!("Internal Error: No template->objects map for\ + {:?}", sym); + vec![] + } + } else { + error!("Internal Error: \ + Wanted to find context {:?} in {:?} which is not \ + a known template object", + first, curr_obj); + return None; + }, + ContextKey::AllWithTemplate(_, templates) => + return self.contexts_to_objs( + self.get_objs_matching_templates( + curr_obj, templates.iter() + .map(|s|s.as_str()) + .collect::>() + .as_slice()), + rest), + }; + self.contexts_to_objs(next_objs, rest) + } + + // Part of constant folding, try to resolve an expression that + // is perhaps a noderef into the symbol of the node it refers to + fn expression_to_resolved_objects<'t, 'c>( + &'c self, + expr: &Expression, + _scope: DMLResolvedObject<'t, 'c>) + -> Vec> { + // TODO: For now, we can only resolve some simple expressions + #[allow(clippy::single_match)] + match expr.as_ref() { + ExpressionKind::AutoObjectRef(key, _) => + if let Some(obj) = self.objects.get(*key) { + return vec![DMLResolvedObject::CompObject(obj)] + }, + _ => (), + } + vec![] + } + + fn resolved_to_symbol<'t, 'c>(&'c self, obj: DMLResolvedObject<'t, 'c>) + -> Option> { + match obj { + DMLResolvedObject::CompObject(comp) => + self.symbol_info.object_symbols.get(&comp.key) + .map(|r|vec![r]), + DMLResolvedObject::ShallowObject(shallow) => + match &shallow.variant { + DMLShallowObjectVariant::Method(m) => + self.symbol_info.method_symbols.get(m.location()) + .map(|r|vec![r]), + DMLShallowObjectVariant::Session(s) | + DMLShallowObjectVariant::Saved(s) => + self.symbol_info.variable_symbols.get(s.loc_span()) + .map(|r|vec![r]), + DMLShallowObjectVariant::Constant(c) => + self.symbol_info.variable_symbols.get(c.loc_span()) + .map(|r|vec![r]), + DMLShallowObjectVariant::Hook(h) => + self.symbol_info.variable_symbols.get(h.loc_span()) + .map(|r|vec![r]), + DMLShallowObjectVariant::Parameter(p) => + self.symbol_info.param_symbols.get( + &(*p.loc_span(), + p.name().val.to_string())) + .map(|m|m.values().collect()), + }, + } + } + + fn lookup_def_in_comp_object<'c>(&'c self, + obj: &'c DMLCompositeObject, + name: &str, + _type_hint: Option) + -> ReferenceMatch { + debug!("Looking up {} in {:?}", name, obj.identity()); + match name { + "this" => ReferenceMatch::Found( + vec![Arc::clone(self.symbol_info + .object_symbols.get(&obj.key).unwrap())]), + _ => obj.get_object(name) + .map(|o|o.resolve(&self.objects)) + .and_then(|r|self.resolved_to_symbol(r)) + .map_or(ReferenceMatch::NotFound(vec![]), + |res|ReferenceMatch::Found(res.into_iter() + .map(Arc::clone) + .collect())) + } + } + + fn lookup_def_in_resolved<'t, 'c>(&'c self, + obj: DMLResolvedObject<'t, 'c>, + name: &str, + type_hint: Option) + -> ReferenceMatch { + match obj { + DMLResolvedObject::CompObject(o) => + self.lookup_def_in_comp_object(o, name, type_hint), + DMLResolvedObject::ShallowObject(o) => match + &o.variant { + DMLShallowObjectVariant::Method(m) => + match name { + "this" => + ReferenceMatch::Found( + vec![Arc::clone( + self.symbol_info.object_symbols.get( + &o.parent).unwrap())]), + "default" => + // NOTE: This is part of the hack that maps default + // references in methods to the corret method decl. + // Here, we simply check if the method has any + // default call, and if so map the reference to the + // method symbol + if m.get_default().is_some() { + self.symbol_info.method_symbols + .get(obj.location()) + .map_or_else( + ||ReferenceMatch::NotFound(vec![]), + |sym|ReferenceMatch::Found(vec![ + Arc::clone(sym)])) + } else { + // fairly sure 'default' cannot be a + // reference otherwise + // TODO: better error message here, somehow? + ReferenceMatch::NotFound(vec![]) + }, + _ => if let Some(sym) = m.args().iter() + .find(|a|a.name().val == name) + .map(|a|self.symbol_info.variable_symbols + .get(a.loc_span()).unwrap()) + { + ReferenceMatch::Found(vec![Arc::clone(sym)]) + } else { + ReferenceMatch::NotFound(vec![]) + }, + }, + DMLShallowObjectVariant::Parameter(p) => { + if let Some(param) = p.get_unambiguous_def() { + // TODO: Remove this when we can resolve 'dev' param + // using constant folding + if param.name().val.as_str() == "dev" { + return self.lookup_def_in_resolved( + self.get_device_obj().resolve( + &self.objects), + name, + type_hint); + } else { + // TODO: pre-evaluate params in objects that are + // noderefs, so it is simple to re-do the lookup + // here + #[allow(clippy::single_match)] + match param.value.as_ref() { + Some(ParamValue::Set(expr)) => + return + collapse_referencematches( + self.expression_to_resolved_objects( + expr, obj) + .into_iter() + .map(|res| + self.lookup_def_in_resolved( + res, + name, + type_hint.clone()))), + _ => (), + } + } + } + ReferenceMatch::NotFound(vec![]) + }, + // TODO: Can we do _anything_ here? Perhaps defer to the + // default value (if any)? + DMLShallowObjectVariant::Session(_) | + DMLShallowObjectVariant::Saved(_) => ReferenceMatch::NotFound(vec![]), + // Special case for hooks, 'send_now' is the only currently + // allowed member + DMLShallowObjectVariant::Hook(_) => if name == "send_now" { + ReferenceMatch::Found( + vec![Arc::clone( + self.symbol_info.variable_symbols + .get(obj.location()).unwrap())]) + } else { + ReferenceMatch::NotFound(vec![]) + }, + _ => { + error!("Internal error: Wanted to lookup symbol {} in \ + {:?}, but that's not something that can \ + contain symbols", + name, obj); + // NOTE: This is not a typeerror, but an internal error + ReferenceMatch::NotFound(vec![]) + }, + } + } + } + + fn lookup_global_from_noderef(&self, node: &NodeRef) + -> ReferenceMatch { + let mut symbols = vec![]; + let suggestions = vec![]; + if let NodeRef::Simple(name) = node { + if let Some(templ) = self.templates.templates.get( + name.val.as_str()) { + if let Some(templ_loc) = &templ.location { + if let Some(templ_sym) = self.symbol_info + .template_symbols.get(templ_loc) { + symbols.push(Arc::clone(templ_sym)); + } else { + error!("Unexpectedly missing a template {}", + name.val); + } + } + } + } + // TODO: types + // TODO: externs + if !symbols.is_empty() { + ReferenceMatch::Found(symbols) + } else { + ReferenceMatch::NotFound(suggestions) + } + } + + fn lookup_global_from_ref(&self, reference: &GlobalReference) + -> ReferenceMatch { + match &reference.kind { + ReferenceKind::Template => { + if let Some(templ) = self.templates + .templates.get(&reference.name) { + // Dummy templates have no loc, do not actually exist + if let Some(templ_loc) = &templ.location { + if let Some(templ_sym) = + self.symbol_info.template_symbols + .get(templ_loc) { + return ReferenceMatch::Found( + vec![Arc::clone(templ_sym)]) + } else { + error!("Unexpectedly missing a template {}", + reference.name); + } + } + } + ReferenceMatch::NotFound(vec![]) + }, + ReferenceKind::Type => + ReferenceMatch::NotFound(vec![]), + _ => { + error!("Invalid global reference kind in {:?}", reference); + ReferenceMatch::NotFound(vec![]) + }, + } + } + + fn lookup_global_symbol(&self, sym: &SimpleSymbol) + -> ReferenceMatch { + match sym.kind { + DMLSymbolKind::Template => { + if let Some(sym) = self.symbol_info.template_symbols.get(&sym.loc) { + ReferenceMatch::Found(vec![Arc::clone(sym)]) + } else { + // I dont think this can happen, so show an error + error!("Unexpectedly missing a template {}", + sym.name); + ReferenceMatch::NotFound(vec![]) + } + }, + _ => ReferenceMatch::NotFound(vec![]), + } + } + + fn lookup_def_in_obj(&self, obj: &DMLObject, sym: &SimpleSymbol) + -> ReferenceMatch { + let resolved = obj.resolve(&self.objects); + if resolved.as_comp().map_or(false, |c|c.kind == CompObjectKind::Device) + && sym.kind == DMLSymbolKind::Template { + self.lookup_global_symbol(sym) + } else { + self.lookup_def_in_resolved(resolved, + sym.name_ref(), + None) + } + } + + fn lookup_global_sym(&self, sym: &SimpleSymbol) -> Vec { + //TODO: being able to fail a re-match here feels silly. consider + // adding sub-enum for DMLGlobalSymbolKind + match sym.kind { + DMLSymbolKind::Template => { + if let Some((templ, _)) = self.templates.get_template( + sym.name.as_str()) { + // This _should_ be guaranteed, since the SimpleSymbol + // ref comes from structure + vec![Arc::clone(self.symbol_info.template_symbols.get( + templ.location.as_ref().unwrap()).unwrap())] + } else { + error!("Unexpectedly missing template matching {:?}", sym); + vec![] + } + }, + // TODO: DMLType lookup + DMLSymbolKind::Typedef => vec![], + // TODO: Extern lookup + DMLSymbolKind::Extern => vec![], + e => { + error!("Internal error: Unexpected symbol kind of global \ + symbol: {:?}", e); + vec![] + }, + } + } + + pub fn lookup_symbols_by_contexted_symbol<'t>(&self, + sym: &ContextedSymbol<'t>) + -> Vec { + if matches!(sym.symbol.kind, DMLSymbolKind::Template | + DMLSymbolKind::Typedef | DMLSymbolKind::Extern) + { + return self.lookup_global_sym(sym.symbol); + } + + debug!("Looking up {:?} in device tree", sym); + + let mb_objs = if sym.contexts.is_empty() { + Some(vec![self.get_device_obj().clone()]) + } else { + self.context_to_objs(self.get_device_comp_obj(), + sym.contexts.iter() + .cloned().cloned() + .collect::>() + .as_slice()) + }; + + if let Some(objs) = mb_objs { + debug!("Found {:?}", objs); + objs.into_iter() + .map(|o|self.lookup_def_in_obj(&o, sym.symbol)) + .filter_map(|rm|match rm { + ReferenceMatch::Found(syms) => Some(syms), + _ => None, + }) + .flatten() + .unique_by(|s|s.lock().unwrap().loc) + .collect() + + } else { + // TODO: Do we need to fall back on globals here? Can we get an + // identifier from a spot that is generic enough to refer to a + // global, but also is in a context where a global makes sense? + error!("Internal Error: Failed to find objects matching {:?}", + sym); + vec![] + } + } + + fn resolve_noderef_in_symbol<'t>(&'t self, + symbol: &'t SymbolRef, + node: &NodeRef) + -> ReferenceMatch { + if let Ok(sym) = symbol.lock() { + match &sym.source { + SymbolSource::DMLObject(key) => { + self.resolve_noderef_in_obj(key, node) + }, + // TODO: Cannot be resolved without constant folding + SymbolSource::MethodArg(_method, _name) => + ReferenceMatch::NotFound(vec![]), + // TODO: Fix once type system is sorted + SymbolSource::Type(_typed) => + ReferenceMatch::NotFound(vec![]), + // TODO: Handle lookups inside templates + SymbolSource::Template(_templ) => + ReferenceMatch::NotFound(vec![]), + } + } else { + error!("Internal Error?: Circular noderef resolve"); + ReferenceMatch::NotFound(vec![]) + } + } + + fn resolve_noderef_in_obj<'c>(&'c self, + obj: &DMLObject, + node: &NodeRef) + -> ReferenceMatch { + match node { + NodeRef::Simple(simple) => { + let resolvedobj = obj.resolve(&self.objects); + self.lookup_def_in_resolved(resolvedobj, + &simple.val, + None) + }, + NodeRef::Sub(subnode, simple, _) => { + let sub = self.resolve_noderef_in_obj(obj, subnode); + match sub { + ReferenceMatch::Found(syms) => { + let wrapped_simple = NodeRef::Simple(simple.clone()); + let mut found_syms = vec![]; + let mut suggestions = vec![]; + for sub_result in syms.into_iter().map( + |sym|self.resolve_noderef_in_symbol( + &sym, &wrapped_simple)) { + match sub_result { + ReferenceMatch::Found(mut sub_syms) => + found_syms.append(&mut sub_syms), + ReferenceMatch::NotFound(mut sub_suggestions) => + suggestions.append(&mut sub_suggestions), + // No need to have this be exhaustive, + // early return is ok + wt @ ReferenceMatch::WrongType(_) => return wt, + } + } + if !found_syms.is_empty() { + ReferenceMatch::Found(found_syms) + } else { + ReferenceMatch::NotFound(suggestions) + } + }, + other => other, + } + } + } + } + + fn lookup_ref_in_obj(&self, + obj: &DMLObject, + reference: &VariableReference) + -> ReferenceMatch { + match &reference.kind { + ReferenceKind::Template | + ReferenceKind::Type => { + error!("Internal Error: Attempted to do a contexted lookup \ + of a global reference {:?}", reference); + return ReferenceMatch::NotFound(vec![]); + }, + _ => (), + } + self.resolve_noderef_in_obj(obj, &reference.reference) + // TODO: Could sanity the result towards the referencekind here + } + + pub fn lookup_symbols_by_contexted_reference( + &self, + context_chain: &[ContextKey], + reference: &VariableReference) -> ReferenceMatch { + debug!("Looking up {:?} : {:?} in device tree", context_chain, + reference); + if let Some(objs) = self.contexts_to_objs( + vec![self.get_device_obj().clone()], + context_chain) { + let mut syms = vec![]; + let mut suggestions = vec![]; + for result in objs.into_iter().map( + |o|self.lookup_ref_in_obj(&o, reference)) { + match result { + ReferenceMatch::Found(mut found_syms) => + syms.append(&mut found_syms), + w @ ReferenceMatch::WrongType(_) => + return w, + ReferenceMatch::NotFound(mut more_suggestions) => + suggestions.append(&mut more_suggestions), + } + } + if syms.is_empty() { + ReferenceMatch::NotFound(suggestions) + } else { + ReferenceMatch::Found(syms) + } + } else { + ReferenceMatch::NotFound(vec![]) + } + } + + fn find_target_for_reference( + &self, + context_chain: &[ContextKey], + reference: &VariableReference, + reference_cache: &Mutex) + -> ReferenceMatch { + let index_key = (context_chain.to_vec(), + reference.clone()); + { + if let Some(cached_result) = reference_cache.lock().unwrap() + .get(index_key.clone()) { + return cached_result.clone(); + } + } + + let result = self.find_target_for_reference_aux( + context_chain, reference, reference_cache); + reference_cache.lock().unwrap() + .insert(index_key, result.clone()); + result + } + + fn find_target_for_reference_aux( + &self, + context_chain: &[ContextKey], + reference: &VariableReference, + reference_cache: &Mutex) + -> ReferenceMatch { + if context_chain.is_empty() { + // Nothing matches the noderef except maybe globals + return self.lookup_global_from_noderef(&reference.reference); + } + + match self.lookup_symbols_by_contexted_reference( + // Ignore first element of chain, it is the device context + &context_chain[1..], reference) { + f @ ReferenceMatch::Found(_) => f, + c @ ReferenceMatch::WrongType(_) => c, + ReferenceMatch::NotFound(mut suggestions) => { + let (_, new_chain) = context_chain.split_last().unwrap(); + match self.find_target_for_reference(new_chain, + reference, + reference_cache) { + f @ ReferenceMatch::Found(_) => f, + c @ ReferenceMatch::WrongType(_) => c, + ReferenceMatch::NotFound(mut rest_suggestions) => { + suggestions.append(&mut rest_suggestions); + ReferenceMatch::NotFound(suggestions) + }, + } + }, + } + } +} + +impl DeviceAnalysis { + fn match_references_in_scope<'c>( + &'c self, + scope_chain: Vec<&'c dyn Scope>, + _report: &Mutex>, + reference_cache: &Mutex) { + let current_scope = scope_chain.last().unwrap(); + let context_chain: Vec = scope_chain + .iter().map(|s|s.create_context()).collect(); + // NOTE: chunk number is arbitrarily picked that benches well + current_scope.defined_references().par_chunks(25).for_each(|references|{ + for reference in references { + debug!("In {:?}, Matching {:?}", context_chain, reference); + let symbol_lookup = match &reference { + Reference::Variable(var) => self.find_target_for_reference( + context_chain.as_slice(), var, reference_cache), + Reference::Global(glob) => + self.lookup_global_from_ref(glob), + }; + + match symbol_lookup { + ReferenceMatch::NotFound(_suggestions) => + // TODO: report suggestions? + // TODO: Uncomment reporting of errors here when + // semantics are strong enough that they are rare + // for correct devices + // report.lock().unwrap().push(DMLError { + // span: reference.span().clone(), + // description: format!("Unknown reference {}", + // reference.to_string()), + // related: vec![], + // }) + (), + // This maps symbols->references, this is later + // used to create the inverse map + // (not done here because of ownership issues) + ReferenceMatch::Found(symbols) => + for symbol in &symbols { + let mut sym = symbol.lock().unwrap(); + sym.references.push(*reference.loc_span()); + if let Some(meth) = sym.source + .as_object() + .and_then(DMLObject::as_shallow) + .and_then(|s|s.variant.as_method()) { + if let Some(default_decl) = meth.get_default() { + if let Some(var) = reference.as_variable_ref() { + if var.reference.to_string().as_str() + == "default" { + sym.default_mappings.insert( + *var.loc_span(), + *default_decl.location()); + } + } + } + } + }, + ReferenceMatch::WrongType(_) => + //TODO: report mismatch, + (), + } + } + }) + } +} + +pub fn parse_file(path: &Path, file: FileSpec<'_>) + -> Result<(parsing::structure::TopAst, + ProvisionalsManager, + Vec), Error> +{ + let content = &file.file.text; + let lexer = TokenKind::lexer(content); + let mut parser = FileParser::new(lexer); + let mut parse_state = FileInfo::default(); + let ast = parsing::structure::parse_toplevel( + &mut parser, &mut parse_state, file); + let mut skipped_errors = parser.report_skips(); + let mut missing_errors = ast.report_missing(); + missing_errors.append(&mut skipped_errors); + parsing::structure::post_parse_toplevel( + &ast, file.file, &mut missing_errors); + // TODO: sort errors + // NOTE: I dont know how to do rust iterators + let errors = missing_errors.into_iter() + .map(|e|e.with_file(path)).collect(); + Ok((ast, parse_state.provisionals, errors)) +} + +fn collect_toplevel(path: &Path, tree: &parsing::structure::TopAst, + errors: &mut Vec, + file: FileSpec<'_>) -> TopLevel { + let mut report = vec![]; + let toplevel = TopLevel::from_ast(tree, &mut report, file); + for error in report { + errors.push(error.with_file(path)); + } + toplevel +} + +pub fn deconstruct_import(import: &ObjectDecl) -> PathBuf { + PathBuf::from(import.obj.imported_name()) +} + +impl fmt::Display for IsolatedAnalysis { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + writeln!(f, "IsolatedAnalysis {{")?; + writeln!(f, "\tpath: {}", self.path.as_str())?; + writeln!(f, "\timports: [\n{}]", + self.toplevel.spec.imports.iter().map(deconstruct_import) + .fold("".to_string(), + |mut s, path|{ + write!(s, "\t\t{:?}\n, ", path) + .expect("write string error"); + s + }))?; + writeln!(f, "\ttoplevel: {}\n}}", self.toplevel)?; + Ok(()) + } +} + +type ResolvedImports = (HashSet<(CanonPath, Import)>, + HashSet<(PathBuf, Import)>); + +impl IsolatedAnalysis { + pub fn new(path: &CanonPath, clientpath: &PathBuf, file: TextFile) + -> Result { + trace!("local analysis: {} at {}", path.as_str(), path.as_str()); + let filespec = FileSpec { + path, file: &file + }; + let (mut ast, provisionals, mut errors) = parse_file(path, filespec)?; + + // Add invalid provisionals to errors + for duped_provisional in &provisionals.duped_provisionals { + errors.push(DMLError { + span: duped_provisional.span, + description: format!("Duplicate activation of provisional '{}'", + duped_provisional.val), + severity: Some(DiagnosticSeverity::ERROR), + // TODO: Could report the original declaration here, but they + // are very close in source so probably unneccessary + related: vec![], + }); + } + for invalid_provisional in &provisionals.invalid_provisionals { + errors.push(DMLError { + span: invalid_provisional.span, + description: format!("Invalid or unknown provisional '{}', \ + will be ignored", + invalid_provisional.val), + severity: Some(DiagnosticSeverity::ERROR), + related: vec![], + }); + } + + // Peek into the ast and read from the file to check if version is 1.4 + let non_14_version = + if let Some(version) = ast.version.version.read_leaf(&file) { + if version != "1.4" { + Some(ast.version.range()) + } else { + None + } + } else { + None + }; + + if let Some(location) = non_14_version { + errors = vec![LocalDMLError { + range: location, + description: + "The language server only supports DML 1.4 files" + .to_string(), + }.with_file(path.as_path())]; + ast = parsing::structure::TopAst { + version: ast.version, + device: None, + provisionals : None, + bitorder: None, + declarations: vec![], + }; + info!("Bailed on further analysis of {} due to it not being a \ + DML 1.4 file", clientpath.display()); + } + + let toplevel = collect_toplevel(path, &ast, + &mut errors, filespec); + // sanity, clientpath and path should be the same file + if CanonPath::from_path_buf(clientpath.clone()).map_or( + true, |cp|&cp != path) { + error!("Clientpath did not describe the same \ + file as the actual path; {:?} vs {:?}", + path, + clientpath); + return Err(Error::InternalError( + "Clientpath did not describe the same file as path")); + } + let top_context = toplevel.to_context(); + let res = IsolatedAnalysis { + ast, + toplevel, + top_context, + path: path.clone(), + clientpath: clientpath.clone(), + errors, + }; + info!("Produced an isolated analysis of {:?}", res.path); + debug!("Produced an isolated analysis: {}", res); + Ok(res) + } + + pub fn get_imports(&self) -> &Vec> { + &self.toplevel.spec.imports + } + + pub fn get_import_names(&self) -> Vec { + self.get_imports().iter().map( + |imp|deconstruct_import(imp)).collect() + } + + pub fn resolve_imports(&self, + resolver: &PathResolver, + context: Option<&CanonPath>) + -> ResolvedImports + { + let mut found = HashSet::default(); + let mut missing = HashSet::default(); + let import_paths = self.get_imports().iter() + .map(|i|(deconstruct_import(i), + i.clone())); + // Patch in implicit dependencies here. These won't affect template + // or file ordering. But we DO want to make sure they are imported + let import_paths = import_paths.chain( + IMPLICIT_IMPORTS.iter().map( + |import|(import.into(), + ObjectDecl::always(&Import { + span: ZeroSpan::invalid(self.path.clone()), + name: DMLString { + val: format!("\"{}\"", import), + span: ZeroSpan::invalid(self.path.clone()), + } + })))); + + for (path, import) in import_paths { + if let Some(found_path) = resolver.resolve_with_maybe_context( + &path, context) { + found.insert((found_path, import.obj)); + } else { + missing.insert((path, import.obj)); + } + } + (found, missing) + } + + pub fn is_device_file(&self) -> bool { + self.toplevel.device.is_some() + } + + pub fn lookup_context_symbol<'t>(&'t self, pos: &ZeroFilePosition) + -> Option> { + self.top_context.lookup_symbol(pos) + } + + pub fn lookup_reference(&self, pos: &ZeroFilePosition) + -> Option<&Reference> { + self.toplevel.reference_at_pos(pos) + } +} + +fn objects_to_symbols(objects: &StructureContainer) -> SymbolStorage { + let mut storage = SymbolStorage::default(); + + for obj in objects.values() { + let new_symbol: SymbolRef = new_symbol_from_object(obj); + debug!("Comp obj symbol is: {:?}", new_symbol); + storage.object_symbols.insert(obj.key, new_symbol); + for subobj in obj.components.values() { + // Non-shallow objects will be handled by the iteration + // over objects + if let DMLObject::ShallowObject(shallow) = subobj { + add_new_symbol_from_shallow(shallow, &mut storage); + } + } + } + storage +} + +fn template_to_symbol(template: &Arc) -> Option { + // Do not create symbols for templates without location, they are dummy + // missing templates + template.location.as_ref().map(|location| { + Arc::new(Mutex::new(Symbol { + loc: *location, + kind: DMLSymbolKind::Template, + references: vec![], + definitions: vec![*location], + declarations: vec![*location], + implementations: vec![], + bases: vec![], + source: SymbolSource::Template(Arc::clone(template)), + default_mappings: HashMap::default(), + }))}) +} + +fn extend_with_templates(storage: &mut SymbolStorage, + templates: &TemplateTraitInfo) { + for template in templates.templates.values() { + if let Some(new_templ) = template_to_symbol(template) { + let loc = new_templ.lock().unwrap().loc; + if let Some(prev) = storage.template_symbols + .insert(loc, new_templ) { + error!("Internal Error: Unexpectedly two template symbols + defined in the same location"); + error!("Previous was {:?}", prev); + error!("New is {:?}", storage.template_symbols.get(&loc)); + } + } + } +} + +fn new_symbol_from_object(object: &DMLCompositeObject) -> SymbolRef { + let all_decl_defs: Vec = object.all_decls.iter().map( + |spec|*spec.loc_span()).collect(); + Arc::new(Mutex::new(Symbol { + loc: object.declloc, + kind: DMLSymbolKind::CompObject(object.kind), + definitions: all_decl_defs.clone(), + declarations: all_decl_defs.clone(), + bases: all_decl_defs, + references: vec![], + implementations: vec![], + source: SymbolSource::DMLObject( + DMLObject::CompObject(object.key)), + default_mappings: HashMap::default(), + })) +} + +fn new_symbol_from_arg(methref: &Arc, + arg: &DMLMethodArg) -> SymbolRef { + let bases = vec![*arg.loc_span()]; + let definitions = vec![*arg.loc_span()]; + let declarations = vec![*arg.loc_span()]; + Arc::new(Mutex::new(Symbol { + loc: *arg.loc_span(), + kind: DMLSymbolKind::MethodArg, + bases, + definitions, + declarations, + references: vec![], + implementations: vec![], + source: SymbolSource::MethodArg(Arc::clone(methref), + arg.name().clone()), + default_mappings: HashMap::default(), + })) +} + +fn log_non_same_insert(map: &mut HashMap, + key: K, + val: SymbolRef) -> bool +where K: std::hash::Hash + Eq + Clone, +{ + // NOTE: We should not need to do these comparisons, when + // object symbol creation is properly guided by structural AST + // this code can be simplified and the comparison discarded + + // NOTE: Insert first rather than checking for key is faster, I think + if let Some(old) = map.insert(key.clone(), val) { + // NOTE: the equivalent operation is a slight-better-than-eq + // comparison, skipping the comparison of structure keys and + // some meta-info + if !old.lock().unwrap().equivalent( + &map.get(&key).unwrap().lock().unwrap()) { + error!("Unexpected Internal Error: Overwrote previous symbol \ + {:?} with non-similar symbol {:?}", + old, map.get(&key)); + return true; + } + } + false +} + +fn add_new_symbol_from_shallow(shallow: &DMLShallowObject, + storage: &mut SymbolStorage) { + let (bases, definitions, declarations) = match &shallow.variant { + DMLShallowObjectVariant::Parameter(param) => + (vec![*param.get_likely_declaration().loc_span()], + param.used_definitions.iter() + .map(|(_, def)|*def.loc_span()).collect(), + param.declarations.iter() + .map(|(_, def)|*def.loc_span()).collect()), + DMLShallowObjectVariant::Method(method_ref) => + (vec![*method_ref.get_base().location()], + method_ref.get_all_defs(), + method_ref.get_all_decls()), + DMLShallowObjectVariant::Constant(constant) => + (vec![*constant.loc_span()], + vec![*constant.loc_span()], + vec![*constant.loc_span()]), + DMLShallowObjectVariant::Session(data) | + DMLShallowObjectVariant::Saved(data) => + (vec![*data.declaration.loc_span()], + vec![*data.declaration.loc_span()], + vec![*data.declaration.loc_span()]), + DMLShallowObjectVariant::Hook(hook) => + (vec![*hook.loc_span()], + vec![*hook.loc_span()], + vec![*hook.loc_span()]), + }; + debug!("Made symbol for {:?}", shallow); + let new_sym = Arc::new(Mutex::new(Symbol { + loc: *shallow.location(), + kind: shallow.kind(), + definitions, + declarations, + implementations: vec![], + references: vec![], + bases, + source: SymbolSource::DMLObject( + // TODO: Inefficient clone. Not terribly so, but worth + // noting + DMLObject::ShallowObject(shallow.clone())), + default_mappings: HashMap::default(), + })); + match &shallow.variant { + DMLShallowObjectVariant::Parameter(_) => { + log_non_same_insert(storage.param_symbols.entry( + (*shallow.location(), + shallow.identity().to_string())) + .or_default(), + shallow.parent, + new_sym); + }, + DMLShallowObjectVariant::Method(method_ref) => + if !log_non_same_insert(&mut storage.method_symbols, + *shallow.location(), + new_sym) { + for arg in method_ref.args() { + let new_argsymbol = new_symbol_from_arg(method_ref, arg); + log_non_same_insert(&mut storage.variable_symbols, + *arg.loc_span(), + new_argsymbol); + } + }, + DMLShallowObjectVariant::Constant(_) | + DMLShallowObjectVariant::Session(_) | + DMLShallowObjectVariant::Saved(_) | + DMLShallowObjectVariant::Hook(_) => { + log_non_same_insert(&mut storage.variable_symbols, + *shallow.location(), + new_sym); + }, + } +} + +impl DeviceAnalysis { + pub fn new(root: IsolatedAnalysis, + timed_bases: Vec>, + imp_map: HashMap) + -> Result { + info!("device analysis: {:?}", root.path); + + if root.toplevel.device.is_none() { + return Err(Error::InternalError( + "Attempted to device analyze a file without device decl")); + } + + let mut bases: Vec<_> = + timed_bases.into_iter().map(|tss|tss.stored).collect(); + + // Fake the implicit imports into the root toplevel + // We do this into bases, because that is the analysises that are + // used in analysis + for base in &mut bases { + if base.path == root.path { + for import in IMPLICIT_IMPORTS { + base.toplevel.spec.imports.push(ObjectDecl::always(&Import { + span: ZeroSpan::invalid(root.path.clone()), + name: DMLString { + val: format!("\"{}\"", import), + span: ZeroSpan::invalid(root.path.clone()), + } + })); + } + } + } + + let base_templates = from_device_and_bases(&root, &bases); + + trace!("base template names: {:?}", base_templates.iter() + .fold("".to_string(), + |mut s, odecl|{ + write!(s, "{}, ", odecl.obj.object.name.val) + .expect("write string error"); + s + })); + + let mut errors = vec![]; + + // Remove duplicate templates + let mut unique_templates: HashMap<&str, &ObjectDecl