diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..1e435d27 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @gsingers @macohen \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/BUG_TEMPLATE.md b/.github/ISSUE_TEMPLATE/BUG_TEMPLATE.md new file mode 100644 index 00000000..1d2ea2d2 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/BUG_TEMPLATE.md @@ -0,0 +1,24 @@ +--- +name: 🐛 Bug report +about: Create a report to help us improve +title: '[BUG]' +labels: 'bug, untriaged' +assignees: '' +--- +### What is the bug? +_A clear and concise description of the bug._ + +### How can one reproduce the bug? +_Steps to reproduce the behavior._ + +### What is the expected behavior? +_A clear and concise description of what you expected to happen._ + +### What is your host/environment? +_Operating system, version._ + +### Do you have any screenshots? +_If applicable, add screenshots to help explain your problem._ + +### Do you have any additional context? +_Add any other context about the problem._ diff --git a/.github/ISSUE_TEMPLATE/FEATURE_REQUEST_TEMPLATE.md b/.github/ISSUE_TEMPLATE/FEATURE_REQUEST_TEMPLATE.md new file mode 100644 index 00000000..55b3ddf3 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/FEATURE_REQUEST_TEMPLATE.md @@ -0,0 +1,18 @@ +--- +name: 🎆 Feature request +about: Request a feature in this project +title: '[FEATURE]' +labels: 'enhancement, untriaged' +assignees: '' +--- +### Is your feature request related to a problem? +_A clear and concise description of what the problem is, e.g. I'm always frustrated when [...]._ + +### What solution would you like? +_A clear and concise description of what you want to happen._ + +### What alternatives have you considered? +_A clear and concise description of any alternative solutions or features you've considered._ + +### Do you have any additional context? +_Add any other context or screenshots about the feature request here._ diff --git a/.github/ISSUE_TEMPLATE/PROPOSAL_TEMPLATE.md b/.github/ISSUE_TEMPLATE/PROPOSAL_TEMPLATE.md new file mode 100644 index 00000000..9276d050 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/PROPOSAL_TEMPLATE.md @@ -0,0 +1,40 @@ +--- +name: 💭 Proposal +about: Suggest an idea for a specific feature you wish to propose to the community for comment +title: '[PROPOSAL]' +labels: proposal +assignees: '' +--- +## What/Why +### What are you proposing? +_In a few sentences, describe the feature and its core capabilities._ + +### What users have asked for this feature? +_Highlight any research, proposals, requests or anecdotes that signal this is the right thing to build. Include links to GitHub Issues, Forums, Stack Overflow, Twitter, Etc_ + +### What problems are you trying to solve? +_Summarize the core use cases and user problems and needs you are trying to solve. Describe the most important user needs, pain points and jobs as expressed by the user asks above. Template: When \ , a \ wants to \, so they can \. (Example: When **searching by postal code**, **a buyer** wants to **be required to enter a valid code** so they **don’t waste time searching for a clearly invalid postal code.**)_ + +### What is the developer experience going to be? +_Does this have a REST API? If so, please describe the API and any impact it may have to existing APIs. In a brief summary (not a spec), highlight what new REST APIs or changes to REST APIs are planned. as well as any other API, CLI or Configuration changes that are planned as part of this feature._ + +#### Are there any security considerations? +_Describe if the feature has any security considerations or impact. What is the security model of the new APIs? Features should be integrated into the OpenSearch security suite and so if they are not, we should highlight the reasons here._ + +#### Are there any breaking changes to the API +_If this feature will require breaking changes to any APIs, ouline what those are and why they are needed. What is the path to minimizing impact? (example, add new API and deprecate the old one)_ + +### What is the user experience going to be? +_Describe the feature requirements and or user stories. You may include low-fidelity sketches, wireframes, APIs stubs, or other examples of how a user would use the feature via CLI, OpenSearch Dashboards, REST API, etc. Using a bulleted list or simple diagrams to outline features is okay. If this is net new functionality, call this out as well._ + +#### Are there breaking changes to the User Experience? +_Will this change the existing user experience? Will this be a breaking change from a user flow or user experience perspective?_ + +### Why should it be built? Any reason not to? +_Describe the value that this feature will bring to the OpenSearch community, as well as what impact it has if it isn't built, or new risks if it is. Highlight opportunities for additional research._ + +### What will it take to execute? +_Describe what it will take to build this feature. Are there any assumptions you may be making that could limit scope or add limitations? Are there performance, cost, or technical constraints that may impact the user experience? Does this feature depend on other feature work? What additional risks are there?_ + +### Any remaining open questions? +_What are known enhancements to this feature? Any enhancements that may be out of scope but that we will want to track long term? List any other open questions that may need to be answered before proceeding with an implementation._ diff --git a/.github/config.yml b/.github/config.yml new file mode 100644 index 00000000..52a7630c --- /dev/null +++ b/.github/config.yml @@ -0,0 +1,7 @@ +contact_links: + - name: OpenSearch Community Support + url: https://forum.opensearch.org + about: Please ask and answer questions here. + - name: AWS/Amazon Security + url: https://aws.amazon.com/security/vulnerability-reporting/ + about: Please report security vulnerabilities here. \ No newline at end of file diff --git a/.github/workflows/add-untriaged-label.yml b/.github/workflows/add-untriaged-label.yml new file mode 100644 index 00000000..bfff875b --- /dev/null +++ b/.github/workflows/add-untriaged-label.yml @@ -0,0 +1,19 @@ +name: Apply 'untriaged' label during issue lifecycle + +on: + issues: + types: [opened, transferred] + +jobs: + apply-label: + runs-on: ubuntu-latest + steps: + - uses: actions/github-script@v6 + with: + script: | + github.rest.issues.addLabels({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + labels: ['untriaged'] + }) \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..4b07d8a9 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,49 @@ +name: Build & Test Learning to Rank Plugin + +on: + schedule: + - cron: '0 0 * * *' # every night + push: + pull_request: + workflow_dispatch: + +jobs: + build-ltr-plugin: + strategy: + matrix: + java: [11, 17] + os: [ubuntu-latest, macos-latest] + + name: Build and Test Learning to Rank Plugin + runs-on: ${{ matrix.os }} + + steps: + - name: Checkout LTR Plugin + uses: actions/checkout@v1 + + - name: Setup Java ${{ matrix.java }} + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + + - name: Run non-Windows + if: ${{ matrix.os != 'windows-latest' }} + run: | + ./gradlew -Dopensearch.version=2.7.0 -Djava.security.manager=allow -Dbuild.snapshot=false -Dtests.security.manager=false clean build test + +# - name: Upload Coverage Report +# if: ${{matrix.os}} == 'ubuntu' +# uses: codecov/codecov-action@v1 +# with: +# token: ${{ secrets.CODECOV_TOKEN }} + + - name: Create Artifact Path + run: | + mkdir -p opensearch-ltr-builds + cp -r ./build/distributions/*.zip opensearch-ltr-builds + + - name: Upload Artifacts + uses: actions/upload-artifact@v1 + with: + name: opensearch-ltr-${{ matrix.os }} + path: opensearch-ltr-builds diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 00000000..f7070a08 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,41 @@ +name: Publish a Docker Image + +on: + release: + types: [published] + workflow_dispatch: + inputs: + opensearch-version: + description: 'OpenSearch Version to use' + required: true + default: 2.7.0 + ltrversion: + description: 'LTR Plugin Version' + required: true + default: 2.7.0 + + +jobs: + build_push_image: + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Set up Docker Buildx + id: setup-docker-buildx + uses: docker/setup-buildx-action@v2 + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Build and push + uses: docker/build-push-action@v3 + with: + file: docker/Dockerfile + build-args: | + opensearch_version=${{inputs.opensearch-version}} + ltrversion=${{inputs.ltrversion}} + push: true + tags: opensearch-project/opensearch-learning-to-rank:${{inputs.ltrversion}} + github-token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index b72199b0..00000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,47 +0,0 @@ -# This workflow will build a Java project with Gradle -# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-gradle - -name: Tag and publish a release - -on: - push: - tags: - - 'v*.*.*' - -jobs: - release: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set release version Name - run: echo ::set-env name=RELEASE_VERSION::${GITHUB_REF#refs/*/} - - name: Set up JDK 14.0 - uses: actions/setup-java@v1 - with: - java-version: 14.0 - - name: Grant execute permission for gradlew - run: chmod +x gradlew - - name: Build with Gradle - run: ./gradlew build - - name: Rename build assets - run: mv ./build/distributions/ltr-*.zip ./ltr-plugin-${{ env.RELEASE_VERSION }}.zip - - name: Create Release - id: create_release - uses: actions/create-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: ${{ env.RELEASE_VERSION }} - release_name: Release ${{ env.RELEASE_VERSION }} - draft: false - prerelease: false - - name: Upload Release Asset - id: upload-release-asset - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: ./ltr-plugin-${{ env.RELEASE_VERSION }}.zip - asset_name: ltr-plugin-${{ env.RELEASE_VERSION }}.zip - asset_content_type: application/zip diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index dd90f7c8..fce79245 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,20 +5,20 @@ name: Run CI tests with Gradle on: push: - branches: [ master ] + branches: [main] pull_request: - branches: [ master ] + branches: [main] jobs: test: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Set up JDK 14.0 + - name: Set up JDK 17.0 uses: actions/setup-java@v1 with: - java-version: 14.0 + java-version: 17.0 - name: Grant execute permission for gradlew run: chmod +x gradlew - name: Build with Gradle - run: ./gradlew clean check + run: ./gradlew -Dtests.security.manager=false clean test diff --git a/.whitesource b/.whitesource new file mode 100644 index 00000000..d465069a --- /dev/null +++ b/.whitesource @@ -0,0 +1,33 @@ +{ + "scanSettings": { + "configMode": "AUTO", + "configExternalURL": "", + "projectToken": "", + "baseBranches": [] + }, + "scanSettingsSAST": { + "enableScan": false, + "scanPullRequests": false, + "incrementalScan": true, + "baseBranches": [], + "snippetSize": 10 + }, + "checkRunSettings": { + "vulnerableCheckRunConclusionLevel": "failure", + "displayMode": "diff", + "useMendCheckNames": true + }, + "checkRunSettingsSAST": { + "checkRunConclusionLevel": "failure", + "severityThreshold": "high" + }, + "issueSettings": { + "minSeverityLevel": "LOW", + "issueType": "DEPENDENCY" + }, + "remediateSettings": { + "workflowRules": { + "enabled": true + } + } +} \ No newline at end of file diff --git a/ADMIN.md b/ADMIN.md new file mode 100644 index 00000000..92290667 --- /dev/null +++ b/ADMIN.md @@ -0,0 +1,39 @@ +- [Overview](#overview) +- [Current Admins](#current-admins) +- [Admin Responsibilities](#admin-responsibilities) + - [Prioritize Security](#prioritize-security) + - [Enforce Code of Conduct](#enforce-code-of-conduct) + - [Add/Remove Maintainers](#addremove-maintainers) + - [Adopt Organizational Best Practices](#adopt-organizational-best-practices) + +## Overview + +This document explains who the admins are (see below), what they do in this repo, and how they should be doing it. If you're interested in becoming a maintainer, see [MAINTAINERS](MAINTAINERS.md). If you're interested in contributing, see [CONTRIBUTING](CONTRIBUTING.md). + +## Current Admins + +| Admin | GitHub ID | Affiliation | +| ------------------ | --------------------------------------- | ----------- | +| Grant Ingersoll | [gsingers](https://github.com/gsingers) | Develomentor + +## Admin Responsibilities + +As an admin you own stewardship of the repository and its settings. Admins have [admin-level permissions on a repository](https://docs.github.com/en/organizations/managing-access-to-your-organizations-repositories/repository-permission-levels-for-an-organization). Use those privileges to serve the community and protect the repository as follows. + +### Prioritize Security + +Security is your number one priority. Manage security keys and safeguard access to the repository. + +Note that this repository is monitored and supported 24/7 by Amazon Security, see [Reporting a Vulnerability](SECURITY.md) for details. + +### Enforce Code of Conduct + +Act on [CODE_OF_CONDUCT](CODE_OF_CONDUCT.md) violations by revoking access, and blocking malicious actors. + +### Add/Remove Maintainers + +Perform administrative tasks, such as [adding](MAINTAINERS.md#adding-a-new-maintainer) and [removing maintainers](MAINTAINERS.md#removing-a-maintainer). + +### Adopt Organizational Best Practices + +Adopt organizational best practices, work in the open, and collaborate with other admins by opening issues before making process changes. Prefer consistency, and avoid diverging from practices in the opensearch-project organization. diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..8c09f91d --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,24 @@ + +This code of conduct applies to all spaces provided by the OpenSource project including in code, documentation, issue trackers, mailing lists, chat channels, wikis, blogs, social media and any other communication channels used by the project. + +**Our open source communities endeavor to:** + +* Be Inclusive: We are committed to being a community where everyone can join and contribute. This means using inclusive and welcoming language. +* Be Welcoming: We are committed to maintaining a safe space for everyone to be able to contribute. +* Be Respectful: We are committed to encouraging differing viewpoints, accepting constructive criticism and work collaboratively towards decisions that help the project grow. Disrespectful and unacceptable behavior will not be tolerated. +* Be Collaborative: We are committed to supporting what is best for our community and users. When we build anything for the benefit of the project, we should document the work we do and communicate to others on how this affects their work. + +**Our Responsibility. As contributors, members, or bystanders we each individually have the responsibility to behave professionally and respectfully at all times. Disrespectful and unacceptable behaviors include, but are not limited to:** + +* The use of violent threats, abusive, discriminatory, or derogatory language; +* Offensive comments related to gender, gender identity and expression, sexual orientation, disability, mental illness, race, political or religious affiliation; +* Posting of sexually explicit or violent content; +* The use of sexualized language and unwelcome sexual attention or advances; +* Public or private harassment of any kind; +* Publishing private information, such as physical or electronic address, without permission; +* Other conduct which could reasonably be considered inappropriate in a professional setting; +* Advocating for or encouraging any of the above behaviors. + +**Enforcement and Reporting Code of Conduct Issues:** + +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported. [Contact us](mailto:opensource-codeofconduct@amazon.com). All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. diff --git a/KNOWN_ISSUES.md b/KNOWN_ISSUES.md index cdcbed0e..8a1bc920 100644 --- a/KNOWN_ISSUES.md +++ b/KNOWN_ISSUES.md @@ -1,15 +1,4 @@ # Known issues +2.7.0: yamlresttests fail; set -Dbuild.snapshot=false to build successfully -## Cache deadlock - -All elasticsearch versions between 5.5.3 (included) and 6.3.0 (excluded) are affected by a [bug](https://github.com/elastic/elasticsearch/pull/30461) in the internal cache that may lead to a serious deadlock that could in theory leak all the search threads. -A possible workaround is to configure the cache to avoid as much as possible its eviction mechanism by setting these options in your `elasticsearch.yml` file: - -``` -ltr.caches.expire_after_write: 0 -ltr.caches.expire_after_read: 0 -ltr.caches.max_mem: 100mb -``` - -elasticsearch versions 5.6.10, 6.3.0 and onwards should include the bugfix for this bug. diff --git a/MAINTAINERS.md b/MAINTAINERS.md new file mode 100644 index 00000000..b1638394 --- /dev/null +++ b/MAINTAINERS.md @@ -0,0 +1,19 @@ +- [Overview](#overview) +- [Current Maintainers](#current-maintainers) +- [Emeritus](#emeritus) + +## Overview + +This document contains a list of maintainers in this repo. See [opensearch-project/opensearch-learning-to-rank-base/RESPONSIBILITIES.md](https://github.com/opensearch-project/opensearch-learning-to-rank-base/blob/main/RESPONSIBILITIES.md#maintainer-responsibilities) that explains what the role of maintainer means, what maintainers do in this and other repos, and how they should be doing it. If you're interested in contributing, and becoming a maintainer, see [CONTRIBUTING](CONTRIBUTING.md). + +## Current Maintainers + +| Maintainer | GitHub ID | Affiliation | +| ------------------ | --------------------------------------------------------- | ----------- | +| Mark Cohen | [macohen](https://github.com/macohen) | Amazon | +| Grant Ingersoll | [gsingers](https://github.com/gsingers) | Develomentor| + +## Emeritus + +| Maintainer | GitHub ID | Affiliation | +| ------------------ | --------------------------------------------------------- | ----------- | diff --git a/NOTICE b/NOTICE new file mode 100644 index 00000000..46ffe3ba --- /dev/null +++ b/NOTICE @@ -0,0 +1,2 @@ +OpenSearch (https://opensearch.org/) +Copyright OpenSearch Contributors \ No newline at end of file diff --git a/README.md b/README.md index 2defe7e0..525e34b6 100644 --- a/README.md +++ b/README.md @@ -1,49 +1,100 @@ +# Installing + +To install, you'd run a command like this but replacing with the appropriate prebuilt version zip: + +| OS | Command | +|-------|--------------------------------------------------------------------------------------------------------------------------------------------| +| 1.0.0 | `bin/opensearch-plugin install https://github.com/aparo/opensearch-learning-to-rank/releases/download/1.0.0/ltr-1.5.4-os1.0.0.zip` | +| 1.1.0 | `bin/opensearch-plugin install https://github.com/aparo/opensearch-learning-to-rank/releases/download/1.1.0/ltr-1.5.4-os1.1.0.zip` | +| 1.2.0 | `bin/opensearch-plugin install https://github.com/aparo/opensearch-learning-to-rank/releases/download/1.2.0/ltr-1.5.4-os1.2.0.zip` | +| 1.2.2 | `bin/opensearch-plugin install https://github.com/aparo/opensearch-learning-to-rank/releases/download/1.2.2/ltr-1.5.4-os1.2.2.zip` | +| 1.2.3 | `bin/opensearch-plugin install https://github.com/aparo/opensearch-learning-to-rank/releases/download/1.2.3/ltr-1.5.4-os1.2.3.zip` | +| 2.2.1 | `bin/opensearch-plugin install https://github.com/aparo/opensearch-learning-to-rank/releases/download/2.2.1/ltr-2.0.0-os2.2.1.zip` | +| 2.5.0 | `bin/opensearch-plugin install https://github.com/gsingers/opensearch-learning-to-rank-base/releases/download/release-v2.1.0/ltr-plugin-v2.1.0.zip` | +| 2.7.0 | `bin/opensearch-plugin install https://github.com/opensearch-project/opensearch-learning-to-rank-base/releases/download/release-v2.7.0/opensearch-ltr-v2.7.0.zip` + +(It's expected you'll confirm some security exceptions, you can pass `-b` to `opensearch-plugin` to automatically install) + +If you already are running OpenSearch, don't forget to restart! + # Important Notice -This is a fork of https://github.com/o19s/elasticsearch-learning-to-rank to work with OpenSearch. It's a rewrite of some parts to be able to work with OpenSearch 1.x. Please refer to official documentation of [Elasticsearch Learning to Rank](http://elasticsearch-learning-to-rank.readthedocs.io) for usage. +This is a fork of https://github.com/o19s/elasticsearch-learning-to-rank to work with OpenSearch. It's a rewrite of some parts to be able to work with OpenSearch. Please refer to official documentation of [Elasticsearch Learning to Rank](http://elasticsearch-learning-to-rank.readthedocs.io) for usage. -The OpenSearch Learning to Rank plugin uses machine learning to improve search relevance ranking. It's powering search at places like Wikimedia Foundation and Snagajob! +The OpenSearch Learning to Rank plugin uses machine learning to improve search relevance ranking. The original Elasticsearch LTR plugin powers search at places like Wikimedia Foundation and Snagajob. -# Installing +# Releases -To install, you'd run a command like this but replacing with the appropriate prebuilt version zip: +Releases can be found at https://github.com/opensearch-project/opensearch-learning-to-rank-base/releases. -| OS | Command | -| ------ | ------- | -| 1.0.0 | `bin/opensearch-plugin install https://github.com/aparo/opensearch-learning-to-rank/releases/download/1.0.0/ltr-1.5.4-os1.0.0.zip` | -| 1.1.0 | `bin/opensearch-plugin install https://github.com/aparo/opensearch-learning-to-rank/releases/download/1.1.0/ltr-1.5.4-os1.1.0.zip` | -| 1.2.0 | `bin/opensearch-plugin install https://github.com/aparo/opensearch-learning-to-rank/releases/download/1.2.0/ltr-1.5.4-os1.2.0.zip` | -| 1.2.2 | `bin/opensearch-plugin install https://github.com/aparo/opensearch-learning-to-rank/releases/download/1.2.2/ltr-1.5.4-os1.2.2.zip` | -| 1.2.3 | `bin/opensearch-plugin install https://github.com/aparo/opensearch-learning-to-rank/releases/download/1.2.3/ltr-1.5.4-os1.2.3.zip` | +## Releasing/Packaging +Releases are done through Github Workflows (see `.github/workflows` in the root directory) on an as needed basis. If you do `./gradlew build` as per above under building, +it will build all the artifacts that are in the release. -(It's expected you'll confirm some security exceptions, you can pass `-b` to `opensearch-plugin` to automatically install) +# Development -If you already are running OpenSearch, don't forget to restart! +To build, you need to explicitly enable Java security and disable snapshot builds (until the YamlRestTests are fixed): + +./gradlew -Dopensearch.version={opensearch-version-to-build-on} -Djava.security.manager=allow -Dbuild.snapshot=false + +# Upgrading the OpenSearch Versions + +1. Build and test as above +2. Update this README with the version info in the table above +3. Upgrade the Docker file versions in the `docker` directory +4. Test the docker image, per below. + +## Development Notes + + + +# Docker + +A custom image of [OpenSearch](https://hub.docker.com/r/opensearchproject/opensearch) with the [OpenSearch Learning to Rank plugin](https://github.com/gsingers/opensearch-learning-to-rank-base) installed. + +This image was created for the [Search with Machine Learning](https://corise.com/course/search-with-machine-learning?utm_source=daniel) course and [Search Fundamentals](https://corise.com/course/search-fundamentals?utm_source=daniel) taught by Grant Ingersoll and Daniel Tunkelang. + +See the [Elasticsearch Learning to Rank](https://elasticsearch-learning-to-rank.readthedocs.io/en/latest/index.html) documentation for details on how to us. + +## Building + +Building the docker image is triggered via the Github Actions workflows automatically (for releases) or via the commands below. + +Note, we are use Docker ARGs to pass through variables via the --build-arg. All args have defaults + +### Using local artifacts + + docker build -f docker/local.Dockerfile . + +### Using official releases, built locally + +#### Using defaults + + docker build -f docker/Dockerfile --tag=YOUR/IMAGE_NAME . + +#### From Versions + + docker build -f docker/Dockerfile --tag=YOUR/IMAGE_NAME --build-arg opensearch_version=2.2.1 --build-arg ltrversion=2.0.0 . + + +#### From a URL + + docker build -f docker/Dockerfile --tag=YOUR/IMAGE_NAME --build-arg plugin="https://github.com/gsingers/opensearch-learning-to-rank-base/releases/download/release-test-release/ltr-plugin-test-release.zip" . + + +## Running the docker image + +See the OpenSearch docs for official instructions, but this should work: + + docker run -p 9200:9200 -p 9600:9600 -e "discovery.type=single-node" YOUR/IMAGE_NAME:latest + + +## Publishing the Docker Image + +### Note: Since moving this into opensearch-project, this doesn't work. TODO: investigate the standard way of pushing plugins to Docker Hub. +To publish the Docker image to Docker Hub, you need to kick off the Docker action workflow: + gh workflow run .github/workflows/docker.yml -## About alpha releases - -These releases are alpha because some issues with the tests due to securemock that depends on ElasticSearch security stuff. -And there are 14 failing tests. - -``` -Tests with failures: -- com.o19s.es.ltr.feature.store.StoredFeatureSetParserTests.testExpressionDoubleQueryParameter -- com.o19s.es.ltr.feature.store.StoredFeatureSetParserTests.testExpressionMissingQueryParameter -- com.o19s.es.ltr.feature.store.StoredFeatureSetParserTests.testExpressionIntegerQueryParameter -- com.o19s.es.ltr.feature.store.StoredFeatureSetParserTests.testExpressionShortQueryParameter -- com.o19s.es.ltr.feature.store.StoredFeatureSetParserTests.testExpressionInvalidQueryParameter -- com.o19s.es.termstat.TermStatQueryBuilderTests.testMustRewrite -- com.o19s.es.termstat.TermStatQueryBuilderTests.testToQuery -- com.o19s.es.termstat.TermStatQueryBuilderTests.testCacheability -- com.o19s.es.ltr.feature.store.StoredFeatureParserTests.testExpressionOptimization -- com.o19s.es.termstat.TermStatQueryTests.testEmptyTerms -- com.o19s.es.termstat.TermStatQueryTests.testUniqueCount -- com.o19s.es.termstat.TermStatQueryTests.testBasicFormula -- com.o19s.es.termstat.TermStatQueryTests.testQuery -- com.o19s.es.termstat.TermStatQueryTests.testMatchCount - -228 tests completed, 14 failed -``` diff --git a/build.gradle b/build.gradle index dc21f536..0246c7bc 100644 --- a/build.gradle +++ b/build.gradle @@ -1,75 +1,151 @@ +import org.opensearch.gradle.test.RestIntegTestTask + +apply plugin: 'java' +apply plugin: 'idea' +apply plugin: 'opensearch.opensearchplugin' +apply plugin: 'opensearch.yaml-rest-test' +apply plugin: 'opensearch.pluginzip' +apply plugin: 'jacoco' + +//group = 'org.opensearch' +group = 'com.o19s.es' +def pluginName = 'ltr' +def pluginDescription = 'Make Opensearch results more relevant.' +def projectPath = 'com.o19s.es' +def pathToPlugin = 'ltr' +def pluginClassName = 'LtrQueryParserPlugin' + +publishing { + publications { + pluginZip(MavenPublication) { publication -> + pom { + name = pluginName + description = pluginDescription + licenses { + license { + name = "The Apache License, Version 2.0" + url = "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + } + developers { + developer { + name = "OpenSearch" + url = "https://github.com/opensearch-project/search-processor" + } + } + } + } + } +} +opensearchplugin { + name "opensearch-${pluginName}-${plugin_version}" + description pluginDescription + classname "${projectPath}.${pathToPlugin}.${pluginClassName}" + licenseFile rootProject.file('LICENSE') + noticeFile rootProject.file('NOTICE') +} + +// This requires an additional Jar not published as part of build-tools +loggerUsageCheck.enabled = false + +// No need to validate pom, as we do not upload to maven/sonatype +validateNebulaPom.enabled = false + buildscript { + ext { + isSnapshot = "true" == System.getProperty("build.snapshot", "true") + opensearch_version = System.getProperty("opensearch.version", "2.7.0") + lucene_version = "9.5.0" + plugin_version = opensearch_version + ".0" + if (isSnapshot) { + opensearch_version += "-SNAPSHOT" + plugin_version += "-SNAPSHOT" + } + println("opensearch_version ${opensearch_version}") + println("plugin_version ${plugin_version}") + + skipIntegTests = System.getProperty("skip.integtests", "false") + } + repositories { - mavenCentral() - jcenter() mavenLocal() + maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } + mavenCentral() + maven { url "https://plugins.gradle.org/m2/" } } dependencies { - classpath "org.opensearch.gradle:build-tools:${opensearchVersion}" + classpath "org.opensearch.gradle:build-tools:${opensearch_version}" } } -plugins { - id "co.riiid.gradle" version "0.4.2" +repositories { + mavenLocal() + maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } + mavenCentral() + maven { url "https://plugins.gradle.org/m2/" } } +dependencies { + implementation "com.github.spullara.mustache.java:compiler:0.9.3" + implementation "org.opensearch:opensearch:${opensearch_version}" + implementation "org.opensearch:opensearch-x-content:${opensearch_version}" + implementation "org.opensearch:opensearch-common:${opensearch_version}" + implementation "org.opensearch:opensearch-core:${opensearch_version}" + implementation "org.opensearch:common-utils:${plugin_version}" + implementation "org.apache.lucene:lucene-expressions:${luceneVersion}" + implementation 'com.o19s:RankyMcRankFace:0.1.1' + implementation "org.antlr:antlr4-runtime:${antlrVersion}" + implementation "org.ow2.asm:asm:${ow2Version}" + implementation "org.ow2.asm:asm-commons:${ow2Version}" + implementation "org.ow2.asm:asm-tree:${ow2Version}" +} -group = 'com.o19s' -version = "${ltrVersion}-os${opensearchVersion}" -apply plugin: 'java' -apply plugin: 'idea' -apply plugin: 'opensearch.opensearchplugin' -apply plugin: 'opensearch.java-rest-test' -apply plugin: 'opensearch.rest-resources' +allprojects { + plugins.withId('jacoco') { + jacoco.toolVersion = '0.8.9' + } +} -// license of this project -licenseFile = rootProject.file('LICENSE.txt') -// copyright notices -noticeFile = rootProject.file('NOTICE.txt') -// disable uploadArchives task for now, no upload happening currently -uploadArchives.enabled = false +test { + include '**/*Tests.class' + finalizedBy jacocoTestReport +} -opensearchplugin { - name 'ltr' - description 'Learning to Rank Query w/ RankLib Models' - classname 'com.o19s.es.ltr.LtrQueryParserPlugin' - // license of the plugin, may be different than the above license - licenseFile = rootProject.file('LICENSE.txt') - // copyright notices, may be different than the above notice - noticeFile = rootProject.file('NOTICE.txt') +task integTest(type: RestIntegTestTask) { + + description = "Run tests against a cluster" + testClassesDirs = sourceSets.test.output.classesDirs + classpath = sourceSets.test.runtimeClasspath +} +if (skipIntegTests == "false") { + tasks.named("check").configure { dependsOn(integTest) } +} else { + println("Skipping Integ Tests") } -// In this section you declare the dependencies for your production and test code -// OpenSearch dependency is included due to the build-tools, test-framework as well -repositories { - mavenCentral() - mavenLocal() - jcenter { - url "https://jcenter.bintray.com/" - metadataSources{ - artifact() - } +integTest { + // The --debug-jvm command-line option makes the cluster debuggable; this makes the tests debuggable + if (System.getProperty("test.debug") != null) { + jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005' } } -dependencies { - compile "org.apache.lucene:lucene-expressions:${luceneVersion}" - compile "org.antlr:antlr4-runtime:${antlrVersion}" - compile "org.ow2.asm:asm:${ow2Version}" - compile "org.ow2.asm:asm-commons:${ow2Version}" - compile "org.ow2.asm:asm-tree:${ow2Version}" - compile 'com.o19s:RankyMcRankFace:0.1.1' - compile "com.github.spullara.mustache.java:compiler:0.9.3" +testClusters.integTest { + testDistribution = "INTEG_TEST" + + // This installs our plugin into the testClusters + plugin(project.tasks.bundlePlugin.archiveFile) } -dependencyLicenses { - mapping from: /lucene-.*/, to: 'lucene' - mapping from: /asm-.*/, to: 'asm' - mapping from: /Ranky-.*/, to: 'lucene' - mapping from: /compiler-.*/, to: 'lucene' +run { + if (skipIntegTests == "false") { + useCluster testClusters.integTest + } else { + println("skipping Integ Tests ${skipIntegTests}") + } } sourceSets { @@ -77,48 +153,21 @@ sourceSets { compileClasspath += sourceSets["main"].output + sourceSets["test"].output + configurations["testRuntimeClasspath"] runtimeClasspath += output + compileClasspath } -} - -// Set to false to not use opensearch checkstyle rules -// checkstyleMain.enabled = true -// checkstyleTest.enabled = true - -// FIXME dependency license check needs to be enabled -// dependencyLicenses.enabled = false - -// FIXME thirdparty audit needs to be enabled -// thirdPartyAudit.enabled = false - -// Uncomment this to skip license header checks -// licenseHeaders.enabled = false - -// No need to validate POM, as we do not upload to sonatype -// validateNebulaPom.enabled = false - -// Elastic tried to remove the logging requirement for plugins, but didn't get it quite right so this is a short term fix until 7.11 -// https://github.com/elastic/opensearch/issues/65247 -loggerUsageCheck.enabled = false - -githubRelease.doFirst { - if (!System.getProperty('GITHUB_TOKEN', '')) { - throw new Exception('Missing property GITHUB_TOKEN') + yamlRestTest { + compileClasspath += sourceSets["main"].output + sourceSets["test"].output + configurations["testRuntimeClasspath"] + runtimeClasspath += output + compileClasspath } +} - // check if zip file is there - assert file("build/distributions/ingest-opennlp-${version}.zip").exists() - - // rename zip file - def currentVersion = version.replace('-SNAPSHOT', '') - def filename = "build/distributions/ltr-1.5.4-os${currentVersion}.zip" - Files.copy(file("build/distributions/ltr-1.5.4-os${version}.zip").toPath(), file(filename).toPath()) - - // configuration - github { - owner = 'aparo' - repo = 'opensearch-learning-to-rank' - token = System.getProperty('GITHUB_TOKEN') - tagName = currentVersion - assets = [ filename ] - targetCommitish = 'main' +jacocoTestReport { + dependsOn test + reports { + xml.required = true + html.required = true } } + +// TODO: Enable these checks +dependencyLicenses.enabled = false +thirdPartyAudit.enabled = false +loggerUsageCheck.enabled = false diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..92fc1b7e --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,4 @@ +ARG build_version=2.7.0 + +FROM opensearchproject/opensearch:${build_version} +RUN /usr/share/opensearch/bin/opensearch-plugin install -b https://github.com/gsingers/opensearch-learning-to-rank-base/releases/download/${build_version}/ltr-${build_version}-os${build_version}.zip diff --git a/docker/local.Dockerfile b/docker/local.Dockerfile new file mode 100644 index 00000000..6eb093a8 --- /dev/null +++ b/docker/local.Dockerfile @@ -0,0 +1,11 @@ +# Build an image from a local distribution + +ARG build_version=2.7.0 +FROM opensearchproject/opensearch:${build_version} +ARG zip_file=ltr-${build_version}-os${build_version}.zip +ARG plugin_file=/usr/share/opensearch/${zip_file} + +COPY --chown=opensearch:opensearch build/distributions/${zip_file} ${plugin_file} +RUN /usr/share/opensearch/bin/opensearch-plugin install -b file:${plugin_file} +# check in to see if there is a better way to do this so we don't have duplicate RUN commands + diff --git a/gradle.properties b/gradle.properties index 7f4aa46d..0985b5ff 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ -ltrVersion = 1.5.4 -opensearchVersion = 1.2.4 -luceneVersion = 8.10.1 -ow2Version = 8.0.1 -antlrVersion=4.5.1-1 +ltrVersion = 2.7.0 +opensearchVersion = 2.7.0 +luceneVersion = 9.5.0 +ow2Version = 9.4 +antlrVersion=4.11.1 diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 490fda85..943f0cbf 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 3ab0b725..f398c33c 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.9.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-bin.zip +networkTimeout=10000 zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index 2fe81a7d..65dcd68d 100755 --- a/gradlew +++ b/gradlew @@ -1,7 +1,7 @@ -#!/usr/bin/env sh +#!/bin/sh # -# Copyright 2015 the original author or authors. +# Copyright © 2015-2021 the original authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,78 +17,113 @@ # ############################################################################## -## -## Gradle start up script for UN*X -## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# ############################################################################## # Attempt to set APP_HOME + # Resolve links: $0 may be a link -PRG="$0" -# Need this for relative symlinks. -while [ -h "$PRG" ] ; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "$PRG"`"/$link" - fi +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac done -SAVED="`pwd`" -cd "`dirname \"$PRG\"`/" >/dev/null -APP_HOME="`pwd -P`" -cd "$SAVED" >/dev/null -APP_NAME="Gradle" -APP_BASE_NAME=`basename "$0"` +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' # Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD="maximum" +MAX_FD=maximum warn () { echo "$*" -} +} >&2 die () { echo echo "$*" echo exit 1 -} +} >&2 # OS specific support (must be 'true' or 'false'). cygwin=false msys=false darwin=false nonstop=false -case "`uname`" in - CYGWIN* ) - cygwin=true - ;; - Darwin* ) - darwin=true - ;; - MINGW* ) - msys=true - ;; - NONSTOP* ) - nonstop=true - ;; +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; esac CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + # Determine the Java command to use to start the JVM. if [ -n "$JAVA_HOME" ] ; then if [ -x "$JAVA_HOME/jre/sh/java" ] ; then # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" + JAVACMD=$JAVA_HOME/jre/sh/java else - JAVACMD="$JAVA_HOME/bin/java" + JAVACMD=$JAVA_HOME/bin/java fi if [ ! -x "$JAVACMD" ] ; then die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME @@ -97,7 +132,7 @@ Please set the JAVA_HOME variable in your environment to match the location of your Java installation." fi else - JAVACMD="java" + JAVACMD=java which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the @@ -105,79 +140,105 @@ location of your Java installation." fi # Increase the maximum file descriptors if we can. -if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then - MAX_FD_LIMIT=`ulimit -H -n` - if [ $? -eq 0 ] ; then - if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then - MAX_FD="$MAX_FD_LIMIT" - fi - ulimit -n $MAX_FD - if [ $? -ne 0 ] ; then - warn "Could not set maximum file descriptor limit: $MAX_FD" - fi - else - warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" - fi +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac fi -# For Darwin, add options to specify how the application appears in the dock -if $darwin; then - GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" -fi +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. # For Cygwin or MSYS, switch paths to Windows format before running java -if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then - APP_HOME=`cygpath --path --mixed "$APP_HOME"` - CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` - JAVACMD=`cygpath --unix "$JAVACMD"` - - # We build the pattern for arguments to be converted via cygpath - ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` - SEP="" - for dir in $ROOTDIRSRAW ; do - ROOTDIRS="$ROOTDIRS$SEP$dir" - SEP="|" - done - OURCYGPATTERN="(^($ROOTDIRS))" - # Add a user-defined pattern to the cygpath arguments - if [ "$GRADLE_CYGPATTERN" != "" ] ; then - OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" - fi +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + # Now convert the arguments - kludge to limit ourselves to /bin/sh - i=0 - for arg in "$@" ; do - CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` - CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option - - if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition - eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` - else - eval `echo args$i`="\"$arg\"" + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) fi - i=`expr $i + 1` + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg done - case $i in - 0) set -- ;; - 1) set -- "$args0" ;; - 2) set -- "$args0" "$args1" ;; - 3) set -- "$args0" "$args1" "$args2" ;; - 4) set -- "$args0" "$args1" "$args2" "$args3" ;; - 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; - 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; - 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; - 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; - 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; - esac fi -# Escape application args -save () { - for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done - echo " " -} -APP_ARGS=`save "$@"` +# Collect all arguments for the java command; +# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of +# shell script including quotes and variable substitutions, so put them in +# double quotes to make sure that they get re-expanded; and +# * put everything else in single quotes, so that it's not re-expanded. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# -# Collect all arguments for the java command, following the shell quoting and substitution rules -eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat index 9109989e..6689b85b 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -14,7 +14,7 @@ @rem limitations under the License. @rem -@if "%DEBUG%" == "" @echo off +@if "%DEBUG%"=="" @echo off @rem ########################################################################## @rem @rem Gradle startup script for Windows @@ -25,7 +25,8 @@ if "%OS%"=="Windows_NT" setlocal set DIRNAME=%~dp0 -if "%DIRNAME%" == "" set DIRNAME=. +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% @@ -40,7 +41,7 @@ if defined JAVA_HOME goto findJavaFromJavaHome set JAVA_EXE=java.exe %JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto init +if %ERRORLEVEL% equ 0 goto execute echo. echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. @@ -54,7 +55,7 @@ goto fail set JAVA_HOME=%JAVA_HOME:"=% set JAVA_EXE=%JAVA_HOME%/bin/java.exe -if exist "%JAVA_EXE%" goto init +if exist "%JAVA_EXE%" goto execute echo. echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% @@ -64,38 +65,26 @@ echo location of your Java installation. goto fail -:init -@rem Get command-line arguments, handling Windows variants - -if not "%OS%" == "Windows_NT" goto win9xME_args - -:win9xME_args -@rem Slurp the command line arguments. -set CMD_LINE_ARGS= -set _SKIP=2 - -:win9xME_args_slurp -if "x%~1" == "x" goto execute - -set CMD_LINE_ARGS=%* - :execute @rem Setup the command line set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + @rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* :end @rem End local scope for the variables with windows NT shell -if "%ERRORLEVEL%"=="0" goto mainEnd +if %ERRORLEVEL% equ 0 goto mainEnd :fail rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of rem the _cmd.exe /c_ return code! -if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 -exit /b 1 +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% :mainEnd if "%OS%"=="Windows_NT" endlocal diff --git a/licenses/RankyMcRankFace-0.1.1.jar.sha1 b/licenses/RankyMcRankFace-0.1.1.jar.sha1 new file mode 100644 index 00000000..36e676a6 --- /dev/null +++ b/licenses/RankyMcRankFace-0.1.1.jar.sha1 @@ -0,0 +1 @@ +d8775154417bacbb6c81a247c3b37c78c86061f8 \ No newline at end of file diff --git a/licenses/antlr4-runtime-4.5.1-1.jar.sha1 b/licenses/antlr4-runtime-4.5.1-1.jar.sha1 deleted file mode 100644 index f15e5006..00000000 --- a/licenses/antlr4-runtime-4.5.1-1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -66144204f9d6d7d3f3f775622c2dd7e9bd511d97 diff --git a/licenses/antlr4-runtime-4.9.3.jar.sha1 b/licenses/antlr4-runtime-4.9.3.jar.sha1 new file mode 100644 index 00000000..13a23674 --- /dev/null +++ b/licenses/antlr4-runtime-4.9.3.jar.sha1 @@ -0,0 +1 @@ +81befc16ebedb8b8aea3e4c0835dd5ca7e8523a8 \ No newline at end of file diff --git a/licenses/asm-5.0.4.jar.sha1 b/licenses/asm-5.0.4.jar.sha1 deleted file mode 100644 index 9223dba3..00000000 --- a/licenses/asm-5.0.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -0da08b8cce7bbf903602a25a3a163ae252435795 diff --git a/licenses/asm-9.4.jar.sha1 b/licenses/asm-9.4.jar.sha1 new file mode 100644 index 00000000..75f2b0fe --- /dev/null +++ b/licenses/asm-9.4.jar.sha1 @@ -0,0 +1 @@ +b4e0e2d2e023aa317b7cfcfc916377ea348e07d1 \ No newline at end of file diff --git a/licenses/asm-commons-5.0.4.jar.sha1 b/licenses/asm-commons-5.0.4.jar.sha1 deleted file mode 100644 index 94fe0cd9..00000000 --- a/licenses/asm-commons-5.0.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5a556786086c23cd689a0328f8519db93821c04c diff --git a/licenses/asm-commons-9.4.jar.sha1 b/licenses/asm-commons-9.4.jar.sha1 new file mode 100644 index 00000000..e0e2a2f4 --- /dev/null +++ b/licenses/asm-commons-9.4.jar.sha1 @@ -0,0 +1 @@ +8fc2810ddbcbbec0a8bbccb3f8eda58321839912 \ No newline at end of file diff --git a/licenses/asm-tree-5.0.4.jar.sha1 b/licenses/asm-tree-5.0.4.jar.sha1 deleted file mode 100644 index 5822a485..00000000 --- a/licenses/asm-tree-5.0.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -396ce0c07ba2b481f25a70195c7c94922f0d1b0b \ No newline at end of file diff --git a/licenses/asm-tree-9.4.jar.sha1 b/licenses/asm-tree-9.4.jar.sha1 new file mode 100644 index 00000000..50ce6d74 --- /dev/null +++ b/licenses/asm-tree-9.4.jar.sha1 @@ -0,0 +1 @@ +a99175a17d7fdc18cbcbd0e8ea6a5d276844190a \ No newline at end of file diff --git a/licenses/compiler-0.9.3.jar.sha1 b/licenses/compiler-0.9.3.jar.sha1 new file mode 100644 index 00000000..2b0fbbc5 --- /dev/null +++ b/licenses/compiler-0.9.3.jar.sha1 @@ -0,0 +1 @@ +2815e016c63bec4f18704ea4f5489106a5b01a99 \ No newline at end of file diff --git a/licenses/lucene-expressions-6.6.0.jar.sha1 b/licenses/lucene-expressions-6.6.0.jar.sha1 deleted file mode 100644 index 860f30cc..00000000 --- a/licenses/lucene-expressions-6.6.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4edcafb90cef9428a6887b92ccb85ff410f1436b \ No newline at end of file diff --git a/licenses/lucene-expressions-9.4.2.jar.sha1 b/licenses/lucene-expressions-9.4.2.jar.sha1 new file mode 100644 index 00000000..61cd306d --- /dev/null +++ b/licenses/lucene-expressions-9.4.2.jar.sha1 @@ -0,0 +1 @@ +f433ede87302221b48703a890e5f5701cf1b021d \ No newline at end of file diff --git a/src/javaRestTest/java/com/o19s/es/ltr/action/BaseIntegrationTest.java b/src/javaRestTest/java/com/o19s/es/ltr/action/BaseIntegrationTest.java index 1c9e6745..68976180 100644 --- a/src/javaRestTest/java/com/o19s/es/ltr/action/BaseIntegrationTest.java +++ b/src/javaRestTest/java/com/o19s/es/ltr/action/BaseIntegrationTest.java @@ -125,7 +125,6 @@ public FeatureStoreResponse addElement(StorableElement element, builder.request().setValidation(validation); FeatureStoreResponse response = builder.execute().get(); assertEquals(1, response.getResponse().getVersion()); - assertEquals(IndexFeatureStore.ES_TYPE, response.getResponse().getType()); assertEquals(DocWriteResponse.Result.CREATED, response.getResponse().getResult()); assertEquals(element.id(), response.getResponse().getId()); assertEquals(store, response.getResponse().getIndex()); diff --git a/src/javaRestTest/java/com/o19s/es/ltr/action/LTRStatsActionIT.java b/src/javaRestTest/java/com/o19s/es/ltr/action/LTRStatsActionIT.java index 5c0c4965..d4ad7310 100644 --- a/src/javaRestTest/java/com/o19s/es/ltr/action/LTRStatsActionIT.java +++ b/src/javaRestTest/java/com/o19s/es/ltr/action/LTRStatsActionIT.java @@ -1,3 +1,18 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ package com.o19s.es.ltr.action; import com.o19s.es.ltr.LtrTestUtils; diff --git a/src/javaRestTest/java/com/o19s/es/ltr/logging/LoggingIT.java b/src/javaRestTest/java/com/o19s/es/ltr/logging/LoggingIT.java index 6eeb7721..4be44573 100644 --- a/src/javaRestTest/java/com/o19s/es/ltr/logging/LoggingIT.java +++ b/src/javaRestTest/java/com/o19s/es/ltr/logging/LoggingIT.java @@ -25,7 +25,7 @@ import com.o19s.es.ltr.query.StoredLtrQueryBuilder; import com.o19s.es.ltr.ranker.parser.LinearRankerParserTests; import org.apache.lucene.search.join.ScoreMode; -import org.apache.lucene.util.TestUtil; +import org.apache.lucene.tests.util.TestUtil; import org.opensearch.ExceptionsHelper; import org.opensearch.action.index.IndexResponse; import org.opensearch.action.search.SearchResponse; @@ -139,7 +139,6 @@ public void testFailures() throws Exception { .addQueryLogging("first_log", "test", false))); assertExcWithMessage(() -> client().prepareSearch("test_index") - .setTypes("test") .setSource(sourceBuilder).get(), IllegalArgumentException.class, "No query named [test] found"); SearchSourceBuilder sourceBuilder2 = new SearchSourceBuilder().query(query) @@ -150,7 +149,6 @@ public void testFailures() throws Exception { .addQueryLogging("first_log", "not_sltr", false))); assertExcWithMessage(() -> client().prepareSearch("test_index") - .setTypes("test") .setSource(sourceBuilder2).get(), IllegalArgumentException.class, "Query named [not_sltr] must be a " + "[sltr] query [TermQuery] found"); @@ -161,7 +159,6 @@ public void testFailures() throws Exception { new LoggingSearchExtBuilder() .addRescoreLogging("first_log", 0, false))); assertExcWithMessage(() -> client().prepareSearch("test_index") - .setTypes("test") .setSource(sourceBuilder3).get(), IllegalArgumentException.class, "rescore index [0] is out of bounds, " + "only [0]"); @@ -173,7 +170,6 @@ public void testFailures() throws Exception { new LoggingSearchExtBuilder() .addRescoreLogging("first_log", 0, false))); assertExcWithMessage(() -> client().prepareSearch("test_index") - .setTypes("test") .setSource(sourceBuilder4).get(), IllegalArgumentException.class, "Expected a [sltr] query but found " + "a [MatchAllDocsQuery] at index [0]"); } @@ -209,7 +205,7 @@ public void testLog() throws Exception { .boost(random().nextInt(3)); QueryBuilder query = QueryBuilders.boolQuery().must(new WrapperQueryBuilder(sbuilder.toString())) - .filter(QueryBuilders.idsQuery("test").addIds(ids)); + .filter(QueryBuilders.idsQuery().addIds(ids)); SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(query) .fetchSource(false) .size(10) @@ -219,7 +215,7 @@ public void testLog() throws Exception { .addQueryLogging("first_log", "test", false) .addRescoreLogging("second_log", 0, true))); - SearchResponse resp = client().prepareSearch("test_index").setTypes("test").setSource(sourceBuilder).get(); + SearchResponse resp = client().prepareSearch("test_index").setSource(sourceBuilder).get(); assertSearchHits(docs, resp); sbuilder.featureSetName(null); sbuilder.modelName("my_model"); @@ -229,7 +225,7 @@ public void testLog() throws Exception { sbuilder_rescore.boost(random().nextInt(3)); query = QueryBuilders.boolQuery().must(new WrapperQueryBuilder(sbuilder.toString())) - .filter(QueryBuilders.idsQuery("test").addIds(ids)); + .filter(QueryBuilders.idsQuery().addIds(ids)); sourceBuilder = new SearchSourceBuilder().query(query) .fetchSource(false) .size(10) @@ -239,7 +235,7 @@ public void testLog() throws Exception { .addQueryLogging("first_log", "test", false) .addRescoreLogging("second_log", 0, true))); - SearchResponse resp2 = client().prepareSearch("test_index").setTypes("test").setSource(sourceBuilder).get(); + SearchResponse resp2 = client().prepareSearch("test_index").setSource(sourceBuilder).get(); assertSearchHits(docs, resp2); query = QueryBuilders.boolQuery() @@ -259,7 +255,7 @@ public void testLog() throws Exception { new LoggingSearchExtBuilder() .addQueryLogging("first_log", "test", false) .addRescoreLogging("second_log", 0, true))); - SearchResponse resp3 = client().prepareSearch("test_index").setTypes("test").setSource(sourceBuilder).get(); + SearchResponse resp3 = client().prepareSearch("test_index").setSource(sourceBuilder).get(); assertSearchHits(docs, resp3); } @@ -285,7 +281,7 @@ public void testLogExtraLogging() throws Exception { .boost(random().nextInt(3)); QueryBuilder query = QueryBuilders.boolQuery().must(new WrapperQueryBuilder(sbuilder.toString())) - .filter(QueryBuilders.idsQuery("test").addIds(ids)); + .filter(QueryBuilders.idsQuery().addIds(ids)); SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(query) .fetchSource(false) .size(10) @@ -295,7 +291,7 @@ public void testLogExtraLogging() throws Exception { .addQueryLogging("first_log", "test", false) .addRescoreLogging("second_log", 0, true))); - SearchResponse resp = client().prepareSearch("test_index").setTypes("test").setSource(sourceBuilder).get(); + SearchResponse resp = client().prepareSearch("test_index").setSource(sourceBuilder).get(); assertSearchHitsExtraLogging(docs, resp); sbuilder.featureSetName(null); sbuilder.modelName("my_model"); @@ -305,7 +301,7 @@ public void testLogExtraLogging() throws Exception { sbuilder_rescore.boost(random().nextInt(3)); query = QueryBuilders.boolQuery().must(new WrapperQueryBuilder(sbuilder.toString())) - .filter(QueryBuilders.idsQuery("test").addIds(ids)); + .filter(QueryBuilders.idsQuery().addIds(ids)); sourceBuilder = new SearchSourceBuilder().query(query) .fetchSource(false) .size(10) @@ -315,7 +311,7 @@ public void testLogExtraLogging() throws Exception { .addQueryLogging("first_log", "test", false) .addRescoreLogging("second_log", 0, true))); - SearchResponse resp2 = client().prepareSearch("test_index").setTypes("test").setSource(sourceBuilder).get(); + SearchResponse resp2 = client().prepareSearch("test_index").setSource(sourceBuilder).get(); assertSearchHitsExtraLogging(docs, resp2); query = QueryBuilders.boolQuery() @@ -335,7 +331,7 @@ public void testLogExtraLogging() throws Exception { new LoggingSearchExtBuilder() .addQueryLogging("first_log", "test", false) .addRescoreLogging("second_log", 0, true))); - SearchResponse resp3 = client().prepareSearch("test_index").setTypes("test").setSource(sourceBuilder).get(); + SearchResponse resp3 = client().prepareSearch("test_index").setSource(sourceBuilder).get(); assertSearchHitsExtraLogging(docs, resp3); } @@ -356,7 +352,7 @@ public void testScriptLogInternalParams() throws Exception { .boost(random().nextInt(3)); QueryBuilder query = QueryBuilders.boolQuery().must(new WrapperQueryBuilder(sbuilder.toString())) - .filter(QueryBuilders.idsQuery("test").addIds(ids)); + .filter(QueryBuilders.idsQuery().addIds(ids)); SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(query) .fetchSource(false) .size(10) @@ -364,7 +360,7 @@ public void testScriptLogInternalParams() throws Exception { new LoggingSearchExtBuilder() .addQueryLogging("first_log", "test", false))); - SearchResponse resp = client().prepareSearch("test_index").setTypes("test").setSource(sourceBuilder).get(); + SearchResponse resp = client().prepareSearch("test_index").setSource(sourceBuilder).get(); SearchHits hits = resp.getHits(); SearchHit testHit = hits.getAt(0); @@ -402,7 +398,7 @@ public void testScriptLogExternalParams() throws Exception { .boost(random().nextInt(3)); QueryBuilder query = QueryBuilders.boolQuery().must(new WrapperQueryBuilder(sbuilder.toString())) - .filter(QueryBuilders.idsQuery("test").addIds(ids)); + .filter(QueryBuilders.idsQuery().addIds(ids)); SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(query) .fetchSource(false) .size(10) @@ -410,7 +406,7 @@ public void testScriptLogExternalParams() throws Exception { new LoggingSearchExtBuilder() .addQueryLogging("first_log", "test", false))); - SearchResponse resp = client().prepareSearch("test_index").setTypes("test").setSource(sourceBuilder).get(); + SearchResponse resp = client().prepareSearch("test_index").setSource(sourceBuilder).get(); SearchHits hits = resp.getHits(); SearchHit testHit = hits.getAt(0); @@ -440,7 +436,7 @@ public void testScriptLogInvalidExternalParams() throws Exception { .boost(random().nextInt(3)); QueryBuilder query = QueryBuilders.boolQuery().must(new WrapperQueryBuilder(sbuilder.toString())) - .filter(QueryBuilders.idsQuery("test").addIds(ids)); + .filter(QueryBuilders.idsQuery().addIds(ids)); SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(query) .fetchSource(false) .size(10) @@ -449,7 +445,6 @@ public void testScriptLogInvalidExternalParams() throws Exception { .addQueryLogging("first_log", "test", false))); assertExcWithMessage(() -> client().prepareSearch("test_index") - .setTypes("test") .setSource(sourceBuilder).get(), IllegalArgumentException.class, "Term Stats injection requires fields and terms"); } @@ -583,10 +578,8 @@ protected void assertSearchHitsExtraLogging(Map docs, SearchRespons public Map buildIndex() { client().admin().indices().prepareCreate("test_index") - .addMapping( - "test", - "{\"properties\":{\"scorefield1\": {\"type\": \"float\"}, \"nesteddocs1\": {\"type\": \"nested\"}}}}", - XContentType.JSON) + .setMapping( + "{\"properties\":{\"scorefield1\": {\"type\": \"float\"}, \"nesteddocs1\": {\"type\": \"nested\"}}}}") .get(); int numDocs = TestUtil.nextInt(random(), 20, 100); @@ -614,7 +607,7 @@ public Map buildIndex() { } public void indexDoc(Doc d) { - IndexResponse resp = client().prepareIndex("test_index", "test") + IndexResponse resp = client().prepareIndex("test_index") .setSource("field1", d.field1, "field2", d.field2, "scorefield1", d.scorefield1, "nesteddocs1", d.getNesteddocs1()) .get(); d.id = resp.getId(); diff --git a/src/javaRestTest/java/com/o19s/es/ltr/query/StoredLtrQueryIT.java b/src/javaRestTest/java/com/o19s/es/ltr/query/StoredLtrQueryIT.java index f664b3ed..b5139c8e 100644 --- a/src/javaRestTest/java/com/o19s/es/ltr/query/StoredLtrQueryIT.java +++ b/src/javaRestTest/java/com/o19s/es/ltr/query/StoredLtrQueryIT.java @@ -296,7 +296,7 @@ public void testInvalidDerived() throws Exception { public void buildIndex() { client().admin().indices().prepareCreate("test_index").get(); - client().prepareIndex("test_index", "test") + client().prepareIndex("test_index") .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .setSource("field1", "hello world", "field2", "bonjour world") .get(); diff --git a/src/main/java/com/o19s/es/explore/ExplorerQuery.java b/src/main/java/com/o19s/es/explore/ExplorerQuery.java index 5b47e07e..76556f0f 100644 --- a/src/main/java/com/o19s/es/explore/ExplorerQuery.java +++ b/src/main/java/com/o19s/es/explore/ExplorerQuery.java @@ -21,6 +21,7 @@ import org.apache.lucene.index.Term; import org.apache.lucene.index.TermStates; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Weight; @@ -60,7 +61,6 @@ private boolean isCollectionScoped() { public String getType() { return this.type; } - @SuppressWarnings("EqualsWhichDoesntCheckParameterClass") @Override public boolean equals(Object other) { return sameClassAs(other) && @@ -94,9 +94,8 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo if (!scoreMode.needsScores()) { return searcher.createWeight(query, scoreMode, boost); } - final Weight subWeight = searcher.createWeight(query, scoreMode, boost); - Set terms = new HashSet<>(); - subWeight.extractTerms(terms); + final Set terms = new HashSet<>(); + this.visit(QueryVisitor.termCollector(terms)); if (isCollectionScoped()) { ClassicSimilarity sim = new ClassicSimilarity(); StatisticsHelper df_stats = new StatisticsHelper(); @@ -105,12 +104,15 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo for (Term term : terms) { TermStates ctx = TermStates.build(searcher.getTopReaderContext(), term, scoreMode.needsScores()); - TermStatistics tStats = searcher.termStatistics(term, ctx); - if(tStats != null){ + if(ctx != null && ctx.docFreq() > 0){ + TermStatistics tStats = searcher.termStatistics(term, ctx.docFreq(), ctx.totalTermFreq()); df_stats.add(tStats.docFreq()); - idf_stats.add(sim.idf(tStats.docFreq(), searcher.getIndexReader().numDocs())); + idf_stats.add(sim.idf(tStats.docFreq(), searcher.collectionStatistics(term.field()).docCount())); ttf_stats.add(tStats.totalTermFreq()); - + } else { + df_stats.add(0.0f); + idf_stats.add(0.0f); + ttf_stats.add(0.0f); } } @@ -238,9 +240,8 @@ static class ExplorerWeight extends Weight { this.type = type; } - @Override public void extractTerms(Set terms) { - weight.extractTerms(terms); + QueryVisitor.termCollector(terms); } @Override @@ -276,4 +277,9 @@ public Scorer scorer(LeafReaderContext context) throws IOException { public String toString(String field) { return query.toString(); } + + @Override + public void visit(QueryVisitor visitor) { + this.query.visit(visitor.getSubVisitor(BooleanClause.Occur.MUST, this)); + } } diff --git a/src/main/java/com/o19s/es/explore/ExplorerQueryBuilder.java b/src/main/java/com/o19s/es/explore/ExplorerQueryBuilder.java index 9a85ea6e..51e49e47 100644 --- a/src/main/java/com/o19s/es/explore/ExplorerQueryBuilder.java +++ b/src/main/java/com/o19s/es/explore/ExplorerQueryBuilder.java @@ -16,14 +16,14 @@ package com.o19s.es.explore; import org.apache.lucene.search.Query; -import org.opensearch.common.ParseField; +import org.opensearch.core.ParseField; import org.opensearch.common.ParsingException; import org.opensearch.common.io.stream.NamedWriteable; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.xcontent.ObjectParser; -import org.opensearch.common.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentParser; +import org.opensearch.core.xcontent.ObjectParser; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.query.AbstractQueryBuilder; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryRewriteContext; diff --git a/src/main/java/com/o19s/es/explore/PostingsExplorerQuery.java b/src/main/java/com/o19s/es/explore/PostingsExplorerQuery.java index 807a7930..0ee2e930 100644 --- a/src/main/java/com/o19s/es/explore/PostingsExplorerQuery.java +++ b/src/main/java/com/o19s/es/explore/PostingsExplorerQuery.java @@ -25,13 +25,14 @@ import org.apache.lucene.index.TermState; import org.apache.lucene.index.TermStates; import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.Explanation; -import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.search.Weight; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.Weight; +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.DocIdSetIterator; import java.io.IOException; import java.util.ArrayList; @@ -62,7 +63,6 @@ public String toString(String field) { return buffer.toString(); } - @SuppressWarnings("EqualsWhichDoesntCheckParameterClass") @Override public boolean equals(Object obj) { return this.sameClassAs(obj) @@ -117,7 +117,6 @@ static class PostingsExplorerWeight extends Weight { this.type = type; } - @Override public void extractTerms(Set terms) { terms.add(term); } @@ -245,4 +244,11 @@ public float getMaxScore(int upTo) throws IOException { return Float.POSITIVE_INFINITY; } } + + @Override + public void visit(QueryVisitor visitor) { + if (visitor.acceptField(this.term.field()) != false) { + visitor.consumeTerms(this, term); + } + } } diff --git a/src/main/java/com/o19s/es/ltr/LtrQueryParserPlugin.java b/src/main/java/com/o19s/es/ltr/LtrQueryParserPlugin.java index d0353036..1428a24f 100644 --- a/src/main/java/com/o19s/es/ltr/LtrQueryParserPlugin.java +++ b/src/main/java/com/o19s/es/ltr/LtrQueryParserPlugin.java @@ -75,7 +75,7 @@ import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.CheckedFunction; -import org.opensearch.common.ParseField; +import org.opensearch.core.ParseField; import org.opensearch.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.io.stream.NamedWriteableRegistry.Entry; import org.opensearch.common.settings.ClusterSettings; @@ -83,8 +83,8 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.SettingsFilter; -import org.opensearch.common.xcontent.NamedXContentRegistry; -import org.opensearch.common.xcontent.XContentParser; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.env.Environment; import org.opensearch.env.NodeEnvironment; import org.opensearch.index.Index; diff --git a/src/main/java/com/o19s/es/ltr/action/AddFeaturesToSetAction.java b/src/main/java/com/o19s/es/ltr/action/AddFeaturesToSetAction.java index 3dea24d9..04ee4e4a 100644 --- a/src/main/java/com/o19s/es/ltr/action/AddFeaturesToSetAction.java +++ b/src/main/java/com/o19s/es/ltr/action/AddFeaturesToSetAction.java @@ -30,7 +30,7 @@ import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.common.io.stream.Writeable.Reader; import org.opensearch.common.xcontent.StatusToXContentObject; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.rest.RestStatus; import java.io.IOException; diff --git a/src/main/java/com/o19s/es/ltr/action/CachesStatsAction.java b/src/main/java/com/o19s/es/ltr/action/CachesStatsAction.java index 30963d6b..8e353499 100644 --- a/src/main/java/com/o19s/es/ltr/action/CachesStatsAction.java +++ b/src/main/java/com/o19s/es/ltr/action/CachesStatsAction.java @@ -30,8 +30,8 @@ import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.common.io.stream.Writeable; -import org.opensearch.common.xcontent.ToXContent; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; import java.io.IOException; import java.util.HashMap; diff --git a/src/main/java/com/o19s/es/ltr/action/CreateModelFromSetAction.java b/src/main/java/com/o19s/es/ltr/action/CreateModelFromSetAction.java index bf28f2b6..c19c6e23 100644 --- a/src/main/java/com/o19s/es/ltr/action/CreateModelFromSetAction.java +++ b/src/main/java/com/o19s/es/ltr/action/CreateModelFromSetAction.java @@ -29,7 +29,7 @@ import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.StatusToXContentObject; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.rest.RestStatus; import java.io.IOException; diff --git a/src/main/java/com/o19s/es/ltr/action/FeatureStoreAction.java b/src/main/java/com/o19s/es/ltr/action/FeatureStoreAction.java index 5c7ca5fb..4e3b1033 100644 --- a/src/main/java/com/o19s/es/ltr/action/FeatureStoreAction.java +++ b/src/main/java/com/o19s/es/ltr/action/FeatureStoreAction.java @@ -31,7 +31,7 @@ import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.common.io.stream.Writeable.Reader; import org.opensearch.common.xcontent.StatusToXContentObject; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.rest.RestStatus; import java.io.IOException; diff --git a/src/main/java/com/o19s/es/ltr/action/LTRStatsAction.java b/src/main/java/com/o19s/es/ltr/action/LTRStatsAction.java index 0052f80d..3564cb8d 100644 --- a/src/main/java/com/o19s/es/ltr/action/LTRStatsAction.java +++ b/src/main/java/com/o19s/es/ltr/action/LTRStatsAction.java @@ -1,3 +1,19 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package com.o19s.es.ltr.action; import org.opensearch.action.ActionRequestBuilder; @@ -12,9 +28,9 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.xcontent.ToXContent; -import org.opensearch.common.xcontent.ToXContentFragment; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.ToXContentFragment; +import org.opensearch.core.xcontent.XContentBuilder; import java.io.IOException; import java.util.HashSet; diff --git a/src/main/java/com/o19s/es/ltr/action/ListStoresAction.java b/src/main/java/com/o19s/es/ltr/action/ListStoresAction.java index d1c604ec..1137e035 100644 --- a/src/main/java/com/o19s/es/ltr/action/ListStoresAction.java +++ b/src/main/java/com/o19s/es/ltr/action/ListStoresAction.java @@ -28,9 +28,9 @@ import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.common.io.stream.Writeable; import org.opensearch.common.io.stream.Writeable.Reader; -import org.opensearch.common.xcontent.ToXContent; -import org.opensearch.common.xcontent.ToXContentObject; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; import java.io.IOException; import java.util.List; diff --git a/src/main/java/com/o19s/es/ltr/action/TransportAddFeatureToSetAction.java b/src/main/java/com/o19s/es/ltr/action/TransportAddFeatureToSetAction.java index 7e1130d0..477fcc55 100644 --- a/src/main/java/com/o19s/es/ltr/action/TransportAddFeatureToSetAction.java +++ b/src/main/java/com/o19s/es/ltr/action/TransportAddFeatureToSetAction.java @@ -144,7 +144,6 @@ private void start() { featuresRef.set(features); } GetRequest getRequest = new GetRequest(store) - .type(IndexFeatureStore.ES_TYPE) .id(StorableElement.generateId(StoredFeatureSet.TYPE, featureSetName)) .routing(routing); @@ -170,7 +169,7 @@ private void fetchFeaturesFromStore() { BoolQueryBuilder bq = QueryBuilders.boolQuery(); bq.must(nameQuery); bq.must(QueryBuilders.matchQuery("type", StoredFeature.TYPE)); - srequest.types(IndexFeatureStore.ES_TYPE); + //srequest.types(IndexFeatureStore.ES_TYPE); srequest.source().query(bq); srequest.source().fetchSource(true); srequest.source().size(StoredFeatureSet.MAX_FEATURES); diff --git a/src/main/java/com/o19s/es/ltr/action/TransportCreateModelFromSetAction.java b/src/main/java/com/o19s/es/ltr/action/TransportCreateModelFromSetAction.java index b0674774..28cc7ac2 100644 --- a/src/main/java/com/o19s/es/ltr/action/TransportCreateModelFromSetAction.java +++ b/src/main/java/com/o19s/es/ltr/action/TransportCreateModelFromSetAction.java @@ -62,7 +62,6 @@ protected void doExecute(Task task, CreateModelFromSetRequest request, ActionLis throw new IllegalArgumentException("Store [" + request.getStore() + "] does not exist, please create it first."); } GetRequest getRequest = new GetRequest(request.getStore()) - .type(IndexFeatureStore.ES_TYPE) .id(StorableElement.generateId(StoredFeatureSet.TYPE, request.getFeatureSetName())); getRequest.setParentTask(clusterService.localNode().getId(), task.getId()); getAction.execute(getRequest, ActionListener.wrap((r) -> this.doStore(task, r, request, listener), listener::onFailure)); diff --git a/src/main/java/com/o19s/es/ltr/action/TransportFeatureStoreAction.java b/src/main/java/com/o19s/es/ltr/action/TransportFeatureStoreAction.java index 08c254c9..7e7c7440 100644 --- a/src/main/java/com/o19s/es/ltr/action/TransportFeatureStoreAction.java +++ b/src/main/java/com/o19s/es/ltr/action/TransportFeatureStoreAction.java @@ -104,7 +104,8 @@ private Optional buildClearCache(FeatureStoreRequest re private IndexRequest buildIndexRequest(Task parentTask, FeatureStoreRequest request) throws IOException { StorableElement elt = request.getStorableElement(); - IndexRequest indexRequest = client.prepareIndex(request.getStore(), IndexFeatureStore.ES_TYPE, elt.id()) + IndexRequest indexRequest = client.prepareIndex(request.getStore()) + .setId(elt.id()) .setCreate(request.getAction() == FeatureStoreRequest.Action.CREATE) .setRouting(request.getRouting()) .setSource(IndexFeatureStore.toSource(elt)) diff --git a/src/main/java/com/o19s/es/ltr/action/TransportLTRStatsAction.java b/src/main/java/com/o19s/es/ltr/action/TransportLTRStatsAction.java index e6f8613b..9ec5df03 100644 --- a/src/main/java/com/o19s/es/ltr/action/TransportLTRStatsAction.java +++ b/src/main/java/com/o19s/es/ltr/action/TransportLTRStatsAction.java @@ -1,3 +1,19 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package com.o19s.es.ltr.action; import com.o19s.es.ltr.action.LTRStatsAction.LTRStatsNodeRequest; diff --git a/src/main/java/com/o19s/es/ltr/feature/FeatureValidation.java b/src/main/java/com/o19s/es/ltr/feature/FeatureValidation.java index d63b7b85..7d615256 100644 --- a/src/main/java/com/o19s/es/ltr/feature/FeatureValidation.java +++ b/src/main/java/com/o19s/es/ltr/feature/FeatureValidation.java @@ -16,15 +16,15 @@ package com.o19s.es.ltr.feature; -import org.opensearch.common.ParseField; +import org.opensearch.core.ParseField; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.common.io.stream.Writeable; -import org.opensearch.common.xcontent.ConstructingObjectParser; -import org.opensearch.common.xcontent.ObjectParser; -import org.opensearch.common.xcontent.ToXContentObject; -import org.opensearch.common.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentParser; +import org.opensearch.core.xcontent.ConstructingObjectParser; +import org.opensearch.core.xcontent.ObjectParser; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; import java.io.IOException; import java.util.Map; diff --git a/src/main/java/com/o19s/es/ltr/feature/PrebuiltFeature.java b/src/main/java/com/o19s/es/ltr/feature/PrebuiltFeature.java index 912b4f76..74603b25 100644 --- a/src/main/java/com/o19s/es/ltr/feature/PrebuiltFeature.java +++ b/src/main/java/com/o19s/es/ltr/feature/PrebuiltFeature.java @@ -18,10 +18,12 @@ import com.o19s.es.ltr.LtrQueryContext; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.Query; -import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.Weight; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.ScoreMode; import org.opensearch.common.Nullable; import java.io.IOException; @@ -84,4 +86,9 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo public Query rewrite(IndexReader reader) throws IOException { return query.rewrite(reader); } + + @Override + public void visit(QueryVisitor visitor) { + this.query.visit(visitor.getSubVisitor(BooleanClause.Occur.MUST, this)); + } } diff --git a/src/main/java/com/o19s/es/ltr/feature/store/ExtraLoggingSupplier.java b/src/main/java/com/o19s/es/ltr/feature/store/ExtraLoggingSupplier.java index de897fef..6279f217 100644 --- a/src/main/java/com/o19s/es/ltr/feature/store/ExtraLoggingSupplier.java +++ b/src/main/java/com/o19s/es/ltr/feature/store/ExtraLoggingSupplier.java @@ -1,3 +1,19 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package com.o19s.es.ltr.feature.store; import java.util.Map; diff --git a/src/main/java/com/o19s/es/ltr/feature/store/FeatureNormDefinition.java b/src/main/java/com/o19s/es/ltr/feature/store/FeatureNormDefinition.java index c131cd56..31bb917d 100644 --- a/src/main/java/com/o19s/es/ltr/feature/store/FeatureNormDefinition.java +++ b/src/main/java/com/o19s/es/ltr/feature/store/FeatureNormDefinition.java @@ -1,8 +1,24 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package com.o19s.es.ltr.feature.store; import com.o19s.es.ltr.ranker.normalizer.Normalizer; import org.opensearch.common.io.stream.Writeable; -import org.opensearch.common.xcontent.ToXContent; +import org.opensearch.core.xcontent.ToXContent; /** * Parsed feature norm from model definition diff --git a/src/main/java/com/o19s/es/ltr/feature/store/FeatureSupplier.java b/src/main/java/com/o19s/es/ltr/feature/store/FeatureSupplier.java index 08f82d4c..2f0e828f 100644 --- a/src/main/java/com/o19s/es/ltr/feature/store/FeatureSupplier.java +++ b/src/main/java/com/o19s/es/ltr/feature/store/FeatureSupplier.java @@ -1,3 +1,19 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package com.o19s.es.ltr.feature.store; import com.o19s.es.ltr.feature.FeatureSet; diff --git a/src/main/java/com/o19s/es/ltr/feature/store/MinMaxFeatureNormDefinition.java b/src/main/java/com/o19s/es/ltr/feature/store/MinMaxFeatureNormDefinition.java index 27de0948..b43c236d 100644 --- a/src/main/java/com/o19s/es/ltr/feature/store/MinMaxFeatureNormDefinition.java +++ b/src/main/java/com/o19s/es/ltr/feature/store/MinMaxFeatureNormDefinition.java @@ -1,12 +1,28 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package com.o19s.es.ltr.feature.store; import com.o19s.es.ltr.ranker.normalizer.MinMaxFeatureNormalizer; import com.o19s.es.ltr.ranker.normalizer.Normalizer; -import org.opensearch.common.ParseField; +import org.opensearch.core.ParseField; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.xcontent.ObjectParser; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.ObjectParser; +import org.opensearch.core.xcontent.XContentBuilder; import java.io.IOException; diff --git a/src/main/java/com/o19s/es/ltr/feature/store/PrecompiledTemplateFeature.java b/src/main/java/com/o19s/es/ltr/feature/store/PrecompiledTemplateFeature.java index 7e200c70..396ca312 100644 --- a/src/main/java/com/o19s/es/ltr/feature/store/PrecompiledTemplateFeature.java +++ b/src/main/java/com/o19s/es/ltr/feature/store/PrecompiledTemplateFeature.java @@ -27,7 +27,7 @@ import org.opensearch.common.ParsingException; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.common.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryShardException; import org.opensearch.index.query.Rewriteable; diff --git a/src/main/java/com/o19s/es/ltr/feature/store/ScriptFeature.java b/src/main/java/com/o19s/es/ltr/feature/store/ScriptFeature.java index 719dd52c..b7763dd2 100644 --- a/src/main/java/com/o19s/es/ltr/feature/store/ScriptFeature.java +++ b/src/main/java/com/o19s/es/ltr/feature/store/ScriptFeature.java @@ -1,3 +1,19 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package com.o19s.es.ltr.feature.store; import com.o19s.es.ltr.LtrQueryContext; @@ -12,19 +28,24 @@ import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.Term; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.Explanation; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.index.TermStates; +import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.search.Weight; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.Weight; +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.MatchAllDocsQuery; + import org.opensearch.common.lucene.search.function.LeafScoreFunction; import org.opensearch.common.lucene.search.function.ScriptScoreFunction; import org.opensearch.common.xcontent.LoggingDeprecationHandler; -import org.opensearch.common.xcontent.NamedXContentRegistry; -import org.opensearch.common.xcontent.XContentParser; + +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.script.ScoreScript; @@ -173,8 +194,8 @@ public Query doToQuery(LtrQueryContext context, FeatureSet featureSet, Map fields = terms.stream().map(Term::field).collect(Collectors.toUnmodifiableSet()); + for (String field : fields) { + if (visitor.acceptField(field) == false) { + return; + } + } + visitor.getSubVisitor(BooleanClause.Occur.SHOULD, this).consumeTerms(this, terms.toArray(new Term[0])); + } } static class LtrScriptWeight extends Weight { @@ -286,18 +319,31 @@ static class LtrScriptWeight extends Weight { private final ScriptScoreFunction function; private final TermStatSupplier termStatSupplier; private final Set terms; + private final HashMap termContexts; LtrScriptWeight(Query query, ScriptScoreFunction function, TermStatSupplier termStatSupplier, Set terms, IndexSearcher searcher, - ScoreMode scoreMode) { + ScoreMode scoreMode) throws IOException { super(query); this.function = function; this.termStatSupplier = termStatSupplier; this.terms = terms; this.searcher = searcher; this.scoreMode = scoreMode; + this.termContexts = new HashMap<>(); + + if (scoreMode.needsScores()) { + for (Term t : terms) { + TermStates ctx = TermStates.build(searcher.getTopReaderContext(), t, true); + if (ctx != null && ctx.docFreq() > 0) { + searcher.collectionStatistics(t.field()); + searcher.termStatistics(t, ctx.docFreq(), ctx.totalTermFreq()); + } + termContexts.put(t, ctx); + } + } } @Override @@ -319,7 +365,7 @@ public int docID() { public float score() throws IOException { // Do the terms magic if the user asked for it if (terms.size() > 0) { - termStatSupplier.bump(searcher, context, docID(), terms, scoreMode); + termStatSupplier.bump(searcher, context, docID(), terms, scoreMode, termContexts); } return (float) leafScoreFunction.score(iterator.docID(), 0F); @@ -343,7 +389,6 @@ public float getMaxScore(int upTo) throws IOException { }; } - @Override public void extractTerms(Set terms) { } diff --git a/src/main/java/com/o19s/es/ltr/feature/store/StandardFeatureNormDefinition.java b/src/main/java/com/o19s/es/ltr/feature/store/StandardFeatureNormDefinition.java index ac13889f..b4c9b91d 100644 --- a/src/main/java/com/o19s/es/ltr/feature/store/StandardFeatureNormDefinition.java +++ b/src/main/java/com/o19s/es/ltr/feature/store/StandardFeatureNormDefinition.java @@ -1,14 +1,30 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package com.o19s.es.ltr.feature.store; import com.o19s.es.ltr.ranker.normalizer.Normalizer; import com.o19s.es.ltr.ranker.normalizer.StandardFeatureNormalizer; import org.opensearch.OpenSearchException; -import org.opensearch.common.ParseField; +import org.opensearch.core.ParseField; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.xcontent.ObjectParser; -import org.opensearch.common.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentParser; +import org.opensearch.core.xcontent.ObjectParser; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; import java.io.IOException; diff --git a/src/main/java/com/o19s/es/ltr/feature/store/StorableElement.java b/src/main/java/com/o19s/es/ltr/feature/store/StorableElement.java index c14b35d1..34ac4d94 100644 --- a/src/main/java/com/o19s/es/ltr/feature/store/StorableElement.java +++ b/src/main/java/com/o19s/es/ltr/feature/store/StorableElement.java @@ -18,8 +18,8 @@ import org.opensearch.common.ParsingException; import org.opensearch.common.io.stream.NamedWriteable; -import org.opensearch.common.xcontent.ToXContent; -import org.opensearch.common.xcontent.XContentParser; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentParser; public interface StorableElement extends ToXContent, NamedWriteable { /** diff --git a/src/main/java/com/o19s/es/ltr/feature/store/StoredFeature.java b/src/main/java/com/o19s/es/ltr/feature/store/StoredFeature.java index 565a9ca6..a69e88c4 100644 --- a/src/main/java/com/o19s/es/ltr/feature/store/StoredFeature.java +++ b/src/main/java/com/o19s/es/ltr/feature/store/StoredFeature.java @@ -24,18 +24,18 @@ import org.apache.lucene.util.Accountable; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.RamUsageEstimator; -import org.opensearch.common.ParseField; +import org.opensearch.core.ParseField; import org.opensearch.common.ParsingException; import org.opensearch.common.Strings; import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.LoggingDeprecationHandler; -import org.opensearch.common.xcontent.NamedXContentRegistry; -import org.opensearch.common.xcontent.ObjectParser; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.ObjectParser; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.common.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.query.QueryShardException; import org.opensearch.index.query.ScriptQueryBuilder; diff --git a/src/main/java/com/o19s/es/ltr/feature/store/StoredFeatureNormalizers.java b/src/main/java/com/o19s/es/ltr/feature/store/StoredFeatureNormalizers.java index 616b6200..8aac603a 100644 --- a/src/main/java/com/o19s/es/ltr/feature/store/StoredFeatureNormalizers.java +++ b/src/main/java/com/o19s/es/ltr/feature/store/StoredFeatureNormalizers.java @@ -1,15 +1,31 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package com.o19s.es.ltr.feature.store; import com.o19s.es.ltr.feature.FeatureSet; import com.o19s.es.ltr.ranker.normalizer.Normalizer; import org.opensearch.OpenSearchException; -import org.opensearch.common.ParseField; +import org.opensearch.core.ParseField; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.xcontent.ObjectParser; -import org.opensearch.common.xcontent.ToXContent; -import org.opensearch.common.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentParser; +import org.opensearch.core.xcontent.ObjectParser; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; import java.io.IOException; import java.util.HashMap; diff --git a/src/main/java/com/o19s/es/ltr/feature/store/StoredFeatureSet.java b/src/main/java/com/o19s/es/ltr/feature/store/StoredFeatureSet.java index afe3620f..65fa328c 100644 --- a/src/main/java/com/o19s/es/ltr/feature/store/StoredFeatureSet.java +++ b/src/main/java/com/o19s/es/ltr/feature/store/StoredFeatureSet.java @@ -23,13 +23,13 @@ import org.apache.lucene.search.Query; import org.apache.lucene.util.Accountable; import org.apache.lucene.util.RamUsageEstimator; -import org.opensearch.common.ParseField; +import org.opensearch.core.ParseField; import org.opensearch.common.ParsingException; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.xcontent.ObjectParser; -import org.opensearch.common.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentParser; +import org.opensearch.core.xcontent.ObjectParser; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; import java.io.IOException; import java.util.ArrayList; diff --git a/src/main/java/com/o19s/es/ltr/feature/store/StoredLtrModel.java b/src/main/java/com/o19s/es/ltr/feature/store/StoredLtrModel.java index 4bed9c04..37954b60 100644 --- a/src/main/java/com/o19s/es/ltr/feature/store/StoredLtrModel.java +++ b/src/main/java/com/o19s/es/ltr/feature/store/StoredLtrModel.java @@ -23,16 +23,16 @@ import com.o19s.es.ltr.ranker.parser.LtrRankerParser; import com.o19s.es.ltr.ranker.parser.LtrRankerParserFactory; import org.opensearch.Version; -import org.opensearch.common.ParseField; +import org.opensearch.core.ParseField; import org.opensearch.common.ParsingException; import org.opensearch.common.Strings; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.common.io.stream.Writeable; import org.opensearch.common.xcontent.LoggingDeprecationHandler; -import org.opensearch.common.xcontent.ObjectParser; -import org.opensearch.common.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentParser; +import org.opensearch.core.xcontent.ObjectParser; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.json.JsonXContent; import java.io.IOException; @@ -40,7 +40,7 @@ import java.util.Map; import java.util.Objects; -import static org.opensearch.common.xcontent.NamedXContentRegistry.EMPTY; +import static org.opensearch.core.xcontent.NamedXContentRegistry.EMPTY; public class StoredLtrModel implements StorableElement { public static final String TYPE = "model"; diff --git a/src/main/java/com/o19s/es/ltr/feature/store/index/IndexFeatureStore.java b/src/main/java/com/o19s/es/ltr/feature/store/index/IndexFeatureStore.java index 5cb25904..7829c02d 100644 --- a/src/main/java/com/o19s/es/ltr/feature/store/index/IndexFeatureStore.java +++ b/src/main/java/com/o19s/es/ltr/feature/store/index/IndexFeatureStore.java @@ -35,17 +35,17 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.MetadataCreateIndexService; import org.opensearch.common.CheckedFunction; -import org.opensearch.common.ParseField; +import org.opensearch.core.ParseField; import org.opensearch.common.bytes.BytesReference; import org.apache.logging.log4j.LogManager; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.LoggingDeprecationHandler; -import org.opensearch.common.xcontent.NamedXContentRegistry; -import org.opensearch.common.xcontent.ObjectParser; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.ObjectParser; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.common.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.internal.io.Streams; @@ -195,7 +195,7 @@ public GetResponse getModel(String name) { } private Supplier internalGet(String id) { - return () -> clientSupplier.get().prepareGet(index, ES_TYPE, id).get(); + return () -> clientSupplier.get().prepareGet(index, id).get(); } /** @@ -264,7 +264,7 @@ void setElement(StorableElement element) { public static CreateIndexRequest buildIndexRequest(String indexName) { return new CreateIndexRequest(indexName) - .mapping(ES_TYPE, readResourceFile(indexName, MAPPING_FILE), XContentType.JSON) + .mapping(readResourceFile(indexName, MAPPING_FILE), XContentType.JSON) .settings(storeIndexSettings(indexName)); } diff --git a/src/main/java/com/o19s/es/ltr/logging/LoggingSearchExtBuilder.java b/src/main/java/com/o19s/es/ltr/logging/LoggingSearchExtBuilder.java index 88ce4760..05e0bb2c 100644 --- a/src/main/java/com/o19s/es/ltr/logging/LoggingSearchExtBuilder.java +++ b/src/main/java/com/o19s/es/ltr/logging/LoggingSearchExtBuilder.java @@ -17,15 +17,15 @@ package com.o19s.es.ltr.logging; import org.opensearch.common.Nullable; -import org.opensearch.common.ParseField; +import org.opensearch.core.ParseField; import org.opensearch.common.ParsingException; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.common.io.stream.Writeable; -import org.opensearch.common.xcontent.ObjectParser; -import org.opensearch.common.xcontent.ToXContentObject; -import org.opensearch.common.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentParser; +import org.opensearch.core.xcontent.ObjectParser; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.search.SearchExtBuilder; import java.io.IOException; diff --git a/src/main/java/com/o19s/es/ltr/query/DerivedExpressionQuery.java b/src/main/java/com/o19s/es/ltr/query/DerivedExpressionQuery.java index d6ebab5b..07394c87 100644 --- a/src/main/java/com/o19s/es/ltr/query/DerivedExpressionQuery.java +++ b/src/main/java/com/o19s/es/ltr/query/DerivedExpressionQuery.java @@ -22,17 +22,23 @@ import org.apache.lucene.expressions.Expression; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.Term; -import org.apache.lucene.search.ConstantScoreScorer; -import org.apache.lucene.search.ConstantScoreWeight; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.DoubleValues; -import org.apache.lucene.search.DoubleValuesSource; -import org.apache.lucene.search.Explanation; -import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.search.Weight; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.Weight; +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.ConstantScoreScorer; +import org.apache.lucene.search.DoubleValuesSource; +import org.apache.lucene.search.DoubleValues; +import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.ConstantScoreWeight; + import java.io.IOException; import java.util.Map; @@ -130,6 +136,11 @@ public int hashCode() { // Should not be called as it is likely an indication that it'll be cached but should not... return Objects.hash(classHash(), query, fvSupplier); } + + @Override + public void visit(QueryVisitor visitor) { + this.query.visit(visitor.getSubVisitor(BooleanClause.Occur.MUST, this)); + } } static class FVWeight extends Weight { @@ -146,7 +157,6 @@ static class FVWeight extends Weight { vectorSupplier = query.fvSupplier; } - @Override public void extractTerms(Set terms) { // No-op } @@ -298,4 +308,9 @@ public boolean isCacheable(LeafReaderContext ctx) { return false; } } + + @Override + public void visit(QueryVisitor visitor) { + // No-op + } } diff --git a/src/main/java/com/o19s/es/ltr/query/LtrQueryBuilder.java b/src/main/java/com/o19s/es/ltr/query/LtrQueryBuilder.java index 17cb4394..342ef2b8 100644 --- a/src/main/java/com/o19s/es/ltr/query/LtrQueryBuilder.java +++ b/src/main/java/com/o19s/es/ltr/query/LtrQueryBuilder.java @@ -24,13 +24,13 @@ import com.o19s.es.ltr.ranker.ranklib.RankLibScriptEngine; import com.o19s.es.ltr.utils.AbstractQueryBuilderUtils; import org.apache.lucene.search.Query; -import org.opensearch.common.ParseField; +import org.opensearch.core.ParseField; import org.opensearch.common.ParsingException; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.xcontent.ObjectParser; -import org.opensearch.common.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentParser; +import org.opensearch.core.xcontent.ObjectParser; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.query.AbstractQueryBuilder; import org.opensearch.index.query.MatchAllQueryBuilder; import org.opensearch.index.query.QueryBuilder; diff --git a/src/main/java/com/o19s/es/ltr/query/LtrRewritableQuery.java b/src/main/java/com/o19s/es/ltr/query/LtrRewritableQuery.java index d4b8baaa..5ce823f2 100644 --- a/src/main/java/com/o19s/es/ltr/query/LtrRewritableQuery.java +++ b/src/main/java/com/o19s/es/ltr/query/LtrRewritableQuery.java @@ -1,3 +1,19 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package com.o19s.es.ltr.query; import org.apache.lucene.search.Query; diff --git a/src/main/java/com/o19s/es/ltr/query/LtrRewriteContext.java b/src/main/java/com/o19s/es/ltr/query/LtrRewriteContext.java index 1ddea240..e6a44129 100644 --- a/src/main/java/com/o19s/es/ltr/query/LtrRewriteContext.java +++ b/src/main/java/com/o19s/es/ltr/query/LtrRewriteContext.java @@ -1,3 +1,19 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package com.o19s.es.ltr.query; import com.o19s.es.ltr.ranker.LogLtrRanker; diff --git a/src/main/java/com/o19s/es/ltr/query/RankerQuery.java b/src/main/java/com/o19s/es/ltr/query/RankerQuery.java index b73fd4e7..f9938071 100644 --- a/src/main/java/com/o19s/es/ltr/query/RankerQuery.java +++ b/src/main/java/com/o19s/es/ltr/query/RankerQuery.java @@ -29,21 +29,24 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.Term; -import org.apache.lucene.search.ConstantScoreScorer; -import org.apache.lucene.search.ConstantScoreWeight; -import org.apache.lucene.search.DisiPriorityQueue; -import org.apache.lucene.search.DisiWrapper; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.Explanation; -import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.ScoreMode; -import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Weight; +import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.ConstantScoreScorer; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.DisiPriorityQueue; +import org.apache.lucene.search.DisiWrapper; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -61,11 +64,14 @@ public class RankerQuery extends Query { private final List queries; private final FeatureSet features; private final LtrRanker ranker; + private final Map featureScoreCache; - private RankerQuery(List queries, FeatureSet features, LtrRanker ranker) { + private RankerQuery(List queries, FeatureSet features, LtrRanker ranker, + Map featureScoreCache) { this.queries = Objects.requireNonNull(queries); this.features = Objects.requireNonNull(features); this.ranker = Objects.requireNonNull(ranker); + this.featureScoreCache = featureScoreCache; } /** @@ -77,7 +83,7 @@ private RankerQuery(List queries, FeatureSet features, LtrRanker ranker) */ public static RankerQuery build(PrebuiltLtrModel model) { return build(model.ranker(), model.featureSet(), - new LtrQueryContext(null, Collections.emptySet()), Collections.emptyMap()); + new LtrQueryContext(null, Collections.emptySet()), Collections.emptyMap(), false); } /** @@ -88,26 +94,31 @@ public static RankerQuery build(PrebuiltLtrModel model) { * @param params the query params * @return the lucene query */ - public static RankerQuery build(LtrModel model, LtrQueryContext context, Map params) { - return build(model.ranker(), model.featureSet(), context, params); + public static RankerQuery build(LtrModel model, LtrQueryContext context, Map params, + Boolean featureScoreCacheFlag) { + return build(model.ranker(), model.featureSet(), context, params, featureScoreCacheFlag); } private static RankerQuery build(LtrRanker ranker, FeatureSet features, - LtrQueryContext context, Map params) { + LtrQueryContext context, Map params, Boolean featureScoreCacheFlag) { List queries = features.toQueries(context, params); - return new RankerQuery(queries, features, ranker); + Map featureScoreCache = null; + if (null != featureScoreCacheFlag && featureScoreCacheFlag) { + featureScoreCache = new HashMap<>(); + } + return new RankerQuery(queries, features, ranker, featureScoreCache); } public static RankerQuery buildLogQuery(LogLtrRanker.LogConsumer consumer, FeatureSet features, LtrQueryContext context, Map params) { List queries = features.toQueries(context, params); return new RankerQuery(queries, features, - new LogLtrRanker(consumer, features.size())); + new LogLtrRanker(consumer, features.size()), null); } public RankerQuery toLoggerQuery(LogLtrRanker.LogConsumer consumer) { NullRanker newRanker = new NullRanker(features.size()); - return new RankerQuery(queries, features, new LogLtrRanker(newRanker, consumer)); + return new RankerQuery(queries, features, new LogLtrRanker(newRanker, consumer), featureScoreCache); } @Override @@ -119,7 +130,7 @@ public Query rewrite(IndexReader reader) throws IOException { rewritten |= rewrittenQuery != query; rewrittenQueries.add(rewrittenQuery); } - return rewritten ? new RankerQuery(rewrittenQueries, features, ranker) : this; + return rewritten ? new RankerQuery(rewrittenQueries, features, ranker, featureScoreCache) : this; } @SuppressWarnings("EqualsWhichDoesntCheckParameterClass") @@ -141,6 +152,7 @@ public boolean equals(Object obj) { Stream stream() { return queries.stream(); } + @Override public int hashCode() { return 31 * classHash() + Objects.hash(features, queries, ranker); @@ -177,7 +189,7 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo @Override public Scorer scorer(LeafReaderContext context) throws IOException { return new ConstantScoreScorer(this, score(), - scoreMode, DocIdSetIterator.all(context.reader().maxDoc())); + scoreMode, DocIdSetIterator.all(context.reader().maxDoc())); } @Override @@ -196,24 +208,27 @@ public boolean isCacheable(LeafReaderContext ctx) { LtrRewriteContext context = new LtrRewriteContext(ranker, vectorSupplier); for (Query q : queries) { if (q instanceof LtrRewritableQuery) { - q = ((LtrRewritableQuery)q).ltrRewrite(context); + q = ((LtrRewritableQuery) q).ltrRewrite(context); } weights.add(searcher.createWeight(q, ScoreMode.COMPLETE, boost)); } - return new RankerWeight(this, weights, ltrRankerWrapper, features); + return new RankerWeight(this, weights, ltrRankerWrapper, features, featureScoreCache); } public static class RankerWeight extends Weight { private final List weights; private final FVLtrRankerWrapper ranker; private final FeatureSet features; + private final Map featureScoreCache; - RankerWeight(RankerQuery query, List weights, FVLtrRankerWrapper ranker, FeatureSet features) { + RankerWeight(RankerQuery query, List weights, FVLtrRankerWrapper ranker, FeatureSet features, + Map featureScoreCache) { super(query); assert weights instanceof RandomAccess; this.weights = weights; this.ranker = Objects.requireNonNull(ranker); this.features = Objects.requireNonNull(features); + this.featureScoreCache = featureScoreCache; } @Override @@ -221,10 +236,10 @@ public boolean isCacheable(LeafReaderContext ctx) { return false; } - @Override public void extractTerms(Set terms) { for (Weight w : weights) { - w.extractTerms(terms); +// w.extractTerms(terms); + QueryVisitor.termCollector(terms); } } @@ -268,8 +283,8 @@ public RankerScorer scorer(LeafReaderContext context) throws IOException { } DisjunctionDISI rankerIterator = new DisjunctionDISI( - DocIdSetIterator.all(context.reader().maxDoc()), disiPriorityQueue); - return new RankerScorer(scorers, rankerIterator, ranker); + DocIdSetIterator.all(context.reader().maxDoc()), disiPriorityQueue, context.docBase, featureScoreCache); + return new RankerScorer(scorers, rankerIterator, ranker, context.docBase, featureScoreCache); } class RankerScorer extends Scorer { @@ -281,12 +296,17 @@ class RankerScorer extends Scorer { private final DisjunctionDISI iterator; private final FVLtrRankerWrapper ranker; private LtrRanker.FeatureVector fv; + private final int docBase; + private final Map featureScoreCache; - RankerScorer(List scorers, DisjunctionDISI iterator, FVLtrRankerWrapper ranker) { + RankerScorer(List scorers, DisjunctionDISI iterator, FVLtrRankerWrapper ranker, + int docBase, Map featureScoreCache) { super(RankerWeight.this); this.scorers = scorers; this.iterator = iterator; this.ranker = ranker; + this.docBase = docBase; + this.featureScoreCache = featureScoreCache; } @Override @@ -297,16 +317,43 @@ public int docID() { @Override public float score() throws IOException { fv = ranker.newFeatureVector(fv); - int ordinal = -1; - // a DisiPriorityQueue could help to avoid - // looping on all scorers - for (Scorer scorer : scorers) { - ordinal++; - // FIXME: Probably inefficient, again we loop over all scorers.. - if (scorer.docID() == docID()) { - // XXX: bold assumption that all models are dense - // do we need a some indirection to infer the featureId? - fv.setFeatureScore(ordinal, scorer.score()); + if (featureScoreCache == null) { // Cache disabled + int ordinal = -1; + // a DisiPriorityQueue could help to avoid + // looping on all scorers + for (Scorer scorer : scorers) { + ordinal++; + // FIXME: Probably inefficient, again we loop over all scorers.. + if (scorer.docID() == docID()) { + // XXX: bold assumption that all models are dense + // do we need a some indirection to infer the featureId? + fv.setFeatureScore(ordinal, scorer.score()); + } + } + } else { + int perShardDocId = docBase + docID(); + if (featureScoreCache.containsKey(perShardDocId)) { // Cache hit + float[] featureScores = featureScoreCache.get(perShardDocId); + int ordinal = -1; + for (float score : featureScores) { + ordinal++; + if (!Float.isNaN(score)) { + fv.setFeatureScore(ordinal, score); + } + } + } else { // Cache miss + int ordinal = -1; + float[] featureScores = new float[scorers.size()]; + for (Scorer scorer : scorers) { + ordinal++; + float score = Float.NaN; + if (scorer.docID() == docID()) { + score = scorer.score(); + fv.setFeatureScore(ordinal, score); + } + featureScores[ordinal] = score; + } + featureScoreCache.put(perShardDocId, featureScores); } } return ranker.score(fv); @@ -343,10 +390,15 @@ public float getMaxScore(int upTo) throws IOException { static class DisjunctionDISI extends DocIdSetIterator { private final DocIdSetIterator main; private final DisiPriorityQueue subIteratorsPriorityQueue; + private final int docBase; + private final Map featureScoreCache; - DisjunctionDISI(DocIdSetIterator main, DisiPriorityQueue subIteratorsPriorityQueue) { + DisjunctionDISI(DocIdSetIterator main, DisiPriorityQueue subIteratorsPriorityQueue, int docBase, + Map featureScoreCache) { this.main = main; this.subIteratorsPriorityQueue = subIteratorsPriorityQueue; + this.docBase = docBase; + this.featureScoreCache = featureScoreCache; } @Override @@ -364,6 +416,9 @@ public int nextDoc() throws IOException { @Override public int advance(int target) throws IOException { int docId = main.advance(target); + if (featureScoreCache != null && featureScoreCache.containsKey(docBase + target)) { + return docId; // Cache hit. No need to advance sub iterators + } advanceSubIterators(docId); return docId; } @@ -425,4 +480,13 @@ public int hashCode() { return Objects.hash(wrapped, vectorSupplier); } } + + @Override + public void visit(QueryVisitor visitor) { + QueryVisitor v = visitor.getSubVisitor(BooleanClause.Occur.SHOULD, this); + for (Query q : queries) { + q.visit(v); + } + } + } diff --git a/src/main/java/com/o19s/es/ltr/query/StoredLtrQueryBuilder.java b/src/main/java/com/o19s/es/ltr/query/StoredLtrQueryBuilder.java index 262b863d..abcc2584 100644 --- a/src/main/java/com/o19s/es/ltr/query/StoredLtrQueryBuilder.java +++ b/src/main/java/com/o19s/es/ltr/query/StoredLtrQueryBuilder.java @@ -24,14 +24,14 @@ import com.o19s.es.ltr.ranker.linear.LinearRanker; import com.o19s.es.ltr.utils.FeatureStoreLoader; import org.opensearch.Version; -import org.opensearch.common.ParseField; +import org.opensearch.core.ParseField; import org.opensearch.common.ParsingException; import org.opensearch.common.io.stream.NamedWriteable; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.xcontent.ObjectParser; -import org.opensearch.common.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentParser; +import org.opensearch.core.xcontent.ObjectParser; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.query.AbstractQueryBuilder; import org.opensearch.index.query.QueryShardContext; @@ -49,6 +49,7 @@ public class StoredLtrQueryBuilder extends AbstractQueryBuilder implements NamedWriteable { public static final String NAME = "sltr"; public static final ParseField MODEL_NAME = new ParseField("model"); + public static final ParseField FEATURE_CACHE_FLAG = new ParseField("cache"); public static final ParseField FEATURESET_NAME = new ParseField("featureset"); public static final ParseField STORE_NAME = new ParseField("store"); public static final ParseField PARAMS = new ParseField("params"); @@ -58,6 +59,7 @@ public class StoredLtrQueryBuilder extends AbstractQueryBuilder(NAME); PARSER.declareString(StoredLtrQueryBuilder::modelName, MODEL_NAME); + PARSER.declareBoolean(StoredLtrQueryBuilder::featureScoreCacheFlag, FEATURE_CACHE_FLAG); PARSER.declareString(StoredLtrQueryBuilder::featureSetName, FEATURESET_NAME); PARSER.declareString(StoredLtrQueryBuilder::storeName, STORE_NAME); PARSER.declareField(StoredLtrQueryBuilder::params, XContentParser::map, PARAMS, ObjectParser.ValueType.OBJECT); @@ -66,10 +68,11 @@ public class StoredLtrQueryBuilder extends AbstractQueryBuilder params; @@ -84,10 +87,13 @@ public StoredLtrQueryBuilder(FeatureStoreLoader storeLoader, StreamInput input) super(input); this.storeLoader = Objects.requireNonNull(storeLoader); modelName = input.readOptionalString(); + featureScoreCacheFlag = input.readOptionalBoolean(); featureSetName = input.readOptionalString(); params = input.readMap(); - String[] activeFeat = input.readOptionalStringArray(); - activeFeatures = activeFeat == null ? null : Arrays.asList(activeFeat); + if (input.getVersion().onOrAfter(Version.V_1_0_0)) {//TODO: check this. In the Elastic LTR plugin this is set to 7_0_0 + String[] activeFeat = input.readOptionalStringArray(); + activeFeatures = activeFeat == null ? null : Arrays.asList(activeFeat); + } storeName = input.readOptionalString(); } @@ -112,9 +118,12 @@ public static StoredLtrQueryBuilder fromXContent(FeatureStoreLoader storeLoader, @Override protected void doWriteTo(StreamOutput out) throws IOException { out.writeOptionalString(modelName); + out.writeOptionalBoolean(featureScoreCacheFlag); out.writeOptionalString(featureSetName); out.writeMap(params); - out.writeOptionalStringArray(activeFeatures != null ? activeFeatures.toArray(new String[0]) : null); + if (out.getVersion().onOrAfter(Version.V_1_0_0)) { + out.writeOptionalStringArray(activeFeatures != null ? activeFeatures.toArray(new String[0]) : null); + } out.writeOptionalString(storeName); } @@ -124,6 +133,9 @@ protected void doXContent(XContentBuilder builder, Params p) throws IOException if (modelName != null) { builder.field(MODEL_NAME.getPreferredName(), modelName); } + if (featureScoreCacheFlag != null) { + builder.field(FEATURE_CACHE_FLAG.getPreferredName(), featureScoreCacheFlag); + } if (featureSetName != null) { builder.field(FEATURESET_NAME.getPreferredName(), featureSetName); } @@ -157,7 +169,7 @@ protected RankerQuery doToQuery(QueryShardContext context) throws IOException { if (modelName != null) { CompiledLtrModel model = store.loadModel(modelName); validateActiveFeatures(model.featureSet(), ltrQueryContext); - return RankerQuery.build(model, ltrQueryContext, params); + return RankerQuery.build(model, ltrQueryContext, params, featureScoreCacheFlag); } else { assert featureSetName != null; FeatureSet set = store.loadSet(featureSetName); @@ -166,13 +178,14 @@ protected RankerQuery doToQuery(QueryShardContext context) throws IOException { LinearRanker ranker = new LinearRanker(weights); CompiledLtrModel model = new CompiledLtrModel("linear", set, ranker); validateActiveFeatures(model.featureSet(), ltrQueryContext); - return RankerQuery.build(model, ltrQueryContext, params); + return RankerQuery.build(model, ltrQueryContext, params, featureScoreCacheFlag); } } @Override protected boolean doEquals(StoredLtrQueryBuilder other) { return Objects.equals(modelName, other.modelName) && + Objects.equals(featureScoreCacheFlag, other.featureScoreCacheFlag) && Objects.equals(featureSetName, other.featureSetName) && Objects.equals(storeName, other.storeName) && Objects.equals(params, other.params) && @@ -181,7 +194,7 @@ protected boolean doEquals(StoredLtrQueryBuilder other) { @Override protected int doHashCode() { - return Objects.hash(modelName, featureSetName, storeName, params, activeFeatures); + return Objects.hash(modelName, featureScoreCacheFlag, featureSetName, storeName, params, activeFeatures); } @Override @@ -198,6 +211,11 @@ public StoredLtrQueryBuilder modelName(String modelName) { return this; } + public StoredLtrQueryBuilder featureScoreCacheFlag(Boolean featureScoreCacheFlag) { + this.featureScoreCacheFlag = featureScoreCacheFlag; + return this; + } + public String featureSetName() { return featureSetName; } diff --git a/src/main/java/com/o19s/es/ltr/query/ValidatingLtrQueryBuilder.java b/src/main/java/com/o19s/es/ltr/query/ValidatingLtrQueryBuilder.java index 874c6f6f..30f8d07f 100644 --- a/src/main/java/com/o19s/es/ltr/query/ValidatingLtrQueryBuilder.java +++ b/src/main/java/com/o19s/es/ltr/query/ValidatingLtrQueryBuilder.java @@ -30,13 +30,13 @@ import com.o19s.es.ltr.ranker.parser.LtrRankerParserFactory; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; -import org.opensearch.common.ParseField; +import org.opensearch.core.ParseField; import org.opensearch.common.ParsingException; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.xcontent.ObjectParser; -import org.opensearch.common.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentParser; +import org.opensearch.core.xcontent.ObjectParser; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.query.AbstractQueryBuilder; import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.query.QueryShardException; @@ -173,10 +173,10 @@ protected Query doToQuery(QueryShardContext queryShardContext) throws IOExceptio FeatureSet set = ((StoredFeatureSet) element).optimize(); LinearRanker ranker = new LinearRanker(new float[set.size()]); CompiledLtrModel model = new CompiledLtrModel("validation", set, ranker); - return RankerQuery.build(model, context, validation.getParams()); + return RankerQuery.build(model, context, validation.getParams(), false); } else if (StoredLtrModel.TYPE.equals(element.type())) { CompiledLtrModel model = ((StoredLtrModel) element).compile(factory); - return RankerQuery.build(model, context, validation.getParams()); + return RankerQuery.build(model, context, validation.getParams(), false); } else { throw new QueryShardException(queryShardContext, "Unknown element type [" + element.type() + "]"); } diff --git a/src/main/java/com/o19s/es/ltr/ranker/LtrRanker.java b/src/main/java/com/o19s/es/ltr/ranker/LtrRanker.java index 64c4fa2f..38ea4c48 100644 --- a/src/main/java/com/o19s/es/ltr/ranker/LtrRanker.java +++ b/src/main/java/com/o19s/es/ltr/ranker/LtrRanker.java @@ -17,9 +17,10 @@ package com.o19s.es.ltr.ranker; import org.opensearch.common.Nullable; +import com.o19s.es.ltr.query.RankerQuery; /** - * A ranker used by {@link com.o19s.es.ltr.query.RankerQuery} + * A ranker used by {@link RankerQuery} * to compute a score based on a set of feature scores. */ public interface LtrRanker { diff --git a/src/main/java/com/o19s/es/ltr/ranker/normalizer/FeatureNormalizingRanker.java b/src/main/java/com/o19s/es/ltr/ranker/normalizer/FeatureNormalizingRanker.java index a3580dfe..999dc16e 100644 --- a/src/main/java/com/o19s/es/ltr/ranker/normalizer/FeatureNormalizingRanker.java +++ b/src/main/java/com/o19s/es/ltr/ranker/normalizer/FeatureNormalizingRanker.java @@ -1,3 +1,19 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package com.o19s.es.ltr.ranker.normalizer; import com.o19s.es.ltr.ranker.LtrRanker; diff --git a/src/main/java/com/o19s/es/ltr/ranker/normalizer/MinMaxFeatureNormalizer.java b/src/main/java/com/o19s/es/ltr/ranker/normalizer/MinMaxFeatureNormalizer.java index 9e41a88c..9e3773f7 100644 --- a/src/main/java/com/o19s/es/ltr/ranker/normalizer/MinMaxFeatureNormalizer.java +++ b/src/main/java/com/o19s/es/ltr/ranker/normalizer/MinMaxFeatureNormalizer.java @@ -1,3 +1,19 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package com.o19s.es.ltr.ranker.normalizer; /** diff --git a/src/main/java/com/o19s/es/ltr/ranker/normalizer/Normalizer.java b/src/main/java/com/o19s/es/ltr/ranker/normalizer/Normalizer.java index 46c0d0ab..fbf766b5 100644 --- a/src/main/java/com/o19s/es/ltr/ranker/normalizer/Normalizer.java +++ b/src/main/java/com/o19s/es/ltr/ranker/normalizer/Normalizer.java @@ -1,3 +1,19 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package com.o19s.es.ltr.ranker.normalizer; /** diff --git a/src/main/java/com/o19s/es/ltr/ranker/normalizer/Normalizers.java b/src/main/java/com/o19s/es/ltr/ranker/normalizer/Normalizers.java index 2ac1ae74..9567f11b 100644 --- a/src/main/java/com/o19s/es/ltr/ranker/normalizer/Normalizers.java +++ b/src/main/java/com/o19s/es/ltr/ranker/normalizer/Normalizers.java @@ -1,3 +1,19 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package com.o19s.es.ltr.ranker.normalizer; import java.util.Collections; diff --git a/src/main/java/com/o19s/es/ltr/ranker/normalizer/StandardFeatureNormalizer.java b/src/main/java/com/o19s/es/ltr/ranker/normalizer/StandardFeatureNormalizer.java index cf24f7d8..9bb9a948 100644 --- a/src/main/java/com/o19s/es/ltr/ranker/normalizer/StandardFeatureNormalizer.java +++ b/src/main/java/com/o19s/es/ltr/ranker/normalizer/StandardFeatureNormalizer.java @@ -1,3 +1,19 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package com.o19s.es.ltr.ranker.normalizer; public class StandardFeatureNormalizer implements Normalizer { diff --git a/src/main/java/com/o19s/es/ltr/ranker/parser/LinearRankerParser.java b/src/main/java/com/o19s/es/ltr/ranker/parser/LinearRankerParser.java index 8b747d6c..fc81cbc8 100644 --- a/src/main/java/com/o19s/es/ltr/ranker/parser/LinearRankerParser.java +++ b/src/main/java/com/o19s/es/ltr/ranker/parser/LinearRankerParser.java @@ -20,12 +20,12 @@ import com.o19s.es.ltr.ranker.linear.LinearRanker; import org.opensearch.common.ParsingException; import org.opensearch.common.xcontent.LoggingDeprecationHandler; -import org.opensearch.common.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.json.JsonXContent; import java.io.IOException; -import static org.opensearch.common.xcontent.NamedXContentRegistry.EMPTY; +import static org.opensearch.core.xcontent.NamedXContentRegistry.EMPTY; public class LinearRankerParser implements LtrRankerParser { public static final String TYPE = "model/linear"; diff --git a/src/main/java/com/o19s/es/ltr/ranker/parser/XGBoostJsonParser.java b/src/main/java/com/o19s/es/ltr/ranker/parser/XGBoostJsonParser.java index 536f7e2b..9d81bd1d 100644 --- a/src/main/java/com/o19s/es/ltr/ranker/parser/XGBoostJsonParser.java +++ b/src/main/java/com/o19s/es/ltr/ranker/parser/XGBoostJsonParser.java @@ -21,13 +21,13 @@ import com.o19s.es.ltr.ranker.dectree.NaiveAdditiveDecisionTree.Node; import com.o19s.es.ltr.ranker.normalizer.Normalizer; import com.o19s.es.ltr.ranker.normalizer.Normalizers; -import org.opensearch.common.ParseField; +import org.opensearch.core.ParseField; import org.opensearch.common.ParsingException; import org.opensearch.common.xcontent.LoggingDeprecationHandler; -import org.opensearch.common.xcontent.NamedXContentRegistry; -import org.opensearch.common.xcontent.ObjectParser; -import org.opensearch.common.xcontent.XContentParseException; -import org.opensearch.common.xcontent.XContentParser; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.ObjectParser; +import org.opensearch.core.xcontent.XContentParseException; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.json.JsonXContent; import java.io.IOException; diff --git a/src/main/java/com/o19s/es/ltr/rest/AutoDetectParser.java b/src/main/java/com/o19s/es/ltr/rest/AutoDetectParser.java index ca64bd9b..274c305a 100644 --- a/src/main/java/com/o19s/es/ltr/rest/AutoDetectParser.java +++ b/src/main/java/com/o19s/es/ltr/rest/AutoDetectParser.java @@ -1,3 +1,19 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package com.o19s.es.ltr.rest; import com.o19s.es.ltr.feature.FeatureValidation; @@ -5,10 +21,10 @@ import com.o19s.es.ltr.feature.store.StoredFeature; import com.o19s.es.ltr.feature.store.StoredFeatureSet; import com.o19s.es.ltr.feature.store.StoredLtrModel; -import org.opensearch.common.ParseField; +import org.opensearch.core.ParseField; import org.opensearch.common.ParsingException; -import org.opensearch.common.xcontent.ObjectParser; -import org.opensearch.common.xcontent.XContentParser; +import org.opensearch.core.xcontent.ObjectParser; +import org.opensearch.core.xcontent.XContentParser; import java.io.IOException; diff --git a/src/main/java/com/o19s/es/ltr/rest/RestAddFeatureToSet.java b/src/main/java/com/o19s/es/ltr/rest/RestAddFeatureToSet.java index 89a6efc7..0da780c7 100644 --- a/src/main/java/com/o19s/es/ltr/rest/RestAddFeatureToSet.java +++ b/src/main/java/com/o19s/es/ltr/rest/RestAddFeatureToSet.java @@ -20,9 +20,9 @@ import com.o19s.es.ltr.feature.FeatureValidation; import com.o19s.es.ltr.feature.store.StoredFeature; import org.opensearch.client.node.NodeClient; -import org.opensearch.common.ParseField; -import org.opensearch.common.xcontent.ObjectParser; -import org.opensearch.common.xcontent.XContentParser; +import org.opensearch.core.ParseField; +import org.opensearch.core.xcontent.ObjectParser; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.rest.RestRequest; import org.opensearch.rest.action.RestStatusToXContentListener; diff --git a/src/main/java/com/o19s/es/ltr/rest/RestCreateModelFromSet.java b/src/main/java/com/o19s/es/ltr/rest/RestCreateModelFromSet.java index 50459252..d492e169 100644 --- a/src/main/java/com/o19s/es/ltr/rest/RestCreateModelFromSet.java +++ b/src/main/java/com/o19s/es/ltr/rest/RestCreateModelFromSet.java @@ -23,10 +23,10 @@ import org.opensearch.ExceptionsHelper; import org.opensearch.action.ActionListener; import org.opensearch.client.node.NodeClient; -import org.opensearch.common.ParseField; +import org.opensearch.core.ParseField; import org.opensearch.common.ParsingException; -import org.opensearch.common.xcontent.ObjectParser; -import org.opensearch.common.xcontent.XContentParser; +import org.opensearch.core.xcontent.ObjectParser; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.engine.VersionConflictEngineException; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestRequest; diff --git a/src/main/java/com/o19s/es/ltr/rest/RestFeatureManager.java b/src/main/java/com/o19s/es/ltr/rest/RestFeatureManager.java index 867eef43..9b2d8a4b 100644 --- a/src/main/java/com/o19s/es/ltr/rest/RestFeatureManager.java +++ b/src/main/java/com/o19s/es/ltr/rest/RestFeatureManager.java @@ -1,3 +1,19 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package com.o19s.es.ltr.rest; import com.o19s.es.ltr.action.ClearCachesAction; @@ -74,7 +90,7 @@ RestChannelConsumer delete(NodeClient client, String type, String indexName, Res String routing = request.param("routing"); return (channel) -> { RestStatusToXContentListener restR = new RestStatusToXContentListener<>(channel, (r) -> r.getLocation(routing)); - client.prepareDelete(indexName, ES_TYPE, id) + client.prepareDelete(indexName, id) .setRouting(routing) .execute(ActionListener.wrap((deleteResponse) -> { // wrap the response so we can send another request to clear the cache @@ -108,7 +124,7 @@ RestChannelConsumer get(NodeClient client, String type, String indexName, RestRe String name = request.param("name"); String routing = request.param("routing"); String id = generateId(type, name); - return (channel) -> client.prepareGet(indexName, ES_TYPE, id) + return (channel) -> client.prepareGet(indexName, id) .setRouting(routing) .execute(new RestToXContentListener(channel) { @Override diff --git a/src/main/java/com/o19s/es/ltr/rest/RestFeatureStoreCaches.java b/src/main/java/com/o19s/es/ltr/rest/RestFeatureStoreCaches.java index b8a701a9..9d3000e0 100644 --- a/src/main/java/com/o19s/es/ltr/rest/RestFeatureStoreCaches.java +++ b/src/main/java/com/o19s/es/ltr/rest/RestFeatureStoreCaches.java @@ -20,7 +20,7 @@ import com.o19s.es.ltr.action.ClearCachesAction; import com.o19s.es.ltr.action.ClearCachesAction.ClearCachesNodesResponse; import org.opensearch.client.node.NodeClient; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; diff --git a/src/main/java/com/o19s/es/ltr/rest/RestLTRStats.java b/src/main/java/com/o19s/es/ltr/rest/RestLTRStats.java index 1a0c8ec6..da8af554 100644 --- a/src/main/java/com/o19s/es/ltr/rest/RestLTRStats.java +++ b/src/main/java/com/o19s/es/ltr/rest/RestLTRStats.java @@ -1,3 +1,19 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package com.o19s.es.ltr.rest; import com.o19s.es.ltr.action.LTRStatsAction; diff --git a/src/main/java/com/o19s/es/ltr/rest/RestSearchStoreElements.java b/src/main/java/com/o19s/es/ltr/rest/RestSearchStoreElements.java index 043ccc88..beef4101 100644 --- a/src/main/java/com/o19s/es/ltr/rest/RestSearchStoreElements.java +++ b/src/main/java/com/o19s/es/ltr/rest/RestSearchStoreElements.java @@ -1,3 +1,18 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ package com.o19s.es.ltr.rest; import com.o19s.es.ltr.feature.store.index.IndexFeatureStore; @@ -48,7 +63,6 @@ RestChannelConsumer search(NodeClient client, String type, String indexName, Res qb.must(matchQuery("name.prefix", prefix)); } return (channel) -> client.prepareSearch(indexName) - .setTypes(IndexFeatureStore.ES_TYPE) .setQuery(qb) .setSize(size) .setFrom(from) diff --git a/src/main/java/com/o19s/es/ltr/rest/RestStoreManager.java b/src/main/java/com/o19s/es/ltr/rest/RestStoreManager.java index ba22b795..d4be858f 100644 --- a/src/main/java/com/o19s/es/ltr/rest/RestStoreManager.java +++ b/src/main/java/com/o19s/es/ltr/rest/RestStoreManager.java @@ -1,3 +1,18 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ package com.o19s.es.ltr.rest; import com.o19s.es.ltr.action.ListStoresAction; @@ -5,7 +20,7 @@ import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; import org.opensearch.action.admin.indices.exists.indices.IndicesExistsResponse; import org.opensearch.client.node.NodeClient; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; diff --git a/src/main/java/com/o19s/es/ltr/stats/LTRStat.java b/src/main/java/com/o19s/es/ltr/stats/LTRStat.java index 43143252..8cc7bc2e 100644 --- a/src/main/java/com/o19s/es/ltr/stats/LTRStat.java +++ b/src/main/java/com/o19s/es/ltr/stats/LTRStat.java @@ -1,3 +1,18 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ package com.o19s.es.ltr.stats; import java.util.function.Supplier; diff --git a/src/main/java/com/o19s/es/ltr/stats/LTRStats.java b/src/main/java/com/o19s/es/ltr/stats/LTRStats.java index 0181f5a8..7fc82738 100644 --- a/src/main/java/com/o19s/es/ltr/stats/LTRStats.java +++ b/src/main/java/com/o19s/es/ltr/stats/LTRStats.java @@ -1,3 +1,18 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ package com.o19s.es.ltr.stats; import java.util.Map; diff --git a/src/main/java/com/o19s/es/ltr/stats/StatName.java b/src/main/java/com/o19s/es/ltr/stats/StatName.java index ba1c68d4..1a45473a 100644 --- a/src/main/java/com/o19s/es/ltr/stats/StatName.java +++ b/src/main/java/com/o19s/es/ltr/stats/StatName.java @@ -1,3 +1,18 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ package com.o19s.es.ltr.stats; import java.util.Collections; diff --git a/src/main/java/com/o19s/es/ltr/stats/suppliers/CacheStatsOnNodeSupplier.java b/src/main/java/com/o19s/es/ltr/stats/suppliers/CacheStatsOnNodeSupplier.java index c08fd7bc..85a5bb61 100644 --- a/src/main/java/com/o19s/es/ltr/stats/suppliers/CacheStatsOnNodeSupplier.java +++ b/src/main/java/com/o19s/es/ltr/stats/suppliers/CacheStatsOnNodeSupplier.java @@ -1,3 +1,18 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ package com.o19s.es.ltr.stats.suppliers; import com.o19s.es.ltr.feature.store.index.Caches; diff --git a/src/main/java/com/o19s/es/ltr/stats/suppliers/PluginHealthStatusSupplier.java b/src/main/java/com/o19s/es/ltr/stats/suppliers/PluginHealthStatusSupplier.java index 73b31d76..ebf588ab 100644 --- a/src/main/java/com/o19s/es/ltr/stats/suppliers/PluginHealthStatusSupplier.java +++ b/src/main/java/com/o19s/es/ltr/stats/suppliers/PluginHealthStatusSupplier.java @@ -1,3 +1,18 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ package com.o19s.es.ltr.stats.suppliers; import com.o19s.es.ltr.feature.store.index.IndexFeatureStore; diff --git a/src/main/java/com/o19s/es/ltr/stats/suppliers/StoreStatsSupplier.java b/src/main/java/com/o19s/es/ltr/stats/suppliers/StoreStatsSupplier.java index ab4c7e33..730bfe06 100644 --- a/src/main/java/com/o19s/es/ltr/stats/suppliers/StoreStatsSupplier.java +++ b/src/main/java/com/o19s/es/ltr/stats/suppliers/StoreStatsSupplier.java @@ -1,3 +1,18 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ package com.o19s.es.ltr.stats.suppliers; import com.o19s.es.ltr.feature.store.index.IndexFeatureStore; diff --git a/src/main/java/com/o19s/es/ltr/utils/AbstractQueryBuilderUtils.java b/src/main/java/com/o19s/es/ltr/utils/AbstractQueryBuilderUtils.java index 60f05326..cfd21f88 100644 --- a/src/main/java/com/o19s/es/ltr/utils/AbstractQueryBuilderUtils.java +++ b/src/main/java/com/o19s/es/ltr/utils/AbstractQueryBuilderUtils.java @@ -1,3 +1,18 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ package com.o19s.es.ltr.utils; import org.opensearch.common.io.stream.StreamInput; diff --git a/src/main/java/com/o19s/es/template/mustache/CustomMustacheFactory.java b/src/main/java/com/o19s/es/template/mustache/CustomMustacheFactory.java index 01340ce0..f00c0344 100644 --- a/src/main/java/com/o19s/es/template/mustache/CustomMustacheFactory.java +++ b/src/main/java/com/o19s/es/template/mustache/CustomMustacheFactory.java @@ -28,7 +28,7 @@ import com.github.mustachejava.codes.IterableCode; import com.github.mustachejava.codes.WriteCode; import org.opensearch.common.Strings; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentType; import java.io.IOException; diff --git a/src/main/java/com/o19s/es/termstat/TermStatQuery.java b/src/main/java/com/o19s/es/termstat/TermStatQuery.java index 926826dd..c124f19d 100644 --- a/src/main/java/com/o19s/es/termstat/TermStatQuery.java +++ b/src/main/java/com/o19s/es/termstat/TermStatQuery.java @@ -1,3 +1,19 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package com.o19s.es.termstat; import com.o19s.es.explore.StatisticsHelper; @@ -6,14 +22,19 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.Term; -import org.apache.lucene.search.Explanation; -import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.index.TermStates; +import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.ScoreMode; -import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Weight; +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.Scorer; import java.io.IOException; +import java.util.HashMap; +import java.util.Map; import java.util.Objects; import java.util.Set; @@ -81,8 +102,14 @@ static class TermStatWeight extends Weight { private final AggrType aggr; private final AggrType posAggr; private final Set terms; - - TermStatWeight(IndexSearcher searcher, TermStatQuery tsq, Set terms, ScoreMode scoreMode, AggrType aggr, AggrType posAggr) { + private final Map termContexts; + + TermStatWeight(IndexSearcher searcher, + TermStatQuery tsq, + Set terms, + ScoreMode scoreMode, + AggrType aggr, + AggrType posAggr) throws IOException { super(tsq); this.searcher = searcher; this.expression = tsq.expr; @@ -90,9 +117,23 @@ static class TermStatWeight extends Weight { this.scoreMode = scoreMode; this.aggr = aggr; this.posAggr = posAggr; + this.termContexts = new HashMap<>(); + + // This is needed for proper DFS_QUERY_THEN_FETCH support + if (scoreMode.needsScores()) { + for (Term t : terms) { + TermStates ctx = TermStates.build(searcher.getTopReaderContext(), t, true); + + if (ctx != null && ctx.docFreq() > 0) { + searcher.collectionStatistics(t.field()); + searcher.termStatistics(t, ctx.docFreq(), ctx.totalTermFreq()); + } + + termContexts.put(t, ctx); + } + } } - @Override public void extractTerms(Set terms) { terms.addAll(terms); } @@ -110,7 +151,7 @@ public Explanation explain(LeafReaderContext context, int doc) throws IOExceptio @Override public Scorer scorer(LeafReaderContext context) throws IOException { - return new TermStatScorer(this, searcher, context, expression, terms, scoreMode, aggr, posAggr); + return new TermStatScorer(this, searcher, context, expression, terms, scoreMode, aggr, posAggr, termContexts); } @Override @@ -118,4 +159,16 @@ public boolean isCacheable(LeafReaderContext ctx) { return true; } } + + @Override + public void visit(QueryVisitor visitor) { + Term[] acceptedTerms = terms.stream().filter( + t -> visitor.acceptField(t.field()) + ).toArray(Term[]::new); + + if (acceptedTerms.length > 0) { + QueryVisitor v = visitor.getSubVisitor(BooleanClause.Occur.SHOULD, this); + v.consumeTerms(this, acceptedTerms); + } + } } diff --git a/src/main/java/com/o19s/es/termstat/TermStatQueryBuilder.java b/src/main/java/com/o19s/es/termstat/TermStatQueryBuilder.java index e8933be3..74dec28a 100644 --- a/src/main/java/com/o19s/es/termstat/TermStatQueryBuilder.java +++ b/src/main/java/com/o19s/es/termstat/TermStatQueryBuilder.java @@ -1,3 +1,18 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ package com.o19s.es.termstat; import com.o19s.es.explore.StatisticsHelper.AggrType; @@ -9,14 +24,14 @@ import org.apache.lucene.expressions.Expression; import org.apache.lucene.index.Term; import org.apache.lucene.search.Query; -import org.opensearch.common.ParseField; +import org.opensearch.core.ParseField; import org.opensearch.common.ParsingException; import org.opensearch.common.io.stream.NamedWriteable; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.xcontent.ObjectParser; -import org.opensearch.common.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentParser; +import org.opensearch.core.xcontent.ObjectParser; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.query.AbstractQueryBuilder; import org.opensearch.index.query.QueryShardContext; diff --git a/src/main/java/com/o19s/es/termstat/TermStatScorer.java b/src/main/java/com/o19s/es/termstat/TermStatScorer.java index 3669e0b5..8382ba71 100644 --- a/src/main/java/com/o19s/es/termstat/TermStatScorer.java +++ b/src/main/java/com/o19s/es/termstat/TermStatScorer.java @@ -1,3 +1,18 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ package com.o19s.es.termstat; import com.o19s.es.explore.StatisticsHelper; @@ -7,6 +22,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermStates; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.DoubleValues; import org.apache.lucene.search.DoubleValuesSource; @@ -16,6 +32,7 @@ import java.io.IOException; import java.util.HashMap; +import java.util.Map; import java.util.Set; public class TermStatScorer extends Scorer { @@ -29,6 +46,7 @@ public class TermStatScorer extends Scorer { private final IndexSearcher searcher; private final Set terms; private final ScoreMode scoreMode; + private final Map termContexts; public TermStatScorer(TermStatQuery.TermStatWeight weight, IndexSearcher searcher, @@ -37,7 +55,8 @@ public TermStatScorer(TermStatQuery.TermStatWeight weight, Set terms, ScoreMode scoreMode, AggrType aggr, - AggrType posAggr) { + AggrType posAggr, + Map termContexts) { super(weight); this.context = context; this.compiledExpression = compiledExpression; @@ -46,6 +65,7 @@ public TermStatScorer(TermStatQuery.TermStatWeight weight, this.scoreMode = scoreMode; this.aggr = aggr; this.posAggr = posAggr; + this.termContexts = termContexts; this.iter = DocIdSetIterator.all(context.reader().maxDoc()); } @@ -65,7 +85,7 @@ public float score() throws IOException { // Refresh the term stats tsq.setPosAggr(posAggr); - tsq.bump(searcher, context, docID(), terms, scoreMode); + tsq.bump(searcher, context, docID(), terms, scoreMode, termContexts); // Prepare computed statistics StatisticsHelper computed = new StatisticsHelper(); diff --git a/src/main/java/com/o19s/es/termstat/TermStatSupplier.java b/src/main/java/com/o19s/es/termstat/TermStatSupplier.java index 2f1c704e..6167b27f 100644 --- a/src/main/java/com/o19s/es/termstat/TermStatSupplier.java +++ b/src/main/java/com/o19s/es/termstat/TermStatSupplier.java @@ -1,3 +1,18 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ package com.o19s.es.termstat; import com.o19s.es.explore.StatisticsHelper; @@ -13,6 +28,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.TermStatistics; import org.apache.lucene.search.similarities.ClassicSimilarity; import java.io.IOException; @@ -22,6 +38,7 @@ import java.util.Arrays; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Set; @@ -47,7 +64,7 @@ public TermStatSupplier() { public void bump (IndexSearcher searcher, LeafReaderContext context, int docID, Set terms, - ScoreMode scoreMode) throws IOException { + ScoreMode scoreMode, Map termContexts) throws IOException { df_stats.getData().clear(); idf_stats.getData().clear(); tf_stats.getData().clear(); @@ -61,22 +78,24 @@ public void bump (IndexSearcher searcher, LeafReaderContext context, break; } - TermStates termStates = TermStates.build(searcher.getTopReaderContext(), term, scoreMode.needsScores()); + TermStates termStates = termContexts.get(term); assert termStates != null && termStates .wasBuiltFor(ReaderUtil.getTopLevelContext(context)); TermState state = termStates.get(context); - if (state == null) { + if (state == null || termStates.docFreq() == 0) { insertZeroes(); // Zero out stats for terms we don't know about in the index continue; } + TermStatistics indexStats = searcher.termStatistics(term, termStates.docFreq(), termStates.totalTermFreq()); + // Collection Statistics - df_stats.add(termStates.docFreq()); - idf_stats.add(sim.idf(termStates.docFreq(), searcher.getIndexReader().numDocs())); - ttf_stats.add(termStates.totalTermFreq()); + df_stats.add(indexStats.docFreq()); + idf_stats.add(sim.idf(indexStats.docFreq(), searcher.collectionStatistics(term.field()).docCount())); + ttf_stats.add(indexStats.totalTermFreq()); // Doc specifics TermsEnum termsEnum = context.reader().terms(term.field()).iterator(); diff --git a/src/main/plugin-metadata/plugin-security.policy b/src/main/plugin-metadata/plugin-security.policy index 4835ab05..979532ef 100644 --- a/src/main/plugin-metadata/plugin-security.policy +++ b/src/main/plugin-metadata/plugin-security.policy @@ -22,11 +22,12 @@ grant { permission java.lang.RuntimePermission "createClassLoader"; // expression runtime - permission org.elasticsearch.script.ClassPermission "java.lang.String"; - permission org.elasticsearch.script.ClassPermission "org.apache.lucene.expressions.Expression"; - permission org.elasticsearch.script.ClassPermission "org.apache.lucene.search.DoubleValues"; + permission org.opensearch.script.ClassPermission "java.lang.String"; + permission org.opensearch.script.ClassPermission "org.apache.lucene.expressions.Expression"; + permission org.opensearch.script.ClassPermission "org.apache.lucene.expressions.js.ExpressionMath"; + permission org.opensearch.script.ClassPermission "org.apache.lucene.search.DoubleValues"; // available functions - permission org.elasticsearch.script.ClassPermission "java.lang.Math"; - permission org.elasticsearch.script.ClassPermission "org.apache.lucene.util.MathUtil"; - permission org.elasticsearch.script.ClassPermission "org.apache.lucene.util.SloppyMath"; + permission org.opensearch.script.ClassPermission "java.lang.Math"; + permission org.opensearch.script.ClassPermission "org.apache.lucene.util.MathUtil"; + permission org.opensearch.script.ClassPermission "org.apache.lucene.util.SloppyMath"; }; diff --git a/src/test/java/com/o19s/es/explore/ExplorerQueryTests.java b/src/test/java/com/o19s/es/explore/ExplorerQueryTests.java index 2ebaa990..ede0ab7b 100644 --- a/src/test/java/com/o19s/es/explore/ExplorerQueryTests.java +++ b/src/test/java/com/o19s/es/explore/ExplorerQueryTests.java @@ -32,7 +32,7 @@ import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.ByteBuffersDirectory; import org.apache.lucene.store.Directory; -import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCase; import org.opensearch.common.lucene.Lucene; import org.junit.After; import org.junit.Before; diff --git a/src/test/java/com/o19s/es/explore/StatisticsHelperTests.java b/src/test/java/com/o19s/es/explore/StatisticsHelperTests.java index febb0aa9..bd79eb0b 100644 --- a/src/test/java/com/o19s/es/explore/StatisticsHelperTests.java +++ b/src/test/java/com/o19s/es/explore/StatisticsHelperTests.java @@ -15,7 +15,7 @@ */ package com.o19s.es.explore; -import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCase; public class StatisticsHelperTests extends LuceneTestCase { private final float[] dataset = new float[] { diff --git a/src/test/java/com/o19s/es/ltr/LearningToRankPluginIT.java b/src/test/java/com/o19s/es/ltr/LearningToRankPluginIT.java new file mode 100644 index 00000000..23ccc526 --- /dev/null +++ b/src/test/java/com/o19s/es/ltr/LearningToRankPluginIT.java @@ -0,0 +1,26 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package com.o19s.es.ltr; + +import org.apache.http.ParseException; +import org.apache.http.util.EntityUtils; +import org.opensearch.test.rest.OpenSearchRestTestCase; +import org.opensearch.client.Request; +import org.opensearch.client.Response; +import java.io.IOException; + +public class LearningToRankPluginIT extends OpenSearchRestTestCase { + public void testPluginInstalled() throws IOException, ParseException { + Response response = client().performRequest(new Request("GET", "/_cat/plugins")); + String body = EntityUtils.toString(response.getEntity()); + + logger.info("response body: {}", body); + assertNotNull(body); + assertTrue(body.contains("ltr")); + } +} diff --git a/src/test/java/com/o19s/es/ltr/LtrQueryContextTests.java b/src/test/java/com/o19s/es/ltr/LtrQueryContextTests.java index 848055bf..7cdf8652 100644 --- a/src/test/java/com/o19s/es/ltr/LtrQueryContextTests.java +++ b/src/test/java/com/o19s/es/ltr/LtrQueryContextTests.java @@ -1,6 +1,6 @@ package com.o19s.es.ltr; -import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCase; import java.util.Arrays; import java.util.Collections; diff --git a/src/test/java/com/o19s/es/ltr/LtrTestUtils.java b/src/test/java/com/o19s/es/ltr/LtrTestUtils.java index 0be01199..766f2942 100644 --- a/src/test/java/com/o19s/es/ltr/LtrTestUtils.java +++ b/src/test/java/com/o19s/es/ltr/LtrTestUtils.java @@ -36,10 +36,10 @@ import com.o19s.es.ltr.ranker.normalizer.Normalizer; import com.o19s.es.ltr.ranker.parser.LinearRankerParser; import com.o19s.es.ltr.utils.FeatureStoreLoader; -import org.apache.lucene.util.TestUtil; +import org.apache.lucene.tests.util.TestUtil; import org.opensearch.common.CheckedFunction; import org.opensearch.common.Strings; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.index.query.WrapperQueryBuilder; @@ -50,7 +50,7 @@ import java.util.function.Function; import java.util.function.IntFunction; -import static org.apache.lucene.util.LuceneTestCase.random; +import static org.apache.lucene.tests.util.LuceneTestCase.random; public class LtrTestUtils { diff --git a/src/test/java/com/o19s/es/ltr/action/TransportLTRStatsActionTests.java b/src/test/java/com/o19s/es/ltr/action/TransportLTRStatsActionTests.java index b45c1250..ce8b98e8 100644 --- a/src/test/java/com/o19s/es/ltr/action/TransportLTRStatsActionTests.java +++ b/src/test/java/com/o19s/es/ltr/action/TransportLTRStatsActionTests.java @@ -1,3 +1,19 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package com.o19s.es.ltr.action; import com.o19s.es.ltr.action.LTRStatsAction.LTRStatsNodeRequest; diff --git a/src/test/java/com/o19s/es/ltr/feature/store/ExtraLoggingSupplierTests.java b/src/test/java/com/o19s/es/ltr/feature/store/ExtraLoggingSupplierTests.java index 7e44025e..245ed8bf 100644 --- a/src/test/java/com/o19s/es/ltr/feature/store/ExtraLoggingSupplierTests.java +++ b/src/test/java/com/o19s/es/ltr/feature/store/ExtraLoggingSupplierTests.java @@ -1,7 +1,23 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package com.o19s.es.ltr.feature.store; import com.o19s.es.ltr.ranker.LogLtrRanker; -import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCase; import java.util.HashMap; import java.util.Map; diff --git a/src/test/java/com/o19s/es/ltr/feature/store/FeatureSupplierTests.java b/src/test/java/com/o19s/es/ltr/feature/store/FeatureSupplierTests.java index e9ceb0ef..0c54be7d 100644 --- a/src/test/java/com/o19s/es/ltr/feature/store/FeatureSupplierTests.java +++ b/src/test/java/com/o19s/es/ltr/feature/store/FeatureSupplierTests.java @@ -1,10 +1,26 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package com.o19s.es.ltr.feature.store; import com.o19s.es.ltr.feature.FeatureSet; import com.o19s.es.ltr.ranker.DenseFeatureVector; import com.o19s.es.ltr.ranker.LtrRanker; import com.o19s.es.ltr.utils.Suppliers; -import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCase; import org.opensearch.index.query.QueryBuilders; import java.util.Collections; diff --git a/src/test/java/com/o19s/es/ltr/feature/store/StoredFeatureParserTests.java b/src/test/java/com/o19s/es/ltr/feature/store/StoredFeatureParserTests.java index 61f61e93..df74b9f9 100644 --- a/src/test/java/com/o19s/es/ltr/feature/store/StoredFeatureParserTests.java +++ b/src/test/java/com/o19s/es/ltr/feature/store/StoredFeatureParserTests.java @@ -16,12 +16,12 @@ package com.o19s.es.ltr.feature.store; -import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCase; import org.opensearch.common.ParsingException; import org.opensearch.common.Strings; import org.opensearch.common.xcontent.LoggingDeprecationHandler; -import org.opensearch.common.xcontent.ToXContent; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.query.AbstractQueryBuilder; @@ -31,7 +31,7 @@ import java.io.IOException; import java.util.Arrays; -import static org.opensearch.common.xcontent.NamedXContentRegistry.EMPTY; +import static org.opensearch.core.xcontent.NamedXContentRegistry.EMPTY; import static org.opensearch.common.xcontent.json.JsonXContent.jsonXContent; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.containsString; @@ -255,6 +255,6 @@ static StoredFeature parse(String featureString, String defaultName) throws IOEx } private String writeAsNonFormattedString(AbstractQueryBuilder builder) { - return Strings.toString(builder, false, false); + return Strings.toString(XContentType.JSON, builder); } } \ No newline at end of file diff --git a/src/test/java/com/o19s/es/ltr/feature/store/StoredFeatureSetParserTests.java b/src/test/java/com/o19s/es/ltr/feature/store/StoredFeatureSetParserTests.java index 15e3eac4..33d44c7b 100644 --- a/src/test/java/com/o19s/es/ltr/feature/store/StoredFeatureSetParserTests.java +++ b/src/test/java/com/o19s/es/ltr/feature/store/StoredFeatureSetParserTests.java @@ -18,12 +18,12 @@ import com.o19s.es.ltr.feature.FeatureSet; import com.o19s.es.ltr.query.DerivedExpressionQuery; -import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCase; import org.opensearch.common.ParsingException; import org.opensearch.common.Strings; import org.opensearch.common.xcontent.LoggingDeprecationHandler; -import org.opensearch.common.xcontent.ToXContent; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.query.MatchQueryBuilder; @@ -37,9 +37,9 @@ import java.util.Set; import java.util.function.Consumer; -import static org.apache.lucene.util.TestUtil.randomRealisticUnicodeString; -import static org.apache.lucene.util.TestUtil.randomSimpleString; -import static org.opensearch.common.xcontent.NamedXContentRegistry.EMPTY; +import static org.apache.lucene.tests.util.TestUtil.randomRealisticUnicodeString; +import static org.apache.lucene.tests.util.TestUtil.randomSimpleString; +import static org.opensearch.core.xcontent.NamedXContentRegistry.EMPTY; import static org.opensearch.common.xcontent.json.JsonXContent.jsonXContent; import static org.hamcrest.CoreMatchers.allOf; import static org.hamcrest.CoreMatchers.containsString; diff --git a/src/test/java/com/o19s/es/ltr/feature/store/StoredFeatureSetTests.java b/src/test/java/com/o19s/es/ltr/feature/store/StoredFeatureSetTests.java index c5508f1c..3436a0fd 100644 --- a/src/test/java/com/o19s/es/ltr/feature/store/StoredFeatureSetTests.java +++ b/src/test/java/com/o19s/es/ltr/feature/store/StoredFeatureSetTests.java @@ -16,7 +16,7 @@ package com.o19s.es.ltr.feature.store; -import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCase; import java.io.IOException; import java.util.List; diff --git a/src/test/java/com/o19s/es/ltr/feature/store/StoredLtrModelParserTests.java b/src/test/java/com/o19s/es/ltr/feature/store/StoredLtrModelParserTests.java index 062eb0e7..227b6a52 100644 --- a/src/test/java/com/o19s/es/ltr/feature/store/StoredLtrModelParserTests.java +++ b/src/test/java/com/o19s/es/ltr/feature/store/StoredLtrModelParserTests.java @@ -23,7 +23,7 @@ import com.o19s.es.ltr.ranker.normalizer.StandardFeatureNormalizer; import com.o19s.es.ltr.ranker.parser.LtrRankerParserFactory; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCase; import org.opensearch.Version; import org.opensearch.common.ParsingException; import org.opensearch.common.Randomness; @@ -32,16 +32,16 @@ import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.xcontent.LoggingDeprecationHandler; -import org.opensearch.common.xcontent.ToXContent; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.common.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import java.io.IOException; import java.util.Base64; -import static org.opensearch.common.xcontent.NamedXContentRegistry.EMPTY; +import static org.opensearch.core.xcontent.NamedXContentRegistry.EMPTY; import static org.opensearch.common.xcontent.json.JsonXContent.jsonXContent; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.equalTo; diff --git a/src/test/java/com/o19s/es/ltr/feature/store/index/CachedFeatureStoreTests.java b/src/test/java/com/o19s/es/ltr/feature/store/index/CachedFeatureStoreTests.java index e1773cfb..1b0b417e 100644 --- a/src/test/java/com/o19s/es/ltr/feature/store/index/CachedFeatureStoreTests.java +++ b/src/test/java/com/o19s/es/ltr/feature/store/index/CachedFeatureStoreTests.java @@ -21,8 +21,8 @@ import com.o19s.es.ltr.feature.store.MemStore; import com.o19s.es.ltr.feature.store.StoredFeature; import com.o19s.es.ltr.feature.store.StoredFeatureSet; -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.TestUtil; +import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.tests.util.TestUtil; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; diff --git a/src/test/java/com/o19s/es/ltr/feature/store/index/IndexFeatureStoreTests.java b/src/test/java/com/o19s/es/ltr/feature/store/index/IndexFeatureStoreTests.java index b192fbf9..ecc49a5e 100644 --- a/src/test/java/com/o19s/es/ltr/feature/store/index/IndexFeatureStoreTests.java +++ b/src/test/java/com/o19s/es/ltr/feature/store/index/IndexFeatureStoreTests.java @@ -23,15 +23,15 @@ import com.o19s.es.ltr.feature.store.StoredFeatureSet; import com.o19s.es.ltr.feature.store.StoredLtrModel; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.TestUtil; +import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.tests.util.TestUtil; import org.opensearch.client.Requests; import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.xcontent.LoggingDeprecationHandler; -import org.opensearch.common.xcontent.NamedXContentRegistry; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.common.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParser; import java.io.IOException; import java.util.HashMap; @@ -41,8 +41,8 @@ import static com.o19s.es.ltr.feature.store.index.IndexFeatureStore.indexName; import static com.o19s.es.ltr.feature.store.index.IndexFeatureStore.isIndexStore; import static com.o19s.es.ltr.feature.store.index.IndexFeatureStore.storeName; -import static org.apache.lucene.util.TestUtil.randomRealisticUnicodeString; -import static org.apache.lucene.util.TestUtil.randomSimpleString; +import static org.apache.lucene.tests.util.TestUtil.randomRealisticUnicodeString; +import static org.apache.lucene.tests.util.TestUtil.randomSimpleString; import static org.hamcrest.CoreMatchers.equalTo; public class IndexFeatureStoreTests extends LuceneTestCase { diff --git a/src/test/java/com/o19s/es/ltr/logging/LoggingFetchSubPhaseTests.java b/src/test/java/com/o19s/es/ltr/logging/LoggingFetchSubPhaseTests.java index 8aa3f5e0..1f35207d 100644 --- a/src/test/java/com/o19s/es/ltr/logging/LoggingFetchSubPhaseTests.java +++ b/src/test/java/com/o19s/es/ltr/logging/LoggingFetchSubPhaseTests.java @@ -42,8 +42,8 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.Weight; import org.apache.lucene.store.Directory; -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.TestUtil; +import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.tests.util.TestUtil; import org.opensearch.common.lucene.search.function.CombineFunction; import org.opensearch.common.lucene.search.function.FieldValueFactorFunction; import org.opensearch.common.lucene.search.function.FunctionScoreQuery; @@ -182,7 +182,6 @@ public void collect(int doc) throws IOException { SearchHit hit = new SearchHit( doc, id, - new Text("text"), random().nextBoolean() ? new HashMap<>() : null, null ); diff --git a/src/test/java/com/o19s/es/ltr/logging/LoggingSearchExtBuilderTests.java b/src/test/java/com/o19s/es/ltr/logging/LoggingSearchExtBuilderTests.java index 6fe56a32..8e11dc9e 100644 --- a/src/test/java/com/o19s/es/ltr/logging/LoggingSearchExtBuilderTests.java +++ b/src/test/java/com/o19s/es/ltr/logging/LoggingSearchExtBuilderTests.java @@ -19,10 +19,10 @@ import org.opensearch.common.ParsingException; import org.opensearch.common.Strings; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.xcontent.ToXContent; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.common.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.test.OpenSearchTestCase; diff --git a/src/test/java/com/o19s/es/ltr/query/LtrQueryTests.java b/src/test/java/com/o19s/es/ltr/query/LtrQueryTests.java index 5e67a584..f970db45 100644 --- a/src/test/java/com/o19s/es/ltr/query/LtrQueryTests.java +++ b/src/test/java/com/o19s/es/ltr/query/LtrQueryTests.java @@ -42,7 +42,7 @@ import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.tests.index.RandomIndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.misc.SweetSpotSimilarity; import org.apache.lucene.queries.BlendedTermQuery; @@ -76,7 +76,7 @@ import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.search.similarities.TFIDFSimilarity; import org.apache.lucene.store.Directory; -import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCase; import org.opensearch.common.lucene.search.function.FunctionScoreQuery; import org.opensearch.common.lucene.search.function.WeightFactorFunction; import org.junit.After; diff --git a/src/test/java/com/o19s/es/ltr/ranker/DenseLtrRankerTests.java b/src/test/java/com/o19s/es/ltr/ranker/DenseLtrRankerTests.java index aa7e9f20..a3ba1d3e 100644 --- a/src/test/java/com/o19s/es/ltr/ranker/DenseLtrRankerTests.java +++ b/src/test/java/com/o19s/es/ltr/ranker/DenseLtrRankerTests.java @@ -16,7 +16,7 @@ package com.o19s.es.ltr.ranker; -import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCase; public class DenseLtrRankerTests extends LuceneTestCase { public void newFeatureVector() throws Exception { diff --git a/src/test/java/com/o19s/es/ltr/ranker/LogLtrRankerTests.java b/src/test/java/com/o19s/es/ltr/ranker/LogLtrRankerTests.java index 358b8eed..9e003f32 100644 --- a/src/test/java/com/o19s/es/ltr/ranker/LogLtrRankerTests.java +++ b/src/test/java/com/o19s/es/ltr/ranker/LogLtrRankerTests.java @@ -17,8 +17,8 @@ package com.o19s.es.ltr.ranker; import com.o19s.es.ltr.ranker.linear.LinearRankerTests; -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.TestUtil; +import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.tests.util.TestUtil; public class LogLtrRankerTests extends LuceneTestCase { public void testNewFeatureVector() throws Exception { diff --git a/src/test/java/com/o19s/es/ltr/ranker/dectree/NaiveAdditiveDecisionTreeTests.java b/src/test/java/com/o19s/es/ltr/ranker/dectree/NaiveAdditiveDecisionTreeTests.java index 6abb1f25..9606a6e2 100644 --- a/src/test/java/com/o19s/es/ltr/ranker/dectree/NaiveAdditiveDecisionTreeTests.java +++ b/src/test/java/com/o19s/es/ltr/ranker/dectree/NaiveAdditiveDecisionTreeTests.java @@ -26,8 +26,8 @@ import com.o19s.es.ltr.ranker.normalizer.Normalizers; import org.apache.logging.log4j.Logger; import org.apache.lucene.search.MatchAllDocsQuery; -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.TestUtil; +import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.tests.util.TestUtil; import org.apache.logging.log4j.LogManager; import java.io.BufferedReader; @@ -47,7 +47,7 @@ import static org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_ARRAY_HEADER; import static org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_OBJECT_HEADER; import static org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_OBJECT_REF; -import static org.apache.lucene.util.TestUtil.nextInt; +import static org.apache.lucene.tests.util.TestUtil.nextInt; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.lessThan; import static org.hamcrest.core.AllOf.allOf; diff --git a/src/test/java/com/o19s/es/ltr/ranker/linear/LinearRankerTests.java b/src/test/java/com/o19s/es/ltr/ranker/linear/LinearRankerTests.java index a02c25ee..ce0d1c17 100644 --- a/src/test/java/com/o19s/es/ltr/ranker/linear/LinearRankerTests.java +++ b/src/test/java/com/o19s/es/ltr/ranker/linear/LinearRankerTests.java @@ -21,12 +21,12 @@ import com.o19s.es.ltr.ranker.dectree.NaiveAdditiveDecisionTreeTests; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.TestUtil; +import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.tests.util.TestUtil; import static org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_ARRAY_HEADER; import static org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_OBJECT_HEADER; -import static org.apache.lucene.util.TestUtil.nextInt; +import static org.apache.lucene.tests.util.TestUtil.nextInt; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.lessThan; import static org.hamcrest.core.AllOf.allOf; diff --git a/src/test/java/com/o19s/es/ltr/ranker/normalizer/NormalizersTests.java b/src/test/java/com/o19s/es/ltr/ranker/normalizer/NormalizersTests.java index 602b5c8e..b360ec9b 100644 --- a/src/test/java/com/o19s/es/ltr/ranker/normalizer/NormalizersTests.java +++ b/src/test/java/com/o19s/es/ltr/ranker/normalizer/NormalizersTests.java @@ -1,6 +1,22 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package com.o19s.es.ltr.ranker.normalizer; -import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCase; import org.hamcrest.CoreMatchers; public class NormalizersTests extends LuceneTestCase { diff --git a/src/test/java/com/o19s/es/ltr/ranker/parser/LinearRankerParserTests.java b/src/test/java/com/o19s/es/ltr/ranker/parser/LinearRankerParserTests.java index 5d2ad359..623b510a 100644 --- a/src/test/java/com/o19s/es/ltr/ranker/parser/LinearRankerParserTests.java +++ b/src/test/java/com/o19s/es/ltr/ranker/parser/LinearRankerParserTests.java @@ -22,10 +22,10 @@ import com.o19s.es.ltr.ranker.DenseFeatureVector; import com.o19s.es.ltr.ranker.linear.LinearRanker; import com.o19s.es.ltr.ranker.linear.LinearRankerTests; -import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCase; import org.opensearch.common.ParsingException; import org.opensearch.common.Strings; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.json.JsonXContent; import org.junit.Assert; diff --git a/src/test/java/com/o19s/es/ltr/ranker/parser/LtrRankerParserFactoryTests.java b/src/test/java/com/o19s/es/ltr/ranker/parser/LtrRankerParserFactoryTests.java index 08ffa9c5..0d329746 100644 --- a/src/test/java/com/o19s/es/ltr/ranker/parser/LtrRankerParserFactoryTests.java +++ b/src/test/java/com/o19s/es/ltr/ranker/parser/LtrRankerParserFactoryTests.java @@ -16,7 +16,7 @@ package com.o19s.es.ltr.ranker.parser; -import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCase; import static org.hamcrest.CoreMatchers.containsString; diff --git a/src/test/java/com/o19s/es/ltr/ranker/parser/XGBoostJsonParserTests.java b/src/test/java/com/o19s/es/ltr/ranker/parser/XGBoostJsonParserTests.java index ec73f0c1..39ae446e 100644 --- a/src/test/java/com/o19s/es/ltr/ranker/parser/XGBoostJsonParserTests.java +++ b/src/test/java/com/o19s/es/ltr/ranker/parser/XGBoostJsonParserTests.java @@ -24,7 +24,7 @@ import com.o19s.es.ltr.ranker.LtrRanker.FeatureVector; import com.o19s.es.ltr.ranker.dectree.NaiveAdditiveDecisionTree; import com.o19s.es.ltr.ranker.linear.LinearRankerTests; -import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCase; import org.opensearch.common.ParsingException; import org.opensearch.core.internal.io.Streams; import org.hamcrest.CoreMatchers; diff --git a/src/test/java/com/o19s/es/ltr/rest/FeaturesParserTests.java b/src/test/java/com/o19s/es/ltr/rest/FeaturesParserTests.java index 36679527..0f36299c 100644 --- a/src/test/java/com/o19s/es/ltr/rest/FeaturesParserTests.java +++ b/src/test/java/com/o19s/es/ltr/rest/FeaturesParserTests.java @@ -16,10 +16,11 @@ package com.o19s.es.ltr.rest; -import org.apache.lucene.util.LuceneTestCase; +import com.o19s.es.ltr.feature.store.StoredFeatureParserTests; +import org.apache.lucene.tests.util.LuceneTestCase; import org.opensearch.common.xcontent.LoggingDeprecationHandler; -import org.opensearch.common.xcontent.NamedXContentRegistry; -import org.opensearch.common.xcontent.XContentParser; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentParser; import java.io.IOException; import java.util.stream.Collectors; @@ -33,7 +34,7 @@ public void testParseArray() throws IOException { RestAddFeatureToSet.FeaturesParserState fparser = new RestAddFeatureToSet.FeaturesParserState(); int nFeat = random().nextInt(18)+1; String featuresArray = IntStream.range(0, nFeat) - .mapToObj((i) -> generateTestFeature("feat" + i)) + .mapToObj((i) -> StoredFeatureParserTests.generateTestFeature("feat" + i)) .collect(Collectors.joining(",")); XContentParser parser = jsonXContent.createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, "{\"features\":[" + featuresArray + "]}"); diff --git a/src/test/java/com/o19s/es/ltr/stats/LTRStatTests.java b/src/test/java/com/o19s/es/ltr/stats/LTRStatTests.java index a25f5650..c0a7cb83 100644 --- a/src/test/java/com/o19s/es/ltr/stats/LTRStatTests.java +++ b/src/test/java/com/o19s/es/ltr/stats/LTRStatTests.java @@ -1,3 +1,18 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ package com.o19s.es.ltr.stats; import org.opensearch.test.OpenSearchTestCase; diff --git a/src/test/java/com/o19s/es/ltr/stats/LTRStatsTests.java b/src/test/java/com/o19s/es/ltr/stats/LTRStatsTests.java index 72c82285..032ae42d 100644 --- a/src/test/java/com/o19s/es/ltr/stats/LTRStatsTests.java +++ b/src/test/java/com/o19s/es/ltr/stats/LTRStatsTests.java @@ -1,3 +1,18 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ package com.o19s.es.ltr.stats; import org.opensearch.test.OpenSearchTestCase; diff --git a/src/test/java/com/o19s/es/ltr/stats/suppliers/CacheStatsOnNodeSupplierTests.java b/src/test/java/com/o19s/es/ltr/stats/suppliers/CacheStatsOnNodeSupplierTests.java index 88238c1f..faea21f6 100644 --- a/src/test/java/com/o19s/es/ltr/stats/suppliers/CacheStatsOnNodeSupplierTests.java +++ b/src/test/java/com/o19s/es/ltr/stats/suppliers/CacheStatsOnNodeSupplierTests.java @@ -1,3 +1,18 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ package com.o19s.es.ltr.stats.suppliers; import com.o19s.es.ltr.LtrTestUtils; diff --git a/src/test/java/com/o19s/es/ltr/stats/suppliers/PluginHealthStatusSupplierTests.java b/src/test/java/com/o19s/es/ltr/stats/suppliers/PluginHealthStatusSupplierTests.java index 9ce590c2..36f14730 100644 --- a/src/test/java/com/o19s/es/ltr/stats/suppliers/PluginHealthStatusSupplierTests.java +++ b/src/test/java/com/o19s/es/ltr/stats/suppliers/PluginHealthStatusSupplierTests.java @@ -1,3 +1,18 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ package com.o19s.es.ltr.stats.suppliers; import com.o19s.es.ltr.feature.store.index.IndexFeatureStore; diff --git a/src/test/java/com/o19s/es/termstat/TermStatQueryBuilderTests.java b/src/test/java/com/o19s/es/termstat/TermStatQueryBuilderTests.java index e4cc13c9..9d2d7e61 100644 --- a/src/test/java/com/o19s/es/termstat/TermStatQueryBuilderTests.java +++ b/src/test/java/com/o19s/es/termstat/TermStatQueryBuilderTests.java @@ -1,3 +1,18 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ package com.o19s.es.termstat; import com.o19s.es.explore.StatisticsHelper.AggrType; diff --git a/src/test/java/com/o19s/es/termstat/TermStatQueryTests.java b/src/test/java/com/o19s/es/termstat/TermStatQueryTests.java index c0b38921..dd953ebc 100644 --- a/src/test/java/com/o19s/es/termstat/TermStatQueryTests.java +++ b/src/test/java/com/o19s/es/termstat/TermStatQueryTests.java @@ -1,3 +1,18 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ package com.o19s.es.termstat; import com.o19s.es.explore.StatisticsHelper.AggrType; @@ -17,7 +32,7 @@ import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.ByteBuffersDirectory; import org.apache.lucene.store.Directory; -import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCase; import org.opensearch.common.lucene.Lucene; import org.junit.After; diff --git a/src/test/resources/rest-api-spec/test/fstore/80_search_w_partial_models.yml b/src/test/resources/rest-api-spec/test/fstore/80_search_w_partial_models.yml index 4434990e..eccc948a 100644 --- a/src/test/resources/rest-api-spec/test/fstore/80_search_w_partial_models.yml +++ b/src/test/resources/rest-api-spec/test/fstore/80_search_w_partial_models.yml @@ -7,14 +7,12 @@ setup: - do: index: index: test - type: test id: 1 body: { "field1": "v1", "field2": "v2", "field3": "some text", "user_rating": 5.2 } - do: index: index: test - type: test id: 2 body: { "field1": "v1 aoeu", "field2": " ua u v2", "field3": "foo bar text", "user_rating": 0.0 } diff --git a/src/test/resources/rest-api-spec/test/fstore/90_get_stats.yml b/src/test/resources/rest-api-spec/test/fstore/90_get_stats.yml index 7bcab57e..3b8347e7 100644 --- a/src/test/resources/rest-api-spec/test/fstore/90_get_stats.yml +++ b/src/test/resources/rest-api-spec/test/fstore/90_get_stats.yml @@ -7,14 +7,12 @@ setup: - do: index: index: test - type: test id: 1 body: { "field1": "v1", "field2": "v2", "field3": "some text", "user_rating": 5.2 } - do: index: index: test - type: test id: 2 body: { "field1": "v1 aoeu", "field2": " ua u v2", "field3": "foo bar text", "user_rating": 0.0 } diff --git a/src/yamlRestTest/java/com/o19s/es/ltr/LearningToRankClientYamlTestSuiteIT.java b/src/yamlRestTest/java/com/o19s/es/ltr/LearningToRankClientYamlTestSuiteIT.java new file mode 100644 index 00000000..fcec1484 --- /dev/null +++ b/src/yamlRestTest/java/com/o19s/es/ltr/LearningToRankClientYamlTestSuiteIT.java @@ -0,0 +1,25 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package com.o19s.es.ltr; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.opensearch.test.rest.yaml.ClientYamlTestCandidate; +import org.opensearch.test.rest.yaml.OpenSearchClientYamlSuiteTestCase; + +public class LearningToRankClientYamlTestSuiteIT extends OpenSearchClientYamlSuiteTestCase { + public LearningToRankClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return OpenSearchClientYamlSuiteTestCase.createParameters(); + } + } + diff --git a/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml b/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml new file mode 100644 index 00000000..ad242c84 --- /dev/null +++ b/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml @@ -0,0 +1,17 @@ +"Test that the plugin is loaded in OpenSearch": + - do: + cat.plugins: + local: true + h: component + + - match: + $body: /^opensearch-ltr-.*\n$/ + + - do: + indices.create: + index: test + + - do: + search: + index: test + body: { }