diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 000000000000..42076be8bdb8 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,43 @@ +[*] +charset = utf-8 +end_of_line = lf +indent_size = 4 +indent_style = space +insert_final_newline = true +max_line_length = 120 +tab_width = 8 +ij_continuation_indent_size = 8 +ij_formatter_off_tag = @formatter:off +ij_formatter_on_tag = @formatter:on +ij_formatter_tags_enabled = true +ij_smart_tabs = false +ij_visual_guides = +ij_wrap_on_typing = false + + +[*.java] +ij_java_class_count_to_use_import_on_demand = 99 +ij_java_imports_layout = $*,java.**,|,javax.**,|,*,|,com.google.refine.**,org.openrefine.** +ij_java_layout_static_imports_separately = true + + +[*.less] +indent_size = 2 + +[{*.bash,*.sh,*.zsh}] +indent_size = 2 +tab_width = 2 +ij_shell_binary_ops_start_line = false +ij_shell_keep_column_alignment_padding = false +ij_shell_minify_program = false +ij_shell_redirect_followed_by_space = false +ij_shell_switch_cases_indented = false +ij_shell_use_unix_line_separator = true + + +[{*.graphqlconfig,*.graphqlrc,*.har,*.jsb2,*.jsb3,*.json,.babelrc,.eslintrc,.prettierrc,.stylelintrc,bowerrc,jest.config}] +indent_size = 2 + + +[{*.yaml,*.yml}] +indent_size = 2 diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 7ff83400eeaf..fc24c5aa44b2 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -2,7 +2,7 @@ name: Bug report about: Create a report to help us improve OpenRefine title: '' -labels: bug, to be reviewed +labels: ['Type: Bug', 'Status: Pending Review'] assignees: '' --- @@ -34,7 +34,7 @@ Steps to reproduce the behavior: ### Datasets +If you are concerned about keeping your data private, you can share it selectively by email to developers who work on the issue --> ### Additional context diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 8dbc191abba8..c543da7e89b6 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,7 +1,7 @@ blank_issues_enabled: false contact_links: - - name: Ask a question (mailing list) - url: https://groups.google.com/d/forum/openrefine + - name: Ask a question (forum) + url: https://forum.openrefine.org/ about: Please ask and answer questions here. - name: Gitter chat url: https://gitter.im/OpenRefine/OpenRefine diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index e386dc313a14..3ab9c56fe8d1 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -2,7 +2,7 @@ name: Feature request about: Suggest an idea for OpenRefine title: '' -labels: enhancement, to be reviewed +labels: ["Type: Feature Request", "Status: Pending Review"] assignees: '' --- diff --git a/.github/SUPPORT.md b/.github/SUPPORT.md index 197b6d6fc51f..4f19e268ec49 100644 --- a/.github/SUPPORT.md +++ b/.github/SUPPORT.md @@ -1,5 +1,5 @@ If you are having trouble with OpenRefine we suggest the following steps: 1. Read through our [FAQ (frequently Asked Questions)](https://github.com/OpenRefine/OpenRefine/wiki/FAQ) -2. Search for similar issues in our community email list archives: http://groups.google.com/group/openrefine/ -3. Send an email to our community mailing list: openrefine@googlegroups.com +2. Search the web for similar issues in our community +3. Ask your question on [our forum](https://forum.openrefine.org) diff --git a/.github/autolabeler.yml b/.github/autolabeler.yml deleted file mode 100644 index f4460f7861ed..000000000000 --- a/.github/autolabeler.yml +++ /dev/null @@ -1 +0,0 @@ -documentation: ["/docs"] diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 21cbc2a5f4a7..290813f82c48 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -3,21 +3,44 @@ version: 2 updates: +# For openrefine java deps - package-ecosystem: maven directory: "/" schedule: - interval: daily - open-pull-requests-limit: 10 + interval: "weekly" + open-pull-requests-limit: 15 ignore: - dependency-name: com.thoughtworks.xstream:xstream versions: - "> 1.4.12" - "< 2" -- package-ecosystem: "npm" # For Yarn - directory: "/" + - dependency-name: "org.eclipse.jetty:jetty-*" + versions: + - ">= 10.0.0" + - dependency-name: "formatter-maven-plugin" + - dependency-name: "impsort-maven-plugin" + target-branch: "master" + +# For main webapp +- package-ecosystem: "npm" + directory: "main/webapp" schedule: - interval: "daily" + interval: "weekly" +# For github actions - package-ecosystem: "github-actions" directory: "/" schedule: - interval: "daily" + interval: "monthly" + groups: + actions: + patterns: + - "*" +# For cypress test_suite +- package-ecosystem: "npm" + directory: "main/tests/cypress" + schedule: + interval: "weekly" + groups: + cypress: + patterns: + - "*" diff --git a/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md b/.github/pull_request_template.md similarity index 100% rename from .github/PULL_REQUEST_TEMPLATE/pull_request_template.md rename to .github/pull_request_template.md diff --git a/.github/workflows/codeql_java.yml b/.github/workflows/codeql_java.yml new file mode 100644 index 000000000000..a41b84f7c26e --- /dev/null +++ b/.github/workflows/codeql_java.yml @@ -0,0 +1,39 @@ +name: "CodeQL Java" + +on: + push: + branches: [ "master" ] + paths: + - '**/*.java' # Only trigger the workflow if Java files have changed + pull_request: + branches: [ "master" ] + paths: + - '**/*.java' + +jobs: + analyze: + if: github.repository == 'OpenRefine/OpenRefine' + name: Analyze Java + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: 'java' + + - name: Build + run: | + ./refine build + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:java" diff --git a/.github/workflows/codeql_javascript.yml b/.github/workflows/codeql_javascript.yml new file mode 100644 index 000000000000..5108d7a0fbbc --- /dev/null +++ b/.github/workflows/codeql_javascript.yml @@ -0,0 +1,39 @@ +name: "CodeQL JavaScript" + +on: + push: + branches: [ "master" ] + paths: + - '**/*.js' # Only trigger the workflow if JavaScript files have changed + pull_request: + branches: [ "master" ] + paths: + - '**/*.js' + +jobs: + analyze: + if: github.repository == 'OpenRefine/OpenRefine' + name: Analyze JavaScript + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: 'javascript' + + - name: Build + run: | + ./refine build + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:javascript" diff --git a/.github/workflows/label_transfer.yml b/.github/workflows/label_transfer.yml new file mode 100644 index 000000000000..a7121a89c2f1 --- /dev/null +++ b/.github/workflows/label_transfer.yml @@ -0,0 +1,26 @@ + +name: Copy labels from issue to pull request + +on: + pull_request_target: + types: [opened, edited] + +jobs: + transfer_tags: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.9 + uses: actions/setup-python@v5 + with: + python-version: 3.9 + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools + pip install -r .github/workflows/label_transfer/requirements.txt + pip freeze + - name: Run Python label transfer script + run: python .github/workflows/label_transfer/script.py ${{ github.event.number }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPO: ${{ github.repository }} diff --git a/.github/workflows/label_transfer/requirements.txt b/.github/workflows/label_transfer/requirements.txt new file mode 100644 index 000000000000..828889b82401 --- /dev/null +++ b/.github/workflows/label_transfer/requirements.txt @@ -0,0 +1,2 @@ +requests +lxml diff --git a/.github/workflows/label_transfer/script.py b/.github/workflows/label_transfer/script.py new file mode 100644 index 000000000000..a408e0090848 --- /dev/null +++ b/.github/workflows/label_transfer/script.py @@ -0,0 +1,78 @@ +import requests +import os +from lxml import html +import sys +import json + +# Config + +# list of labels that should not be transferred to PRs +do_not_transfer = [ + 'good first issue', + 'good second issue', + 'imported from old code repo', + 'help wanted', + 'duplicate', + 'invalid', + 'question', + 'to be reviewed', +] + +# Internal + +repo = os.environ.get('GITHUB_REPO') +github_token = os.environ.get('GITHUB_TOKEN') + +headers = { + 'Accept': 'application/vnd.github.v3+json', +} + +if github_token: + headers['Authorization'] = 'Bearer '+github_token + +def get_linked_issues(pr_number): + """ + Given a PR number, extract all the linked issue numbers from it. + Sadly this is not supported by the API yet, so we just scrape the web UI. + """ + url = f'https://github.com/{repo}/pull/{pr_number}' + page = requests.get(url) + page.raise_for_status() + parsed = html.document_fromstring(page.text) + matches = parsed.xpath('//form/span[@class="Truncate truncate-with-responsive-width my-1"]/a') + for match in matches: + yield int(match.attrib['href'].split('/')[-1]) + +def get_issue_labels(issue_number): + """ + Returns all the labels in a given issue / PR + """ + url = f'https://api.github.com/repos/{repo}/issues/{issue_number}/labels' + response = requests.get(url, headers=headers) + response.raise_for_status() + return [ tag['name'] for tag in response.json() ] + +def transfer_issue_labels(pr_number): + """ + Transfers labels from all the linked issues to the PR + """ + linked_issues = get_linked_issues(pr_number) + if not linked_issues: + print('No linked issues found') + all_labels = [ label for issue in linked_issues for label in get_issue_labels(issue) ] + to_transfer = [ label for label in all_labels if label not in do_not_transfer ] + current_labels = get_issue_labels(pr_number) + missing_labels = [ label for label in to_transfer if label not in current_labels ] + if not missing_labels: + return + new_labels = current_labels + missing_labels + url = f'https://api.github.com/repos/{repo}/issues/{pr_number}/labels' + print(f'adding {missing_labels} to PR #{pr_number}') + if not github_token: + print('no GITHUB_TOKEN, skipping') + else: + resp = requests.put(url, headers=headers, data=json.dumps({'labels':new_labels})) + resp.raise_for_status() + +if __name__ == '__main__': + transfer_issue_labels(sys.argv[1]) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml deleted file mode 100644 index aff51029be15..000000000000 --- a/.github/workflows/pull_request.yml +++ /dev/null @@ -1,74 +0,0 @@ -name: Java CI - -on: [pull_request_target] - -jobs: - build: - strategy: - matrix: - java: [ 8, 14 ] - - runs-on: ubuntu-latest - - services: - postgres: - image: postgres - ports: - - 5432 - env: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: 'postgres' - POSTGRES_DB: test_db - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - mysql: - image: mysql:8 - ports: - - 3306 - env: - MYSQL_ROOT_PASSWORD: root - options: >- - --health-cmd "mysqladmin ping" - --health-interval 5s - --health-timeout 2s - --health-retries 3 - - steps: - - uses: actions/checkout@v2.3.4 - - - name: Restore dependency cache - uses: actions/cache@v2.1.3 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- - - - name: Set up Java ${{ matrix.java }} - uses: actions/setup-java@v1 - with: - java-version: ${{ matrix.java }} - - - name: Configure connections to databases - id: configure_db_connections - run: cat extensions/database/tests/conf/github_actions_tests.xml | sed -e "s/MYSQL_PORT/${{ job.services.mysql.ports[3306] }}/g" | sed -e "s/POSTGRES_PORT/${{ job.services.postgres.ports[5432] }}/g" > extensions/database/tests/conf/tests.xml - - - name: Populate databases with test data - id: populate_databases_with_test_data - run: | - mysql -u root -h 127.0.0.1 -P ${{ job.services.mysql.ports[3306] }} -proot -e 'CREATE DATABASE test_db;' - mysql -u root -h 127.0.0.1 -P ${{ job.services.mysql.ports[3306] }} -proot < extensions/database/tests/conf/test-mysql.sql - psql -U postgres test_db -h 127.0.0.1 -p ${{ job.services.postgres.ports[5432] }} < extensions/database/tests/conf/test-pgsql.sql - env: - PGPASSWORD: postgres - - - name: Build and test with Maven - run: mvn jacoco:prepare-agent test - - - name: Submit test coverage to Coveralls - run: | - mvn prepare-package -DskipTests=true - mvn jacoco:report coveralls:report -DrepoToken=${{ secrets.COVERALLS_TOKEN }} -DpullRequest=${{ github.event.number }} diff --git a/.github/workflows/pull_request_e2e.yml b/.github/workflows/pull_request_e2e.yml new file mode 100644 index 000000000000..fb09561a57f8 --- /dev/null +++ b/.github/workflows/pull_request_e2e.yml @@ -0,0 +1,92 @@ +name: CI + +on: + pull_request: + paths-ignore: + - 'main/IDEs/**' + - 'main/webapp/modules/core/langs/**' + - 'extensions/database/module/langs/**' + - 'extensions/gdata/module/langs/**' + - 'extensions/pc-axis/module/langs/**' + - 'extensions/phonetic/module/langs/**' + - 'extensions/wikidata/module/langs/**' + branches: + - master + - '4.0' + +permissions: read-all + +jobs: + prepare_e2e_test_matrix: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v4 + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: '20' + - id: set-matrix + run: npm install --save glob@8.1.0 && node main/tests/cypress/build-test-matrix.js >> $GITHUB_OUTPUT + env: + browsers: chrome + e2e_test: + name: e2e_test ${{ matrix.specs.group }} + needs: prepare_e2e_test_matrix + runs-on: ubuntu-latest + strategy: + matrix: ${{fromJSON(needs.prepare_e2e_test_matrix.outputs.matrix)}} + steps: + - uses: actions/checkout@v4 + + - name: Set up secrets + run: | + echo "CYPRESS_RECORD_KEY=$(echo YzE3ZDU4OGItZTBkOC00ZjJmLTg4NjYtNzJmNmFmYmRhNGQxCg== | base64 -d)" >> $GITHUB_ENV + echo "CYPRESS_PROJECT_ID=s5du3k" >> $GITHUB_ENV + + - name: Set up Java 17 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 17 + cache: 'maven' + + - name: Build OpenRefine + run: ./refine build + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Restore Tests dependency cache + uses: actions/cache@v4 + with: + path: | + ~/cache + ~/.cache + **/node_modules + !~/cache/exclude + key: ${{ runner.os }}-modules-${{ hashFiles('**/yarn.lock') }} + restore-keys: | + ${{ runner.os }}-yarn + + - name: Install test dependencies + run: | + cd ./main/tests/cypress + npm i -g yarn + yarn install + + - name: Test with Cypress on ${{ matrix.browser }} + run: | + echo REFINE_MIN_MEMORY=1400M >> ./refine.ini + echo REFINE_MEMORY=4096M >> ./refine.ini + ./refine e2e_tests + env: + CYPRESS_BROWSER: ${{ matrix.browser }} + CYPRESS_RECORD_KEY: ${{ env.CYPRESS_RECORD_KEY }} + CYPRESS_PROJECT_ID: ${{ env.CYPRESS_PROJECT_ID }} + CYPRESS_CI_BUILD_ID: '${{ github.run_id }}' + CYPRESS_SPECS: ${{ matrix.specs.paths }} + CYPRESS_GROUP: ${{ matrix.specs.group }} diff --git a/.github/workflows/pull_request_server.yml b/.github/workflows/pull_request_server.yml new file mode 100644 index 000000000000..80526c5118f2 --- /dev/null +++ b/.github/workflows/pull_request_server.yml @@ -0,0 +1,210 @@ +name: CI + +on: + pull_request: + paths: + - 'benchmark/**' + - 'conf/**' + - 'extensions/database/src/**' + - 'extensions/database/tests/**' + - 'extensions/database/pom.xml' + - 'extensions/gdata/src/**' + - 'extensions/gdata/tests/**' + - 'extensions/gdata/pom.xml' + - 'extensions/jython/src/**' + - 'extensions/jython/tests/**' + - 'extensions/jython/pom.xml' + - 'extensions/sample/src/**' + - 'extensions/sample/pom.xml' + - 'extensions/pc-axis/src/**' + - 'extensions/pc-axis/pom.xml' + - 'extensions/phonetic/src/**' + - 'extensions/phonetic/tests/**' + - 'extensions/phonetic/pom.xml' + - 'extensions/wikibase/src/**' + - 'extensions/wikibase/tests/**' + - 'extensions/wikibase/pom.xml' + - 'extensions/pom.xml' + - 'graphics/**' + - 'main/resources/**' + - 'main/src/**' + - 'main/tests/data/**' + - 'main/tests/server/**' + - 'main/pom.xml' + - 'packaging/**' + - 'server/**' + - 'pom.xml' + - 'refine' + - 'refine.bat' + - 'refine.ini' + branches: + - master + - '4.0' + +permissions: read-all + +jobs: + linux_server_tests: + strategy: + matrix: + java: [ 21 ] + + runs-on: ubuntu-latest + + services: + postgres: + image: postgres + ports: + - 5432 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: 'postgres' + POSTGRES_DB: test_db + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + mysql: + image: mysql:8 + ports: + - 3306 + env: + MYSQL_ROOT_PASSWORD: root + options: >- + --health-cmd "mysqladmin ping" + --health-interval 5s + --health-timeout 2s + --health-retries 3 + + steps: + - name: Set up secrets + run: | + echo "COVERALLS_TOKEN=$(echo eUVUVGRHOFJhQm9GMFJBYTNibjVhcWFEblpac1lmMlE3Cg== | base64 -d)" >> $GITHUB_ENV + + - uses: actions/checkout@v4 + + - name: Set up Java ${{ matrix.java }} + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: ${{ matrix.java }} + cache: 'maven' + + - name: Check Java linting + id: java_linting + run: | + mvn -B compile formatter:validate impsort:check javadoc:javadoc -Ddoclint=all + + - name: Configure connections to databases + id: configure_db_connections + run: cat extensions/database/tests/conf/github_actions_tests.xml | sed -e "s/MYSQL_PORT/${{ job.services.mysql.ports[3306] }}/g" | sed -e "s/POSTGRES_PORT/${{ job.services.postgres.ports[5432] }}/g" > extensions/database/tests/conf/tests.xml + + - name: Populate databases with test data + id: populate_databases_with_test_data + run: | + mysql -u root -h 127.0.0.1 -P ${{ job.services.mysql.ports[3306] }} -proot -e 'CREATE DATABASE test_db;' + mysql -u root -h 127.0.0.1 -P ${{ job.services.mysql.ports[3306] }} -proot < extensions/database/tests/conf/test-mysql.sql + psql -U postgres test_db -h 127.0.0.1 -p ${{ job.services.postgres.ports[5432] }} < extensions/database/tests/conf/test-pgsql.sql + env: + PGPASSWORD: postgres + + - name: Build and test with Maven + run: mvn -B jacoco:prepare-agent test jacoco:report + + - name: Coveralls main + uses: coverallsapp/github-action@v2 + with: + base-path: main/src + files: main/target/site/jacoco/jacoco.xml + format: jacoco + flag-name: Java-${{ matrix.java }}-main + fail-on-error: false + parallel: true + + - name: Coveralls database + uses: coverallsapp/github-action@v2 + with: + base-path: extensions/database/src + files: extensions/database/target/site/jacoco/jacoco.xml + github-token: ${{ env.COVERALLS_TOKEN }} + format: jacoco + flag-name: Java-${{ matrix.java }}-database + fail-on-error: false + parallel: true + + - name: Coveralls phonetic + uses: coverallsapp/github-action@v2 + with: + base-path: extensions/phonetic/src + files: extensions/phonetic/target/site/jacoco/jacoco.xml + github-token: ${{ env.COVERALLS_TOKEN }} + format: jacoco + flag-name: Java-${{ matrix.java }}-phonetic + fail-on-error: false + parallel: true + + - name: Coveralls gdata + uses: coverallsapp/github-action@v2 + with: + base-path: extensions/gdata/src + files: extensions/gdata/target/site/jacoco/jacoco.xml + github-token: ${{ env.COVERALLS_TOKEN }} + format: jacoco + flag-name: Java-${{ matrix.java }}-gdata + fail-on-error: false + parallel: true + + - name: Coveralls wikibase + uses: coverallsapp/github-action@v2 + with: + base-path: extensions/wikibase/src + files: extensions/wikibase/target/site/jacoco/jacoco.xml + github-token: ${{ env.COVERALLS_TOKEN }} + format: jacoco + flag-name: Java-${{ matrix.java }}-wikibase + fail-on-error: false + parallel: true + + - name: Coveralls jython + uses: coverallsapp/github-action@v2 + with: + base-path: extensions/jython/src + files: extensions/jython/target/site/jacoco/jacoco.xml + github-token: ${{ env.COVERALLS_TOKEN }} + format: jacoco + flag-name: Java-${{ matrix.java }}-jython + fail-on-error: false + parallel: true + + windows_server_tests: + runs-on: windows-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Java 21 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 21 + cache: 'maven' + + - name: Check Java linting + id: java_linting + run: | + mvn -B formatter:validate impsort:check + + - name: Build and test with Maven + run: mvn -B jacoco:prepare-agent test + + finish: + needs: linux_server_tests + if: ${{ always() }} + runs-on: ubuntu-latest + steps: + # Only really needed if we're testing in separate jobs (e.g. multiple Java versions) + - name: Close parallel build + uses: coverallsapp/github-action@v2 + with: + parallel-finished: true diff --git a/.github/workflows/release/add_apple_keys.sh b/.github/workflows/release/add_apple_keys.sh new file mode 100755 index 000000000000..62983d6d648d --- /dev/null +++ b/.github/workflows/release/add_apple_keys.sh @@ -0,0 +1,20 @@ +#!/bin/sh + +# Create a custom keychain +security create-keychain -p gh_actions refine-build.keychain + +# Make the custom keychain default, so xcodebuild will use it for signing +security default-keychain -s refine-build.keychain + +# Unlock the keychain +security unlock-keychain -p gh_actions refine-build.keychain + +# Set keychain timeout to 1 hour for long builds +security set-keychain-settings -t 3600 -l ~/Library/Keychains/refine-build.keychain + +# Add certificates to keychain and allow codesign to access them +security import ./packaging/apple_certs/AppleWWDRCA.cer -k ~/Library/Keychains/refine-build.keychain -T /usr/bin/codesign +security import ./.github/workflows/release/apple_cert.cer -k ~/Library/Keychains/refine-build.keychain -T /usr/bin/codesign +security import ./.github/workflows/release/apple_cert.p12 -k ~/Library/Keychains/refine-build.keychain -P $P12_PASSPHRASE -T /usr/bin/codesign + +security set-key-partition-list -S apple-tool:,apple: -s -k gh_actions refine-build.keychain diff --git a/.github/workflows/release/gdata.patch.asc b/.github/workflows/release/gdata.patch.asc new file mode 100644 index 000000000000..1a066e4615ef Binary files /dev/null and b/.github/workflows/release/gdata.patch.asc differ diff --git a/.github/workflows/release_manager.sh b/.github/workflows/release_manager.sh deleted file mode 100755 index 4d551a5fa67c..000000000000 --- a/.github/workflows/release_manager.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - - -# Build Release Message -API_JSON=$(printf '{"tag_name": "%s","target_commitish": "master","name": "OpenRefine snapshot %s","body": "This is a snapshot of the development version of OpenRefine, made on %s.\\n\\nThis contains the latest new features and bug fixes, but might not have been tested as thoroughly as official releases.\\n\\nMake sure you [back up your workspace](https://github.com/OpenRefine/OpenRefine/wiki/Back-Up-OpenRefine-Data) to avoid data loss and report any issues found with this version on [the mailing list](https://groups.google.com/forum/#!forum/openrefine).","draft": false,"prerelease": true}' ${OR_VERSION} ${OR_VERSION} "$(date -u +"%c")" ) - -# Create Release -echo $( curl --silent --data "${API_JSON}" "https://api.github.com/repos/${RELEASE_REPO_OWNER}/OpenRefine-snapshot-releases/releases?access_token=${RELEASE_REPO_TOKEN}" | jq -r '.upload_url' ) diff --git a/.github/workflows/snapshot_release.yml b/.github/workflows/snapshot_release.yml index 0d8da984f3a3..5cc592e9a03e 100644 --- a/.github/workflows/snapshot_release.yml +++ b/.github/workflows/snapshot_release.yml @@ -3,11 +3,99 @@ name: Snapshot release on: push: branches: - - master + - 'master' + - '4.0' + paths-ignore: + - 'docs/**' + release: + types: [created] jobs: - build: + prepare_e2e_test_matrix: + if: github.repository == 'OpenRefine/OpenRefine' + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.ref }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: '20' + - id: set-matrix + run: npm install --save glob@8.1.0 && node main/tests/cypress/build-test-matrix.js >> $GITHUB_OUTPUT + env: + browsers: chrome + e2e_test: + if: github.repository == 'OpenRefine/OpenRefine' + name: e2e_test ${{ matrix.specs.group }} + needs: prepare_e2e_test_matrix + runs-on: ubuntu-latest + strategy: + matrix: ${{fromJSON(needs.prepare_e2e_test_matrix.outputs.matrix)}} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.ref }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + + - name: Set up secrets + run: | + echo "CYPRESS_RECORD_KEY=$(echo YzE3ZDU4OGItZTBkOC00ZjJmLTg4NjYtNzJmNmFmYmRhNGQxCg== | base64 -d)" >> $GITHUB_ENV + echo "CYPRESS_PROJECT_ID=s5du3k" >> $GITHUB_ENV + + - name: Set up Java 11 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 11 + cache: 'maven' + + - name: Build OpenRefine + run: ./refine build + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Restore Tests dependency cache + uses: actions/cache@v4 + with: + path: | + ~/cache + ~/.cache + **/node_modules + !~/cache/exclude + key: ${{ runner.os }}-modules-${{ hashFiles('**/yarn.lock') }} + restore-keys: | + ${{ runner.os }}-yarn + + - name: Install test dependencies + run: | + cd ./main/tests/cypress + npm i -g yarn + yarn install + + - name: Test with Cypress on ${{ matrix.browser }} + run: | + echo REFINE_MIN_MEMORY=1400M >> ./refine.ini + echo REFINE_MEMORY=4096M >> ./refine.ini + ./refine e2e_tests + env: + CYPRESS_BROWSER: ${{ matrix.browser }} + CYPRESS_RECORD_KEY: ${{ env.CYPRESS_RECORD_KEY }} + CYPRESS_PROJECT_ID: ${{ env.CYPRESS_PROJECT_ID }} + CYPRESS_CI_BUILD_ID: '${{ github.run_id }}' + CYPRESS_SPECS: ${{ matrix.specs.paths }} + CYPRESS_GROUP: ${{ matrix.specs.group }} + + + build: + if: github.repository == 'OpenRefine/OpenRefine' services: postgres: image: postgres @@ -37,22 +125,24 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2.3.4 + - uses: actions/checkout@v4 with: fetch-depth: 0 # This is wasteful, but needed for git describe - - name: Restore dependency cache - uses: actions/cache@v2.1.3 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- + - name: Set up secrets + run: | + echo "COVERALLS_TOKEN=$(echo eUVUVGRHOFJhQm9GMFJBYTNibjVhcWFEblpac1lmMlE3Cg== | base64 -d)" >> $GITHUB_ENV - - name: Set up JDK 1.8 - uses: actions/setup-java@v1 + - name: Set up Java 11 + uses: actions/setup-java@v4 with: - java-version: 1.8 + distribution: 'temurin' + java-version: 11 + cache: 'maven' + server-id: ossrh + server-username: OSSRH_USER + server-password: OSSRH_PASS + - name: Install genisoimage and jq run: sudo apt-get install genisoimage jq @@ -70,79 +160,245 @@ jobs: env: PGPASSWORD: postgres + - name: Install webapp dependencies + run: cd main/webapp && npm install + - name: Build and test with Maven - run: mvn jacoco:prepare-agent test + run: mvn -B jacoco:prepare-agent test jacoco:report - - name: Submit test coverage to Coveralls - run: | - mvn prepare-package -DskipTests=true - mvn jacoco:report coveralls:report -DrepoToken=${{ secrets.COVERALLS_TOKEN }} -DpullRequest=${{ github.event.number }} -DserviceName="GitHub Actions" -DserviceBuildNumber=${{ env.GITHUB_RUN_ID }} -Dbranch=master + - name: Coveralls + uses: coverallsapp/github-action@v2 + with: + github-token: ${{ env.COVERALLS_TOKEN }} + format: jacoco + flag-name: Java ${{ matrix.java }} + pull-request: ${{ github.event.number }} + fail-on-error: false - - name: Get the OpenRefine snapshot version - run: echo ::set-env name=OR_VERSION::$(cat ./main/webapp/WEB-INF/classes/git.properties | jq -r '.["git.commit.id.describe"]') - env: - ACTIONS_ALLOW_UNSECURE_COMMANDS: true + - name: Install webapp dependencies + run: cd main/webapp && npm install - name: Generate dist files - run: ./refine dist ${{ env.OR_VERSION }} + run: mvn -B package -DskipTests=true + + mac_test_and_deploy: + if: github.repository == 'OpenRefine/OpenRefine' + runs-on: macos-12 + + outputs: + version_number_string: ${{ steps.version_string_variable.outputs.output_value }} + + steps: + - uses: actions/checkout@v4 + + - name: Set up Java 17 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 17 + cache: 'maven' + server-id: ossrh + server-username: OSSRH_USER + server-password: OSSRH_PASS + gpg-private-key: ${{ secrets.GPG_PRIVATE_KEY }} + gpg-passphrase: GPG_PASSPHRASE + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3' + + - name: Install dmgbuild + run: pip install dmgbuild + + - name: Decrypt Apple certificates + run: | + gpg --pinentry-mode loopback --passphrase "${{ secrets.GPG_PASSPHRASE }}" --output .github/workflows/release/apple_cert.cer --decrypt packaging/apple_certs/apple_cert.cer.enc + gpg --pinentry-mode loopback --passphrase "${{ secrets.GPG_PASSPHRASE }}" --output .github/workflows/release/apple_cert.p12 --decrypt packaging/apple_certs/apple_cert.p12.enc + .github/workflows/release/add_apple_keys.sh + env: + P12_PASSPHRASE: ${{ secrets.APPLE_P12_PASSPHRASE }} + + - name: Apply patch to insert Google API credentials + run: | + gpg --pinentry-mode loopback --passphrase "${{ secrets.GPG_PASSPHRASE }}" --output .github/workflows/release/gdata.patch --decrypt .github/workflows/release/gdata.patch.asc + git am .github/workflows/release/gdata.patch + git reset HEAD^ + + - name: Install webapp dependencies + run: cd main/webapp && npm install - - name: Create Release - if: github.repository == 'OpenRefine/OpenRefine' - id: create_release - run: echo ::set-env name=API_RELEASE::$(./.github/workflows/release_manager.sh) + - name: Get release upload URL + id: get_release_upload_url + if: github.event_name == 'release' + uses: bruceadams/get-release@v1.3.2 env: - ACTIONS_ALLOW_UNSECURE_COMMANDS: true - OR_VERSION: ${{ env.OR_VERSION }} - RELEASE_REPO_OWNER: OpenRefine - RELEASE_REPO_TOKEN: ${{ secrets.RELEASE_REPO_TOKEN }} - - - name: Upload Release Asset Windows - id: upload-release-asset-win - if: github.repository == 'OpenRefine/OpenRefine' - uses: actions/upload-release-asset@v1.0.2 + GITHUB_TOKEN: ${{ github.token }} + + - name: Set version + if: github.event_name == 'release' + run: mvn versions:set -DnewVersion=$(echo '${{ steps.get_release_upload_url.outputs.tag_name }}' | sed 's/[^a-zA-Z0-9_\.\-]/_/g') + + - name: Build and test + run: mvn -B test env: - GITHUB_TOKEN: ${{ secrets.RELEASE_REPO_TOKEN }} + GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} + + - name: Set version string output variable + id: version_string_variable + run: echo "output_value=$(mvn org.apache.maven.plugins:maven-help-plugin:3.1.0:evaluate -Dexpression=project.version -q -DforceStdout)" >> "$GITHUB_OUTPUT" + + - name: Package and upload to OSSRH + run: | + mvn -B deploy -DskipTests=true -DrepositoryId=ossrh -Dapple.notarization.username=$APPLE_USERNAME -Dapple.notarization.password=$APPLE_PASSWORD -Dapple.notarization.team.id=$APPLE_TEAM_ID -Dgpg.signer=bc + env: + OSSRH_USER: ${{ secrets.OSSRH_USER }} + OSSRH_PASS: ${{ secrets.OSSRH_PASS }} + MAVEN_GPG_KEY: ${{ secrets.GPG_PRIVATE_KEY }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} + APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }} + APPLE_USERNAME: ${{ secrets.APPLE_USERNAME }} + APPLE_PASSWORD: ${{ secrets.APPLE_PASSWORD }} + + - name: Upload release asset for Windows (with Java) + id: upload-release-asset-win-with-java + if: github.event_name == 'release' + uses: shogo82148/actions-upload-release-asset@v1.7.5 with: - upload_url: ${{ env.API_RELEASE }} - asset_path: ./packaging/target/openrefine-win-${{ env.OR_VERSION }}.zip - asset_name: openrefine-win-${{ env.OR_VERSION }}.zip + upload_url: ${{ steps.get_release_upload_url.outputs.upload_url }} + asset_path: ./packaging/target/openrefine-win-with-java-${{ steps.version_string_variable.outputs.output_value }}.zip + asset_name: openrefine-win-with-java-${{ steps.version_string_variable.outputs.output_value }}.zip asset_content_type: application/zip + - name: Upload build artifact for Windows (with Java) + id: upload-build-artifact-win-with-java + uses: actions/upload-artifact@v4 + with: + path: "./packaging/target/openrefine-win-with-java-${{ steps.version_string_variable.outputs.output_value }}.zip" + name: openrefine-win-with-java-archive-${{ steps.version_string_variable.outputs.output_value }} + + - name: Unzip build artifact for Windows (with Java) for Installer + id: unzip-build-artifact-win-with-java + run: unzip ./packaging/target/openrefine-win-with-java-${{ steps.version_string_variable.outputs.output_value }}.zip -d ./packaging/target/openrefine-win-with-java-${{ steps.version_string_variable.outputs.output_value }} + + - name: Upload build artifact for Windows (with Java) for Installer + id: upload-build-artifact-win-with-java-for-installer + uses: actions/upload-artifact@v4 + with: + name: installertemp-win-with-java-${{ steps.version_string_variable.outputs.output_value }} + path: ./packaging/target/openrefine-win-with-java-${{ steps.version_string_variable.outputs.output_value }}/openrefine-${{ steps.version_string_variable.outputs.output_value }} + - name: Upload Release Asset Linux id: upload-release-asset-linux - if: github.repository == 'OpenRefine/OpenRefine' - uses: actions/upload-release-asset@v1.0.2 - env: - GITHUB_TOKEN: ${{ secrets.RELEASE_REPO_TOKEN }} + if: github.event_name == 'release' + uses: shogo82148/actions-upload-release-asset@v1.7.5 with: - upload_url: ${{ env.API_RELEASE }} - asset_path: ./packaging/target/openrefine-linux-${{ env.OR_VERSION }}.tar.gz - asset_name: openrefine-linux-${{ env.OR_VERSION }}.tar.gz + upload_url: ${{ steps.get_release_upload_url.outputs.upload_url }} + asset_path: ./packaging/target/openrefine-linux-${{ steps.version_string_variable.outputs.output_value }}.tar.gz + asset_name: openrefine-linux-${{ steps.version_string_variable.outputs.output_value }}.tar.gz asset_content_type: application/tar+gzip + - name: Upload build artifact for Linux + id: upload-build-artifact-linux + uses: actions/upload-artifact@v4 + with: + path: "./packaging/target/openrefine-linux-${{ steps.version_string_variable.outputs.output_value }}.tar.gz" + name: openrefine-linux-${{ steps.version_string_variable.outputs.output_value }} + - name: Upload Release Asset Mac id: upload-release-asset-mac - if: github.repository == 'OpenRefine/OpenRefine' - uses: actions/upload-release-asset@v1.0.2 - env: - GITHUB_TOKEN: ${{ secrets.RELEASE_REPO_TOKEN }} + if: github.event_name == 'release' + uses: shogo82148/actions-upload-release-asset@v1.7.5 with: - upload_url: ${{ env.API_RELEASE }} - asset_path: ./packaging/target/openrefine-mac-${{ env.OR_VERSION }}.dmg - asset_name: openrefine-mac-${{ env.OR_VERSION }}.dmg + upload_url: ${{ steps.get_release_upload_url.outputs.upload_url }} + asset_path: ./packaging/target/openrefine-mac-${{ steps.version_string_variable.outputs.output_value }}.dmg + asset_name: openrefine-mac-${{ steps.version_string_variable.outputs.output_value }}.dmg asset_content_type: application/x-apple-diskimage - - name: Delete older releases - id: delete-older-releases - if: github.repository == 'OpenRefine/OpenRefine' - uses: dev-drprasad/delete-older-releases@v0.1.0 - with: - repo: OpenRefine/OpenRefine-snapshot-releases - # Specifies number of latest releases (sorted by created_at) to keep. Pass 0 if you want to delete all releases - keep_latest: 10 - # Specifies whether to delete tags associated to older releases or not. - # Older tags without any associated releases will not be deleted - delete_tags: true + - name: Upload build artifact for Mac + id: upload-build-artifact-mac + uses: actions/upload-artifact@v4 + with: + path: "./packaging/target/openrefine-mac-${{ steps.version_string_variable.outputs.output_value }}.dmg" + name: openrefine-mac-${{ steps.version_string_variable.outputs.output_value }} + + windows_installer: + if: github.repository == 'OpenRefine/OpenRefine' + runs-on: windows-2022 + needs: mac_test_and_deploy + + steps: + - uses: actions/checkout@v4 + + - name: Get release upload URL + id: get_release_upload_url + if: github.event_name == 'release' + uses: bruceadams/get-release@v1.3.2 env: - GITHUB_TOKEN: ${{ secrets.RELEASE_REPO_TOKEN }} + GITHUB_TOKEN: ${{ github.token }} + - name: Download build artifact for Windows Installer (with Java) + id: download-build-artifact-win-with-java + uses: actions/download-artifact@v4 + with: + name: installertemp-win-with-java-${{ needs.mac_test_and_deploy.outputs.version_number_string }} + path: ./packaging/target/openrefine-win-with-java-${{ needs.mac_test_and_deploy.outputs.version_number_string }} + +# - name: Sign the executables +# uses: nextgens/authenticode-sign-action@v1.0.1 +# with: +# certificate: '${{ secrets.CERTIFICATES }}' +# credentials: '${{ secrets.CERTIFICATE_CREDENTIALS }}' +# key-uri: "projects/myProject/locations/europe-west2/keyRings/code-signing/cryptoKeys/ev/cryptoKeyVersions/1" +# timestamp-url: 'http://timestamp.digicert.com' +# description: 'OpenRefine' +# description-url: 'https://openrefine.org' +# folder: "./packaging/target/openrefine-win-with-java-${{ needs.mac_test_and_deploy.outputs.version_number_string }}" +# recursive: true + + - name: Build self-containing installer with InnoSetup 6 for Windows + id: build-installer-innosetup-win-with-java + run: | + "%ProgramFiles(x86)%\Inno Setup 6\iscc.exe" -DMyAppVersion="${{ needs.mac_test_and_deploy.outputs.version_number_string }}" -DMyProgramFiles="target\openrefine-win-with-java-${{ needs.mac_test_and_deploy.outputs.version_number_string }}" "packaging\openrefine.iss" + shell: cmd + + - name: Cleanup unused installer build artifact (delete) + id: delete-build-artifact-win-with-java + uses: geekyeggo/delete-artifact@v5 + with: + name: installertemp-win-with-java-${{ needs.mac_test_and_deploy.outputs.version_number_string }} + useGlob: false + failOnError: false + +# - name: Sign the installer +# uses: nextgens/authenticode-sign-action@v1.0.1 +# with: +# certificate: '${{ secrets.CERTIFICATES }}' +# credentials: '${{ secrets.CERTIFICATE_CREDENTIALS }}' +# key-uri: "projects/myProject/locations/europe-west2/keyRings/code-signing/cryptoKeys/ev/cryptoKeyVersions/1" +# timestamp-url: 'http://timestamp.digicert.com' +# description: 'OpenRefine Installer' +# description-url: 'https://openrefine.org' +# folder: "./packaging/target" +# recursive: false + +# - name: Verify the signature +# run: call "%ProgramFiles(x86)%\Windows Kits\10\bin\10.0.17763.0\x86\signtool.exe" verify /v /pa "OpenRefine-Installer-${{ needs.mac_test_and_deploy.outputs.version_number_string }}.exe" +# shell: cmd + + - name: Upload the installer as an artifact + id: upload-build-artifact-win-with-java-installer + uses: actions/upload-artifact@v4 + with: + path: "./packaging/target/openrefine-win-with-java-installer-${{ needs.mac_test_and_deploy.outputs.version_number_string }}.exe" + name: openrefine-win-with-java-installer-${{ needs.mac_test_and_deploy.outputs.version_number_string }} + + - name: Upload release asset for Windows (with Java) + id: upload-release-asset-win-with-java-installer + if: github.event_name == 'release' + uses: shogo82148/actions-upload-release-asset@v1.7.5 + with: + upload_url: ${{ steps.get_release_upload_url.outputs.upload_url }} + asset_path: ./packaging/target/openrefine-win-with-java-installer-${{ needs.mac_test_and_deploy.outputs.version_number_string }}.exe + asset_name: openrefine-win-with-java-installer-${{ needs.mac_test_and_deploy.outputs.version_number_string }}.exe + asset_content_type: application/vnd.microsoft.portable-executable diff --git a/.github/workflows/unassign_issues.yml b/.github/workflows/unassign_issues.yml new file mode 100644 index 000000000000..b438b0d4ad79 --- /dev/null +++ b/.github/workflows/unassign_issues.yml @@ -0,0 +1,21 @@ +name: Unassign contributors from issues after a delay + +on: + schedule: + - cron: '39 4 * * *' + +jobs: + # Best used in combination with actions/stale to assign a Stale label + + unassign-good-first-issues-after-90-days-of-inactivity: + if: github.repository == 'OpenRefine/OpenRefine' + name: Unassign issues labeled "good first issue" after 45 days of inactivity. + runs-on: ubuntu-latest + steps: + - uses: boundfoxstudios/action-unassign-contributor-after-days-of-inactivity@v1.0.3 + with: + last-activity: 45 # After how many days the issue should get unassigned + labels: 'good first issue' # Only search for issues containing this labels (comma-separated) + exempt-assignees: '' # Exempt some assignees from being unassigned (comma-separated) + labels-to-remove: '' # Labels to remove after unassigning an issue (comma-separated) + message: 'Automatically unassigned after 45 days of inactivity.' diff --git a/.gitignore b/.gitignore index bd9d695977d4..a2ea5f4dccbe 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,8 @@ .idea/ +# Leave .idea settings ignored and force users to use `git add -f` to update +#!*/ +#!.idea/codeStyles/ +*.iml *.log logs *~ @@ -15,13 +19,7 @@ server/classes/ main/webapp/WEB-INF/classes/ main/tests/server/classes/ main/test-output/ -appengine/classes/ tools/ -broker/appengine/module/MOD-INF/classes/ -broker/core/module/MOD-INF/classes/ -broker/core/WEB-INF/lib/ -broker/core/data/ -broker/core/test-output/ tmp/ /test-output test-out/ @@ -33,11 +31,9 @@ open-refine.log # Locally stored "Eclipse launch configurations" *.launch -.idea -*.iml - main/target/ main/webapp/WEB-INF/lib/ +main/webapp/modules/core/3rdparty/ server/target/ extensions/*/target/ extensions/*/module/MOD-INF/classes/ @@ -51,6 +47,9 @@ target/ *.classpath *.project *.settings +#.settings/ +#!/* +#!.settings/org.eclipse.jdt.ui.prefs # Ignore Apache Maven default download path apache-maven-*/ @@ -67,3 +66,17 @@ refine-dev.ini # STS (Spring Tool Suite) .springBeans + +# Ignore Node modules that might inadvertently install at current path instead of user or configured env var NODE_PATH +node_modules/ + +# Ignore the node_modules folder as they are dependencies to be installed by end-user +main/webapp/node_modules + +# Yarn internal files +**/.yarn +**/.yarnrc.yml + +# Ignore .gitignore among extensions as it can be used to ignore custom extensions + +extensions/.gitignore diff --git a/.idea/codeStyles/Project.xml b/.idea/codeStyles/Project.xml new file mode 100644 index 000000000000..31d1a1409fbc --- /dev/null +++ b/.idea/codeStyles/Project.xml @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/codeStyles/codeStyleConfig.xml b/.idea/codeStyles/codeStyleConfig.xml new file mode 100644 index 000000000000..0f7bc519db61 --- /dev/null +++ b/.idea/codeStyles/codeStyleConfig.xml @@ -0,0 +1,5 @@ + + + + diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 000000000000..287fc443a356 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "java.completion.importOrder": ["#", "java", "javax", "", "com.google.refine", "org.openrefine"], +} diff --git a/AUTHORS.md b/AUTHORS.md deleted file mode 100644 index 41883611832e..000000000000 --- a/AUTHORS.md +++ /dev/null @@ -1,78 +0,0 @@ -This file lists the contributors to OpenRefine. - -EMERITUS CREATORS ---------------------- -No longer active but honored for bringing OpenRefine to life. - - David Huynh - - Stefano Mazzocchi - -EMERITUS CONTRIBUTORS ---------------------- -No longer active but honored here for their previous contributions - - - Vishal Talwar - - Jeff Fry - - Will Moffat - - James Home - - Heather Campbell - -CURRENT CONTRIBUTORS --------------------- - -See up to date list at https://github.com/OpenRefine/OpenRefine/graphs/contributors - - - Iain Sproat - - Tom Morris - - Thad Guidry - - Martin Magdinier - - Paul Makepeace - - Tomaž Šolc - - Gabriel Sjoberg - - Rod Salazar - - pxb - - Qi - - Antonin Delpeuch - - Owen Stephens - - Ettore Rizza - - Fabio Tacchelli - - noamoss - - ROMitat - - Jesus M. Castagnetto - - Pablo Moyano - - David Leoni - - Cora Johnson-Roberson - - Pei Long Hui - - Rudy Alvarez - - Mateja Verlic Bruncic - - nestorjal - - Alexey Medvetsky - - Remi Rampin - - lispc - - Bob Harper - - Felix Lohmeier - - Shixiong Zhu - - Ankit Sardesai - - Scott Wiedemann - - Ryo Yamazaki - - Ralf Claussnitzer - - Charles Pritchard - - Maxim Galushka - - Evelyn Mitchell - - Andreas Kohn - - Honza - - Ram Ezrach - - Daniel Berthereau - - Gideon Thomas - - José Vitor Hisse - - Vitor Baptista - - Joe Wicentowski - - mpc9000 - - Dan Michael O. - - Adi Eyal - - Shrubhra Sharma - - Fadi Maali - - Aubrey Mcfato - - Matthias Findeisen - - Mathieu Saby - - Allan Nordhøy - - Tony Opara diff --git a/CHANGES.txt b/CHANGES.txt deleted file mode 100644 index 0a4cd3615e34..000000000000 --- a/CHANGES.txt +++ /dev/null @@ -1,284 +0,0 @@ -2.6 Release (TBD) - First release as OpenRefine - Maintenance release - -Features: - -- Issue 144: Use subdirectory for archives (tar, zip) -- Issue 467: Provide feedback to user on JVM heap memory usage -- Issue 524: Shorten name of anonymous JSON nodes in column names - -Bugs fixed: - -- Issue 154: Can't import RDF/XML Data -- Issue 226: Load into freebase: date objects are not converted into proper freebase dates -- Issue 361: Headless operation on OS X leaves Dock icon bouncing -- Issue 390: Latest POI fixes some import issues -- Issue 432: cross() failing -- Issue 436: Google refine don't start on debian squeeze -- Issue 490: Unable to guess filetype is CSV when importing a simple CSV file -- Issue 496: google-refine-2.5-rc2-r2379 failed to start if using refine.ini -- Issue 515: "Open Project" page shows negative interval for "Last Modified" -- Issue 517: Combin() function does not work as intended -- Issue 521: Machine readable string (/type/rawstring) missing -- Issue 523: Return all available HTTP error information on when Fetch URL fails -- Issue 535: Refine fails to import Excel 2010 XLSX file with null hyperlinks -- Issue 537: ToNumber should try to convert to integer first -- Issue 541: Can't constrain a Freebase key query when using add column -- Issue 542: ToDate should work on integers -- Issue 543: Project creation from URL can't handle gzip Content-Encoding -- Issue 544: FileNotFound exception when importing archive from URL -- Issue 548: escape(value, encoding) returns "null" for non-string values -- Issue 551: MQL preview missing "insert" clause -- Issue 553: Upload Error - Not a directory -- Issue 554: Format guesser ranking isn't working correctly in some cases -- Issue 558: 'refine windows_dist' does not work on MacOS Lion -- Issue 559: Deadlock between autosave thread and history code -- Issue 578: json import on create project deletes useful blank data from arrays -- Issue 586: Only one parse date format is attempted from list in toDate(format1,format2) -- Issue 594: Date diff function doesn't work for two Calendar objects -- Issue 596: JSON importer should not deserialize nulls as the string "null" -- Issue 597: Deselect all button on Custom Tabular Exporter dialog doesn't work -- Issue 599: RDF/XML parser preview not wired up -- Issue 601: Allow selection of root element on JSON import -- Issue 604: The common transform “Trim leading and trailing whitespace” doesn’t trim non-breaking spaces - - -2.5 Release (December 11, 2011) - -Major changes: -- Entirely new importer architecture and user interface for Create Project -- New import/export formats: Google Fusion Tables, OpenOffice Calc, - and fixed width (import only) - -Features: - -- Issue 131: Read multiple files in a directory -- Issue 179: Progress feedback during upload -- Issue 260: XML Importer silently fails if file doesn't meet its criteria -- Issue 279: Add support for reading from non-public Google Spreadsheets docs -- Issue 280: Spreadsheet/table importers should allow selection of sheet/table to import -- Issue 281: Add sheet selection support to Google Spreadsheet importer -- Issue 31: Maximum number of facet values should be configurable. -- Issue 38: Fix the table header so that it's always visible when scrolling a long page -- Issue 433: Usability: use HTML