diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 14cbc46648..f43b03dea2 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -104,7 +104,7 @@ pipeline { 'check-packages-with-kind': generateTestCommandStage(command: 'test-check-packages-with-kind', artifacts: ['build/test-results/*.xml', 'build/kubectl-dump.txt', 'build/elastic-stack-dump/check-*/logs/*.log', 'build/elastic-stack-dump/check-*/logs/fleet-server-internal/*'], junitArtifacts: true, publishCoverage: true), 'check-packages-other': generateTestCommandStage(command: 'test-check-packages-other', artifacts: ['build/test-results/*.xml', 'build/elastic-stack-dump/check-*/logs/*.log', 'build/elastic-stack-dump/check-*/logs/fleet-server-internal/*'], junitArtifacts: true, publishCoverage: true), 'check-packages-with-custom-agent': generateTestCommandStage(command: 'test-check-packages-with-custom-agent', artifacts: ['build/test-results/*.xml', 'build/elastic-stack-dump/check-*/logs/*.log', 'build/elastic-stack-dump/check-*/logs/fleet-server-internal/*'], junitArtifacts: true, publishCoverage: true), - 'build-zip': generateTestCommandStage(command: 'test-build-zip', artifacts: ['build/elastic-stack-dump/build-zip/logs/*.log', 'build/integrations/*.sig']), + 'build-zip': generateTestCommandStage(command: 'test-build-zip', artifacts: ['build/elastic-stack-dump/build-zip/logs/*.log', 'build/packages/*.sig']), 'profiles-command': generateTestCommandStage(command: 'test-profiles-command') ] diff --git a/.ci/package-storage-publish.groovy b/.ci/package-storage-publish.groovy index 412ed19ee1..35b0ab179b 100644 --- a/.ci/package-storage-publish.groovy +++ b/.ci/package-storage-publish.groovy @@ -60,14 +60,14 @@ pipeline { } } } - stash(allowEmpty: true, name: 'build-package', includes: "${BASE_DIR}/build/integrations/*.zip", useDefaultExcludes: false) + stash(allowEmpty: true, name: 'build-package', includes: "${BASE_DIR}/build/packages/*.zip", useDefaultExcludes: false) } } stage('Sign and publish package') { steps { cleanup(source: 'build-package') dir("${BASE_DIR}") { - packageStoragePublish('build/integrations') + packageStoragePublish('build/packages') } } } diff --git a/docs/howto/dependency_management.md b/docs/howto/dependency_management.md index cc47a1cb6f..16d96f97b7 100644 --- a/docs/howto/dependency_management.md +++ b/docs/howto/dependency_management.md @@ -42,7 +42,7 @@ The tool will try to download and cache locally referenced schemas (e.g. `git@0b Cached files are stored in a dedicated directory - `~/.elastic-package/cache/fields/`. It's assumed that schema (versioned) files do not change. -To verify if building process went well, you can open `build` directory and compare fields (e.g. `./build/integrations/nginx/1.2.3/access/fields/ecs.yml`): +To verify if building process went well, you can open `build` directory and compare fields (e.g. `./build/packages/nginx/1.2.3/access/fields/ecs.yml`): ```yaml - description: |- diff --git a/internal/builder/packages.go b/internal/builder/packages.go index e936d95a6c..7f34c48bfa 100644 --- a/internal/builder/packages.go +++ b/internal/builder/packages.go @@ -9,15 +9,16 @@ import ( "os" "path/filepath" - "github.com/elastic/package-spec/code/go/pkg/validator" "github.com/pkg/errors" + "github.com/elastic/package-spec/code/go/pkg/validator" + "github.com/elastic/elastic-package/internal/files" "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/packages" ) -const buildIntegrationsFolder = "integrations" +const builtPackagesFolder = "packages" type BuildOptions struct { PackageRoot string @@ -102,7 +103,7 @@ func buildPackagesRootDirectory() (string, error) { return "", errors.Wrap(err, "can't locate build directory") } if !found { - buildDir, err = createBuildDirectory(buildIntegrationsFolder) // TODO add support for other package types + buildDir, err = createBuildDirectory(builtPackagesFolder) if err != nil { return "", errors.Wrap(err, "can't create new build directory") } @@ -118,7 +119,7 @@ func FindBuildPackagesDirectory() (string, bool, error) { } if found { - path := filepath.Join(buildDir, buildIntegrationsFolder) // TODO add support for other package types + path := filepath.Join(buildDir, builtPackagesFolder) fileInfo, err := os.Stat(path) if errors.Is(err, os.ErrNotExist) { return "", false, nil diff --git a/internal/packages/packages.go b/internal/packages/packages.go index 72c4d40a57..87dbf25752 100644 --- a/internal/packages/packages.go +++ b/internal/packages/packages.go @@ -85,7 +85,12 @@ type Conditions struct { // PolicyTemplate is a configuration of inputs responsible for collecting log or metric data. type PolicyTemplate struct { - Inputs []Input `config:"inputs" json:"inputs" yaml:"inputs"` + Inputs []Input `config:"inputs,omitempty" json:"inputs,omitempty" yaml:"inputs,omitempty"` + + // For purposes of "input packages" + Input string `config:"input,omitempty" json:"input,omitempty" yaml:"input,omitempty"` + Type string `config:"type,omitempty" json:"type,omitempty" yaml:"type,omitempty"` + TemplatePath string `config:"template_path,omitempty" json:"template_path,omitempty" yaml:"template_path,omitempty"` } // Owner defines package owners, either a single person or a team. @@ -273,7 +278,7 @@ func isPackageManifest(path string) (bool, error) { if err != nil { return false, errors.Wrapf(err, "reading package manifest failed (path: %s)", path) } - return m.Type == "integration" && m.Version != "", nil // TODO add support for other package types + return (m.Type == "integration" || m.Type == "input") && m.Version != "", nil } func isDataStreamManifest(path string) (bool, error) { diff --git a/scripts/test-build-zip.sh b/scripts/test-build-zip.sh index ab2bde529e..2287f01a43 100755 --- a/scripts/test-build-zip.sh +++ b/scripts/test-build-zip.sh @@ -38,7 +38,7 @@ done cd - # Remove unzipped built packages, leave .zip files -rm -r build/integrations/*/ +rm -r build/packages/*/ # Boot up the stack eval "$(elastic-package stack shellinit)" diff --git a/test/packages/other/sql_input/agent/input/input.yml.hbs b/test/packages/other/sql_input/agent/input/input.yml.hbs new file mode 100644 index 0000000000..6eb03670bf --- /dev/null +++ b/test/packages/other/sql_input/agent/input/input.yml.hbs @@ -0,0 +1,9 @@ +metricsets: ["query"] +period: {{period}} +hosts: +{{#each hosts}} + - {{this}} +{{/each}} +driver: {{driver}} +sql_query: {{sql_query} +sql_response_format: {{sql_response_format}} \ No newline at end of file diff --git a/test/packages/other/sql_input/changelog.yml b/test/packages/other/sql_input/changelog.yml new file mode 100644 index 0000000000..37b3d9d3f6 --- /dev/null +++ b/test/packages/other/sql_input/changelog.yml @@ -0,0 +1,6 @@ +# newer versions go on top +- version: "0.2.0" + changes: + - description: Initial draft of the package + type: enhancement + link: https://github.com/elastic/package-spec/pull/325 diff --git a/test/packages/other/sql_input/docs/README.md b/test/packages/other/sql_input/docs/README.md new file mode 100644 index 0000000000..2023396ba2 --- /dev/null +++ b/test/packages/other/sql_input/docs/README.md @@ -0,0 +1 @@ +# SQL Input \ No newline at end of file diff --git a/test/packages/other/sql_input/fields/input.yml b/test/packages/other/sql_input/fields/input.yml new file mode 100644 index 0000000000..fe6b1af669 --- /dev/null +++ b/test/packages/other/sql_input/fields/input.yml @@ -0,0 +1,4 @@ +- name: input.name + type: constant_keyword + description: Sample field to be added. + value: logs diff --git a/test/packages/other/sql_input/img/sample-logo.svg b/test/packages/other/sql_input/img/sample-logo.svg new file mode 100644 index 0000000000..6268dd88f3 --- /dev/null +++ b/test/packages/other/sql_input/img/sample-logo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/test/packages/other/sql_input/img/sample-screenshot.png b/test/packages/other/sql_input/img/sample-screenshot.png new file mode 100644 index 0000000000..d7a56a3ecc Binary files /dev/null and b/test/packages/other/sql_input/img/sample-screenshot.png differ diff --git a/test/packages/other/sql_input/manifest.yml b/test/packages/other/sql_input/manifest.yml new file mode 100644 index 0000000000..49540cc9ec --- /dev/null +++ b/test/packages/other/sql_input/manifest.yml @@ -0,0 +1,67 @@ +format_version: 1.0.0 +name: sql_input +title: SQL Input +description: >- + Execute custom queries against an SQL database and store the results in Elasticsearch. +type: input +version: 0.2.0 +license: basic +categories: + - custom + - datastore +policy_templates: + - name: sql_query + type: metrics + title: SQL Query + description: Query the database to capture metrics. + input: sql + template_path: input.yml.hbs + vars: + - name: hosts + type: text + title: Hosts + multi: true + required: true + show_user: true + default: + - http://127.0.0.1 + - name: period + type: text + title: Period + multi: false + required: true + show_user: true + default: 10s + - name: driver + type: text + title: Driver + description: "Supported database drivers: mssql, mysql, oracle, postgres" + multi: false + required: true + show_user: true + default: "mysql" + - name: sql_query + type: text + title: Query + multi: false + required: true + show_user: true + default: "SHOW GLOBAL STATUS LIKE 'Innodb_system%'" + - name: sql_response_format + type: text + title: Response format + description: "Supported response formats: variables, table" + multi: false + required: true + show_user: false + default: "variables" +icons: + - src: "/img/sample-logo.svg" + type: "image/svg+xml" +screenshots: + - src: "/img/sample-screenshot.png" + title: "Sample screenshot" + size: "600x600" + type: "image/png" +owner: + github: elastic/integrations diff --git a/test/packages/parallel/gcp/data_stream/compute/fields/agent.yml b/test/packages/parallel/gcp/data_stream/compute/fields/agent.yml index 2a31d79f49..8e686410af 100644 --- a/test/packages/parallel/gcp/data_stream/compute/fields/agent.yml +++ b/test/packages/parallel/gcp/data_stream/compute/fields/agent.yml @@ -5,49 +5,11 @@ footnote: 'Examples: If Metricbeat is running on an GCP Compute VM and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - name: instance.name level: extended type: keyword ignore_above: 1024 description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - name: project.id type: keyword description: Name of the project in Google Cloud. diff --git a/test/packages/parallel/gcp/docs/compute.md b/test/packages/parallel/gcp/docs/compute.md index c3c4b42f54..1037395d81 100644 --- a/test/packages/parallel/gcp/docs/compute.md +++ b/test/packages/parallel/gcp/docs/compute.md @@ -101,14 +101,14 @@ An example event for `compute` looks as following: | cloud | Fields related to the cloud or infrastructure the events are coming from. | group | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | | cloud.instance.id | Instance ID of the host machine. | keyword | | cloud.instance.name | Instance name of the host machine. | keyword | | cloud.machine.type | Machine type of the host machine. | keyword | | cloud.project.id | Name of the project in Google Cloud. | keyword | | cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | +| cloud.region | Region in which this host, resource, or service is located. | keyword | | container.id | Unique container id. | keyword | | container.image.name | Name of the image the container was built on. | keyword | | container.labels | Image labels. | object |