From c142bfd6f2bb9ce48b0b1dd47473d91c21c2b247 Mon Sep 17 00:00:00 2001 From: Tim Pavlik Date: Tue, 26 Oct 2021 11:58:13 -0700 Subject: [PATCH 1/5] Move modinputs to bin, cleanup setup.py imports --- examples/random_numbers/{ => bin}/random_numbers.py | 0 setup.py | 4 ---- 2 files changed, 4 deletions(-) rename examples/random_numbers/{ => bin}/random_numbers.py (100%) diff --git a/examples/random_numbers/random_numbers.py b/examples/random_numbers/bin/random_numbers.py similarity index 100% rename from examples/random_numbers/random_numbers.py rename to examples/random_numbers/bin/random_numbers.py diff --git a/setup.py b/setup.py index 93540373..284c5098 100755 --- a/setup.py +++ b/setup.py @@ -15,13 +15,9 @@ # under the License. from setuptools import setup, Command -from contextlib import closing -from subprocess import check_call, STDOUT import os import sys -import shutil -import tarfile import splunklib From 5c8b2565f684bfe212de57b1a55ee0e56221d752 Mon Sep 17 00:00:00 2001 From: Tim Pavlik Date: Tue, 26 Oct 2021 12:01:14 -0700 Subject: [PATCH 2/5] Move mod inputs to bin, add splunklib dependency --- docker-compose.yml | 2 ++ examples/github_forks/{ => bin}/github_forks.py | 0 2 files changed, 2 insertions(+) rename examples/github_forks/{ => bin}/github_forks.py (100%) diff --git a/docker-compose.yml b/docker-compose.yml index a93a14c0..6885cfd5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,7 +11,9 @@ services: - SPLUNK_APPS_URL=https://github.com/splunk/sdk-app-collection/releases/download/v1.1.0/sdkappcollection.tgz volumes: - ./examples/github_forks:/opt/splunk/etc/apps/github_forks + - ./splunklib:/opt/splunk/etc/apps/github_forks/lib/splunklib - ./examples/random_numbers:/opt/splunk/etc/apps/random_numbers + - ./splunklib:/opt/splunk/etc/apps/random_numbers/lib/splunklib - ./examples/searchcommands_app/package:/opt/splunk/etc/apps/searchcommands_app - ./splunklib:/opt/splunk/etc/apps/searchcommands_app/lib/splunklib - ./examples/twitted/twitted:/opt/splunk/etc/apps/twitted diff --git a/examples/github_forks/github_forks.py b/examples/github_forks/bin/github_forks.py similarity index 100% rename from examples/github_forks/github_forks.py rename to examples/github_forks/bin/github_forks.py From 8da1679bfcb18238ad8b67e34151d2377b259f86 Mon Sep 17 00:00:00 2001 From: Tim Pavlik Date: Tue, 26 Oct 2021 12:26:23 -0700 Subject: [PATCH 3/5] Random numbers mod input example working --- examples/random_numbers/README.md | 8 ++++++++ examples/random_numbers/bin/random_numbers.py | 4 ++++ 2 files changed, 12 insertions(+) create mode 100644 examples/random_numbers/README.md diff --git a/examples/random_numbers/README.md b/examples/random_numbers/README.md new file mode 100644 index 00000000..90172dff --- /dev/null +++ b/examples/random_numbers/README.md @@ -0,0 +1,8 @@ +splunk-sdk-python random_numbers example +======================================== + +This app provides an example of a modular input that generates a random number between the min and max values provided by the user during setup of the input. + +To run this example locally run `SPLUNK_VERSION=latest docker compose up -d` from the root of this repository which will mount this example alongside the latest version of splunklib within `/opt/splunk/etc/apps/random_numbers` and `/opt/splunk/etc/apps/random_numbers/lib/splunklib` within the `splunk` container. + +Once the docker container is up and healthy log into the Splunk UI and setup a new `Random Numbers` input by visiting this page: http://localhost:8000/en-US/manager/random_numbers/datainputstats and selecting the "Add new..." button next to the Local Inputs > Random Inputs. If no Random Numbers input appears then the script is likely not running properly, see https://docs.splunk.com/Documentation/SplunkCloud/latest/AdvancedDev/ModInputsDevTools for more details on debugging the modular input using the command line and relevant logs. \ No newline at end of file diff --git a/examples/random_numbers/bin/random_numbers.py b/examples/random_numbers/bin/random_numbers.py index f0727f0d..b9673db9 100755 --- a/examples/random_numbers/bin/random_numbers.py +++ b/examples/random_numbers/bin/random_numbers.py @@ -17,6 +17,10 @@ from __future__ import absolute_import import random, sys import os +# NOTE: splunklib must exist within random_numbers/lib/splunklib for this +# example to run! To run this locally use `SPLUNK_VERSION=latest docker compose up -d` +# from the root of this repo which mounts this example and the latest splunklib +# code together at /opt/splunk/etc/apps/random_numbers sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "lib")) from splunklib.modularinput import * From d9e7044dbf817d79acd35f54d90d35eba2c7272d Mon Sep 17 00:00:00 2001 From: Tim Pavlik Date: Tue, 26 Oct 2021 16:55:03 -0700 Subject: [PATCH 4/5] Fix github_forks example. --- examples/github_forks/README.md | 12 ++++++++ examples/github_forks/bin/github_forks.py | 35 +++++++++++++++++------ examples/random_numbers/README.md | 6 +++- 3 files changed, 44 insertions(+), 9 deletions(-) create mode 100644 examples/github_forks/README.md diff --git a/examples/github_forks/README.md b/examples/github_forks/README.md new file mode 100644 index 00000000..6ba51ba6 --- /dev/null +++ b/examples/github_forks/README.md @@ -0,0 +1,12 @@ +splunk-sdk-python github_forks example +======================================== + +This app provides an example of a modular input that generates a random number between the min and max values provided by the user during setup of the input. + +To run this example locally run `SPLUNK_VERSION=latest docker compose up -d` from the root of this repository which will mount this example alongside the latest version of splunklib within `/opt/splunk/etc/apps/github_forks` and `/opt/splunk/etc/apps/github_forks/lib/splunklib` within the `splunk` container. + +Once the docker container is up and healthy log into the Splunk UI and setup a new `Github Repository Forks` input by visiting this page: http://localhost:8000/en-US/manager/github_forks/datainputstats and selecting the "Add new..." button next to the Local Inputs > Random Inputs. Enter values for a Github Repository owner and repo_name, for example owner = `splunk` repo_name = `splunk-sdk-python`. + +NOTE: If no Github Repository Forks input appears then the script is likely not running properly, see https://docs.splunk.com/Documentation/SplunkCloud/latest/AdvancedDev/ModInputsDevTools for more details on debugging the modular input using the command line and relevant logs. + +Once the input is created you should be able to see an event when running the following search: `source="github_forks://*"` the event should contain fields for `owner` and `repository` matching the values you input during setup and then a `fork_count` field corresponding to the number of forks the repo has according to the Github API. \ No newline at end of file diff --git a/examples/github_forks/bin/github_forks.py b/examples/github_forks/bin/github_forks.py index 2349bd68..5ffa4e40 100755 --- a/examples/github_forks/bin/github_forks.py +++ b/examples/github_forks/bin/github_forks.py @@ -15,10 +15,18 @@ # under the License. from __future__ import absolute_import -import sys, urllib2, json +import os +import sys +import json +# NOTE: splunklib must exist within github_forks/lib/splunklib for this +# example to run! To run this locally use `SPLUNK_VERSION=latest docker compose up -d` +# from the root of this repo which mounts this example and the latest splunklib +# code together at /opt/splunk/etc/apps/github_forks +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "lib")) from splunklib.modularinput import * from splunklib import six +from six.moves import http_client class MyScript(Script): """All modular inputs should inherit from the abstract base class Script @@ -87,11 +95,9 @@ def validate_input(self, validation_definition): # Get the values of the parameters, and construct a URL for the Github API owner = validation_definition.parameters["owner"] repo_name = validation_definition.parameters["repo_name"] - repo_url = "https://api.github.com/repos/%s/%s" % (owner, repo_name) - # Read the response from the Github API, then parse the JSON data into an object - response = urllib2.urlopen(repo_url).read() - jsondata = json.loads(response) + # Call Github to retrieve repo information + jsondata = _get_github_repos(owner, repo_name) # If there is only 1 field in the jsondata object,some kind or error occurred # with the Github API. @@ -125,9 +131,7 @@ def stream_events(self, inputs, ew): repo_name = input_item["repo_name"] # Get the fork count from the Github API - repo_url = "https://api.github.com/repos/%s/%s" % (owner, repo_name) - response = urllib2.urlopen(repo_url).read() - jsondata = json.loads(response) + jsondata = _get_github_repos(owner, repo_name) fork_count = jsondata["forks_count"] # Create an Event object, and set its fields @@ -139,5 +143,20 @@ def stream_events(self, inputs, ew): # Tell the EventWriter to write this event ew.write_event(event) + +def _get_github_repos(owner, repo_name): + # Read the response from the Github API, then parse the JSON data into an object + repo_path = "/repos/%s/%s" % (owner, repo_name) + connection = http_client.HTTPSConnection('api.github.com') + headers = { + 'Content-type': 'application/json', + 'User-Agent': 'splunk-sdk-python', + } + connection.request('GET', repo_path, headers=headers) + response = connection.getresponse() + body = response.read().decode() + return json.loads(body) + + if __name__ == "__main__": sys.exit(MyScript().run(sys.argv)) diff --git a/examples/random_numbers/README.md b/examples/random_numbers/README.md index 90172dff..7ff4069f 100644 --- a/examples/random_numbers/README.md +++ b/examples/random_numbers/README.md @@ -5,4 +5,8 @@ This app provides an example of a modular input that generates a random number b To run this example locally run `SPLUNK_VERSION=latest docker compose up -d` from the root of this repository which will mount this example alongside the latest version of splunklib within `/opt/splunk/etc/apps/random_numbers` and `/opt/splunk/etc/apps/random_numbers/lib/splunklib` within the `splunk` container. -Once the docker container is up and healthy log into the Splunk UI and setup a new `Random Numbers` input by visiting this page: http://localhost:8000/en-US/manager/random_numbers/datainputstats and selecting the "Add new..." button next to the Local Inputs > Random Inputs. If no Random Numbers input appears then the script is likely not running properly, see https://docs.splunk.com/Documentation/SplunkCloud/latest/AdvancedDev/ModInputsDevTools for more details on debugging the modular input using the command line and relevant logs. \ No newline at end of file +Once the docker container is up and healthy log into the Splunk UI and setup a new `Random Numbers` input by visiting this page: http://localhost:8000/en-US/manager/random_numbers/datainputstats and selecting the "Add new..." button next to the Local Inputs > Random Inputs. Enter values for the `min` and `max` values which the random number should be generated between. + +NOTE: If no Random Numbers input appears then the script is likely not running properly, see https://docs.splunk.com/Documentation/SplunkCloud/latest/AdvancedDev/ModInputsDevTools for more details on debugging the modular input using the command line and relevant logs. + +Once the input is created you should be able to see an event when running the following search: `source="random_numbers://*"` the event should contain a `number` field with a float between the min and max specified when the input was created. \ No newline at end of file From f93129fa44d540ad1efcab62bdced525caddb8c9 Mon Sep 17 00:00:00 2001 From: Tim Pavlik Date: Tue, 26 Oct 2021 16:58:07 -0700 Subject: [PATCH 5/5] Fix description --- examples/github_forks/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/github_forks/README.md b/examples/github_forks/README.md index 6ba51ba6..1a05c862 100644 --- a/examples/github_forks/README.md +++ b/examples/github_forks/README.md @@ -1,7 +1,7 @@ splunk-sdk-python github_forks example ======================================== -This app provides an example of a modular input that generates a random number between the min and max values provided by the user during setup of the input. +This app provides an example of a modular input that generates the number of repository forks according to the Github API based on the owner and repo_name provided by the user during setup of the input. To run this example locally run `SPLUNK_VERSION=latest docker compose up -d` from the root of this repository which will mount this example alongside the latest version of splunklib within `/opt/splunk/etc/apps/github_forks` and `/opt/splunk/etc/apps/github_forks/lib/splunklib` within the `splunk` container.