Skip to content

Commit

Permalink
feat: add github_repository_content table
Browse files Browse the repository at this point in the history
  • Loading branch information
@aminvielledebatAtBedrock
aminvielledebatAtBedrock committed Oct 5, 2022
1 parent 150a7e5 commit 96cfadc
Show file tree
Hide file tree
Showing 3 changed files with 234 additions and 0 deletions.
134 changes: 134 additions & 0 deletions docs/tables/github_repository_content.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
# Table: github_repository_content

Gets the contents of a file or directory in a repository.

Specify the file path or directory in `repository_content_path`.
If you omit `repository_content_path`, you will receive the contents of the repository's root directory.
See the description below regarding what the response includes for directories.

The `github_repository_content` table can be used to query information about **ANY** repository, and **you must specify which repository** in the where or join clause (`where repository_full_name=`, `join github_repository_content on repository_full_name=`).

NOTE : file content is only available when the filter field `repository_content_path` is equal to your file path in the repository.

## Examples

### List a repository

```sql
select
repository_full_name,
path,
type,
size,
sha,
html_url
from
github_public.github_repository_content
where
repository_full_name = 'github/docs'
```

**ResultSet**

| repository\_full\_name | path | type | size | sha | html\_url |
| ---------------------- | -------------------------------- | ---- | ------- | ---------------------------------------- | ------------------------------------------------------------------------- |
| github/docs | .dockerignore | file | 353 | aa9b89a06425fc2f21f673e976ff721365892ea5 | https://github.com/github/docs/blob/main/.dockerignore |
| github/docs | .devcontainer | dir | 0 | 6a929511b06913dbc123320eb0ed5b880a7cedc5 | https://github.com/github/docs/tree/main/.devcontainer |
| github/docs | codespaces-settings.json | file | 19 | 840c49b326de81028baeced9678851ec46f5be02 | https://github.com/github/docs/blob/main/codespaces-settings.json |
| github/docs | tsconfig.json | file | 641 | beedb054164da3b1ada92f449aee69b8a0a3013e | https://github.com/github/docs/blob/main/tsconfig.json |
| github/docs | LICENSE-CODE | file | 1059 | c9802266ce3239fc222b1c803308535712b2e3be | https://github.com/github/docs/blob/main/LICENSE-CODE |
| github/docs | jest.config.js | file | 1464 | 404781c32232722730480118de89b8dc8e0125f4 | https://github.com/github/docs/blob/main/jest.config.js |
| github/docs | data | dir | 0 | 569ebd957ef4dfacad0bf26c552db7d36cbbc017 | https://github.com/github/docs/tree/main/data |
| github/docs | .babelrc | file | 68 | 34abf3fffeb8d879e40057d78d1f4d639c252bed | https://github.com/github/docs/blob/main/.babelrc |
| github/docs | README.md | file | 3345 | fbbaf2e15695e62c7f8e992b70dd67f95260a2a9 | https://github.com/github/docs/blob/main/README.md |
| github/docs | jest.setup.js | file | 487 | a7f994411e5dbaa8d880b0b1de0fed0411d280df | https://github.com/github/docs/blob/main/jest.setup.js |
| github/docs | docker-compose.prod.tmpl.yaml | file | 870 | 0c986eca9312d4ef114c1f388db2be28415bdc5a | https://github.com/github/docs/blob/main/docker-compose.prod.tmpl.yaml |
| github/docs | .prettierrc.json | file | 357 | a76d08f0287d26d78a8215b1880b8f0c218ea84a | https://github.com/github/docs/blob/main/.prettierrc.json |
| github/docs | components | dir | 0 | eac3eccf4214887e41cbb012ea4a7a621e8a235c | https://github.com/github/docs/tree/main/components |
| github/docs | .prettierignore | file | 85 | 80eac85c27be1e4df71d9be9eb39f0381026f178 | https://github.com/github/docs/blob/main/.prettierignore |
| github/docs | .github | dir | 0 | c9a5878760cee1010de687606243511d408c35f7 | https://github.com/github/docs/tree/main/.github |
| github/docs | docker-compose.staging.tmpl.yaml | file | 849 | 0ca9b2d0fae1606db13fd77803cd5bb6e0aacf6b | https://github.com/github/docs/blob/main/docker-compose.staging.tmpl.yaml |
| github/docs | assets | dir | 0 | 235e877b2c731969eb9cf62e852c254973ca6507 | https://github.com/github/docs/tree/main/assets |
| github/docs | lib | dir | 0 | 6b63b0503e89fd7008ffb80d7f4f313a3474af11 | https://github.com/github/docs/tree/main/lib |
| github/docs | .vscode | dir | 0 | 1242eaf1a1d4b73a04fa39170e718cc4715ec4a0 | https://github.com/github/docs/tree/main/.vscode |
| github/docs | content | dir | 0 | 2cebbc04564c9c3301c0fa84a0e4099ecbd3b11d | https://github.com/github/docs/tree/main/content |
| github/docs | .editorconfig | file | 171 | beffa3084e7a693343fff8701f413ec3efa87b24 | https://github.com/github/docs/blob/main/.editorconfig |
| github/docs | .gitignore | file | 614 | d610611db9497f61f6a28d4981def15f75130b0d | https://github.com/github/docs/blob/main/.gitignore |
| github/docs | docker-compose.yaml | file | 141 | 68470e45a7ff46af90f07e549e4e416eab721011 | https://github.com/github/docs/blob/main/docker-compose.yaml |
| github/docs | azure-preview-env-template.json | file | 2800 | f63f318358de73ed720aeb1b17c3827bbf99ab1a | https://github.com/github/docs/blob/main/azure-preview-env-template.json |
| github/docs | middleware | dir | 0 | 54ac42077acdeab7230ee179589c235cebef03a0 | https://github.com/github/docs/tree/main/middleware |
| github/docs | CODE\_OF\_CONDUCT.md | file | 5363 | e66f6d941d8c806dde598112670761e8ad205f99 | https://github.com/github/docs/blob/main/CODE\_OF\_CONDUCT.md |
| github/docs | .env.example | file | 27 | 983c975bb3680a2da59ccfd34b53c5c96bba4520 | https://github.com/github/docs/blob/main/.env.example |
| github/docs | docs | dir | 0 | 743da72211bf44ab6c1c7b0868bafdb41bb3415b | https://github.com/github/docs/tree/main/docs |
| github/docs | feature-flags.json | file | 63 | 64c69f19d789c3f681400959dc1b3985bd9edf9e | https://github.com/github/docs/blob/main/feature-flags.json |
| github/docs | next-env.d.ts | file | 201 | 4f11a03dc6cc37f2b5105c08f2e7b24c603ab2f4 | https://github.com/github/docs/blob/main/next-env.d.ts |
| github/docs | .husky | dir | 0 | 99cc69a3f9a2c9606eaff8ef21c81a37857737d4 | https://github.com/github/docs/tree/main/.husky |
| github/docs | jest-puppeteer.config.cjs | file | 109 | a66133dd35ea74a0afc69c34da4bbce8052684d4 | https://github.com/github/docs/blob/main/jest-puppeteer.config.cjs |
| github/docs | .eslintrc.cjs | file | 1061 | 2e7e063533be52faaa54efd855f9ef5a57e97bba | https://github.com/github/docs/blob/main/.eslintrc.cjs |
| github/docs | CONTRIBUTING.md | file | 7797 | babc51f666d5d0d361944e2901616a03d2fbe051 | https://github.com/github/docs/blob/main/CONTRIBUTING.md |
| github/docs | .gitattributes | file | 267 | 94f95b93826d614321f8db400bf27e0ec84c6283 | https://github.com/github/docs/blob/main/.gitattributes |
| github/docs | next.config.js | file | 1832 | 50aa6fcec002869b3635c1a046609674d688abb1 | https://github.com/github/docs/blob/main/next.config.js |
| github/docs | LICENSE | file | 18620 | 9238c8f9388066fe7cb3b308de35104bb3c9596b | https://github.com/github/docs/blob/main/LICENSE |
| github/docs | .npmrc | file | 105 | 9b816f6d44a58c89ae93cebddd940208422b0894 | https://github.com/github/docs/blob/main/.npmrc |
| github/docs | Dockerfile.openapi\_decorator | file | 525 | 6014681b41da5fd88a60cc72d502ba46ca961730 | https://github.com/github/docs/blob/main/Dockerfile.openapi\_decorator |
| github/docs | script | dir | 0 | c76e0e28601f650c498482594499fc01e480f62c | https://github.com/github/docs/tree/main/script |
| github/docs | .node-version | file | 6 | 1c2b43343f0b94485b880b21d7cb9c6ee8593b8d | https://github.com/github/docs/blob/main/.node-version |
| github/docs | contributing | dir | 0 | dcee2b717757136f5c5554ee2d9528962902adf4 | https://github.com/github/docs/tree/main/contributing |
| github/docs | package.json | file | 7842 | 43f22e9aa7d7831115b17e63e3cc8581486be110 | https://github.com/github/docs/blob/main/package.json |
| github/docs | nodemon.json | file | 153 | e5c16a7537db31b3f97b02eab64574af6ff74bb9 | https://github.com/github/docs/blob/main/nodemon.json |
| github/docs | package-lock.json | file | 1141381 | a871e8fa9175668edc875fd086f8d83cfb161c00 | https://github.com/github/docs/blob/main/package-lock.json |
| github/docs | stylesheets | dir | 0 | 61229708c49cd5733becd8907516f8582f8b70a8 | https://github.com/github/docs/tree/main/stylesheets |
| github/docs | pages | dir | 0 | c1fa0161d0b55f41c0316ff69d5d3fe5d576d923 | https://github.com/github/docs/tree/main/pages |
| github/docs | tests | dir | 0 | be635b1a62e00e23077a22e0cc2aa3005a663518 | https://github.com/github/docs/tree/main/tests |
| github/docs | server.js | file | 49 | a48da58794021cf7db6f7b9ee270c657be6adcf9 | https://github.com/github/docs/blob/main/server.js |
| github/docs | translations | dir | 0 | c2f6dc20ce63483cab222e1d9e8b922dae06807e | https://github.com/github/docs/tree/main/translations |
| github/docs | Dockerfile | file | 2989 | cb9dcc3ad98d6ea9c4112ed7904604b828087578 | https://github.com/github/docs/blob/main/Dockerfile |
| github/docs | ownership.yaml | file | 727 | 8bcc8188283f89caa6eea115461a6122acb6c8a8 | https://github.com/github/docs/blob/main/ownership.yaml |
| github/docs | start-server.js | file | 1801 | 99f4872fbbbd786a0b8fe3d1fcc38fe3d875bfa8 | https://github.com/github/docs/blob/main/start-server.js |+----------------------+----------------------------------+------+---------+------------------------------------------+---------------------------------------------------------------------------+

### List a directory in a repository

```sql
select
repository_full_name,
path,
type,
size,
sha,
html_url
from
github_public.github_repository_content
where
repository_full_name = 'github/docs'
and repository_content_path = 'docs'
```

**ResultSet**

| repository_full_name | path | type | size | sha | html_url |
|----------------------|-----------------|------|------|------------------------------------------|----------------------------------------------------------|
| github/docs | docs/index.yaml | file | 1963 | 91461a8c9f53fd0bc1b3b278cdfd5b262ac2c343 | https://github.com/github/docs/blob/main/docs/index.yaml |

### Get a file in a repository

```sql
select
repository_full_name,
path,
type,
size,
sha,
content,
html_url
from
github_public.github_repository_content
where
repository_full_name = 'github/docs'
and repository_content_path = '.vscode/settings.json'
```

**ResultSet**

| repository\_full\_name | path | type | size | sha | content | html\_url |
| ---------------------- | --------------------- | ---- | ---- | ---------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------- |
| github/docs | .vscode/settings.json | file | 156 | d331b10571f739b2298c95870acc0ef22026a15a | {<br>"files.exclude": {<br>"translations/\*\*": true<br>},<br>"workbench.editor.enablePreview": false,<br>"workbench.editor.enablePreviewFromQuickOpen": false<br>} | https://github.com/github/docs/blob/main/.vscode/settings.json |
1 change: 1 addition & 0 deletions github/plugin.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ func Plugin(ctx context.Context) *plugin.Plugin {
"github_rate_limit": tableGitHubRateLimit(ctx),
"github_release": tableGitHubRelease(ctx),
"github_repository": tableGitHubRepository(),
"github_repository_content": tableGitHubRepositoryContent(),
"github_search_code": tableGitHubSearchCode(ctx),
"github_search_commit": tableGitHubSearchCommit(ctx),
"github_search_issue": tableGitHubSearchIssue(ctx),
Expand Down
99 changes: 99 additions & 0 deletions github/table_github_repository_content.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
package github

import (
"context"
"github.com/google/go-github/v45/github"
"github.com/turbot/steampipe-plugin-sdk/v4/grpc/proto"
"github.com/turbot/steampipe-plugin-sdk/v4/plugin"
"github.com/turbot/steampipe-plugin-sdk/v4/plugin/transform"
)

//// TABLE DEFINITION

func tableGitHubRepositoryContent() *plugin.Table {
return &plugin.Table{
Name: "github_repository_content",
Description: "List the content in a repository (list directory, or get file content",
List: &plugin.ListConfig{
Hydrate: tableGitHubRepositoryContentList,
KeyColumns: []*plugin.KeyColumn{
{Name: "repository_full_name", Require: plugin.Required},
{Name: "repository_content_path", Require: plugin.Optional, CacheMatch: "exact"},
},
},
Columns: []*plugin.Column{
{Name: "repository_full_name", Description: "The full name of the repository (login/repo-name).", Type: proto.ColumnType_STRING, Transform: transform.FromQual("repository_full_name")},
{Name: "type", Description: "The file type (directory or file).", Type: proto.ColumnType_STRING},
{Name: "name", Description: "The file name.", Type: proto.ColumnType_STRING},
{Name: "repository_content_path", Description: "The requested path in repository search.", Type: proto.ColumnType_STRING, Transform: transform.FromQual("repository_content_path")},
{Name: "path", Description: "The path of the file.", Type: proto.ColumnType_STRING},
{Name: "size", Description: "The size of the file.", Type: proto.ColumnType_INT},
{Name: "content", Description: "The decoded file content (if the element is a file).", Type: proto.ColumnType_STRING, Transform: transform.FromMethod("GetContent")},
{Name: "target", Description: "Target is only set if the type is \"symlink\" and the target is not a normal file. If Target is set, Path will be the symlink path.", Type: proto.ColumnType_STRING},
{Name: "sha", Description: "The sha of the file.", Type: proto.ColumnType_STRING, Transform: transform.FromField("SHA")},
{Name: "url", Description: "Url of file's metadata", Type: proto.ColumnType_STRING},
{Name: "git_url", Description: "Git url (with SHA) of the file", Type: proto.ColumnType_STRING},
{Name: "html_url", Description: "Raw file url in GitHub", Type: proto.ColumnType_STRING},
{Name: "download_url", Description: "Download URL : it expires and can be be used just once.", Type: proto.ColumnType_STRING},
},
}
}

//// LIST FUNCTION

func tableGitHubRepositoryContentList(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) {
owner, repo := parseRepoFullName(d.KeyColumnQuals["repository_full_name"].GetStringValue())
var filterPath string
if d.KeyColumnQuals["repository_content_path"] != nil {
filterPath = d.KeyColumnQuals["repository_content_path"].GetStringValue()
}
plugin.Logger(ctx).Trace("tableGitHubRepositoryContentList", "owner", owner, "repo", repo, "path", filterPath)

type ListPageResponse struct {
repositoryContent []*github.RepositoryContent
resp *github.Response
}
client := connect(ctx, d)
opt := &github.RepositoryContentGetOptions{}
listPage := func(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) {
fileContent, directoryContent, resp, err := client.Repositories.GetContents(ctx, owner, repo, filterPath, opt)

if err != nil {
plugin.Logger(ctx).Error("tableGitHubRepositoryContentList", "api_error", err, "path", filterPath)
return nil, err
}

if fileContent != nil {
directoryContent = []*github.RepositoryContent{fileContent}
}

return ListPageResponse{
repositoryContent: directoryContent,
resp: resp,
}, err
}

for {
listPageResponse, err := retryHydrate(ctx, d, h, listPage)
if err != nil {
plugin.Logger(ctx).Error("tableGitHubRepositoryContentList", "retry_hydrate_error", err)
return nil, err
}

for _, i := range listPageResponse.(ListPageResponse).repositoryContent {
if i != nil {
d.StreamListItem(ctx, i)
}

// Context can be cancelled due to manual cancellation or the limit has been hit
if d.QueryStatus.RowsRemaining(ctx) == 0 {
return nil, nil
}
}

if listPageResponse.(ListPageResponse).resp.NextPage == 0 {
break
}
}
return nil, nil
}

0 comments on commit 96cfadc

Please sign in to comment.