Detect proprietary toolchain (#42)

.github/workflows/deploy.yml

@@ -33,6 +33,8 @@ jobs:
           bundle config path vendor/bundle
           bundle install --jobs 4 --retry 3
           bundle exec jekyll build --trace
+        with:
+          GITHUB_TOKEN : ${{ secrets.GITHUB_TOKEN }}
 
       - name: Upload to GitHub Pages
         uses: JamesIves/github-pages-deploy-action@releases/v3

README.md

@@ -10,6 +10,11 @@ Jekyll is a static site generator that takes markdown files and transforms them
 To run Jekyll locally, install `ruby` `>=2.5.0` and the `bundler` gem (`gem install bundler`).
 Then install dependencies by running `bundle install` and build the site by calling `bundle exec jekyll serve --livereload`.
 
+We use the GitHub API to fetch a list of projects using Padauk µCs and the latest activity in the `free-pdk` organization.
+The API has a rate limit of 60 requests per hour for unauthenticated requests, which may not be sufficient for the amount of API requests we make when building the website.
+Please set a `GITHUB_TOKEN` environment variable with a personal access token [you can create here (no scopes necessary)](https://github.com/settings/tokens).
+If you don't set a `GITHUB_TOKEN` environment variable, we automatically make less requests to the API, but some features like detecting projects using Padauk's proprietary toolchain will be skipped.
+
 ## Deployment
 
 Every commit on the `production` branch is built and deployed by a [GitHub Action](https://github.com/free-pdk/free-pdk.github.io/actions) and the result is force-pushed to the `master` branch, which is deployed to https://free-pdk.github.io by GitHub Pages.

_includes/community_projects.html

@@ -1,27 +1,37 @@
 <div markdown="0" class="community-projects">
 {% for project in site.projects_using_padauk %}
+{% assign repo = project.data %}
 <div>
-  <a class="card" href="{{ project.html_url }}">
+  <a class="card" href="{{ repo.html_url }}">
     <h4>
-      <span class="owner">{{ project.owner.login | escape }}</span>
+      <span class="owner">{{ repo.owner.login | escape }}</span>
       <br />
-      <strong>{{ project.name | escape }}</strong>
+      <strong>{{ repo.name | escape }}</strong>
     </h4>
+    {% if project.type == 'free-pdk' %}
+      <div class="badges">
+        <span class="badge green">Uses Free PDK toolchain</span>
+      </div>
+    {% elsif project.type == 'proprietary' %}
+      <div class="badges">
+        <span class="badge orange">Uses proprietary toolchain</span>
+      </div>
+    {% endif %}
     <p>
-      {{ project.description | escape }}
+      {{ repo.description | escape }}
     </p>
     <div class="spacer"></div>
     <div class="meta">
-      {% assign topics = project.topics | where_exp:"topic","topic != 'padauk' and topic != 'pdk'" %}
-      {% if topics %}
+      {% assign topics = repo.topics | where_exp:"topic","topic != 'padauk' and topic != 'pdk' and topic != 'free-pdk'" %}
+      {% if topics != empty %}
         {% for topic in topics %}
           <code>{{ topic }}</code>
         {% endfor %}
         <br />
       {% endif %}
-      updated {{ project.pushed_at | timeago }}
-      {% if project.stargazers_count > 0 %}
-        · {{ project.stargazers_count }} :star:
+      updated {{ repo.pushed_at | timeago }}
+      {% if repo.stargazers_count > 0 %}
+        · {{ repo.stargazers_count }} :star:
       {% endif %}
     </div>
   </a>

_plugins/find_github_projects_using_padauk.rb

@@ -1,6 +1,18 @@
 require 'octokit'
 require 'jekyll'
 
+# Stable sort functions by user tokland
+# https://stackoverflow.com/a/15442966/2560557
+module Enumerable
+  def stable_sort
+    sort_by.with_index { |x, idx| [x, idx] }
+  end
+
+  def stable_sort_by
+    sort_by.with_index { |x, idx| [yield(x), idx] }
+  end
+end
+
 module GitHubPadaukTopics
   class Generator < Jekyll::Generator
     safe true
@@ -33,29 +45,53 @@ def log_rate_limit(client)
     end
 
     def generate(site)
-      repos, org_events = cache.getset("repos") do
+      projects, org_events = cache.getset("repos") do
+        access_token = ENV['GITHUB_TOKEN']
         client = Octokit::Client.new()
+        client.access_token=access_token
 
         # https://docs.github.com/en/rest/reference/search#search-repositories
         Jekyll.logger.info "Fetching GitHub repositories with topic 'padauk'"
-        result = client.search_repositories(QUERY, { 
-            per_page: 100, sort: 'updated', 
+        result = client.search_repositories(QUERY, {
+            per_page: 100, sort: 'updated',
             # Set preview header to also get a list of topics for each repository
-            accept: ::Octokit::Preview::PREVIEW_TYPES[:topics] 
+            accept: ::Octokit::Preview::PREVIEW_TYPES[:topics]
         })
         log_rate_limit(client)
         repos = stringify_keys(result.items.map(&:to_hash))
 
+        projects = Parallel.map(repos, in_threads: 10) do |repo|
+          type = "unknown"
+          # Only fetch repository files if we have an access token, because we run into rate limits otherwise.
+          if access_token then
+            Jekyll.logger.info "Listing files of #{repo["full_name"]}"
+            result = client.tree(repo["full_name"], repo["default_branch"], :recursive => true)
+            log_rate_limit(client)
+
+            has_pre = result.tree.detect {|entry| entry.type == "blob" and entry.path.end_with?(".PRE") }
+            if has_pre then
+              type = "proprietary"
+            elsif repo["topics"].include?('free-pdk') then
+              type = "free-pdk"
+            end
+          else
+            Jekyll.logger.warn "Skipping detection of proprietary toolchain because no GITHUB_TOKEN was found."
+          end
+          { "type" => type, "data" => repo }
+        end
+
+        projects = projects.stable_sort_by { |project| project["type"] == "free-pdk" ? 0 : project["type"] == "unknown" ? 1 : 2 }
+
         # https://docs.github.com/en/rest/reference/activity#list-public-organization-events
         Jekyll.logger.info "Fetching latest events of the free-pdk organization"
         result = client.organization_public_events('free-pdk', {per_page: 100})
         log_rate_limit(client)
         org_events = stringify_keys(result.map(&:to_hash))
 
-        [repos, org_events]
+        [projects, org_events]
       end
 
-      site.config['projects_using_padauk'] = repos
+      site.config['projects_using_padauk'] = projects
       site.config['projects_using_padauk_query_url'] = "https://github.com/search?q=" + ERB::Util.url_encode(QUERY)
       site.config['latest_free_pdk_events'] = org_events
     end

assets/main.scss

@@ -8,11 +8,13 @@ $content-width: 1040px;
 @import "minima";
 
 // custom variables
-$radius: 0.5em;
-$card-margin: 1em;
-$card-padding: 1em;
+$radius: 8px;
+$card-margin: 16px;
+$card-padding: 16px;
 $border: 1px solid #ececec;
 
+$callout-color: #FFC04C;
+
 #pinout-diagram {
   margin: 0 $spacing-unit;
 
@@ -96,6 +98,23 @@ a.card {
   }
 }
 
+.badge {
+  font-size: $small-font-size;
+  padding: 4px 8px;
+  border-radius: 6px;
+  white-space: nowrap;
+
+  &.green {
+    color: #155724;
+    background-color: lighten(#d4edda, 5%);
+  }
+
+  &.orange {
+    color: #856404;
+    background-color: #fff3cd;
+  }
+}
+
 .community-projects {
   display: flex;
   flex: 0 1 auto;
@@ -131,6 +150,13 @@ a.card {
       flex-grow: 1;
     }
 
+    .badges {
+      margin-top: -$spacing-unit / 2;
+      margin-bottom: $spacing-unit / 2;
+
+    }
+
+
     .owner {
       font-size: $small-font-size;
     }
@@ -189,8 +215,6 @@ a.card {
   }
 }
 
-$callout-color: #FFC04C;
-
 div.callout {
   @extend .card;
   margin-bottom: $spacing-unit;

index.md

@@ -100,7 +100,11 @@ More information, including information on the programming sequence, can be foun
 ## Projects from the Community
 
 These projects are auto-populated once per day by
-[searching GitHub for repositories with the `padauk` tag]({{ site.projects_using_padauk_query_url }}).
+[searching GitHub for repositories with the `padauk` topic]({{ site.projects_using_padauk_query_url }}).
+Projects that additionally have the `free-pdk` topic are highlighted as
+<span class="badge green">Uses Free PDK toolchain</span>.
+Projects that contain `.PRE` files are marked as
+<span class="badge orange">Uses proprietary toolchain</span>.
 
 {% include community_projects.html %}