Pull from a github folder

apps/remix-ide-e2e/src/tests/url.test.ts

@@ -336,5 +336,18 @@ module.exports = {
       .waitForElementVisible('*[data-shared="tooltipPopup"]')
       .waitForElementContainsText('*[data-shared="tooltipPopup"]', 'initiating fileManager and calling "open" ...')
       .waitForElementContainsText('*[data-shared="tooltipPopup"]', 'initiating terminal and calling "log" ...')
+  },
+
+  'Import Github folder from URL params #group4': function (browser: NightwatchBrowser) {
+    browser
+      .url('http://127.0.0.1:8080/#ghfolder=https://github.com/ethereum/remix-project/tree/master/apps/remix-ide/contracts/hardhat')
+      .refreshPage()
+      .waitForElementVisible('*[data-id="treeViewLitreeViewItemcontracts"]', 40000)
+      .currentWorkspaceIs('code-sample')
+      .openFile('contracts')
+      .openFile('contracts/Lock.sol')
+      .getEditorValue((content) => {
+        browser.assert.ok(content.indexOf('contract Lock {') !== -1, 'content does contain "contract Lock {"')
+      })      
   }
 }

libs/remix-core-plugin/src/lib/compiler-content-imports.ts

@@ -1,12 +1,12 @@
 'use strict'
 import { Plugin } from '@remixproject/engine'
-import { RemixURLResolver } from '@remix-project/remix-url-resolver'
+import { RemixURLResolver, githubFolderResolver } from '@remix-project/remix-url-resolver'
 
 const profile = {
   name: 'contentImport',
   displayName: 'content import',
   version: '0.0.1',
-  methods: ['resolve', 'resolveAndSave', 'isExternalUrl']
+  methods: ['resolve', 'resolveAndSave', 'isExternalUrl', 'resolveGithubFolder']
 }
 
 export type ResolvedImport = {
@@ -212,4 +212,10 @@ export class CompilerImports extends Plugin {
       throw new Error(e)
     }
   }
+
+  async resolveGithubFolder (url) {
+    const ghFolder = {}
+    await githubFolderResolver(url, ghFolder, 3)
+    return ghFolder
+  }
 }

libs/remix-ui/workspace/src/lib/actions/index.ts

@@ -29,6 +29,7 @@ export type UrlParametersType = {
   address: string
   opendir: string,
   blockscout: string,
+  ghfolder: string
 }
 
 const basicWorkspaceInit = async (workspaces: { name: string; isGitRepo: boolean; }[], workspaceProvider) => {
@@ -80,7 +81,7 @@ export const initWorkspace = (filePanelPlugin) => async (reducerDispatch: React.
       plugin.setWorkspace({ name, isLocalhost: false })
       dispatch(setCurrentWorkspace({ name, isGitRepo: false }))
       await loadWorkspacePreset('gist-template')
-    } else if (params.code || params.url || params.shareCode) {
+    } else if (params.code || params.url || params.shareCode || params.ghfolder) {
       await createWorkspaceTemplate('code-sample', 'code-template')
       plugin.setWorkspace({ name: 'code-sample', isLocalhost: false })
       dispatch(setCurrentWorkspace({ name: 'code-sample', isGitRepo: false }))

libs/remix-ui/workspace/src/lib/actions/workspace.ts

@@ -238,6 +238,7 @@ export type UrlParametersType = {
   shareCode: string
   url: string
   language: string
+  ghfolder: string
 }
 
 export const loadWorkspacePreset = async (template: WorkspaceTemplate = 'remixDefault', opts?) => {
@@ -285,10 +286,8 @@ export const loadWorkspacePreset = async (template: WorkspaceTemplate = 'remixDe
       }
       if (params.url) {
         const data = await plugin.call('contentImport', 'resolve', params.url)
-
         path = data.cleanUrl
         content = data.content
-
         try {
           content = JSON.parse(content) as any
           if (content.language && content.language === 'Solidity' && content.sources) {
@@ -307,6 +306,17 @@ export const loadWorkspacePreset = async (template: WorkspaceTemplate = 'remixDe
           await workspaceProvider.set(path, content)
         }
       }
+      if (params.ghfolder) {
+        try {
+          const files = await plugin.call('contentImport', 'resolveGithubFolder', params.ghfolder)
+          for (const [path, content] of Object.entries(files)) {
+            await workspaceProvider.set(path, content)
+          }
+        } catch (e) {
+          console.log(e)
+        }
+      }
+
       return path
     } catch (e) {
       console.error(e)

libs/remix-url-resolver/src/github-folder-resolver.ts

@@ -0,0 +1,49 @@
+// eslint-disable-next-line no-unused-vars
+import axios, { AxiosResponse } from 'axios'
+
+export type GithubItem = {
+  name: string
+  path: string
+  sha: string
+  size: number
+  url: string
+  html_url: string
+  git_url: string
+  download_url: string | null
+  type: 'dir' | 'file'
+  _links: {
+    self: string
+    git: string
+    html: string
+  }
+}
+
+export const githubFolderResolver = async (url, obj = {}, maxDepth, depth?, rootPath?) => {
+  depth = depth ? depth : 0
+  const child = await pullFolder(url)
+  depth = depth++
+  const urlObj = new URL(url);
+  const pathname = urlObj.pathname;
+  const pathParts = pathname.split('/');
+  const folderPath = pathParts.slice(5).join('/');
+  rootPath = rootPath || folderPath
+
+  if (!Array.isArray(child)) return obj
+  for (const item of child) {
+    if (item.type === 'file') {
+      const response: AxiosResponse = await axios.get(item.download_url)
+      obj[item.path.replace(rootPath, '')] = response.data
+    } else if (maxDepth > depth) {
+      // dir
+      await githubFolderResolver(item.html_url, obj, maxDepth, depth, rootPath)
+    }
+  }
+  return obj
+}
+
+const pullFolder = async (url) => {
+  const response = await axios.get('https://ghfolderpull.remixproject.org', { params: { ghfolder: url } });
+  const data: Array<GithubItem> = await response.data;
+  return data
+}
+

libs/remix-url-resolver/src/index.ts

@@ -1 +1,2 @@
 export { RemixURLResolver } from './resolve'
+export { githubFolderResolver } from './github-folder-resolver'