Merge pull request #6 from d70-t/test_local_gateway

Test local gateway

.github/workflows/local_gateway.yml

@@ -0,0 +1,38 @@
+name: Test with local gateway
+
+on: [push]
+
+jobs:
+  test:
+
+    runs-on: ubuntu-latest
+    strategy:
+      max-parallel: 4
+      matrix:
+        python-version: [3.9]
+        ipfs-version: ["0.8.0"]
+    env:
+      IPFSSPEC_GATEWAYS: "http://127.0.0.1:8080" # use only localhost as gateway
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -e .
+    - name: Install ipfs
+      run: |
+        wget https://dist.ipfs.io/go-ipfs/v${{ matrix.ipfs-version }}/go-ipfs_v${{ matrix.ipfs-version }}_linux-amd64.tar.gz
+        tar -xvzf go-ipfs_v${{ matrix.ipfs-version }}_linux-amd64.tar.gz
+        cd go-ipfs
+        sudo bash install.sh
+        ipfs --version
+        ipfs init --profile server
+        ipfs daemon > ipfs.log &
+    - name: Test with pytest
+      run: |
+        pip install pytest
+        pytest

README.md

@@ -12,3 +12,5 @@ with fsspec.open("ipfs://QmZ4tDuvesekSs4qM5ZBKpXiZGun7S2CYtEZRB3DYXkjGx", "r") a
 ```
 
 The current implementation uses a HTTP gateway to access the data. It tries to use a local one (which is expected to be found at `http://127.0.0.1:8080`) and falls back to `ipfs.io` if the local gateway is not available.
+
+You can modify the list of gateways using the space separated environment variable `IPFSSPEC_GATEWAYS`.

ipfsspec/core.py

@@ -9,6 +9,7 @@
 import functools
 import time
 import json
+import os
 
 import logging
 
@@ -116,12 +117,19 @@ def get_state(self):
 ]
 
 
+def get_default_gateways():
+    try:
+        return os.environ["IPFSSPEC_GATEWAYS"].split()
+    except KeyError:
+        return GATEWAYS
+
+
 class IPFSFileSystem(AbstractFileSystem):
     protocol = "ipfs"
 
     def __init__(self, *args, gateways=None, timeout=10, **kwargs):
         super(IPFSFileSystem, self).__init__(*args, **kwargs)
-        gateways = gateways or GATEWAYS
+        gateways = gateways or get_default_gateways()
         self._gateways = [IPFSGateway(g) for g in gateways]
         self.timeout = timeout