Play nicely with go-ipfs 0.5.0

* deps: use latest py-multicodec

* Embed go-ipfs 0.5.0 and the latest fs-repo-migrations
in the AppImage. Automatic repo migration is on by
default.

* Add galacteek.ipfs.cidhelpers.ipnsKeyCidV1(), which
converts an IPNS key into a CIDv1 with the right multicodec,
that is usable with /ipns/<cid> with go-ipfs 0.5.0

* Make the ipns:// scheme work natively with such CIDs.
All across the browser any IPNS object opened should go
through this scheme (and not dweb:/).

AppImage/galacteek-appimage-build

@@ -89,8 +89,9 @@ cp ../share/icons/galacteek.png $APPDIR/usr/share/icons/hicolor/512x512/apps
 find /usr/lib -iname 'libzbar.so*' -exec cp -av {} $APPDIR/usr/lib \;
 find /usr/lib -iname 'libjpeg.so*' -exec cp -av {} $APPDIR/usr/lib \;
 
-# Copy go-ipfs
+# Copy go-ipfs and fs-repo-migrations
 cp /tmp/go-ipfs/ipfs $APPDIR/usr/bin
+cp $HOME/bin/fs-repo-migrations $APPDIR/usr/bin
 
 # Remove unnecessary stuff
 pushd "$APPDIR"/usr

Pipfile

@@ -10,7 +10,7 @@ jinja2 = ">=2.10.1"
 base58 = ">=1.0.2,<2.0"
 gitpython = ">=2.1.11"
 py-multibase = "==1.0.1"
-py-multicodec = "==0.1.3"
+py-multicodec = "==0.2.1"
 py-multihash = ">=0.2.3"
 aiofiles = ">=0.4.0"
 aiohttp = ">=3.4.4"
@@ -34,7 +34,7 @@ async-timeout = ">=3.0.1"
 aiosqlite = ">=0.10.0"
 aiodns = ">=2.0.0"
 pyld = ">=1.0.5"
-aioipfs = ">=0.4.4"
+aioipfs = ">=0.4.5"
 cachetools = ">=3.1.1"
 tortoise-orm = "==0.16.5"
 pyyaml = ">=5.3.0"

galacteek/application.py

@@ -59,7 +59,6 @@
 from galacteek.core.analyzer import ResourceAnalyzer
 
 from galacteek.core.schemes import SCHEME_MANUAL
-from galacteek.core.schemes import DWebSchemeHandlerNative
 from galacteek.core.schemes import DWebSchemeHandlerGateway
 from galacteek.core.schemes import EthDNSSchemeHandler
 from galacteek.core.schemes import EthDNSProxySchemeHandler
@@ -155,7 +154,7 @@ def ipfsVersion():
 async def fetchGoIpfsWrapper(app, timeout=60 * 10):
     try:
         await asyncio.wait_for(fetchIpfsSoft(
-            app, 'go-ipfs', 'ipfs', '0.4.23'), timeout)
+            app, 'go-ipfs', 'ipfs', '0.5.0'), timeout)
     except asyncio.TimeoutError:
         app.mainWindow.statusMessage(iGoIpfsFetchTimeout())
         return None
@@ -787,7 +786,7 @@ async def fetchGoIpfs(self):
         ipfsPath = self.which('ipfs')
         fsMigratePath = self.which('fs-repo-migrations')
 
-        if fsMigratePath is None and self.cmdArgs.forcegoipfsdl:
+        if fsMigratePath is None or self.cmdArgs.forcegoipfsdl:
             await fetchFsMigrateWrapper(self)
 
         if ipfsPath is None or self.cmdArgs.forcegoipfsdl:
@@ -1050,8 +1049,7 @@ def initDapps(self):
         self.dappsRegistry = DappsRegistry(self.ethereum, parent=self)
 
     def setupSchemeHandlers(self):
-        self.dwebGwSchemeHandler = DWebSchemeHandlerGateway(self)
-        self.dwebSchemeHandler = DWebSchemeHandlerNative(self)
+        self.dwebSchemeHandler = DWebSchemeHandlerGateway(self)
         self.ensSchemeHandler = EthDNSSchemeHandler(self)
         self.ensProxySchemeHandler = EthDNSProxySchemeHandler(self)
         self.nativeIpfsSchemeHandler = NativeIPFSSchemeHandler(

galacteek/core/models/mfs.py

@@ -106,8 +106,8 @@ def ipfsPath(self):
         return IPFSPath(self.fullPath)
 
     @property
-    def dwebUrl(self):
-        return QUrl('dweb:{}'.format(self.fullPath))
+    def qIpfsUrl(self):
+        return QUrl(self.ipfsPath.ipfsUrl)
 
     @property
     def mimeType(self):
@@ -279,7 +279,7 @@ def mimeData(self, indexes):
             nameItem = self.getNameItemFromIdx(idx)
 
             if nameItem:
-                mimedata.setUrls([nameItem.dwebUrl])
+                mimedata.setUrls([nameItem.qIpfsUrl])
                 break
 
         return mimedata

galacteek/core/profile.py

@@ -155,6 +155,14 @@ async def identityDagGet(self, ipfsop, path, identityUid=None):
             path)
         )
 
+    @ipfsOp
+    async def identityGetRaw(self, ipfsop, path, identityUid=None):
+        return await self.cat(os.path.join(
+            'identities',
+            identityUid if identityUid else self.root['currentIdentityUid'],
+            path)
+        )
+
     @ipfsOp
     async def getAvatar(self, ipfsop):
         return await self.get('avatar')

galacteek/core/userdag.py

@@ -108,7 +108,7 @@ def ipnsKey(self):
 
     @property
     def siteUrl(self):
-        return self.sitePath.dwebUrl
+        return self.sitePath.ipfsUrl
 
     @property
     def sitePath(self):
@@ -274,7 +274,7 @@ def createFeed(self):
         feed.author({
             'name': self.profile.userInfo.iphandle
         })
-        feed.link(href=self.atomFeedPath.dwebUrl, rel='self')
+        feed.link(href=self.atomFeedPath.ipfsUrl, rel='self')
         feed.language('en')
         return feed
 
@@ -409,8 +409,8 @@ def feedAddPinRequests(self, requests, feed):
 
             fEntry = feed.add_entry()
             fEntry.title('Pin request: {}'.format(req['title']))
-            fEntry.id(rpath.dwebUrl)
-            fEntry.link(href=rpath.dwebUrl, rel='alternate')
+            fEntry.id(rpath.ipfsUrl)
+            fEntry.link(href=rpath.ipfsUrl, rel='alternate')
             fEntry.published(req['date_published'])
             fEntry.author({'name': self.profile.userInfo.iphandle})
 
@@ -421,8 +421,8 @@ def feedAddPosts(self, blogPosts, feed):
 
             fEntry = feed.add_entry()
             fEntry.title(post['title'])
-            fEntry.id(ppath.dwebUrl)
-            fEntry.link(href=ppath.dwebUrl, rel='alternate')
+            fEntry.id(ppath.ipfsUrl)
+            fEntry.link(href=ppath.ipfsUrl, rel='alternate')
             fEntry.updated(post['date_modified'])
             fEntry.published(post['date_published'])
             fEntry.author({'name': self.profile.userInfo.iphandle})
@@ -439,7 +439,7 @@ async def tmplRender(self, tmpl, contained=False, **kw):
                           profile=self.profile,
                           dag=self.edag.dagRoot,
                           siteIpns=self.ipnsKey,
-                          atomFeedrUrl=self.atomFeedPath.dwebUrl,
+                          atomFeedrUrl=self.atomFeedPath.ipfsUrl,
                           **kw)
 
     async def renderLink(self, tmpl, contained=False, **kw):

galacteek/core/webprofiles.py

@@ -8,7 +8,6 @@
 from galacteek.dweb.webscripts import ethereumClientScripts
 
 from galacteek.core.schemes import SCHEME_DWEB
-from galacteek.core.schemes import SCHEME_DWEBGW
 from galacteek.core.schemes import SCHEME_ENS
 from galacteek.core.schemes import SCHEME_ENSR
 from galacteek.core.schemes import SCHEME_FS
@@ -64,8 +63,6 @@ def installIpfsSchemeHandlers(self):
         for scheme in [SCHEME_DWEB, SCHEME_FS]:
             self.installHandler(scheme, self.app.dwebSchemeHandler)
 
-        self.installHandler(SCHEME_DWEBGW, self.app.dwebGwSchemeHandler)
-
         for scheme in [SCHEME_IPFS, SCHEME_IPNS]:
             self.installHandler(scheme, self.app.nativeIpfsSchemeHandler)
 

galacteek/database/models/core.py

@@ -121,7 +121,8 @@ def preferredUrl(self):
             iPath = IPFSPath(self.path, autoCidConv=True)
             if self.schemepreferred == 'dweb':
                 return iPath.dwebUrl
-            elif not self.schemepreferred or self.schemepreferred == 'ipfs':
+            elif not self.schemepreferred or self.schemepreferred in \
+                    ['ipfs', 'ipns']:
                 return iPath.ipfsUrl
         else:
             return self.url

galacteek/guientrypoint.py

@@ -131,6 +131,7 @@ def start():
         '--migrate',
         action='store_true',
         dest='migrate',
+        default=True,
         help='Activate automatic repository migration')
     parser.add_argument(
         '--no-release-check',

galacteek/hashmarks/default/hashmarks.ipfs.yaml

@@ -54,7 +54,7 @@ hashmarks:
     description: Hardbin is an encrypted pastebin for IPFS
     category: ipfs/apps
     icon: /ipfs/QmQZ1PLTZKekXbcLErY2bSnHiqB9vXxkLtPtbqhEPbDHgg
-    schemepreferred: dwebgw
+    schemepreferred: dweb
     tags:
       - '#pastebin'
       - '#dapp'
@@ -75,7 +75,7 @@ hashmarks:
     date: "2019-07-04T12:00:00-00:00"
     category: ipfs/apps
     icon: /ipfs/bafybeihbmylhr5aplden7ijet4m6izxn6svq27eykqyiq5tik6mjxsoxjy/mstile-144x144.png
-    schemepreferred: dwebgw
+    schemepreferred: dweb
     tags:
       - '#ipfs'
       - '#essays'

galacteek/ipfs/cidhelpers.py

@@ -6,6 +6,7 @@
 from PyQt5.QtCore import QUrl
 
 from galacteek import log
+from galacteek.ipfs.cid import CIDv1
 from galacteek.ipfs.cid import make_cid
 from galacteek.ipfs.cid import BaseCID
 from galacteek.ipfs.stat import StatInfo
@@ -167,6 +168,21 @@ def cidConvertBase32(cid):
     return cid.encode('base32').decode()
 
 
+def ipnsKeyCidV1(ipnsKey):
+    """
+    Converts a base58 IPNS key to a CIDv1 with the
+    'libp2p-key' multicodec.
+    """
+
+    cid = getCID(ipnsKey)
+    if not cid:
+        return None
+
+    # Need to use the libp2p-key multicodec
+    cidV1 = CIDv1('libp2p-key', cid.multihash)
+    return cidV1.encode('base32').decode()
+
+
 def cidValid(cid):
     """
     Check if the passed argument is a valid IPFS CID
@@ -209,6 +225,10 @@ def domainValid(domain):
         r'^(\s*)?(?:fs:|dweb:|dwebgw:|https?://[\w:.-]+)?(?P<fullpath>(/ipfs/)?(?P<rootcid>[a-zA-Z0-9]{46,113})/?(?P<subpath>[\w<>":;,?!\*%&=@\$~/\s\.\-_\\\'()\+]{1,1024})?)#?(?P<fragment>[\w_\.\-\+,=/]{1,256})?$',  # noqa
     flags=re.UNICODE)
 
+ipfsDomainPathRe = re.compile(
+        r'^(\s*)?(?:http)://(?P<rootcid>[a-z2-7]{59,113})\.ipfs\.(?:[\w]+):(?:[\d]+)/(?P<subpath>[\w<>"\/:;,?!\*%&=@\$~/\s\.\-_\\\'()\+]{0,1024})#?(?P<fragment>[\w_\.\-\+,=/]{1,256})?$',  # noqa
+    flags=re.UNICODE)
+
 # For ipfs://<cid-base32>
 ipfsPathDedRe = re.compile(
     r'^(\s*)?(?:ipfs://)(?P<fullpath>(?P<rootcid>[a-z2-7]{59,113})/?(?P<subpath>[\w<>"*:;,?!%&=@\$~/\s\.\-_\\\'\+()]{1,1024})?)#?(?P<fragment>[\w_\+\.\-,=/]{1,256})?$',  # noqa
@@ -393,7 +413,10 @@ def ipfsUrl(self):
                     path=stripIpns(self.fullPath)
                 )
             else:
-                return self.dwebUrl
+                return '{scheme}://{path}'.format(
+                    scheme=self.scheme,
+                    path=stripIpns(self.fullPath)
+                )
         else:
             return self.dwebUrl
 
@@ -416,6 +439,28 @@ def __analyze(self):
         if len(self.input) > self.maxLength:
             return False
 
+        ma = ipfsRegSearchDomainPath(self.input)
+        if ma:
+            gdict = ma.groupdict()
+            cidStr = gdict.get('rootcid')
+            subpath = gdict.get('subpath')
+
+            if not self.parseCid(cidStr):
+                return False
+
+            if subpath:
+                self._rscPath = os.path.join(
+                    joinIpfs(cidStr),
+                    subpath
+                )
+            else:
+                self._rscPath = joinIpfs(self.rootCidRepr) + '/'
+
+            self._subPath = subpath
+            self._scheme = 'ipfs'
+            self._fragment = gdict.get('fragment')
+            return True
+
         ma = ipfsDedSearchPath(self.input)
         if ma:
             gdict = ma.groupdict()
@@ -472,15 +517,18 @@ def __analyze(self):
             gdict = ma.groupdict()
 
             subpath = gdict.get('subpath')
+            ipnsId32 = ipnsKeyCidV1(gdict.get('fqdn'))
+            _id = ipnsId32 if ipnsId32 else gdict.get('fqdn')
+
             if subpath:
                 self._rscPath = os.path.join(
-                    joinIpns(gdict.get('fqdn')),
+                    joinIpns(_id),
                     subpath
                 )
             else:
-                self._rscPath = joinIpns(gdict.get('fqdn'))
+                self._rscPath = joinIpns(_id)
 
-            self._ipnsId = gdict.get('fqdn')
+            self._ipnsId = _id
             self._fragment = gdict.get('fragment')
             self._subPath = gdict.get('subpath')
             self._scheme = 'ipns'
@@ -624,6 +672,10 @@ def ipfsDedSearchPath(text):
     return ipfsPathDedRe.match(text)
 
 
+def ipfsRegSearchDomainPath(text):
+    return ipfsDomainPathRe.match(text)
+
+
 def ipfsDedSearchPath58(text):
     # Like ipfsDedSearchPath() but allows any type of CID (including
     # CIDv0) as root CID. Only used to be able to extract the root CID

galacteek/ipfs/distipfsfetch.py

@@ -15,7 +15,7 @@
 
 @async_generator
 async def distIpfsExtract(dstdir='.', software='go-ipfs', executable='ipfs',
-                          site='dist.ipfs.io', version='0.4.23', loop=None,
+                          site='dist.ipfs.io', version='0.5.0', loop=None,
                           sslverify=True):
 
     """ Fetch a distribution archive from dist.ipfs.io and extracts the
@@ -79,14 +79,21 @@ def statusMessage(code, msg):
                     return False
                 else:
                     await yield_(statusMessage(0, 'File found!'))
+
+                csize = 524288
+                ccount = 0
                 while True:
-                    data = await resp.content.read(262144)
+                    data = await resp.content.read(csize)
                     if not data:
                         break
+
+                    ccount += 1
                     await fd.write(data)
                     bytesRead += len(data)
-                    await yield_(statusMessage(
-                        0, 'received {} bytes'.format(bytesRead)))
+
+                    if divmod(ccount, 32)[1] == 0:
+                        await yield_(statusMessage(
+                            0, 'received {} bytes'.format(bytesRead)))
                     await asyncio.sleep(0)
 
     def extract(path, dest):

galacteek/ipfs/ipfsops.py

@@ -188,11 +188,13 @@ class IPFSOperator(object):
     """
 
     def __init__(self, client, ctx=None, rsaAgent=None, debug=False,
-                 offline=False, nsCachePath=None):
+                 offline=False, nsCachePath=None,
+                 objectMapping=False):
         self._lock = asyncio.Lock()
         self._id = uuid.uuid1()
         self._cache = {}
         self._offline = offline
+        self._objectMapping = objectMapping
         self._rsaAgent = rsaAgent
         self._nsCache = {}
         self._nsCachePath = nsCachePath
@@ -529,7 +531,7 @@ async def keysRemove(self, name):
             return False
         return True
 
-    async def publish(self, path, key='self', timeout=60 * 4,
+    async def publish(self, path, key='self', timeout=60 * 6,
                       allow_offline=None, lifetime='24h',
                       resolve=True,
                       ttl=None, cache=None, cacheOrigin='unknown'):
@@ -1067,6 +1069,10 @@ def ipld(self, cid):
             return {"/": cid['Hash']}
 
     def objectPathMap(self, path):
+        if self._objectMapping is False and 0:
+            # No object mapping (don't use any cache)
+            return path
+
         ipfsPath = path if isinstance(path, IPFSPath) else \
             IPFSPath(path, autoCidConv=True)