Skip to content
Permalink
Browse files

feat: upload blobs to infura

  • Loading branch information...
satazor committed Jun 21, 2019
1 parent aa234f1 commit 35a6fa4af741d930aeee17e8e12919af216e6da2

Some generated files are not rendered by default. Learn more.

@@ -110,6 +110,7 @@
"p-map": "^2.1.0",
"p-reduce": "^2.1.0",
"p-series": "^2.1.0",
"p-timeout": "^3.1.0",
"pico-signals": "^1.0.0",
"pify": "^4.0.1",
"scrypt-async": "^2.0.1",
@@ -17,7 +17,7 @@ beforeEach(() => {
it('should create wallet successfully', async () => {
const wallet = await createWallet();

expect(Object.keys(wallet)).toEqual(['didm', 'storage', 'locker', 'identities', 'sessions']);
expect(Object.keys(wallet)).toEqual(['ipfs', 'didm', 'storage', 'locker', 'identities', 'sessions']);
expect(Ipfs).toHaveBeenCalledTimes(1);
});

@@ -1,7 +1,7 @@
import signal from 'pico-signals';
import pDelay from 'delay';
import pSeries from 'p-series';
import { pick, get, has, isEqual } from 'lodash';
import { pick, has, isEqual } from 'lodash';
import { loadStore, dropStore, dropOrbitDbIfEmpty, waitStoreReplication } from './utils/orbitdb';
import createBlobStore from './utils/blob-store';
import { assertProfileProperty, assertNonMandatoryProfileProperty } from './utils/asserts';
@@ -106,9 +106,7 @@ class Profile {

PROFILE_BLOB_PROPERTIES.forEach((key) => {
if (details[key]) {
const blobRef = this.#blobStore.get(key);

details[key] = get(blobRef, 'content.dataUri', null);
details[key] = this.#blobStore.getUrl(key);
}
});

@@ -1,6 +1,10 @@
import { Buffer } from 'buffer';
import signal from 'pico-signals';
import { difference } from 'lodash';
import pTimeout from 'p-timeout';
import infuraIpfs from './infura-ipfs';
import { INFURA_IPFS_ENDPOINT, INFURA_ADD_FILE_TIMEOUT, INFURA_HAS_FILE_TIMEOUT } from './constants/infura';
import { InfuraHashMismatch } from '../../../utils/errors';

class BlobStore {
#ipfs;
@@ -15,27 +19,31 @@ class BlobStore {
return this.#refs.get(key);
}

getUrl(key) {
const ref = this.#refs.get(key);

return ref && ref.status === 'synced' ? `${INFURA_IPFS_ENDPOINT}/cat?arg=${ref.hash}` : undefined;
}

async put(key, blob) {
const { type, data } = blob;
const buffer = Buffer.from(data);
const { type } = blob;

const hash = await this.#addToIpfs(blob);

const [{ hash }] = await this.#ipfs.add(buffer, { pin: true });
await this.#addToInfura(key, hash, blob.data).catch((err) => console.warn(`Unable to add "${key}" to infura`, err));

let ref = this.#refs.get(key);

// Skip if hash & type are the same and it's already loaded to avoid triggering change
if (ref && ref.type === type && ref.hash === hash && ref.content.status === 'fulfilled') {
if (ref && ref.type === type && ref.hash === hash && ref.status === 'synced') {
return ref;
}

ref = {
type,
hash,
content: {
status: 'fulfilled',
data,
dataUri: this.#getDataUri(type, buffer),
},
status: 'synced',
error: undefined,
};

this.#refs.set(key, ref);
@@ -90,58 +98,90 @@ class BlobStore {
#syncAdded = async (key, ref) => {
ref = {
...ref,
content: {
status: 'pending',
error: undefined,
data: undefined,
dataUri: undefined,
},
status: 'syncing',
error: undefined,
};

// Mark the blob as pending
this.#refs.set(key, ref);
this.#notifyChange(key);

// Attempt to load the blob
let content;
let updatedRef;

// Attempt to pin the blob && ensure it's on infura
try {
const [{ content: buffer }] = await this.#ipfs.get(ref.hash);

// Pin the hash so that we can serve it to others
await this.#ipfs.pin.add(ref.hash);
await this.#maybeAddToInfura(key, ref.hash).catch((err) => console.warn(`Unable to check and add "${key}" to infura`, err));

const data = buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);

content = {
status: 'fulfilled',
updatedRef = {
...ref,
status: 'synced',
error: undefined,
data,
dataUri: this.#getDataUri(ref.type, buffer),
};
} catch (error) {
content = {
status: 'rejected',
updatedRef = {
...ref,
status: 'error',
error,
data: undefined,
dataUri: undefined,
};
}

const stillExactSame = this.#refs.get(key) === ref;

if (stillExactSame) {
this.#refs.set(key, { ...ref, content });
this.#refs.set(key, updatedRef);
this.#notifyChange(key, true);
}
}

#isSame = (ref1, ref2) => ref1.type === ref2.type && ref1.hash === ref2.hash;
#notifyChange = (key, async = false) => {
const ref = this.#refs.get(key);

#getDataUri = (type, buffer) => `data:${type};base64,${buffer.toString('base64')}`;
this.#onChange.dispatch(key, ref, async);
}

#notifyChange = (key, async = false) => {
this.#onChange.dispatch(key, this.#refs.get(key), async);
#addToIpfs = async (blob) => {
const buffer = Buffer.from(blob.data);

const [{ hash }] = await this.#ipfs.add(buffer, { cidVersion: 0, pin: true });

return hash;
}

#maybeAddToInfura = async (key, hash, data) => {
// Check if files exists in infura
const exists = await pTimeout(
infuraIpfs.block.stat(hash).then(() => true),
INFURA_HAS_FILE_TIMEOUT,
() => false
);

if (exists) {
return;
}

// If there's no data yet, grab it
if (!data) {
const [{ content: buffer }] = await this.#ipfs.get(hash);

data = buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
}

await this.#addToInfura(key, hash, data);
}

#addToInfura = async (key, hash, data) => {
const buffer = Buffer.from(data);

const [{ hash: infuraHash }] = await pTimeout(
infuraIpfs.add(buffer, { cidVersion: 0, pin: true }),
INFURA_ADD_FILE_TIMEOUT,
);

if (infuraHash !== hash) {
throw new InfuraHashMismatch(infuraHash, hash);
}

return hash;
}
}

@@ -0,0 +1,5 @@
export const INFURA_IPFS_ENDPOINT = 'https://ipfs.infura.io:5001/api/v0';

export const INFURA_HAS_FILE_TIMEOUT = 10000;

export const INFURA_ADD_FILE_TIMEOUT = 60000;
@@ -0,0 +1,11 @@
import createIpfsClient from 'ipfs-http-client';
import { INFURA_IPFS_ENDPOINT } from './constants/infura';

const endpointUrl = new URL(INFURA_IPFS_ENDPOINT);

export default createIpfsClient({
host: endpointUrl.hostname,
port: endpointUrl.port,
protocol: endpointUrl.protocol.slice(0, -1),
'api-url': endpointUrl.pathname,
});
@@ -45,17 +45,18 @@ const createWallet = async (options) => {
...options,
};

const ipfsNode = await createIpfs(options.ipfs);
const ipfs = await createIpfs(options.ipfs);

const secret = createSecret();

const didm = createDidm(ipfsNode);
const didm = createDidm(ipfs);
const storage = await createStorage(secret);
const locker = await createLocker(storage, secret);
const identities = createIdentities(storage, didm, ipfsNode);
const identities = createIdentities(storage, didm, ipfs);
const sessions = await createSessions(storage, identities);

const idmWallet = {
ipfs,
didm,
storage,
locker,
@@ -23,3 +23,9 @@ export class ProfileReplicationTimeoutError extends BaseError {
super('Profile replication timed out', 'PROFILE_REPLICATION_TIMEOUT');
}
}

export class InfuraHashMismatch extends BaseError {
constructor(infuraHash, expectedHash) {
super(`Expecting infura response hash to be "${expectedHash}" but got "${infuraHash} instead`, 'INFURA_HASH_MISMATCH');
}
}

0 comments on commit 35a6fa4

Please sign in to comment.
You can’t perform that action at this time.