Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Move onion requests to typescript #1221

Merged
merged 3 commits into from Jul 7, 2020
Merged

Move onion requests to typescript #1221

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
12f73e2
Move onion requests to typescript
msgmaxim Jul 6, 2020
6919f53
Move constants to preload.js
msgmaxim Jul 7, 2020
ccc4875
Fix edkey extraction
msgmaxim Jul 7, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion js/background.js
View file
Open in desktop
Expand Up @@ -292,7 +292,7 @@
window.lokiFeatureFlags.useFileOnionRequests
) {
// Initialize paths for onion requests
window.lokiSnodeAPI.buildNewOnionPaths();
window.OnionAPI.buildNewOnionPaths();
}

const currentPoWDifficulty = storage.get('PoWDifficulty', null);
Expand Down
2 changes: 1 addition & 1 deletion js/modules/data.d.ts
View file
Open in desktop
Expand Up @@ -159,7 +159,7 @@ export function getPairingAuthorisationsFor(
export function removePairingAuthorisationsFor(pubKey: string): Promise<void>;

// Guard Nodes
export function getGuardNodes(): Promise<GuardNode>;
export function getGuardNodes(): Promise<Array<GuardNode>>;
export function updateGuardNodes(nodes: Array<string>): Promise<void>;

// Storage Items
Expand Down
6 changes: 3 additions & 3 deletions js/modules/loki_app_dot_net_api.js
View file
Open in desktop
@@ -1,6 +1,6 @@
/* global log, textsecure, libloki, Signal, Whisper, ConversationController,
clearTimeout, MessageController, libsignal, StringView, window, _,
dcodeIO, Buffer, lokiSnodeAPI, TextDecoder, process */
dcodeIO, Buffer, TextDecoder, process */
const nodeFetch = require('node-fetch');
const { URL, URLSearchParams } = require('url');
const FormData = require('form-data');
Expand Down Expand Up @@ -59,7 +59,7 @@ const sendViaOnion = async (srvPubKey, url, fetchOptions, options = {}) => {
// eslint-disable-next-line no-param-reassign
options.retry = 0;
// eslint-disable-next-line no-param-reassign
options.requestNumber = window.lokiSnodeAPI.assignOnionRequestNumber();
options.requestNumber = window.OnionAPI.assignOnionRequestNumber();
}

const payloadObj = {
Expand Down Expand Up @@ -92,7 +92,7 @@ const sendViaOnion = async (srvPubKey, url, fetchOptions, options = {}) => {

let pathNodes = [];
try {
pathNodes = await lokiSnodeAPI.getOnionPath();
pathNodes = await window.OnionAPI.getOnionPath();
} catch (e) {
log.error(
`loki_app_dot_net:::sendViaOnion #${options.requestNumber} - getOnionPath Error ${e.code} ${e.message}`
Expand Down
316 changes: 5 additions & 311 deletions js/modules/loki_snode_api.js
View file
Open in desktop
@@ -1,317 +1,11 @@
/* eslint-disable class-methods-use-this */
/* global window, textsecure, log, process, Buffer, StringView, dcodeIO */

// not sure I like this name but it's been than util
const primitives = require('./loki_primitives');

const is = require('@sindresorhus/is');
const nodeFetch = require('node-fetch');

const MIN_GUARD_COUNT = 2;
/* global window, Buffer, StringView, dcodeIO */

class LokiSnodeAPI {
constructor({ serverUrl, localUrl }) {
if (!is.string(serverUrl)) {
throw new Error('LokiSnodeAPI.initialize: Invalid server url');
}
this.serverUrl = serverUrl; // random.snode
this.localUrl = localUrl; // localhost.loki
this.swarmsPendingReplenish = {};
this.stopGetAllVersionPromiseControl = false;

this.onionPaths = [];
this.guardNodes = [];
this.onionRequestCounter = 0; // Request index for debugging
}

assignOnionRequestNumber() {
this.onionRequestCounter += 1;
return this.onionRequestCounter;
}

async testGuardNode(snode) {
log.info('Testing a candidate guard node ', snode);

// Send a post request and make sure it is OK
const endpoint = '/storage_rpc/v1';

const url = `https://${snode.ip}:${snode.port}${endpoint}`;

const ourPK = textsecure.storage.user.getNumber();
const pubKey = window.getStoragePubKey(ourPK); // truncate if testnet

const method = 'get_snodes_for_pubkey';
const params = { pubKey };
const body = {
jsonrpc: '2.0',
id: '0',
method,
params,
};

const fetchOptions = {
method: 'POST',
body: JSON.stringify(body),
headers: { 'Content-Type': 'application/json' },
timeout: 10000, // 10s, we want a smaller timeout for testing
};

process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0';

let response;

try {
// Log this line for testing
// curl -k -X POST -H 'Content-Type: application/json' -d '"+fetchOptions.body.replace(/"/g, "\\'")+"'", url
response = await nodeFetch(url, fetchOptions);
} catch (e) {
if (e.type === 'request-timeout') {
log.warn(`test timeout for node,`, snode);
}
return false;
} finally {
process.env.NODE_TLS_REJECT_UNAUTHORIZED = '1';
}

if (!response.ok) {
log.info(`Node failed the guard test:`, snode);
}

return response.ok;
}

async selectGuardNodes() {
const _ = window.Lodash;

// FIXME: handle rejections
let nodePool = await window.SnodePool.getRandomSnodePool();
if (nodePool.length === 0) {
log.error(`Could not select guard nodes: node pool is empty`);
return [];
}

let shuffled = _.shuffle(nodePool);

let guardNodes = [];

const DESIRED_GUARD_COUNT = 3;

if (shuffled.length < DESIRED_GUARD_COUNT) {
log.error(
`Could not select guard nodes: node pool is not big enough, pool size ${shuffled.length}, need ${DESIRED_GUARD_COUNT}, attempting to refresh randomPool`
);
await window.SnodePool.refreshRandomPool();
nodePool = await window.SnodePool.getRandomSnodePool();
shuffled = _.shuffle(nodePool);
if (shuffled.length < DESIRED_GUARD_COUNT) {
log.error(
`Could not select guard nodes: node pool is not big enough, pool size ${shuffled.length}, need ${DESIRED_GUARD_COUNT}, failing...`
);
return [];
}
}
// The use of await inside while is intentional:
// we only want to repeat if the await fails
// eslint-disable-next-line-no-await-in-loop
while (guardNodes.length < 3) {
if (shuffled.length < DESIRED_GUARD_COUNT) {
log.error(`Not enought nodes in the pool`);
break;
}

const candidateNodes = shuffled.splice(0, DESIRED_GUARD_COUNT);

// Test all three nodes at once
// eslint-disable-next-line no-await-in-loop
const idxOk = await Promise.all(
candidateNodes.map(n => this.testGuardNode(n))
);

const goodNodes = _.zip(idxOk, candidateNodes)
.filter(x => x[0])
.map(x => x[1]);

guardNodes = _.concat(guardNodes, goodNodes);
}

if (guardNodes.length < DESIRED_GUARD_COUNT) {
log.error(
`COULD NOT get enough guard nodes, only have: ${guardNodes.length}`
);
}

log.info('new guard nodes: ', guardNodes);

const edKeys = guardNodes.map(n => n.pubkey_ed25519);

await window.libloki.storage.updateGuardNodes(edKeys);

return guardNodes;
}

async getOnionPath(toExclude = null) {
const _ = window.Lodash;

let goodPaths = this.onionPaths.filter(x => !x.bad);

let attemptNumber = 0;
while (goodPaths.length < MIN_GUARD_COUNT) {
log.error(
`Must have at least 2 good onion paths, actual: ${goodPaths.length}, attempt #${attemptNumber} fetching more...`
);
// eslint-disable-next-line no-await-in-loop
await this.buildNewOnionPaths();
// should we add a delay? buildNewOnionPaths should act as one

// reload goodPaths now
attemptNumber += 1;
goodPaths = this.onionPaths.filter(x => !x.bad);
}

const paths = _.shuffle(goodPaths);

if (!toExclude) {
if (!paths[0]) {
log.error('LokiSnodeAPI::getOnionPath - no path in', paths);
return [];
}
if (!paths[0].path) {
log.error('LokiSnodeAPI::getOnionPath - no path in', paths[0]);
}
return paths[0].path;
}

// Select a path that doesn't contain `toExclude`
const otherPaths = paths.filter(
path =>
!_.some(path, node => node.pubkey_ed25519 === toExclude.pubkey_ed25519)
);

if (otherPaths.length === 0) {
// This should never happen!
// well it did happen, should we
// await this.buildNewOnionPaths();
// and restart call?
log.error(
`LokiSnodeAPI::getOnionPath - no paths without`,
toExclude.pubkey_ed25519,
'path count',
paths.length,
'goodPath count',
goodPaths.length,
'paths',
paths
);
throw new Error('No onion paths available after filtering');
}

if (!otherPaths[0].path) {
log.error(
'LokiSnodeAPI::getOnionPath - otherPaths no path in',
otherPaths[0]
);
}

return otherPaths[0].path;
}

markPathAsBad(path) {
this.onionPaths.forEach(p => {
if (!p.path) {
log.error('LokiSnodeAPI::markPathAsBad - no path in', p);
}
if (p.path === path) {
// eslint-disable-next-line no-param-reassign
p.bad = true;
}
});
}

async buildNewOnionPathsWorker() {
const _ = window.Lodash;

log.info('LokiSnodeAPI::buildNewOnionPaths - building new onion paths');

const allNodes = await window.SnodePool.getRandomSnodePool();

if (this.guardNodes.length === 0) {
// Not cached, load from DB
const nodes = await window.libloki.storage.getGuardNodes();

if (nodes.length === 0) {
log.warn(
'LokiSnodeAPI::buildNewOnionPaths - no guard nodes in DB. Will be selecting new guards nodes...'
);
} else {
// We only store the nodes' keys, need to find full entries:
const edKeys = nodes.map(x => x.ed25519PubKey);
this.guardNodes = allNodes.filter(
x => edKeys.indexOf(x.pubkey_ed25519) !== -1
);

if (this.guardNodes.length < edKeys.length) {
log.warn(
`LokiSnodeAPI::buildNewOnionPaths - could not find some guard nodes: ${this.guardNodes.length}/${edKeys.length} left`
);
}
}

// If guard nodes is still empty (the old nodes are now invalid), select new ones:
if (this.guardNodes.length < MIN_GUARD_COUNT) {
// TODO: don't throw away potentially good guard nodes
this.guardNodes = await this.selectGuardNodes();
}
}

// TODO: select one guard node and 2 other nodes randomly
let otherNodes = _.difference(allNodes, this.guardNodes);

if (otherNodes.length < 2) {
log.warn(
'LokiSnodeAPI::buildNewOnionPaths - Too few nodes to build an onion path! Refreshing pool and retrying'
);
await window.SnodePool.refreshRandomPool();
await this.buildNewOnionPaths();
return;
}

otherNodes = _.shuffle(otherNodes);
const guards = _.shuffle(this.guardNodes);

// Create path for every guard node:
const nodesNeededPerPaths = window.lokiFeatureFlags.onionRequestHops - 1;

// Each path needs X (nodesNeededPerPaths) nodes in addition to the guard node:
const maxPath = Math.floor(
Math.min(
guards.length,
nodesNeededPerPaths
? otherNodes.length / nodesNeededPerPaths
: otherNodes.length
)
);

// TODO: might want to keep some of the existing paths
this.onionPaths = [];

for (let i = 0; i < maxPath; i += 1) {
const path = [guards[i]];
for (let j = 0; j < nodesNeededPerPaths; j += 1) {
path.push(otherNodes[i * nodesNeededPerPaths + j]);
}
this.onionPaths.push({ path, bad: false });
}

log.info(`Built ${this.onionPaths.length} onion paths`, this.onionPaths);
}

async buildNewOnionPaths() {
// this function may be called concurrently make sure we only have one inflight
return primitives.allowOnlyOneAtATime('buildNewOnionPaths', async () => {
await this.buildNewOnionPathsWorker();
});
}

// ************** NOTE ***************
// This is not used by anything yet,
// but should be. Do not remove!!!
// ***********************************
async getLnsMapping(lnsName, timeout) {
// Returns { pubkey, error }
// pubkey is
Expand Down
10 changes: 5 additions & 5 deletions preload.js
View file
Open in desktop
Expand Up @@ -99,6 +99,8 @@ window.CONSTANTS = new (function() {
// https://loki.network/2020/03/25/loki-name-system-the-facts/
this.LNS_REGEX = `^[a-zA-Z0-9_]([a-zA-Z0-9_-]{0,${this.LNS_MAX_LENGTH -
2}}[a-zA-Z0-9_]){0,1}$`;
this.MIN_GUARD_COUNT = 2;
this.DESIRED_GUARD_COUNT = 3;
})();

window.versionInfo = {
Expand Down Expand Up @@ -332,14 +334,12 @@ const { initialize: initializeWebAPI } = require('./js/modules/web_api');
window.WebAPI = initializeWebAPI();

window.seedNodeList = JSON.parse(config.seedNodeList);
const LokiSnodeAPI = require('./js/modules/loki_snode_api');

window.SenderKeyAPI = require('./js/modules/loki_sender_key_api');

window.lokiSnodeAPI = new LokiSnodeAPI({
serverUrl: config.serverUrl,
localUrl: config.localUrl,
});
const { OnionAPI } = require('./ts/session/onions');

window.OnionAPI = OnionAPI;

if (process.env.USE_STUBBED_NETWORK) {
const StubMessageAPI = require('./integration_test/stubs/stub_message_api');
Expand Down