fix: Demote connection media validations (at least temporarily)

Errors thrown during Media retrievals no longer terminate the
connection process, but output the error instead.

logging: Hide no-mediainfo ref-count warnings
logging: Add error context to PartitionConnection media ops

Author: iridiankin

packages/prophet/Oracle/_connectionOps.js

@@ -6,7 +6,7 @@ import type ValaaURI from "~/raem/ValaaURI";
 import type { NarrateOptions } from "~/prophet/api/Prophet";
 import PartitionConnection from "~/prophet/api/PartitionConnection";
 
-import { invariantifyNumber, vdon } from "~/tools";
+import { dumpObject, invariantifyNumber, isPromise, vdon } from "~/tools";
 
 import Oracle from "./Oracle";
 import OraclePartitionConnection from "./OraclePartitionConnection";
@@ -86,15 +86,14 @@ export async function _connect (connection: OraclePartitionConnection,
     throw new Error(`Existing actions found when trying to create a new partition '${
         connection.partitionURI().toString()}'`);
   }
-
   if ((onConnectData.requireLatestMediaContents !== false)
       && (ret.mediaRetrievalStatus || { latestFailures: [] }).latestFailures.length) {
-    throw new Error(`Failed to connect to partition: encountered ${
-            onConnectData.mediaRetrievalStatus.latestFailures.length
+    // FIXME(iridian): This error temporarily demoted to log error
+    connection.outputErrorEvent(new Error(`Failed to connect to partition: encountered ${
+          ret.mediaRetrievalStatus.latestFailures.length
         } latest media content retrieval failures (and acquirePartitionConnection.${
-        ""}options.requireLatestMediaContents does not equal false).`);
+        ""}options.requireLatestMediaContents does not equal false).`));
   }
-
   connection._isConnected = true;
   return ret;
 }
@@ -133,19 +132,35 @@ export async function _narrateEventLog (connection: OraclePartitionConnection,
     const collection = connection.createReceiveTruthCollection("initialAuthorityNarration", {
       retrieveBatchContents: authorityConnection.requestMediaContents.bind(authorityConnection),
     });
+
     const authorityNarration = Promise.resolve(authorityConnection.narrateEventLog({
       narrateRemote: true, subscribeRemote: options.subscribeRemote,
       firstEventId: connection._lastAuthorizedEventId + 1,
       callback: collection.receiveTruth,
     })).then(async (remoteNarration) =>
         ((await collection.finalize(remoteNarration)) || remoteNarration));
+
     if ((options.fullNarrate === true) || (!(ret.eventLog || []).length
         && !(ret.scribeEventLog || []).length && !(ret.scribeCommandQueue || []).length)) {
       // Handle step 2 of the opportunistic narration if local narration didn't find any events.
       const authorityResults = await authorityNarration;
       for (const key of Object.keys(authorityResults)) {
-        if (Array.isArray(authorityResults[key])) {
-          authorityResults[key] = await Promise.all(authorityResults[key]);
+        const resultEntry = authorityResults[key];
+        if (!Array.isArray(resultEntry)) continue;
+        for (let i = 0; i !== resultEntry.length; ++i) {
+          const value = resultEntry[i];
+          if (!isPromise(value)) continue;
+          try {
+            resultEntry[i] = await value;
+          } catch (error) {
+            const wrapped = connection.wrapErrorEvent(error,
+                    `narrateEventLog.authorityResults[${key}][${i}]`,
+                "\n\toptions:", ...dumpObject(options),
+                "\n\tcurrent ret:", ...dumpObject(ret));
+            if (error.blocksNarration) throw wrapped;
+            connection.outputErrorEvent(wrapped);
+            resultEntry[i] = wrapped;
+          }
         }
       }
       ret.mediaRetrievalStatus = collection.analyzeRetrievals();

packages/prophet/Oracle/_downstreamOps.js

@@ -78,7 +78,9 @@ export function _createReceiveTruthCollection (connection: OraclePartitionConnec
           if (pendingRetrieval) pendingRetrievals.push(ofMedia.ongoingRetrieval.pendingContent);
         }
         if (!pendingRetrievals.length) break; // No retrieval queue, no pending retrievals - finish.
-        await Promise.all(pendingRetrievals);
+        try { await Promise.all(pendingRetrievals); } catch (error) {
+          // Ignore retrieval failures - we're just waiting for graceful retries.
+        }
       }
     }
   };
@@ -126,7 +128,7 @@ export function _createReceiveTruthCollection (connection: OraclePartitionConnec
                 mediaInfo.name}), ${thisRetrieval.retries}. attempt`;
             if (thisRetrieval.retries <= _maxOnConnectRetrievalRetries) {
               connection.warnEvent(`${description} retrying after ignoring an exception: ${
-                  error.originalMessage || error.message}`);
+                  error.originalMessage || error.message}`, ...dumpObject({ error }));
             } else {
               thisRetrieval.error = connection.wrapErrorEvent(error, description,
                   "\n\tretrievals:", ...dumpObject(collection.retrievals),

packages/prophet/Oracle/_mediaOps.js

@@ -47,9 +47,10 @@ export function _requestMediaContents (connection: OraclePartitionConnection,
     }
     ret = retrieveMediaContent(mediaInfo.mediaId, mediaInfo);
     if (ret !== undefined) {
+      // Initiate write but don't wait for it to complete. (really?).
       thenChainEagerly(ret,
         (content) => _prepareBvob(connection, content, mediaInfo, { remotePersist: false }),
-        onError.bind(this));
+        onError);
     }
     return ret;
   });

packages/prophet/Scribe/_databaseOps.js

@@ -113,17 +113,17 @@ export function _persistMediaEntry (connection: ScribePartitionConnection, newMe
         // TODO(iridian): Are there race conditions here? The refcount operations are not awaited on
         if (newInfo.bvobId) {
           if (!connection._prophet._bvobLookup[newInfo.bvobId]) {
-            console.log(`Can't find Media "${newInfo.name}" Bvob info for ${newInfo.bvobId
-                } when adding new content references`);
+            // console.log(`Can't find Media "${newInfo.name}" Bvob info for ${newInfo.bvobId
+            //    } when adding new content references`);
           } else {
             if (newMediaEntry.isInMemory) connection._prophet._addContentInMemoryReference(newInfo);
             if (newMediaEntry.isPersisted) connection._prophet._addContentPersistReference(newInfo);
           }
         }
         if (oldBvobId) {
           if (!connection._prophet._bvobLookup[oldBvobId]) {
-            console.log(`Can't find Media "${newInfo.name}" Bvob info for ${oldBvobId
-                } when removing old content references`);
+            // console.log(`Can't find Media "${newInfo.name}" Bvob info for ${oldBvobId
+            //    } when removing old content references`);
           } else {
             if (oldEntry.isInMemory) connection._prophet._removeContentInMemoryReference(oldBvobId);
             if (oldEntry.isPersisted) connection._prophet._removeContentPersistReference(oldBvobId);

packages/prophet/api/PartitionConnection.js

@@ -164,7 +164,12 @@ export default class PartitionConnection extends LogEventGenerator {
     delete mediaInfo.mime;
     delete mediaInfo.type;
     delete mediaInfo.subtype;
-    return thenChainEagerly(this.requestMediaContents([mediaInfo]), results => results[0]);
+    return thenChainEagerly(
+        this.requestMediaContents([mediaInfo]),
+        results => results[0],
+        (error) => this.wrapErrorEvent(error, `readMediaContent(${mediaInfo.name})`,
+            "\n\tmediaInfo:", ...dumpObject(mediaInfo))
+    );
   }
 
   /**
@@ -187,7 +192,13 @@ export default class PartitionConnection extends LogEventGenerator {
           `decodeMediaContent('${mediaInfo.name || "<unnamed>"}')`,
               "\n\tmediaInfo:", ...dumpObject(mediaInfo));
     }
-    return thenChainEagerly(this.requestMediaContents([mediaInfo]), results => results[0]);
+    return thenChainEagerly(
+        this.requestMediaContents([mediaInfo]),
+        results => results[0],
+        (error) => this.wrapErrorEvent(error, `decodeMediaContent(${mediaInfo.name} as ${
+                mediaInfo.mime})`,
+            "\n\tmediaInfo:", ...dumpObject(mediaInfo)),
+    );
   }
 
   /**
@@ -204,7 +215,12 @@ export default class PartitionConnection extends LogEventGenerator {
    */
   getMediaURL (mediaInfo: MediaInfo): any {
     if (!mediaInfo.asURL) mediaInfo.asURL = true;
-    return thenChainEagerly(this.requestMediaContents([mediaInfo]), results => results[0]);
+    return thenChainEagerly(
+        this.requestMediaContents([mediaInfo]),
+        results => results[0],
+        (error) => this.wrapErrorEvent(error, `getMediaURL(${mediaInfo.name})`,
+            "\n\tmediaInfo:", ...dumpObject(mediaInfo)),
+    );
   }
 
   requestMediaContents (mediaInfos: MediaInfo[]): Promise<(Promise | any)[]> | (Promise | any)[] {