Skip to content

Commit

Permalink
Merge pull request #567 from nats-io/dev
Browse files Browse the repository at this point in the history 
dev to main
  • Loading branch information
@aricart
aricart committed Aug 16, 2023
2 parents 54e4791 + 0ee931d commit f0f7e1d
Show file tree
Hide file tree
Showing 77 changed files with 839 additions and 412 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/test.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ jobs:
environment: CI
strategy:
matrix:
deno-version: [1.33.4]
deno-version: [1.36.1]

steps:
- name: Git Checkout Deno Module
Expand All @@ -30,7 +30,7 @@ jobs:
deno-version: ${{ matrix.deno-version }}

- name: Set NATS Server Version
run: echo "NATS_VERSION=v2.9.19" >> $GITHUB_ENV
run: echo "NATS_VERSION=v2.9.21" >> $GITHUB_ENV

# this here because dns seems to be wedged on gha
# - name: Add hosts to /etc/hosts
Expand Down
152 changes: 100 additions & 52 deletions bin/fix-os.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
* limitations under the License.
*/
import { parse } from "https://deno.land/std@0.190.0/flags/mod.ts";
import { ObjectStoreImpl } from "https://raw.githubusercontent.com/nats-io/nats.deno/main/nats-base-client/objectstore.ts";
import { ObjectStoreImpl, ServerObjectInfo } from "../jetstream/objectstore.ts";
import {
connect,
ConnectionOptions,
Expand All @@ -33,7 +33,7 @@ const argv = parse(
c: 1,
i: 0,
},
boolean: ["dryrun"],
boolean: ["check"],
string: ["server", "creds", "bucket"],
},
);
Expand All @@ -56,9 +56,7 @@ if (argv.h || argv.help) {
}

if (argv.creds) {
const f = await Deno.open(argv.creds, { read: true });
const data = await Deno.readFile(f);
Deno.close(f.rid);
const data = await Deno.readFile(argv.creds);
copts.authenticator = credsAuthenticator(data);
}

Expand All @@ -85,64 +83,114 @@ if (!found) {
Deno.exit(1);
}
const os = await js.views.os(argv.bucket) as ObjectStoreImpl;
await fixHashes(os);
await metaFix(os);

// `$${osPrefix}${os.name}.M.>`
const osInfo = await os.status({ subjects_filter: "$O.*.M.*" });
const entries = Object.getOwnPropertyNames(
osInfo.streamInfo.state.subjects || {},
);
let fixes = 0;
async function fixHashes(os: ObjectStoreImpl): Promise<void> {
let fixes = 0;
// `$${osPrefix}${os.name}.M.>`
const osInfo = await os.status({ subjects_filter: "$O.*.M.*" });
const entries = Object.getOwnPropertyNames(
osInfo.streamInfo.state.subjects || {},
);

for (let i = 0; i < entries.length; i++) {
const chunks = entries[i].split(".");
const key = chunks[3];
if (key.endsWith("=")) {
// this is already padded
continue;
}
const pad = key.length % 4;
if (pad === 0) {
continue;
}
// this entry is incorrect fix it
fixes++;
if (argv.check) {
continue;
}
const padding = pad > 0 ? "=".repeat(pad) : "";
chunks[3] += padding;
const fixedKey = chunks.join(".");
for (let i = 0; i < entries.length; i++) {
const chunks = entries[i].split(".");
const key = chunks[3];
if (key.endsWith("=")) {
// this is already padded
continue;
}
const pad = key.length % 4;
if (pad === 0) {
continue;
}
// this entry is incorrect fix it
fixes++;
if (argv.check) {
continue;
}
const padding = pad > 0 ? "=".repeat(pad) : "";
chunks[3] += padding;
const fixedKey = chunks.join(".");

let m;
try {
m = await jsm.streams.getMessage(os.stream, {
last_by_subj: entries[i],
});
} catch (err) {
console.error(`[ERR] failed to update ${entries[i]}: ${err.message}`);
continue;
}
if (m) {
let m;
try {
await js.publish(fixedKey, m.data);
m = await jsm.streams.getMessage(os.stream, {
last_by_subj: entries[i],
});
} catch (err) {
console.error(`[ERR] failed to update ${entries[i]}: ${err.message}`);
continue;
}
if (m) {
try {
await js.publish(fixedKey, m.data);
} catch (err) {
console.error(`[ERR] failed to update ${entries[i]}: ${err.message}`);
continue;
}
try {
const seq = m.seq;
await jsm.streams.deleteMessage(os.stream, seq);
} catch (err) {
console.error(
`[WARN] failed to delete bad entry ${
entries[i]
}: ${err.message} - new entry was added`,
);
}
}
}

const verb = argv.check ? "are" : "were";
console.log(`${fixes} hash fixes ${verb} required on bucket ${argv.bucket}`);
}

// metaFix addresses an issue where keys that contained `.` were serialized
// using a subject meta that replaced the `.` with `_`.
async function metaFix(os: ObjectStoreImpl): Promise<void> {
let fixes = 0;
const osInfo = await os.status({ subjects_filter: "$O.*.M.*" });
const subjects = Object.getOwnPropertyNames(
osInfo.streamInfo.state.subjects || {},
);
for (let i = 0; i < subjects.length; i++) {
const metaSubj = subjects[i];
try {
const seq = m.seq;
await jsm.streams.deleteMessage(os.stream, seq);
const m = await os.jsm.streams.getMessage(os.stream, {
last_by_subj: metaSubj,
});
const soi = m.json<ServerObjectInfo>();
const calcMeta = os._metaSubject(soi.name);
if (calcMeta !== metaSubj) {
fixes++;
if (argv.check) {
continue;
}
try {
await js.publish(calcMeta, m.data);
} catch (err) {
console.error(`[ERR] failed to update ${metaSubj}: ${err.message}`);
continue;
}
try {
const seq = m.seq;
await jsm.streams.deleteMessage(os.stream, seq);
} catch (err) {
console.error(
`[WARN] failed to delete bad entry ${metaSubj}: ${err.message} - new entry was added`,
);
}
}
} catch (err) {
console.error(
`[WARN] failed to delete bad entry ${
entries[i]
}: ${err.message} - new entry was added`,
);
console.error(`[ERR] failed to update ${metaSubj}: ${err.message}`);
}
}
const verb = argv.check ? "are" : "were";
console.log(
`${fixes} meta fixes ${verb} required on bucket ${argv.bucket}`,
);
}

const verb = argv.check ? "are" : "were";
console.log(`${fixes} fixes ${verb} required on bucket ${argv.bucket}`);

await nc.close();
await nc.drain();
2 changes: 1 addition & 1 deletion docs/assets/search.js
View file

Large diffs are not rendered by default.

30 changes: 15 additions & 15 deletions docs/enums/AdvisoryKind.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ <h1>Enumeration AdvisoryKind</h1></div>
<div class="tsd-comment tsd-typography"><p>The different kinds of Advisories</p>
</div></section><aside class="tsd-sources">
<ul>
<li>Defined in cjs/jetstream/types.ts:833</li></ul></aside>
<li>Defined in cjs/jetstream/types.ts:834</li></ul></aside>
<section class="tsd-panel-group tsd-index-group">
<section class="tsd-panel tsd-index-panel">
<details class="tsd-index-content tsd-index-accordion" open><summary class="tsd-accordion-summary tsd-index-summary">
Expand Down Expand Up @@ -48,72 +48,72 @@ <h2>Enumeration Members</h2>
<h3 class="tsd-anchor-link"><span>API</span><a href="#API" aria-label="Permalink" class="tsd-anchor-icon"><svg class="icon icon-tabler icon-tabler-link" viewBox="0 0 24 24" stroke-width="2" stroke="currentColor" fill="none" stroke-linecap="round" stroke-linejoin="round"><path stroke="none" d="M0 0h24v24H0z" fill="none" id="icon-anchor-a"></path><path d="M10 14a3.5 3.5 0 0 0 5 0l4 -4a3.5 3.5 0 0 0 -5 -5l-.5 .5" id="icon-anchor-b"></path><path d="M14 10a3.5 3.5 0 0 0 -5 0l-4 4a3.5 3.5 0 0 0 5 5l.5 -.5" id="icon-anchor-c"></path></svg></a></h3>
<div class="tsd-signature">API<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-type">&quot;api_audit&quot;</span></div><aside class="tsd-sources">
<ul>
<li>Defined in cjs/jetstream/types.ts:834</li></ul></aside></section>
<li>Defined in cjs/jetstream/types.ts:835</li></ul></aside></section>
<section class="tsd-panel tsd-member tsd-kind-enum-member tsd-parent-kind-enum"><a id="Ack" class="tsd-anchor"></a>
<h3 class="tsd-anchor-link"><span>Ack</span><a href="#Ack" aria-label="Permalink" class="tsd-anchor-icon"><svg class="icon icon-tabler icon-tabler-link" viewBox="0 0 24 24" stroke-width="2" stroke="currentColor" fill="none" stroke-linecap="round" stroke-linejoin="round"><use href="#icon-anchor-a"></use><use href="#icon-anchor-b"></use><use href="#icon-anchor-c"></use></svg></a></h3>
<div class="tsd-signature">Ack<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-type">&quot;consumer_ack&quot;</span></div><aside class="tsd-sources">
<ul>
<li>Defined in cjs/jetstream/types.ts:843</li></ul></aside></section>
<li>Defined in cjs/jetstream/types.ts:844</li></ul></aside></section>
<section class="tsd-panel tsd-member tsd-kind-enum-member tsd-parent-kind-enum"><a id="ConsumerAction" class="tsd-anchor"></a>
<h3 class="tsd-anchor-link"><span>Consumer<wbr/>Action</span><a href="#ConsumerAction" aria-label="Permalink" class="tsd-anchor-icon"><svg class="icon icon-tabler icon-tabler-link" viewBox="0 0 24 24" stroke-width="2" stroke="currentColor" fill="none" stroke-linecap="round" stroke-linejoin="round"><use href="#icon-anchor-a"></use><use href="#icon-anchor-b"></use><use href="#icon-anchor-c"></use></svg></a></h3>
<div class="tsd-signature">Consumer<wbr/>Action<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-type">&quot;consumer_action&quot;</span></div><aside class="tsd-sources">
<ul>
<li>Defined in cjs/jetstream/types.ts:836</li></ul></aside></section>
<li>Defined in cjs/jetstream/types.ts:837</li></ul></aside></section>
<section class="tsd-panel tsd-member tsd-kind-enum-member tsd-parent-kind-enum"><a id="ConsumerLeaderElected" class="tsd-anchor"></a>
<h3 class="tsd-anchor-link"><span>Consumer<wbr/>Leader<wbr/>Elected</span><a href="#ConsumerLeaderElected" aria-label="Permalink" class="tsd-anchor-icon"><svg class="icon icon-tabler icon-tabler-link" viewBox="0 0 24 24" stroke-width="2" stroke="currentColor" fill="none" stroke-linecap="round" stroke-linejoin="round"><use href="#icon-anchor-a"></use><use href="#icon-anchor-b"></use><use href="#icon-anchor-c"></use></svg></a></h3>
<div class="tsd-signature">Consumer<wbr/>Leader<wbr/>Elected<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-type">&quot;consumer_leader_elected&quot;</span></div><aside class="tsd-sources">
<ul>
<li>Defined in cjs/jetstream/types.ts:846</li></ul></aside></section>
<li>Defined in cjs/jetstream/types.ts:847</li></ul></aside></section>
<section class="tsd-panel tsd-member tsd-kind-enum-member tsd-parent-kind-enum"><a id="ConsumerQuorumLost" class="tsd-anchor"></a>
<h3 class="tsd-anchor-link"><span>Consumer<wbr/>Quorum<wbr/>Lost</span><a href="#ConsumerQuorumLost" aria-label="Permalink" class="tsd-anchor-icon"><svg class="icon icon-tabler icon-tabler-link" viewBox="0 0 24 24" stroke-width="2" stroke="currentColor" fill="none" stroke-linecap="round" stroke-linejoin="round"><use href="#icon-anchor-a"></use><use href="#icon-anchor-b"></use><use href="#icon-anchor-c"></use></svg></a></h3>
<div class="tsd-signature">Consumer<wbr/>Quorum<wbr/>Lost<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-type">&quot;consumer_quorum_lost&quot;</span></div><aside class="tsd-sources">
<ul>
<li>Defined in cjs/jetstream/types.ts:847</li></ul></aside></section>
<li>Defined in cjs/jetstream/types.ts:848</li></ul></aside></section>
<section class="tsd-panel tsd-member tsd-kind-enum-member tsd-parent-kind-enum"><a id="MaxDeliver" class="tsd-anchor"></a>
<h3 class="tsd-anchor-link"><span>Max<wbr/>Deliver</span><a href="#MaxDeliver" aria-label="Permalink" class="tsd-anchor-icon"><svg class="icon icon-tabler icon-tabler-link" viewBox="0 0 24 24" stroke-width="2" stroke="currentColor" fill="none" stroke-linecap="round" stroke-linejoin="round"><use href="#icon-anchor-a"></use><use href="#icon-anchor-b"></use><use href="#icon-anchor-c"></use></svg></a></h3>
<div class="tsd-signature">Max<wbr/>Deliver<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-type">&quot;max_deliver&quot;</span></div><aside class="tsd-sources">
<ul>
<li>Defined in cjs/jetstream/types.ts:841</li></ul></aside></section>
<li>Defined in cjs/jetstream/types.ts:842</li></ul></aside></section>
<section class="tsd-panel tsd-member tsd-kind-enum-member tsd-parent-kind-enum"><a id="RestoreComplete" class="tsd-anchor"></a>
<h3 class="tsd-anchor-link"><span>Restore<wbr/>Complete</span><a href="#RestoreComplete" aria-label="Permalink" class="tsd-anchor-icon"><svg class="icon icon-tabler icon-tabler-link" viewBox="0 0 24 24" stroke-width="2" stroke="currentColor" fill="none" stroke-linecap="round" stroke-linejoin="round"><use href="#icon-anchor-a"></use><use href="#icon-anchor-b"></use><use href="#icon-anchor-c"></use></svg></a></h3>
<div class="tsd-signature">Restore<wbr/>Complete<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-type">&quot;restore_complete&quot;</span></div><aside class="tsd-sources">
<ul>
<li>Defined in cjs/jetstream/types.ts:840</li></ul></aside></section>
<li>Defined in cjs/jetstream/types.ts:841</li></ul></aside></section>
<section class="tsd-panel tsd-member tsd-kind-enum-member tsd-parent-kind-enum"><a id="RestoreCreate" class="tsd-anchor"></a>
<h3 class="tsd-anchor-link"><span>Restore<wbr/>Create</span><a href="#RestoreCreate" aria-label="Permalink" class="tsd-anchor-icon"><svg class="icon icon-tabler icon-tabler-link" viewBox="0 0 24 24" stroke-width="2" stroke="currentColor" fill="none" stroke-linecap="round" stroke-linejoin="round"><use href="#icon-anchor-a"></use><use href="#icon-anchor-b"></use><use href="#icon-anchor-c"></use></svg></a></h3>
<div class="tsd-signature">Restore<wbr/>Create<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-type">&quot;restore_create&quot;</span></div><aside class="tsd-sources">
<ul>
<li>Defined in cjs/jetstream/types.ts:839</li></ul></aside></section>
<li>Defined in cjs/jetstream/types.ts:840</li></ul></aside></section>
<section class="tsd-panel tsd-member tsd-kind-enum-member tsd-parent-kind-enum"><a id="SnapshotComplete" class="tsd-anchor"></a>
<h3 class="tsd-anchor-link"><span>Snapshot<wbr/>Complete</span><a href="#SnapshotComplete" aria-label="Permalink" class="tsd-anchor-icon"><svg class="icon icon-tabler icon-tabler-link" viewBox="0 0 24 24" stroke-width="2" stroke="currentColor" fill="none" stroke-linecap="round" stroke-linejoin="round"><use href="#icon-anchor-a"></use><use href="#icon-anchor-b"></use><use href="#icon-anchor-c"></use></svg></a></h3>
<div class="tsd-signature">Snapshot<wbr/>Complete<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-type">&quot;snapshot_complete&quot;</span></div><aside class="tsd-sources">
<ul>
<li>Defined in cjs/jetstream/types.ts:838</li></ul></aside></section>
<li>Defined in cjs/jetstream/types.ts:839</li></ul></aside></section>
<section class="tsd-panel tsd-member tsd-kind-enum-member tsd-parent-kind-enum"><a id="SnapshotCreate" class="tsd-anchor"></a>
<h3 class="tsd-anchor-link"><span>Snapshot<wbr/>Create</span><a href="#SnapshotCreate" aria-label="Permalink" class="tsd-anchor-icon"><svg class="icon icon-tabler icon-tabler-link" viewBox="0 0 24 24" stroke-width="2" stroke="currentColor" fill="none" stroke-linecap="round" stroke-linejoin="round"><use href="#icon-anchor-a"></use><use href="#icon-anchor-b"></use><use href="#icon-anchor-c"></use></svg></a></h3>
<div class="tsd-signature">Snapshot<wbr/>Create<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-type">&quot;snapshot_create&quot;</span></div><aside class="tsd-sources">
<ul>
<li>Defined in cjs/jetstream/types.ts:837</li></ul></aside></section>
<li>Defined in cjs/jetstream/types.ts:838</li></ul></aside></section>
<section class="tsd-panel tsd-member tsd-kind-enum-member tsd-parent-kind-enum"><a id="StreamAction" class="tsd-anchor"></a>
<h3 class="tsd-anchor-link"><span>Stream<wbr/>Action</span><a href="#StreamAction" aria-label="Permalink" class="tsd-anchor-icon"><svg class="icon icon-tabler icon-tabler-link" viewBox="0 0 24 24" stroke-width="2" stroke="currentColor" fill="none" stroke-linecap="round" stroke-linejoin="round"><use href="#icon-anchor-a"></use><use href="#icon-anchor-b"></use><use href="#icon-anchor-c"></use></svg></a></h3>
<div class="tsd-signature">Stream<wbr/>Action<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-type">&quot;stream_action&quot;</span></div><aside class="tsd-sources">
<ul>
<li>Defined in cjs/jetstream/types.ts:835</li></ul></aside></section>
<li>Defined in cjs/jetstream/types.ts:836</li></ul></aside></section>
<section class="tsd-panel tsd-member tsd-kind-enum-member tsd-parent-kind-enum"><a id="StreamLeaderElected" class="tsd-anchor"></a>
<h3 class="tsd-anchor-link"><span>Stream<wbr/>Leader<wbr/>Elected</span><a href="#StreamLeaderElected" aria-label="Permalink" class="tsd-anchor-icon"><svg class="icon icon-tabler icon-tabler-link" viewBox="0 0 24 24" stroke-width="2" stroke="currentColor" fill="none" stroke-linecap="round" stroke-linejoin="round"><use href="#icon-anchor-a"></use><use href="#icon-anchor-b"></use><use href="#icon-anchor-c"></use></svg></a></h3>
<div class="tsd-signature">Stream<wbr/>Leader<wbr/>Elected<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-type">&quot;stream_leader_elected&quot;</span></div><aside class="tsd-sources">
<ul>
<li>Defined in cjs/jetstream/types.ts:844</li></ul></aside></section>
<li>Defined in cjs/jetstream/types.ts:845</li></ul></aside></section>
<section class="tsd-panel tsd-member tsd-kind-enum-member tsd-parent-kind-enum"><a id="StreamQuorumLost" class="tsd-anchor"></a>
<h3 class="tsd-anchor-link"><span>Stream<wbr/>Quorum<wbr/>Lost</span><a href="#StreamQuorumLost" aria-label="Permalink" class="tsd-anchor-icon"><svg class="icon icon-tabler icon-tabler-link" viewBox="0 0 24 24" stroke-width="2" stroke="currentColor" fill="none" stroke-linecap="round" stroke-linejoin="round"><use href="#icon-anchor-a"></use><use href="#icon-anchor-b"></use><use href="#icon-anchor-c"></use></svg></a></h3>
<div class="tsd-signature">Stream<wbr/>Quorum<wbr/>Lost<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-type">&quot;stream_quorum_lost&quot;</span></div><aside class="tsd-sources">
<ul>
<li>Defined in cjs/jetstream/types.ts:845</li></ul></aside></section>
<li>Defined in cjs/jetstream/types.ts:846</li></ul></aside></section>
<section class="tsd-panel tsd-member tsd-kind-enum-member tsd-parent-kind-enum"><a id="Terminated" class="tsd-anchor"></a>
<h3 class="tsd-anchor-link"><span>Terminated</span><a href="#Terminated" aria-label="Permalink" class="tsd-anchor-icon"><svg class="icon icon-tabler icon-tabler-link" viewBox="0 0 24 24" stroke-width="2" stroke="currentColor" fill="none" stroke-linecap="round" stroke-linejoin="round"><use href="#icon-anchor-a"></use><use href="#icon-anchor-b"></use><use href="#icon-anchor-c"></use></svg></a></h3>
<div class="tsd-signature">Terminated<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-type">&quot;terminated&quot;</span></div><aside class="tsd-sources">
<ul>
<li>Defined in cjs/jetstream/types.ts:842</li></ul></aside></section></section></div>
<li>Defined in cjs/jetstream/types.ts:843</li></ul></aside></section></section></div>
<div class="col-4 col-menu menu-sticky-wrap menu-highlight">
<div class="tsd-navigation settings">
<details class="tsd-index-accordion"><summary class="tsd-accordion-summary">
Expand Down
Loading

0 comments on commit f0f7e1d

Please sign in to comment.