Skip to content

Commit

Permalink
fixed: checking internal storage res as admin users
Browse files Browse the repository at this point in the history
  • Loading branch information
@t83714
t83714 committed Apr 17, 2024
1 parent f5dea46 commit c24c9c4
Show file tree
Hide file tree
Showing 4 changed files with 90 additions and 30 deletions.
2 changes: 2 additions & 0 deletions src/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,8 @@ function sleuthBrokenLinks() {
registry,
argv.storageApiBaseUrl,
argv.datasetBucketName,
argv.jwtSecret,
argv.userId,
argv.externalRetries,
1,
argv.domainWaitTimeConfig as any,
Expand Down
81 changes: 61 additions & 20 deletions src/onRecordFound.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,15 @@ import {
} from "./HttpRequests.js";
import getUrlWaitTime from "./getUrlWaitTime.js";
import wait from "./wait.js";
import { buildJwt } from "@magda/utils";

export default async function onRecordFound(
record: Record,
registry: Registry,
storageApiBaseUrl: string,
datasetBucketName: string,
jwtSecret: string,
actionUserId: string,
retries: number = 1,
baseRetryDelaySeconds: number = 1,
domainWaitTimeConfig: { [domain: string]: number } = {},
Expand Down Expand Up @@ -49,7 +52,9 @@ export default async function onRecordFound(
_.partialRight(getUrlWaitTime, domainWaitTimeConfig),
requestOpts,
storageApiBaseUrl,
datasetBucketName
datasetBucketName,
jwtSecret,
actionUserId
)
);

Expand Down Expand Up @@ -172,7 +177,9 @@ function checkDistributionLink(
getUrlWaitTime: (url: string) => number,
requestOpts: CoreOptions,
storageApiBaseUrl: string,
datasetBucketName: string
datasetBucketName: string,
jwtSecret: string,
actionUserId: string
): DistributionLinkCheck[] {
type DistURL = {
url?: URI;
Expand All @@ -190,15 +197,7 @@ function checkDistributionLink(
}
]
.map((urlObj) => {
const url =
typeof urlObj.url === "string"
? getStorageApiResourceAccessUrl(
urlObj.url,
storageApiBaseUrl,
datasetBucketName
)
: urlObj.url;
return { ...urlObj, url: parseUriSafe(url) };
return { ...urlObj, url: parseUriSafe(urlObj.url) };
})
.filter((x) => x.url && x.url.protocol().length > 0);

Expand Down Expand Up @@ -232,7 +231,11 @@ function checkDistributionLink(
retries,
ftpHandler,
getUrlWaitTime,
requestOpts
requestOpts,
storageApiBaseUrl,
datasetBucketName,
jwtSecret,
actionUserId
)
.then((aspect) => {
console.info("Finished retrieving " + parsedURL);
Expand Down Expand Up @@ -262,15 +265,28 @@ function retrieve(
retries: number,
ftpHandler: FTPHandler,
getUrlWaitTime: (url: string) => number,
requestOpts: CoreOptions
requestOpts: CoreOptions,
storageApiBaseUrl: string,
datasetBucketName: string,
jwtSecret: string,
actionUserId: string
): Promise<BrokenLinkAspect> {
if (parsedURL.protocol() === "http" || parsedURL.protocol() === "https") {
if (
parsedURL.protocol() === "http" ||
parsedURL.protocol() === "https" ||
(parsedURL.protocol() === "magda" &&
parsedURL.hostname() === "storage-api")
) {
return retrieveHttp(
parsedURL.toString(),
baseRetryDelay,
retries,
getUrlWaitTime,
requestOpts
requestOpts,
storageApiBaseUrl,
datasetBucketName,
jwtSecret,
actionUserId
);
} else if (parsedURL.protocol() === "ftp") {
return retrieveFtp(parsedURL, ftpHandler);
Expand Down Expand Up @@ -319,16 +335,41 @@ async function retrieveHttp(
baseRetryDelay: number,
retries: number,
getUrlWaitTime: (url: string) => number,
requestOpts: CoreOptions
requestOpts: CoreOptions,
storageApiBaseUrl: string,
datasetBucketName: string,
jwtSecret: string,
actionUserId: string
): Promise<BrokenLinkAspect> {
const isInternalStorageRes = url.indexOf("magda://storage-api/") === 0;
const resUrl = getStorageApiResourceAccessUrl(
url,
storageApiBaseUrl,
datasetBucketName
);
const runtimeRequestOpts = { ...requestOpts };
if (requestOpts?.headers) {
runtimeRequestOpts.headers = {
...requestOpts.headers
};
}
if (isInternalStorageRes) {
if (!runtimeRequestOpts?.headers) {
runtimeRequestOpts.headers = {};
}
runtimeRequestOpts.headers = {
...runtimeRequestOpts.headers,
"X-Magda-Session": buildJwt(jwtSecret, actionUserId)
};
}
async function operation() {
try {
await wait(getUrlWaitTime(url));
return await headRequest(url, requestOpts);
await wait(getUrlWaitTime(resUrl));
return await headRequest(resUrl, runtimeRequestOpts);
} catch (e) {
// --- HEAD Method not allowed
await wait(getUrlWaitTime(url));
return await getRequest(url, requestOpts);
await wait(getUrlWaitTime(resUrl));
return await getRequest(resUrl, runtimeRequestOpts);
}
}

Expand Down
20 changes: 15 additions & 5 deletions src/test/onRecordFound.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,9 @@ import {
} from "../getUrlWaitTime.js";

const defaultStorageApiBaseUrl = "http://storage-api/v0";
const defaultDatsetBucketName = "magda-datasets";
const defaultDatasetBucketName = "magda-datasets";
const jwtSecret = "sdsfsfdsfsddsfsdfdsfds2323432423";
const actionUserId = "user-id-1";
const schema = require("@magda/registry-aspects/source-link-status.schema.json");

describe("onRecordFound", function (this: Mocha.Suite) {
Expand Down Expand Up @@ -379,7 +381,9 @@ describe("onRecordFound", function (this: Mocha.Suite) {
record,
registry,
defaultStorageApiBaseUrl,
defaultDatsetBucketName,
defaultDatasetBucketName,
jwtSecret,
actionUserId,
0,
0,
{},
Expand Down Expand Up @@ -592,7 +596,9 @@ describe("onRecordFound", function (this: Mocha.Suite) {
record,
registry,
defaultStorageApiBaseUrl,
defaultDatsetBucketName,
defaultDatasetBucketName,
jwtSecret,
actionUserId,
retryCount,
0
)
Expand Down Expand Up @@ -716,7 +722,9 @@ describe("onRecordFound", function (this: Mocha.Suite) {
record,
registry,
defaultStorageApiBaseUrl,
defaultDatsetBucketName,
defaultDatasetBucketName,
jwtSecret,
actionUserId,
failures.length,
0,
delayConfig
Expand Down Expand Up @@ -763,7 +771,9 @@ describe("onRecordFound", function (this: Mocha.Suite) {
record,
registry,
defaultStorageApiBaseUrl,
defaultDatsetBucketName
defaultDatasetBucketName,
jwtSecret,
actionUserId
).then(() => {
afterEachProperty();

Expand Down
17 changes: 12 additions & 5 deletions src/test/testStorageUrl.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,12 @@ import {
setDefaultDomainWaitTime,
getDefaultDomainWaitTime
} from "../getUrlWaitTime.js";
import { buildJwt } from "@magda/utils";

const defaultStorageApiBaseUrl = "http://storage-api/v0";
const defaultDatasetBucketName = "magda-datasets";
const jwtSecret = "sdsfsfdsfsddsfsdfdsfds2323432423";
const actionUserId = "user-id-1";
const schema = require("@magda/registry-aspects/source-link-status.schema.json");

describe("Test Internal Storage URL", function (this: Mocha.Suite) {
Expand Down Expand Up @@ -109,23 +112,25 @@ describe("Test Internal Storage URL", function (this: Mocha.Suite) {
};

const defaultStorageApiBaseUri = urijs(defaultStorageApiBaseUrl);

const jwt = buildJwt(jwtSecret, actionUserId);
const storageApiScope = nock(
defaultStorageApiBaseUri.clone().path("").toString()
defaultStorageApiBaseUri.clone().path("").toString(),
{
reqheaders: {
"X-Magda-Session": jwt
}
}
);

storageApiScope
.head(
`${defaultStorageApiBaseUri.path()}/${defaultDatasetBucketName}/ds-1/dist-1/test-file1.pdf`
)
.query(true)
.reply(200);

storageApiScope
.head(
`${defaultStorageApiBaseUri.path()}/${defaultDatasetBucketName}/ds-1/dist-2/test-file2.pdf`
)
.query(true)
.reply(200);

["dist-1", "dist-2"].forEach((distId) => {
Expand Down Expand Up @@ -160,6 +165,8 @@ describe("Test Internal Storage URL", function (this: Mocha.Suite) {
registry,
defaultStorageApiBaseUrl,
defaultDatasetBucketName,
jwtSecret,
actionUserId,
0,
0,
{},
Expand Down

0 comments on commit c24c9c4

Please sign in to comment.