Skip to content

fix: use new URL for URL normalization #365

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Mar 20, 2023
Merged

fix: use new URL for URL normalization #365

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
24b6a44
fix: use `new URL` for URL normalization
B4nan Mar 16, 2023
d88f2fb
refactor: keep spaces in the URL before parsing
B4nan Mar 17, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 25 additions & 19 deletions packages/utilities/src/utilities.client.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,9 @@ interface Uri {
fragmentKey?: Record<string, unknown>;
}

/**
* @deprecated use `new URL()` instead
*/
export function parseUrl(str: string): Uri {
if (typeof str !== 'string') return {};
const o = {
Expand Down Expand Up @@ -127,28 +130,31 @@ export function normalizeUrl(url: string, keepFragment?: boolean) {
return null;
}

const urlObj = parseUrl(url.trim());
if (!urlObj.protocol || !urlObj.host) {
let urlObj;

try {
urlObj = new URL(url.trim());
} catch {
return null;
}

const path = urlObj.path!.replace(/\/$/, '');
const params = (urlObj.query
? urlObj.query
.split('&')
.filter((param) => {
return !/^utm_/.test(param);
})
.sort()
: []
);

return `${urlObj.protocol.trim().toLowerCase()
}://${
urlObj.host.trim().toLowerCase()
}${path.trim()
}${params.length ? `?${params.join('&').trim()}` : ''
}${keepFragment && urlObj.fragment ? `#${urlObj.fragment.trim()}` : ''}`;
const { searchParams } = urlObj;

for (const key of [...searchParams.keys()]) {
if (key.startsWith('utm_')) {
searchParams.delete(key);
}
}

searchParams.sort();

const protocol = urlObj.protocol.toLowerCase();
const host = urlObj.host.toLowerCase();
const path = urlObj.pathname.replace(/\/$/, '');
const search = searchParams.toString() ? `?${searchParams}` : '';
const hash = keepFragment ? urlObj.hash : '';

return `${protocol}//${host}${path}${search}${hash}`;
}

// Helper function for markdown rendered marked
Expand Down
27 changes: 22 additions & 5 deletions test/utilities.client.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -487,6 +487,16 @@ describe('utilities.client', () => {
expect(normalizeUrl('https://example.com')).toEqual('https://example.com');
});

it('edge cases', () => {
expect(normalizeUrl('a https://example.com b')).toEqual(null);
expect(normalizeUrl('https://example.com?q=foo bar')).toEqual('https://example.com?q=foo+bar');
expect(normalizeUrl('https://example.com?q=foo+bar')).toEqual('https://example.com?q=foo+bar');
expect(normalizeUrl('https://google.com/maps/search/restaurant prague/@39.1029725,39.5483593,4z'))
.toEqual('https://google.com/maps/search/restaurant%20prague/@39.1029725,39.5483593,4z');
expect(normalizeUrl('https://google.com/maps/search/restaurantprague/@39.1029725,39.5483593,4z'))
.toEqual('https://google.com/maps/search/restaurantprague/@39.1029725,39.5483593,4z');
});

it('should lowercase hostname and protocols', () => {
expect(normalizeUrl('httpS://EXAMPLE.cOm')).toEqual('https://example.com');
expect(normalizeUrl('hTTp://www.EXAMPLE.com')).toEqual('http://www.example.com');
Expand Down Expand Up @@ -527,7 +537,12 @@ describe('utilities.client', () => {
});

it('should not touch invalid or empty params', () => {
expect(normalizeUrl('http://example.com/?neco=&dalsi')).toEqual('http://example.com?dalsi&neco=');
expect(normalizeUrl('http://example.com/?neco=&dalsi')).toEqual('http://example.com?dalsi=&neco=');
});

it('should work with @ inside query', () => {
expect(normalizeUrl('https://www.google.com/maps/search/restaurant/@39.102972537998426,39.54835927707177,4z?foo=bar&aaa=bbb'))
.toEqual('https://www.google.com/maps/search/restaurant/@39.102972537998426,39.54835927707177,4z?aaa=bbb&foo=bar');
});

it('should normalize real-world URLs', () => {
Expand All @@ -541,10 +556,12 @@ describe('utilities.client', () => {
expect(normalizeUrl('http://notebooky.heureka.cz/f:2111:25235;2278:9720,9539;p:579,580/')).toEqual(expected);
});

it('should trim all parts of URL', () => {
expect(normalizeUrl(' http :// test # fragment ')).toEqual('http://test');
expect(normalizeUrl(' http :// test # fragment ', true)).toEqual('http://test#fragment');
});
// this is no longer a valid URL and results in `null`, if we want to support it,
// we will need some regexp magic to first remove the spaces
// it('should trim all parts of URL', () => {
// expect(normalizeUrl(' http :// test # fragment ')).toEqual('http://test');
// expect(normalizeUrl(' http :// test # fragment ', true)).toEqual('http://test#fragment');
// });
});
describe('#buildOrVersionNumberIntToStr()', () => {
it('should convert build number to string', () => {
Expand Down