Skip to content

Commit

Permalink
feat: page scraping, url add form,
Browse files Browse the repository at this point in the history
observable collection
  • Loading branch information
pixelmord committed Mar 30, 2020
1 parent 4269d26 commit b7c26b2
Show file tree
Hide file tree
Showing 6 changed files with 406 additions and 149 deletions.
7 changes: 5 additions & 2 deletions packages/poolbase-app/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
"lint": "npm-run-all -p -r lint-app lint-functions"
},
"keywords": [],
"author": "Andreas Sahle <andreas.sahle@gmail.com> (https://pixelmord.de/)",
"author": "Andreas Adam <andreas.sahle@gmail.com> (https://pixelmord.de/)",
"license": "MIT",
"engines": {
"node": "10"
Expand All @@ -34,11 +34,14 @@
"firebase-functions": "^3.3.0",
"isomorphic-unfetch": "^3.0.0",
"lodash": "^4.17.15",
"next": "^9.2.1",
"next": "^9.3.2",
"puppeteer": "^2.1.1",
"react": "^16.12.0",
"react-dom": "^16.12.0",
"react-firebaseui": "^4.1.0",
"react-hook-form": "^5.2.0",
"rxfire": "^3.11.1",
"rxjs": "^6.5.4",
"theme-ui": "^0.3.1"
},
"devDependencies": {
Expand Down
68 changes: 68 additions & 0 deletions packages/poolbase-app/src/app/pages/add-url.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
/** @jsx jsx */
import { jsx, Styled } from 'theme-ui';
import { useState, useEffect } from 'react';
import { useForm } from 'react-hook-form';
import { NextPage } from 'next';

import withAuthUser from '../utils/pageWrappers/withAuthUser';
import PageLayout from '../components/PageLayout';
import { PropsWithAuthUserInfo } from '../interfaces';
import { firestore, collectionData, functions } from '../utils/auth/initFirebase';

interface AddUrlProps extends PropsWithAuthUserInfo {
data: Datum[];
}

interface Datum {
id: string;
metaTitle: string;
}

const AddUrlPage: NextPage<{}> = (props): JSX.Element => {
const [data, setData] = useState([]);
const [loading, setLoading] = useState(false);
const [error, setError] = useState('');
const { register, handleSubmit } = useForm();

const onSubmit = async (data): Promise<void> => {
try {
setLoading(true);
const addURL = functions.httpsCallable('addURL');
await addURL(data);
setLoading(false);
} catch (error) {
console.error(error);
setError(error.message);
setLoading(false);
}
};

useEffect(() => {
const query = firestore
.collection('pages')
.orderBy('created', 'desc')
.limit(3);
const subscription = collectionData(query, 'id').subscribe(setData);
// Specify how to clean up after this effect:
return function cleanup(): void {
subscription.unsubscribe();
};
}, []);
return (
<PageLayout>
<form onSubmit={handleSubmit(onSubmit)}>
{error ? <h3 className="error">{error}</h3> : null}
<label htmlFor="url">URL</label>
<input name="url" required ref={register} />
<button disabled={loading}>{loading ? 'Loading...' : 'Add URL'}</button>
</form>
{data.map(
(page): JSX.Element => (
<Styled.h3 key={page.id}>{!page.metaTitle ? page.url : page.metaTitle}</Styled.h3>
)
)}
</PageLayout>
);
};

export default withAuthUser(AddUrlPage);
25 changes: 25 additions & 0 deletions packages/poolbase-app/src/functions/handlers/pageCreateHandler.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
import * as functions from 'firebase-functions';
import { scrapeHTML } from '../processing';
import { firestore } from '../initFirebase';

export const pageCreateHandler = functions
.region('europe-west1')
.runWith({
timeoutSeconds: 540,
memory: '1GB',
})
.firestore.document('pages/{pageId}')
.onCreate(async (snapshot, context) => {
const page = snapshot.data();
try {
if (typeof page !== 'undefined' && typeof page.url !== 'undefined') {
const data = await scrapeHTML(page.url);
firestore
.collection('pages')
.doc(context.params.pageId)
.update(data);
}
} catch (e) {
console.log(e);
}
});
12 changes: 8 additions & 4 deletions packages/poolbase-app/src/functions/processing/scrapeHTML.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ export const scrapeHTML = async (url: string): Promise<ScrapeData> => {
metaTitle: null,
status: null,
};
const browser = await puppeteer.launch({ headless: false });
const browser = await puppeteer.launch({ headless: true, args: ['--no-sandbox', '--disable-setuid-sandbox'] });

const page = await browser.newPage();
try {
Expand All @@ -29,7 +29,13 @@ export const scrapeHTML = async (url: string): Promise<ScrapeData> => {
await page.waitForSelector('head > title');
const extractedFromHEAD = await page.evaluate(() => {
return {
...(!!document.querySelector('title') && { metaTitle: document!.querySelector('title')!.innerText.trim() }),
...(!!document.querySelector('title') && { metaTitle: document?.querySelector('title')?.innerText.trim() }),
...(!!document.querySelector('meta[name=keywords]') && {
metaKeywords: document?.querySelector('meta[name=author]')?.getAttribute('content'),
}),
...(!!document.querySelector('meta[name=author]') && {
metaAuthor: document?.querySelector('meta[name=author]')?.getAttribute('content'),
}),
};
});
data = {
Expand All @@ -42,5 +48,3 @@ export const scrapeHTML = async (url: string): Promise<ScrapeData> => {

return data;
};

scrapeHTML('https://github.com/drejohnson/sapper-graphql-firebase');
14 changes: 13 additions & 1 deletion packages/poolbase-app/yarn.lock
Original file line number Diff line number Diff line change
Expand Up @@ -7699,6 +7699,11 @@ react-firebaseui@^4.1.0:
dependencies:
firebaseui "^4.1.0"

react-hook-form@^5.2.0:
version "5.2.0"
resolved "https://registry.yarnpkg.com/react-hook-form/-/react-hook-form-5.2.0.tgz#b5b654516ee03d55d78b7b9e194c7f4632885426"
integrity sha512-EqGCSl3DxSUBtL/9lFvrFQLJ7ICdVKrfjcMHay2SvmU4trR8aqrd7YuiLSojBKmZBRdBnCcxG+LzLWF9z474NA==

react-is@16.8.6:
version "16.8.6"
resolved "https://registry.yarnpkg.com/react-is/-/react-is-16.8.6.tgz#5bbc1e2d29141c9fbdfed456343fe2bc430a6a16"
Expand Down Expand Up @@ -8036,7 +8041,14 @@ run-queue@^1.0.0, run-queue@^1.0.3:
dependencies:
aproba "^1.1.1"

rxjs@^6.5.3:
rxfire@^3.11.1:
version "3.11.1"
resolved "https://registry.yarnpkg.com/rxfire/-/rxfire-3.11.1.tgz#84e7874f71bba47930158591cb7b7375000517ab"
integrity sha512-5kxDE2nvnRt0dnIRp0NYqnUDaVuV4kV5lijAppbRwxQFTWq501sJ4Ijzxurfck/g6VwZO9YrzrKOZaqaMyByIA==
dependencies:
tslib "1.11.1"

rxjs@^6.5.3, rxjs@^6.5.4:
version "6.5.4"
resolved "https://registry.yarnpkg.com/rxjs/-/rxjs-6.5.4.tgz#e0777fe0d184cec7872df147f303572d414e211c"
integrity sha512-naMQXcgEo3csAEGvw/NydRA0fuS2nDZJiw1YUWFKU7aPPAPGZEsD4Iimit96qwCieH6y614MCLYwdkrWx7z/7Q==
Expand Down
Loading

0 comments on commit b7c26b2

Please sign in to comment.