Better script gen & fix recording stop listening

baptisteArno · Mar 17, 2021 · a01737e · a01737e
1 parent f9252db
commit a01737e
Show file tree

Hide file tree

Showing 5 changed files with 30 additions and 13 deletions.
diff --git a/public/content.js b/public/content.js
@@ -269,8 +269,8 @@ const stopSelectNode = () => {
   for (const handler of handlers) {
     document.removeEventListener("click", handler, { capture: true });
   }
-  handlers = [];
   stopClicksKeysRecording();
+  handlers = [];
 };
 
 const onRecordClick = function (stepIndex) {

diff --git a/public/manifest.json b/public/manifest.json
@@ -1,6 +1,6 @@
 {
   "name": "Tinking - Scrapping Tool",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "manifest_version": 2,
   "description": "Extract data from any website without code, just clicks",
   "icons": {

diff --git a/src/StepItem/OptionItem.tsx b/src/StepItem/OptionItem.tsx
@@ -224,7 +224,7 @@ const SelectOption = ({
   optionType?: OptionType;
   onOptionChange: (val: OptionType) => void;
 }) => (
-  <Menu>
+  <Menu defaultIsOpen={optionType === undefined}>
     <MenuButton
       size="sm"
       as={Button}

diff --git a/src/StepItem/StepItem.tsx b/src/StepItem/StepItem.tsx
@@ -495,7 +495,7 @@ const SelectAction = ({
   step: Step;
   onActionChange: (val: StepAction) => void;
 }) => (
-  <Menu>
+  <Menu defaultIsOpen={step.action === undefined}>
     <MenuButton size="sm" as={Button} rightIcon={<ChevronDownIcon />}>
       {step.action ?? "Select an action"}
     </MenuButton>

diff --git a/src/lib/scriptGenerator.ts b/src/lib/scriptGenerator.ts
@@ -246,7 +246,7 @@ const parseSingleCommandFromStep = (
           const element = document.querySelector("${step.selector}")
           return element.src || null;
         }
-        let ${variableName} = await page.evaluate(() => ${variableName}Eval);
+        let ${variableName} = await page.evaluate(${variableName}Eval);
         if(${variableName} === null || ${variableName} === ""){
           // The content could be dynamically loaded. Waiting a bit...
           await page.waitForTimeout(4000)
@@ -341,9 +341,13 @@ const parseLoopFromStep = (step: Step) => {
       await page.waitForSelector("${step.selector}")
       let urls = []
       urls = await page.evaluate(() => {
-        return [...document.querySelectorAll("${step.selector}")].map((node) => node.href);
+        return [...document.querySelectorAll("${
+          step.selector
+        }")].map((node) => node.href);
       });
-      if(urls.length >= ${amountToExtract}){
+      if(${
+        amountToExtract === "" ? "false" : `urls.length >= ${amountToExtract}`
+      }){
         urls = urls.slice(0, ${amountToExtract})
       } else {
         let i = 0
@@ -366,18 +370,30 @@ const parseLoopFromStep = (step: Step) => {
           }catch{
             break;
           }
-          const firstLinkInNewPage = (
-            await page.$("${step.selector}")
-          ).href;
-          if (!firstLinkInNewPage || firstLinkInNewPage === firstLinkInCurrentPage) {
+          let firstLinkInNewPage = await page.evaluate(() => {return document.querySelector("${
+            step.selector
+          }").href});
+          if (firstLinkInNewPage === firstLinkInCurrentPage) {
             // There is some kind of loading state we need to wait for
             await page.waitForTimeout(4000);
+            firstLinkInNewPage = await page.evaluate(() => {return document.querySelector("${
+              step.selector
+            }").href});
+            if (firstLinkInNewPage === firstLinkInCurrentPage) {
+              break;
+            }
           }
           const newUrls = await page.evaluate(() => {
-            return [...document.querySelectorAll("${step.selector}")].map(node => node.href);
+            return [...document.querySelectorAll("${
+              step.selector
+            }")].map(node => node.href);
           })
           urls = urls.concat(newUrls)
-          if (urls.length >= ${amountToExtract}) {
+          if (${
+            amountToExtract === ""
+              ? "false"
+              : `urls.length >= ${amountToExtract}`
+          }) {
             urls = urls.slice(0, ${amountToExtract})
             break;
           }
@@ -460,6 +476,7 @@ const parseLibrarySettings = (library: "puppeteer" | "playwright") => {
         "--no-sandbox",
         "--disable-setuid-sandbox",
         "--disable-dev-shm-usage",
+        "--window-size=1300,1024"
       ],
     });`;
   }