feat: add Actor.useState() helper (#98)

apify · Oct 13, 2022 · 27dc413 · 27dc413
1 parent 8bfeb8e
commit 27dc413
Show file tree

Hide file tree

Showing 11 changed files with 292 additions and 184 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/packages/actor-scraper/cheerio-scraper/package.json b/packages/actor-scraper/cheerio-scraper/package.json
@@ -6,7 +6,7 @@
     "type": "module",
     "dependencies": {
         "@apify/scraper-tools": "^1.0.0",
-        "@crawlee/cheerio": "^3.0.0",
+        "@crawlee/cheerio": "^3.1.0",
         "apify": "^3.0.0"
     },
     "devDependencies": {

diff --git a/packages/actor-scraper/playwright-scraper/package.json b/packages/actor-scraper/playwright-scraper/package.json
@@ -6,9 +6,9 @@
     "type": "module",
     "dependencies": {
         "@apify/scraper-tools": "^1.0.0",
-        "@crawlee/core": "^3.0.0",
-        "@crawlee/playwright": "^3.0.0",
-        "@crawlee/utils": "^3.0.0",
+        "@crawlee/core": "^3.1.0",
+        "@crawlee/playwright": "^3.1.0",
+        "@crawlee/utils": "^3.1.0",
         "apify": "^3.0.0",
         "playwright": "*"
     },

diff --git a/packages/actor-scraper/playwright-scraper/src/internals/crawler_setup.ts b/packages/actor-scraper/playwright-scraper/src/internals/crawler_setup.ts
@@ -11,7 +11,7 @@ import {
     PlaywrightCrawler,
     PlaywrightCrawlerOptions,
     PlaywrightLaunchContext,
-    BrowserCrawlerEnqueueLinksOptions,
+    EnqueueLinksOptions,
     log,
 } from '@crawlee/playwright';
 import { Awaitable, Dictionary } from '@crawlee/utils';
@@ -342,7 +342,7 @@ export class CrawlerSetup implements CrawlerSetupOptions {
             return;
         }
 
-        const enqueueOptions: BrowserCrawlerEnqueueLinksOptions = {
+        const enqueueOptions: EnqueueLinksOptions = {
             globs: this.input.globs,
             pseudoUrls: this.input.pseudoUrls,
             transformRequestFunction: (requestOptions) => {

diff --git a/packages/actor-scraper/puppeteer-scraper/package.json b/packages/actor-scraper/puppeteer-scraper/package.json
@@ -6,7 +6,7 @@
     "type": "module",
     "dependencies": {
         "@apify/scraper-tools": "^1.0.0",
-        "@crawlee/puppeteer": "^3.0.0",
+        "@crawlee/puppeteer": "^3.1.0",
         "apify": "^3.0.0",
         "puppeteer": "*"
     },

diff --git a/packages/actor-scraper/puppeteer-scraper/src/internals/crawler_setup.ts b/packages/actor-scraper/puppeteer-scraper/src/internals/crawler_setup.ts
@@ -11,7 +11,7 @@ import {
     PuppeteerCrawlingContext,
     PuppeteerCrawler,
     PuppeteerCrawlerOptions,
-    BrowserCrawlerEnqueueLinksOptions,
+    EnqueueLinksOptions,
     log,
 } from '@crawlee/puppeteer';
 import { Awaitable, Dictionary } from '@crawlee/utils';
@@ -341,7 +341,7 @@ export class CrawlerSetup implements CrawlerSetupOptions {
             return;
         }
 
-        const enqueueOptions: BrowserCrawlerEnqueueLinksOptions = {
+        const enqueueOptions: EnqueueLinksOptions = {
             globs: this.input.globs,
             pseudoUrls: this.input.pseudoUrls,
             transformRequestFunction: (requestOptions) => {

diff --git a/packages/actor-scraper/web-scraper/package.json b/packages/actor-scraper/web-scraper/package.json
@@ -7,7 +7,7 @@
     "type": "module",
     "dependencies": {
         "@apify/scraper-tools": "^1.0.0",
-        "@crawlee/puppeteer": "^3.0.0",
+        "@crawlee/puppeteer": "^3.1.0",
         "apify": "^3.0.0",
         "content-type": "^1.0.4",
         "devtools-server": "^0.0.2",

diff --git a/packages/apify/package.json b/packages/apify/package.json
@@ -59,9 +59,9 @@
         "@apify/log": "^2.1.4",
         "@apify/timeout": "^0.3.0",
         "@apify/utilities": "^2.2.2",
-        "@crawlee/core": "^3.0.0",
-        "@crawlee/types": "^3.0.0",
-        "@crawlee/utils": "^3.0.0",
+        "@crawlee/core": "^3.1.0",
+        "@crawlee/types": "^3.1.0",
+        "@crawlee/utils": "^3.1.0",
         "apify-client": "^2.6.0",
         "ow": "^0.28.1",
         "semver": "^7.3.7",

diff --git a/packages/apify/src/actor.ts b/packages/apify/src/actor.ts
@@ -22,6 +22,7 @@ import type {
     EventTypeName,
     IStorage,
     RecordOptions,
+    UseStateOptions,
 } from '@crawlee/core';
 import {
     Configuration as CoreConfiguration,
@@ -823,6 +824,41 @@ export class Actor<Data extends Dictionary = Dictionary> {
         return !!process.env[ENV_VARS.IS_AT_HOME];
     }
 
+    /**
+     * Easily create and manage state values. All state values are automatically persisted.
+     *
+     * Values can be modified by simply using the assignment operator.
+     *
+     * @param name The name of the store to use.
+     * @param defaultValue If the store does not yet have a value in it, the value will be initialized with the `defaultValue` you provide.
+     * @param options An optional object parameter where a custom `keyValueStoreName` and `config` can be passed in.
+     */
+    async useState<State extends Dictionary = Dictionary>(
+        name?: string,
+        defaultValue = {} as State,
+        options?: UseStateOptions,
+    ) {
+        const kvStore = await KeyValueStore.open(options?.keyValueStoreName, { config: options?.config || Configuration.getGlobalConfig() });
+        return kvStore.getAutoSavedValue<State>(name || 'APIFY_GLOBAL_STATE', defaultValue);
+    }
+
+    /**
+     * Easily create and manage state values. All state values are automatically persisted.
+     *
+     * Values can be modified by simply using the assignment operator.
+     *
+     * @param name The name of the store to use.
+     * @param defaultValue If the store does not yet have a value in it, the value will be initialized with the `defaultValue` you provide.
+     * @param options An optional object parameter where a custom `keyValueStoreName` and `config` can be passed in.
+     */
+    static async useState<State extends Dictionary = Dictionary>(
+        name?: string,
+        defaultValue = {} as State,
+        options?: UseStateOptions,
+    ) {
+        return Actor.getDefaultInstance().useState<State>(name, defaultValue, options);
+    }
+
     /**
      * Runs the main user function that performs the job of the actor
      * and terminates the process when the user function finishes.

diff --git a/packages/scraper-tools/package.json b/packages/scraper-tools/package.json
@@ -40,17 +40,17 @@
     },
     "devDependencies": {
         "apify": "^3.0.0",
-        "@crawlee/browser-pool": "^3.0.0",
-        "@crawlee/core": "^3.0.0",
-        "@crawlee/types": "^3.0.0",
-        "@crawlee/utils": "^3.0.0"
+        "@crawlee/browser-pool": "^3.1.0",
+        "@crawlee/core": "^3.1.0",
+        "@crawlee/types": "^3.1.0",
+        "@crawlee/utils": "^3.1.0"
     },
     "peerDependencies": {
         "apify": "^3.0.0",
-        "@crawlee/browser-pool": "^3.0.0",
-        "@crawlee/core": "^3.0.0",
-        "@crawlee/types": "^3.0.0",
-        "@crawlee/utils": "^3.0.0"
+        "@crawlee/browser-pool": "^3.1.0",
+        "@crawlee/core": "^3.1.0",
+        "@crawlee/types": "^3.1.0",
+        "@crawlee/utils": "^3.1.0"
     },
     "peerDependenciesMeta": {
         "@crawlee/puppeteer": {

diff --git a/test/apify/actor.test.ts b/test/apify/actor.test.ts
@@ -1009,7 +1009,6 @@ describe('Actor', () => {
             const defaultStore = await KeyValueStore.open();
 
             const oldGet = defaultStore.getValue;
-            // @ts-expect-error TODO use spyOn instead of this
             defaultStore.getValue = async (key) => expect(key).toBe('key-1');
 
             await Actor.getValue('key-1');
@@ -1023,7 +1022,6 @@ describe('Actor', () => {
             const defaultStore = await KeyValueStore.open();
 
             const oldGet = defaultStore.getValue;
-            // @ts-expect-error TODO use spyOn instead of this
             defaultStore.getValue = async (key) => expect(key).toBe('key-1');
 
             await Actor.getValue('key-1');