Skip to content
43 changes: 24 additions & 19 deletions components/apify/actions/get-dataset-items/get-dataset-items.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ export default {
key: "apify-get-dataset-items",
name: "Get Dataset Items",
description: "Returns data stored in a dataset. [See the documentation](https://docs.apify.com/api/v2/dataset-items-get)",
version: "0.0.6",
version: "0.0.7",
annotations: {
destructiveHint: false,
openWorldHint: true,
Expand Down Expand Up @@ -52,32 +52,37 @@ export default {
},
},
async run({ $ }) {
const params = {
limit: LIMIT,
offset: this.offset,
clean: this.clean,
fields: this.fields,
omit: this.omit,
};
const {
clean, fields, omit, limit,
} = this;
const datasetId = this.datasetId?.replace("/", "~");
const offset = this.offset ?? 0;

const results = [];
let total;
let currentOffset = offset;

do {
while (limit === undefined || results.length < limit) {
const pageSize = limit === undefined
? LIMIT
: Math.min(LIMIT, limit - results.length);
const { items } = await this.apify.listDatasetItems({
datasetId: this.datasetId,
params,
datasetId,
params: {
offset: currentOffset,
limit: pageSize,
clean,
fields,
omit,
},
});
if (!items?.length) {
break;
}
results.push(...items);
if (results.length >= this.limit) {
currentOffset += items.length;
if (items.length < pageSize) {
break;
}
total = items?.length;
params.offset += LIMIT;
} while (total);

if (results.length > this.limit) {
results.length = this.limit;
}

if (results.length > 0) {
Expand Down
10 changes: 5 additions & 5 deletions components/apify/apify.app.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ export default {
datasetId: {
type: "string",
label: "Dataset ID",
description: "The ID of the dataset to retrieve items within",
description: "Select a dataset, or enter a Dataset ID or `username/dataset-name`",
async options({ page }) {
const { items } = await this.listDatasets({
offset: LIMIT * page,
Expand Down Expand Up @@ -119,15 +119,15 @@ export default {
limit: {
type: "integer",
label: "Limit",
description: "The maximum number of items to return",
default: LIMIT,
description: "The maximum number of items to return. Leave empty to return all items",
min: 1,
optional: true,
},
offset: {
type: "integer",
label: "Offset",
description: "The number records to skip before returning results",
default: 0,
description: "The number of records to skip before returning results. Leave empty to start from the first item",
min: 0,
optional: true,
},
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ const props = adjustPropDefinitions(others.props, app);
export default {
...others,
key: "apify_oauth-get-dataset-items",
version: "0.0.2",
version: "0.0.3",
name,
description,
type,
Expand Down