Skip to content

Commit e872704

Browse files
tobiutobiu
committed
feat: Implement folder-scoped dynamic loading for Data Worker pipelines (#9551)
1 parent 488e6ca commit e872704

9 files changed

Lines changed: 224 additions & 5 deletions

File tree

‎docs/examples.json‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -317,5 +317,19 @@
317317
"name" : "Big Data Tree",
318318
"parentId": 44,
319319
"path" : "../../../examples/grid/treeBigData/MainContainer.mjs"
320+
},
321+
{
322+
"id" : 46,
323+
"isLeaf" : false,
324+
"name" : "data",
325+
"parentId": 1,
326+
"path" : null
327+
},
328+
{
329+
"id" : 47,
330+
"isLeaf" : true,
331+
"name" : "Pipeline",
332+
"parentId": 46,
333+
"path" : "../../../examples/data/pipeline/MainContainer.mjs"
320334
}
321335
]

‎examples/data/pipeline/BeatlesModel.mjs‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
import Model from '../../../src/data/Model.mjs';
2+
3+
/**
4+
* @class Neo.examples.data.pipeline.BeatlesModel
5+
* @extends Neo.data.Model
6+
*/
7+
class BeatlesModel extends Model {
8+
static config = {
9+
className: 'Neo.examples.data.pipeline.BeatlesModel',
10+
fields : [{
11+
name: 'first',
12+
type: 'String'
13+
}, {
14+
name: 'last',
15+
type: 'String'
16+
}, {
17+
name: 'dob',
18+
type: 'Date'
19+
}]
20+
}
21+
}
22+
23+
export default Neo.setupClass(BeatlesModel);

‎examples/data/pipeline/MainContainer.mjs‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
import BeatlesModel from './BeatlesModel.mjs';
2+
import ConnectionFetch from '../../../src/data/connection/Fetch.mjs';
3+
import TabContainer from '../../../src/tab/Container.mjs';
4+
import Table from '../../../src/table/Container.mjs';
5+
import Viewport from '../../../src/container/Viewport.mjs';
6+
7+
/**
8+
* @class Neo.examples.data.pipeline.MainContainer
9+
* @extends Neo.container.Viewport
10+
*/
11+
class MainContainer extends Viewport {
12+
static config = {
13+
className: 'Neo.examples.data.pipeline.MainContainer',
14+
layout : {ntype: 'fit'},
15+
style : {padding: '20px'},
16+
17+
items: [{
18+
module: TabContainer,
19+
items : [{
20+
module : Table,
21+
reference : 'table-app-worker',
22+
wrapperStyle: {height: '100%'},
23+
header: {
24+
iconCls: 'fa fa-microchip',
25+
text : 'App Worker Pipeline'
26+
},
27+
store: {
28+
autoLoad: true,
29+
model : BeatlesModel,
30+
pipeline: {
31+
workerExecution: 'app',
32+
connection: {
33+
module: ConnectionFetch,
34+
url : '../../../resources/data/theBeatles.json'
35+
}
36+
}
37+
},
38+
columns: [{
39+
dataField: 'first',
40+
text : 'First Name'
41+
}, {
42+
dataField: 'last',
43+
text : 'Last Name'
44+
}, {
45+
dataField: 'dob',
46+
text : 'Date of Birth',
47+
renderer : data => data.value ? Intl.DateTimeFormat('default').format(new Date(data.value)) : ''
48+
}]
49+
}, {
50+
module : Table,
51+
reference : 'table-data-worker',
52+
wrapperStyle: {height: '100%'},
53+
header: {
54+
iconCls: 'fa fa-server',
55+
text : 'Data Worker Pipeline'
56+
},
57+
store: {
58+
autoLoad: true,
59+
model : BeatlesModel,
60+
pipeline: {
61+
workerExecution: 'data',
62+
connection: {
63+
className: 'Neo.data.connection.Fetch',
64+
url : '../../../resources/data/theBeatles.json'
65+
}
66+
}
67+
},
68+
columns: [{
69+
dataField: 'first',
70+
text : 'First Name'
71+
}, {
72+
dataField: 'last',
73+
text : 'Last Name'
74+
}, {
75+
dataField: 'dob',
76+
text : 'Date of Birth',
77+
renderer : data => data.value ? Intl.DateTimeFormat('default').format(new Date(data.value)) : ''
78+
}]
79+
}]
80+
}]
81+
}
82+
}
83+
84+
export default Neo.setupClass(MainContainer);

‎examples/data/pipeline/app.mjs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
import MainContainer from './MainContainer.mjs';
2+
3+
export const onStart = () => Neo.app({
4+
mainView: MainContainer,
5+
name : 'Neo.examples.data.pipeline'
6+
});

‎examples/data/pipeline/index.html‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
<!DOCTYPE HTML>
2+
<html>
3+
<head>
4+
<meta name="viewport" content="width=device-width, initial-scale=1">
5+
<meta charset="UTF-8">
6+
<title>Neo Data Pipeline</title>
7+
</head>
8+
<body>
9+
<script src="../../../src/MicroLoader.mjs" type="module"></script>
10+
</body>
11+
</html>

‎examples/data/pipeline/neo-config.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"appPath" : "examples/data/pipeline/app.mjs",
3+
"basePath" : "../../../",
4+
"environment": "development",
5+
"mainPath" : "./Main.mjs"
6+
}

‎src/data/Pipeline.mjs‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -309,6 +309,23 @@ class Pipeline extends Base {
309309
};
310310

311311
try {
312+
// Pre-load required modules in the Data Worker using strictly scoped imports
313+
const modulesToLoad = [];
314+
315+
if (remoteConfig.connection?.className) {
316+
modulesToLoad.push(Neo.worker.Data.loadDataModule({className: remoteConfig.connection.className}))
317+
}
318+
if (remoteConfig.normalizer?.className) {
319+
modulesToLoad.push(Neo.worker.Data.loadDataModule({className: remoteConfig.normalizer.className}))
320+
}
321+
if (remoteConfig.parser?.className) {
322+
modulesToLoad.push(Neo.worker.Data.loadDataModule({className: remoteConfig.parser.className}))
323+
}
324+
325+
if (modulesToLoad.length > 0) {
326+
await Promise.all(modulesToLoad)
327+
}
328+
312329
const data = await Neo.worker.Data.createInstance({
313330
config: remoteConfig,
314331
path : 'src/data/Pipeline.mjs'

‎src/worker/Data.mjs‎

Lines changed: 62 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
1-
import Neo from '../Neo.mjs';
2-
import Base from './Base.mjs';
3-
import Fetch from '../Fetch.mjs';
4-
import Xhr from '../Xhr.mjs';
1+
import Neo from '../Neo.mjs';
2+
import Base from './Base.mjs';
3+
import Fetch from '../Fetch.mjs';
4+
import Instance from '../manager/Instance.mjs';
5+
import Xhr from '../Xhr.mjs';
56

67
/**
78
* The Data worker is responsible to handle all the communication to the backend (e.g. Ajax-calls).
@@ -25,6 +26,7 @@ class Data extends Base {
2526
remote: {
2627
app: [
2728
'createInstance',
29+
'loadDataModule',
2830
'loadModule'
2931
]
3032
},
@@ -103,6 +105,62 @@ class Data extends Base {
103105
}
104106
}
105107

108+
/**
109+
* @summary Dynamically loads a data module into the Data Worker using folder-scoped imports.
110+
* This restricts Webpack's context to specific sub-folders (connection, parser, normalizer),
111+
* preventing bundle bloat.
112+
*
113+
* @param {Object} msg
114+
* @param {String} msg.className The fully qualified class name (e.g., 'Neo.data.connection.Fetch')
115+
* @returns {Promise<Object>} {success: true, className} or {success: false, error}
116+
*/
117+
async loadDataModule({className}) {
118+
const parts = className.split('.');
119+
120+
if (parts[0] !== 'Neo' || parts[1] !== 'data') {
121+
return {success: false, error: 'Not a Neo.data class'};
122+
}
123+
124+
const
125+
type = parts[2],
126+
name = parts.slice(3).join('/');
127+
128+
try {
129+
switch (type) {
130+
case 'connection':
131+
await import(
132+
/* webpackInclude: /src\/data\/connection\/.*\.mjs$/ */
133+
/* webpackExclude: /(?:\/|\\)(buildScripts|dist|node_modules)/ */
134+
/* webpackMode: "lazy" */
135+
`../data/connection/${name}.mjs`
136+
);
137+
break;
138+
case 'parser':
139+
await import(
140+
/* webpackInclude: /src\/data\/parser\/.*\.mjs$/ */
141+
/* webpackExclude: /(?:\/|\\)(buildScripts|dist|node_modules)/ */
142+
/* webpackMode: "lazy" */
143+
`../data/parser/${name}.mjs`
144+
);
145+
break;
146+
case 'normalizer':
147+
await import(
148+
/* webpackInclude: /src\/data\/normalizer\/.*\.mjs$/ */
149+
/* webpackExclude: /(?:\/|\\)(buildScripts|dist|node_modules)/ */
150+
/* webpackMode: "lazy" */
151+
`../data/normalizer/${name}.mjs`
152+
);
153+
break;
154+
default:
155+
return {success: false, error: `Unsupported data module type: ${type}`};
156+
}
157+
return {success: true, className};
158+
} catch (e) {
159+
console.error(`Data Worker: Failed to load data module ${className}`, e);
160+
return {success: false, className, error: e.message}
161+
}
162+
}
163+
106164
/**
107165
* @summary Remotely loads an ES module into the Data Worker.
108166
* This method uses a scoped dynamic import to ensure Webpack only bundles

‎src/worker/mixin/RemoteMethodAccess.mjs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -210,7 +210,7 @@ class RemoteMethodAccess extends Base {
210210
*/
211211
onRemoteMethod(msg) {
212212
let me = this,
213-
pkg = msg.remoteId ? Neo.manager.Instance.get(msg.remoteId) : Neo.ns(msg.remoteClassName),
213+
pkg = msg.remoteId ? Neo.get(msg.remoteId) : Neo.ns(msg.remoteClassName),
214214
out, method;
215215

216216
if (!pkg) {

0 commit comments

Comments
 (0)