diff --git a/LICENSE b/LICENSE index 9071541d86bb..88872bb1d73d 100644 --- a/LICENSE +++ b/LICENSE @@ -1,13 +1,13 @@ Source code in this repository is variously licensed under the Apache License -Version 2.0 (see file ./LICENSE-APACHE),or the AGPLv3 License (see file ./LICENSE-AGPL) +Version 2.0 (see file ./LICENSE-APACHE), or the AGPLv3 License (see file ./LICENSE-AGPL) Every file is under copyright (c) Windmill Labs, Inc 2022 unless otherwise specified. Every file is under License AGPL unless otherwise specified or belonging to one of the below cases: -The files under backend/ are AGPL Licensed. -The files under frontend/ are AGPL Licensed. +The files under backend/ are AGPLv3 Licensed. +The files under frontend/ are AGPLv3 Licensed. The files under python-client/ deno-client/ go-client/ are Apache 2.0 Licensed. The openapi files, including the OpenFlow spec is Apache 2.0 Licensed. diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 43cd56699734..546fda96a96f 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -74,7 +74,23 @@ pub async fn pip_compile( ) -> error::Result { logs.push_str(&format!("\nresolving dependencies...")); set_logs(logs, job_id, db).await; - logs.push_str(&format!("\ncontent of requirements:\n{}", requirements)); + logs.push_str(&format!("\ncontent of requirements:\n{}\n", requirements)); + let requirements = if let Some(pip_local_dependencies) = PIP_LOCAL_DEPENDENCIES.as_ref() { + let deps = pip_local_dependencies.clone(); + requirements + .lines() + .filter(|s| { + if !deps.contains(&s.to_string()) { + return true; + } else { + logs.push_str(&format!("\nignoring local dependency: {}", s)); + return false; + } + }) + .join("\n") + } else { + requirements.to_string() + }; let req_hash = calculate_hash(&requirements); if let Some(cached) = sqlx::query_scalar!( "SELECT lockfile FROM pip_resolution_cache WHERE hash = $1", @@ -87,15 +103,7 @@ pub async fn pip_compile( return Ok(cached); } let file = "requirements.in"; - let requirements = if let Some(pip_local_dependencies) = PIP_LOCAL_DEPENDENCIES.as_ref() { - let deps = pip_local_dependencies.clone(); - requirements - .lines() - .filter(|s| !deps.contains(&s.to_string())) - .join("\n") - } else { - requirements.to_string() - }; + write_file(job_dir, file, &requirements).await?; let mut args = vec!["-q", "--no-header", file, "--resolver=backtracking"]; diff --git a/docker/DockerfileOpenbb b/docker/DockerfileOpenbb index 82e20e81ebcb..ec84ac2a826b 100644 --- a/docker/DockerfileOpenbb +++ b/docker/DockerfileOpenbb @@ -90,7 +90,7 @@ FROM python:3.10.8-slim-buster ARG APP=/usr/src/app RUN apt-get update \ - && apt-get install -y ca-certificates wget curl git jq libprotobuf-dev libnl-route-3-dev unzip build-essential pkg-config libcairo2-dev \ + && apt-get install -y ca-certificates wget curl git jq libprotobuf-dev libnl-route-3-dev unzip build-essential pkg-config libcairo2-dev libwebkit2gtk-4.0-37 \ && rm -rf /var/lib/apt/lists/* RUN arch="$(dpkg --print-architecture)"; arch="${arch##*-}"; \ diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 63fe17490354..9e01c6bb01c8 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -22,6 +22,7 @@ "chartjs-plugin-zoom": "^2.0.0", "d3-zoom": "^3.0.0", "date-fns": "^2.29.3", + "esm-env": "^1.0.0", "fast-equals": "^5.0.1", "highlight.js": "^11.8.0", "lodash": "^4.17.21", @@ -2940,8 +2941,7 @@ "node_modules/esm-env": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/esm-env/-/esm-env-1.0.0.tgz", - "integrity": "sha512-Cf6VksWPsTuW01vU9Mk/3vRue91Zevka5SjyNf3nEpokFRuqt/KjUQoGAwq9qMmhpLTHmXzSIrFRw8zxWzmFBA==", - "dev": true + "integrity": "sha512-Cf6VksWPsTuW01vU9Mk/3vRue91Zevka5SjyNf3nEpokFRuqt/KjUQoGAwq9qMmhpLTHmXzSIrFRw8zxWzmFBA==" }, "node_modules/esm-env-robust": { "version": "0.0.3", @@ -9452,8 +9452,7 @@ "esm-env": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/esm-env/-/esm-env-1.0.0.tgz", - "integrity": "sha512-Cf6VksWPsTuW01vU9Mk/3vRue91Zevka5SjyNf3nEpokFRuqt/KjUQoGAwq9qMmhpLTHmXzSIrFRw8zxWzmFBA==", - "dev": true + "integrity": "sha512-Cf6VksWPsTuW01vU9Mk/3vRue91Zevka5SjyNf3nEpokFRuqt/KjUQoGAwq9qMmhpLTHmXzSIrFRw8zxWzmFBA==" }, "esm-env-robust": { "version": "0.0.3", diff --git a/frontend/package.json b/frontend/package.json index 17d969726131..e48a3357bde7 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -82,6 +82,7 @@ "chartjs-plugin-zoom": "^2.0.0", "d3-zoom": "^3.0.0", "date-fns": "^2.29.3", + "esm-env": "^1.0.0", "fast-equals": "^5.0.1", "highlight.js": "^11.8.0", "lodash": "^4.17.21", @@ -157,6 +158,16 @@ "svelte": "./package/components/FlowViewer.svelte", "default": "./package/components/FlowViewer.svelte" }, + "./components/FlowBuilder.svelte": { + "types": "./package/components/FlowBuilder.svelte.d.ts", + "svelte": "./package/components/FlowBuilder.svelte", + "default": "./package/components/FlowBuilder.svelte" + }, + "./components/FlowEditor.svelte": { + "types": "./package/components/flows/FlowEditor.svelte.d.ts", + "svelte": "./package/components/flows/FlowEditor.svelte", + "default": "./package/components/flows/FlowEditor.svelte" + }, "./components/SchemaViewer.svelte": { "types": "./package/components/SchemaViewer.svelte.d.ts", "svelte": "./package/components/SchemaViewer.svelte", @@ -171,6 +182,10 @@ "types": "./package/common.d.ts", "default": "./package/common.js" }, + "./stores": { + "types": "./package/stores.d.ts", + "default": "./package/stores.js" + }, "./components/icons": { "types": "./package/components/icons/index.d.ts", "svelte": "./package/components/icons/index.js", @@ -193,7 +208,7 @@ "dist", "package" ], - "license": "Apache-2.0", + "license": "AGPL-3.0", "svelte": "./dist/index.js", "typesVersions": { ">4.0": { @@ -227,6 +242,12 @@ "components/FlowViewer.svelte": [ "./package/components/FlowViewer.svelte.d.ts" ], + "components/FlowBuilder.svelte": [ + "./package/components/FlowBuilder.svelte.d.ts" + ], + "components/FlowEditor.svelte": [ + "./package/components/flows/FlowEditor.svelte.d.ts" + ], "components/SchemaViewer.svelte": [ "./package/components/SchemaViewer.svelte.d.ts" ], @@ -236,6 +257,9 @@ "common": [ "./package/common.d.ts" ], + "stores": [ + "./package/stores.d.ts" + ], "components/icons": [ "./package/components/icons/index.d.ts" ], diff --git a/frontend/src/lib/cloud.ts b/frontend/src/lib/cloud.ts new file mode 100644 index 000000000000..57afa10b80a5 --- /dev/null +++ b/frontend/src/lib/cloud.ts @@ -0,0 +1,43 @@ +import { BROWSER } from 'esm-env' +import { page } from '$app/stores' +import { get } from 'svelte/store' +import { premiumStore, userStore, workspaceStore } from './stores' +import { getUserExt } from './user' +import { WorkspaceService } from './gen' + +export function isCloudHosted(): boolean { + return get(page)?.url?.hostname == 'app.windmill.dev' +} + +if (BROWSER) { + workspaceStore.subscribe(async (workspace) => { + if (workspace) { + try { + localStorage.setItem('workspace', String(workspace)) + } catch (e) { + console.error('Could not persist workspace to local storage', e) + } + const user = await getUserExt(workspace) + userStore.set(user) + if (isCloudHosted() && user?.is_admin) { + premiumStore.set(await WorkspaceService.getPremiumInfo({ workspace })) + } + } else { + userStore.set(undefined) + } + }) + + setInterval(async () => { + try { + const workspace = get(workspaceStore) + const user = get(userStore) + + if (workspace && user && !user.is_super_admin && !user.is_admin) { + userStore.set(await getUserExt(workspace)) + console.log('refreshed user') + } + } catch (e) { + console.error('Could not refresh user', e) + } + }, 30000) +} diff --git a/frontend/src/lib/components/AddUser.svelte b/frontend/src/lib/components/AddUser.svelte index e1dbf442f3c2..46066896f4f8 100644 --- a/frontend/src/lib/components/AddUser.svelte +++ b/frontend/src/lib/components/AddUser.svelte @@ -1,10 +1,10 @@ { e.preventDefault() - copyToClipboard( - `${$page.url.protocol}//${$page.url.hostname}/api/w/${$workspaceStore}/capture_u/${$flowStore.path}` - ) + copyToClipboard(`${hostname}/api/w/${$workspaceStore}/capture_u/${$flowStore.path}`) }} - href="{$page.url.protocol}//{$page.url - .hostname}/api/w/{$workspaceStore}/capture_u/{$flowStore.path}" - >{$page.url.protocol}//{$page.url - .hostname}/api/w/{$workspaceStore}/capture_u/{$flowStore.path} + href="{hostname}/api/w/{$workspaceStore}/capture_u/{$flowStore.path}" + >{hostname}/api/w/{$workspaceStore}/capture_u/{$flowStore.path} @@ -78,7 +76,7 @@ 
{`curl -X POST ${$page.url.protocol}//${$page.url.hostname}/api/w/${$workspaceStore}/capture_u/${$flowStore.path} \\
+				>{`curl -X POST ${hostname}/api/w/${$workspaceStore}/capture_u/${$flowStore.path} \\
    -H 'Content-Type: application/json' \\
    -d '{"foo": 42}'`}
diff --git a/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte b/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte index 4c8d85a2e5e6..7a6c48af4e6d 100644 --- a/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte +++ b/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte @@ -10,7 +10,7 @@ import { RawScript, type FlowModule, type PathFlow, type PathScript } from '$lib/gen' import FlowCard from '../common/FlowCard.svelte' import FlowModuleHeader from './FlowModuleHeader.svelte' - import { getLatestHashForScript, schemaToObject, scriptLangToEditorLang } from '$lib/utils' + import { getLatestHashForScript, scriptLangToEditorLang } from '$lib/scripts' import PropPickerWrapper from '../propPicker/PropPickerWrapper.svelte' import { afterUpdate, getContext } from 'svelte' import type { FlowEditorContext } from '../types' @@ -27,6 +27,7 @@ import FlowModuleSleep from './FlowModuleSleep.svelte' import FlowPathViewer from './FlowPathViewer.svelte' import InputTransformSchemaForm from '$lib/components/InputTransformSchemaForm.svelte' + import { schemaToObject } from '$lib/schema' const { selectedId, previewArgs, flowStateStore, flowStore, saveDraft } = getContext('FlowEditorContext') diff --git a/frontend/src/lib/components/flows/content/FlowModuleHeader.svelte b/frontend/src/lib/components/flows/content/FlowModuleHeader.svelte index 217c36ea3087..c1a05632fbe7 100644 --- a/frontend/src/lib/components/flows/content/FlowModuleHeader.svelte +++ b/frontend/src/lib/components/flows/content/FlowModuleHeader.svelte @@ -6,8 +6,9 @@ import { Bed, PhoneIncoming, Repeat, Square } from 'lucide-svelte' import Popover from '../../Popover.svelte' import type { FlowEditorContext } from '../types' - import { getLatestHashForScript, sendUserToast } from '$lib/utils' + import { sendUserToast } from '$lib/utils' import { workerTags } from '$lib/stores' + import { getLatestHashForScript } from '$lib/scripts' export let module: FlowModule const { scriptEditorDrawer } = getContext('FlowEditorContext') diff --git a/frontend/src/lib/components/flows/content/FlowModuleScript.svelte b/frontend/src/lib/components/flows/content/FlowModuleScript.svelte index 71c487470e7e..60be6d7e6d83 100644 --- a/frontend/src/lib/components/flows/content/FlowModuleScript.svelte +++ b/frontend/src/lib/components/flows/content/FlowModuleScript.svelte @@ -1,8 +1,8 @@
@@ -107,7 +108,7 @@ e.preventDefault() copyToClipboard(url) }} - href={$page.url.protocol + '//' + url} + href={url} class="whitespace-nowrap text-ellipsis overflow-hidden mr-1" > {url} @@ -123,7 +124,7 @@ e.preventDefault() copyToClipboard(syncedUrl) }} - href={$page.url.protocol + '//' + syncedUrl} + href={syncedUrl} class="whitespace-nowrap text-ellipsis overflow-hidden mr-1" > {syncedUrl} diff --git a/frontend/src/lib/components/flows/flowStateUtils.ts b/frontend/src/lib/components/flows/flowStateUtils.ts index d875ae051b7c..85178e692e1d 100644 --- a/frontend/src/lib/components/flows/flowStateUtils.ts +++ b/frontend/src/lib/components/flows/flowStateUtils.ts @@ -10,7 +10,7 @@ import { } from '$lib/gen' import { initialCode } from '$lib/script_helpers' import { userStore, workspaceStore } from '$lib/stores' -import { getScriptByPath } from '$lib/utils' +import { getScriptByPath } from '$lib/scripts' import { get, type Writable } from 'svelte/store' import type { FlowModuleState, FlowState } from './flowState' import { charsToNumber, numberToChars } from './idUtils' diff --git a/frontend/src/lib/components/flows/pickers/PickHubApp.svelte b/frontend/src/lib/components/flows/pickers/PickHubApp.svelte index 568c652039b4..97c4461cbec0 100644 --- a/frontend/src/lib/components/flows/pickers/PickHubApp.svelte +++ b/frontend/src/lib/components/flows/pickers/PickHubApp.svelte @@ -2,10 +2,10 @@ import { createEventDispatcher, onMount } from 'svelte' import { Badge, Skeleton } from '$lib/components/common' import SearchItems from '$lib/components/SearchItems.svelte' - import { loadHubApps } from '$lib/utils' import ListFilters from '$lib/components/home/ListFilters.svelte' import NoItemFound from '$lib/components/home/NoItemFound.svelte' import RowIcon from '$lib/components/common/table/RowIcon.svelte' + import { loadHubApps } from '$lib/hub' export let filter = '' @@ -53,7 +53,7 @@
-
+
{#if item.marked} {@html item.marked ?? ''} diff --git a/frontend/src/lib/components/flows/pickers/PickHubFlow.svelte b/frontend/src/lib/components/flows/pickers/PickHubFlow.svelte index debcdd2c208a..e04b0357927c 100644 --- a/frontend/src/lib/components/flows/pickers/PickHubFlow.svelte +++ b/frontend/src/lib/components/flows/pickers/PickHubFlow.svelte @@ -2,10 +2,10 @@ import { createEventDispatcher, onMount } from 'svelte' import { Badge, Skeleton } from '$lib/components/common' import SearchItems from '$lib/components/SearchItems.svelte' - import { loadHubFlows } from '$lib/utils' import ListFilters from '$lib/components/home/ListFilters.svelte' import NoItemFound from '$lib/components/home/NoItemFound.svelte' import RowIcon from '$lib/components/common/table/RowIcon.svelte' + import { loadHubFlows } from '$lib/hub' export let filter = '' @@ -53,7 +53,7 @@
-
+
{#if item.marked} {@html item.marked ?? ''} diff --git a/frontend/src/lib/components/flows/pickers/PickHubScript.svelte b/frontend/src/lib/components/flows/pickers/PickHubScript.svelte index 2189e34b348a..652b40a58933 100644 --- a/frontend/src/lib/components/flows/pickers/PickHubScript.svelte +++ b/frontend/src/lib/components/flows/pickers/PickHubScript.svelte @@ -4,10 +4,11 @@ import type { HubItem } from './model' import { Badge, Skeleton } from '$lib/components/common' import SearchItems from '$lib/components/SearchItems.svelte' - import { capitalize, classNames, loadHubScripts } from '$lib/utils' + import { capitalize, classNames } from '$lib/utils' import NoItemFound from '$lib/components/home/NoItemFound.svelte' import { APP_TO_ICON_COMPONENT } from '$lib/components/icons' import ListFilters from '$lib/components/home/ListFilters.svelte' + import { loadHubScripts } from '$lib/scripts' export let kind: 'script' | 'trigger' | 'approval' | 'failure' = 'script' export let filter = '' @@ -61,7 +62,7 @@ />
-
+
{#if item.marked} {@html item.marked ?? ''} @@ -69,7 +70,7 @@ {item.summary ?? ''} {/if}
-
+
{item.path}
diff --git a/frontend/src/lib/components/flows/previousResults.ts b/frontend/src/lib/components/flows/previousResults.ts index 6a7283ed07a6..f477a45d22ae 100644 --- a/frontend/src/lib/components/flows/previousResults.ts +++ b/frontend/src/lib/components/flows/previousResults.ts @@ -1,12 +1,12 @@ import type { Schema } from '$lib/common' import type { Flow, FlowModule } from '$lib/gen' -import { schemaToObject } from '$lib/utils' +import { schemaToObject } from '$lib/schema' import type { FlowState } from './flowState' export type PickableProperties = { flow_input: Object priorIds: Record - previousId: string | undefined, + previousId: string | undefined hasResume: boolean } @@ -15,7 +15,6 @@ type StepPropPicker = { extraLib: string } - type ModuleBranches = FlowModule[][] function getSubModules(flowModule: FlowModule): ModuleBranches { @@ -24,19 +23,18 @@ function getSubModules(flowModule: FlowModule): ModuleBranches { } else if (flowModule.value.type === 'branchall') { return flowModule.value.branches.map((branch) => branch.modules) } else if (flowModule.value.type == 'branchone') { - return [ - ...flowModule.value.branches.map((branch) => branch.modules), - flowModule.value.default - ] + return [...flowModule.value.branches.map((branch) => branch.modules), flowModule.value.default] } return [] } function getAllSubmodules(flowModule: FlowModule): ModuleBranches { return getSubModules(flowModule).map((modules) => { - return modules.map((module) => { - return [module, ...getAllSubmodules(module).flat()] - }).flat() + return modules + .map((module) => { + return [module, ...getAllSubmodules(module).flat()] + }) + .flat() }) } @@ -47,8 +45,6 @@ function dfs(id: string | undefined, flow: Flow, getParents: boolean = true): Fl function rec(id: string, moduleBranches: ModuleBranches): FlowModule[] | undefined { for (let modules of moduleBranches) { - - for (const [i, module] of modules.entries()) { if (module.id === id) { return getParents ? [module] : modules.slice(0, i + 1).reverse() @@ -93,7 +89,7 @@ function getFlowInput( value: "Iteration's value", index: "Iteration's index" }, - ...parentFlowInput, + ...parentFlowInput } } else { return parentFlowInput @@ -111,38 +107,40 @@ export function getStepPropPicker( id: string, flow: Flow, args: any, - include_node: boolean, + include_node: boolean ): StepPropPicker { const flowInput = getFlowInput(dfs(parentModule?.id, flow), flowState, args, flow.schema) - const previousIds = dfs(id, flow, false).map((x) => { - let submodules = getAllSubmodules(x).flat().map((x) => x.id) + const previousIds = dfs(id, flow, false) + .map((x) => { + let submodules = getAllSubmodules(x) + .flat() + .map((x) => x.id) - if (submodules.includes(id)) { - return [x.id] - } else { - return [x.id, ...submodules] - } - }).flat() + if (submodules.includes(id)) { + return [x.id] + } else { + return [x.id, ...submodules] + } + }) + .flat() if (!include_node) { previousIds.shift() } - - let priorIds = Object.fromEntries(previousIds.map((id) => [id, flowState[id]?.previewResult ?? {}]).reverse()) - + let priorIds = Object.fromEntries( + previousIds.map((id) => [id, flowState[id]?.previewResult ?? {}]).reverse() + ) const pickableProperties = { flow_input: flowInput, priorIds: priorIds, previousId: previousIds[0], - hasResume: previousModule?.suspend != undefined, + hasResume: previousModule?.suspend != undefined } - - if (pickableProperties.hasResume) { - pickableProperties["approvers"] = "The list of approvers" + pickableProperties['approvers'] = 'The list of approvers' } return { @@ -151,7 +149,11 @@ export function getStepPropPicker( } } -export function buildExtraLib(flowInput: Record, results: Record, resume: boolean): string { +export function buildExtraLib( + flowInput: Record, + results: Record, + resume: boolean +): string { return ` /** * get variable (including secret) at path @@ -180,7 +182,9 @@ declare const params: any; */ declare const results = ${JSON.stringify(results)}; -${resume ? ` +${ + resume + ? ` /** * resume payload */ @@ -190,7 +194,8 @@ declare const resume: any * The list of approvers separated by , */ declare const approvers: string -` : ''} ` - + : '' +} +` } diff --git a/frontend/src/lib/components/flows/utils.ts b/frontend/src/lib/components/flows/utils.ts index c0525658c2fe..f2fbf90bb4bd 100644 --- a/frontend/src/lib/components/flows/utils.ts +++ b/frontend/src/lib/components/flows/utils.ts @@ -10,11 +10,12 @@ import { import { inferArgs } from '$lib/infer' import { loadSchema, loadSchemaFlow } from '$lib/scripts' import { workspaceStore } from '$lib/stores' -import { emptySchema, sendUserToast } from '$lib/utils' +import { emptySchema } from '$lib/utils' import { get } from 'svelte/store' import type { FlowModuleState } from './flowState' import type { PickableProperties } from './previousResults' import { NEVER_TESTED_THIS_FAR } from './models' +import { sendUserToast } from '$lib/toast' function create_context_function_template(eval_string: string, context: Record) { return ` diff --git a/frontend/src/lib/components/home/ItemsList.svelte b/frontend/src/lib/components/home/ItemsList.svelte index 4e4161e7ba4a..efb29d3e3ae9 100644 --- a/frontend/src/lib/components/home/ItemsList.svelte +++ b/frontend/src/lib/components/home/ItemsList.svelte @@ -14,7 +14,6 @@ RawAppService } from '$lib/gen' import { userStore, workspaceStore } from '$lib/stores' - import { canWrite } from '$lib/utils' import type uFuzzy from '@leeoniya/ufuzzy' import { Code2, LayoutDashboard } from 'lucide-svelte' @@ -35,6 +34,7 @@ import ToggleButton from '../common/toggleButton-v2/ToggleButton.svelte' import FlowIcon from './FlowIcon.svelte' import RawAppRow from '../common/table/RawAppRow.svelte' + import { canWrite } from '$lib/utils' type TableItem = T & { canWrite: boolean diff --git a/frontend/src/lib/components/scriptEditor/LogPanel.svelte b/frontend/src/lib/components/scriptEditor/LogPanel.svelte index 33f150cb691c..eea1292b72b9 100644 --- a/frontend/src/lib/components/scriptEditor/LogPanel.svelte +++ b/frontend/src/lib/components/scriptEditor/LogPanel.svelte @@ -20,12 +20,10 @@ import SplitPanesWrapper from '../splitPanes/SplitPanesWrapper.svelte' import { Loader2 } from 'lucide-svelte' - export let path: string | undefined export let lang: Preview.language export let previewIsLoading = false export let previewJob: Job | undefined export let pastPreviews: CompletedJob[] = [] - export let lastSave: string | null type DrawerContent = { mode: 'json' | Preview.language | 'plain' @@ -67,9 +65,8 @@ - Logs/Result + Logs & Result History - Last save