-
-
Notifications
You must be signed in to change notification settings - Fork 277
/
resources.py
1471 lines (1196 loc) · 63.2 KB
/
resources.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#
# Copyright (c) 2024 YunoHost Contributors
#
# This file is part of YunoHost (see https://yunohost.org)
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import os
import copy
import shutil
import random
import tempfile
import subprocess
from typing import Dict, Any, List, Union, Callable
from moulinette import m18n
from moulinette.utils.text import random_ascii
from moulinette.utils.process import check_output
from moulinette.utils.log import getActionLogger
from moulinette.utils.filesystem import mkdir, chown, chmod, write_to_file
from moulinette.utils.filesystem import (
rm,
)
from yunohost.utils.system import system_arch, debian_version, debian_version_id
from yunohost.utils.error import YunohostError, YunohostValidationError
logger = getActionLogger("yunohost.app_resources")
class AppResourceManager:
def __init__(self, app: str, current: Dict, wanted: Dict):
self.app = app
self.current = current
self.wanted = wanted
if "resources" not in self.current:
self.current["resources"] = {}
if "resources" not in self.wanted:
self.wanted["resources"] = {}
def apply(
self, rollback_and_raise_exception_if_failure, operation_logger=None, **context
):
todos = list(self.compute_todos())
completed = []
rollback = False
exception = None
for todo, name, old, new in todos:
try:
if todo == "deprovision":
# FIXME : i18n, better info strings
logger.info(f"Deprovisioning {name}...")
old.deprovision(context=context)
elif todo == "provision":
logger.info(f"Provisioning {name}...")
new.provision_or_update(context=context)
elif todo == "update":
logger.info(f"Updating {name}...")
new.provision_or_update(context=context)
except (KeyboardInterrupt, Exception) as e:
exception = e
if isinstance(e, KeyboardInterrupt):
logger.error(m18n.n("operation_interrupted"))
else:
logger.warning(f"Failed to {todo} {name} : {e}")
if rollback_and_raise_exception_if_failure:
rollback = True
completed.append((todo, name, old, new))
break
else:
pass
else:
completed.append((todo, name, old, new))
if rollback:
for todo, name, old, new in completed:
try:
# (NB. here we want to undo the todo)
if todo == "deprovision":
# FIXME : i18n, better info strings
logger.info(f"Reprovisioning {name}...")
old.provision_or_update(context=context)
elif todo == "provision":
logger.info(f"Deprovisioning {name}...")
new.deprovision(context=context)
elif todo == "update":
logger.info(f"Reverting {name}...")
old.provision_or_update(context=context)
except (KeyboardInterrupt, Exception) as e:
if isinstance(e, KeyboardInterrupt):
logger.error(m18n.n("operation_interrupted"))
else:
logger.error(f"Failed to rollback {name} : {e}")
if exception:
if rollback_and_raise_exception_if_failure:
logger.error(
m18n.n("app_resource_failed", app=self.app, error=exception)
)
if operation_logger:
failure_message_with_debug_instructions = operation_logger.error(
str(exception)
)
raise YunohostError(
failure_message_with_debug_instructions, raw_msg=True
)
else:
raise YunohostError(str(exception), raw_msg=True)
else:
logger.error(exception)
def compute_todos(self):
for name, infos in reversed(self.current["resources"].items()):
if name not in self.wanted["resources"].keys():
resource = AppResourceClassesByType[name](infos, self.app, self)
yield ("deprovision", name, resource, None)
for name, infos in self.wanted["resources"].items():
wanted_resource = AppResourceClassesByType[name](infos, self.app, self)
if name not in self.current["resources"].keys():
yield ("provision", name, None, wanted_resource)
else:
infos_ = self.current["resources"][name]
current_resource = AppResourceClassesByType[name](
infos_, self.app, self
)
yield ("update", name, current_resource, wanted_resource)
class AppResource:
type: str = ""
default_properties: Dict[str, Any] = {}
def __init__(self, properties: Dict[str, Any], app: str, manager=None):
self.app = app
self.manager = manager
properties = self.default_properties | properties
# It's not guaranteed that this info will be defined, e.g. during unit tests, only small resource snippets are used, not proper manifests
app_upstream_version = ""
if manager and manager.wanted and "version" in manager.wanted:
app_upstream_version = manager.wanted["version"].split("~")[0]
elif manager and manager.current and "version" in manager.current:
app_upstream_version = manager.current["version"].split("~")[0]
replacements: dict[str, str] = {
"__APP__": self.app,
"__YNH_ARCH__": system_arch(),
"__YNH_DEBIAN_VERSION__": debian_version(),
"__YNH_DEBIAN_VERSION_ID__": debian_version_id(),
"__YNH_APP_UPSTREAM_VERSION__": app_upstream_version,
}
def recursive_apply(function: Callable, data: Any) -> Any:
if isinstance(data, dict): # FIXME: hashable?
return {
key: recursive_apply(function, value) for key, value in data.items()
}
if isinstance(data, list): # FIXME: iterable?
return [recursive_apply(function, value) for value in data]
return function(data)
def replace_tokens_in_strings(data: Any):
if not isinstance(data, str):
return data
for token, replacement in replacements.items():
data = data.replace(token, replacement)
return data
properties = recursive_apply(replace_tokens_in_strings, properties)
for key, value in properties.items():
setattr(self, key, value)
def get_setting(self, key):
from yunohost.app import app_setting
return app_setting(self.app, key)
def set_setting(self, key, value):
from yunohost.app import app_setting
app_setting(self.app, key, value=value)
def delete_setting(self, key):
from yunohost.app import app_setting
app_setting(self.app, key, delete=True)
def check_output_bash_snippet(self, snippet, env={}):
from yunohost.app import (
_make_environment_for_app_script,
)
env_ = _make_environment_for_app_script(
self.app,
force_include_app_settings=True,
)
env_.update(env)
with tempfile.NamedTemporaryFile(prefix="ynh_") as fp:
fp.write(snippet.encode())
fp.seek(0)
with tempfile.TemporaryFile() as stderr:
out = check_output(f"bash {fp.name}", env=env_, stderr=stderr)
stderr.seek(0)
err = stderr.read().decode()
return out, err
def _run_script(self, action, script, env={}):
from yunohost.app import (
_make_tmp_workdir_for_app,
_make_environment_for_app_script,
)
from yunohost.hook import hook_exec_with_script_debug_if_failure
tmpdir = _make_tmp_workdir_for_app(app=self.app)
env_ = _make_environment_for_app_script(
self.app,
workdir=tmpdir,
action=f"{action}_{self.type}",
force_include_app_settings=True,
)
env_.update(env)
script_path = f"{tmpdir}/{action}_{self.type}"
script = f"""
source /usr/share/yunohost/helpers
ynh_abort_if_errors
{script}
"""
write_to_file(script_path, script)
from yunohost.log import OperationLogger
# FIXME ? : this is an ugly hack :(
active_operation_loggers = [
o for o in OperationLogger._instances if o.ended_at is None
]
if active_operation_loggers:
operation_logger = active_operation_loggers[-1]
else:
operation_logger = OperationLogger(
"resource_snippet", [("app", self.app)], env=env_
)
operation_logger.start()
try:
(
call_failed,
failure_message_with_debug_instructions,
) = hook_exec_with_script_debug_if_failure(
script_path,
env=env_,
operation_logger=operation_logger,
error_message_if_script_failed="An error occured inside the script snippet",
error_message_if_failed=lambda e: f"{action} failed for {self.type} : {e}",
)
finally:
if call_failed:
raise YunohostError(
failure_message_with_debug_instructions, raw_msg=True
)
else:
# FIXME: currently in app install code, we have
# more sophisticated code checking if this broke something on the system etc.
# dunno if we want to do this here or manage it elsewhere
pass
# print(ret)
class SourcesResource(AppResource):
"""
Declare what are the sources / assets used by this app. Typically, this corresponds to some tarball published by the upstream project, that needs to be downloaded and extracted in the install dir using the ynh_setup_source helper.
This resource is intended both to declare the assets, which will be parsed by ynh_setup_source during the app script runtime, AND to prefetch and validate the sha256sum of those asset before actually running the script, to be able to report an error early when the asset turns out to not be available for some reason.
Various options are available to accomodate the behavior according to the asset structure
### Example
```toml
[resources.sources]
[resources.sources.main]
url = "https://github.com/foo/bar/archive/refs/tags/v1.2.3.tar.gz"
sha256 = "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b"
autoupdate.strategy = "latest_github_tag"
```
Or more complex examples with several element, including one with asset that depends on the arch
```toml
[resources.sources]
[resources.sources.main]
in_subdir = false
amd64.url = "https://github.com/foo/bar/archive/refs/tags/v1.2.3.amd64.tar.gz"
amd64.sha256 = "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b"
i386.url = "https://github.com/foo/bar/archive/refs/tags/v1.2.3.386.tar.gz"
i386.sha256 = "53c234e5e8472b6ac51c1ae1cab3fe06fad053beb8ebfd8977b010655bfdd3c3"
armhf.url = "https://github.com/foo/bar/archive/refs/tags/v1.2.3.arm.tar.gz"
armhf.sha256 = "4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865"
autoupdate.strategy = "latest_github_release"
autoupdate.asset.amd64 = ".*\\.amd64.tar.gz"
autoupdate.asset.i386 = ".*\\.386.tar.gz"
autoupdate.asset.armhf = ".*\\.arm.tar.gz"
[resources.sources.zblerg]
url = "https://zblerg.com/download/zblerg"
sha256 = "1121cfccd5913f0a63fec40a6ffd44ea64f9dc135c66634ba001d10bcf4302a2"
format = "script"
rename = "zblerg.sh"
```
### Properties (for each source)
- `prefetch` : `true` (default) or `false`, wether or not to pre-fetch this asset during the provisioning phase of the resource. If several arch-dependent url are provided, YunoHost will only prefetch the one for the current system architecture.
- `url` : the asset's URL
- If the asset's URL depend on the architecture, you may instead provide `amd64.url`, `i386.url`, `armhf.url` and `arm64.url` (depending on what architectures are supported), using the same `dpkg --print-architecture` nomenclature as for the supported architecture key in the manifest
- `sha256` : the asset's sha256sum. This is used both as an integrity check, and as a layer of security to protect against malicious actors which could have injected malicious code inside the asset...
- Same as `url` : if the asset's URL depend on the architecture, you may instead provide `amd64.sha256`, `i386.sha256`, ...
- `format` : The "format" of the asset. It is typically automatically guessed from the extension of the URL (or the mention of "tarball", "zipball" in the URL), but can be set explicitly:
- `tar.gz`, `tar.xz`, `tar.bz2` : will use `tar` to extract the archive
- `zip` : will use `unzip` to extract the archive
- `docker` : useful to extract files from an already-built docker image (instead of rebuilding them locally). Will use `docker-image-extract`
- `whatever`: whatever arbitrary value, not really meaningful except to imply that the file won't be extracted (eg because it's a .deb to be manually installed with dpkg/apt, or a script, or ...)
- `in_subdir`: `true` (default) or `false`, depending on if there's an intermediate subdir in the archive before accessing the actual files. Can also be `N` (an integer) to handle special cases where there's `N` level of subdir to get rid of to actually access the files
- `extract` : `true` or `false`. Defaults to `true` for archives such as `zip`, `tar.gz`, `tar.bz2`, ... Or defaults to `false` when `format` is not something that should be extracted. When `extract = false`, the file will only be `mv`ed to the location, possibly renamed using the `rename` value
- `rename`: some string like `whatever_your_want`, to be used for convenience when `extract` is `false` and the default name of the file is not practical
- `platform`: for example `linux/amd64` (defaults to `linux/$YNH_ARCH`) to be used in conjonction with `format = "docker"` to specify which architecture to extract for
#### Regarding `autoupdate`
Strictly speaking, this has nothing to do with the actual app install. `autoupdate` is expected to contain metadata for automatic maintenance / update of the app sources info in the manifest. It is meant to be a simpler replacement for "autoupdate" Github workflow mechanism.
The infos are used by this script : <https://github.com/YunoHost/apps/blob/master/tools/autoupdate_app_sources/autoupdate_app_sources.py> which is ran by the YunoHost infrastructure periodically and will create the corresponding pull request automatically.
The script will rely on the code repo specified in `code` in the upstream section of the manifest.
The `autoupdate.strategy` is expected to be constructed like this: `latest_<gitforge>_<strategy>`
You need to replace the `<gitforge>` in the strategy name by either `github`, `gitlab`, `gitea` or `forgejo`, as the autoupdater supports:
- GitHub
- GitLab (official and self-hosted instances)
- Gitea & Forgejo instances
And choose one strategy in the following ones:
- `latest_<gitforge>_release` : similar to `latest_<gitforge>_tag`, but starting from the list of releases. Note that it's the only strategy that provides the changelog link in the PR message. Pre- or draft releases are ignored. Releases may have assets attached to them, in which case you can define:
- `autoupdate.asset = "some regex"` (when there's only one asset to use). The regex is used to find the appropriate asset among the list of all assets
- or several `autoupdate.asset.$arch = "some_regex"` (when the asset is arch-specific). The regex is used to find the appropriate asset for the specific arch among the list of assets
- `latest_<gitforge>_tag` : look for the latest tag (by sorting tags and finding the "largest" version). Then using the corresponding tar.gz url. Tags containing `rc`, `beta`, `alpha`, `start` are ignored, and actually any tag which doesn't look like `x.y.z` or `vx.y.z`
- `latest_<gitforge>_commit` : will use the latest commit on github, and the corresponding tarball. If this is used for the 'main' source, it will also assume that the version is YYYY.MM.DD corresponding to the date of the commit.
It is also possible to define `autoupdate.upstream` to use a different Git repository instead of the code repository from the upstream section of the manifest. This can be useful when, for example, the app uses other assets such as plugin from a different repository.
If the upstream project provides non-standard tag or release names, you can fix this, with a regex with a matching group.
For example, if tags look like `release-v4.1`, put:
```toml
autoupdate.version_regex = "^release-v(.*)$"
```
And the autoupdater will use the matched group (here: `4.1`) as the version.
### Provision/Update
- For elements with `prefetch = true`, will download the asset (for the appropriate architecture) and store them in `/var/cache/yunohost/download/$app/$source_id`, to be later picked up by `ynh_setup_source`. (NB: this only happens during install and upgrade, not restore)
### Deprovision
- Nothing (just cleanup the cache)
"""
type = "sources"
priority = 10
default_sources_properties: Dict[str, Any] = {
"prefetch": True,
"url": None,
"sha256": None,
}
sources: Dict[str, Dict[str, Any]] = {}
def __init__(self, properties: Dict[str, Any], *args, **kwargs):
for source_id, infos in properties.items():
properties[source_id] = copy.copy(self.default_sources_properties)
properties[source_id].update(infos)
super().__init__({"sources": properties}, *args, **kwargs)
def deprovision(self, context: Dict = {}):
if os.path.isdir(f"/var/cache/yunohost/download/{self.app}/"):
rm(f"/var/cache/yunohost/download/{self.app}/", recursive=True)
def provision_or_update(self, context: Dict = {}):
# Don't prefetch stuff during restore
if context.get("action") == "restore":
return
for source_id, infos in self.sources.items():
if not infos["prefetch"]:
continue
if infos["url"] is None:
arch = system_arch()
if (
arch in infos
and isinstance(infos[arch], dict)
and isinstance(infos[arch].get("url"), str)
and isinstance(infos[arch].get("sha256"), str)
):
self.prefetch(source_id, infos[arch]["url"], infos[arch]["sha256"])
else:
raise YunohostError(
f"In resources.sources: it looks like you forgot to define url/sha256 or {arch}.url/{arch}.sha256",
raw_msg=True,
)
else:
if infos["sha256"] is None:
raise YunohostError(
f"In resources.sources: it looks like the sha256 is missing for {source_id}",
raw_msg=True,
)
self.prefetch(source_id, infos["url"], infos["sha256"])
def prefetch(self, source_id, url, expected_sha256):
logger.debug(f"Prefetching asset {source_id}: {url} ...")
if not os.path.isdir(f"/var/cache/yunohost/download/{self.app}/"):
mkdir(f"/var/cache/yunohost/download/{self.app}/", parents=True)
filename = f"/var/cache/yunohost/download/{self.app}/{source_id}"
# NB: we use wget and not requests.get() because we want to output to a file (ie avoid ending up with the full archive in RAM)
# AND the nice --tries, --no-dns-cache, --timeout options ...
p = subprocess.Popen(
[
"/usr/bin/wget",
"--tries=3",
"--no-dns-cache",
"--timeout=900",
"--no-verbose",
"--output-document=" + filename,
url,
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
out, _ = p.communicate()
returncode = p.returncode
if returncode != 0:
if os.path.exists(filename):
rm(filename)
raise YunohostError(
"app_failed_to_download_asset",
source_id=source_id,
url=url,
app=self.app,
out=out.decode(),
)
assert os.path.exists(
filename
), f"For some reason, wget worked but {filename} doesnt exists?"
computed_sha256 = check_output(f"sha256sum {filename}").split()[0]
if computed_sha256 != expected_sha256:
size = check_output(f"du -hs {filename}").split()[0]
rm(filename)
raise YunohostError(
"app_corrupt_source",
source_id=source_id,
url=url,
app=self.app,
expected_sha256=expected_sha256,
computed_sha256=computed_sha256,
size=size,
)
class PermissionsResource(AppResource):
"""
Configure the SSO permissions/tiles. Typically, webapps are expected to have a 'main' permission mapped to '/', meaning that a tile pointing to the `$domain/$path` will be available in the SSO for users allowed to access that app.
Additional permissions can be created, typically to have a specific tile and/or access rules for the admin part of a webapp.
The list of allowed user/groups may be initialized using the content of the `init_{perm}_permission` question from the manifest, hence `init_main_permission` replaces the `is_public` question and shall contain a group name (typically, `all_users` or `visitors`).
### Example
```toml
[resources.permissions]
main.url = "/"
# (these two previous lines should be enough in the majority of cases)
admin.url = "/admin"
admin.show_tile = false
admin.allowed = "admins" # Assuming the "admins" group exists (cf future developments ;))
```
### Properties (for each perm name)
- `url`: The relative URI corresponding to this permission. Typically `/` or `/something`. This property may be omitted for non-web permissions.
- `show_tile`: (default: `true` if `url` is defined) Wether or not a tile should be displayed for that permission in the user portal
- `allowed`: (default: nobody) The group initially allowed to access this perm, if `init_{perm}_permission` is not defined in the manifest questions. Note that the admin may tweak who is allowed/unallowed on that permission later on, this is only meant to **initialize** the permission.
- `auth_header`: (default: `true`) Define for the URL of this permission, if SSOwat pass the authentication header to the application. Default is true
- `protected`: (default: `false`) Define if this permission is protected. If it is protected the administrator won't be able to add or remove the visitors group of this permission. Defaults to 'false'.
- `additional_urls`: (default: none) List of additional URL for which access will be allowed/forbidden
### Provision/Update
- Delete any permissions that may exist and be related to this app yet is not declared anymore
- Loop over the declared permissions and create them if needed or update them with the new values
### Deprovision
- Delete all permission related to this app
### Legacy management
- Legacy `is_public` setting will be deleted if it exists
"""
# Notes for future ?
# deep_clean -> delete permissions for any __APP__.foobar where app not in app list...
# backup -> handled elsewhere by the core, should be integrated in there (dump .ldif/yml?)
# restore -> handled by the core, should be integrated in there (restore .ldif/yml?)
type = "permissions"
priority = 80
default_properties: Dict[str, Any] = {}
default_perm_properties: Dict[str, Any] = {
"url": None,
"additional_urls": [],
"auth_header": True,
"allowed": None,
"show_tile": None, # To be automagically set to True by default if an url is defined and show_tile not provided
"protected": False,
}
permissions: Dict[str, Dict[str, Any]] = {}
def __init__(self, properties: Dict[str, Any], *args, **kwargs):
# FIXME : if url != None, we should check that there's indeed a domain/path defined ? ie that app is a webapp
# Validate packager-provided infos
for perm, infos in properties.items():
if "auth_header" in infos and not isinstance(
infos.get("auth_header"), bool
):
raise YunohostError(
f"In manifest, for permission '{perm}', 'auth_header' should be a boolean",
raw_msg=True,
)
if "show_tile" in infos and not isinstance(infos.get("show_tile"), bool):
raise YunohostError(
f"In manifest, for permission '{perm}', 'show_tile' should be a boolean",
raw_msg=True,
)
if "protected" in infos and not isinstance(infos.get("protected"), bool):
raise YunohostError(
f"In manifest, for permission '{perm}', 'protected' should be a boolean",
raw_msg=True,
)
if "additional_urls" in infos and not isinstance(
infos.get("additional_urls"), list
):
raise YunohostError(
f"In manifest, for permission '{perm}', 'additional_urls' should be a list",
raw_msg=True,
)
if "main" not in properties:
properties["main"] = copy.copy(self.default_perm_properties)
for perm, infos in properties.items():
properties[perm] = copy.copy(self.default_perm_properties)
properties[perm].update(infos)
if properties[perm]["show_tile"] is None:
properties[perm]["show_tile"] = bool(properties[perm]["url"])
if properties["main"]["url"] is not None and (
not isinstance(properties["main"].get("url"), str)
or properties["main"]["url"] != "/"
):
raise YunohostError(
"URL for the 'main' permission should be '/' for webapps (or left undefined for non-webapps). Note that / refers to the install url of the app, i.e $domain.tld/$path/",
raw_msg=True,
)
super().__init__({"permissions": properties}, *args, **kwargs)
from yunohost.app import _get_app_settings, _hydrate_app_template
settings = _get_app_settings(self.app)
for perm, infos in self.permissions.items():
if infos.get("url") and "__" in infos.get("url"):
infos["url"] = _hydrate_app_template(infos["url"], settings)
if infos.get("additional_urls"):
infos["additional_urls"] = [
_hydrate_app_template(url, settings)
for url in infos["additional_urls"]
]
def provision_or_update(self, context: Dict = {}):
from yunohost.permission import (
permission_create,
permission_url,
permission_delete,
user_permission_list,
user_permission_update,
permission_sync_to_user,
)
# Delete legacy is_public setting if not already done
self.delete_setting("is_public")
# Detect that we're using a full-domain app,
# in which case we probably need to automagically
# define the "path" setting with "/"
if (
isinstance(self.permissions["main"]["url"], str)
and self.get_setting("domain")
and not self.get_setting("path")
):
self.set_setting("path", "/")
existing_perms = user_permission_list(short=True, apps=[self.app])[
"permissions"
]
for perm in existing_perms:
if perm.split(".")[1] not in self.permissions.keys():
permission_delete(perm, force=True, sync_perm=False)
for perm, infos in self.permissions.items():
perm_id = f"{self.app}.{perm}"
if perm_id not in existing_perms:
# Use the 'allowed' key from the manifest,
# or use the 'init_{perm}_permission' from the install questions
# which is temporarily saved as a setting as an ugly hack to pass the info to this piece of code...
init_allowed = (
infos["allowed"]
or self.get_setting(f"init_{perm}_permission")
or []
)
# If we're choosing 'visitors' from the init_{perm}_permission question, add all_users too
if not infos["allowed"] and init_allowed == "visitors":
init_allowed = ["visitors", "all_users"]
permission_create(
perm_id,
allowed=init_allowed,
# This is why the ugly hack with self.manager exists >_>
label=self.manager.wanted["name"] if perm == "main" else perm,
url=infos["url"],
additional_urls=infos["additional_urls"],
auth_header=infos["auth_header"],
sync_perm=False,
)
self.delete_setting(f"init_{perm}_permission")
user_permission_update(
perm_id,
show_tile=infos["show_tile"],
protected=infos["protected"],
sync_perm=False,
)
permission_url(
perm_id,
url=infos["url"],
set_url=infos["additional_urls"],
auth_header=infos["auth_header"],
sync_perm=False,
)
permission_sync_to_user()
def deprovision(self, context: Dict = {}):
from yunohost.permission import (
permission_delete,
user_permission_list,
permission_sync_to_user,
)
existing_perms = user_permission_list(short=True, apps=[self.app])[
"permissions"
]
for perm in existing_perms:
permission_delete(perm, force=True, sync_perm=False)
permission_sync_to_user()
class SystemuserAppResource(AppResource):
"""
Provision a system user to be used by the app. The username is exactly equal to the app id
### Example
```toml
[resources.system_user]
# (empty - defaults are usually okay)
```
### Properties
- `allow_ssh`: (default: False) Adds the user to the ssh.app group, allowing SSH connection via this user
- `allow_sftp`: (default: False) Adds the user to the sftp.app group, allowing SFTP connection via this user
- `allow_email`: (default: False) Enable authentication on the mail stack for the system user and send mail using `__APP__@__DOMAIN__`. A `mail_pwd` setting is automatically defined (similar to `db_pwd` for databases). You can then configure the app to use `__APP__` and `__MAIL_PWD__` as SMTP credentials (with host 127.0.0.1). You can also tweak the user-part of the domain-part of the email used by manually defining a custom setting `mail_user` or `mail_domain`
- `home`: (default: `/var/www/__APP__`) Defines the home property for this user. NB: unfortunately you can't simply use `__INSTALL_DIR__` or `__DATA_DIR__` for now
### Provision/Update
- will create the system user if it doesn't exists yet
- will add/remove the ssh/sftp.app groups
### Deprovision
- deletes the user and group
"""
# Notes for future?
#
# deep_clean -> uuuuh ? delete any user that could correspond to an app x_x ?
#
# backup -> nothing
# restore -> provision
type = "system_user"
priority = 20
default_properties: Dict[str, Any] = {
"allow_ssh": False,
"allow_sftp": False,
"allow_email": False,
"home": "/var/www/__APP__",
}
# FIXME : wat do regarding ssl-cert, multimedia, and other groups
allow_ssh: bool = False
allow_sftp: bool = False
allow_email: bool = False
home: str = ""
def provision_or_update(self, context: Dict = {}):
from yunohost.app import regen_mail_app_user_config_for_dovecot_and_postfix
# FIXME : validate that no yunohost user exists with that name?
# and/or that no system user exists during install ?
if os.system(f"getent passwd {self.app} >/dev/null 2>/dev/null") != 0:
# FIXME: improve logging ? os.system wont log stdout / stderr
cmd = f"useradd --system --user-group {self.app} --home-dir {self.home} --no-create-home"
ret = os.system(cmd)
assert ret == 0, f"useradd command failed with exit code {ret}"
if os.system(f"getent passwd {self.app} >/dev/null 2>/dev/null") != 0:
raise YunohostError(
f"Failed to create system user for {self.app}", raw_msg=True
)
# Update groups
groups = set(check_output(f"groups {self.app}").strip().split()[2:])
if self.allow_ssh:
groups.add("ssh.app")
elif "ssh.app" in groups:
groups.remove("ssh.app")
if self.allow_sftp:
groups.add("sftp.app")
elif "sftp.app" in groups:
groups.remove("sftp.app")
os.system(f"usermod -G {','.join(groups)} {self.app}")
# Update home dir
raw_user_line_in_etc_passwd = check_output(f"getent passwd {self.app}").strip()
user_infos = raw_user_line_in_etc_passwd.split(":")
current_home = user_infos[5]
if current_home != self.home:
ret = os.system(f"usermod --home {self.home} {self.app} 2>/dev/null")
# Most of the time this won't work because apparently we can't change the home dir while being logged-in -_-
# So we gotta brute force by replacing the line in /etc/passwd T_T
if ret != 0:
user_infos[5] = self.home
new_raw_user_line_in_etc_passwd = ":".join(user_infos)
os.system(
f"sed -i 's@{raw_user_line_in_etc_passwd}@{new_raw_user_line_in_etc_passwd}@g' /etc/passwd"
)
# Update mail-related stuff
if self.allow_email:
mail_pwd = self.get_setting("mail_pwd")
if not mail_pwd:
mail_pwd = random_ascii(24)
self.set_setting("mail_pwd", mail_pwd)
regen_mail_app_user_config_for_dovecot_and_postfix()
else:
self.delete_setting("mail_pwd")
if (
os.system(
f"grep --quiet ' {self.app}$' /etc/postfix/app_senders_login_maps"
)
== 0
or os.system(
f"grep --quiet '^{self.app}:' /etc/dovecot/app-senders-passwd"
)
== 0
):
regen_mail_app_user_config_for_dovecot_and_postfix()
def deprovision(self, context: Dict = {}):
from yunohost.app import regen_mail_app_user_config_for_dovecot_and_postfix
if os.system(f"getent passwd {self.app} >/dev/null 2>/dev/null") == 0:
os.system(f"deluser {self.app} >/dev/null")
if os.system(f"getent passwd {self.app} >/dev/null 2>/dev/null") == 0:
raise YunohostError(
f"Failed to delete system user for {self.app}", raw_msg=True
)
if os.system(f"getent group {self.app} >/dev/null 2>/dev/null") == 0:
os.system(f"delgroup {self.app} >/dev/null")
if os.system(f"getent group {self.app} >/dev/null 2>/dev/null") == 0:
raise YunohostError(
f"Failed to delete system user for {self.app}", raw_msg=True
)
self.delete_setting("mail_pwd")
if (
os.system(
f"grep --quiet ' {self.app}$' /etc/postfix/app_senders_login_maps"
)
== 0
or os.system(f"grep --quiet '^{self.app}:' /etc/dovecot/app-senders-passwd")
== 0
):
regen_mail_app_user_config_for_dovecot_and_postfix()
# FIXME : better logging and error handling, add stdout/stderr from the deluser/delgroup commands...
class InstalldirAppResource(AppResource):
"""
Creates a directory to be used by the app as the installation directory, typically where the app sources and assets are located. The corresponding path is stored in the settings as `install_dir`
### Example
```toml
[resources.install_dir]
# (empty - defaults are usually okay)
```
### Properties
- `dir`: (default: `/var/www/__APP__`) The full path of the install dir
- `owner`: (default: `__APP__:rwx`) The owner (and owner permissions) for the install dir
- `group`: (default: `__APP__:rx`) The group (and group permissions) for the install dir
### Provision/Update
- during install, the folder will be deleted if it already exists (FIXME: is this what we want?)
- if the dir path changed and a folder exists at the old location, the folder will be `mv`'ed to the new location
- otherwise, creates the directory if it doesn't exists yet
- (re-)apply permissions (only on the folder itself, not recursively)
- save the value of `dir` as `install_dir` in the app's settings, which can be then used by the app scripts (`$install_dir`) and conf templates (`__INSTALL_DIR__`)
### Deprovision
- recursively deletes the directory if it exists
### Legacy management
- In the past, the setting was called `final_path`. The code will automatically rename it as `install_dir`.
- As explained in the 'Provision/Update' section, the folder will also be moved if the location changed
"""
# Notes for future?
# deep_clean -> uuuuh ? delete any dir in /var/www/ that would not correspond to an app x_x ?
# backup -> cp install dir
# restore -> cp install dir
type = "install_dir"
priority = 30
default_properties: Dict[str, Any] = {
"dir": "/var/www/__APP__",
"owner": "__APP__:rwx",
"group": "__APP__:rx",
}
dir: str = ""
owner: str = ""
group: str = ""
# FIXME: change default dir to /opt/stuff if app ain't a webapp...
def provision_or_update(self, context: Dict = {}):
assert self.dir.strip() # Be paranoid about self.dir being empty...
assert self.owner.strip()
assert self.group.strip()
current_install_dir = self.get_setting("install_dir") or self.get_setting(
"final_path"
)
# If during install, /var/www/$app already exists, assume that it's okay to remove and recreate it
# FIXME : is this the right thing to do ?
if not current_install_dir and os.path.isdir(self.dir):
rm(self.dir, recursive=True)
# isdir will be True if the path is a symlink pointing to a dir
# This should cover cases where people moved the data dir to another place via a symlink (ie we dont enter the if)
if not os.path.isdir(self.dir):
# Handle case where install location changed, in which case we shall move the existing install dir
# FIXME: confirm that's what we wanna do
# Maybe a middle ground could be to compute the size, check that it's not too crazy (eg > 1G idk),
# and check for available space on the destination
if current_install_dir and os.path.isdir(current_install_dir):
logger.warning(
f"Moving {current_install_dir} to {self.dir}... (this may take a while)"
)
shutil.move(current_install_dir, self.dir)
else:
mkdir(self.dir, parents=True)
owner, owner_perm = self.owner.split(":")
group, group_perm = self.group.split(":")
owner_perm_octal = (
(4 if "r" in owner_perm else 0)
+ (2 if "w" in owner_perm else 0)
+ (1 if "x" in owner_perm else 0)
)
group_perm_octal = (
(4 if "r" in group_perm else 0)
+ (2 if "w" in group_perm else 0)
+ (1 if "x" in group_perm else 0)
)
perm_octal = 0o100 * owner_perm_octal + 0o010 * group_perm_octal
# NB: we use realpath here to cover cases where self.dir could actually be a symlink
# in which case we want to apply the perm to the pointed dir, not to the symlink
chmod(os.path.realpath(self.dir), perm_octal)
chown(os.path.realpath(self.dir), owner, group)
# FIXME: shall we apply permissions recursively ?
self.set_setting("install_dir", self.dir)
self.delete_setting("final_path") # Legacy
def deprovision(self, context: Dict = {}):
assert self.dir.strip() # Be paranoid about self.dir being empty...
assert self.owner.strip()
assert self.group.strip()
# FIXME : check that self.dir has a sensible value to prevent catastrophes
if os.path.isdir(self.dir):
rm(self.dir, recursive=True)
# FIXME : in fact we should delete settings to be consistent
class DatadirAppResource(AppResource):
"""
Creates a directory to be used by the app as the data store directory, typically where the app multimedia or large assets added by users are located. The corresponding path is stored in the settings as `data_dir`. This resource behaves very similarly to install_dir.
### Example
```toml
[resources.data_dir]
# (empty - defaults are usually okay)
```
### Properties
- `dir`: (default: `/home/yunohost.app/__APP__`) The full path of the data dir
- `subdirs`: (default: empty list) A list of subdirs to initialize inside the data dir. For example, `['foo', 'bar']`
- `owner`: (default: `__APP__:rwx`) The owner (and owner permissions) for the data dir
- `group`: (default: `__APP__:rx`) The group (and group permissions) for the data dir
### Provision/Update
- if the dir path changed and a folder exists at the old location, the folder will be `mv`'ed to the new location
- otherwise, creates the directory if it doesn't exists yet