Skip to content

Commit

Permalink
Merge pull request openSUSE#3058 from gyr/SLSA
Browse files Browse the repository at this point in the history
SLSA support
  • Loading branch information
gleidi-suse committed Mar 4, 2024
2 parents f941f9d + 3a3c97e commit fd5b7a6
Show file tree
Hide file tree
Showing 16 changed files with 358 additions and 6 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/ci-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ jobs:
run: |
for f in $(find . -maxdepth 1 -type f -executable -print); do
# skip completely broken scripts or those without --help
[[ " ./checknewer.py ./repo2fileprovides.py ./openqa-maintenance.py ./docker_publisher.py ./publish_distro ./findfileconflicts ./write_repo_susetags_file.pl ./issue-diff.py " =~ "$f" ]] || "$f" --help
[[ " ./checknewer.py ./repo2fileprovides.py ./openqa-maintenance.py ./docker_publisher.py ./publish_distro ./findfileconflicts ./write_repo_susetags_file.pl ./issue-diff.py ./generate-release-packages ./verify-build-and-generatelists " =~ "$f" ]] || "$f" --help
done
linters:
Expand Down
6 changes: 4 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,14 @@ include Makefile.common

pkgdata_BINS = $(shell find * -maxdepth 0 -executable -type f)
pkgdata_SCRIPTS=$(wildcard *.py *.pl *.sh)
pkgdata_SCRIPTS+=findfileconflicts publish_distro
pkgdata_SCRIPTS+=findfileconflicts publish_distro generate-release-packages verify-build-and-generatelists gocd/verify-repo-built-successful.py
pkgdata_DATA+=bs_copy metrics osclib pkglistgen $(wildcard *.pm *.testcase)
VERSION = "build-$(shell date +%F)"

all:

install:
install -d -m 755 $(DESTDIR)$(bindir) $(DESTDIR)$(pkgdatadir) $(DESTDIR)$(unitdir) $(DESTDIR)$(oscplugindir) $(DESTDIR)$(sysconfdir)/$(package_name) $(DESTDIR)$(grafana_provisioning_dir)/dashboards $(DESTDIR)$(grafana_provisioning_dir)/datasources
install -d -m 755 $(DESTDIR)$(bindir) $(DESTDIR)$(pkgdatadir) $(DESTDIR)$(unitdir) $(DESTDIR)$(oscplugindir) $(DESTDIR)$(sysconfdir)/$(package_name) $(DESTDIR)$(grafana_provisioning_dir)/dashboards $(DESTDIR)$(grafana_provisioning_dir)/datasources $(DESTDIR)$(logdir)/$(package_name) $(DESTDIR)$(varlibdir)/osrt-slsa/pkglistgen $(DESTDIR)$(varlibdir)/osrt-slsa/relpkggen
for i in $(pkgdata_SCRIPTS); do install -m 755 $$i $(DESTDIR)$(pkgdatadir); done
chmod 644 $(DESTDIR)$(pkgdatadir)/osc-*.py
for i in $(pkgdata_DATA); do cp -a $$i $(DESTDIR)$(pkgdatadir); done
Expand All @@ -24,6 +24,8 @@ install:
ln -s $(pkgdatadir)/metrics/access/aggregate.php $(DESTDIR)$(bindir)/osrt-metrics-access-aggregate
ln -s $(pkgdatadir)/metrics/access/ingest.php $(DESTDIR)$(bindir)/osrt-metrics-access-ingest
cp -R config/* $(DESTDIR)$(sysconfdir)/$(package_name)
rm $(DESTDIR)$(sysconfdir)/$(package_name)/logrotate
install -Dpm0644 config/logrotate $(DESTDIR)$(sysconfdir)/logrotate.d/$(package_name)
for dir in dashboards datasources ; do ln -s $(pkgdatadir)/metrics/grafana/provisioning/$$dir.yaml \
$(DESTDIR)$(grafana_provisioning_dir)/$$dir/$(package_name).yaml ; done
sed -i "s|OSRT_DATA_DIR|$(pkgdatadir)|" \
Expand Down
2 changes: 2 additions & 0 deletions Makefile.common
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,5 @@ grafana_provisioning_dir="$(sysconfdir)/grafana/provisioning"
oscplugindir=$(prefix)/lib/osc-plugins
apachevhostsdir=$(sysconfdir)/apache2/vhosts.d
tmpfilesdir=$(prefix)/lib/tmpfiles.d
logdir=/var/log
varlibdir=/var/lib
1 change: 1 addition & 0 deletions config/ibsapi
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
API_URL="https://api.suse.de"
10 changes: 10 additions & 0 deletions config/logrotate
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
/var/log/openSUSE-release-tools/*/*.log {
compress
dateext
maxage 365
rotate 99
size=+4096k
missingok
notifempty
delaycompress
}
139 changes: 139 additions & 0 deletions config/oscrc
Original file line number Diff line number Diff line change
@@ -0,0 +1,139 @@
[general]
# URL to access API server, e.g. https://api.suse.de/
# you also need a section [https://api.suse.de/] with the credentials
apiurl = https://api.suse.de/
# Downloaded packages are cached here. Must be writable by you.
#packagecachedir = /var/tmp/osbuild-packagecache
# Wrapper to call build as root (sudo, su -, ...)
#su-wrapper = sudo
# set it empty to run build script as user (works only with KVM atm):
#su-wrapper =
# rootdir to setup the chroot environment
# can contain %(repo)s, %(arch)s, %(project)s, %(package)s and %(apihost)s (apihost is the hostname
# extracted from currently used apiurl) for replacement, e.g.
# /srv/oscbuild/%(repo)s-%(arch)s or
# /srv/oscbuild/%(repo)s-%(arch)s-%(project)s-%(package)s
#build-root = /var/tmp/build-root/%(repo)s-%(arch)s
# compile with N jobs (default: "getconf _NPROCESSORS_ONLN")
#build-jobs = N
# build-type to use - values can be (depending on the capabilities of the 'build' script)
# empty - chroot build
# kvm - kvm VM build (needs build-device, build-swap, build-memory)
# xen - xen VM build (needs build-device, build-swap, build-memory)
# experimental:
# qemu - qemu VM build
# lxc - lxc build
#build-type =
# Execute always a shell prompt on build failure inside of the build environment
#build-shell-after-fail = 1
# build-device is the disk-image file to use as root for VM builds
# e.g. /var/tmp/FILE.root
#build-device = /var/tmp/FILE.root
# build-swap is the disk-image to use as swap for VM builds
# e.g. /var/tmp/FILE.swap
#build-swap = /var/tmp/FILE.swap
# build-kernel is the boot kernel used for VM builds
#build-kernel = /boot/vmlinuz
# build-initrd is the boot initrd used for VM builds
#build-initrd = /boot/initrd
# build-memory is the amount of memory used in the VM
# value in MB - e.g. 512
#build-memory = 512
# build-vmdisk-rootsize is the size of the disk-image used as root in a VM build
# values in MB - e.g. 4096
#build-vmdisk-rootsize = 4096
# build-vmdisk-swapsize is the size of the disk-image used as swap in a VM build
# values in MB - e.g. 1024
#build-vmdisk-swapsize = 1024
# build-vmdisk-filesystem is the file system type of the disk-image used in a VM build
# values are ext3(default) ext4 xfs reiserfs btrfs
#build-vmdisk-filesystem = ext4
# Numeric uid:gid to assign to the "abuild" user in the build-root
# or "caller" to use the current users uid:gid
# This is convenient when sharing the buildroot with ordinary userids
# on the host.
# This should not be 0
# build-uid =
# strip leading build time information from the build log
# buildlog_strip_time = 1
# Enable ccache in build roots.
# ccache = 1
# Enable sccache in build roots. Conflicts with ccache.
# Equivalent to sccache_uri = file:///var/tmp/osbuild-sccache-{pkgname}.tar
# sccache = 1
# Optional URI for sccache storage. Maybe a file://, redis:// or other URI supported
# by the configured sccache install. This uri MAY take {pkgname} as a special parameter
# which will be replaced with the name of the package to be built.
# sccache_uri = file:///var/tmp/osbuild-sccache-{pkgname}.tar.lzop
# sccache_uri = file:///var/tmp/osbuild-sccache-{pkgname}.tar
# sccache_uri = redis://127.0.0.1:6379
# extra packages to install when building packages locally (osc build)
# this corresponds to osc build's -x option and can be overridden with that
# -x '' can also be given on the command line to override this setting, or
# you can have an empty setting here. This global setting may leads to
# dependency problems when the base distro is not providing the package.
# => using server side definition via cli_debug_packages substitute rule is
# recommended therefore.
#extra-pkgs =
# build platform is used if the platform argument is omitted to osc build
#build_repository = openSUSE_Factory
# default project for getpac or bco
#getpac_default_project = openSUSE:Factory
# alternate filesystem layout: have multiple subdirs, where colons were.
#checkout_no_colon = 0
# change filesystem layout: avoid checkout within a project or package dir.
#checkout_rooted = 0
# local files to ignore with status, addremove, ....
#exclude_glob = .osc CVS .svn .* _linkerror *~ #*# *.orig *.bak *.changes.vctmp.*
# limit the age of requests shown with 'osc req list'.
# this is a default only, can be overridden by 'osc req list -D NNN'
# Use 0 for unlimted.
#request_list_days = 0
# show info useful for debugging
#debug = 1
# show HTTP traffic useful for debugging
#http_debug = 1
# number of retries on HTTP transfer
#http_retries = 3
# Skip signature verification of packages used for build.
#no_verify = 1
# jump into the debugger in case of errors
#post_mortem = 1
# print call traces in case of errors
#traceback = 1
# check for unversioned/removed files before commit
#check_filelist = 1
# check for pending requests after executing an action (e.g. checkout, update, commit)
#check_for_request_on_action = 1
# what to do with the source package if the submitrequest has been accepted. If
# nothing is specified the API default is used
#submitrequest_on_accept_action = cleanup|update|noupdate
# template for an accepted submitrequest
#submitrequest_accepted_template = Hi %(who)s,\n
# thanks for working on:\t%(tgt_project)s/%(tgt_package)s.
# SR %(reqid)s has been accepted.\n\nYour maintainers
# template for a declined submitrequest
#submitrequest_declined_template = Hi %(who)s,\n
# sorry your SR %(reqid)s (request type: %(type)s) for
# %(tgt_project)s/%(tgt_package)s has been declined because...
#review requests interactively (default: off)
#request_show_review = 1
# if a review is accepted in interactive mode and a group
# was specified the review will be accepted for this group (default: off)
#review_inherit_group = 1

[https://api.suse.de/]
# set aliases for this apiurl
# aliases = foo, bar
# real name used in .changes, unless the one from osc meta prj <user> will be used
# realname =
# email used in .changes, unless the one from osc meta prj <user> will be used
# email =
# additional headers to pass to a request, e.g. for special authentication
#http_headers = Host: foofoobar,
# User: mumblegack
# Plain text password
#pass =
user=osrt-slsa
pass=
credentials_mgr_class=osc.credentials.ObfuscatedConfigFileCredentialsManager
54 changes: 54 additions & 0 deletions dist/package/openSUSE-release-tools.spec
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
%global __provides_exclude ^perl.*
%define source_dir openSUSE-release-tools
%define announcer_filename factory-package-news
%define services osrt-slsa.target osrt-relpkggen@.timer osrt-relpkggen@.service osrt-pkglistgen@.timer osrt-pkglistgen@.service
Name: openSUSE-release-tools
Version: 0
Release: 0
Expand Down Expand Up @@ -49,6 +50,9 @@ BuildRequires: apache2-devel
BuildRequires: rsyslog
BuildRequires: systemd-rpm-macros

BuildRequires: sysuser-shadow
BuildRequires: sysuser-tools

Requires: python3-PyYAML
Requires: python3-cmdln
Requires: python3-colorama
Expand Down Expand Up @@ -242,6 +246,19 @@ BuildArch: noarch
Generates package lists based on 000package-groups and puts them
in 000product, resp 000release-packages

%package slsa-build-service
Summary: Build service
Group: Development/Tools/Other
# TODO Update requirements, but for now base deps.
Requires: %{name} = %{version}
Requires: openSUSE-release-tools-pkglistgen
%sysusers_requires
Recommends: logrotate
BuildArch: noarch

%description slsa-build-service
Service to run repo-checker and pkglistgen.

%package -n osclib
Summary: Supplemental osc libraries
Group: Development/Tools/Other
Expand Down Expand Up @@ -351,6 +368,21 @@ getent passwd osrt-repo-checker > /dev/null || \
useradd -r -m -s /sbin/nologin -c "user for openSUSE-release-tools-repo-checker" osrt-repo-checker
exit 0

%pre slsa-build-service
%service_add_pre %{services}
getent passwd osrt-slsa > /dev/null || \
useradd -r -d /var/lib/osrt-slsa -s /sbin/nologin -c "user for openSUSE-release-tools-slsa-build-service" osrt-slsa
exit 0

%post slsa-build-service
%service_add_post %{services}

%preun slsa-build-service
%service_del_preun %{services}

%postun slsa-build-service
%service_del_postun_with_restart %{services}

%pre staging-bot
getent passwd osrt-staging-bot > /dev/null || \
useradd -r -m -s /sbin/nologin -c "user for openSUSE-release-tools-staging-bot" osrt-staging-bot
Expand Down Expand Up @@ -400,6 +432,9 @@ exit 0
%exclude %{_datadir}/%{source_dir}/osc-staging.py
%exclude %{_datadir}/%{source_dir}/publish_distro
%exclude %{_datadir}/%{source_dir}/findfileconflicts
%exclude %{_datadir}/%{source_dir}/generate-release-packages
%exclude %{_datadir}/%{source_dir}/verify-build-and-generatelists
%exclude %{_datadir}/%{source_dir}/verify-repo-built-successful.py
%exclude %{_datadir}/%{source_dir}/write_repo_susetags_file.pl
%dir %{_sysconfdir}/openSUSE-release-tools

Expand Down Expand Up @@ -427,6 +462,25 @@ exit 0
%{_unitdir}/osrt-docker-publisher.service
%{_unitdir}/osrt-docker-publisher.timer

%files slsa-build-service
%{_bindir}/osrt-generate-release-packages
%{_bindir}/osrt-verify-build-and-generatelists
%{_datadir}/%{source_dir}/generate-release-packages
%{_datadir}/%{source_dir}/verify-build-and-generatelists
%{_datadir}/%{source_dir}/verify-repo-built-successful.py
%{_sysconfdir}/openSUSE-release-tools/ibsapi
%{_unitdir}/osrt-pkglistgen@.service
%{_unitdir}/osrt-pkglistgen@.timer
%{_unitdir}/osrt-relpkggen@.service
%{_unitdir}/osrt-relpkggen@.timer
%{_unitdir}/osrt-slsa.target
%config(noreplace) %attr(0640,osrt-slsa,osrt-slsa) %{_sysconfdir}/openSUSE-release-tools/oscrc
%config(noreplace) %{_sysconfdir}/logrotate.d/%{name}
%dir %attr(755,osrt-slsa,osrt-slsa) %{_localstatedir}/log/openSUSE-release-tools/
%dir %attr(750,osrt-slsa,osrt-slsa) %{_sharedstatedir}/osrt-slsa
%dir %attr(750,osrt-slsa,osrt-slsa) %{_sharedstatedir}/osrt-slsa/pkglistgen
%dir %attr(750,osrt-slsa,osrt-slsa) %{_sharedstatedir}/osrt-slsa/relpkggen

%files maintenance
%{_bindir}/osrt-check_maintenance_incidents
%{_datadir}/%{source_dir}/check_maintenance_incidents.py
Expand Down
14 changes: 14 additions & 0 deletions generate-release-packages
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
#!/usr/bin/env sh

PROJECT=$1
LOG_DIR="/var/log/openSUSE-release-tools/${PROJECT}"
[ ! -d "${LOG_DIR}" ] && mkdir ${LOG_DIR}

logger() {
date -Is >> ${LOG_DIR}/relpkggen.log
echo "$1" >> ${LOG_DIR}/relpkggen.log
}

logger "[START] Start osrt-relpkggen@${PROJECT}.service"
/usr/bin/osrt-pkglistgen -A ${API_URL} --debug update_and_solve -p ${PROJECT} -s target --only-release-packages --custom-cache-tag releasepackages --force >> ${LOG_DIR}/relpkggen.log 2>&1
logger "[FINISH] Finish osrt-relpkggen@${PROJECT}.service"
4 changes: 3 additions & 1 deletion pkglistgen/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ def do_handle_update_repos(self, subcmd, opts, project):
@cmdln.option('--stop-after-solve', action='store_true', help='only create group files')
@cmdln.option('--staging', help='Only solve that one staging')
@cmdln.option('--only-release-packages', action='store_true', help='Generate 000release-packages only')
@cmdln.option('--custom-cache-tag', help='add custom tag to cache dir to avoid issues when running in parallel')
def do_update_and_solve(self, subcmd, opts):
"""${cmd_name}: update and solve for given scope
Expand Down Expand Up @@ -94,7 +95,8 @@ def solve_project(project, scope: str):
project=project, scope=scope, force=opts.force,
no_checkout=opts.no_checkout,
only_release_packages=opts.only_release_packages,
stop_after_solve=opts.stop_after_solve)
stop_after_solve=opts.stop_after_solve,
custom_cache_tag=opts.custom_cache_tag)
except MismatchedRepoException:
logging.error("Failed to create weakremovers.inc due to mismatch in repos - project most likey started building again.")
# for stagings we have to be strict on the exit value
Expand Down
8 changes: 6 additions & 2 deletions pkglistgen/tool.py
Original file line number Diff line number Diff line change
Expand Up @@ -611,7 +611,8 @@ def update_and_solve_target(
force: bool,
no_checkout: bool,
only_release_packages: bool,
stop_after_solve: bool
stop_after_solve: bool,
custom_cache_tag
):
self.all_architectures = target_config.get('pkglistgen-archs').split(' ')
self.use_newest_version = str2bool(target_config.get('pkglistgen-use-newest-version', 'False'))
Expand Down Expand Up @@ -651,7 +652,10 @@ def update_and_solve_target(

# Cache dir specific to hostname and project.
host = urlparse(api.apiurl).hostname
cache_dir = CacheManager.directory('pkglistgen', host, project)
prefix_dir = 'pkglistgen'
if custom_cache_tag:
prefix_dir += '-' + custom_cache_tag
cache_dir = CacheManager.directory(prefix_dir, host, project)

if not no_checkout:
if os.path.exists(cache_dir):
Expand Down
13 changes: 13 additions & 0 deletions systemd/osrt-pkglistgen@.service
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
[Unit]
PartOf=osrt-slsa.target

[Service]
User=osrt-slsa
SyslogIdentifier=osrt-slsa
EnvironmentFile=/etc/openSUSE-release-tools/ibsapi
Environment="OSC_CONFIG=/etc/openSUSE-release-tools/oscrc"
WorkingDirectory=/var/lib/osrt-slsa/pkglistgen
ExecStart=/usr/bin/osrt-verify-build-and-generatelists %i

[Install]
WantedBy=osrt-slsa.target
7 changes: 7 additions & 0 deletions systemd/osrt-pkglistgen@.timer
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
[Timer]
OnBootSec=420
OnCalendar=*:05,15,25,35,45,55
Unit=osrt-pkglistgen@.service

[Install]
WantedBy=timers.target
15 changes: 15 additions & 0 deletions systemd/osrt-relpkggen@.service
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
[Unit]
PartOf=osrt-slsa.target

[Service]
User=osrt-slsa
SyslogIdentifier=osrt-slsa
EnvironmentFile=/etc/openSUSE-release-tools/ibsapi
Environment="OSC_CONFIG=/etc/openSUSE-release-tools/oscrc"
WorkingDirectory=/var/lib/osrt-slsa/relpkggen
ExecStartPre=/bin/bash -xc '/usr/bin/systemctl is-active --quiet osrt-pkglistgen@%i.service && exit 1 || exit 0'
ExecStart=/usr/bin/osrt-generate-release-packages %i
RuntimeMaxSec=120 hour

[Install]
WantedBy=osrt-slsa.target
7 changes: 7 additions & 0 deletions systemd/osrt-relpkggen@.timer
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
[Timer]
OnBootSec=120
OnCalendar=hourly
Unit=osrt-relpkggen@.service

[Install]
WantedBy=timers.target
5 changes: 5 additions & 0 deletions systemd/osrt-slsa.target
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
[Unit]
Description=Target to restart all parts of openSUSE-release-tools-slsa-build-service

[Install]
WantedBy=multi-user.target
Loading

0 comments on commit fd5b7a6

Please sign in to comment.