Skip to content

Commit

Permalink
Refactor build to be parallelized and stages with a Jenkinsfile (#121)
Browse files Browse the repository at this point in the history
Fixes #120
  • Loading branch information
jonahgraham committed Feb 28, 2024
1 parent af613f4 commit 727fd60
Show file tree
Hide file tree
Showing 9 changed files with 473 additions and 262 deletions.
122 changes: 122 additions & 0 deletions Jenkinsfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
pipeline {
agent any
options {
timestamps()
disableConcurrentBuilds()
}
tools {
maven 'apache-maven-latest'
jdk 'temurin-jdk17-latest'
}
environment {
BUILDING='/home/data/httpd/download.eclipse.org/technology/epp/building/'
STAGING='/home/data/httpd/download.eclipse.org/technology/epp/staging/'
SSHUSER='genie.packaging@projects-storage.eclipse.org'
MAVEN_OPTS='-XX:MaxRAMPercentage=50.0'
}
stages {
stage('Prepare shared building space on download.e.o') {
steps {
sshagent ( ['projects-storage.eclipse.org-bot-ssh']) {
sh "ssh ${SSHUSER} rm -rf ${BUILDING}"
sh "ssh ${SSHUSER} mkdir -p ${BUILDING}/repository"
}
}
}
stage('Build p2') {
steps {
sh "mvn verify -Depp.p2.all -Depp.product.all --batch-mode --show-version -Dmaven.repo.local=.repository -Dtycho.disableP2Mirrors=true -Peclipse-sign-jar"
sshagent ( ['projects-storage.eclipse.org-bot-ssh']) {
sh "scp -rpv archive/repository/* ${SSHUSER}:${BUILDING}/repository/"
}
}
}
// TODO make this a loop/matrix please
// For now I can't use matrix because that runs all in parallel and I don't want that here (I could if each one was on a separate node though)
stage('Build committers') {
steps {
sh "mvn verify -Pepp.product.committers -Pepp.materialize-products -Depp.p2.repourl=https://download.eclipse.org/technology/epp/building/repository/ --batch-mode --show-version -Dmaven.repo.local=.repository -Dtycho.disableP2Mirrors=true -Peclipse-sign-jar -Peclipse-sign-mac -Peclipse-sign-dmg -Peclipse-sign-windows -Peclipse-package-dmg"
sshagent ( ['projects-storage.eclipse.org-bot-ssh']) {
sh "./releng/org.eclipse.epp.config/tools/upload-to-building.sh ${BUILDING} committers"
}
}
}
stage('Build cpp') {
steps {
sh "mvn verify -Pepp.product.cpp -Pepp.materialize-products -Depp.p2.repourl=https://download.eclipse.org/technology/epp/building/repository/ --batch-mode --show-version -Dmaven.repo.local=.repository -Dtycho.disableP2Mirrors=true -Peclipse-sign-jar -Peclipse-sign-mac -Peclipse-sign-dmg -Peclipse-sign-windows -Peclipse-package-dmg"
sshagent ( ['projects-storage.eclipse.org-bot-ssh']) {
sh "./releng/org.eclipse.epp.config/tools/upload-to-building.sh ${BUILDING} cpp"
}
}
}
stage('Build embedcpp') {
steps {
sh "mvn verify -Pepp.product.embedcpp -Pepp.materialize-products -Depp.p2.repourl=https://download.eclipse.org/technology/epp/building/repository/ --batch-mode --show-version -Dmaven.repo.local=.repository -Dtycho.disableP2Mirrors=true -Peclipse-sign-jar -Peclipse-sign-mac -Peclipse-sign-dmg -Peclipse-sign-windows -Peclipse-package-dmg"
sshagent ( ['projects-storage.eclipse.org-bot-ssh']) {
sh "./releng/org.eclipse.epp.config/tools/upload-to-building.sh ${BUILDING} embedcpp"
}
}
}
stage('Build dsl') {
steps {
sh "mvn verify -Pepp.product.dsl -Pepp.materialize-products -Depp.p2.repourl=https://download.eclipse.org/technology/epp/building/repository/ --batch-mode --show-version -Dmaven.repo.local=.repository -Dtycho.disableP2Mirrors=true -Peclipse-sign-jar -Peclipse-sign-mac -Peclipse-sign-dmg -Peclipse-sign-windows -Peclipse-package-dmg"
sshagent ( ['projects-storage.eclipse.org-bot-ssh']) {
sh "./releng/org.eclipse.epp.config/tools/upload-to-building.sh ${BUILDING} dsl"
}
}
}
stage('Build java') {
steps {
sh "mvn verify -Pepp.product.java -Pepp.materialize-products -Depp.p2.repourl=https://download.eclipse.org/technology/epp/building/repository/ --batch-mode --show-version -Dmaven.repo.local=.repository -Dtycho.disableP2Mirrors=true -Peclipse-sign-jar -Peclipse-sign-mac -Peclipse-sign-dmg -Peclipse-sign-windows -Peclipse-package-dmg"
sshagent ( ['projects-storage.eclipse.org-bot-ssh']) {
sh "./releng/org.eclipse.epp.config/tools/upload-to-building.sh ${BUILDING} java"
}
}
}
stage('Build jee') {
steps {
sh "mvn verify -Pepp.product.jee -Pepp.materialize-products -Depp.p2.repourl=https://download.eclipse.org/technology/epp/building/repository/ --batch-mode --show-version -Dmaven.repo.local=.repository -Dtycho.disableP2Mirrors=true -Peclipse-sign-jar -Peclipse-sign-mac -Peclipse-sign-dmg -Peclipse-sign-windows -Peclipse-package-dmg"
sshagent ( ['projects-storage.eclipse.org-bot-ssh']) {
sh "./releng/org.eclipse.epp.config/tools/upload-to-building.sh ${BUILDING} jee"
}
}
}
stage('Build modeling') {
steps {
sh "mvn verify -Pepp.product.modeling -Pepp.materialize-products -Depp.p2.repourl=https://download.eclipse.org/technology/epp/building/repository/ --batch-mode --show-version -Dmaven.repo.local=.repository -Dtycho.disableP2Mirrors=true -Peclipse-sign-jar -Peclipse-sign-mac -Peclipse-sign-dmg -Peclipse-sign-windows -Peclipse-package-dmg"
sshagent ( ['projects-storage.eclipse.org-bot-ssh']) {
sh "./releng/org.eclipse.epp.config/tools/upload-to-building.sh ${BUILDING} modeling"
}
}
}
stage('Build php') {
steps {
sh "mvn verify -Pepp.product.php -Pepp.materialize-products -Depp.p2.repourl=https://download.eclipse.org/technology/epp/building/repository/ --batch-mode --show-version -Dmaven.repo.local=.repository -Dtycho.disableP2Mirrors=true -Peclipse-sign-jar -Peclipse-sign-mac -Peclipse-sign-dmg -Peclipse-sign-windows -Peclipse-package-dmg"
sshagent ( ['projects-storage.eclipse.org-bot-ssh']) {
sh "./releng/org.eclipse.epp.config/tools/upload-to-building.sh ${BUILDING} php"
}
}
}
stage('Build rcp') {
steps {
sh "mvn verify -Pepp.product.rcp -Pepp.materialize-products -Depp.p2.repourl=https://download.eclipse.org/technology/epp/building/repository/ --batch-mode --show-version -Dmaven.repo.local=.repository -Dtycho.disableP2Mirrors=true -Peclipse-sign-jar -Peclipse-sign-mac -Peclipse-sign-dmg -Peclipse-sign-windows -Peclipse-package-dmg"
sshagent ( ['projects-storage.eclipse.org-bot-ssh']) {
sh "./releng/org.eclipse.epp.config/tools/upload-to-building.sh ${BUILDING} rcp"
}
}
}
stage('Build scout') {
steps {
sh "mvn verify -Pepp.product.scout -Pepp.materialize-products -Depp.p2.repourl=https://download.eclipse.org/technology/epp/building/repository/ --batch-mode --show-version -Dmaven.repo.local=.repository -Dtycho.disableP2Mirrors=true -Peclipse-sign-jar -Peclipse-sign-mac -Peclipse-sign-dmg -Peclipse-sign-windows -Peclipse-package-dmg"
sshagent ( ['projects-storage.eclipse.org-bot-ssh']) {
sh "./releng/org.eclipse.epp.config/tools/upload-to-building.sh ${BUILDING} scout"
}
}
}
}
post {
cleanup {
cleanWs()
}
}
}
72 changes: 35 additions & 37 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,48 +1,50 @@
The EPP Build
=============

The [Eclipse Packaging Project (EPP)](http://www.eclipse.org/epp/) provides
the download packages based on the content of the yearly Simultaneous Release.
The download packages are provided from
[www.eclipse.org/downloads/eclipse-packages/](https://www.eclipse.org/downloads/eclipse-packages/).
The [Eclipse Packaging Project (EPP)](https://www.eclipse.org/epp/) provides the download packages based on the content of the yearly Simultaneous Release.
The download packages are provided from [www.eclipse.org/downloads/eclipse-packages/](https://www.eclipse.org/downloads/eclipse-packages/).

Creating and releasing packages
-------------------------------
## Creating and releasing packages

Please see [RELEASING.md](RELEASING.md) in this repo for instructions on the release process for the EPP project.

Build a Package Locally
-----------------------
## Build Locally

It's *easy* to run the build locally! All you need is Maven and then you need
to tell Maven which package(s) to build via profile. As an example, the following
command from the root of the Git repository builds the RCP/RAP package against
the Simultaneous Release staging p2 repository:
It's *easy* to run the build locally! All you need is Maven:

mvn clean verify -Pepp.package.rcp
mvn clean verify

To build all the packages:
will build all the packages (using automatically activated profiles) and the resulting zip/tar/dmg will be in `packages/org.eclipse.epp.package.${PACKAGE}.product/target/products`.
In addition the combined p2 site will be in `archive/repository`.

mvn clean verify -Pepp.package.cpp,epp.package.dsl,epp.package.embedcpp,epp.package.java,epp.package.jee,epp.package.modeling,epp.package.php,epp.package.rcp,epp.package.scout,epp.package.committers
### Build a single package

If you want to build just a single package add the profile for the package you want to build, along with the profile to materialize the product:

mvn verify -Pepp.product.rcp -Pepp.materialize-products

This build creates output in two places:

1. tar.gz/zip/dmg archives with the packages in `archive/` and
2. a p2 repository with the EPP artifacts in `archive/repository/`.

Windows users
-------------
### Build a single platform

By default the maven build runs the build for all platforms, this can be time consuming and can be changed to only build a limited number of platforms, a useful thing to do for testing changes locally.
There is no profile (PRs welcome!) to disabled other platforms, instead modify `releng/org.eclipse.epp.config/parent/pom.xml` to exclude the unwanted platforms in `target-platform-configuration`'s configuration.

### Windows users

In the past the last step in the build process used to fail.
For further details see [bug 426416](https://bugs.eclipse.org/bugs/show_bug.cgi?id=426416).
If that happens again
If that happens again you can omit the `verify` stage and simply `package`.

mvn clean package -P"epp.package.rcp"
mvn clean package -Pepp.package.rcp -Pepp.materialize-products

Available Profiles
------------------
### Available Profiles

Each package uses its own profile:
Each package uses its own profile, with the zip/tar/dmg in `packages/org.eclipse.epp.package.${PACKAGE}.product/target/products`.
With the `epp.materialize-products` profile the zip/tar/dmg will be created, otherwise only the p2 site will be created.

- epp.package.committers
- epp.package.cpp
Expand All @@ -56,33 +58,31 @@ Each package uses its own profile:
- epp.package.scout

macOS dmg files can only be created within the Eclipse Foundation network. To enable creating
dmg files enable the eclipse-package-dmg profile. Without eclipse-package-dmg enabled, the .tar.gz
dmg files enable the `eclipse-package-dmg` profile. Without `eclipse-package-dmg` enabled, the .tar.gz
for macOS will be created regardless.

With the signing profiles enabled, the build artifacts (bundles, features) and the
Windows and macOS executables are signed. This is done by using the Eclipse Foundation
Windows and macOS executables are signed. This is done by using the Eclipse Foundation
internal signing service and can be activated only if the build is running there.

- eclipse-sign-jar profile enables signing of the EPP bundles and jar files
- eclipse-sign-mac profile enables usage of macOS signing service
- eclipse-sign-dmg profile enables signing of the DMG files for the macOS platform (the eclipse-package-dmg needs to be enabled too!)
- eclipse-sign-windows profile enables usage of Windows signing service
- `eclipse-sign-jar` profile enables signing of the EPP bundles and jar files
- `eclipse-sign-mac` profile enables usage of macOS signing service
- `eclipse-sign-dmg` profile enables signing of the DMG files for the macOS platform (the `eclipse-package-dmg` needs to be enabled too!)
- `eclipse-sign-windows` profile enables usage of Windows signing service

Additional Configuration Possibilities
--------------------------------------
### Additional Configuration Possibilities

By default, the EPP build uses the content of the Eclipse Simultaneous Release *Staging*
repository at <http://download.eclipse.org/staging/2020-03/> as input. Sometimes it is
repository at <https://download.eclipse.org/staging/2024-03/> as input. Sometimes it is
desired to build against another release (e.g. a different milestone), or against a local
mirror of this repository. This can be achieved by setting the Java property
`eclipse.simultaneous.release.repository`to another URL. As an example, by adding the
following argument to the Maven command line, the EPP build will read its input from the
composite Eclipse 2020-03 repository:
composite Eclipse 2024-03 repository:

-Declipse.simultaneous.release.repository="http://download.eclipse.org/releases/2020-03"
-Declipse.simultaneous.release.repository="https://download.eclipse.org/releases/2024-03"

EPP Configuration File format
-----------------------------
### EPP Configuration File format

The individual EPP packages have a special file called epp.website.xml that defines various
pieces of information about the package. The format of the file is:
Expand Down Expand Up @@ -119,5 +119,3 @@ pieces of information about the package. The format of the file is:
</configuration>
```

This content in this section has been migrated from the [wiki](https://wiki.eclipse.org/EPP/Packaging_Site)
11 changes: 6 additions & 5 deletions RELEASING.md
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ This checklist is only used once per release cycle. Scroll down for the per-mile
- [ ] Check for bad links to Bugzilla (other things?) especially in `epp.website.xml`
- [ ] Make sure any outstanding reviews are progressing - e.g. file IP logs, get PMC approval, etc.
- For 2022-03 there is no review planned, next review expected to be a progress review around 2022-06
- [ ] Ensure that the [CI build](https://ci.eclipse.org/packaging/job/simrel.epp-tycho-build/) is green<!-- or yellow (yellow means some files have failed to notarize which can be handled later on in this process)-->. Resolving non-green builds will require tracking down problems and incompatibilities across all Eclipse participating projects. [cross-project-issues-dev](https://accounts.eclipse.org/mailing-list/cross-project-issues-dev) mailing list is a good place to start when tracking such problems.
- [ ] Ensure that the [CI build](https://ci.eclipse.org/packaging/job/epp/job/master/) is green<!-- or yellow (yellow means some files have failed to notarize which can be handled later on in this process)-->. Resolving non-green builds will require tracking down problems and incompatibilities across all Eclipse participating projects. [cross-project-issues-dev](https://accounts.eclipse.org/mailing-list/cross-project-issues-dev) mailing list is a good place to start when tracking such problems.
- [ ] Check that packages containing incubating projects have that information reflected in Help -> About dialog. See near the end of build output for report of check-incubating.sh script.
- `-incubation` and ` (includes Incubating components)` are not used in packageMetaData anymore (See [Bug 564214](https://bugs.eclipse.org/bugs/show_bug.cgi?id=564214))
- [ ] On RC1 check "new and noteworthy" version numbers - If any N&N are out of date, remove the N&N entries and notify the corresponding package maintainer.
Expand All @@ -64,8 +64,9 @@ This checklist is only used once per release cycle. Scroll down for the per-mile
- [ ] Wait for announcement that the staging repo is ready on [cross-project-issues-dev](https://accounts.eclipse.org/mailing-list/cross-project-issues-dev). An [example announcement](https://www.eclipse.org/lists/cross-project-issues-dev/msg17420.html).
- [ ] Update `SIMREL_REPO` in `releng/org.eclipse.epp.config/parent/pom.xml` if not done above.
- [ ] Update the build qualifiers to ensure that packages are all updated. See this [gerrit](https://git.eclipse.org/r/#/c/161075/) for an example. To do this run `releng/org.eclipse.epp.config/tools/setGitDate` ([link](https://github.com/eclipse-packaging/packages/blob/master/releng/org.eclipse.epp.config/tools/setGitDate)) script. This script will make a local commit you need to push.
- [ ] Run a [CI build](https://ci.eclipse.org/packaging/job/simrel.epp-tycho-build/) that includes the above changes.
- [ ] Disable the [CI build](https://ci.eclipse.org/packaging/job/simrel.epp-tycho-build/) so that the build results are not overwritten while doing the promotion. You can disable the project once it has fully started running, you don't have to wait for the build to finish.
- [ ] Run a [CI build](https://ci.eclipse.org/packaging/job/epp/job/master/) that includes the above changes.
- If the build fails there may be the opportunity to continue the build rather than restart it. This is relatively underused option but enabled by the multi-step Jenkins build in the Jenkinsfile. For example, running the build with the previously successful steps commented out can produce a build.
- [ ] Disable the [CI build](https://ci.eclipse.org/packaging/job/epp/job/master/) so that the build results are not overwritten while doing the promotion. You can disable the project once it has fully started running, you don't have to wait for the build to finish.
- [ ] Check that there are no unexpected warnings in the console output. Especially look for warnings about failure to sign.
- The following warnings are known to be OK but ideally should be fixed at some point
- Warnings about Mirror tool seem to be ok and can be ignored. In a historically good build there is one `[WARNING] Mirror tool: Messages while mirroring artifact descriptors.` per package
Expand All @@ -89,7 +90,7 @@ This checklist is only used once per release cycle. Scroll down for the per-mile
- `https://download.eclipse.org/staging/2024-03/` - _NOTE_ Use `SIMREL_REPO` if the staging repo has been updated since the `SIMREL_REPO` location was created.
- `https://download.eclipse.org/technology/epp/staging/repository/`
- [ ] Verify no non-EPP content is in the p2 repo (especially justj, update [remove-justj-from-p2.xml](https://github.com/eclipse-packaging/packages/blob/master/releng/org.eclipse.epp.config/tools/remove-justj-from-p2.xml) if needed)
- [ ] Edit the [Jenkins build](https://ci.eclipse.org/packaging/job/simrel.epp-tycho-build/)
- [ ] Edit the [Jenkins build](https://ci.eclipse.org/packaging/job/epp/job/master/)
- [ ] Mark build as Keep forever
- [ ] Edit Jenkins Build Information and name it (e.g. `2020-03 M3`)
- [ ] In the evening Ottawa time, run the [Promote a Build](https://ci.eclipse.org/packaging/job/promote-a-build/) CI job to prepare build artifacts and copy them to download.eclipse.org
Expand All @@ -98,7 +99,7 @@ This checklist is only used once per release cycle. Scroll down for the per-mile
- The `DRY_RUN` can be done earlier in the day and is a good way to increase the chance that the final promotion step will be successful.
- [ ] Run the [Notarize MacOSX Downloads](https://ci.eclipse.org/packaging/job/notarize-downloads/) CI job to notarize DMG packages on download.eclipse.org if the promoted build was unstable
- [ ] Update `SIMREL_REPO` to the staging repo so CI builds run against CI of SimRel (e.g. [see this gerrit](https://git.eclipse.org/r/c/epp/org.eclipse.epp.packages/+/189618))
- [ ] Re-enable the [CI build](https://ci.eclipse.org/packaging/job/simrel.epp-tycho-build/)
- [ ] Re-enable the [CI build](https://ci.eclipse.org/packaging/job/epp/job/master/)
- [ ] Send email to epp-dev to request package maintainers test it. The email is templated in email.txt in the release directory.
- [ ] Archive old milestones/RCs so that they don't accumulate on the mirrors
- [ ] **24 Hours before Final release** Make sure files are in final location to allow downloads to mirror
Expand Down
Loading

0 comments on commit 727fd60

Please sign in to comment.