Skip to content
This repository has been archived by the owner on May 24, 2018. It is now read-only.

Commit

Permalink
Merge pull request #359 from Unity-Technologies/CDSK-926-customArtifa…
Browse files Browse the repository at this point in the history 
…ctPathRef

CDSK-926 - add customArtifactPath
  • Loading branch information
@warcholprzemo
warcholprzemo committed Apr 24, 2018
2 parents 4f054c6 + a721c8d commit 98fe357
Show file tree
Hide file tree
Showing 3 changed files with 177 additions and 41 deletions.
49 changes: 25 additions & 24 deletions master/buildbot/steps/artifact.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,6 @@
from twisted.python import log
import ntpath

# Change artifact location in August
# datetime.datetime(2017, 7, 31, 23, 59, 59, tzinfo=UTC)
ARTIFACT_LOCATION_CHANGE_DATE = epoch2datetime(1501545599)

def FormatDatetime(value):
return value.strftime("%d_%m_%Y_%H_%M_%S_%z")

Expand Down Expand Up @@ -182,7 +178,8 @@ class CheckArtifactExists(ShellCommandResumeBuild, FindPreviousSuccessBuildMixin
descriptionDone="Searching complete."

def __init__(self, artifact=None, artifactDirectory=None, artifactServer=None, artifactServerDir=None,
artifactServerURL=None, artifactServerPort=None, stopBuild=True, resumeBuild=None, **kwargs):
artifactServerURL=None, artifactServerPort=None, stopBuild=True, resumeBuild=None,
customArtifactPath=None, **kwargs):
self.master = None
self.build_sourcestamps = []
if not isinstance(artifact, list):
Expand All @@ -197,6 +194,7 @@ def __init__(self, artifact=None, artifactDirectory=None, artifactServer=None, a
self.artifactPath = None
self.artifactURL = None
self.stopBuild = stopBuild
self.customArtifactPath = customArtifactPath
resume_build_val = stopBuild if resumeBuild is None else resumeBuild
ShellCommandResumeBuild.__init__(self, resumeBuild=resume_build_val, **kwargs)

Expand Down Expand Up @@ -270,16 +268,15 @@ def start(self):
self.step_status.setText(["Artifact not found."])
self.finished(SUCCESS)

@defer.inlineCallbacks
def _previousBuildFound(self, prevBuildRequest):
self.artifactBuildrequest = prevBuildRequest
self.step_status.setText(["Artifact has been already generated."])

if self.artifactBuildrequest["submitted_at"] > ARTIFACT_LOCATION_CHANGE_DATE:
self.artifactPath = "%s/%s_%s" % (self.build.builder.config.builddir,
self.artifactBuildrequest['brid'],
FormatDatetime(self.artifactBuildrequest['submitted_at']))
if self.customArtifactPath:
self.artifactPath = yield self.build.render(self.customArtifactPath)
else:
self.artifactPath = "%s_%s_%s" % (self.build.builder.config.builddir,
self.artifactPath = "%s/%s_%s" % (self.build.builder.config.builddir,
self.artifactBuildrequest['brid'],
FormatDatetime(self.artifactBuildrequest['submitted_at']))

Expand Down Expand Up @@ -312,20 +309,21 @@ class CreateArtifactDirectory(ShellCommand):
descriptionDone="Remote artifact directory created."

def __init__(self, artifactDirectory=None, artifactServer=None, artifactServerDir=None, artifactServerPort=None,
**kwargs):
customArtifactPath=None, **kwargs):
self.artifactDirectory = artifactDirectory
self.artifactServer = artifactServer
self.artifactServerDir = artifactServerDir
self.artifactServerPort = artifactServerPort
self.customArtifactPath = customArtifactPath
ShellCommand.__init__(self, **kwargs)

@defer.inlineCallbacks
def start(self):
br = self.build.requests[0]
if mkdt(br.submittedAt) > ARTIFACT_LOCATION_CHANGE_DATE:
artifactPath = "%s/%s_%s" % (self.build.builder.config.builddir,
br.id, FormatDatetime(mkdt(br.submittedAt)))
if self.customArtifactPath:
artifactPath = yield self.build.render(self.customArtifactPath)
else:
artifactPath = "%s_%s_%s" % (self.build.builder.config.builddir,
artifactPath = "%s/%s_%s" % (self.build.builder.config.builddir,
br.id, FormatDatetime(mkdt(br.submittedAt)))

if (self.artifactDirectory):
Expand Down Expand Up @@ -393,7 +391,8 @@ class UploadArtifact(ShellCommand):
descriptionDone="Artifact(s) uploaded."

def __init__(self, artifact=None, artifactDirectory=None, artifactServer=None, artifactServerDir=None,
artifactServerURL=None, artifactServerPort=None, usePowerShell=True, **kwargs):
artifactServerURL=None, artifactServerPort=None, usePowerShell=True,
customArtifactPath=None, **kwargs):
self.artifact=artifact
self.artifactURL = None
self.artifactDirectory = artifactDirectory
Expand All @@ -402,6 +401,7 @@ def __init__(self, artifact=None, artifactDirectory=None, artifactServer=None, a
self.artifactServerURL = artifactServerURL
self.artifactServerPort = artifactServerPort
self.usePowerShell = usePowerShell
self.customArtifactPath = customArtifactPath
ShellCommand.__init__(self, **kwargs)

@defer.inlineCallbacks
Expand All @@ -413,10 +413,10 @@ def start(self):
master = self.build.builder.botmaster.parent
reuse = yield master.db.buildrequests.updateMergedBuildRequest(self.build.requests)

if mkdt(br.submittedAt) > ARTIFACT_LOCATION_CHANGE_DATE:
artifactPath = "%s/%s_%s" % (self.build.builder.config.builddir, br.id, FormatDatetime(mkdt(br.submittedAt)))
if self.customArtifactPath:
artifactPath = yield self.build.render(self.customArtifactPath)
else:
artifactPath = "%s_%s_%s" % (self.build.builder.config.builddir, br.id, FormatDatetime(mkdt(br.submittedAt)))
artifactPath = "%s/%s_%s" % (self.build.builder.config.builddir, br.id, FormatDatetime(mkdt(br.submittedAt)))

artifactServerPath = self.build.getProperty("artifactServerPath", None)
if artifactServerPath is None:
Expand Down Expand Up @@ -445,7 +445,8 @@ class DownloadArtifact(ShellCommand):
descriptionDone="Artifact(s) downloaded."

def __init__(self, artifactBuilderName=None, artifact=None, artifactDirectory=None, artifactDestination=None,
artifactServer=None, artifactServerDir=None, artifactServerPort=None, usePowerShell=True, **kwargs):
artifactServer=None, artifactServerDir=None, artifactServerPort=None, usePowerShell=True,
customArtifactPath=None, **kwargs):
self.artifactBuilderName = artifactBuilderName
self.artifact = artifact
self.artifactDirectory = artifactDirectory
Expand All @@ -455,6 +456,7 @@ def __init__(self, artifactBuilderName=None, artifact=None, artifactDirectory=No
self.artifactDestination = artifactDestination or artifact
self.master = None
self.usePowerShell = usePowerShell
self.customArtifactPath = customArtifactPath
name = "Download Artifact for '%s'" % artifactBuilderName
description = "Downloading artifact '%s'..." % artifactBuilderName
descriptionDone="Downloaded '%s'." % artifactBuilderName
Expand All @@ -469,11 +471,10 @@ def start(self):
#find artifact dependency
br = yield self._getBuildRequest()

if br["submitted_at"] > ARTIFACT_LOCATION_CHANGE_DATE:
artifactPath = "%s/%s_%s" % (safeTranslate(self.artifactBuilderName),
br['brid'], FormatDatetime(br["submitted_at"]))
if self.customArtifactPath:
artifactPath = yield self.build.render(self.customArtifactPath)
else:
artifactPath = "%s_%s_%s" % (safeTranslate(self.artifactBuilderName),
artifactPath = "%s/%s_%s" % (safeTranslate(self.artifactBuilderName),
br['brid'], FormatDatetime(br["submitted_at"]))

if (self.artifactDirectory):
Expand Down
149 changes: 142 additions & 7 deletions master/buildbot/test/unit/test_step_artifact.py
View file
Original file line number Diff line number Diff line change
@@ -1,24 +1,32 @@
from twisted.trial import unittest
from buildbot.test.util import steps

from buildbot.process.properties import Interpolate
from buildbot.steps import artifact
from buildbot.status.results import SUCCESS
from buildbot.test.fake.remotecommand import ExpectShell
from buildbot.test.fake import fakemaster, fakedb


class StubSourceStamp(object):
def asDict(self):
return {'codebase': 'fmod', 'revision': 'abcde'}


class TestArtifactSteps(steps.BuildStepMixin, unittest.TestCase):
def setUp(self):
return self.setUpBuildStep()

def tearDown(self):
return self.tearDownBuildStep()

def setupStep(self, step, brqs=None, winslave=False):
def setupStep(self, step, brqs=None, winslave=False, sources=None):
steps.BuildStepMixin.setupStep(self, step)

self.remote = '\'usr@srv.com:/home/srv/web/dir/build_1_17_12_2014_13_31_26_+0000/mydir/myartifact.py\''
self.remote_2 = '\'usr@srv.com:/home/srv/web/dir/B_2_01_01_1970_00_00_00_+0000/mydir/myartifact.py\''
self.remote = '\'usr@srv.com:/home/srv/web/dir/build/1_17_12_2014_13_31_26_+0000/mydir/myartifact.py\''
self.remote_2 = '\'usr@srv.com:/home/srv/web/dir/B/2_01_01_1970_00_00_00_+0000/mydir/myartifact.py\''
self.remote_custom = '\'usr@srv.com:/home/srv/web/dir/foobar/mydir/myartifact.py\''
self.remote_custom_interpolate = '\'usr@srv.com:/home/srv/web/dir/Art-abcde/mydir/myartifact.py\''
self.local = '\'myartifact.py\''

fake_br = fakedb.BuildRequest(id=1, buildsetid=1, buildername="A", complete=1, results=0)
Expand All @@ -35,6 +43,7 @@ def setupStep(self, step, brqs=None, winslave=False):
breqs += brqs

self.build.slavebuilder.slave.slave_environ = {}
self.build.sources = sources or {}

if winslave:
self.build.slavebuilder.slave.slave_environ['os'] = 'Windows_NT'
Expand All @@ -48,7 +57,7 @@ def test_create_artifact_directory(self):
self.expectCommands(
ExpectShell(workdir='wkdir', usePTY='slave-config',
command=['ssh', 'usr@srv.com', 'cd /home/srv/web/dir;', 'mkdir -p ',
'build_1_17_12_2014_13_31_26_+0000/mydir'])
'build/1_17_12_2014_13_31_26_+0000/mydir'])
+ ExpectShell.log('stdio', stdout='')
+ 0
)
Expand All @@ -63,13 +72,51 @@ def test_create_artifact_directory_with_port(self):
self.expectCommands(
ExpectShell(workdir='wkdir', usePTY='slave-config',
command=['ssh', 'usr@srv.com', '-p 222', 'cd /home/srv/web/dir;', 'mkdir -p ',
'build_1_17_12_2014_13_31_26_+0000/mydir'])
'build/1_17_12_2014_13_31_26_+0000/mydir'])
+ ExpectShell.log('stdio', stdout='')
+ 0
)
self.expectOutcome(result=SUCCESS, status_text=['Remote artifact directory created.'])
return self.runStep()

def test_create_artifact_directory_with_customArtifactPath(self):
self.setupStep(artifact.CreateArtifactDirectory(artifactDirectory="mydir",
artifactServer='usr@srv.com',
artifactServerDir='/home/srv/web/dir',
artifactServerPort=222,
customArtifactPath='foobar'))
self.expectCommands(
ExpectShell(workdir='wkdir', usePTY='slave-config',
command=['ssh', 'usr@srv.com', '-p 222', 'cd /home/srv/web/dir;',
'mkdir -p ',
'foobar/mydir'])
+ ExpectShell.log('stdio', stdout='')
+ 0
)
self.expectOutcome(result=SUCCESS, status_text=['Remote artifact directory created.'])
return self.runStep()

def test_create_artifact_directory_with_customArtifactPath_as_Interpolate(self):
customArtifactPath = Interpolate("Art-%(src:fmod:revision)s")
sourcestamp = StubSourceStamp()
self.setupStep(artifact.CreateArtifactDirectory(artifactDirectory="mydir",
artifactServer='usr@srv.com',
artifactServerDir='/home/srv/web/dir',
artifactServerPort=222,
customArtifactPath=customArtifactPath),
sources={'fmod': sourcestamp})
self.expectCommands(
ExpectShell(workdir='wkdir', usePTY='slave-config',
command=['ssh', 'usr@srv.com', '-p 222', 'cd /home/srv/web/dir;',
'mkdir -p ',
'Art-abcde/mydir'])
+ ExpectShell.log('stdio', stdout='')
+ 0
)
self.expectOutcome(result=SUCCESS, status_text=['Remote artifact directory created.'])
return self.runStep()


def test_upload_artifact(self):
self.setupStep(artifact.UploadArtifact(artifact="myartifact.py", artifactDirectory="mydir",
artifactServer='usr@srv.com', artifactServerDir='/home/srv/web/dir',
Expand All @@ -84,7 +131,7 @@ def test_upload_artifact(self):
)
self.expectOutcome(result=SUCCESS, status_text=['Artifact(s) uploaded.'])
self.expectProperty('artifactServerPath',
'http://srv.com/dir/build_1_17_12_2014_13_31_26_+0000',
'http://srv.com/dir/build/1_17_12_2014_13_31_26_+0000',
'UploadArtifact')
return self.runStep()

Expand All @@ -103,7 +150,50 @@ def test_upload_artifact_with_port(self):
)
self.expectOutcome(result=SUCCESS, status_text=['Artifact(s) uploaded.'])
self.expectProperty('artifactServerPath',
'http://srv.com/dir/build_1_17_12_2014_13_31_26_+0000',
'http://srv.com/dir/build/1_17_12_2014_13_31_26_+0000',
'UploadArtifact')
return self.runStep()

def test_upload_artifact_with_customArtifactPath(self):
self.setupStep(artifact.UploadArtifact(artifact="myartifact.py", artifactDirectory="mydir",
artifactServer='usr@srv.com',
artifactServerDir='/home/srv/web/dir',
artifactServerURL="http://srv.com/dir",
customArtifactPath="foobar"))
self.expectCommands(
ExpectShell(workdir='wkdir', usePTY='slave-config',
command='for i in 1 2 3 4 5; do rsync -var --progress --partial ' +
self.local + ' ' + self.remote_custom +
'; if [ $? -eq 0 ]; then exit 0; else sleep 5; fi; done; exit -1')
+ ExpectShell.log('stdio', stdout='')
+ 0
)
self.expectOutcome(result=SUCCESS, status_text=['Artifact(s) uploaded.'])
self.expectProperty('artifactServerPath',
'http://srv.com/dir/foobar',
'UploadArtifact')
return self.runStep()

def test_upload_artifact_with_customArtifactPath_as_Interpolate(self):
customArtifactPath = Interpolate("Art-%(src:fmod:revision)s")
sourcestamp = StubSourceStamp()
self.setupStep(artifact.UploadArtifact(artifact="myartifact.py", artifactDirectory="mydir",
artifactServer='usr@srv.com',
artifactServerDir='/home/srv/web/dir',
artifactServerURL="http://srv.com/dir",
customArtifactPath=customArtifactPath),
sources={'fmod': sourcestamp})
self.expectCommands(
ExpectShell(workdir='wkdir', usePTY='slave-config',
command='for i in 1 2 3 4 5; do rsync -var --progress --partial ' +
self.local + ' ' + self.remote_custom_interpolate +
'; if [ $? -eq 0 ]; then exit 0; else sleep 5; fi; done; exit -1')
+ ExpectShell.log('stdio', stdout='')
+ 0
)
self.expectOutcome(result=SUCCESS, status_text=['Artifact(s) uploaded.'])
self.expectProperty('artifactServerPath',
'http://srv.com/dir/Art-abcde',
'UploadArtifact')
return self.runStep()

Expand Down Expand Up @@ -214,6 +304,51 @@ def test_download_artifact_with_port(self):
self.expectOutcome(result=SUCCESS, status_text=["Downloaded 'B'."])
return self.runStep()

def test_download_artifact_with_customArtifactPath(self):
fake_trigger = fakedb.BuildRequest(id=2, buildsetid=2, buildername="B", complete=1,
results=0, triggeredbybrid=1, startbrid=1)
self.setupStep(artifact.DownloadArtifact(artifactBuilderName="B", artifact="myartifact.py",
artifactDirectory="mydir",
artifactServer='usr@srv.com',
artifactServerDir='/home/srv/web/dir',
customArtifactPath='foobar'),
[fake_trigger])

self.expectCommands(
ExpectShell(workdir='wkdir', usePTY='slave-config',
command='for i in 1 2 3 4 5; do rsync -var --progress --partial ' +
self.remote_custom + ' ' + self.local +
'; if [ $? -eq 0 ]; then exit 0; else sleep 5; fi; done; exit -1')
+ ExpectShell.log('stdio', stdout='')
+ 0
)
self.expectOutcome(result=SUCCESS, status_text=["Downloaded 'B'."])
return self.runStep()

def test_download_artifact_with_customArtifactPath_as_Interpolate(self):
customArtifactPath = Interpolate("Art-%(src:fmod:revision)s")
sourcestamp = StubSourceStamp()
fake_trigger = fakedb.BuildRequest(id=2, buildsetid=2, buildername="B", complete=1,
results=0, triggeredbybrid=1, startbrid=1)
self.setupStep(artifact.DownloadArtifact(artifactBuilderName="B", artifact="myartifact.py",
artifactDirectory="mydir",
artifactServer='usr@srv.com',
artifactServerDir='/home/srv/web/dir',
customArtifactPath=customArtifactPath),
[fake_trigger],
sources={'fmod': sourcestamp})

self.expectCommands(
ExpectShell(workdir='wkdir', usePTY='slave-config',
command='for i in 1 2 3 4 5; do rsync -var --progress --partial ' +
self.remote_custom_interpolate + ' ' + self.local +
'; if [ $? -eq 0 ]; then exit 0; else sleep 5; fi; done; exit -1')
+ ExpectShell.log('stdio', stdout='')
+ 0
)
self.expectOutcome(result=SUCCESS, status_text=["Downloaded 'B'."])
return self.runStep()

def test_download_artifact_Win_DOS(self):
fake_trigger = fakedb.BuildRequest(id=2, buildsetid=2, buildername="B", complete=1,
results=0, triggeredbybrid=1, startbrid=1)
Expand Down
Loading

0 comments on commit 98fe357

Please sign in to comment.