Skip to content

Commit

Permalink
Initial commit
Browse files Browse the repository at this point in the history
  • Loading branch information
unknown committed Jun 3, 2011
0 parents commit ac9b655
Show file tree
Hide file tree
Showing 16 changed files with 1,016 additions and 0 deletions.
11 changes: 11 additions & 0 deletions .gitignore
@@ -0,0 +1,11 @@
.checkstyle
.classpath
.pmd
.project
.ruleset
.settings/
.swp
target/
velocity.log
*.i??

161 changes: 161 additions & 0 deletions cxf/cxf-performance-soapui-project.xml
@@ -0,0 +1,161 @@
<?xml version="1.0" encoding="UTF-8"?>
<con:soapui-project name="cxf performance" resourceRoot="" soapui-version="4.0-beta2" abortOnError="false" runType="SEQUENTIAL" xmlns:con="http://eviware.com/soapui/config"><con:settings/><con:interface xsi:type="con:WsdlInterface" wsaVersion="NONE" name="CustomerServiceServiceSoapBinding" type="wsdl" bindingName="{http://customerservice.example.com/}CustomerServiceServiceSoapBinding" soapVersion="1_1" anonymous="optional" definition="file:/C:/checkout/performance-tests/cxf/src/main/resources/CustomerService.wsdl" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"><con:settings/><con:definitionCache type="TEXT" rootPart="file:/C:/checkout/performance-tests/cxf/src/main/resources/CustomerService.wsdl"><con:part><con:url>file:/C:/checkout/performance-tests/cxf/src/main/resources/CustomerService.wsdl</con:url><con:content><![CDATA[<!--Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.-->
<wsdl:definitions name="CustomerServiceService" targetNamespace="http://customerservice.example.com/" xmlns:wsdl="http://schemas.xmlsoap.org/wsdl/" xmlns:tns="http://customerservice.example.com/" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:soap="http://schemas.xmlsoap.org/wsdl/soap/">
<wsdl:types>
<xs:schema attributeFormDefault="unqualified" elementFormDefault="unqualified" targetNamespace="http://customerservice.example.com/" xmlns:xs="http://www.w3.org/2001/XMLSchema">
<xs:element name="getCustomersByName" type="tns:getCustomersByName"/>
<xs:element name="getCustomersByNameResponse" type="tns:getCustomersByNameResponse"/>
<xs:element name="updateCustomer" type="tns:updateCustomer"/>
<xs:complexType name="updateCustomer">
<xs:sequence>
<xs:element minOccurs="0" name="customer" type="tns:customer"/>
</xs:sequence>
</xs:complexType>
<xs:complexType name="customer">
<xs:sequence>
<xs:element name="customerId" type="xs:int"/>
<xs:element minOccurs="0" name="name" type="xs:string"/>
<xs:element maxOccurs="unbounded" minOccurs="0" name="address" nillable="true" type="xs:string"/>
<xs:element minOccurs="0" name="numOrders" type="xs:int"/>
<xs:element name="revenue" type="xs:double"/>
<xs:element minOccurs="0" name="test" type="xs:decimal"/>
<xs:element minOccurs="0" name="birthDate" type="xs:date"/>
<xs:element minOccurs="0" name="type" type="tns:customerType"/>
</xs:sequence>
</xs:complexType>
<xs:complexType name="getCustomersByName">
<xs:sequence>
<xs:element minOccurs="0" name="name" type="xs:string"/>
</xs:sequence>
</xs:complexType>
<xs:complexType name="getCustomersByNameResponse">
<xs:sequence>
<xs:element maxOccurs="unbounded" minOccurs="0" name="return" type="tns:customer"/>
</xs:sequence>
</xs:complexType>
<xs:simpleType name="customerType">
<xs:restriction base="xs:string">
<xs:enumeration value="PRIVATE"/>
<xs:enumeration value="BUSINESS"/>
</xs:restriction>
</xs:simpleType>
<xs:element name="NoSuchCustomer" type="tns:NoSuchCustomer"/>
<xs:complexType name="NoSuchCustomer">
<xs:sequence>
<xs:element name="customerName" nillable="true" type="xs:string"/>
</xs:sequence>
</xs:complexType>
</xs:schema>
</wsdl:types>
<wsdl:message name="getCustomersByNameResponse">
<wsdl:part name="parameters" element="tns:getCustomersByNameResponse"></wsdl:part>
</wsdl:message>
<wsdl:message name="getCustomersByName">
<wsdl:part name="parameters" element="tns:getCustomersByName"></wsdl:part>
</wsdl:message>
<wsdl:message name="updateCustomer">
<wsdl:part name="parameters" element="tns:updateCustomer"></wsdl:part>
</wsdl:message>
<wsdl:message name="NoSuchCustomerException">
<wsdl:part name="NoSuchCustomerException" element="tns:NoSuchCustomer"></wsdl:part>
</wsdl:message>
<wsdl:portType name="CustomerService">
<wsdl:operation name="updateCustomer">
<wsdl:input name="updateCustomer" message="tns:updateCustomer"></wsdl:input>
</wsdl:operation>
<wsdl:operation name="getCustomersByName">
<wsdl:input name="getCustomersByName" message="tns:getCustomersByName"></wsdl:input>
<wsdl:output name="getCustomersByNameResponse" message="tns:getCustomersByNameResponse"></wsdl:output>
<wsdl:fault name="NoSuchCustomerException" message="tns:NoSuchCustomerException"></wsdl:fault>
</wsdl:operation>
</wsdl:portType>
<wsdl:binding name="CustomerServiceServiceSoapBinding" type="tns:CustomerService">
<soap:binding style="document" transport="http://schemas.xmlsoap.org/soap/http"/>
<wsdl:operation name="updateCustomer">
<soap:operation soapAction="" style="document"/>
<wsdl:input name="updateCustomer">
<soap:body use="literal"/>
</wsdl:input>
</wsdl:operation>
<wsdl:operation name="getCustomersByName">
<soap:operation soapAction="" style="document"/>
<wsdl:input name="getCustomersByName">
<soap:body use="literal"/>
</wsdl:input>
<wsdl:output name="getCustomersByNameResponse">
<soap:body use="literal"/>
</wsdl:output>
<wsdl:fault name="NoSuchCustomerException">
<soap:fault name="NoSuchCustomerException" use="literal"/>
</wsdl:fault>
</wsdl:operation>
</wsdl:binding>
<wsdl:service name="CustomerServiceService">
<wsdl:port name="CustomerServicePort" binding="tns:CustomerServiceServiceSoapBinding">
<soap:address location="http://localhost:9090/CustomerServicePort"/>
</wsdl:port>
</wsdl:service>
</wsdl:definitions>]]></con:content><con:type>http://schemas.xmlsoap.org/wsdl/</con:type></con:part></con:definitionCache><con:endpoints><con:endpoint>http://localhost:9090/CustomerServicePort</con:endpoint></con:endpoints><con:operation isOneWay="false" action="" name="getCustomersByName" bindingOperationName="getCustomersByName" type="Request-Response" outputName="getCustomersByNameResponse" inputName="getCustomersByName" receivesAttachments="false" sendsAttachments="false" anonymous="optional"><con:settings/><con:call name="Request 1"><con:settings/><con:encoding>UTF-8</con:encoding><con:endpoint>http://localhost:9090/CustomerServicePort</con:endpoint><con:request><![CDATA[<soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/" xmlns:cus="http://customerservice.example.com/">
<soapenv:Header/>
<soapenv:Body>
<cus:getCustomersByName>
<!--Optional:-->
<name>?</name>
</cus:getCustomersByName>
</soapenv:Body>
</soapenv:Envelope>]]></con:request><con:wsaConfig mustUnderstand="NONE" version="200508" action="http://customerservice.example.com/CustomerService/getCustomersByName"/></con:call></con:operation><con:operation isOneWay="false" action="" name="updateCustomer" bindingOperationName="updateCustomer" type="One-Way" inputName="updateCustomer" sendsAttachments="false" anonymous="optional"><con:settings/><con:call name="Request 1"><con:settings/><con:encoding>UTF-8</con:encoding><con:endpoint>http://localhost:9090/CustomerServicePort</con:endpoint><con:request><![CDATA[<soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/" xmlns:cus="http://customerservice.example.com/">
<soapenv:Header/>
<soapenv:Body>
<cus:updateCustomer>
<!--Optional:-->
<customer>
<customerId>?</customerId>
<!--Optional:-->
<name>?</name>
<!--Zero or more repetitions:-->
<address>?</address>
<!--Optional:-->
<numOrders>?</numOrders>
<revenue>?</revenue>
<!--Optional:-->
<test>?</test>
<!--Optional:-->
<birthDate>?</birthDate>
<!--Optional:-->
<type>?</type>
</customer>
</cus:updateCustomer>
</soapenv:Body>
</soapenv:Envelope>]]></con:request><con:wsaConfig mustUnderstand="NONE" version="200508" action="http://customerservice.example.com/CustomerService/updateCustomer"/></con:call></con:operation></con:interface><con:testSuite name="CustomerServiceServiceSoapBinding TestSuite"><con:settings/><con:runType>SEQUENTIAL</con:runType><con:testCase failOnError="true" failTestCaseOnErrors="true" keepSession="false" maxResults="0" name="getCustomersByName TestCase" searchProperties="true"><con:settings/><con:testStep type="request" name="getCustomersByName" id="f1e5925c-c3b6-40e9-8eec-1ecb65202bf8"><con:settings/><con:config xsi:type="con:RequestStep" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"><con:interface>CustomerServiceServiceSoapBinding</con:interface><con:operation>getCustomersByName</con:operation><con:request name="getCustomersByName"><con:settings><con:setting id="com.eviware.soapui.impl.wsdl.WsdlRequest@request-headers">&lt;xml-fragment/></con:setting></con:settings><con:encoding>UTF-8</con:encoding><con:endpoint>http://localhost:9090/CustomerServicePort</con:endpoint><con:request><![CDATA[<soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/" xmlns:cus="http://customerservice.example.com/">
<soapenv:Header/>
<soapenv:Body>
<cus:getCustomersByName>
<name>Test</name>
</cus:getCustomersByName>
</soapenv:Body>
</soapenv:Envelope>]]></con:request><con:jmsConfig JMSDeliveryMode="PERSISTENT"/><con:jmsPropertyConfig/><con:wsaConfig action="http://customerservice.example.com/CustomerService/getCustomersByName" mustUnderstand="NONE" version="200508"/><con:wsrmConfig version="1.2"/></con:request></con:config></con:testStep><con:loadTest name="LoadTest 1"><con:settings><con:setting id="HttpSettings@close-connections">false</con:setting></con:settings><con:threadCount>2</con:threadCount><con:startDelay>0</con:startDelay><con:sampleInterval>250</con:sampleInterval><con:calculateTPSOnTimePassed>true</con:calculateTPSOnTimePassed><con:resetStatisticsOnThreadCountChange>true</con:resetStatisticsOnThreadCountChange><con:historyLimit>-1</con:historyLimit><con:testLimit>60</con:testLimit><con:limitType>TIME</con:limitType><con:loadStrategy><con:type>Simple</con:type><con:config><testDelay>0</testDelay><randomFactor>0.0</randomFactor></con:config></con:loadStrategy><con:assertion type="Step Status" name="Step Status"/><con:maxAssertionErrors>100</con:maxAssertionErrors><con:statisticsLogFolder/><con:statisticsLogInterval>0</con:statisticsLogInterval><con:logStatisticsOnThreadChange>false</con:logStatisticsOnThreadChange><con:cancelOnReachedLimit>false</con:cancelOnReachedLimit><con:cancelExcessiveThreads>true</con:cancelExcessiveThreads><con:strategyInterval>500</con:strategyInterval><con:updateStatisticsPerTestStep>false</con:updateStatisticsPerTestStep></con:loadTest><con:properties/></con:testCase><con:testCase failOnError="true" failTestCaseOnErrors="true" keepSession="false" maxResults="0" name="updateCustomer TestCase" searchProperties="true"><con:settings/><con:testStep type="request" name="updateCustomer"><con:settings/><con:config xsi:type="con:RequestStep" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"><con:interface>CustomerServiceServiceSoapBinding</con:interface><con:operation>updateCustomer</con:operation><con:request name="updateCustomer"><con:settings><con:setting id="com.eviware.soapui.impl.wsdl.WsdlRequest@request-headers">&lt;xml-fragment/></con:setting></con:settings><con:encoding>UTF-8</con:encoding><con:endpoint>http://localhost:9090/CustomerServicePort</con:endpoint><con:request><![CDATA[<soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/" xmlns:cus="http://customerservice.example.com/">
<soapenv:Header/>
<soapenv:Body>
<cus:updateCustomer>
<customer>
<customerId>1</customerId>
<name>Test</name>
</customer>
</cus:updateCustomer>
</soapenv:Body>
</soapenv:Envelope>]]></con:request><con:jmsConfig JMSDeliveryMode="PERSISTENT"/><con:jmsPropertyConfig/><con:wsaConfig action="http://customerservice.example.com/CustomerService/updateCustomer" mustUnderstand="NONE" version="200508"/><con:wsrmConfig version="1.2"/></con:request></con:config></con:testStep><con:loadTest name="LoadTest 1"><con:settings><con:setting id="HttpSettings@close-connections">false</con:setting></con:settings><con:threadCount>5</con:threadCount><con:startDelay>0</con:startDelay><con:sampleInterval>250</con:sampleInterval><con:calculateTPSOnTimePassed>true</con:calculateTPSOnTimePassed><con:resetStatisticsOnThreadCountChange>true</con:resetStatisticsOnThreadCountChange><con:historyLimit>-1</con:historyLimit><con:testLimit>60</con:testLimit><con:limitType>TIME</con:limitType><con:loadStrategy><con:type>Simple</con:type></con:loadStrategy><con:assertion type="Step Status" name="Step Status"/><con:maxAssertionErrors>100</con:maxAssertionErrors><con:cancelExcessiveThreads>true</con:cancelExcessiveThreads><con:strategyInterval>500</con:strategyInterval></con:loadTest><con:properties/></con:testCase><con:properties/></con:testSuite><con:properties/><con:wssContainer/></con:soapui-project>
84 changes: 84 additions & 0 deletions cxf/performance_article.txt
@@ -0,0 +1,84 @@
h1. CXF Performance Measurements

From time to time people ask how fast is CXF? Of course this is a difficult question as the measuered speed depends very much in the Hardware of the test setup and on the whole definition of the test.
So I am trying to explain how you can do your own tests and what to do to make sure you get clean results.

What should you keep in mind when doing performance tests with Java

* Thread count is a typical number you should experiment with
* As long as you have not maxed out a resource you can improve the results. Typical resources to check are processor load, memory and network
* Increase the thread count until you max out a resource. But do not go much higher
* Always use a warmup phase (~1-2 minutes). Java needs to load classes the first time. On the Sun VM additionally the Hotspot compiler will kick in after some time

h2. Prerequisites

The test project can be found on my github account. You can either download a zip or clone the project with git:
[https://github.com/cschneider/performance-tests}

As a load generator and measurement tool we use soapUI. Download the free version from the link below:
[http://www.soapui.org/]

h2. Our test plan

We test SOAP/HTTP and SOAP/JMS performance using a small but non trivial service. For this case the CustomerService from the wsdl_first example will be used.

h2. Customerservice SOAP/HTTP performance

For the server side I have prepared a maven project which start the CustomerService implementation from the wsdl_first example on an embedded jetty. We could
also use an external server but in my tests the results were similar and the embedded version can be started very easily.

The number of listener threads can be adjusted in the file src/main/resources/server-applicationContext.xml :

{code}
<httpj:threadingParameters minThreads="5" maxThreads="5" />
{code}

Start the server:

cd cxf
mvn -Pserver

Start soapUI and load the soapUI project from the file cxf/cxf-performance-soapui-project.xml. The project was built using the wsdl of the CustomerService and contains
test requests and a load test definition.

Now navigate to the Loadtest 1 like shown in screenshot and start the loadtest by clicking on the green arrow. The intersting result is tps (Transactions per seconds). It measures how many Requests/Resonses are processed per second.
At first the number will be quite low but increase steadily. That is because of class loading and optimizations in Java. Let the test run 60 seconds. This was the warmup phase. Now start the test again.

h2. Customerservice SOAP/JMS performance



h2. Environment

It is always important to describe excatly on which configuration the test was run.
All the tests below were run on a Intel Core i5 / 8GB System. Client and Server where on the same machine.

h2. SOAP/HTTP Results

Threads are listener and client threads. CPU load is read from the windows Task Manager. Transactions per Second are the highest number from soapUI.

The payload size can be adjusted by the number of customer records the server sends:

|| Customers || payload size ||
| 0 | 190 |
| 2 | 662 |
| 200 | 44222 | 44 KB |
| 20000 | 4400222 | 4,4 MB |
| 200000 | 44000222 | 44 MB |

|| Threads || Payload size || CPU Load || Transations per Second ||
| 10 | 190 | 100% | 3770 |
| 5 | 662 | 57% | 2480 |
| 10 | 662 | 100% | 3660 |
| 20 | 662 | 100% | 3590 |
| 10 | 44 KB | 100% | 804 |
| 10 | 4,4 MB | 100% | 9 |
| 4 | 44 MB | 100% | 0,6 |

So it looks like 10 threads is ideal for the test machine with 2 cores and 4 virtual cores. This is quite near the rule of thumb to use double the amount of cores as optimal thread number.
When scaling up the payload size performance drops with the same factor. In the last line the drop is a little higher as I was not able to use 10 threads as soapUI then exits with an out of memory error.

When looking at the number with large payload it is important to know that we are sending very fine grained xml. When using mtom attachments for larger data performance is much better.

h2. SOAP/JMS results

0 comments on commit ac9b655

Please sign in to comment.