From 8176e3d37554734d44f52f1a1233f9ebaaa9bd6a Mon Sep 17 00:00:00 2001 From: Jeff Pei Date: Sun, 1 Nov 2015 23:58:47 -0800 Subject: [PATCH] update --- CHANGELOG.md | 22 +- CONTRIBUTING.md | 9 + README.md | 213 +++++++++++++++++- pom.xml | 17 +- .../java/io/parallec/core/FilterRegex.java | 4 +- .../java/io/parallec/core/ParallecHeader.java | 5 +- .../core/ParallecResponseHandler.java | 1 - .../java/io/parallec/core/ParallelClient.java | 30 ++- .../java/io/parallec/core/ParallelTask.java | 20 +- .../io/parallec/core/ParallelTaskBuilder.java | 26 ++- .../io/parallec/core/RequestProtocol.java | 2 +- .../parallec/core/ResponseOnSingleTask.java | 15 +- .../core/actor/AssistantExecutionManager.java | 11 +- .../parallec/core/actor/ExecutionManager.java | 2 +- .../io/parallec/core/actor/HttpWorker.java | 24 +- .../parallec/core/actor/OperationWorker.java | 36 ++- .../io/parallec/core/actor/PingWorker.java | 22 +- .../io/parallec/core/actor/SshWorker.java | 13 +- .../io/parallec/core/actor/TcpWorker.java | 8 +- .../core/actor/message/NodeReqResponse.java | 6 +- ...ResponseCountToBatchSenderAsstManager.java | 2 +- .../actor/message/ResponseOnSingeRequest.java | 7 +- .../core/actor/poll/HttpPollerProcessor.java | 2 +- .../parallec/core/actor/poll/PollerData.java | 2 +- .../io/parallec/core/app/ParallecAppMin.java | 2 +- .../io/parallec/core/app/package-info.java | 2 +- .../java/io/parallec/core/bean/HttpMeta.java | 2 +- .../io/parallec/core/bean/SetAndCount.java | 2 +- .../core/bean/SingleTargetTaskStatus.java | 2 +- .../java/io/parallec/core/bean/StrStrMap.java | 2 +- .../io/parallec/core/bean/TargetHostMeta.java | 2 +- .../io/parallec/core/bean/TaskRequest.java | 2 +- .../io/parallec/core/bean/ping/PingMeta.java | 2 +- .../io/parallec/core/bean/ping/PingMode.java | 2 +- .../io/parallec/core/bean/ssh/SshMeta.java | 39 ++-- .../io/parallec/core/bean/tcp/TcpMeta.java | 4 +- .../workflow/InternalDataProvider.java | 35 +-- .../workflow/VarReplacementProvider.java | 14 +- .../commander/workflow/ssh/SshProvider.java | 2 +- .../core/config/ParallecGlobalConfig.java | 7 +- .../core/config/ParallelTaskConfig.java | 8 +- .../ActorMessageTypeInvalidException.java | 2 +- .../core/monitor/MonitorProvider.java | 10 +- .../AsyncHttpClientFactoryEmbed.java | 2 +- .../core/resources/HttpClientStore.java | 13 +- .../core/resources/HttpClientType.java | 2 +- .../resources/TcpSshPingResourceStore.java | 2 +- .../core/task/CapacityAwareTaskScheduler.java | 10 +- .../parallec/core/task/ParallelTaskBean.java | 2 + .../core/task/ParallelTaskManager.java | 42 ++-- .../TargetHostsBuilderHelperCms.java | 2 +- .../io/parallec/core/util/BeanMapper.java | 2 +- .../io/parallec/core/util/PcConstants.java | 2 +- .../io/parallec/core/util/PcHttpUtils.java | 2 +- .../core/util/PcTargetHostsUtils.java | 2 +- .../io/parallec/core/ParallelClientTest.java | 2 +- src/test/java/io/parallec/core/TestBase.java | 6 +- .../parallec/core/actor/HttpWorkerTest.java | 12 +- .../core/actor/OperationWorkerTest.java | 2 +- .../httpclient/async/HttpClientStoreTest.java | 2 +- ...arallelClientHttpBasicMoreOptionsTest.java | 2 +- .../http/ParallelClientHttpBasicTest.java | 2 +- ...rallelClientHttpBasicTestInvalidInput.java | 6 +- .../main/http/ParallelClientHttpLogTest.java | 2 +- .../ParallelClientHttpOtherMethodsTest.java | 8 +- .../ParallelTaskCancelOnTargetHostsTest.java | 2 +- .../http/ParallelTaskCancelWholeTaskTest.java | 2 +- .../sampleserver/ServerWithPollableJobs.java | 2 +- ...lClientVarReplacementHostSpecificTest.java | 36 ++- ...rallelClientVarReplacementUniformTest.java | 2 +- .../ParallelClientSchedulerAndCancelTest.java | 3 + .../core/main/ping/PingOnTmpTest.java | 4 +- .../io/parallec/core/main/ping/PingTest.java | 6 +- .../parallec/core/main/ping/package-info.java | 2 +- .../parallec/core/main/tcp/package-info.java | 2 +- .../main/tcp/sampleserver/TcpEchoServer.java | 2 +- .../main/tcp/sampleserver/package-info.java | 2 +- .../io/parallec/core/task/package-info.java | 2 +- .../targethosts/TargetHostsBuilderTest.java | 8 +- .../core/util/FileNetworkIoUtilsTest.java | 1 + userdata/sample_target_hosts_top100_old.txt | 8 +- 81 files changed, 543 insertions(+), 308 deletions(-) create mode 100644 CONTRIBUTING.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 32c33d1..2ae4544 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,26 +1,32 @@ -Change Log +Parallec Change Log ========== +## Version 0.9.0 -## Version 0.8.11 +_2015-11-01_ -_2015-10-28_ +* Add: Late initialize CapacityAwareTaskScheduler only when it is used. Added shutdown for the scheduler. +* Test: Update some tests on variable replacements. +* Doc: Update javadoc. -* Javadoc: refine javadoc. -* Coverage: add code coverage. -* Maven: setup with maven central deployment. Added codecov and travis CI. +## Version 0.8.12-beta +_2015-10-29_ +* Javadoc: refine javadoc. Fixed errors in javadoc. +* Coverage: add code coverage. +* Maven: setup with maven central deployment. Added codecov and travis CI. -## Version 0.8.10 +## Version 0.8.11-beta +## Version 0.8.10 _2015-10-27_ * Change: refactor and removed command director. Renamed command manager to execution manager. * Change: remove duplicated functions of prepareHttp* in builder and client classes. Refactor internal type of http method. * Add: ParallelTask add getAggregatedResultHumanStr() to display human readable results. -* Change: enforce safeguard of concurrency limit for ssh as 400. +* Change: enforce safeguard of concurrency limit for SSH as 400. ## Version 0.8.9 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..4d8e2ea --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,9 @@ +## Contributing to Eagle + +Contributions via GitHub pull requests are gladly accepted from their original +author. Along with any pull requests, please state that the contribution is +your original work and that you license the work to the project under the +project's open source license. Whether or not you state this explicitly, by +submitting any copyrighted material via pull request, email, or other means +you agree to license the material under the project's open source license and +warrant that you have the legal authority to do so. diff --git a/README.md b/README.md index 0f4ebc1..f2a1530 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,213 @@ -##Parallec +Parallec-logo -![build status](https://img.shields.io/badge/build-info =>-green.svg) [![Build Status](https://travis-ci.org/eBay/parallec.svg?branch=master)](https://travis-ci.org/eBay/parallec) [![Coverage Status](https://img.shields.io/codecov/c/github/eBay/parallec.svg)](https://codecov.io/github/eBay/parallec) [![MIT License](https://img.shields.io/hexpm/l/plug.svg) ](https://github.com/eBay/parallec/blob/master/LICENSE) -![latest 0.8.x](https://img.shields.io/badge/latest stable-0.8.x =>-green.svg) [ ![latest beta maven central](https://maven-badges.herokuapp.com/maven-central/com.ebay.parallec/parallec-core/badge.svg?style=flat)](http://search.maven.org/#artifactdetails|com.ebay.parallec|parallec-core|0.8.11-beta|) +![build status](https://img.shields.io/badge/build-info=>-green.svg) [![Build Status](https://travis-ci.org/eBay/parallec.svg?branch=master)](https://travis-ci.org/eBay/parallec) [![Coverage Status](https://img.shields.io/codecov/c/github/eBay/parallec.svg)](https://codecov.io/github/eBay/parallec) [![Apache V2.0 License](http://www.parallec.io/images/apache2.svg) ](https://github.com/eBay/parallec/blob/master/LICENSE) -## Stay Tuned +![latest 0.9.x](http://img.shields.io/badge/latest_stable-0.9.x=>-green.svg) [ ![latest beta maven central](https://maven-badges.herokuapp.com/maven-central/io.parallec/parallec-core/badge.svg?style=flat)](http://search.maven.org/#artifactdetails|io.parallec|parallec-core|0.9.0|) + +[![Javadoc](http://www.parallec.io/images/parallec-javadoc-blue.svg)](http://www.parallec.io/javadoc/index.html?io/parallec/core/ParallelClient.html) [![Documentation](http://www.parallec.io/images/parallec-documentation-red.svg)](http://www.parallec.io/docs/) [![Samples](http://www.parallec.io/images/parallec-samples-brightgreen.svg)](https://github.com/eBay/parallec-samples) + + +Parallec is a performant parallel async http/ssh/tcp/ping client java library. Scalably aggregate and handle API responses **anyway** and send it **anywhere** by writing [20 lines](https://www.youtube.com/watch?v=QcavegPMDms) of code. Response handler with context enables you conduct scalable API calls, then pass aggregated data anywhere to elastic search, kafka, MongoDB, graphite, memcached, etc. + +Parallec means **Paralle**l **C**lient, and is pronounced as "Para-like". Parallec is built on Akka actors and Async HTTP Client / Netty / Jsch. The library focuses on HTTP while also enables scalable communication over SSH/Ping/TCP. + +![Workflow Overview](http://www.parallec.io/images/parallec-flow.svg) + + +####Motivation +- Flexible response handling and immediate processing embedded in other applications. +- Handle async APIs with auto progress polling for task level concurrency control. +- Support of other protocols, and [more](#features).. + +With the feedbacks, lessons, and improvements from the past year of internal usage and open source of **[REST Commander](www.restcommander.com)**, we now made the core of REST Commander as an easy to use standalone library. We added [**15+ new**](#compare) features, rewritten 70%+ of the code, with [**90%+ test coverage**](https://codecov.io/github/eBay/parallec) for confident usage and contribution. This time we also structure it better so that most internal development can be directly made here. + +###Use Cases + +1. Scalable web server monitoring, management, and configuration push, ping check. +1. Asset / server status discovery, remote task execution in agent-less(parallel SSH) or agent based (parallel HTTP/TCP) method. +1. Scalable API aggregation and processing with flexible destination with your favorate message queue / storage / alert engine. +1. Orchestration and work flows on multiple web servers. +1. Parallel different requests with controlled concurrency to a single server. + + +## Features + +**90%+ Test coverage** assures you always find an example of each of feature. + +1. **Exceedingly intuitive** interface with builder pattern similar to that in [Async HTTP Client](https://github.com/AsyncHttpClient/async-http-client), but handles concurrency behind the scenes. +1. **Generic response handler with context**. Enable total freedom of processing each response your way. Process and aggregate data **anywhere** to Kafka, Redis, Elastic Search, mongoDB, CMS and etc. +1. **Flexible on when to invoke the handler**: before (in worker thread) or after the aggregation (in master/manager thread). +1. **Flexible Input of target hosts**: Input target hosts from a list, string, JSON Path from local files or a remote URL +1. **Scalable and fast**, **infinitely scalable** with built-in **Concurrency control**. +1. **Auto-progress polling** to enable task level concurrency with **Async API** for long jobs and orchestrations. +1. **Request template** to handle non-uniform requests. +1. **Convenient single place handling success and failure cases**. Handle in a single function where you can get the response including the actual response if success; or stacktrace and error details if failures. +1. **Capacity aware task scheduler** helps you to auto queue up and fire tasks when capacity is insufficient. (e.g. submit consecutively 5 tasks each hitting 100K websites with default concurrency will result in a queue up) +1. **Fine-grained task progress tracking** helps you track the the progress each individual task status. Of a parallel task on 1000 target hosts, you may check status on any single host task, and percentage progress on how many are completed. +1. **Fine-grained task cancelation** on whole/individual request level. Of a parallel task on 1000 target hosts, you may cancel a subset of target hosts or cancel the whole parallel task anytime. +1. **Status-code-aggregation** is provided out of the box. +1. **Parallel Ping** supports both InetAddress.reachable ICMP (requires root) and Process based ping with retries. Performance testing shows it is ** 67% faster** than best-effort tuned FPing on pinging on 1500 targets. (2.7 vs 4.5 sec) +1. **Parallel SSH** supports both key and password based login and task cancellation. +1. **Parallel TCP** supports idle timeout based channel closes. + + +### Get Started + +#### Maven / Gradle Import + +Please replace the version with the latest version available. + +######Maven + +```xml + + io.parallec + parallec-core + 0.9.0 + +``` + +######Gradle + +```xml +compile 'io.parallec:parallec-core:0.9.0' +``` + +####Examples + +- **List of Code Examples** please check [here](https://github.com/ebay/parallec-samples#http). + +In the example below, simply changing **prepareHttpGet()** to **prepareSsh()**, **prepareTcp()**, **preparePing()** enables you to conduct parallel SSH/TCP/Ping. Details please refer to [Java Doc](http://www.parallec.io/javadoc/index.html?io/parallec/core/ParallelClient.html). + +```java +import io.parallec.core.*; +import java.util.Map; + +ParallelClient pc = new ParallelClient(); +pc.prepareHttpGet("").setTargetHostsFromString("www.google.com www.ebay.com www.yahoo.com") +.execute(new ParallecResponseHandler() { + public void onCompleted(ResponseOnSingleTask res, + Map responseContext) { + System.out.println( res.toString() ); } +}); +pc.releaseExternalResources(); +``` + +Here is another example with response parsing and a little more parameters. + +```java +import io.parallec.core.*; +import java.util.Map; + +ParallelClient pc= new ParallelClient(); +pc.prepareGet("/validateInternals.html") + .setConcurrency(1000) + .setTargetHostsFromString("parallec.github.io www.jeffpei.com www.restcommander.com") + .execute(new ParallecResponseHandler() { + @Override + public void onCompleted(ResponseOnSingleTask res, + Map responseContext) { + String cpu = new FilterRegex(".*CPU-Usage-Percent\\s*(.*?).*") + .filter(res.getResponseContent()); + System.out.println("cpu:" + cpu + " host: " + res.getHost() ); + } + }); +pc.releaseExternalResources(); +``` + + +## Watch Parallec in Action + +[**Watch Demo**](https://www.youtube.com/watch?v=QcavegPMDms"Parallec demo - Click to Watch!"): Parallec Aggregates 100 websites status with 20 lines of code. + +20 lines parallec to elastic search demo + + + +## Performance + +Note that speed varies based on network speed, API response time, the slowest servers, and concurrency settings. + +#####HTTP + +We conducted remote task execution API on 3,000 servers with response aggregated to elastic search, visualized within 15 seconds, by writing 25 lines of code. + +With another faster API, calls to 8,000 servers in the same datacenter with response aggregated in memory in 12 seconds. + +#####Ping +Parallec 2.7 seconds vs FPing 4.5 seconds on 1500 servers. Parallec is 67% faster than [FPing](http://fping.org/) (after best-effort tuning : -i 1 -r 0 v3.12) of pinging 1500 servers while getting the same ping results. While FPing consistently crashing (seg fault) when it pings 2000 or more servers, Parallec pings 8000 servers within 11.8 seconds with breeze. + +As usual, don't rely on these numbers and perform your own benchmarks. + + + + + +## Compare Parallec vs REST Commander vs ThreadPools+Async Client + + +| Features | Parallec | REST Commander | Thread Pools + Async Client | +|:---------------------------------------------------------------------------------------------------------------------:|:--------:|:--------------:|:---------------------------:| +| Embedded library with intuitive builder pattern interface | Yes | No | No | +| Ready to use application with GUI wizard based request submission and response aggregation | No | Yes | No | +| Simple concurrency control not limited by thread size | Yes | Yes | No | +| Immediate response handler without waiting all response return | Yes | No | Yes | +| Capacity aware task scheduler and global capacity control | Yes | No | No | +| Total freedom of response processing and API aggregation: Pluggable and generic response handler and response context | Yes | No | No* | +| 1 line plugin to enable SSL Client auth | Yes | No | No | +| 90% Test Coverage | Yes | No | No | +| Load target hosts from CMS query, JSON Path, text, list, string from URL/local | Yes | No | No | +| Task level concurrency and orchestration for Async APIs: auto polling task progress | Yes | No | No | +| Task level configuration on timeout and replacing Async HTTP Client | Yes | No | No | +| Async and sync task control with progress polling and cancellation | Yes | No | No | +| Scalable Parallel SSH with password and key based login | Yes | No | No | +| Proven scalability and speed on 100,000+ target hosts in Production environment | Yes | Yes | No | +| Generic request template with variable replacement for sending different requests to same/different target hosts | Yes | Yes | No | +| Scalable Ping with Retries | Yes | No | No | +| Scalable TCP with idle timeout | Yes | No | No | +| Flexible handler location at either worker (in parallel) or manager thread | Yes | No | No | +| Out-of-the-box two-level response aggregation on status code| Yes | Yes | No | +| Configurable response log trimming on intervals| Yes | No | No | +| Cancel task on a list of target hosts | Yes | No | No | + +#### Actor Based Concurrency Control + +![System Design](http://www.parallec.io/images/parallec-actors-v2.svg) + +This is similar to the [model](www.ebaytechblog.com/2014/03/11/rest-commander-scalable-web-server-management-and-monitoring/#akka) in REST Commander, except that: + +- The user defined response handler can be triggerred in either the manager after aggregation, or in parallel when each operation worker receives the response. +- When handling async APIs, a single task may contains one job submission request, plus *1-n* progress polling requests. +- Worker now have more protocols such as SSH/Ping/TCP to support. + + +## [Plugins](https://github.com/eBay/parallec-plugins) + +#### [SSL Client Auth Plugin](https://github.com/eBay/parallec-plugins) + +## Authors + +Parallec is served to you by [Yuanteng (Jeff) Pei](https://www.linkedin.com/in/peiyuant) and [Teng Song](https://www.linkedin.com/pub/teng-song/49/763/713), [Cloud Infrastructure & Platform Services (CIPS)](https://helpusbuild.ebayc3.com/) at eBay Inc. + +## Credits & Acknowledgement + +- We thanks our manager [Andy Santosa](https://www.linkedin.com/pub/andy-santosa/0/230/305), project manager [Marco Rotelli](https://www.linkedin.com/pub/marco-rotelli/2/25/54), [Cloud Infrastructure & Platform Services (CIPS)](https://helpusbuild.ebayc3.com/) and legal for the big support on this project and the open source effort. +- The auto-progress polling is inspired by [lightflow](https://github.com/yubin154/lightflow). +- We thank [openpojo](https://github.com/oshoukry/openpojo) and the author Osman Shoukry for his help on making the openpojo more accessible for us to use in our project. +- We thank [AsyncHttpClient](https://github.com/AsyncHttpClient/async-http-client) and Stephane Landelle for guidance. + + + +##Contributions + +Any helpful feedback is more than welcome. This includes feature requests, bug reports, pull requests, constructive feedback, and etc. + + +## Licenses + +Code licensed under Apache License v2.0 + +© 2015 eBay Software Foundation -Scheduled to release by 11/02/2015: working on documentations and more. \ No newline at end of file diff --git a/pom.xml b/pom.xml index 2afa494..6db7648 100644 --- a/pom.xml +++ b/pom.xml @@ -2,15 +2,15 @@ 4.0.0 - com.ebay.parallec + io.parallec parallec-core - 0.8.11-beta + 0.9.0 jar - com.ebay.parallec:parallec-core + io.parallec:parallec-core https://github.com/eBay/parallec Parallec: Parallel Async HTTP/SSH/PING/TCP Client library. - More at https://github.com/eBay/parallec + Details at: http://www.parallec.io @@ -39,7 +39,10 @@ www.ebay.com - + + eBay + http://www.ebay.com/ + 2015 @@ -204,10 +207,6 @@ - - - - diff --git a/src/main/java/io/parallec/core/FilterRegex.java b/src/main/java/io/parallec/core/FilterRegex.java index 3dea119..f28a354 100644 --- a/src/main/java/io/parallec/core/FilterRegex.java +++ b/src/main/java/io/parallec/core/FilterRegex.java @@ -22,8 +22,8 @@ import org.slf4j.LoggerFactory; /** - * The Class FilterRegex. - * @author Yuanteng Jeff Pei + * A simple class to apply a regular expression to . + * @author Yuanteng (Jeff) Pei */ public class FilterRegex { diff --git a/src/main/java/io/parallec/core/ParallecHeader.java b/src/main/java/io/parallec/core/ParallecHeader.java index 1ddbd8d..2f82f0d 100644 --- a/src/main/java/io/parallec/core/ParallecHeader.java +++ b/src/main/java/io/parallec/core/ParallecHeader.java @@ -19,10 +19,9 @@ /** - * easy to use builder to add header key/value pair then get the results - * directly for use. + * Easy to use HTTP Header builder to add header key/value pair. * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei */ public class ParallecHeader { diff --git a/src/main/java/io/parallec/core/ParallecResponseHandler.java b/src/main/java/io/parallec/core/ParallecResponseHandler.java index 1fb3e03..4227efa 100644 --- a/src/main/java/io/parallec/core/ParallecResponseHandler.java +++ b/src/main/java/io/parallec/core/ParallecResponseHandler.java @@ -18,7 +18,6 @@ /** * The Interface ParallecResponseHandler. * - * Consider make it serializable if we need to pass this for akka clustering. */ public interface ParallecResponseHandler { diff --git a/src/main/java/io/parallec/core/ParallelClient.java b/src/main/java/io/parallec/core/ParallelClient.java index d769edf..d6e375c 100644 --- a/src/main/java/io/parallec/core/ParallelClient.java +++ b/src/main/java/io/parallec/core/ParallelClient.java @@ -27,25 +27,22 @@ import com.ning.http.client.AsyncHttpClient; - /** * * This is the starting point of starting a parallel client, which create a * {@link ParallelTaskBuilder} after a prepareGet functions. The Parallel * Task builder is to build a parallel task. * - * - * * This is more a wrapper class and does not hold object specific resources. You - * may create a single or multiple instance. + * may create a single or multiple instances. * * Only call {@link ParallelClient#releaseExternalResources()} before your app - * shutdown. will mark as closed after it is closed + * shutdown and when you no longer need to use it. Will mark as closed after it is closed * {@link ParallelClient#isClosed} * - * Will auto reinitialize if is closed but will be executed again. + * Will auto reinitialize if is closed. * - *

Maven Import

+ *

Maven Import

* * 
  *     <dependency>
@@ -67,11 +64,10 @@
  * import java.util.Map;
  * 
  * ParallelClient pc= new ParallelClient();
- * pc.prepareGet("/validateInternals.html")
- *     .setMaxConcurrency(1000)
- *     .setTargetHostsFromString("parallec.github.io www.jeffpei.com www.restcommander.com")
+ * pc.prepareHttpGet("/validateInternals.html")
+ *     .setConcurrency(1000)
+ *     .setTargetHostsFromString("www.parallec.io www.jeffpei.com www.restcommander.com")
  *     .execute(new ParallecResponseHandler() {
- *         
  *         public void onCompleted(ResponseOnSingleTask res,
  *                 Map<String, Object> responseContext) {
  *             String cpu = new FilterRegex(".*<td>CPU-Usage-Percent</td>\\s*<td>(.*?)</td>.*")
@@ -83,7 +79,7 @@
  * 
  *
* - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei */ public class ParallelClient { @@ -99,7 +95,7 @@ public class ParallelClient { public HttpClientStore httpClientStore = HttpClientStore.getInstance(); /** The tcp client store. */ - public TcpSshPingResourceStore tcpClientStore = TcpSshPingResourceStore.getInstance(); + public TcpSshPingResourceStore tcpSshPingResourceStore = TcpSshPingResourceStore.getInstance(); /** The is closed is marked when all resources are released/not initialized. */ public static AtomicBoolean isClosed = new AtomicBoolean(true); @@ -119,7 +115,7 @@ public void initialize() { logger.info("Initialing Parallel Client Resources: actor system, HttpClientStore, Task Manager ...."); ActorConfig.createAndGetActorSystem(); httpClientStore.init(); - tcpClientStore.init(); + tcpSshPingResourceStore.init(); ParallelTaskManager.getInstance(); isClosed.set(false); logger.info("Parallel Client Resources has been initialized."); @@ -131,7 +127,7 @@ public void initialize() { /** * Releases the external resources that this object depends on. You should * not call this method if you still want to use the external resources - * (e.g. akka system, sync and async http client store, thread pool for + * (e.g. akka system, async http client store, thread pool for * SSH/TCP) are in use by other objects. * * @@ -144,7 +140,7 @@ public void releaseExternalResources() { logger.info("Releasing all ParallelClient resources... "); ActorConfig.shutDownActorSystemForce(); httpClientStore.shutdown(); - tcpClientStore.shutdown(); + tcpSshPingResourceStore.shutdown(); taskManager.cleanWaitTaskQueue(); taskManager.cleanInprogressJobMap(); isClosed.set(true); @@ -168,7 +164,7 @@ public void reinitIfClosed() { ActorConfig.createAndGetActorSystem(); httpClientStore.reinit(); - tcpClientStore.reinit(); + tcpSshPingResourceStore.reinit(); try { Thread.sleep(1000l); } catch (InterruptedException e) { diff --git a/src/main/java/io/parallec/core/ParallelTask.java b/src/main/java/io/parallec/core/ParallelTask.java index 8c3ebcf..f19c0e2 100644 --- a/src/main/java/io/parallec/core/ParallelTask.java +++ b/src/main/java/io/parallec/core/ParallelTask.java @@ -58,7 +58,7 @@ * The key class represents a onetime execution on multiple requests. It contains all the task * and request metadata, target hosts, configs, and the responses. * - * A ParallelTask is the returned object from the {@link ParallelTaskBuilder#execute()} + * A ParallelTask is the returned object from the {@link ParallelTaskBuilder#execute} * *
  • The metadata on this whole task, including config, running state, progress, request count, task id. 
    • @@ -70,7 +70,7 @@
* * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei */ public class ParallelTask { @@ -120,7 +120,12 @@ public class ParallelTask { /** The aggregate result map. */ private final Map> aggregateResultMap = new LinkedHashMap>(); - /** The parallel task result. */ + /** The parallel task result: + * a hashmap to store the request parameters, host name, ResponseOnSingleTask. + * Note that by default, the response content is not saved into the ResponseOnSingleTask. + * Unless the user changes the config by calling {@link ParallelTaskBuilder#setSaveResponseToTask} + * + * */ private Map parallelTaskResult = new ConcurrentHashMap(); /** The http meta. */ @@ -189,7 +194,6 @@ public ParallelTask() { * @param pingMeta the ping meta * @param handler the handler * @param responseContext the response context - * @param asyncHttpClient the async http client * @param replacementVarMapNodeSpecific the replacement var map node specific * @param replacementVarMap the replacement var map * @param requestReplacementType the request replacement type @@ -605,6 +609,9 @@ public List getTaskErrorMetas() { + /* (non-Javadoc) + * @see java.lang.Object#toString() + */ @Override public String toString() { return "ParallelTask [config=" + config + ", submitTime=" + submitTime @@ -822,6 +829,11 @@ public Map getAggregateResultFullSummary() { return summaryMap; } + /** + * Gets the aggregated result human str. + * + * @return the aggregated result human str + */ public String getAggregatedResultHumanStr() { return PcStringUtils.getAggregatedResultHuman(aggregateResultMap); } diff --git a/src/main/java/io/parallec/core/ParallelTaskBuilder.java b/src/main/java/io/parallec/core/ParallelTaskBuilder.java index ec4d03e..5bfed72 100644 --- a/src/main/java/io/parallec/core/ParallelTaskBuilder.java +++ b/src/main/java/io/parallec/core/ParallelTaskBuilder.java @@ -48,6 +48,7 @@ import com.google.common.base.Strings; import com.ning.http.client.AsyncHttpClient; + /** * * This store all the metadata to build the ParallelTask @@ -56,7 +57,7 @@ * validation, certain missing parameters will use the default values. execute() * is the key function * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei */ public class ParallelTaskBuilder { @@ -240,7 +241,9 @@ public ParallelTask execute(ParallecResponseHandler handler) { if (task.getConfig().isEnableCapacityAwareTaskScheduler()) { - // add to the task + //late initialize the task scheduler + ParallelTaskManager.getInstance().initTaskSchedulerIfNot(); + // add task to the wait queue ParallelTaskManager.getInstance().getWaitQ().add(task); logger.info("Enabled CapacityAwareTaskScheduler. Submitted task to waitQ in builder.. " @@ -267,7 +270,6 @@ public void run() { task.getTaskId()); while (task != null && !task.isCompleted()) { - try { Thread.sleep(500L); } catch (InterruptedException e) { @@ -589,7 +591,7 @@ public void setTargetHostBuilder(ITargetHostsBuilder thb) { } /** - * Gets the mode of either sync or async + * Gets the mode of either sync or async. * * @return the mode */ @@ -929,11 +931,12 @@ public ParallelTaskBuilder setSshPrivKeyRelativePath( return this; } + /** - * Sets the ssh priv key need passphrase and set value. + * Sets the ssh priv key relative path wtih passphrase. * - * @param userName - * the user name + * @param privKeyRelativePath the priv key relative path + * @param passphrase the passphrase * @return the parallel task builder */ public ParallelTaskBuilder setSshPrivKeyRelativePathWtihPassphrase( @@ -1009,7 +1012,6 @@ public ParallelTaskBuilder setTcpChannelFactory( return this; } - /** * Sets the config. * @@ -1060,6 +1062,11 @@ public ParallelTaskBuilder setEnableCapacityAwareTaskScheduler( /** * Sets the auto save log to local. + * Will auto save logs to the local file system. + * + * The logs by default are written to path "userdata/task/logs" folder. + * + * Note that it is user's responsibility to clearn these logs. * * OPTIONAL. DEFAULT: false. * @param autoSaveLogToLocal @@ -1111,7 +1118,8 @@ public int getConcurrency() { /** - * Sets the ping mode. + * Sets the ping mode. Process or INET_ADDRESS_REACHABLE based. + * Default as InetAddress mode. InetAddress requires Root privilege. * * @param mode the mode * @return the parallel task builder diff --git a/src/main/java/io/parallec/core/RequestProtocol.java b/src/main/java/io/parallec/core/RequestProtocol.java index 8dcfdef..78ac019 100644 --- a/src/main/java/io/parallec/core/RequestProtocol.java +++ b/src/main/java/io/parallec/core/RequestProtocol.java @@ -18,7 +18,7 @@ * * Currently values include HTTP, HTTPS, SSH, PING, TCP * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei */ public enum RequestProtocol { diff --git a/src/main/java/io/parallec/core/ResponseOnSingleTask.java b/src/main/java/io/parallec/core/ResponseOnSingleTask.java index 09db473..001fa61 100644 --- a/src/main/java/io/parallec/core/ResponseOnSingleTask.java +++ b/src/main/java/io/parallec/core/ResponseOnSingleTask.java @@ -19,14 +19,17 @@ import java.util.LinkedHashMap; import java.util.Map; - /** - * this includes the request - * each target host will have one. + * this includes the request each target host will have one. + * + * {@link ResponseOnSingleTask#isError()} is true: means fail to receive + * response. + * + * Note that by default, the response content is not saved into the + * singleTaskResponse. Unless the user change the config by calling + * {@link ParallelTaskBuilder#setSaveResponseToTask} * - * {@link ResponseOnSingleTask#isError()} is true: means fail to receive resposne - * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei * */ public class ResponseOnSingleTask { diff --git a/src/main/java/io/parallec/core/actor/AssistantExecutionManager.java b/src/main/java/io/parallec/core/actor/AssistantExecutionManager.java index f8ccfa5..08e5e4d 100644 --- a/src/main/java/io/parallec/core/actor/AssistantExecutionManager.java +++ b/src/main/java/io/parallec/core/actor/AssistantExecutionManager.java @@ -33,16 +33,9 @@ /** * - * 20131014; update major logic to enable concurrency control + * The assistant manager is for sending out requests in batch; * - * 20130730 The assistant manager is soley for sending out requests in batch; - * with interval based sleep 1. Enables IN PARALLEL sending batch requests and - * geting response 2. Enables > 5K nodes per requests. The manager gives all - * request sending task to this AssistantExecutionManager. this guy - * tells workers to directly reply back to the original manager. So that - * requests and replies can happen the same time. - * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei * */ public class AssistantExecutionManager extends UntypedActor { diff --git a/src/main/java/io/parallec/core/actor/ExecutionManager.java b/src/main/java/io/parallec/core/actor/ExecutionManager.java index d32302d..654276b 100644 --- a/src/main/java/io/parallec/core/actor/ExecutionManager.java +++ b/src/main/java/io/parallec/core/actor/ExecutionManager.java @@ -73,7 +73,7 @@ * * Will create a list of operation workers to match each of the target hosts. * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei * */ public class ExecutionManager extends UntypedActor { diff --git a/src/main/java/io/parallec/core/actor/HttpWorker.java b/src/main/java/io/parallec/core/actor/HttpWorker.java index 969dea7..6560ca9 100644 --- a/src/main/java/io/parallec/core/actor/HttpWorker.java +++ b/src/main/java/io/parallec/core/actor/HttpWorker.java @@ -46,13 +46,15 @@ import com.ning.http.client.Response; + /** * This is an akka actor with async http client. * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei */ public class HttpWorker extends UntypedActor { + /** The actor max operation timeout sec. */ private int actorMaxOperationTimeoutSec; /** The client. */ @@ -95,22 +97,17 @@ public class HttpWorker extends UntypedActor { private boolean sentReply = false; /** The response future. */ - // 20150221 ListenableFuture responseFuture = null; - + /** * Instantiates a new http worker. * - * @param client - * the client - * @param requestUrl - * the request url - * @param httpMethod - * the http method - * @param postData - * the post data - * @param httpHeaderMap - * the http header map + * @param actorMaxOperationTimeoutSec the actor max operation timeout sec + * @param client the client + * @param requestUrl the request url + * @param httpMethod the http method + * @param postData the post data + * @param httpHeaderMap the http header map */ public HttpWorker(final int actorMaxOperationTimeoutSec, final AsyncHttpClient client, final String requestUrl, @@ -289,6 +286,7 @@ public void onReceive(Object message) throws Exception { } /** + * Cancel cancellable. */ public void cancelCancellable() { diff --git a/src/main/java/io/parallec/core/actor/OperationWorker.java b/src/main/java/io/parallec/core/actor/OperationWorker.java index 284aab9..d00e029 100644 --- a/src/main/java/io/parallec/core/actor/OperationWorker.java +++ b/src/main/java/io/parallec/core/actor/OperationWorker.java @@ -44,10 +44,11 @@ import com.ning.http.client.AsyncHttpClient; + /** * AHC based. * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei */ public class OperationWorker extends UntypedActor { @@ -83,6 +84,7 @@ public class OperationWorker extends UntypedActor { /** The timeout duration. */ private FiniteDuration timeoutDuration = null; + /** The actor max operation timeout sec. */ private int actorMaxOperationTimeoutSec = ParallelTaskConfigDefault.actorMaxOperationTimeoutSec; /** The start time millis. */ @@ -434,6 +436,9 @@ private final void operationTimeout() { } + /** + * Cancel. + */ @SuppressWarnings("deprecation") private final void cancel() { /** @@ -475,14 +480,11 @@ private final void cancelCancellable() { /** * Reply. * - * @param error - * the error - * @param errorMessage - * the error message - * @param stackTrace - * the stack trace - * @param statusCode - * the status code + * @param error the error + * @param errorMessage the error message + * @param stackTrace the stack trace + * @param statusCode the status code + * @param statusCodeInt the status code int */ private final void reply(final boolean error, final String errorMessage, final String stackTrace, final String statusCode, @@ -495,16 +497,12 @@ private final void reply(final boolean error, final String errorMessage, /** * Reply. * - * @param error - * the error - * @param errorMessage - * the error message - * @param stackTrace - * the stack trace - * @param statusCode - * the status code - * @param receiveTime - * the receive time + * @param error the error + * @param errorMessage the error message + * @param stackTrace the stack trace + * @param statusCode the status code + * @param statusCodeInt the status code int + * @param receiveTime the receive time */ @SuppressWarnings("deprecation") private final void reply(final boolean error, final String errorMessage, diff --git a/src/main/java/io/parallec/core/actor/PingWorker.java b/src/main/java/io/parallec/core/actor/PingWorker.java index 65e7a50..c91a3dd 100644 --- a/src/main/java/io/parallec/core/actor/PingWorker.java +++ b/src/main/java/io/parallec/core/actor/PingWorker.java @@ -37,13 +37,14 @@ import akka.actor.Cancellable; import akka.actor.UntypedActor; + /** * * Using callable to start a new thread to run the Ping execution task (which * make take minutes). Check if future is read every 0.1 seconds. Make sure the * message are handled async. * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei */ public class PingWorker extends UntypedActor { @@ -51,6 +52,7 @@ public class PingWorker extends UntypedActor { /** The logger. */ private static Logger logger = LoggerFactory.getLogger(PingWorker.class); + /** The actor max operation timeout sec. */ private int actorMaxOperationTimeoutSec; /** The sender. */ @@ -81,10 +83,11 @@ public class PingWorker extends UntypedActor { private Future responseFuture; /** - * Instantiates a new ssh worker. + * Instantiates a new ping worker. * - * @param targetHost - * the target host + * @param actorMaxOperationTimeoutSec the actor max operation timeout sec + * @param pingMeta the ping meta + * @param targetHost the target host */ public PingWorker(int actorMaxOperationTimeoutSec,PingMeta pingMeta, String targetHost) { @@ -365,14 +368,23 @@ public static void setLogger(Logger logger) { /** * thread to return the future. * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei */ private static class PingTask implements Callable { /** The target host. */ private String targetHost; + + /** The ping meta. */ private PingMeta pingMeta; + + /** + * Instantiates a new ping task. + * + * @param targetHost the target host + * @param pingMeta the ping meta + */ public PingTask( String targetHost, PingMeta pingMeta) { this.targetHost = targetHost; this.pingMeta = pingMeta; diff --git a/src/main/java/io/parallec/core/actor/SshWorker.java b/src/main/java/io/parallec/core/actor/SshWorker.java index 7ab7780..f76e558 100644 --- a/src/main/java/io/parallec/core/actor/SshWorker.java +++ b/src/main/java/io/parallec/core/actor/SshWorker.java @@ -39,6 +39,7 @@ import akka.actor.UntypedActor; + /** * * Using callable to start a new thread to run the SSH execution task (which @@ -47,13 +48,14 @@ * subsequent messages. * * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei */ public class SshWorker extends UntypedActor { /** The logger. */ private static Logger logger = LoggerFactory.getLogger(SshWorker.class); + /** The actor max operation timeout sec. */ private int actorMaxOperationTimeoutSec; /** The sender. */ @@ -86,10 +88,9 @@ public class SshWorker extends UntypedActor { /** * Instantiates a new ssh worker. * - * @param sshMeta - * the ssh meta - * @param targetHost - * the target host + * @param actorMaxOperationTimeoutSec the actor max operation timeout sec + * @param sshMeta the ssh meta + * @param targetHost the target host */ public SshWorker(int actorMaxOperationTimeoutSec, SshMeta sshMeta, String targetHost) { @@ -373,7 +374,7 @@ public static void setLogger(Logger logger) { /** * thread to return the future. * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei */ private static class SshTask implements Callable { diff --git a/src/main/java/io/parallec/core/actor/TcpWorker.java b/src/main/java/io/parallec/core/actor/TcpWorker.java index 476b4a3..381054a 100644 --- a/src/main/java/io/parallec/core/actor/TcpWorker.java +++ b/src/main/java/io/parallec/core/actor/TcpWorker.java @@ -59,7 +59,7 @@ /** * A worker for single TCP request class * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei */ public class TcpWorker extends UntypedActor { @@ -395,7 +395,7 @@ public void setResponseSb(StringBuilder responseSb) { /** * define the list of handlers for this channel. * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei */ public static class MyPipelineFactory implements ChannelPipelineFactory{ @@ -446,7 +446,7 @@ public ChannelPipeline getPipeline() { /** * how to pass the idle event back to the worker. * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei */ public static class MyIdleHandler extends IdleStateAwareChannelHandler { @@ -487,7 +487,7 @@ public void channelIdle(ChannelHandlerContext ctx, IdleStateEvent e) { /** * The Netty response/channel handler. * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei */ public static class TcpChannelHandler extends SimpleChannelHandler { diff --git a/src/main/java/io/parallec/core/actor/message/NodeReqResponse.java b/src/main/java/io/parallec/core/actor/message/NodeReqResponse.java index 7db778d..781caf2 100644 --- a/src/main/java/io/parallec/core/actor/message/NodeReqResponse.java +++ b/src/main/java/io/parallec/core/actor/message/NodeReqResponse.java @@ -24,11 +24,9 @@ /** - * To save the content request and response By default the responseContent is - * NULL because dont need it in InProgress entries. But must init!! when in - * valid entries. + * To save the request parameter and the task response for this target host. * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei * */ public class NodeReqResponse { diff --git a/src/main/java/io/parallec/core/actor/message/ResponseCountToBatchSenderAsstManager.java b/src/main/java/io/parallec/core/actor/message/ResponseCountToBatchSenderAsstManager.java index 42ec2fa..a1d93c9 100644 --- a/src/main/java/io/parallec/core/actor/message/ResponseCountToBatchSenderAsstManager.java +++ b/src/main/java/io/parallec/core/actor/message/ResponseCountToBatchSenderAsstManager.java @@ -16,7 +16,7 @@ /** * Receive count from the manager to asst manager; for flow control. * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei */ public class ResponseCountToBatchSenderAsstManager { diff --git a/src/main/java/io/parallec/core/actor/message/ResponseOnSingeRequest.java b/src/main/java/io/parallec/core/actor/message/ResponseOnSingeRequest.java index 0b44cd7..d8bf9d3 100644 --- a/src/main/java/io/parallec/core/actor/message/ResponseOnSingeRequest.java +++ b/src/main/java/io/parallec/core/actor/message/ResponseOnSingeRequest.java @@ -17,15 +17,16 @@ /** - * a single HTTP response for each http request + * A single HTTP response for each http request * * Note that the failObtainResponse in the response is whether work successfully * received. it is not about 200 code. * * The Single Response will be in error when the HTTP Worker fail to obtain the - * response back on condition of CANCEL PROCESS_ON_EXCEPTION PROCESS_ON_TIMEOUT + * response back on condition of + * [CANCEL or PROCESS_ON_EXCEPTION or PROCESS_ON_TIMEOUT] * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei * */ public class ResponseOnSingeRequest { diff --git a/src/main/java/io/parallec/core/actor/poll/HttpPollerProcessor.java b/src/main/java/io/parallec/core/actor/poll/HttpPollerProcessor.java index 6145d02..ad8da94 100644 --- a/src/main/java/io/parallec/core/actor/poll/HttpPollerProcessor.java +++ b/src/main/java/io/parallec/core/actor/poll/HttpPollerProcessor.java @@ -31,7 +31,7 @@ * Poll API: assume the HTTP Method for poll: is GET / no body/ and with the * same header as the 1st reqwuest * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei * */ public class HttpPollerProcessor { diff --git a/src/main/java/io/parallec/core/actor/poll/PollerData.java b/src/main/java/io/parallec/core/actor/poll/PollerData.java index d0bb443..1eafa08 100644 --- a/src/main/java/io/parallec/core/actor/poll/PollerData.java +++ b/src/main/java/io/parallec/core/actor/poll/PollerData.java @@ -21,7 +21,7 @@ /** * the currentProgress is just for display. * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei * */ public class PollerData { diff --git a/src/main/java/io/parallec/core/app/ParallecAppMin.java b/src/main/java/io/parallec/core/app/ParallecAppMin.java index 3c2db43..5a5fa99 100644 --- a/src/main/java/io/parallec/core/app/ParallecAppMin.java +++ b/src/main/java/io/parallec/core/app/ParallecAppMin.java @@ -35,7 +35,7 @@ public static void main(String[] args) { pc.prepareHttpGet("/validateInternals.html") .setConcurrency(1000) .setTargetHostsFromString( - "parallec.github.io www.jeffpei.com www.restcommander.com") + "www.parallec.io www.jeffpei.com www.restcommander.com") .execute(new ParallecResponseHandler() { @Override diff --git a/src/main/java/io/parallec/core/app/package-info.java b/src/main/java/io/parallec/core/app/package-info.java index 3da64c6..527f4ff 100644 --- a/src/main/java/io/parallec/core/app/package-info.java +++ b/src/main/java/io/parallec/core/app/package-info.java @@ -1,6 +1,6 @@ /** * The package for storing sample applications using Parallec. - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei * */ package io.parallec.core.app; \ No newline at end of file diff --git a/src/main/java/io/parallec/core/bean/HttpMeta.java b/src/main/java/io/parallec/core/bean/HttpMeta.java index 8864d3b..676c271 100644 --- a/src/main/java/io/parallec/core/bean/HttpMeta.java +++ b/src/main/java/io/parallec/core/bean/HttpMeta.java @@ -28,7 +28,7 @@ * The metadata about the HTTP request (url/port/header/concurrency etc) It does * not include the target hosts information. * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei */ public class HttpMeta { /** The logger. */ diff --git a/src/main/java/io/parallec/core/bean/SetAndCount.java b/src/main/java/io/parallec/core/bean/SetAndCount.java index 849a7c9..9a5292e 100644 --- a/src/main/java/io/parallec/core/bean/SetAndCount.java +++ b/src/main/java/io/parallec/core/bean/SetAndCount.java @@ -18,7 +18,7 @@ /** * The class used to represent the target hosts and a count when aggregation * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei */ public class SetAndCount { diff --git a/src/main/java/io/parallec/core/bean/SingleTargetTaskStatus.java b/src/main/java/io/parallec/core/bean/SingleTargetTaskStatus.java index e3c3cba..7c5efc8 100644 --- a/src/main/java/io/parallec/core/bean/SingleTargetTaskStatus.java +++ b/src/main/java/io/parallec/core/bean/SingleTargetTaskStatus.java @@ -15,7 +15,7 @@ /** * The Enum SingleTargetTaskStatus. * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei */ public enum SingleTargetTaskStatus { diff --git a/src/main/java/io/parallec/core/bean/StrStrMap.java b/src/main/java/io/parallec/core/bean/StrStrMap.java index a076080..c2d338c 100644 --- a/src/main/java/io/parallec/core/bean/StrStrMap.java +++ b/src/main/java/io/parallec/core/bean/StrStrMap.java @@ -23,7 +23,7 @@ * variable name : e.g. GET /checkProgress/$JOBID value: the actual value string * that will replace the JOBID * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei */ public class StrStrMap { diff --git a/src/main/java/io/parallec/core/bean/TargetHostMeta.java b/src/main/java/io/parallec/core/bean/TargetHostMeta.java index 2b38e2c..097fd41 100644 --- a/src/main/java/io/parallec/core/bean/TargetHostMeta.java +++ b/src/main/java/io/parallec/core/bean/TargetHostMeta.java @@ -22,7 +22,7 @@ * The metadata about the targetHost, which is mainly a list of hostnames (IP or FQDN) * the primary key is the targetHostId, generated with timestamp * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei */ public class TargetHostMeta { diff --git a/src/main/java/io/parallec/core/bean/TaskRequest.java b/src/main/java/io/parallec/core/bean/TaskRequest.java index a3bc64c..d533ef1 100644 --- a/src/main/java/io/parallec/core/bean/TaskRequest.java +++ b/src/main/java/io/parallec/core/bean/TaskRequest.java @@ -29,7 +29,7 @@ * This is the request send to the operation worker. * It contains the actual request that has been replaced if there are variables defined. * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei */ public class TaskRequest { diff --git a/src/main/java/io/parallec/core/bean/ping/PingMeta.java b/src/main/java/io/parallec/core/bean/ping/PingMeta.java index a91a490..251c5e0 100644 --- a/src/main/java/io/parallec/core/bean/ping/PingMeta.java +++ b/src/main/java/io/parallec/core/bean/ping/PingMeta.java @@ -22,7 +22,7 @@ /** * The Ping metadata on mode,timeout and retries. * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei * */ public class PingMeta { diff --git a/src/main/java/io/parallec/core/bean/ping/PingMode.java b/src/main/java/io/parallec/core/bean/ping/PingMode.java index 42fd755..52dd241 100644 --- a/src/main/java/io/parallec/core/bean/ping/PingMode.java +++ b/src/main/java/io/parallec/core/bean/ping/PingMode.java @@ -15,7 +15,7 @@ /** * The Enum of Ping Mode. Process or INET_ADDRESS_REACHABLE based. * Default as InetAddress mode. InetAddress requires Root privilege. - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei */ public enum PingMode { diff --git a/src/main/java/io/parallec/core/bean/ssh/SshMeta.java b/src/main/java/io/parallec/core/bean/ssh/SshMeta.java index 768e5dc..6c5075e 100644 --- a/src/main/java/io/parallec/core/bean/ssh/SshMeta.java +++ b/src/main/java/io/parallec/core/bean/ssh/SshMeta.java @@ -16,11 +16,12 @@ import io.parallec.core.exception.ParallelTaskInvalidException; + /** * all ssh metadata except for the target host name. also those timeout configs * and * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei * */ public class SshMeta { @@ -92,25 +93,19 @@ public void setPassphrase(String passphrase) { this.passphrase = passphrase; } + /** * Instantiates a new ssh meta. * - * @param commandLine - * the command line - * @param userName - * the user name - * @param sshPort - * the ssh port - * @param sshLoginType - * the ssh login type - * @param privKeyRelativePath - * the priv key relative path - * @param password - * the password - * @param privKeyUsePassphrase - * the priv key use passphrase - * @param passphrase - * the passphrase + * @param commandLine the command line + * @param userName the user name + * @param sshPort the ssh port + * @param sshLoginType the ssh login type + * @param privKeyRelativePath the priv key relative path + * @param password the password + * @param privKeyUsePassphrase the priv key use passphrase + * @param passphrase the passphrase + * @param sshConnectionTimeoutMillis the ssh connection timeout millis */ public SshMeta(String commandLine, String userName, int sshPort, SshLoginType sshLoginType, String privKeyRelativePath, @@ -316,10 +311,20 @@ public String getPrivKeyAbsPath() { return privKeyAbsPath; } + /** + * Gets the ssh connection timeout millis. + * + * @return the ssh connection timeout millis + */ public int getSshConnectionTimeoutMillis() { return sshConnectionTimeoutMillis; } + /** + * Sets the ssh connection timeout millis. + * + * @param sshConnectionTimeoutMillis the new ssh connection timeout millis + */ public void setSshConnectionTimeoutMillis(int sshConnectionTimeoutMillis) { this.sshConnectionTimeoutMillis = sshConnectionTimeoutMillis; } diff --git a/src/main/java/io/parallec/core/bean/tcp/TcpMeta.java b/src/main/java/io/parallec/core/bean/tcp/TcpMeta.java index d9d9c88..6d3dcba 100644 --- a/src/main/java/io/parallec/core/bean/tcp/TcpMeta.java +++ b/src/main/java/io/parallec/core/bean/tcp/TcpMeta.java @@ -30,7 +30,7 @@ * all ssh metadata except for the target host name. also those timeout configs * and * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei * */ public class TcpMeta { @@ -92,7 +92,7 @@ public boolean validation() throws ParallelTaskInvalidException { if (this.tcpConnectTimeoutMillis == null) { logger.info("SET DEFAULT TCP CONNECT TIMEOUT: TCP tcpConnectTimeoutMillis is set as default"); - this.tcpConnectTimeoutMillis= ParallecGlobalConfig.tcpConnectionTimeoutMillisDefault; + this.tcpConnectTimeoutMillis= ParallecGlobalConfig.tcpConnectTimeoutMillisDefault; } if (this.tcpIdleTimeoutSec == null) { diff --git a/src/main/java/io/parallec/core/commander/workflow/InternalDataProvider.java b/src/main/java/io/parallec/core/commander/workflow/InternalDataProvider.java index 9b3c376..ecf6a6f 100644 --- a/src/main/java/io/parallec/core/commander/workflow/InternalDataProvider.java +++ b/src/main/java/io/parallec/core/commander/workflow/InternalDataProvider.java @@ -28,7 +28,7 @@ /** * generate the node data; make sure it is not null. * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei * */ public class InternalDataProvider { @@ -56,14 +56,14 @@ private InternalDataProvider() { } /** - * Gen node data map. + * Generate node data map. * * @param task * the job info */ public void genNodeDataMap(ParallelTask task) { - TargetHostMeta nodeGroupSourceMetadata = task.getTargetHostMeta(); + TargetHostMeta targetHostMeta = task.getTargetHostMeta(); HttpMeta httpMeta = task.getHttpMeta(); @@ -72,11 +72,10 @@ public void genNodeDataMap(ParallelTask task) { String requestContent = HttpMeta .replaceDefaultFullRequestContent(entityBody); - // 2015 new Map parallelTaskResult = task .getParallelTaskResult(); try { - for (String fqdn : nodeGroupSourceMetadata.getHosts()) { + for (String fqdn : targetHostMeta.getHosts()) { NodeReqResponse nodeReqResponse = new NodeReqResponse(fqdn); nodeReqResponse.setDefaultReqestContent(requestContent); parallelTaskResult.put(fqdn, nodeReqResponse); @@ -90,18 +89,10 @@ public void genNodeDataMap(ParallelTask task) { }// end func. /** - * Check if it will execute VarUtils.NODE_REQUEST_WILL_EXECUTE each target - * host is unique as in the key of the hash map. - * - * So there will not be duplicated target hosts. - * - * - * @param nodeDataMapValidSource - * the node data map valid source - * @param nodeDataMapValidSafe - * the node data map valid safe - * @param commandType - * the command type + * Filter unsafe or unnecessary request. + * + * @param nodeDataMapValidSource the node data map valid source + * @param nodeDataMapValidSafe the node data map valid safe */ public void filterUnsafeOrUnnecessaryRequest( Map nodeDataMapValidSource, @@ -114,12 +105,6 @@ public void filterUnsafeOrUnnecessaryRequest( String hostName = entry.getKey(); NodeReqResponse nrr = entry.getValue(); - if (nrr == null) { - logger.error("NodeReqResponse is NULL " - + "in filterUnsafeRequest for host {}", hostName); - continue; - } - // now get the content. Map map = nrr.getRequestParameters(); /** @@ -127,7 +112,6 @@ public void filterUnsafeOrUnnecessaryRequest( * field and this field is false */ if (map.containsKey(PcConstants.NODE_REQUEST_WILL_EXECUTE)) { - Boolean willExecute = Boolean.parseBoolean(map .get(PcConstants.NODE_REQUEST_WILL_EXECUTE)); @@ -140,9 +124,6 @@ public void filterUnsafeOrUnnecessaryRequest( } // now safely to add this node in. - - // note that this is shallow copy; put the pointer of the source - // in... nodeDataMapValidSafe.put(hostName, nrr); }// end for loop diff --git a/src/main/java/io/parallec/core/commander/workflow/VarReplacementProvider.java b/src/main/java/io/parallec/core/commander/workflow/VarReplacementProvider.java index fac6722..cabbf22 100644 --- a/src/main/java/io/parallec/core/commander/workflow/VarReplacementProvider.java +++ b/src/main/java/io/parallec/core/commander/workflow/VarReplacementProvider.java @@ -14,9 +14,7 @@ import io.parallec.core.ParallelTask; import io.parallec.core.actor.message.NodeReqResponse; -import io.parallec.core.bean.HttpMeta; import io.parallec.core.bean.StrStrMap; -import io.parallec.core.bean.TargetHostMeta; import io.parallec.core.util.PcConstants; import io.parallec.core.util.PcDateUtils; @@ -31,7 +29,7 @@ * * extending from replacing only WISB based. now to also API based. * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei * */ public class VarReplacementProvider { @@ -78,7 +76,6 @@ private VarReplacementProvider() { * boolean: use target host specific replacement map * @param replacementVarMapNodeSpecific * the replacement variable map node specific - * @return the string */ public void updateRequestWithReplacement( ParallelTask task, boolean useReplacementVarMap, @@ -88,15 +85,6 @@ public void updateRequestWithReplacement( try { - TargetHostMeta targetHostMeta = task.getTargetHostMeta(); - HttpMeta cmd = task.getHttpMeta(); - if (targetHostMeta == null || cmd == null) { - - String errMessage = "nodeGroupSourceMetadata is NULL or agentCommandType is NULL in updateRequestContentGenericHelper. EXIT!!" - + PcDateUtils.getNowDateTimeStrStandard(); - logger.error(errMessage); - } - /** * * ENABLE CRETIRIA: useReplacementVarMap==true diff --git a/src/main/java/io/parallec/core/commander/workflow/ssh/SshProvider.java b/src/main/java/io/parallec/core/commander/workflow/ssh/SshProvider.java index 8043299..b855c56 100644 --- a/src/main/java/io/parallec/core/commander/workflow/ssh/SshProvider.java +++ b/src/main/java/io/parallec/core/commander/workflow/ssh/SshProvider.java @@ -38,7 +38,7 @@ * a basic one that works with password. If needed we may replace this * SshProvider after defining an interface. * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei * */ public class SshProvider { diff --git a/src/main/java/io/parallec/core/config/ParallecGlobalConfig.java b/src/main/java/io/parallec/core/config/ParallecGlobalConfig.java index 0e52714..d1e5d16 100644 --- a/src/main/java/io/parallec/core/config/ParallecGlobalConfig.java +++ b/src/main/java/io/parallec/core/config/ParallecGlobalConfig.java @@ -19,7 +19,7 @@ * Different from those settings defined in ParallelTaskConfig, * settings here are effective to all executions, and cannot be overwritten for a particular task. * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei */ public class ParallecGlobalConfig { @@ -58,10 +58,9 @@ public class ParallecGlobalConfig { /** The ssh buffer size. */ public static int sshBufferSize = 1024; - /** The tcp connection timeout millis default. */ - public static int tcpConnectionTimeoutMillisDefault = 2000; + public static int tcpConnectTimeoutMillisDefault = 2000; /** The tcp idle timeout seconds default. */ public static int tcpIdleTimeoutSecDefault = 5; @@ -92,7 +91,7 @@ public class ParallecGlobalConfig { public static double pingFutureCheckIntervalSec = 0.1; /** The log response interval. */ - public static int logResponseInterval = 1; + public static int logResponseInterval = 5; /** The log all response after percent. */ public static double logAllResponseAfterPercent = 95.0; diff --git a/src/main/java/io/parallec/core/config/ParallelTaskConfig.java b/src/main/java/io/parallec/core/config/ParallelTaskConfig.java index 86b7ea2..36bc5ad 100644 --- a/src/main/java/io/parallec/core/config/ParallelTaskConfig.java +++ b/src/main/java/io/parallec/core/config/ParallelTaskConfig.java @@ -13,14 +13,18 @@ package io.parallec.core.config; import io.parallec.core.ParallelTask; +import io.parallec.core.ParallelTaskBuilder; /** * Configuration class to use with a {@link ParallelTask}. * * This can be overwritten for each task. * - * Configs about various timeout, whether to auto save responses, - * whether to enable the s + * Configs about various timeout, whether to auto save logs / save responses. + * whether to enable the capacity scheduler. + * + * Some of the most useful ones we directly have APIs by the {@link ParallelTaskBuilder} + * * will load from the default value from {@link ParallelTaskConfigDefault} * */ diff --git a/src/main/java/io/parallec/core/exception/ActorMessageTypeInvalidException.java b/src/main/java/io/parallec/core/exception/ActorMessageTypeInvalidException.java index 32fc28f..1e38822 100644 --- a/src/main/java/io/parallec/core/exception/ActorMessageTypeInvalidException.java +++ b/src/main/java/io/parallec/core/exception/ActorMessageTypeInvalidException.java @@ -15,7 +15,7 @@ /** * The Class ActorMessageTypeInvalidException. * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei */ public class ActorMessageTypeInvalidException extends RuntimeException { diff --git a/src/main/java/io/parallec/core/monitor/MonitorProvider.java b/src/main/java/io/parallec/core/monitor/MonitorProvider.java index ce874e2..d5f8ad3 100644 --- a/src/main/java/io/parallec/core/monitor/MonitorProvider.java +++ b/src/main/java/io/parallec/core/monitor/MonitorProvider.java @@ -28,6 +28,7 @@ import com.google.gson.GsonBuilder; + /** * The Class MonitorProvider. */ @@ -93,6 +94,11 @@ public ThreadInfo[] getThreadDump() { return threadMxBean.dumpAllThreads(true, true); } + /** + * Gets the live thread count. + * + * @return the live thread count + */ public int getLiveThreadCount() { return ManagementFactory.getThreadMXBean().getThreadCount(); } @@ -123,7 +129,9 @@ public ThreadUsage getThreadUsage() { } /** - * Log health. + * Gets the health memory. + * + * @return the health memory */ public String getHealthMemory() { StringBuilder sb = new StringBuilder(); diff --git a/src/main/java/io/parallec/core/resources/AsyncHttpClientFactoryEmbed.java b/src/main/java/io/parallec/core/resources/AsyncHttpClientFactoryEmbed.java index 293ebd8..b9ac0cc 100644 --- a/src/main/java/io/parallec/core/resources/AsyncHttpClientFactoryEmbed.java +++ b/src/main/java/io/parallec/core/resources/AsyncHttpClientFactoryEmbed.java @@ -42,7 +42,7 @@ /** * The Class HttpClientFactoryEmbed. * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei */ public final class AsyncHttpClientFactoryEmbed { diff --git a/src/main/java/io/parallec/core/resources/HttpClientStore.java b/src/main/java/io/parallec/core/resources/HttpClientStore.java index 1ea1af6..0d6349f 100644 --- a/src/main/java/io/parallec/core/resources/HttpClientStore.java +++ b/src/main/java/io/parallec/core/resources/HttpClientStore.java @@ -20,15 +20,18 @@ /** - * this stores a pair of default fast/slow AsyncHttpClient, and another pair of - * customized fast/slow AsyncHttpClient the default. + * this stores a pair of default embedded fast/slow AsyncHttpClient, and another pair of + * customized fast/slow AsyncHttpClient. + * By default the pair of customized ones are just references (duplicates of) the embedded ones. * * For each Parallel Task, will by default load the Embed-fast client in AsyncHttpClientStore * unless you load your specific async http client. * - * Apache HTTP Client is only used for CMS query or load target hosts for URLs. + * You may call {@link #setHttpClientTypeCurrentDefault(HttpClientType)} to change the default one * - * @author Yuanteng Jeff Pei + * (default one is the embedded fast) + * + * @author Yuanteng (Jeff) Pei */ public class HttpClientStore { @@ -83,7 +86,7 @@ public synchronized void reinit() { } /** - * Shutdown. + * Shutdown each AHC client in the map. */ public void shutdown() { diff --git a/src/main/java/io/parallec/core/resources/HttpClientType.java b/src/main/java/io/parallec/core/resources/HttpClientType.java index d6a29cd..09a8da6 100644 --- a/src/main/java/io/parallec/core/resources/HttpClientType.java +++ b/src/main/java/io/parallec/core/resources/HttpClientType.java @@ -14,7 +14,7 @@ /** - * The Enum HCTYPE. + * The Enum of HttyClient Type. Default is the EMBED_FAST. */ public enum HttpClientType { diff --git a/src/main/java/io/parallec/core/resources/TcpSshPingResourceStore.java b/src/main/java/io/parallec/core/resources/TcpSshPingResourceStore.java index 64bb268..87c12a2 100644 --- a/src/main/java/io/parallec/core/resources/TcpSshPingResourceStore.java +++ b/src/main/java/io/parallec/core/resources/TcpSshPingResourceStore.java @@ -23,7 +23,7 @@ * Provide external resources needed for netty based TCP worker: * ChannelFactory and HashedWheelTimer timer * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei */ public class TcpSshPingResourceStore { diff --git a/src/main/java/io/parallec/core/task/CapacityAwareTaskScheduler.java b/src/main/java/io/parallec/core/task/CapacityAwareTaskScheduler.java index 5f50a00..0907747 100644 --- a/src/main/java/io/parallec/core/task/CapacityAwareTaskScheduler.java +++ b/src/main/java/io/parallec/core/task/CapacityAwareTaskScheduler.java @@ -20,16 +20,16 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - /** * - * TODO Doc: The Class EvalTaskInWaitQRunner. - * + * A task scheduler runnable to check if there are capacity to run a task + * from the waitQ. * + * Note that this scheduler is not enabled by default. * - * A task scheduler runnable to check if there are capacity to run a task in - * from the waitQ + * Enable it by {@link ParallelTaskBuilder#setEnableCapacityAwareTaskScheduler(true)} * + * @author Yuanteng (Jeff) Pei */ public class CapacityAwareTaskScheduler implements Runnable { diff --git a/src/main/java/io/parallec/core/task/ParallelTaskBean.java b/src/main/java/io/parallec/core/task/ParallelTaskBean.java index c3c7ed1..5a385f8 100644 --- a/src/main/java/io/parallec/core/task/ParallelTaskBean.java +++ b/src/main/java/io/parallec/core/task/ParallelTaskBean.java @@ -35,6 +35,8 @@ /** * The Class ParallelTaskBean. this is for serialization and see as a snapshot + * + * @author Yuanteng (Jeff) Pei */ public class ParallelTaskBean { diff --git a/src/main/java/io/parallec/core/task/ParallelTaskManager.java b/src/main/java/io/parallec/core/task/ParallelTaskManager.java index 51f28e5..d5d23d9 100644 --- a/src/main/java/io/parallec/core/task/ParallelTaskManager.java +++ b/src/main/java/io/parallec/core/task/ParallelTaskManager.java @@ -53,7 +53,7 @@ * * It has access to the waiting task queue and the currently running map of ParallelTasks. * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei * @author Teng Song * */ @@ -95,19 +95,18 @@ private ParallelTaskManager() { synchronized (this) { logger = LoggerFactory.getLogger(ParallelTaskManager.class); } - - initTaskScheduler(); logger.info("Initialized ParallelTaskManager..."); } /** - * as it is daemon thread, save to use. no need to stop either. + * as it is daemon thread + * + * TODO when release external resources should shutdown the scheduler. */ - public void initTaskScheduler() { + public synchronized void initTaskSchedulerIfNot() { if (scheduler == null) { - // TODO scheduler = Executors .newSingleThreadScheduledExecutor(DaemonThreadFactory .getInstance()); @@ -117,13 +116,20 @@ public void initTaskScheduler() { ParallecGlobalConfig.schedulerCheckInterval, TimeUnit.MILLISECONDS); logger.info("initialized daemon task scheduler to evaluate waitQ tasks."); + + } + } + + /** + * Shutdown task scheduler. + */ + public synchronized void shutdownTaskScheduler(){ + if (scheduler != null && !scheduler.isShutdown()) { + scheduler.shutdown(); + logger.info("shutdowned the task scheduler. No longer accepting new tasks"); + scheduler = null; } } - - - - - // waiting queue /** * Gets the task from in progress map. @@ -238,10 +244,9 @@ public synchronized boolean removeTaskFromWaitQ(ParallelTask taskTobeRemoved) { } /** - * key function to execute a parallel task + * key function to execute a parallel task. * - * @param task - * the parallel task + * @param task the parallel task * @return the batch response from manager */ public ResponseFromManager generateUpdateExecuteTask(ParallelTask task) { @@ -308,6 +313,12 @@ public ResponseFromManager generateUpdateExecuteTask(ParallelTask task) { public Queue getWaitQ() { return waitQ; } + + /** + * Gets the inprogress task map. + * + * @return the inprogress task map + */ public ConcurrentHashMap getInprogressTaskMap() { return inprogressTaskMap; } @@ -331,9 +342,6 @@ public ResponseFromManager sendTaskToExecutionManager(ParallelTask task) { + task.getTaskId() + " at " + PcDateUtils.getNowDateTimeStr()); - // Get the singleton actor system - // create the master - // command smart upgrade: executionManager = ActorConfig.createAndGetActorSystem().actorOf( Props.create(ExecutionManager.class, task), "ExecutionManager-" + task.getTaskId()); diff --git a/src/main/java/io/parallec/core/taskbuilder/targethosts/TargetHostsBuilderHelperCms.java b/src/main/java/io/parallec/core/taskbuilder/targethosts/TargetHostsBuilderHelperCms.java index fcd53ce..607fbfd 100644 --- a/src/main/java/io/parallec/core/taskbuilder/targethosts/TargetHostsBuilderHelperCms.java +++ b/src/main/java/io/parallec/core/taskbuilder/targethosts/TargetHostsBuilderHelperCms.java @@ -40,7 +40,7 @@ /** * load node from CMS. * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei */ public class TargetHostsBuilderHelperCms { diff --git a/src/main/java/io/parallec/core/util/BeanMapper.java b/src/main/java/io/parallec/core/util/BeanMapper.java index 273d5bb..c0d0e0a 100644 --- a/src/main/java/io/parallec/core/util/BeanMapper.java +++ b/src/main/java/io/parallec/core/util/BeanMapper.java @@ -8,7 +8,7 @@ /** * TODO * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei * */ public class BeanMapper { diff --git a/src/main/java/io/parallec/core/util/PcConstants.java b/src/main/java/io/parallec/core/util/PcConstants.java index 2d1fb39..cd9466e 100644 --- a/src/main/java/io/parallec/core/util/PcConstants.java +++ b/src/main/java/io/parallec/core/util/PcConstants.java @@ -15,7 +15,7 @@ /** * Static variable names... * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei * */ public class PcConstants { diff --git a/src/main/java/io/parallec/core/util/PcHttpUtils.java b/src/main/java/io/parallec/core/util/PcHttpUtils.java index 0f6e4f6..1e74198 100644 --- a/src/main/java/io/parallec/core/util/PcHttpUtils.java +++ b/src/main/java/io/parallec/core/util/PcHttpUtils.java @@ -27,7 +27,7 @@ * Adding http header logic TODO; should finally be data driven and specific for * each command. Now it is defined here and user can easily change. * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei * */ public class PcHttpUtils { diff --git a/src/main/java/io/parallec/core/util/PcTargetHostsUtils.java b/src/main/java/io/parallec/core/util/PcTargetHostsUtils.java index 92efa21..2b76045 100644 --- a/src/main/java/io/parallec/core/util/PcTargetHostsUtils.java +++ b/src/main/java/io/parallec/core/util/PcTargetHostsUtils.java @@ -24,7 +24,7 @@ /** * The Class PcTargetHostsUtils. * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei */ public class PcTargetHostsUtils { diff --git a/src/test/java/io/parallec/core/ParallelClientTest.java b/src/test/java/io/parallec/core/ParallelClientTest.java index aaa19d7..6ba2992 100644 --- a/src/test/java/io/parallec/core/ParallelClientTest.java +++ b/src/test/java/io/parallec/core/ParallelClientTest.java @@ -32,7 +32,7 @@ public void testReInit() { .setConcurrency(1700) .handleInWorker() .setTargetHostsFromString( - "parallec.github.io www.jeffpei.com www.restcommander.com") + "www.parallec.io www.jeffpei.com www.restcommander.com") .execute(new ParallecResponseHandler() { @Override diff --git a/src/test/java/io/parallec/core/TestBase.java b/src/test/java/io/parallec/core/TestBase.java index 6f501ef..5643719 100644 --- a/src/test/java/io/parallec/core/TestBase.java +++ b/src/test/java/io/parallec/core/TestBase.java @@ -23,9 +23,9 @@ public class TestBase { public final String FILEPATH_JSON_PATH = "userdata/sample_target_hosts_json_path.json"; public final String URL_TOP_100 = "http://www.restcommander.com/docs/sample_target_hosts_top100.txt"; - public final String URL_JSON_PATH = "http://parallec.github.io/userdata/sample_target_hosts_json_path.json"; - public final String URL_CMS_QUERY_SINGLE_PAGE = "http://parallec.github.io/cms/repositories/cmsdb/branches/main/query/sample_cms_query_results_single_page.json"; - public final String URL_CMS_QUERY_MULTI_PAGE = "http://parallec.github.io/cms/repositories/cmsdb/branches/main/query/sample_cms_query_results_multi_page_1.json"; + public final String URL_JSON_PATH = "http://www.parallec.io/userdata/sample_target_hosts_json_path.json"; + public final String URL_CMS_QUERY_SINGLE_PAGE = "http://www.parallec.io/cms/repositories/cmsdb/branches/main/query/sample_cms_query_results_single_page.json"; + public final String URL_CMS_QUERY_MULTI_PAGE = "http://www.parallec.io/cms/repositories/cmsdb/branches/main/query/sample_cms_query_results_multi_page_1.json"; public final HostsSourceType SOURCE_LOCAL = HostsSourceType.LOCAL_FILE; public final HostsSourceType SOURCE_URL = HostsSourceType.URL; diff --git a/src/test/java/io/parallec/core/actor/HttpWorkerTest.java b/src/test/java/io/parallec/core/actor/HttpWorkerTest.java index e1df482..e5c5680 100644 --- a/src/test/java/io/parallec/core/actor/HttpWorkerTest.java +++ b/src/test/java/io/parallec/core/actor/HttpWorkerTest.java @@ -47,7 +47,7 @@ public void testSshWorkerCreateRequestException() { // Start new job int actorMaxOperationTimeoutSec = 15; - String urlComplete = "http://parallec.github.io/v**``\"..,++08alidateInternals.html"; + String urlComplete = "http://www.parallec.io/v**``\"..,++08alidateInternals.html"; asyncWorker = ActorConfig.createAndGetActorSystem().actorOf( Props.create(HttpWorker.class, actorMaxOperationTimeoutSec, HttpClientStore.getInstance() @@ -77,7 +77,7 @@ public void testHttpWorkerNormalCheckComplete() { // Start new job int actorMaxOperationTimeoutSec = 15; - String urlComplete = "http://parallec.github.io/validateInternals.html"; + String urlComplete = "http://www.parallec.io/validateInternals.html"; pc.getHttpClientStore(); asyncWorker = ActorConfig.createAndGetActorSystem().actorOf( Props.create(HttpWorker.class, actorMaxOperationTimeoutSec, @@ -109,7 +109,7 @@ public void testHttpWorkerDupAndCancel() { int actorMaxOperationTimeoutSec = 15; - String urlComplete = "http://parallec.github.io/validateInternals.html"; + String urlComplete = "http://www.parallec.io/validateInternals.html"; pc.getHttpClientStore(); asyncWorker = ActorConfig.createAndGetActorSystem().actorOf( Props.create(HttpWorker.class, actorMaxOperationTimeoutSec, @@ -150,7 +150,7 @@ public void testHttpWorkerException() { int actorMaxOperationTimeoutSec = 15; HttpWorker.setLogger(null); - String urlComplete = "http://parallec.github.io/validateInternals.html"; + String urlComplete = "http://www.parallec.io/validateInternals.html"; asyncWorker = ActorConfig.createAndGetActorSystem().actorOf( Props.create(HttpWorker.class, actorMaxOperationTimeoutSec, HttpClientStore.getInstance() @@ -182,7 +182,7 @@ public void testHttpWorkerTimeoutException() { // made a timeout int actorMaxOperationTimeoutSec = 0; - String urlComplete = "http://parallec.github.io/validateInternals.html"; + String urlComplete = "http://www.parallec.io/validateInternals.html"; pc.getHttpClientStore(); asyncWorker = ActorConfig.createAndGetActorSystem().actorOf( Props.create(HttpWorker.class, actorMaxOperationTimeoutSec, @@ -209,7 +209,7 @@ public void testHttpWorkerTimeoutException() { public void testHttpWorkerBadMsgType() { ActorRef asyncWorker = null; try { - String urlComplete = "http://parallec.github.io/validateInternals.html"; + String urlComplete = "http://www.parallec.io/validateInternals.html"; pc.getHttpClientStore(); int actorMaxOperationTimeoutSec = 0; asyncWorker = ActorConfig.createAndGetActorSystem().actorOf( diff --git a/src/test/java/io/parallec/core/actor/OperationWorkerTest.java b/src/test/java/io/parallec/core/actor/OperationWorkerTest.java index 4802962..056f0c9 100644 --- a/src/test/java/io/parallec/core/actor/OperationWorkerTest.java +++ b/src/test/java/io/parallec/core/actor/OperationWorkerTest.java @@ -49,7 +49,7 @@ public void testOperationWorkerWrongMsgType() { new TaskRequest(actorMaxOperationTimeoutSec, RequestProtocol.valueOf("HTTP" .toUpperCase()), - "parallec.github.io", null, 80, "", "", + "www.parallec.io", null, 80, "", "", HttpMethod.GET, false, null, null, new HashMap(), diff --git a/src/test/java/io/parallec/core/httpclient/async/HttpClientStoreTest.java b/src/test/java/io/parallec/core/httpclient/async/HttpClientStoreTest.java index 0c33a26..f6e5104 100644 --- a/src/test/java/io/parallec/core/httpclient/async/HttpClientStoreTest.java +++ b/src/test/java/io/parallec/core/httpclient/async/HttpClientStoreTest.java @@ -33,7 +33,7 @@ public void testDirectorForException() { HttpClientStore.getInstance().getCustomClientSlow(); HttpClientStore.getInstance().getEmbedClientFast(); HttpClientStore.getInstance().getEmbedClientSlow(); - + HttpClientStore.getInstance().setHttpClientTypeCurrentDefault(HttpClientType.CUSTOM_FAST); } catch (Exception ex) { logger.error("Expected Exception : " + ex); } diff --git a/src/test/java/io/parallec/core/main/http/ParallelClientHttpBasicMoreOptionsTest.java b/src/test/java/io/parallec/core/main/http/ParallelClientHttpBasicMoreOptionsTest.java index efc9fb8..f0f881c 100644 --- a/src/test/java/io/parallec/core/main/http/ParallelClientHttpBasicMoreOptionsTest.java +++ b/src/test/java/io/parallec/core/main/http/ParallelClientHttpBasicMoreOptionsTest.java @@ -74,7 +74,7 @@ public void hitWebsitesMinSync() { .prepareHttpGet("/validateInternals.html") .setConcurrency(1700) .setTargetHostsFromString( - "parallec.github.io www.jeffpei.com www.restcommander.com") + "www.parallec.io www.jeffpei.com www.restcommander.com") .setSaveResponseToTask(true).setAutoSaveLogToLocal(true) .setEnableCapacityAwareTaskScheduler(true) diff --git a/src/test/java/io/parallec/core/main/http/ParallelClientHttpBasicTest.java b/src/test/java/io/parallec/core/main/http/ParallelClientHttpBasicTest.java index 5011ee4..73fc816 100644 --- a/src/test/java/io/parallec/core/main/http/ParallelClientHttpBasicTest.java +++ b/src/test/java/io/parallec/core/main/http/ParallelClientHttpBasicTest.java @@ -75,7 +75,7 @@ public void hitWebsitesMinSync() { .setConcurrency(1700) .handleInWorker() .setTargetHostsFromString( - "parallec.github.io www.jeffpei.com www.restcommander.com") + "www.parallec.io www.jeffpei.com www.restcommander.com") .execute(new ParallecResponseHandler() { @Override diff --git a/src/test/java/io/parallec/core/main/http/ParallelClientHttpBasicTestInvalidInput.java b/src/test/java/io/parallec/core/main/http/ParallelClientHttpBasicTestInvalidInput.java index 995f5a3..0f0d47b 100644 --- a/src/test/java/io/parallec/core/main/http/ParallelClientHttpBasicTestInvalidInput.java +++ b/src/test/java/io/parallec/core/main/http/ParallelClientHttpBasicTestInvalidInput.java @@ -34,7 +34,7 @@ public void hitWebsitesUrlWithValidation() { boolean validate = pc.prepareHttpGet(" /validateInternals.html ") .setConcurrency(1700) - // .setTargetHostsFromString("parallec.github.io www.jeffpei.com www.restcommander.com") + // .setTargetHostsFromString("www.parallec.io www.jeffpei.com www.restcommander.com") .validation(); System.out.println("validation: " + validate); @@ -44,7 +44,7 @@ public void hitWebsitesUrlWithValidation() { .async() .sync() .setTargetHostsFromString( - "parallec.github.io www.jeffpei.com www.restcommander.com") + "www.parallec.io www.jeffpei.com www.restcommander.com") .validation(); System.out.println("validation2: " + validate2); } @@ -56,7 +56,7 @@ public void hitWebsitesUrlEndingWSInput() { pc.prepareHttpGet(" /validateInternals.html ") .setConcurrency(1700) .setTargetHostsFromString( - "parallec.github.io www.jeffpei.com www.restcommander.com") + "www.parallec.io www.jeffpei.com www.restcommander.com") .execute(new ParallecResponseHandler() { @Override diff --git a/src/test/java/io/parallec/core/main/http/ParallelClientHttpLogTest.java b/src/test/java/io/parallec/core/main/http/ParallelClientHttpLogTest.java index 056bbf2..2e0f74c 100644 --- a/src/test/java/io/parallec/core/main/http/ParallelClientHttpLogTest.java +++ b/src/test/java/io/parallec/core/main/http/ParallelClientHttpLogTest.java @@ -74,7 +74,7 @@ public void hitWebsitesMinSync() { .prepareHttpGet("/validateInternals.html") .setConcurrency(1700) .setTargetHostsFromString( - "parallec.github.io www.jeffpei.com www.restcommander.com") + "www.parallec.io www.jeffpei.com www.restcommander.com") .execute(new ParallecResponseHandler() { @Override diff --git a/src/test/java/io/parallec/core/main/http/ParallelClientHttpOtherMethodsTest.java b/src/test/java/io/parallec/core/main/http/ParallelClientHttpOtherMethodsTest.java index 41ed22a..ebd2973 100644 --- a/src/test/java/io/parallec/core/main/http/ParallelClientHttpOtherMethodsTest.java +++ b/src/test/java/io/parallec/core/main/http/ParallelClientHttpOtherMethodsTest.java @@ -31,7 +31,7 @@ public void hitWebsitesInvalidHttpMethods() { pc.prepareHttpPut("/validateInternals.html") .setConcurrency(1700) .setTargetHostsFromString( - "parallec.github.io www.jeffpei.com www.restcommander.com") + "www.parallec.io www.jeffpei.com www.restcommander.com") .execute(new ParallecResponseHandler() { @Override public void onCompleted(ResponseOnSingleTask res, @@ -44,7 +44,7 @@ public void onCompleted(ResponseOnSingleTask res, pc.prepareHttpDelete("/validateInternals.html") .setConcurrency(1700) .setTargetHostsFromString( - "parallec.github.io www.jeffpei.com www.restcommander.com") + "www.parallec.io www.jeffpei.com www.restcommander.com") .execute(new ParallecResponseHandler() { @Override public void onCompleted(ResponseOnSingleTask res, @@ -57,7 +57,7 @@ public void onCompleted(ResponseOnSingleTask res, pc.prepareHttpHead("/validateInternals.html") .setConcurrency(1700) .setTargetHostsFromString( - "parallec.github.io www.jeffpei.com www.restcommander.com") + "www.parallec.io www.jeffpei.com www.restcommander.com") .execute(new ParallecResponseHandler() { @Override public void onCompleted(ResponseOnSingleTask res, @@ -70,7 +70,7 @@ public void onCompleted(ResponseOnSingleTask res, pc.prepareHttpOptions("/validateInternals.html") .setConcurrency(1700) .setTargetHostsFromString( - "parallec.github.io www.jeffpei.com www.restcommander.com") + "www.parallec.io www.jeffpei.com www.restcommander.com") .execute(new ParallecResponseHandler() { @Override public void onCompleted(ResponseOnSingleTask res, diff --git a/src/test/java/io/parallec/core/main/http/ParallelTaskCancelOnTargetHostsTest.java b/src/test/java/io/parallec/core/main/http/ParallelTaskCancelOnTargetHostsTest.java index 6a8e619..2071ff1 100644 --- a/src/test/java/io/parallec/core/main/http/ParallelTaskCancelOnTargetHostsTest.java +++ b/src/test/java/io/parallec/core/main/http/ParallelTaskCancelOnTargetHostsTest.java @@ -19,7 +19,7 @@ /** * TODO Testing the enabled capacity control * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei * */ public class ParallelTaskCancelOnTargetHostsTest extends TestBase { diff --git a/src/test/java/io/parallec/core/main/http/ParallelTaskCancelWholeTaskTest.java b/src/test/java/io/parallec/core/main/http/ParallelTaskCancelWholeTaskTest.java index 14c2dcb..5934edf 100644 --- a/src/test/java/io/parallec/core/main/http/ParallelTaskCancelWholeTaskTest.java +++ b/src/test/java/io/parallec/core/main/http/ParallelTaskCancelWholeTaskTest.java @@ -15,7 +15,7 @@ /** * TODO Testing the enabled capacity control * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei * */ public class ParallelTaskCancelWholeTaskTest extends TestBase { diff --git a/src/test/java/io/parallec/core/main/http/pollable/sampleserver/ServerWithPollableJobs.java b/src/test/java/io/parallec/core/main/http/pollable/sampleserver/ServerWithPollableJobs.java index 75f6cca..746c95c 100644 --- a/src/test/java/io/parallec/core/main/http/pollable/sampleserver/ServerWithPollableJobs.java +++ b/src/test/java/io/parallec/core/main/http/pollable/sampleserver/ServerWithPollableJobs.java @@ -18,7 +18,7 @@ * * note that the job map will never be clean up until it reaches 8192 * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei * */ public class ServerWithPollableJobs extends NanoHTTPD { diff --git a/src/test/java/io/parallec/core/main/http/request/template/ParallelClientVarReplacementHostSpecificTest.java b/src/test/java/io/parallec/core/main/http/request/template/ParallelClientVarReplacementHostSpecificTest.java index 318353b..c89f524 100644 --- a/src/test/java/io/parallec/core/main/http/request/template/ParallelClientVarReplacementHostSpecificTest.java +++ b/src/test/java/io/parallec/core/main/http/request/template/ParallelClientVarReplacementHostSpecificTest.java @@ -10,6 +10,7 @@ import io.parallec.core.bean.StrStrMap; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -42,7 +43,7 @@ public static void shutdown() throws Exception { public void hitWebsitesMinTargetHostSpecificReplacement() { Map replacementVarMapNodeSpecific = new HashMap(); - replacementVarMapNodeSpecific.put("parallec.github.io", + replacementVarMapNodeSpecific.put("www.parallec.io", new StrStrMap().addPair("JOB_ID", "job_a")); replacementVarMapNodeSpecific.put("www.jeffpei.com", new StrStrMap().addPair("JOB_ID", "job_b")); @@ -52,7 +53,7 @@ public void hitWebsitesMinTargetHostSpecificReplacement() { pc.prepareHttpGet("/$JOB_ID.html") .setConcurrency(1700) .setTargetHostsFromString( - "parallec.github.io www.jeffpei.com www.restcommander.com") + "www.parallec.io www.jeffpei.com www.restcommander.com") .setReplacementVarMapNodeSpecific(replacementVarMapNodeSpecific) .execute(new ParallecResponseHandler() { @Override @@ -71,14 +72,11 @@ public void onCompleted(ResponseOnSingleTask res, /** * note that the target host must match the key this is the new way new - * http://parallec.github.io/userdata/sample_weather_48824.txt - * http://parallec.github.io/userdata/sample_weather_95037.txt + * http://www.parallec.io/userdata/sample_weather_48824.txt + * http://www.parallec.io/userdata/sample_weather_95037.txt */ @Test public void differentRequestsToSameTargetHost() { - List replaceList = new ArrayList(); - replaceList.add("95037"); - replaceList.add("48824"); Map responseContext = new HashMap(); responseContext.put("temp", null); @@ -88,8 +86,8 @@ public void differentRequestsToSameTargetHost() { new ParallecHeader().addPair("content-type", "text/xml;charset=UTF-8")) .setConcurrency(1700) - .setReplaceVarMapToSingleTargetSingleVar("ZIP", replaceList, - "parallec.github.io") + .setReplaceVarMapToSingleTargetSingleVar("ZIP", Arrays.asList("95037","48824"), + "www.parallec.io") .setResponseContext(responseContext) .execute(new ParallecResponseHandler() { @@ -130,8 +128,8 @@ public void differentRequestsToSameTargetHostMultiVarInvalidReplaceEmptyTarget() /** * Use this api if need to match multiple variables new - * http://parallec.github.io/userdata/MI/sample_weather_48824.txt - * http://parallec.github.io/userdata/CA/sample_weather_95037.txt + * http://www.parallec.io/userdata/MI/sample_weather_48824.txt + * http://www.parallec.io/userdata/CA/sample_weather_95037.txt */ @Test public void differentRequestsToSameTargetHostMultiVar() { @@ -151,7 +149,7 @@ public void differentRequestsToSameTargetHostMultiVar() { "text/xml;charset=UTF-8")) .setConcurrency(1700) .setReplaceVarMapToSingleTarget(replacementVarMapList, - "parallec.github.io") + "www.parallec.io") .setResponseContext(responseContext) .execute(new ParallecResponseHandler() { @@ -179,8 +177,8 @@ public void onCompleted(ResponseOnSingleTask res, /** * note that the target host must match the key * - * new http://parallec.github.io/userdata/sample_weather_48824.txt - * http://parallec.github.io/userdata/sample_weather_95037.txt + * new http://www.parallec.io/userdata/sample_weather_48824.txt + * http://www.parallec.io/userdata/sample_weather_95037.txt */ @Test public void differentRequestsToSameTargetHostWithMap() { @@ -202,7 +200,7 @@ public void differentRequestsToSameTargetHostWithMap() { .setConcurrency(1700) .setTargetHostsFromString("api1 api2") .setReplaceVarMapToSingleTargetFromMap( - replacementVarMapNodeSpecific, "parallec.github.io") + replacementVarMapNodeSpecific, "www.parallec.io") .setResponseContext(responseContext) .execute(new ParallecResponseHandler() { @@ -299,7 +297,7 @@ public void onCompleted(ResponseOnSingleTask res, public void hitWebsitesMinTargetHostSpecificReplacementWithNA() { Map replacementVarMapNodeSpecific = new HashMap(); - replacementVarMapNodeSpecific.put("parallec.github.io", + replacementVarMapNodeSpecific.put("www.parallec.io", new StrStrMap().addPair("JOB_ID", "job_a")); replacementVarMapNodeSpecific.put("www.jeffpei.com", new StrStrMap() .addPair("JOB_ID", "job_b").addPair("NA", "NA")); @@ -310,7 +308,7 @@ public void hitWebsitesMinTargetHostSpecificReplacementWithNA() { .prepareHttpGet("/$JOB_ID.html") .setConcurrency(1700) .setTargetHostsFromString( - "parallec.github.io www.jeffpei.com www.restcommander.com") + "www.parallec.io www.jeffpei.com www.restcommander.com") .setReplacementVarMapNodeSpecific(replacementVarMapNodeSpecific) .execute(new ParallecResponseHandler() { @Override @@ -335,7 +333,7 @@ public void onCompleted(ResponseOnSingleTask res, public void hitWebsitesMinTargetHostSpecificReplacementWithAllNA() { Map replacementVarMapNodeSpecific = new HashMap(); - replacementVarMapNodeSpecific.put("parallec.github.io", new StrStrMap() + replacementVarMapNodeSpecific.put("www.parallec.io", new StrStrMap() .addPair("JOB_ID", "job_a").addPair("NA", "NA")); replacementVarMapNodeSpecific.put("www.jeffpei.com", new StrStrMap() .addPair("JOB_ID", "job_b").addPair("NA", "NA")); @@ -346,7 +344,7 @@ public void hitWebsitesMinTargetHostSpecificReplacementWithAllNA() { .prepareHttpGet("/$JOB_ID.html") .setConcurrency(1700) .setTargetHostsFromString( - "parallec.github.io www.jeffpei.com www.restcommander.com") + "www.parallec.io www.jeffpei.com www.restcommander.com") .setReplacementVarMapNodeSpecific(replacementVarMapNodeSpecific) .execute(new ParallecResponseHandler() { @Override diff --git a/src/test/java/io/parallec/core/main/http/request/template/ParallelClientVarReplacementUniformTest.java b/src/test/java/io/parallec/core/main/http/request/template/ParallelClientVarReplacementUniformTest.java index 4a494af..0d9a28a 100644 --- a/src/test/java/io/parallec/core/main/http/request/template/ParallelClientVarReplacementUniformTest.java +++ b/src/test/java/io/parallec/core/main/http/request/template/ParallelClientVarReplacementUniformTest.java @@ -51,7 +51,7 @@ public void hitWebsitesMinTargeUniformReplacement() { pc.prepareHttpGet("/$URL_VARIABLE.html") .setConcurrency(1700) .setTargetHostsFromString( - "parallec.github.io www.jeffpei.com www.restcommander.com") + "www.parallec.io www.jeffpei.com www.restcommander.com") .setReplacementVarMap(replacementVarMap) .execute(new ParallecResponseHandler() { @Override diff --git a/src/test/java/io/parallec/core/main/http/scheduler/ParallelClientSchedulerAndCancelTest.java b/src/test/java/io/parallec/core/main/http/scheduler/ParallelClientSchedulerAndCancelTest.java index 7919c97..512ff2a 100644 --- a/src/test/java/io/parallec/core/main/http/scheduler/ParallelClientSchedulerAndCancelTest.java +++ b/src/test/java/io/parallec/core/main/http/scheduler/ParallelClientSchedulerAndCancelTest.java @@ -94,6 +94,7 @@ public void onCompleted(ResponseOnSingleTask res, } ParallecGlobalConfig.maxCapacity = 2500; + ParallelTaskManager.getInstance().shutdownTaskScheduler(); }// end func /** @@ -105,6 +106,8 @@ public void onCompleted(ResponseOnSingleTask res, @Test(timeout = 60000) public void hitTop100WebsitesCapacityProtectionCancel2ndTask() { + ParallelTaskManager.getInstance().shutdownTaskScheduler(); + ParallecGlobalConfig.maxCapacity = 120; ParallelTask pt = pc .prepareHttpGet("") diff --git a/src/test/java/io/parallec/core/main/ping/PingOnTmpTest.java b/src/test/java/io/parallec/core/main/ping/PingOnTmpTest.java index 2e5d348..06e35f2 100644 --- a/src/test/java/io/parallec/core/main/ping/PingOnTmpTest.java +++ b/src/test/java/io/parallec/core/main/ping/PingOnTmpTest.java @@ -73,7 +73,7 @@ public void pingWebsitesMinBasicSyncOn1500() { .setPingNumRetries(0) .setTargetHostsFromLineByLineText(FILEPATH_TMP1, SOURCE_LOCAL) // .setTargetHostsFromString( - // "parallec.github.io www.jeffpei.com www.restcommander.com bad.c21tom") + // "www.parallec.io www.jeffpei.com www.restcommander.com bad.c21tom") .execute(new ParallecResponseHandler() { @Override public void onCompleted(ResponseOnSingleTask res, @@ -98,7 +98,7 @@ public void pingWebsitesMinAsync() { ParallelTask task = pc.preparePing().async().setConcurrency(1500) .setTargetHostsFromLineByLineText(FILEPATH_TMP1, SOURCE_LOCAL) // .setTargetHostsFromString( - // "parallec.github.io www.jeffpei.com www.restcommander.com bad.c21tom") + // "www.parallec.io www.jeffpei.com www.restcommander.com bad.c21tom") .execute(new ParallecResponseHandler() { @Override public void onCompleted(ResponseOnSingleTask res, diff --git a/src/test/java/io/parallec/core/main/ping/PingTest.java b/src/test/java/io/parallec/core/main/ping/PingTest.java index 4132171..0e8040f 100644 --- a/src/test/java/io/parallec/core/main/ping/PingTest.java +++ b/src/test/java/io/parallec/core/main/ping/PingTest.java @@ -71,7 +71,7 @@ public void pingWebsitesMinBasicSync() { ParallelTask task = pc.preparePing().setConcurrency(1500) .setTargetHostsFromString( - "parallec.github.io www.jeffpei.com www.restcommander.com bad.c21tom") + "www.parallec.io www.jeffpei.com www.restcommander.com bad.c21tom") .execute(new ParallecResponseHandler() { @Override public void onCompleted(ResponseOnSingleTask res, @@ -90,7 +90,7 @@ public void pingWebsitesMoreOptions() { ParallelTask task = pc.preparePing().setConcurrency(1500) .setTargetHostsFromString( - "parallec.github.io www.jeffpei.com www.restcommander.com") + "www.parallec.io www.jeffpei.com www.restcommander.com") .setPingMode(PingMode.PROCESS) .setPingNumRetries(3) .setPingTimeoutMillis(500) @@ -115,7 +115,7 @@ public void testInvalidPoller() { pc.preparePing().setConcurrency(1500) .setHttpPollable(true) .setTargetHostsFromString( - "parallec.github.io www.jeffpei.com www.restcommander.com bad.c21tom") + "www.parallec.io www.jeffpei.com www.restcommander.com bad.c21tom") .execute(new ParallecResponseHandler() { @Override public void onCompleted(ResponseOnSingleTask res, diff --git a/src/test/java/io/parallec/core/main/ping/package-info.java b/src/test/java/io/parallec/core/main/ping/package-info.java index 75b63ae..3cd9f50 100644 --- a/src/test/java/io/parallec/core/main/ping/package-info.java +++ b/src/test/java/io/parallec/core/main/ping/package-info.java @@ -2,7 +2,7 @@ * */ /** - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei * */ package io.parallec.core.main.ping; \ No newline at end of file diff --git a/src/test/java/io/parallec/core/main/tcp/package-info.java b/src/test/java/io/parallec/core/main/tcp/package-info.java index 3b7751d..abb16d2 100644 --- a/src/test/java/io/parallec/core/main/tcp/package-info.java +++ b/src/test/java/io/parallec/core/main/tcp/package-info.java @@ -2,7 +2,7 @@ * */ /** - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei * */ package io.parallec.core.main.tcp; \ No newline at end of file diff --git a/src/test/java/io/parallec/core/main/tcp/sampleserver/TcpEchoServer.java b/src/test/java/io/parallec/core/main/tcp/sampleserver/TcpEchoServer.java index 709cdd8..a5f3190 100644 --- a/src/test/java/io/parallec/core/main/tcp/sampleserver/TcpEchoServer.java +++ b/src/test/java/io/parallec/core/main/tcp/sampleserver/TcpEchoServer.java @@ -28,7 +28,7 @@ * will echo back 3 lines of response that include the request string, * then auto close the connection * - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei */ public class TcpEchoServer { diff --git a/src/test/java/io/parallec/core/main/tcp/sampleserver/package-info.java b/src/test/java/io/parallec/core/main/tcp/sampleserver/package-info.java index bbe17df..ae519f9 100644 --- a/src/test/java/io/parallec/core/main/tcp/sampleserver/package-info.java +++ b/src/test/java/io/parallec/core/main/tcp/sampleserver/package-info.java @@ -2,7 +2,7 @@ * */ /** - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei * */ package io.parallec.core.main.tcp.sampleserver; \ No newline at end of file diff --git a/src/test/java/io/parallec/core/task/package-info.java b/src/test/java/io/parallec/core/task/package-info.java index aedbc65..9dfa4e9 100644 --- a/src/test/java/io/parallec/core/task/package-info.java +++ b/src/test/java/io/parallec/core/task/package-info.java @@ -1,6 +1,6 @@ /** - * @author Yuanteng Jeff Pei + * @author Yuanteng (Jeff) Pei * */ package io.parallec.core.task; \ No newline at end of file diff --git a/src/test/java/io/parallec/core/taskbuilder/targethosts/TargetHostsBuilderTest.java b/src/test/java/io/parallec/core/taskbuilder/targethosts/TargetHostsBuilderTest.java index 616f57a..11ddddd 100644 --- a/src/test/java/io/parallec/core/taskbuilder/targethosts/TargetHostsBuilderTest.java +++ b/src/test/java/io/parallec/core/taskbuilder/targethosts/TargetHostsBuilderTest.java @@ -95,7 +95,7 @@ public void setTargetHostsFromUrl() { } - // http://parallec.github.io/userdata/sample_target_hosts_json_path.json + // http://www.parallec.io/userdata/sample_target_hosts_json_path.json @Test public void setTargetHostsFromJsonPathTest() { String jsonPath = "$.sample.small-target-hosts[*].hostName"; @@ -146,7 +146,7 @@ public void setTargetHostsFromCmsQueryUrTest() { // bad query try { - thb.setTargetHostsFromCmsQueryUrl("http://1parallec.github.io/cms/repositories/cmsdb/branches" + thb.setTargetHostsFromCmsQueryUrl("http://1www.parallec.io/cms/repositories/cmsdb/branches" + "/main/query/sample_cms_query_results_single_page.json"); } catch (TargetHostsLoadException e) { logger.info("expected exception: " + e); @@ -154,11 +154,11 @@ public void setTargetHostsFromCmsQueryUrTest() { // with projection null/empty thb.setTargetHostsFromCmsQueryUrl( - "http://parallec.github.io/cms/repositories/cmsdb/branches" + "http://www.parallec.io/cms/repositories/cmsdb/branches" + "/main/query/sample_cms_query_results_single_page.json", null); thb.setTargetHostsFromCmsQueryUrl( - "http://parallec.github.io/cms/repositories/cmsdb/branches" + "http://www.parallec.io/cms/repositories/cmsdb/branches" + "/main/query/sample_cms_query_results_single_page.json", ""); diff --git a/src/test/java/io/parallec/core/util/FileNetworkIoUtilsTest.java b/src/test/java/io/parallec/core/util/FileNetworkIoUtilsTest.java index 6df3edb..0598511 100644 --- a/src/test/java/io/parallec/core/util/FileNetworkIoUtilsTest.java +++ b/src/test/java/io/parallec/core/util/FileNetworkIoUtilsTest.java @@ -17,6 +17,7 @@ public void setUp() throws Exception { public void test() throws FileNotFoundException { PcFileNetworkIoUtils.isFileExist(FILEPATH_TOP_100); PcFileNetworkIoUtils.readFileToInputStream(FILEPATH_TOP_100); + PcFileNetworkIoUtils.readFileToInputStream(FILEPATH_TOP_100+"badpath"); PcFileNetworkIoUtils.getListFromLineByLineText(FILEPATH_TOP_100, SOURCE_LOCAL); PcFileNetworkIoUtils.getListFromLineByLineText(FILEPATH_TOP_100 + "a", diff --git a/userdata/sample_target_hosts_top100_old.txt b/userdata/sample_target_hosts_top100_old.txt index e448cac..c90cfaf 100644 --- a/userdata/sample_target_hosts_top100_old.txt +++ b/userdata/sample_target_hosts_top100_old.txt @@ -31,7 +31,7 @@ www.mail.ru www.pinterest.com www.apple.com www.google.com.br -www.soso.com +www.loc.gov www.PayPal.com www.google.ru www.google.es @@ -46,7 +46,6 @@ www.bbc.co.uk www.go.com www.amazon.co.jp www.google.com.mx -www.odnoklassniki.ru www.google.ca www.amazon.de www.adobe.com @@ -75,7 +74,6 @@ www.dailymotion.com www.imgur.com www.zedo.com www.google.pl -www.uol.com.br www.deviantart.com www.google.nl www.dropbox.com @@ -97,4 +95,6 @@ www.toptenvideos.com www.walgreens.com www.liveleak.com www.discover.com -www.mosthappy.com \ No newline at end of file +www.mosthappy.com +www.reverbnation.com +www.irs.gov \ No newline at end of file