multiple remote bots on different machines #2179
Comments
kbiz862
changed the title
|
Hi, Did you guys find time to look at it? |
|
Somehow remote bot is not able to accept the fuzzing job. remote bot logs: 2020-12-28 21:04:55,204 - run_bot - ERROR - Task exited with exception (payload="None"). |
|
We cannot replicate your connection failures, from stacktrace, your pubsub cannot connect to server, make sure port 9006 is open on your server and client can connect to it. https://github.com/google/clusterfuzz/blob/master/src/local/butler/constants.py#L70 |
Hi,
This is in continuation of closed ticket #2151.
I tried to connect remote bots to the server. it shows the list of remote bots but when i submit the job, remote jobs seems to be not taking the job.
I followed the steps mentioned in https://www.youtube.com/watch?v=65aywI2JcLA. Using these steps i am able to upload the jobs from other machines as well(where server is not running).
Server constant.py (vm-cfz1 is my server hostname)
DEV_APPSERVER_PORT = 9000
DEV_APPSERVER_HOST = 'vm-cfz1:' + str(DEV_APPSERVER_PORT)
CRON_SERVICE_PORT = 9009
CRON_SERVICE_HOST = 'vm-cfz1:' + str(CRON_SERVICE_PORT)
DEV_APPSERVER_ADMIN_PORT = 9002
DATASTORE_EMULATOR_PORT = 9004
DATASTORE_EMULATOR_HOST = 'vm-cfz1:' + str(DATASTORE_EMULATOR_PORT)
PUBSUB_EMULATOR_PORT = 9006
PUBSUB_EMULATOR_HOST = 'localhost:' + str(PUBSUB_EMULATOR_PORT)
LOCAL_GCS_SERVER_PORT = 9008
LOCAL_GCS_SERVER_HOST = 'http://vm-cfz1:' + str(LOCAL_GCS_SERVER_PORT)
remote bot(whose hostname is vm-cfz2) constant.py
DEV_APPSERVER_PORT = 9000
DEV_APPSERVER_HOST = 'vm-cfz1:' + str(DEV_APPSERVER_PORT)
CRON_SERVICE_PORT = 9009
CRON_SERVICE_HOST = 'vm-cfz1:' + str(CRON_SERVICE_PORT)
DEV_APPSERVER_ADMIN_PORT = 9002
DATASTORE_EMULATOR_PORT = 9004
DATASTORE_EMULATOR_HOST = 'vm-cfz1:' + str(DATASTORE_EMULATOR_PORT)
PUBSUB_EMULATOR_PORT = 9006
PUBSUB_EMULATOR_HOST = 'vm-cfz1:' + str(PUBSUB_EMULATOR_PORT)
LOCAL_GCS_SERVER_PORT = 9008
LOCAL_GCS_SERVER_HOST = 'http://vm-cfz1:' + str(LOCAL_GCS_SERVER_PORT)
command to run server:
python3.7 butler.py run_server --storage-path /prj/clusterfuzz --bootstrap
command to run bot:
python3.7 butler.py run_bot --server-storage-path /prj/clusterfuzz/ --name remote_bot ./remote_bot/
Note: /prj/clusterfuzz is shared path which is accessible to both server and remote bot.
bot.log on remote bot:
run_bot - INFO - Retrying on google_cloud_utils.pubsub._execute_with_retry failed with [Errno 111] Connection refused. Retrying again.
StackTrace:
2020-12-23 16:40:55,697 - run_bot - ERROR - Error occurred while working on task.
Traceback (most recent call last):
File "src/python/bot/startup/run_bot.py", line 78, in task_loop
task = tasks.get_task()
File "/local/mnt/workspace/clusterfuzz/remote_bot/clusterfuzz/src/python/base/tasks.py", line 202, in get_task
task = get_regular_task()
File "/local/mnt/workspace/clusterfuzz/remote_bot/clusterfuzz/src/python/base/tasks.py", line 165, in get_regular_task
pubsub.subscription_name(application_id, queue), max_messages=1)
File "/local/mnt/workspace/clusterfuzz/remote_bot/clusterfuzz/src/python/google_cloud_utils/pubsub.py", line 157, in pull_from_subscription
response = self._execute_with_retry(request)
File "/local/mnt/workspace/clusterfuzz/remote_bot/clusterfuzz/src/python/base/retry.py", line 88, in _wrapper
result = func(*args, **kwargs)
File "/local/mnt/workspace/clusterfuzz/remote_bot/clusterfuzz/src/python/google_cloud_utils/pubsub.py", line 107, in _execute_with_retry
return request.execute()
File "/local/mnt/workspace/clusterfuzz/remote_bot/clusterfuzz/src/third_party/googleapiclient/_helpers.py", line 134, in positional_wrapper
return wrapped(*args, **kwargs)
File "/local/mnt/workspace/clusterfuzz/remote_bot/clusterfuzz/src/third_party/googleapiclient/http.py", line 901, in execute
headers=self.headers,
File "/local/mnt/workspace/clusterfuzz/remote_bot/clusterfuzz/src/third_party/googleapiclient/http.py", line 204, in _retry_request
raise exception
File "/local/mnt/workspace/clusterfuzz/remote_bot/clusterfuzz/src/third_party/googleapiclient/http.py", line 177, in _retry_request
resp, content = http.request(uri, method, *args, **kwargs)
File "/local/mnt/workspace/clusterfuzz/remote_bot/clusterfuzz/src/third_party/google_auth_httplib2.py", line 201, in request
uri, method, body=body, headers=request_headers, **kwargs)
File "/local/mnt/workspace/clusterfuzz/remote_bot/clusterfuzz/src/third_party/httplib2/init.py", line 1994, in request
cachekey,
File "/local/mnt/workspace/clusterfuzz/remote_bot/clusterfuzz/src/third_party/httplib2/init.py", line 1651, in _request
conn, request_uri, method, body, headers
File "/local/mnt/workspace/clusterfuzz/remote_bot/clusterfuzz/src/third_party/httplib2/init.py", line 1557, in _conn_request
conn.connect()
File "/local/mnt/workspace/clusterfuzz/remote_bot/clusterfuzz/src/third_party/httplib2/init.py", line 1236, in connect
raise socket_err
File "/local/mnt/workspace/clusterfuzz/remote_bot/clusterfuzz/src/third_party/httplib2/init.py", line 1210, in connect
self.sock.connect((self.host, self.port) + sa[2:])
ConnectionRefusedError: [Errno 111] Connection refused
Also i tried to change PUBSUB_EMULATOR_HOST from 'localhost:' + str(PUBSUB_EMULATOR_PORT) to 'vm-cfz1:' + str(PUBSUB_EMULATOR_PORT)
i get below error:
Traceback (most recent call last):
File "butler.py", line 316, in
main()
File "butler.py", line 302, in main
command.execute(args)
File "src/local/butler/run_server.py", line 155, in execute
test_utils.setup_pubsub(constants.TEST_APP_ID)
File "/local/mnt/workspace/clusterfuzz/src/python/tests/test_libs/test_utils.py", line 293, in setup_pubsub
create_pubsub_topic(client, project, queue['name'])
File "/local/mnt/workspace/clusterfuzz/src/python/tests/test_libs/test_utils.py", line 266, in create_pubsub_topic
if client.get_topic(full_name):
File "/local/mnt/workspace/clusterfuzz/src/python/google_cloud_utils/pubsub.py", line 197, in get_topic
response = self._execute_with_retry(request)
File "/local/mnt/workspace/clusterfuzz/src/python/base/retry.py", line 88, in _wrapper
result = func(*args, **kwargs)
File "/local/mnt/workspace/clusterfuzz/src/python/google_cloud_utils/pubsub.py", line 107, in _execute_with_retry
return request.execute()
File "/local/mnt/workspace/clusterfuzz/src/third_party/googleapiclient/_helpers.py", line 134, in positional_wrapper
return wrapped(*args, **kwargs)
File "/local/mnt/workspace/clusterfuzz/src/third_party/googleapiclient/http.py", line 907, in execute
raise HttpError(resp, content, uri=self.uri)
googleapiclient.errors.HttpError: <HttpError 503 when requesting http://vm-cfz1:9006/v1/projects/test-clusterfuzz/topics/jobs-linux?alt=json returned "io exception">
Thanks in advance.
The text was updated successfully, but these errors were encountered: