Skip to content
Permalink
Browse files Browse the repository at this point in the history
fix remote execute
  • Loading branch information
Germey committed Dec 26, 2021
1 parent f1cd46d commit 49bcb19
Showing 1 changed file with 56 additions and 40 deletions.
96 changes: 56 additions & 40 deletions gerapy/server/core/views.py
Expand Up @@ -2,7 +2,12 @@
from pathlib import Path
from urllib.parse import unquote
import base64
import json, os, requests, time, pytz, pymongo
import json
import os
import requests
import time
import pytz
import pymongo
from shutil import rmtree
from requests.exceptions import ConnectionError
from os.path import join, exists
Expand Down Expand Up @@ -173,7 +178,8 @@ def spider_list(request, client_id, project_name):
client = Client.objects.get(id=client_id)
scrapyd = get_scrapyd(client)
spiders = scrapyd.list_spiders(project_name)
spiders = [{'name': spider, 'id': index + 1} for index, spider in enumerate(spiders)]
spiders = [{'name': spider, 'id': index + 1}
for index, spider in enumerate(spiders)]
return JsonResponse(spiders)


Expand Down Expand Up @@ -242,23 +248,25 @@ def project_configure(request, project_name):
if request.method == 'GET':
project = Project.objects.get(name=project_name)
project = model_to_dict(project)
project['configuration'] = json.loads(project['configuration']) if project['configuration'] else None
project['configuration'] = json.loads(
project['configuration']) if project['configuration'] else None
return JsonResponse(project)

# update configuration
elif request.method == 'POST':
project = Project.objects.filter(name=project_name)
data = json.loads(request.body)
configuration = json.dumps(data.get('configuration'), ensure_ascii=False)
configuration = json.dumps(
data.get('configuration'), ensure_ascii=False)
project.update(**{'configuration': configuration})

# for safe protection
project_name = re.sub('[\!\@\#\$\;\&\*\~\"\'\{\}\]\[\-\+\%\^]+', '', project_name)
project_name = re.sub(
'[\s\!\@\#\$\;\&\*\~\"\'\{\}\]\[\-\+\%\^]+', '', project_name)
# execute generate cmd
cmd = ' '.join(['gerapy', 'generate', project_name])
p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE)
cmd = ['gerapy', 'generate', project_name]
p = Popen(cmd, shell=False, stdin=PIPE, stdout=PIPE, stderr=PIPE)
stdout, stderr = bytes2str(p.stdout.read()), bytes2str(p.stderr.read())

if not stderr:
return JsonResponse({'status': '1'})
else:
Expand Down Expand Up @@ -294,7 +302,8 @@ def project_create(request):
data['configurable'] = 1
project, result = Project.objects.update_or_create(**data)
# generate a single project folder
path = join(os.path.abspath(join(os.getcwd(), PROJECTS_FOLDER)), data['name'])
path = join(os.path.abspath(
join(os.getcwd(), PROJECTS_FOLDER)), data['name'])
os.mkdir(path)
return JsonResponse(model_to_dict(project))

Expand Down Expand Up @@ -334,12 +343,13 @@ def project_clone(request):
if not address.startswith('http'):
return JsonResponse({'status': False})
address = address + '.git' if not address.endswith('.git') else address
cmd = 'git clone {address} {target}'.format(address=address, target=join(PROJECTS_FOLDER, Path(address).stem))
cmd = ['git', 'clone', 'address', join(PROJECTS_FOLDER, Path(address).stem)]
logger.debug('clone cmd %s', cmd)
p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE)
p = Popen(cmd, shell=False, stdin=PIPE, stdout=PIPE, stderr=PIPE)
stdout, stderr = bytes2str(p.stdout.read()), bytes2str(p.stderr.read())
logger.debug('clone run result %s', stdout)
if stderr: logger.error(stderr)
if stderr:
logger.error(stderr)
return JsonResponse({'status': True}) if not stderr else JsonResponse({'status': False})


Expand Down Expand Up @@ -393,10 +403,12 @@ def project_version(request, client_id, project_name):
return JsonResponse({'message': 'Connect Error'}, status=500)
if len(versions) > 0:
version = versions[-1]
deployed_at = timezone.datetime.fromtimestamp(int(version), tz=pytz.timezone(TIME_ZONE))
deployed_at = timezone.datetime.fromtimestamp(
int(version), tz=pytz.timezone(TIME_ZONE))
else:
deployed_at = None
deploy, result = Deploy.objects.update_or_create(client=client, project=project, deployed_at=deployed_at)
deploy, result = Deploy.objects.update_or_create(
client=client, project=project, deployed_at=deployed_at)
# return deploy json info
return JsonResponse(model_to_dict(deploy))

Expand Down Expand Up @@ -446,7 +458,7 @@ def project_build(request, project_name):
# get project folder
path = os.path.abspath(join(os.getcwd(), PROJECTS_FOLDER))
project_path = join(path, project_name)

# get build version
if request.method == 'GET':
egg = find_egg(project_path)
Expand All @@ -470,7 +482,7 @@ def project_build(request, project_name):
# transfer model to dict then dumps it to json
data = model_to_dict(model)
return JsonResponse(data)

# build operation manually by clicking button
elif request.method == 'POST':
data = json.loads(request.body)
Expand All @@ -483,7 +495,8 @@ def project_build(request, project_name):
built_at = timezone.now()
# if project does not exists in db, create it
if not Project.objects.filter(name=project_name):
Project(name=project_name, description=description, built_at=built_at, egg=egg).save()
Project(name=project_name, description=description,
built_at=built_at, egg=egg).save()
model = Project.objects.get(name=project_name)
# if project exists, update egg, description, built_at info
else:
Expand Down Expand Up @@ -526,17 +539,16 @@ def project_parse(request, project_name):
body = data.get('body', '')
if args.get('method').lower() != 'get':
args['body'] = "'" + json.dumps(body, ensure_ascii=False) + "'"

args_cmd = ' '.join(
['--{arg} {value}'.format(arg=arg, value=value) for arg, value in args.items()])
logger.debug('args cmd %s', args_cmd)
cmd = 'gerapy parse {args_cmd} {project_path} {spider_name}'.format(
args_cmd=args_cmd,
project_path=project_path,
spider_name=spider_name
)

args_array = []
for arg, value in args.items():
args_array.append(f'--{arg}')
args_array.append(f'{value}')
cmd = ['gerapy', 'parse'] + args_array + [project_path] + [spider_name]
print('cmd', cmd)
logger.debug('parse cmd %s', cmd)
p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True)
p = Popen(cmd, shell=False, stdin=PIPE,
stdout=PIPE, stderr=PIPE, close_fds=True)
stdout, stderr = bytes2str(p.stdout.read()), bytes2str(p.stderr.read())
logger.debug('stdout %s, stderr %s', stdout, stderr)
if not stderr:
Expand Down Expand Up @@ -645,7 +657,6 @@ def job_list(request, client_id, project_name):
job['status'] = status
jobs.append(job)
return JsonResponse(jobs)



@api_view(['GET'])
Expand All @@ -663,7 +674,8 @@ def job_log(request, client_id, project_name, spider_name, job_id):
if request.method == 'GET':
client = Client.objects.get(id=client_id)
# get log url
url = log_url(client.ip, client.port, project_name, spider_name, job_id)
url = log_url(client.ip, client.port,
project_name, spider_name, job_id)
# get last 1000 bytes of log
response = requests.get(url, timeout=5, headers={
'Range': 'bytes=-1000'
Expand Down Expand Up @@ -765,7 +777,8 @@ def monitor_create(request):
if request.method == 'POST':
data = json.loads(request.body)
data = data['form']
data['configuration'] = json.dumps(data['configuration'], ensure_ascii=False)
data['configuration'] = json.dumps(
data['configuration'], ensure_ascii=False)
monitor = Monitor.objects.create(**data)
return JsonResponse(model_to_dict(monitor))

Expand All @@ -785,7 +798,8 @@ def task_create(request):
name=data.get('name'),
spider=data.get('spider'),
trigger=data.get('trigger'),
configuration=json.dumps(data.get('configuration'), ensure_ascii=False),
configuration=json.dumps(
data.get('configuration'), ensure_ascii=False),
modified=1)
return JsonResponse({'result': '1', 'data': model_to_dict(task)})

Expand All @@ -803,7 +817,8 @@ def task_update(request, task_id):
task = Task.objects.filter(id=task_id)
data = json.loads(request.body)
data['clients'] = json.dumps(data.get('clients'), ensure_ascii=False)
data['configuration'] = json.dumps(data.get('configuration'), ensure_ascii=False)
data['configuration'] = json.dumps(
data.get('configuration'), ensure_ascii=False)
data['modified'] = 1
task.update(**data)
return JsonResponse(model_to_dict(Task.objects.get(id=task_id)))
Expand All @@ -823,11 +838,10 @@ def task_remove(request, task_id):
clients = clients_of_task(task)
for client in clients:
job_id = get_job_id(client, task)
DjangoJob.objects.filter(name=job_id).delete()
DjangoJob.objects.filter(id=job_id).delete()
# delete task
Task.objects.filter(id=task_id).delete()
return JsonResponse({'result': '1'})



@api_view(['GET'])
Expand Down Expand Up @@ -875,12 +889,14 @@ def task_status(request, task_id):
clients = clients_of_task(task)
for client in clients:
job_id = get_job_id(client, task)
jobs = DjangoJob.objects.filter(name=job_id)
jobs = DjangoJob.objects.filter(id=job_id)
logger.debug('jobs from djangojob %s', jobs)
# if job does not exist, for date mode exceed time
if not jobs: continue
job = DjangoJob.objects.get(name=job_id)
executions = serialize('json', DjangoJobExecution.objects.filter(job=job))
if not jobs:
continue
job = DjangoJob.objects.get(id=job_id)
executions = serialize(
'json', DjangoJobExecution.objects.filter(job=job))
result.append({
'client': model_to_dict(client),
'next': job.next_run_time,
Expand Down

0 comments on commit 49bcb19

Please sign in to comment.