diff --git a/README.rst b/README.rst index 9e5e9d4..36d6b1c 100644 --- a/README.rst +++ b/README.rst @@ -23,82 +23,86 @@ Installation **Usage:** -First you need to login into your account +First you need to login into your Syncano account :: syncano login -It will ask you for your email and password and store account key in -${HOME}/.syncano file. You can also override account key with --key option. +It will ask you for your email and password. After successfully logging in your Account Key (admin key) +will be stored in *${HOME}/.syncano* file. You can also override an Account Key later with *--key* option. -You can specify the default instance name - that would be used in all CLI calls:: +You can specify the default instance name that will be used in all consecutive CLI calls:: syncano login --instance-name patient-resonance-4283 -After when you want to override this setting use --instance-name in particular command, eg:: +If you want to override this setting for a specific command, use --instance-name eg:: syncano sync pull --instance-name new-instance-1234 +Documentation +------------- + +You can read detailed documentation `here `_. + Pulling your instance classes and scripts ----------------------------------------- -In order to pull your instance configuration you can execute +In order to pull your instance configuration, execute :: syncano sync pull -This will fetch all Classes and Scripts into current working directory, and -store configuration in syncano.yml file. If you want to pull only selected -classes/scripts you can add -c/--class or -s/--script option eg. +This will fetch all Classes and Scripts into the current working directory, and +store configuration in *syncano.yml* file. If you want to pull only selected +classes/scripts you can add *-c/--class* or *-s/--script* option e.g. :: syncano sync pull -c Class1 -c Class2 -s script_label_1 -s "script label 2" -Scripts source code is stored in scripts subdirectory, and names are based on -script labels. Keep in mind that script labels in syncano are not unique, and -this tools can't handle this kind of situation. +The Scripts' source code is stored in the scripts subdirectory. Their names are based on +script labels. Keep in mind that script labels in Syncano are not unique, and +this tool cannot yet handle this kind of situation when pulling a Script from Syncano. -Classes and Scripts configuration is stored in syncano.yml file. If file -syncano.yml already exists only classes and scripts stored in this file, will -be pulled and updated. If you want to pull whole instance you can use -a/--all -flag. +Classes and Scripts configuration is stored in *syncano.yml* file. If this file already +exists, only classes and scripts stored in this file will be pulled and updated. +If you want to pull the whole instance you can use *-a/--all* switch flag. Pushing your changes -------------------- -When you have made some changes to syncano.yml or some script source code you -can push the changes to syncano using +After you have made changes to *syncano.yml* or any of the script's source code, +you can push the changes to Syncano using :: syncano sync push -It will push only changes newer then last synchronization time. This time is -recorded using .sync file last modification time. If syncano.yml has changed -it will try to push all data to syncano. Otherwise it will just push changed -source code files for scripts. If you want to force push all changes you can -use -a/--all option. +It will push only changes newer than the last synchronization time. +As last synchronization time we use *.sync* file last modification time. +If *syncano.yml* has changed, it will try to push all data to Syncano. Otherwise, +it will just push the source code files for scripts that were changed. +If you want to force push all changes you can use *-a/--all* option. -If you want to just push selected classes/scripts changes you can provide them -with -c/--class or -s/--script options like in the pull example above. +If you only want to push changes from selected classes/scripts you can provide them +with *-c/--class* or *-s/--script* options like in the pull example above. -Synchronization of changes in realtime +Synchronize changes in real-time -------------------------------------- -There is also an option to synchronize your project live. When you change -syncano.yml or some script source code pointed to by syncano.yml your changes -will be automatically pushed to syncano. +There is an option to synchronize your project in real-time. When you change +syncano.yml or the source code of a script described in *syncano.yml*, your changes +will be automatically pushed to Syncano. :: syncano sync watch -This command will push all your project configuration to syncano and will -wait for changes made to project files. When it detects file modification -it will push those changes to syncano. +This command will push all of your project's configuration to Syncano and will +wait for changes made to project files. When it detects that any file was modified, +it will push those changes to Syncano. Syncano Parse migration tool @@ -108,8 +112,8 @@ This tool will help you to move your data from Parse to Syncano. **Usage:** -Currently supports only transferring data. This tool takes the Parse schemas and transform them to Syncano classes. -Next step is to move all of the data between Parse and Syncano. The last step is rebuilding the relations between +Currently supports only transferring data. This tool takes the Parse schemas and transforms them to Syncano classes. +The next step is to move all of the data between Parse and Syncano. The last step is rebuilding the relations between objects. @@ -132,7 +136,7 @@ Will run the configuration that will ask you for the following variables: * -c (--current) which will display the current configuration; * -f (--force) which allow to override the previously set configuration; -The configuration will be stored in your home directory in .syncano file under the P2S section. +The configuration will be stored in your home directory in the .syncano file under the P2S section. It's used to call the Parse API and Syncano API as well. Run migration @@ -142,50 +146,102 @@ Run migration syncano migrate parse -This command will run the synchronization process between Parse and Syncano. Sit comfortably in your chair and read +This command will run the synchronization process between Parse and Syncano. Sit back, relax, and read the output. +Tips & Troubleshooting +---------------------- + +1. This tool currently does not support checking if an object is already present in the Syncano instance. + If the sync is run twice, the data will be duplicated. To avoid this, + simply remove your instance using Syncano dashboard; + +2. The whole process can be quite slow because of the throttling on both sides: Parse and Syncano on free trial accounts (which is the bottom boundary for scripts); + Syncano Hosting =============== -Syncano Hosting is a simple way to host the static files. CLI supports it in the following way: +Syncano Hosting is a simple way to host your static files on Syncano servers. +The CLI supports it in the following way: -:: +This command will list files for currently hosted website:: - syncano hosting --list-files + syncano hosting list -This command will list files in hosting which match the default hosting. +This command will publish all files inside ** to the default Syncano Hosting instance. +When publishing the whole directory, the structure will be mapped on Syncano.:: -:: + syncano hosting publish - syncano hosting --publish +This command will unpublish currently published hosting:: -This command will publish all files inside and will publish it to the Syncano Hosting (default one). -The whole directory structure - will be mapped in Syncano Hosting. + syncano hosting unpublish -Tips & Troubleshooting ----------------------- +This command will permamently delete the hosting:: -1. This tool currently does not support checking if some object is already present in the Syncano instance, - so if sync is run twice the end results is that data is duplicated. To avoid such cases, - simply remove your instance in using Syncano dashboard; + syncano hosting delete -2. The process can be quite slow - it's because of the throttling on both sides: Parse and Syncano on free accounts - (which is the bottom boundary for scripts); +This command will delete the specified file:: -3. If you encounter any problems, have some improvements proposal or just wanna talk, - please write me: sebastian.opalczynski@syncano.com; + syncano hosting delete hosting/file/path -4. The Syncano can be found on - please do not hesitate to ask for help or share your thoughts; +This command will update single file:: -* Github: - * https://github.com/Syncano/ -* Gitter: - * https://gitter.im/Syncano/community - * https://gitter.im/Syncano/community-pl -* Slack: - * http://syncano-community.github.io/slack-invite/ + syncano hosting update hosting/file/path local/file/path + +Custom Sockets +-------------- + +This is a list of commands available for Custom Sockets. +If you want to know more about Custom Sockets, `read the detailed docs here `_. + +To install a Custom Socket from a local file:: + + syncano sockets install /path/to/dir + +To install a Custom Socket from a URL:: + + syncano sockets install https://web.path.to/your.file + +List all Custom Sockets:: + + syncano sockets list + +List all defined endpoints (for all Custom Sockets):: + + syncano sockets list endpoints + +Display chosen Custom Socket details:: + + syncano sockets details socket_name + +Delete a Custom Socket:: + + syncano sockets delete socket_name + +Create a template from a template stored in Syncano CLI:: + + syncano sockets template /path/to/output_dir + +Create a template from an existing Custom Socket:: + + syncano sockets template /path/to/out --socket socket_name + +Run endpoint defined in Custom Socket::s + + syncano sockets run socket_name/endpoint_name + +Run endpoint providing POST data:: + + syncano sockets run socket_name/my_endpoint_12 POST --data '{"one": 1}' + +In all of the above cases you can override the Syncano instance being used:: + + --instance-name my_instance_name + +Providing the instance name this way will override the default instance name +defined during initial setup (*syncano login --instance-name my_instance*) Running scripts @@ -198,3 +254,22 @@ This command will allow you to execute any script (Script Endpoint) with optiona :: syncano execute --payload="" + + +Issues +======== + +1. If you encounter any problems, have some improvement ideas or just wanna talk, + please write to me directly: sebastian.opalczynski@syncano.com; + +2. Syncano team can be reached in multiple ways. Please do not hesitate to ask for help or share your thoughts. You can find us on: + +* Github: + * https://github.com/Syncano/ +* Slack: + * http://syncano-community.github.io/slack-invite/ +* Gitter: + * https://gitter.im/Syncano/community + * https://gitter.im/Syncano/community-pl +* Support e-mail: + * `support@syncano.io `_ diff --git a/docs/README.md b/docs/README.md index 17a8002..5a41959 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,123 +1,17 @@ -# How does the data transfer from Parse to Syncano work? +# Syncano CLI documentation -## Overview +## Migrations -The provided tool uses both Syncano and Parse API. How does it work? +Migration allows you to transfer data from Parse to Syncano. Read the following guide carefully, to understand +what can be transferred from Parse to Syncano, and what the limitations are. -1. Parse schemas are fetched and transferred to Syncano as classes. All field types are supported. -2. Data is transferred for each schema/class. This can be divided into following steps: +[Migrations](migrations/docs.md) - 1. The call to the Parse API is made - to obtain 1000 objects of particular class; - 2. The conversion is made - the Object from Parse is transformed to the Syncano Data Object; - 3. The batch API call is made to the Syncano API - with 50 elements (API limitations); - 4. All the object relations are restored. +## Custom Sockets -Ad iv) This is the most challenging task. During the first and second step, all the information about relations are saved. The map of references are built -- which binds Parse objects with Syncano Data Objects -- and based on that the relations are restored. It's a time consuming process due to Parse API limitations and Syncano throttling functionality. More information on that can be found in the [data transfer](#data-transfer) section below. +Custom Sockets are a powerful Syncano feature, which allows you to create your own API. +They can be used to provide an integration with other applications (e.g. sending emails with SendGrid) or to define custom behavior for your endpoints e.g. for password reset. +Sockets are built on the top of other Syncano solutions - like Script Endpoints. -### Schemas to classes transfer +[Custom Sockets](custom_sockets/docs.md) -**Name normalization** - -For each Parse schema, the Syncano Class schema is created. The class name and name of all fields are normalized. -The normalization process usually just makes a lowercase name. So when you transfer schema from Parse, where class name name is `SomeOutstandingClass`, in Syncano it will be called: `someoutstandingclass`. -The same applies to the fields. - -There are some exceptions. If Parse schema name starts from an underscore: `_`, the name will be changed to `internal_`. That's because Syncano does not support names which start with an underscore. - -**Created at and updated at** - -Syncano classes have two special fields `created_at` and `updated_at`, which store the dates indicating when object was created and last updated. The Parse has fields `createdAt` and `updatedAt` - which are used for the same purpose. - -Syncano `created_at` and `updated_at` fields are read-only, so it's impossible to transfer there values from Parse. -To resolve this, two additional fields are created on Syncano class: `original_createdat` and `original_updatedat` - -which have filter and order index added to them and can be used to find and sort data easily. -These fields store the original creation and update date from Parse (but update date will **NOT** be updated automatically next time). - -**objectId field** - -Syncano and Parse use different methods of object identification. Syncano uses integer ID field, Parse uses string ID field. -That means that simple one-to-one conversion isn't possible. When creating the Syncano schema, additional 'objectid' field is made, which stores the Parse string ID. This field has a filter index added to it, so can be used for filtering data in the Syncano class. - -**ACL** - -The ACL system in both Parse and Syncano is widely different - transferring ACL is **NOT** supported yet. -Currently we also are not able to provide any deadline when support for it will be added. - -**Field mapping** - -For the field types, the following map is used - the key is the Parse field type, the value is the Syncano field type: - -```python -class ClassProcessor(object): - map = { - 'Number': 'integer', - 'Date': 'datetime', - 'Boolean': 'boolean', - 'String': 'string', - 'Array': 'array', - 'Object': 'object', - 'Pointer': 'reference', - 'File': 'file', - 'GeoPoint': 'geopoint', - 'Relation': 'relation', - } -``` - -### Data transfer - -When Parse schemas are transferred as Syncano classes - the data migration process start. - -Class by class is processed: -* get the 1000 objects from Parse (max value of `limit` parameter) -* translate Parse objects to Syncano objects (and make a batch call with 10 Syncano Data Object, to avoid throttling); - -During this process for each class the reference map is made. This reference map stores the class name and connects Parse Object ID with Syncano Object ID. - -It is possible that this structure will use a lot of your local machine memory. - -Last step of data transfer is transferring the files. In previous steps we use batch calls when translating Parse objects into Syncano objects. Unfortunately it's not possible to send files using batch calls in Syncano, so during initial transfer files are not being moved. -What we do here, is we download those files localy to your machine and then we attach them to the right Syncano objects. As a result - all your Parse objects with files end up being Syncano objects with files with a bonus of files are now being stored on Syncano servers. - -### Relations rebuild - -For each Parse object that has a relation field a query is made -- to obtain the related objects. -Then those related objects are added to the Syncano Data Object. Relations are transformed as a whole. In Syncano, relation fields store IDs of the related objects like this: - -```json -books=[1, 2, 3] -``` - -which means that related books are those with ids: 1, 2 and 3. This is why the relations rebuilding process is the last one in the queue. - -Information about Syncano IDs must be known beforehand - we can't make a relation knowing only Parse IDs. - -Whole process can be very time consuming. First, because there's a need to query each parse object with relations about their related objects and second, because the Syncano free Builder account will throttles all requests when 15 request per second limit is reached. - -## Limitations - -The main limitations are: - -1. Syncano free Builder account throttling: 15 requests per second. -2. Parse `limit` parameter max value equal to 1000. -3. Parse API calls for obtaining the related objects - can not be obtained as a large set - only one by one. - -## Things you should know - -### Parse - -The only operations that are made to Parse API are **GET** calls. Your Parse Application is not affected after or -during the process of the data transfer. - -**Master Key** - -Your Parse Master Key - which is required by this tool - is stored locally on your machine, under the home directory in -`.syncano` file. It is used **ONLY** for communication with Parse. - -### Syncano - -Each time the transfer is run - data will be duplicated. It's because Syncano does not support the unique constraint - and -thus it's impossible to check if object with particular Parse ID is already present in the Syncano Data Objects. Before -re-running the data import process, it's a good idea to remove your instance or already trasferred classes, or use a new one in our tool configuration. - -Syncano credentials are stored in `.syncano` file under your home directory, and are used **ONLY** for communication -with Syncano services. diff --git a/docs/custom_sockets/docs.md b/docs/custom_sockets/docs.md new file mode 100644 index 0000000..f0996fc --- /dev/null +++ b/docs/custom_sockets/docs.md @@ -0,0 +1,117 @@ +# Syncano Custom Sockets + +## YAML file structure (socket.yml) + + name: my_integration + description: Example integration + author: + name: Sebastian + email: sebastian@syncano.com + icon: + name: icon_name + color: red + endpoints: + my_endpoint_1: + script: script_endpoint_1 + + my_endpoint_2: + POST: + script: script_endpoint_2 + GET: + script: script_endpoint_3 + + dependencies: + scripts: + script_endpoint_1: + runtime_name: python_library_v5.0 + file: scripts/script1.py + + script_endpoint_2: + runtime_name: python_library_v5.0 + file: scripts/script2.py + + script_endpoint_3: + runtime_name: python_library_v5.0 + file: scripts/script3.py + +### YAML file structure explanation + +* `name` is the name of your new Custom Socket - this should be unique; +* `description` is a description of your Custom Socket - it allows you to easily identify what your custom socket does; +* `author` is metadata information about the Custom Socket author; Under the hood: all fields that are not + * `name`, + * `description`, + * `endpoints`, + * `dependencies` + * can be found in `metadata` field on Custom Socket in Syncano Dasboard. +* `icon` is metadata information about your Custom Socket - it stores the icon name used and its color (used in Syncano Dashboard) +* `endpoints` - definition of the endpoints in a Custom Socket; Currently supported endpoints can be only of `script` type. + + Consider this example: + + my_endpoint_1: + script: script_endpoint_1 + + In the YAML snippet: + * `my_endpoint_1` is an endpoint name - it will be used in the url path; + * `script` is a type of the endpoint; + * `script_endpoint_1` is the dependency name which will be called when endpoint + `my_endpoint_1` will be requested; + + In the example above we didn't specify HTTP method - so no matter what HTTP method + is used to call `my_endpoint_1` (can be PATCH, POST, GET, etc.), script endpoint `script_endpoint_1` will be called; + + Consider yet another example: + + my_endpoint_2: + POST: + script: script_endpoint_2 + GET: + script: script_endpoint_3 + + In the above YAML snippet: + * `my_endpoint_2` is an endpoint name. + * GET, POST - define HTTP method type used to call our endpoint + + The difference is that we now define what happens for the different HTTP methods. When the GET HTTP method is used, + `script_endpoint_3` script endpoint will be run. When the POST HTTP method is used - `script_endpoint_2` endpoint will be executed. + + Currently only Script Endpoints are supported, which run scripts under the hood. But don't worry, + we are working on adding more options! + +* `dependencies` - the definition of your Custom Socket dependencies. They define all dependency objects +which will be called when the endpoint is requested. + + Consider the example: + + dependencies: + scripts: + script_endpoint_1: + runtime_name: python_library_v5.0 + file: scripts/script1.py + + Above YAML snippet defines one dependency: + * `script` - type of the dependency (defined using `scripts` keyword). + * `script_endpoint_1` - name of the dependency; it's an important element, because that's the place where you connect a dependency to an endpoint. + * `runtime_name` is name of the runtime used in a script; + * `file` stores the source code that will be executed. + + It should be noted that when defining Custom Scripts, we suggest following some basic directory structure- for + better work organization. We recommend storing scripts under the `scripts` directory - this is why the filename + is a relative path: `scripts/script1.py`. Of course your can also follow your own rules, e.g. by using a flat file structure. + + +## Custom Socket directory structure + +Below is a sample Custom Socket structure for the above YAML definition: + +![](images/tree_socket.png) + +`socket.yml` file stores the YAML definition mentioned above; `scripts` directory stores all scripts source +code used in `script` dependency type. + + +## Custom Socket examples + +* ['HelloWorld' example](examples/hello_world.md) +* [Advanced example: provide title here](examples/advanced.md) diff --git a/docs/custom_sockets/examples/advanced.md b/docs/custom_sockets/examples/advanced.md new file mode 100644 index 0000000..2eba3eb --- /dev/null +++ b/docs/custom_sockets/examples/advanced.md @@ -0,0 +1,286 @@ +# Advanced Custom Socket example + +## Abstract + +This in an advanced example of Syncano Custom Sockets, showing an integration with Mailgun (https://mailgun.com/). + +We will use Mailgun's sandbox environment, which allows sending up to 300 emails per day - it should be more than enough for testing. + +We will implement two endpoints inside the Custom Socket - one for sending emails, and a second one to obtain some basic statistics. + +## Repository link + +The whole example can be found under: [Syncano/custom-socket-advanced-example](https://github.com/Syncano/custom-socket-advanced-example). +It can be installed to your Syncano instance using the `install from url` functionality in the CLI. The url is: + +https://github.com/Syncano/custom-socket-advanced-example/blob/master/socket.yml + +## Prerequisites + +* Syncano Account - [Create one here](https://www.syncano.io/). +* GIT - If you want to edit files locally, clone our repository using: + +```bash +git clone git@github.com:Syncano/custom-socket-advanced-example.git` +``` + +* Syncano [CLI tool](https://pypi.python.org/pypi/syncano-cli/0.5) in version 0.5 or higher. + + > Note: + > It is nice to use virtualenv to separate your tools: `sudo pip install virtualenv` + > Then create virtual env: `virtualenv cli` and active it: `source cli/bin/activate` + > Install Syncano CLI: `pip install syncano_cli>=0.5` + +* Your favorite text editor. + +## YML definition + + name: mailgun_integration + description: An advanced example of Custom Socket - mailgun integration. + author: + name: Info at Syncano + email: info@syncano.com + endpoints: + send_mail: + POST: + script: send_mail + + get_stats: + GET: + script: get_stats + + dependencies: + scripts: + send_mail: + runtime_name: python_library_v5.0 + file: scripts/send_mail.py + + get_stats: + runtime_name: python_library_v5.0 + file: scripts/get_stats.py + +Above YAML file defines one Custom Socket with two endpoints: +* `send_mail` for sending emails, it's run on POST HTTP method call; we want to pass some basic information about who it should be sent to, what subject should be used and what text should be in the email itself; +* `get_stats` - second endpoint is for obtaining basic stats from Mailgun service. + +There are also two `script` dependencies defined. + +## Scripts definition + +### scripts/send_mail.py + + import requests + import json + + api_key = CONFIG.get('mailgun_api_key') + + to_email = ARGS.get('to_email') + subject = ARGS.get('subject') + email_body = ARGS.get('email_body') + + response = requests.post( + "https://api.mailgun.net/v3/sandboxa8ccfb01296d4b19bace47fb8102d130.mailgun.org/messages", + auth=("api", api_key), + data={ + "from": "Mailgun Sandbox ", + "to": to_email, + "subject": subject, + "text": email_body + } + ) + + if response.status_code == 200: + success_content = json.dumps( + { + 'status_code': 200, + 'info': u'Mail successfully sent to {}'.format(to_email) + } + ) + set_response(HttpResponse(status_code=200, content=success_content, content_type='application/json')) + else: + fail_content = json.dumps( + { + 'status_code': response.status_code, + 'info': response.text + } + ) + set_response(HttpResponse(status_code=400, content=fail_content, content_type='application/json')) + +Script above will send a request to Mailgun service - and this service will send an email to a user. +It's worth noting the `CONFIG` variable - it's an Instance global config dictionary/map - you can define its content inside +Syncano Dasboard under `Global Config` menu on the left, or using Syncano Libraries - more about it can be found +[in our docs](http://docs.syncano.io/docs/snippets-scripts#section-global-config-dictionary). + +### scripts/get_stats.py + + import requests + + api_key = CONFIG.get('mailgun_api_key') + + response = requests.get( + "https://api.mailgun.net/v3/sandboxa8ccfb01296d4b19bace47fb8102d130.mailgun.org/stats/total", + auth=("api", api_key), + params={ + "event": ["accepted", "delivered", "failed"], + "duration": "1m"} + ) + + if response.status_code == 200: + set_response(HttpResponse(status_code=200, content=response.text, content_type='application/json')) + else: + set_response(HttpResponse(status_code=400, content=response.text, content_type='application/json')) + +Above script will work as a proxy to the Mailgun service - will send a request about stats and pass the result as +a response. + + +## Custom Socket directory structure + +The directory structure in my favourite editor looks like this: + +![](../images/project_struct_adv.png) + +or in tree format: + + . + ├── scripts + │   ├── get_stats.py + │   └── send_mail.py + └── socket.yml + + +## Putting everything together + +### Steps: + +1. Assuming that you have Syncano CLI installed, log in using: `syncano login --instance-name your-instance-name` + In my case it is: + + syncano login --instance-name patient-resonance-4283 + + Next you will see a prompt for `username` and `password`; provide both and confirm with `enter`. + +2. There are two ways of installing a Custom Socket - one is using your local files and the second one is by using a URL. + + To install the Custom Socket from a URL do: + + syncano sockets install https://raw.githubusercontent.com/Syncano/custom-socket-advanced-example/master/socket.yml --name mailgun_integration + + In such scenario - you do not even need to clone the repository to your local machine. The `--name` parameter and name here are needed - because under the hood, empty Custom Socket is created - and code fetching from repository is done asynchronously in the second step. + + To install Custom Socket from local files do: + + syncano sockets install + + In my case it is: + + syncano sockets install ../syncano_scripts/repos/custom-socket-advanced-example/ + +3. Try a newly created Custom Socket: + + To list Custom Sockets, do: + + syncano sockets list + + In the output you should find: + + - socket: + info: '' + name: mailgun_integration + status: ok + + This means that Custom Socket `mailgun_integration` was created successfuly - the status is `ok`. In any other case you will see an `error` and detailed information in `info` about what went wrong. + + Now, list all defined endpoints: + + syncano sockets list endpoints + + In the output you should find: + + - endpoint: + methods: + - GET + name: mailgun_integration/get_stats + path: /v1.1/instances/patient-resonance-4283/endpoints/sockets/mailgun_integration/get_stats/ + - endpoint: + methods: + - POST + name: mailgun_integration/send_mail + path: /v1.1/instances/patient-resonance-4283/endpoints/sockets/mailgun_integration/send_mail/ + +4. Before you run an endpoint - be sure that you have Mailgun `api_key` inside your Instance Global Config. See the +`send_mail.py` description for more details. My config looks like this: + + {"mailgun_api_key": "key-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"} + + Of course you need to replace `key-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx` with your Mailgun API Key. + + +5. Run the endpoint defined in your Custom Socket: + + First, run the stats endpoint - it's easier, as it is a simple GET request without any arguments required. + + syncano sockets run mailgun_integration/get_stats + + In the output you should see something like this (probably not formatted): + + { + "stats": [ + { + "delivered": { + "smtp": 5, + "total": 5, + "http": 0 + }, + "accepted": { + "outgoing": 5, + "total": 5, + "incoming": 0 + }, + "time": "Mon, 01 Aug 2016 00:00:00 UTC", + "failed": { + "permanent": { + "suppress-complaint": 0, + "suppress-bounce": 0, + "total": 0, + "bounce": 0, + "suppress-unsubscribe": 0 + }, + "temporary": { + "espblock": 0 + } + } + } + ], + "end": "Mon, 01 Aug 2016 00:00:00 UTC", + "resolution": "month", + "start": "Mon, 01 Aug 2016 00:00:00 UTC" + } + + Above response is one-to-one to the response provided by Mailgun. + + Now let's send an e-mail! + + Run: + + syncano sockets run mailgun_integration/send_mail POST --data '{"subject": "CustomSocket MailGun test", "to_email": "FirstName LastName ", "email_body": "So nice to create Custom Sockets!"}' + + Do not forget to change e-mail address in JSON data. + + It should return: + + { + u'info': u'Mail successfully sent to {to_email_value}', + u'status_code': 200 + } + + You can now call stats again and see if anything changed. + +6. To delete mailgun Custom Socket do: + + syncano sockets delete mailgun_integration + +## Summary + +Hope this was helpful! If you have any question or issues, do no hesitate to contact me directly: sebastian.opalczynski@syncano.com +I am also available on the [Syncano Slack community](http://syncano-community.github.io/slack-invite/). See you there! diff --git a/docs/custom_sockets/examples/hello_world.md b/docs/custom_sockets/examples/hello_world.md new file mode 100644 index 0000000..d806c3a --- /dev/null +++ b/docs/custom_sockets/examples/hello_world.md @@ -0,0 +1,183 @@ +# HelloWorld example + +## Abstract + +In this example we will create a simple Custom Socket. The idea here is to create an endpoint which will return +a `Hello world` message. + +## Repository link + +The whole example can be found under: [Syncano/custom-socket-hello-world](https://github.com/Syncano/custom-socket-hello-world) +It's possible to install it to Syncano Instance using `install from url` functionality in CLI. The URL is: + +`https://github.com/Syncano/custom-socket-hello-world/blob/master/socket.yml` + +## Prerequisites + +* Syncano Account - [Create one here](https://www.syncano.io/). +* GIT - If you want to edit files locally, clone our repository using: +```bash +git clone git@github.com:Syncano/custom-socket-hello-world.git +```` +* Syncano [CLI tool](https://pypi.python.org/pypi/syncano-cli/0.5) in version 0.5 or higher. + + > Note: + > It is nice to use virtualenv to separate your tools: `sudo pip install virtualenv` + > Then create virtual env: `virtualenv cli` and active it: `source cli/bin/activate` + > Install Syncano CLI: `pip install syncano_cli>=0.5` + +4. Your favorite text editor. + +## YML definition + + name: hello_world + author: + name: Info + email: info@synano.com + description: Hello World example + endpoints: + hello_endpoint: + script: hello_world + + dependencies: + scripts: + hello_world: + runtime_name: python_library_v5.0 + file: scripts/hello_world.py + +Above YAML file defines one Custom Socket with one endpoint: +* `hello_endpoint` for printing hello world on every HTTP method call. + +There is also one `script` dependency defined, to `hello_world` script. + +In my favorite editor the project look as follows: + +![](../images/project_struct.png) + +## Scripts definition + +The script (`scripts/hello_world.py`) consists of a few lines: + + content = """ + + + + Hello world! + + + + Hello World! + + + """ + + set_response(HttpResponse(status_code=200, content=content, content_type='text/html')) + +The above code executed in Syncano will return a valid HTML response with the `Hello World!` message inside. +The `set_response` is a function which returns a custom response (e.g. in JSON, CSV or HTML format) from the script. + +## Custom Socket directory structure + +As can be seen in the example above, the basic structure of this Custom Socket is: + + . + ├── scripts + │   └── hello_world.py + └── socket.yml + +`socket.yml` file stores YAML definition of the Custom Socket, and under `scripts` directory there is a definition +of Custom Socket dependencies (currently of type `script`). + +## Putting everything together + +### Steps: + +1. Assuming that you have Syncano CLI installed, log in using: `syncano login --instance-name your-instance-name` + In my case it is: + + syncano login --instance-name patient-resonance-4283 + + Next you will see a prompt for `username` and `password`; provide both and confirm with `enter`. + +2. There are two ways of installing a Custom Socket - one is using your local files and the second one is by using a URL. + + To install the Custom Socket from url do: + + syncano sockets install https://raw.githubusercontent.com/Syncano/custom-socket-hello-world/master/socket.yml --name hello_world + + In this scenario - you do not even need to clone the repository to your local machine. The `--name` parameter and name here are needed - because under the hood, an empty Custom Socket is created - and fetching code from the repository is done asynchronously in the second step. + + To install Custom Socket from local files do: + + syncano sockets install + + In my case it is: + + syncano sockets install ../syncano_scripts/repos/custom-socket-hello-world/ + + So you need to point to the parent directory of your `socket.yml` definition. + +3. Try a newly created Custom Socket: + + To list Custom Sockets, do: + + syncano sockets list + + In the output you should find: + + - socket: + info: '' + name: hello_world + status: ok + + This means that Custom Socket `hello_world` was created successfuly - the status is `ok`. In any other case you will see here an `error` and detailed information in `info` about what went wrong. + + Now, list all defined endpoints: + + syncano sockets list endpoints + + In the output you should find: + + - endpoint: + methods: + - POST + - PUT + - PATCH + - GET + - DELETE + name: hello_world/hello_endpoint + path: /v1.1/instances/your-instance-name/endpoints/sockets/hello_world/hello_endpoint/ + +4. Run the endpoint defined in Custom Socket: + + syncano sockets run hello_world/hello_endpoint + + You should see in the output raw html file: + + + + + Hello world! + + + + Hello World! + + + + Lets see if output can be seen in the browser: + Go to: `https://api.syncano.io/v1.1/instances/your-instance-name/endpoints/sockets/hello_world/hello_endpoint/?api_key=your_api_key` + We defined the endpoint to handle GET HTTP method. + + In my case: + + ![](../images/hello_world.png) + +5. To delete Custom Socket do: + + syncano sockets delete hello_world + +## Summary + +Hope this was helpful! If you have any question or issues, do no hesitate to contact me directly: sebastian.opalczynski@syncano.com +I am also available on the [Syncano Slack community](http://syncano-community.github.io/slack-invite/). See you there! diff --git a/docs/custom_sockets/images/hello_world.png b/docs/custom_sockets/images/hello_world.png new file mode 100644 index 0000000..54ceea3 Binary files /dev/null and b/docs/custom_sockets/images/hello_world.png differ diff --git a/docs/custom_sockets/images/project_struct.png b/docs/custom_sockets/images/project_struct.png new file mode 100644 index 0000000..1b4414b Binary files /dev/null and b/docs/custom_sockets/images/project_struct.png differ diff --git a/docs/custom_sockets/images/project_struct_adv.png b/docs/custom_sockets/images/project_struct_adv.png new file mode 100644 index 0000000..2866f53 Binary files /dev/null and b/docs/custom_sockets/images/project_struct_adv.png differ diff --git a/docs/custom_sockets/images/tree_socket.png b/docs/custom_sockets/images/tree_socket.png new file mode 100644 index 0000000..bf566da Binary files /dev/null and b/docs/custom_sockets/images/tree_socket.png differ diff --git a/docs/migrations/docs.md b/docs/migrations/docs.md new file mode 100644 index 0000000..17a8002 --- /dev/null +++ b/docs/migrations/docs.md @@ -0,0 +1,123 @@ +# How does the data transfer from Parse to Syncano work? + +## Overview + +The provided tool uses both Syncano and Parse API. How does it work? + +1. Parse schemas are fetched and transferred to Syncano as classes. All field types are supported. +2. Data is transferred for each schema/class. This can be divided into following steps: + + 1. The call to the Parse API is made - to obtain 1000 objects of particular class; + 2. The conversion is made - the Object from Parse is transformed to the Syncano Data Object; + 3. The batch API call is made to the Syncano API - with 50 elements (API limitations); + 4. All the object relations are restored. + +Ad iv) This is the most challenging task. During the first and second step, all the information about relations are saved. The map of references are built -- which binds Parse objects with Syncano Data Objects -- and based on that the relations are restored. It's a time consuming process due to Parse API limitations and Syncano throttling functionality. More information on that can be found in the [data transfer](#data-transfer) section below. + +### Schemas to classes transfer + +**Name normalization** + +For each Parse schema, the Syncano Class schema is created. The class name and name of all fields are normalized. +The normalization process usually just makes a lowercase name. So when you transfer schema from Parse, where class name name is `SomeOutstandingClass`, in Syncano it will be called: `someoutstandingclass`. +The same applies to the fields. + +There are some exceptions. If Parse schema name starts from an underscore: `_`, the name will be changed to `internal_`. That's because Syncano does not support names which start with an underscore. + +**Created at and updated at** + +Syncano classes have two special fields `created_at` and `updated_at`, which store the dates indicating when object was created and last updated. The Parse has fields `createdAt` and `updatedAt` - which are used for the same purpose. + +Syncano `created_at` and `updated_at` fields are read-only, so it's impossible to transfer there values from Parse. +To resolve this, two additional fields are created on Syncano class: `original_createdat` and `original_updatedat` - +which have filter and order index added to them and can be used to find and sort data easily. +These fields store the original creation and update date from Parse (but update date will **NOT** be updated automatically next time). + +**objectId field** + +Syncano and Parse use different methods of object identification. Syncano uses integer ID field, Parse uses string ID field. +That means that simple one-to-one conversion isn't possible. When creating the Syncano schema, additional 'objectid' field is made, which stores the Parse string ID. This field has a filter index added to it, so can be used for filtering data in the Syncano class. + +**ACL** + +The ACL system in both Parse and Syncano is widely different - transferring ACL is **NOT** supported yet. +Currently we also are not able to provide any deadline when support for it will be added. + +**Field mapping** + +For the field types, the following map is used - the key is the Parse field type, the value is the Syncano field type: + +```python +class ClassProcessor(object): + map = { + 'Number': 'integer', + 'Date': 'datetime', + 'Boolean': 'boolean', + 'String': 'string', + 'Array': 'array', + 'Object': 'object', + 'Pointer': 'reference', + 'File': 'file', + 'GeoPoint': 'geopoint', + 'Relation': 'relation', + } +``` + +### Data transfer + +When Parse schemas are transferred as Syncano classes - the data migration process start. + +Class by class is processed: +* get the 1000 objects from Parse (max value of `limit` parameter) +* translate Parse objects to Syncano objects (and make a batch call with 10 Syncano Data Object, to avoid throttling); + +During this process for each class the reference map is made. This reference map stores the class name and connects Parse Object ID with Syncano Object ID. + +It is possible that this structure will use a lot of your local machine memory. + +Last step of data transfer is transferring the files. In previous steps we use batch calls when translating Parse objects into Syncano objects. Unfortunately it's not possible to send files using batch calls in Syncano, so during initial transfer files are not being moved. +What we do here, is we download those files localy to your machine and then we attach them to the right Syncano objects. As a result - all your Parse objects with files end up being Syncano objects with files with a bonus of files are now being stored on Syncano servers. + +### Relations rebuild + +For each Parse object that has a relation field a query is made -- to obtain the related objects. +Then those related objects are added to the Syncano Data Object. Relations are transformed as a whole. In Syncano, relation fields store IDs of the related objects like this: + +```json +books=[1, 2, 3] +``` + +which means that related books are those with ids: 1, 2 and 3. This is why the relations rebuilding process is the last one in the queue. + +Information about Syncano IDs must be known beforehand - we can't make a relation knowing only Parse IDs. + +Whole process can be very time consuming. First, because there's a need to query each parse object with relations about their related objects and second, because the Syncano free Builder account will throttles all requests when 15 request per second limit is reached. + +## Limitations + +The main limitations are: + +1. Syncano free Builder account throttling: 15 requests per second. +2. Parse `limit` parameter max value equal to 1000. +3. Parse API calls for obtaining the related objects - can not be obtained as a large set - only one by one. + +## Things you should know + +### Parse + +The only operations that are made to Parse API are **GET** calls. Your Parse Application is not affected after or +during the process of the data transfer. + +**Master Key** + +Your Parse Master Key - which is required by this tool - is stored locally on your machine, under the home directory in +`.syncano` file. It is used **ONLY** for communication with Parse. + +### Syncano + +Each time the transfer is run - data will be duplicated. It's because Syncano does not support the unique constraint - and +thus it's impossible to check if object with particular Parse ID is already present in the Syncano Data Objects. Before +re-running the data import process, it's a good idea to remove your instance or already trasferred classes, or use a new one in our tool configuration. + +Syncano credentials are stored in `.syncano` file under your home directory, and are used **ONLY** for communication +with Syncano services. diff --git a/setup.py b/setup.py index c8dfaad..7e31d14 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name='syncano-cli', - version='0.5', + version='0.6', description='Syncano command line utilities', long_description=README, author='Marcin Swiderski, Sebastian Opalczynski', @@ -14,7 +14,7 @@ url='https://github.com/Syncano/syncano-cli', packages=find_packages(), license='MIT', - install_requires=['syncano>=5.4.0', 'PyYaml>=3.11', 'watchdog>=0.8.3', 'click>=6.6'], + install_requires=['syncano>=5.4.1', 'PyYaml>=3.11', 'watchdog>=0.8.3', 'click>=6.6'], test_suite='tests', entry_points=""" [console_scripts] diff --git a/syncano_cli/base/connection.py b/syncano_cli/base/connection.py index 3ad403a..d27ab5a 100644 --- a/syncano_cli/base/connection.py +++ b/syncano_cli/base/connection.py @@ -11,7 +11,13 @@ def create_connection(config): config = config or ACCOUNT_CONFIG_PATH ACCOUNT_CONFIG.read(config) api_key = ACCOUNT_CONFIG.get('DEFAULT', 'key') - return syncano.connect(api_key=api_key) + connection_dict = { + 'api_key': api_key, + } + instance_name = ACCOUNT_CONFIG.get('DEFAULT', 'instance_name', None) + if instance_name: + connection_dict['instance_name'] = instance_name + return syncano.connect(**connection_dict) def get_instance_name(config, instance_name): diff --git a/syncano_cli/commands.py b/syncano_cli/commands.py index b612677..2dfa4ab 100644 --- a/syncano_cli/commands.py +++ b/syncano_cli/commands.py @@ -39,5 +39,6 @@ def login(context, config, instance_name): ACCOUNT_CONFIG.set('DEFAULT', 'instance_name', instance_name) with open(context.obj['config'], 'wb') as fp: ACCOUNT_CONFIG.write(fp) + click.echo("INFO: Login successful.") except SyncanoException as error: print(error) diff --git a/syncano_cli/custom_sockets/command.py b/syncano_cli/custom_sockets/command.py index 36c7707..a407a49 100644 --- a/syncano_cli/custom_sockets/command.py +++ b/syncano_cli/custom_sockets/command.py @@ -32,7 +32,7 @@ def list_endpoints(self): def delete(self, socket_name): custom_socket = CustomSocket.please.get(name=socket_name, instance_name=self.instance.name) custom_socket.delete() - click.echo("INFO: Custom Socket {} delted.".format(socket_name)) + click.echo("INFO: Custom Socket {} deleted.".format(socket_name)) def install_from_dir(self, dir_path): with open(os.path.join(dir_path, self.SOCKET_FILE_NAME)) as socket_file: diff --git a/syncano_cli/custom_sockets/commands.py b/syncano_cli/custom_sockets/commands.py index 42037f9..9bfc8a1 100644 --- a/syncano_cli/custom_sockets/commands.py +++ b/syncano_cli/custom_sockets/commands.py @@ -17,7 +17,7 @@ def top_sockets(): @click.pass_context @click.option('--config', help=u'Account configuration file.') @click.option('--instance-name', help=u'Instance name.') -def sockets(ctx, config, instance_name, **kwargs): +def sockets(ctx, config, instance_name): """ Allow to create a custom socket. """ diff --git a/syncano_cli/hosting/utils.py b/syncano_cli/hosting/command.py similarity index 54% rename from syncano_cli/hosting/utils.py rename to syncano_cli/hosting/command.py index 8ba38e7..0653f1c 100644 --- a/syncano_cli/hosting/utils.py +++ b/syncano_cli/hosting/command.py @@ -17,22 +17,24 @@ def list_hosting(self): def list_hosting_files(self, domain): hosting = self._get_hosting(domain=domain) - if not hosting: - click.echo(u'WARN: No default hosting found. Exit.') - sys.exit(1) - files_list = hosting.list_files() return files_list def publish(self, domain, base_dir): uploaded_files = [] - hosting = self._get_hosting(domain=domain) + hosting = self._get_hosting(domain=domain, is_new=True) + upload_method_name = 'update_file' if not hosting: # create a new hosting if no default is present; hosting = self.create_hosting(label='Default hosting', domain=domain) + upload_method_name = 'upload_file' for folder, subs, files in os.walk(base_dir): - path = folder.split(base_dir)[1][1:] # skip the / + path = folder.split(base_dir)[1] + + if path.startswith('/'): # skip the / + path = path[1:] + for single_file in files: if path: file_path = '{}/{}'.format(path, single_file) @@ -44,11 +46,43 @@ def publish(self, domain, base_dir): sys_path = os.path.join(folder, single_file) with open(sys_path, 'rb') as upload_file: click.echo(u'INFO: Uploading file: {}'.format(file_path)) - hosting.upload_file(path=file_path, file=upload_file) + getattr(hosting, upload_method_name)(path=file_path, file=upload_file) uploaded_files.append(file_path) return uploaded_files + def unpublish(self, domain): + hosting = self._get_hosting(domain=domain) + hosting.domains = ['unpublished'] + hosting.save() + click.echo('INFO: Hosting `{}` unpublished.'.format(hosting.label)) + + def delete_hosting(self, domain, path=None): + hosting = self._get_hosting(domain=domain) + deleted_label = hosting.label + hosting.delete() + click.echo('INFO: Hosting `{}` deleted.'.format(deleted_label)) + + def delete_path(self, domain, path=None): + hosting = self._get_hosting(domain=domain) + hosting_files = hosting.list_files() + is_deleted = False + for hosting_file in hosting_files: + if hosting_file.path == path: + is_deleted = True + hosting_file.delete() + break + + if is_deleted: + click.echo('INFO: File `{}` deleted.'.format(path)) + sys.exit(1) + click.echo('INFO: File `{}` not found.'.format(path)) + + def update_single_file(self, domain, path, file): + hosting = self._get_hosting(domain=domain) + hosting.update_file(path, file) + click.echo('INFO: File `{}` updated.'.format(path)) + def create_hosting(self, label, domain): hosting = self.instance.hostings.create( label=label, @@ -56,11 +90,20 @@ def create_hosting(self, label, domain): ) return hosting - def _get_hosting(self, domain): + def _get_hosting(self, domain, is_new=False): hostings = self.instance.hostings.all() + to_return = None + for hosting in hostings: if domain in hosting.domains: - return hosting + to_return = hosting + break + + if not to_return and not is_new: + click.echo(u'WARN: No default hosting found. Exit.') + sys.exit(1) + + return to_return def _validate_path(self, file_path): try: @@ -72,8 +115,8 @@ def _validate_path(self, file_path): def print_hosting_files(self, hosting_files): print('Hosting files:') self._print_separator() - for file_path in hosting_files: - print(file_path) + for hosting_file in hosting_files: + print(hosting_file.path) @staticmethod def _print_separator(): diff --git a/syncano_cli/hosting/commands.py b/syncano_cli/hosting/commands.py index 8b0a2a7..95ce604 100644 --- a/syncano_cli/hosting/commands.py +++ b/syncano_cli/hosting/commands.py @@ -1,11 +1,11 @@ # -*- coding: utf-8 -*- -import os import sys import click from syncano_cli.base.connection import create_connection, get_instance_name from syncano_cli.config import ACCOUNT_CONFIG_PATH -from syncano_cli.hosting.utils import HostingCommands +from syncano_cli.hosting.command import HostingCommands +from syncano_cli.hosting.validators import validate_domain, validate_publish @click.group() @@ -13,24 +13,15 @@ def top_hosting(): pass -@top_hosting.command() +@top_hosting.group() +@click.pass_context @click.option('--config', help=u'Account configuration file.') @click.option('--instance-name', help=u'Instance name.') -@click.option('--list-files', is_flag=True, help='List files within the hosting.') -@click.option('--publish', type=str, help='Publish files from the local directory to the Syncano Hosting.') -def hosting(config, instance_name, list_files, publish): +def hosting(ctx, config, instance_name): """ Handle hosting and hosting files. Allow to publish static pages to the Syncano Hosting. """ - def validate_domain(provided_domain=None): - return 'default' if not provided_domain else provided_domain - - def validate_publish(base_dir): - if not os.path.isdir(base_dir): - click.echo(u'ERROR: You should provide a project root directory here.') - sys.exit(1) - config = config or ACCOUNT_CONFIG_PATH instance_name = get_instance_name(config, instance_name) @@ -39,20 +30,87 @@ def validate_publish(base_dir): instance = connection.Instance.please.get(name=instance_name) hosting_commands = HostingCommands(instance) + ctx.obj['hosting_commands'] = hosting_commands + + except Exception as e: + click.echo(u'ERROR: {}'.format(e)) + sys.exit(1) + + +@hosting.command() +@click.pass_context +@click.argument('directory') +def publish(ctx, directory): + + validate_publish(directory) + domain = validate_domain() # prepared for user defined domains; + + hosting_commands = ctx.obj['hosting_commands'] + try: + hosting_commands.publish(domain=domain, base_dir=directory) + except Exception as e: + click.echo(u'ERROR: {}'.format(e)) + sys.exit(1) - if list_files: - domain = validate_domain() - click.echo(u'INFO: List the hosting files: {} in instance: {}'.format(domain, instance_name)) - hosting_files = hosting_commands.list_hosting_files(domain=domain) - hosting_commands.print_hosting_files(hosting_files) - if publish: - domain = validate_domain() - click.echo(u'INFO: Publish the hosting files: {} in instance: {}'.format(domain, instance_name)) - validate_publish(base_dir=publish) - uploaded_files = hosting_commands.publish(domain=domain, base_dir=publish) - hosting_commands.print_hosting_files(uploaded_files) +@hosting.command() +@click.pass_context +def unpublish(ctx): + domain = validate_domain() + hosting_commands = ctx.obj['hosting_commands'] + try: + hosting_commands.unpublish(domain=domain) + except Exception as e: + click.echo(u'ERROR: {}'.format(e)) + sys.exit(1) + +@hosting.command() +@click.pass_context +def list(ctx): + hosting_commands = ctx.obj['hosting_commands'] + domain = validate_domain() + try: + hosting_commands.print_hosting_files( + hosting_files=hosting_commands.list_hosting_files(domain) + ) + except Exception as e: + click.echo(u'ERROR: {}'.format(e)) + sys.exit(1) + + +@hosting.command() +@click.pass_context +@click.argument('path', required=False) +def delete(ctx, path): + domain = validate_domain() + hosting_commands = ctx.obj['hosting_commands'] + if not path: + if click.confirm('Do you want to remove whole hosting?'): + try: + hosting_commands.delete_hosting(domain=domain, path=path) + except Exception as e: + click.echo(u'ERROR: {}'.format(e)) + sys.exit(1) + else: + click.echo("INFO: Deleting aborted.") + else: + try: + hosting_commands.delete_path(domain=domain, path=path) + except Exception as e: + click.echo(u'ERROR: {}'.format(e)) + sys.exit(1) + + +@hosting.command() +@click.pass_context +@click.argument('path') +@click.argument('file') +def update(ctx, path, file): + domain = validate_domain() + hosting_commands = ctx.obj['hosting_commands'] + try: + hosting_commands.update_single_file(domain=domain, path=path, file=file) except Exception as e: click.echo(u'ERROR: {}'.format(e)) sys.exit(1) diff --git a/syncano_cli/hosting/validators.py b/syncano_cli/hosting/validators.py new file mode 100644 index 0000000..70ff800 --- /dev/null +++ b/syncano_cli/hosting/validators.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +import os +import sys + +import click + + +def validate_publish(directory): + if not os.path.isdir(directory): + click.echo(u'ERROR: You should provide a project root directory here.') + sys.exit(1) + + +def validate_domain(provided_domain=None): + return 'default' if not provided_domain else provided_domain diff --git a/tests/test_hosting_commands.py b/tests/test_hosting_commands.py index 716c780..4ec2a53 100644 --- a/tests/test_hosting_commands.py +++ b/tests/test_hosting_commands.py @@ -5,20 +5,65 @@ class HostingCommandsTest(BaseCLITest): - def test_file_list(self): + def test_hosting_commands(self): + # test publishing; self._publish_files() + result = self._get_list_output() + self.assertIn('index.html', result.output) + self.assertIn('css/page.css', result.output) - result = self.runner.invoke(cli, args=[ - 'hosting', '--list-files' - ]) + # test single file deletion; + self._delete_single_file('css/page.css') + result = self._get_list_output() + self.assertNotIn('css/page.css', result.output) - self.assertIn('index.html', result.output) + # test update file which do not exist; + result = self.runner.invoke(cli, args=[ + 'hosting', 'update', 'css/page.css', 'tests/hosting_files_examples/css/page.css' + ], obj={}) self.assertIn('css/page.css', result.output) + # test hosting delete; + self._delete_hosting() + result = self._get_list_output() + self.assertIn('No default hosting found. Exit.', result.output) + + # recreate hosting; + self._publish_files() + + # test unpublish; + result = self.runner.invoke(cli, args=[ + 'hosting', 'unpublish' + ], obj={}) + self.assertIn('unpublished', result.output) + result = self._get_list_output() + self.assertIn('No default hosting found. Exit.', result.output) + + def _get_list_output(self): + result = self.runner.invoke(cli, args=[ + 'hosting', 'list' + ], obj={}) + return result + def _publish_files(self): result = self.runner.invoke(cli, args=[ - 'hosting', '--publish', 'tests/hosting_files_examples' + 'hosting', 'publish', 'tests/hosting_files_examples' ], obj={}) self.assertIn('index.html', result.output) self.assertIn('css/page.css', result.output) + + def _delete_single_file(self, path): + result = self.runner.invoke(cli, args=[ + 'hosting', 'delete', path + ], obj={}) + + self.assertIn('deleted.', result.output) + + def _delete_hosting(self): + result = self.runner.invoke(cli, args=[ + 'hosting', 'delete' + ], input='y', obj={}) + + self.assertIn('Hosting', result.output) + self.assertIn('deleted.', result.output)