Problems with response after GET request

[espruino-discuss]

Posted at 2017-07-20 by Hansi

Hello,
I'm trying to create a counter for Etsy platform which has its own API. I have already tried it via console ( with XMLHttpRequest()) and everything works fine. And have started working with ESP8266.
Wifi connected, the request is sent, but I receive not the same answer as for example when putting the same URL in the browser.

My function:

function getPage() {
  require("http").get("https://openapi.ets­y.com/v2/shops", function(res) {
   console.log("Response: ",res);
     res.on('data', function(data) { contents += data; });
     res.on('close', function(contents) { console.log(contents); });
  });
}
getPage();

in console I see:

=undefined
Response:  httpCRs {
  "headers": {
    "Location": "https://openapi.etsy.com/v2/shops",
    "Server": "BigIP",
    "Content-Length": "0",
    "Accept-Ranges": "bytes",
    "Date": "Thu, 20 Jul 2017 03:06:32 GMT",
    "Via": "1.1 varnish",
    "Age": "0",
    "Connection": "close",
    "X-Served-By": "cache-yyz8330-YYZ",
    "X-Cache": "MISS",
    "X-Cache-Hits": "0",
    "X-Timer": "S1500519992.012622,VS0,VE37"
   },
  "httpVersion": "1.1",
  "statusCode": "302",
  "statusMessage": "Found"
 }
false

But if I put the same request in browser "https://openapi.etsy.com/v2/shops" I at least see "API request missing api_key or valid OAuth parameters" (just don't want to show my ApiKey).

As for other links "http://www.pur3.co.uk/hello.txt" or "http://samples.openweathermap.org/data/2­.5/weather?q=London,uk&appid=b1b15e88fa7­97225412429c1c50c122a1" everything is ok.

So in result I'm see 'the wall' before next steps.

Could anyone help to solve it?

Thanks

[espruino-discuss]

Posted at 2017-07-20 by @gfwilliams

What happens if you try that same code from Node.js on the desktop?

You could try setting 'User Agent' in the headers? It's possible that Etsy blocks anything that it isn't sure is a web browser

---

Posted at 2017-07-20 by Hansi

Hi, thanks for the answer

I've tried in such a way:

var https = require('https');

var options = { host: 'openapi.etsy.com',  path: '/v2/shops',  headers: {'user-agent': 'Mozilla/5.0'}};

https.get(options, function(res) {  console.log("Got response: " + res.statusCode);  console.log(res.headers);}).on('error', function(e) {console.log("Got error: " + e.message);});

For this request, I got a response from Apache server 200 and see 'content-type': 'application/json', but don't know how to read it(it's the next step).

But with 'http' I receive the response as before even with using Agent in request

---

Posted at 2017-07-21 by Hansi

Finally it works in node.js

var https = require('https');

var options = { host: 'openapi.etsy.com',  
				path: '/v2/shops/shopID?method=GET&api_key=api-key=num_favorers',  
				headers: {'user-agent': 'Mozilla/5.0'}
			};


https.get(options, function(res) {  
	console.log("Got response: " + res.statusCode);
	var dataQueue = "";
	res.on('data', function(d) {
		//process.stdout.write(d);
		dataQueue += d;
		console.log(dataQueue);
		myObj = JSON.parse(dataQueue);
		console.log(myObj.results[0].num_favorers)
	});
	//res.on('end', function(){
		//console.log(dataQueue);
	//});

}).on('error', function(e) {
	console.log("Got error: " + e.message);
});

but it is Https - there is no support of Https?

---

Posted at 2017-07-21 by dave_irvine

Not on ESP8266. Not enough RAM for HTTPS.

---

Posted at 2017-07-21 by Hansi

Eh, it's a pitty)
So it is no variant to receive 'body' in my case using http?

I found this on site:
HTTPS
The only boards currently supporting this are the Espruino Pico and WiFi. To use HTTPS simply use it in the URL of any normal HTTP request:

require("http").get("https://www.google.com", function(res) {
  res.on('data', function(data) { /* ... */ });
});

To specify keys and certificates, you can use an options object - see require('tls').connect(...)

So if I will use Pico or WIFI it should work?

---

Posted at 2017-07-21 by @MaBecker

Hi Hansi,

check MrTimcakes http2https

---

Posted at 2017-07-21 by Hansi

Oh, thank you so much!

Now it works in my desktop node.js

var http = require('http');

var options = { host: 'httptohttps.mrtimcakes.com', 
			path: '/https://openapi.etsy.com/v2/shops/shopID?method=GET&api_key=apikey',  
			headers: {'user-agent': 'Mozilla/5.0'}
			};

http.get(options, function(res) {  
	console.log("Got response: " + res.statusCode);
	var dataQueue = "";
	res.on('data', function(d) {
		dataQueue += d;
		console.log(dataQueue);
		myObj = JSON.parse(dataQueue);
		console.log(myObj.results[0].num_favorers)
	});

}).on('error', function(e) {
	console.log("Got error: " + e.message);
});

but when I try the same in ESP8266 I receive

=undefined
Got error: not found

But now it's closer to the solution!)

UPD
And ordered Espruino WIFI to try it later when it comes, but before I want to finish this problem with ESP8266

---

Posted at 2017-07-22 by @MaBecker

Hmm,

this is how far I get with your code and without a key

Got response: 200
Invalid api_key: apikey
Uncaught SyntaxError: Expecting a valid value, got ID
 at line 1 col 24
Invalid api_key: apikey
                       ^
in function called from system

---

Posted at 2017-07-22 by Hansi

Have you checked via desktop node.js or your microcontroller?

Oh, finally I've found my mistake, current result:

=undefined
Got response: 200
{"count":1,"results":[{"num_favorers":34,"include_dispute_form_link":false}],
Uncaught SyntaxError: Got EOF expected '}'
 at line 1 col 78
...dispute_form_link":false}],
                              ^
in function called from system
{"count":1,"results":[{"num_favorers":34,"include_dispute_form_link":false}],"params":{"shop_id":"FoxyGami"},"type":"Shop","pagination":{}}
34

Now I can receive what I want. Just don't know why it always shows such messages:

{"count":1,"results":[{"num_favorers":34,"include_dispute_form_link":false}],
Uncaught SyntaxError: Got EOF expected '}'
 at line 1 col 78
...dispute_form_link":false}],
                              ^
in function called from system

In node.js is just:

Got response: 200
{"count":1,"results":[{"num_favorers":34,"include_dispute_form_link":false}],"params":{"shop_id":"FoxyGami"},"type":"Shop","pagination":{}}
34

---

Posted at 2017-07-24 by @gfwilliams

It's because you're parsing the data into JSON as it arrives. In Node.js it's arriving in one big chunk but in Espruino it's a few smaller ones.

Do like it suggests here and do stuff on the close event, not the data one: http://www.espruino.com/Internet#client

require("http").get("http://www.espruino.com", function(res) {
  var contents = "";
  res.on('data', function(data) { contents += data; });
  res.on('close', function() {
   //  do JSON.parse(contents)
  });
});

---

Posted at 2017-07-24 by Hansi

oh, thanks, Gordon, good support on your forum.
Now my project is possible!!!

---

Posted at 2017-08-13 by Hansi

Could anyone explain this message in console:

ets Jan  8 2013,rst cause:2, boot mode:(3,0)
load 0x40100000, len 2408, room 16
tail 8
chksum 0xe5
load 0x3ffe8000, len 776, room 0
tail 8
chksum 0x84
load 0x3ffe8310, len 632, room 0
tail 8
chksum 0xd8
csum 0xd8
2nd boot version : 1.6
  SPI Speed      : 40MHz
  SPI Mode       : DIO
  SPI Flash Size & Map: 32Mbit(512KB+512KB)
jump to run user1 @ 1000
�â�ìp��Nä�{�òn|ìd�l ��â{�p��üLoading 1298 bytes from flash...

Is it mean that I run out the memory?

---

Posted at 2017-08-13 by @MaBecker

you can find reset causes explained in this document

The most root cause is running out of heap.

Use this statement to get debug info on console like heap size

require('ESP8266').logDebug(true);
require('ESP8266').setLog(2);

// output
>325651> Sun Aug 13 07:54:45 2017, heap: 10416

---

Posted at 2017-08-15 by Hansi

Could you please explain what 'heap' in the output means?

Problem is that after starting of program everything is ok, I make 3 requests at the same time every 180seconds. But on 10th time it shows error - out of memory, and every time it happens on different try. Probably I should clean ram somehow after each request or something like this? (using of non-custom font helps a little bit, but errors still happen).

ERROR: Out of Memory!
ERROR: Error processing Serial data handler - removing it.
Execution Interrupted during event processing.
Uncaught SyntaxError: Expecting a valid value, got EOF
 at line 1 col 22
{"count":1,"results":[

Thank you for answers!