diff --git a/Examples/walmart.s b/Examples/walmart.s index 0c4c1e0..461b8e9 100644 --- a/Examples/walmart.s +++ b/Examples/walmart.s @@ -65,15 +65,33 @@ each(var p in fetchPage) from expand (p.first to p.last) } -create buffer product(url string, description string, price string) +create buffer product(url string, detailUrl string, description string, price string, upc string) insert overwrite product select url, + 'http://www.walmart.com' + pick '.prod-ProductCard--Image a' take attribute 'href', pick '.prod-ProductTitle div', - pick '.Price-characteristic' + '.' + pick '.Price-mantissa' -from download page (select url from urls) with (js|thread(5)) + pick '.Price-characteristic' + '.' + pick '.Price-mantissa', + '' +from download page (select url from urls) with (js|thread(10)) where nodes = '.search-result-gridview-item' + +update p +set p.upc = u.upc +from (select upc, url + from download page (select detailUrl from product) with (js|thread(10)) => { + " + var primaryProductId = __WML_REDUX_INITIAL_STATE__.product.primaryProduct; + + var primaryProduct = __WML_REDUX_INITIAL_STATE__.product.products[primaryProductId]; + + return [{upc:primaryProduct.upc, url:url}]; + " + }) u +join product p on p.detailUrl = u.url + + select * from product \ No newline at end of file diff --git a/Pickaxe.Runtime/Http.cs b/Pickaxe.Runtime/Http.cs index 3ba1126..e56c9e4 100644 --- a/Pickaxe.Runtime/Http.cs +++ b/Pickaxe.Runtime/Http.cs @@ -71,23 +71,30 @@ private static RuntimeTable ProcesJson(IHttpRequestFactory factor json = string.Empty; dynamic objectValue = JsonConvert.DeserializeObject(json); - if (!(objectValue is JArray)) - throw new InvalidOperationException("must return array from java script"); - var dynamics = new List(); - var properties = new List>(); - foreach (var v in objectValue) + + if (objectValue != null) { - var dynamic = new DynamicObject(); - Dictionary values = v.ToObject>(); + if (!(objectValue is JArray)) + throw new InvalidOperationException("must return array from java script"); - foreach (var p in values.Keys) - dynamic.Add(p, values[p].ToString()); + var properties = new List>(); + foreach (var v in objectValue) + { + var dynamic = new DynamicObject(); + Dictionary values = v.ToObject>(); - dynamics.Add(dynamic); + foreach (var p in values.Keys) + dynamic.Add(p, values[p].ToString()); + + dynamics.Add(dynamic); + } } var table = new RuntimeTable(); + if(dynamics.Count == 0) + dynamics.Add(new DynamicObject()); + table.SetRows(dynamics); return table; } diff --git a/Pickaxe.Runtime/Internal/SeleniumExecJsHttpWire.cs b/Pickaxe.Runtime/Internal/SeleniumExecJsHttpWire.cs index 0c164f3..4430a88 100644 --- a/Pickaxe.Runtime/Internal/SeleniumExecJsHttpWire.cs +++ b/Pickaxe.Runtime/Internal/SeleniumExecJsHttpWire.cs @@ -48,7 +48,7 @@ protected override object RunPostDownload(IWebDriver driver) ", _js); - string json = (string)jsExecutor.ExecuteScript(script, driver.Url); + string json = (string)jsExecutor.ExecuteScript(script, Url); return json; } } diff --git a/Pickaxe.Studio/Highlighting/Scrape.xshd b/Pickaxe.Studio/Highlighting/Scrape.xshd index 3bd9e89..45469ec 100644 --- a/Pickaxe.Studio/Highlighting/Scrape.xshd +++ b/Pickaxe.Studio/Highlighting/Scrape.xshd @@ -14,6 +14,12 @@ ' + + + " + " + + //