diff --git a/MKMTool/MKMBot.cs b/MKMTool/MKMBot.cs index bdf1ef9..ab94113 100644 --- a/MKMTool/MKMBot.cs +++ b/MKMTool/MKMBot.cs @@ -428,7 +428,7 @@ public void generatePrices(List cardList, bool useMyStock) MainView.Instance.LogMainWindow("Found myStock.csv, parsing minimal prices..."); try { - DataTable stock = MKMDbManager.ConvertCSVtoDataTable(@".//myStock.csv"); + DataTable stock = MKMCsvUtils.ConvertCSVtoDataTable(@".//myStock.csv"); if (stock.Columns.Contains(MCAttribute.MinPrice)) { foreach (DataRow dr in stock.Rows) @@ -460,72 +460,42 @@ public void updatePrices() { MainView.Instance.LogMainWindow("Setting price according to lowest price is very risky - specify limits for maximal price change first!"); return; - } - -#if (DEBUG) - var debugCounter = 0; -#endif - List articles = new List(); - string sRequestXML = ""; - XmlNodeList result; - var start = 1; - // load file with lowest prices - Dictionary> myStock = LoadMyStock(); + } + List articles; try { - do - { - var doc = MKMInteract.RequestHelper.readStock(start); - - result = doc.GetElementsByTagName("article"); - foreach (XmlNode article in result) - { - articles.Add(article); - } - start += result.Count; - } while (result.Count == 100); + articles = MKMInteract.RequestHelper.getAllStockSingles(MainView.Instance.Config.UseStockGetFile); } catch (Exception error) { MKMHelpers.LogError("reading own stock, cannot continue price update", error.Message, true); return; } + // load file with lowest prices + Dictionary> myStock = LoadMyStock(); MainView.Instance.LogMainWindow("Updating Prices..."); int putCounter = 0; - foreach (XmlNode article in articles) + string sRequestXML = ""; + foreach (MKMMetaCard MKMCard in articles) { -#if (DEBUG) - debugCounter++; - if (debugCounter > 3) - { - MainView.Instance.logMainWindow("DEBUG MODE - EXITING AFTER 3\n"); - break; - } -#endif - // according to the API documentation, "The 'condition' key is only returned for single cards. " - // -> check if condition exists to see if this is a single card or something else - if (article["condition"] != null && article["idArticle"].InnerText != null && article["price"].InnerText != null) + XmlNodeList similarItems = getSimilarItems(MKMCard); + if (similarItems != null) { - MKMMetaCard MKMCard = new MKMMetaCard(article); - XmlNodeList similarItems = getSimilarItems(MKMCard); - if (similarItems != null) + appraiseArticle(MKMCard, similarItems, myStock); + string newPrice = MKMCard.GetAttribute(MCAttribute.MKMToolPrice); + if (newPrice != "") { - appraiseArticle(MKMCard, similarItems, myStock); - string newPrice = MKMCard.GetAttribute(MCAttribute.MKMToolPrice); - if (newPrice != "") + sRequestXML += MKMInteract.RequestHelper.changeStockArticleBody(MKMCard, newPrice); + // max 100 articles is allowed to be part of a PUT call - if we are there, call it + if (putCounter > 98 && !settings.testMode) { - sRequestXML += MKMInteract.RequestHelper.changeStockArticleBody(MKMCard, newPrice); - // max 100 articles is allowed to be part of a PUT call - if we are there, call it - if (putCounter > 98 && !settings.testMode) - { - MKMInteract.RequestHelper.SendStockUpdate(sRequestXML, "PUT"); - putCounter = 0; - sRequestXML = ""; - } - else - putCounter++; + MKMInteract.RequestHelper.SendStockUpdate(sRequestXML, "PUT"); + putCounter = 0; + sRequestXML = ""; } + else + putCounter++; } } } @@ -542,7 +512,7 @@ public void updatePrices() MainView.Instance.LogMainWindow("Done. No valid/meaningful price updates created."); } - String timeStamp = GetTimestamp(DateTime.Now); + string timeStamp = GetTimestamp(DateTime.Now); MainView.Instance.LogMainWindow("Last Run finished: " + timeStamp); } @@ -565,6 +535,7 @@ private XmlNodeList getSimilarItems(MKMMetaCard card, int maxNbItems = 150) string isFoil = card.GetAttribute(MCAttribute.Foil); string isSigned = card.GetAttribute(MCAttribute.Signed); string isAltered = card.GetAttribute(MCAttribute.Altered); + string isFirstEd = card.GetAttribute(MCAttribute.FirstEd); string articleName = card.GetAttribute(MCAttribute.Name); try { @@ -572,7 +543,7 @@ private XmlNodeList getSimilarItems(MKMMetaCard card, int maxNbItems = 150) (languageID != "" ? "?idLanguage=" + card.GetAttribute(MCAttribute.LanguageID) : "") + (condition != "" ? "&minCondition=" + condition : "") + (isFoil != "" ? "&isFoil=" + isFoil : "") + (isSigned != "" ? "&isSigned=" + isSigned : "") + (isAltered != "" ? "&isAltered=" + isAltered : "") + - "&start=0&maxResults=" + maxNbItems; + (isFirstEd != "" ? "&isFirstEd=" + isFirstEd : "") + "&start=0&maxResults=" + maxNbItems; return MKMInteract.RequestHelper.makeRequest(sUrl, "GET").GetElementsByTagName("article"); } diff --git a/MKMTool/MKMCsvUtils.cs b/MKMTool/MKMCsvUtils.cs new file mode 100644 index 0000000..aeed0f2 --- /dev/null +++ b/MKMTool/MKMCsvUtils.cs @@ -0,0 +1,191 @@ +using System; +using System.Collections.Generic; +using System.Data; +using System.IO; +using System.Linq; +using static MKMTool.MKMHelpers; + +namespace MKMTool +{ + /// + /// Helper class for processing CSV files. + /// + class MKMCsvUtils + { + + /// + /// Parses a row in a CSV file. + /// + /// The row from a CSV file. Each value can be enclosed by double quotes (i.e. the character ") + /// and if a double quote is part of the value, it is escaped by another double quote. + /// If a field contains an even number of double quotes and nothing else, it will be considered as not enclosed, so a field that contains """""" will + /// be parsed as """. The only exception is when there are exactly two double quotes, in that case it will be considered an enclosed empty string. + /// This is done as in practice, excel and similar might enclose the empty fields and the user probably wants to keep them empty. + /// The character used as separator between columns. + /// A list of the individual parsed values in the order they appear on the row. All enclosing quotes are trimmed + /// from the value and all escape characters are removed. + private static List parseCSVRow(string rowToParse, char separator) + { + List ret = new List(); + string[] split = rowToParse.Split(separator); + + for (int i = 0; i < split.Length; i++) + { + string columnValue = split[i]; + // we need to account for entires that have commas in their own name (so the split will split them among multiple + // columns, even though they should be in one) and also for entries that can contain quotes. + // MKM escapes the double quotes by another double quote, so the ending double quote is really ending only if it is not preceded by + // something else. + // So far worst case scenario is a card from the Force of Will game called: "I", the pilot. This in the database looks like this: + // "304732","""I"", the Pilot","1018","Force of Will Single","1775","229401","2017-10-04 17:48:59" + // so it has a comma, double quote precedes it, so it looks like the entry should end there, but it does not, it is in the middle of the name. + // in general, double quote is to be considered opening/ending only if there is an odd number of them, otherwise they are escaped + bool quoteEnclosed = false; + for (int j = 0; j < columnValue.Length; j++) + { + if (columnValue[j] == '"') + quoteEnclosed = !quoteEnclosed; + else break; + } + if (quoteEnclosed) // starts by a double quote -> can contain comma itself, merge until the last double quote is found + { + while (true) + { + quoteEnclosed = false; + for (int j = columnValue.Length - 1; j >= 0; j--) // check if it ends with an odd number of double quotes + { + if (columnValue[j] == '"') + quoteEnclosed = !quoteEnclosed; + else break; + } + if (!quoteEnclosed) // the closing quote was not found yet + columnValue += separator + split[++i]; // append the next column value + else break; + } + } + if (quoteEnclosed) + columnValue = columnValue.Substring(1, columnValue.Length - 2); + columnValue = columnValue.Replace("\"\"", "\""); // un-escape double quotes + // let's handle one corner case: if somebody is exporting a list from excel and says "enclose each field in double quotes", + // empty fields will have the value "". Since that is an even number of double quotes, our algorithm will evaluate it as + // not being quoteEnclosed even though it actually is an enclosed empty string. After the above replacement, we will now + // have a string that is a single double quote. If that is the case, replace it with actual empty string. + // Note that this still does not handle all corner cases, in general, if there is an even number X of double quotes in the field and no other text, + // it can either be X/2 not-enclosed double quotes, or (X-1)/2 enclosed double quotes, there is no way to tell since we are allowing + // mixed format (i.e. enclosed and unenclosed in the same file). Hopefully this never has any practical impact. + if (!quoteEnclosed && columnValue == "\"") + columnValue = ""; + ret.Add(columnValue); + } + return ret; + } + + /// + /// Writes the table as CSV. + /// + /// Path to the file as which to write the table. + /// The data table to write. + public static void WriteTableAsCSV(string filePath, DataTable dt) + { + try + { + using (StreamWriter exp = new StreamWriter(filePath)) + { + // we know there will be at least one column, otherwise there would be no valid imported items and therefore no export enabled + string row = "\"" + (dt.Columns[0].ColumnName).Replace("\"", "\"\"") + "\""; // don't forget to escape all " by doubling them + for (int i = 1; i < dt.Columns.Count; i++) + row += ",\"" + (dt.Columns[i].ColumnName).Replace("\"", "\"\"") + "\""; + exp.WriteLine(row); + foreach (DataRow card in dt.Rows) + { + row = "\"" + card[0].ToString().Replace("\"", "\"\"") + "\""; + for (int i = 1; i < dt.Columns.Count; i++) + row += ",\"" + card[i].ToString().Replace("\"", "\"\"") + "\""; + exp.WriteLine(row); + } + } + } + catch (Exception eError) + { + LogError("writing CSV file " + filePath, eError.Message, true); + } + } + + /// + /// Converts the csv to DataTable. + /// + /// The string file path. + /// Each row of the file as a row in the returned DataTable. + public static DataTable ConvertCSVtoDataTable(string strFilePath) + { + using (var sr = new StreamReader(strFilePath)) + { + return ConvertCSVtoDataTable(sr); + } + } + + /// + /// Converts the csv to DataTable. + /// + /// Raw data containing the csv file. + /// Each row of the file as a row in the returned DataTable. + public static DataTable ConvertCSVtoDataTable(byte[] data) + { + MemoryStream ms = new MemoryStream(data); + using (var sr = new StreamReader(ms)) + { + return ConvertCSVtoDataTable(sr); + } + } + + /// + /// Converts a CSV file to a DataTable. + /// http://stackoverflow.com/questions/1050112/how-to-read-a-csv-file-into-a-net-datatable + /// + /// Stream with the CSV file. It is assumed that the file has a header on the first line with names of the columns. + /// See parseCSVRow on details on the format of the CSV. + /// Each row of the file as a row in the returned DataTable. + /// + /// Wrong format of the header of CSV file " + strFilePath + ": " + eError.Message + /// or + /// Wrong format of the CSV file on row " + (dt.Rows.Count + 1) + ": " + eError.Message + /// + public static DataTable ConvertCSVtoDataTable(StreamReader sr) + { + DataTable dt = new DataTable(); + char separator = ','; + try + { + // detect the separator - this assumes it's ether semicolon or comma and that semicolon cannot be part of column names + string firstLine = sr.ReadLine(); + if (firstLine.Contains(';')) + separator = ';'; + List headers = parseCSVRow(firstLine, separator); + foreach (string header in headers) + dt.Columns.Add(header); + } + catch (Exception eError) + { + throw new FormatException("Wrong format of the header of CSV file: " + eError.Message); + } + while (!sr.EndOfStream) + { + try + { + List row = parseCSVRow(sr.ReadLine(), separator); + DataRow dr = dt.NewRow(); + for (int i = 0; i < row.Count; i++) + dr[i] = row[i]; + dt.Rows.Add(dr); + } + catch (Exception eError) + { + // technically it is the (dt.Rows.Count + 1)th row, but in the file the first row is the header so this should + // give the user the number of the row in the actual file + throw new FormatException("Wrong format of the CSV file on row " + (dt.Rows.Count + 2) + ": " + eError.Message); + } + } + return dt; + } + } +} diff --git a/MKMTool/MKMDatabaseManager.cs b/MKMTool/MKMDatabaseManager.cs index 60d5a9c..3067cc6 100644 --- a/MKMTool/MKMDatabaseManager.cs +++ b/MKMTool/MKMDatabaseManager.cs @@ -39,7 +39,7 @@ using System.Xml; using System.IO; using System.Windows.Forms; -using System.Collections.Generic; +using static MKMTool.MKMCsvUtils; using static MKMTool.MKMHelpers; namespace MKMTool @@ -290,152 +290,6 @@ private void buildDatabase() #endregion #region utilities - /// - /// Parses a row in a CSV file. - /// - /// The row from a CSV file. Assumes it is comma-separated, each value can be - /// enclosed by double quotes (i.e. the character ") and if a double quote is part of the value, it is escaped by another double quote. - /// If a field contains an even number of double quotes and nothing else, it will be considered as not enclosed, so a field that contains """""" will - /// be parsed as """. The only exception is when there are exactly two double quotes, in that case it will be considered an enclosed empty string. - /// This is done as in practice, excel and similar might enclose the empty fields and the user probably wants to keep them empty. - /// A list of the individual parsed values in the order they appear on the row. All enclosing quotes are trimmed - /// from the value and all escape characters are removed. - private static List parseCSVRow(string rowToParse) - { - List ret = new List(); - string[] split = rowToParse.Split(','); - - for (int i = 0; i < split.Length; i++) - { - string columnValue = split[i]; - // we need to account for entires that have commas in their own name (so the split will split them among multiple - // columns, even though they should be in one) and also for entries that can contain quotes. - // MKM escapes the double quotes by another double quote, so the ending double quote is really ending only if it is not preceded by - // something else. - // So far worst case scenario is a card from the Force of Will game called: "I", the pilot. This in the database looks like this: - // "304732","""I"", the Pilot","1018","Force of Will Single","1775","229401","2017-10-04 17:48:59" - // so it has a comma, double quote precedes it, so it looks like the entry should end there, but it does not, it is in the middle of the name. - // in general, double quote is to be considered opening/ending only if there is an odd number of them, otherwise they are escaped - bool quoteEnclosed = false; - for (int j = 0; j < columnValue.Length; j++) - { - if (columnValue[j] == '"') - quoteEnclosed = !quoteEnclosed; - else break; - } - if (quoteEnclosed) // starts by a double quote -> can contain comma itself, merge until the last double quote is found - { - while (true) - { - quoteEnclosed = false; - for (int j = columnValue.Length - 1; j >= 0; j--) // check if it ends with an odd number of double quotes - { - if (columnValue[j] == '"') - quoteEnclosed = !quoteEnclosed; - else break; - } - if (!quoteEnclosed) // the closing quote was not found yet - columnValue += "," + split[++i]; // append the next column value - else break; - } - } - if (quoteEnclosed) - columnValue = columnValue.Substring(1, columnValue.Length - 2); - columnValue = columnValue.Replace("\"\"", "\""); // un-escape double quotes - // let's handle one corner case: if somebody is exporting a list from excel and says "enclose each field in double quotes", - // empty fields will have the value "". Since that is an even number of double quotes, our algorithm will evaluate it as - // not being quoteEnclosed even though it actually is an enclosed empty string. After the above replacement, we will now - // have a string that is a single double quote. If that is the case, replace it with actual empty string. - // Note that this still does not handle all corner cases, in general, if there is an even number X of double quotes in the field and no other text, - // it can either be X/2 not-enclosed double quotes, or (X-1)/2 enclosed double quotes, there is no way to tell since we are allowing - // mixed format (i.e. enclosed and unenclosed in the same file). Hopefully this never has any practical impact. - if (!quoteEnclosed && columnValue == "\"") - columnValue = ""; - ret.Add(columnValue); - } - return ret; - } - - /// - /// Writes the table as CSV. - /// - /// Path to the file as which to write the table. - /// The data table to write. - public static void WriteTableAsCSV(string filePath, DataTable dt) - { - try - { - using (StreamWriter exp = new StreamWriter(filePath)) - { - // we know there will be at least one column, otherwise there would be no valid imported items and therefore no export enabled - string row = "\"" + (dt.Columns[0].ColumnName).Replace("\"", "\"\"") + "\""; // don't forget to escape all " by doubling them - for (int i = 1; i < dt.Columns.Count; i++) - row += ",\"" + (dt.Columns[i].ColumnName).Replace("\"", "\"\"") + "\""; - exp.WriteLine(row); - foreach (DataRow card in dt.Rows) - { - row = "\"" + card[0].ToString().Replace("\"", "\"\"") + "\""; - for (int i = 1; i < dt.Columns.Count; i++) - row += ",\"" + card[i].ToString().Replace("\"", "\"\"") + "\""; - exp.WriteLine(row); - } - } - } - catch (Exception eError) - { - LogError("writing CSV file " + filePath, eError.Message, true); - } - } - - /// - /// Converts a CSV file to a DataTable. - /// http://stackoverflow.com/questions/1050112/how-to-read-a-csv-file-into-a-net-datatable - /// - /// Path to the CSV file. It is assumed that the file has a header on the first line with names of the columns. - /// See parseCSVRow on details on the format of the CSV. - /// Each row of the file as a row in the returned DataTable. - /// - /// Wrong format of the header of CSV file " + strFilePath + ": " + eError.Message - /// or - /// Wrong format of the CSV file on row " + (dt.Rows.Count + 1) + ": " + eError.Message - /// - public static DataTable ConvertCSVtoDataTable(string strFilePath) - { - DataTable dt = new DataTable(); - using (var sr = new StreamReader(strFilePath)) - { - try - { - List headers = parseCSVRow(sr.ReadLine()); - foreach (string header in headers) - dt.Columns.Add(header); - } - catch (Exception eError) - { - throw new FormatException("Wrong format of the header of CSV file " + strFilePath + ": " + eError.Message); - } - while (!sr.EndOfStream) - { - try - { - List row = parseCSVRow(sr.ReadLine()); - DataRow dr = dt.NewRow(); - for (int i = 0; i < row.Count; i++) - dr[i] = row[i]; - dt.Rows.Add(dr); - } - catch (Exception eError) - { - // technically it is the (dt.Rows.Count + 1)th row, but in the file the first row is the header so this should - // give the user the number of the row in the actual file - throw new FormatException("Wrong format of the CSV file on row " + (dt.Rows.Count + 2) + ": " + eError.Message); - } - } - } - - return dt; - } - // Reference: // http://stackoverflow.com/questions/665754/inner-join-of-datatables-in-c-sharp public static DataTable JoinDataTables(DataTable t1, DataTable t2, params Func[] joinOn) diff --git a/MKMTool/MKMInteract.cs b/MKMTool/MKMInteract.cs index 3248d26..58b3c72 100644 --- a/MKMTool/MKMInteract.cs +++ b/MKMTool/MKMInteract.cs @@ -31,6 +31,8 @@ */ using System; +using System.Collections.Generic; +using System.Data; using System.IO; using System.Net; using System.Xml; @@ -344,6 +346,112 @@ public static XmlDocument readStock(int start = 1) return makeRequest("https://api.cardmarket.com/ws/v2.0/stock/" + start, "GET"); } + /// + /// Gets the stock file csv. + /// Warning: the Language column in the returned csv is actually Language ID and boolean vars (foil etc. are empty for "false"). + /// Prefer using the wrapper getAllStockSingles. + /// + /// Decompressed data containing the stock file. Can either be written directly to an output stream. + /// Null if the reading failed (this method logs the error). + private static byte[] getStockFile() + { + var doc = makeRequest("https://api.cardmarket.com/ws/v2.0/stock/file", "GET"); + + var node = doc.GetElementsByTagName("response"); + + if (node.Count > 0 && node.Item(0)["stock"].InnerText != null) + { + var data = Convert.FromBase64String(node.Item(0)["stock"].InnerText); + var aDecompressed = MKMHelpers.gzDecompress(data); + + return aDecompressed; + } + else + { + MKMHelpers.LogError("getting stock file", "failed to get the stock file from MKM.", false); + return null; + } + } + + /// + /// Returns all single cards in our stock as meta cards. This is just a convenience wrapper on getStockFile/readStock. + /// + /// This is for legacy support. If set to false, it will use the old way of getting stock + /// by the readStock method. New way is to use getStockFile as it takes only a single API request. + /// List of all single cards in our stock + public static List getAllStockSingles(bool useFile) + { + List cards = new List(); + if (useFile) + { + byte[] stock = getStockFile(); + if (stock != null) + { + var articleTable = MKMCsvUtils.ConvertCSVtoDataTable(stock); + // the GET STOCK FILE has language ID named Language, fix that + articleTable.Columns["Language"].ColumnName = MCAttribute.LanguageID; + foreach (DataRow row in articleTable.Rows) + { + MKMMetaCard mc = new MKMMetaCard(row); + // according to the API documentation, "The 'condition' key is only returned for single cards. " + // -> check if condition exists to see if this is a single card or something else + if (mc.GetAttribute(MCAttribute.Condition) != "" && mc.GetAttribute(MCAttribute.ArticleID) != "") + { + // sanitize the false booleans - the empty ones mean no, while in MKMMEtaCard empty means "any" + if (articleTable.Columns.Contains("Foil?") && mc.GetAttribute(MCAttribute.Foil) == "") + { + mc.SetBoolAttribute(MCAttribute.Foil, "false"); + } + if (articleTable.Columns.Contains("Altered?") && mc.GetAttribute(MCAttribute.Altered) == "") + { + mc.SetBoolAttribute(MCAttribute.Altered, "false"); + } + if (articleTable.Columns.Contains("Signed?") && mc.GetAttribute(MCAttribute.Signed) == "") + { + mc.SetBoolAttribute(MCAttribute.Signed, "false"); + } + if (articleTable.Columns.Contains("Playset?") && mc.GetAttribute(MCAttribute.Playset) == "") + { + mc.SetBoolAttribute(MCAttribute.Playset, "false"); + } + // this is just a guess, not sure how isFirstEd is written there, or if at all + if (articleTable.Columns.Contains("FirstEd?") && mc.GetAttribute(MCAttribute.FirstEd) == "") + { + mc.SetBoolAttribute(MCAttribute.FirstEd, "false"); + } + cards.Add(mc); + } + } + } + } + else + { + var start = 1; + XmlNodeList result; + var count = 0; + do + { + var doc = readStock(start); + if (doc.HasChildNodes) + { + result = doc.GetElementsByTagName("article"); + foreach (XmlNode article in result) + { + // according to the API documentation, "The 'condition' key is only returned for single cards. " + // -> check if condition exists to see if this is a single card or something else + if (article["condition"] != null && article["idArticle"].InnerText != null) + { + cards.Add(new MKMMetaCard(article)); + } + } + count = result.Count; + start += count; + } + } while (count == 100); + } + return cards; + } + /// /// From MKM documentation: Empties the authenticated user's shopping cart. /// diff --git a/MKMTool/MKMMetaCard.cs b/MKMTool/MKMMetaCard.cs index 9c9ff26..919ac16 100644 --- a/MKMTool/MKMMetaCard.cs +++ b/MKMTool/MKMMetaCard.cs @@ -69,6 +69,11 @@ public class MCAttribute /// public static string ExpansionID { get { return "Expansion ID"; } } + /// + /// The MKM's expansion abbreviation. + /// + public static string ExpansionAbb { get { return "abbreviation"; } } + /// /// English name of the language, first letters capital, i.e. one of the following strings: /// (English;French; German; Spanish; Italian; Simplified Chinese; Japanese; Portuguese; Russian; Korean; Traditional Chinese) @@ -110,7 +115,13 @@ public class MCAttribute /// The string "true" (lowercase) if the card is treated as a playset, "false" if it isn't and an empty string if either is accepted. /// public static string Playset { get { return "Playset"; } } - + + /// + /// The string "true" (lowercase) if the card is treated as first edition, "false" if it isn't and an empty string if either is accepted. + /// Only relevant for Yu-Gi-Oh + /// + public static string FirstEd { get { return "FirstEd"; } } + /// /// The MKM's product ID. /// @@ -157,6 +168,16 @@ public class MCAttribute /// public static string MKMPrice { get { return "Price"; } } + /// + /// The string EUR or GBP. + /// + public static string MKMCurrencyCode { get { return "Currency Code"; } } + + /// + /// The currency id, 1 for EUR, 2 for GBP. + /// + public static string MKMCurrencyId { get { return "idCurrency"; } } + /// /// For MKM articles, this is what is inside the "comments" field. /// @@ -231,7 +252,12 @@ public class MKMMetaCard // If a recognized attribute has a synonym, add it to this dictionary, key == synonym, value == the recognized attribute to which this synonym maps. private static Dictionary synonyms = new Dictionary { - { "enName", MCAttribute.Name }, { "Edition", MCAttribute.Expansion }, { "Altered Art", MCAttribute.Altered } + { "enName", MCAttribute.Name }, { "Edition", MCAttribute.Expansion }, { "Altered Art", MCAttribute.Altered }, + // the following 10 are used by GET STOCK FILE + { "Foil?", MCAttribute.Foil }, { "Signed?", MCAttribute.Signed }, { "Playset?", MCAttribute.Playset }, + { "Altered?", MCAttribute.Altered }, { "FirstEd?", MCAttribute.FirstEd }, { "English Name", MCAttribute.Name }, + { "Exp. Name", MCAttribute.Expansion }, { "Amount", MCAttribute.Count }, { "Exp.", MCAttribute.ExpansionAbb }, + { "Local Name", MCAttribute.LocName } }; // literal dictionary of conditions - translates any supported condition denomination to its equivalent in the two-letter format MKM uses @@ -310,7 +336,7 @@ public MKMMetaCard(DataRow card) else if (attName == MCAttribute.LanguageID) SetLanguageID(columnVal); else if (attName == MCAttribute.Foil || attName == MCAttribute.Signed - || attName == MCAttribute.Altered || attName == MCAttribute.Playset) + || attName == MCAttribute.Altered || attName == MCAttribute.Playset || attName == MCAttribute.FirstEd) SetBoolAttribute(attName, columnVal); // all other attributes else @@ -434,6 +460,8 @@ public MKMMetaCard(XmlNode MKMArticle) data[MCAttribute.Altered] = MKMArticle["isAltered"].InnerText; if (MKMArticle["isPlayset"] != null) data[MCAttribute.Playset] = MKMArticle["isPlayset"].InnerText; + if (MKMArticle["isFirstEd"] != null) + data[MCAttribute.FirstEd] = MKMArticle["isFirstEd"].InnerText; if (MKMArticle["product"] != null) // based on which API call was used, this can be null { data[MCAttribute.Name] = MKMArticle["product"]["enName"].InnerText; @@ -614,51 +642,45 @@ public void SetMinCondition(string condition) } /// - /// Sets an attribute that has boolean value. + /// Sets an attribute that has boolean value. Also sets all existing synonyms to that value. /// /// The name. /// The value. Can be in any recognized format, any capitalization, it will be transformed to lowercase true/false/"". /// Recognized values: empty string always == "any"; true/false/null; yes/no/any; foil/signed/altered all equal true. public void SetBoolAttribute(string name, string value) { - value = value.ToLower(); // MKM API is returning boolean values always as "false/true/null" + string toWrite = value; switch (value) { - case "false": - case "true": - data[name] = value; - break; case "null": // means "any" - data[name] = ""; + toWrite = ""; break; case "yes": - data[name] = "true"; + case "x": // this is used by GET STOCK FILE + toWrite = "true"; break; case "no": - data[name] = "false"; + toWrite = "false"; break; case "any": - data[name] = ""; + toWrite = ""; break; // deckbox.org says the word if it is true and leaves the field blank if it is false // note that we are deviating for the empty field case from their system because it is wrong: // for example, they leave blank foil field even for cards that are available only in foil like all kinds of promos // therefore we understand empty string or "null" as meaning "Any" case "foil": - data[name] = "true"; - break; case "signed": - data[name] = "true"; - break; case "altered": - data[name] = "true"; + toWrite = "true"; break; } + SetAttribute(name, toWrite); } /// - /// Sets the specified attribute to the specified value. + /// Sets the specified attribute to the specified value. Also sets all existing synonyms to that value. /// DOES NOT PERFORM ANY CHECKS ON VALIDITY - if there is a Set*** method available for a particular attribute (like language, condition), /// use that instead of this method to perform a check that the value is valid and that all related attributes will be set to correct values. /// Use SetBoolAttribute where applicable: foil, playset, signed, altered... @@ -668,6 +690,24 @@ public void SetBoolAttribute(string name, string value) public void SetAttribute(string name, string value) { data[name] = value; + // get the main name + string mainName; + if (synonyms.TryGetValue(name, out mainName)) // check if this is a registered synonym + { + data[mainName] = value; // don't forget to write it in the main name, not just other synonyms + } + else // if it fails, then this is a main name + { + mainName = name; + } + // now look for all synonyms for the main name + foreach (var entry in synonyms) + { + if (entry.Value == mainName) + { + data[entry.Key] = value; // key are synonyms, values are main names + } + } } /// @@ -724,7 +764,10 @@ public Bool3 IsOfMinCondition(MKMMetaCard otherCard) /// in the table will be written. If this card does not have a value for that attribute, an empty string will be set as the value. /// If set to true, all attributes will be written - new columns will be added if they are missing. /// Specific format - if not set to MKM, some fields will have reformatted output. - public void WriteItselfIntoTable(DataTable table, bool writeAllAttributes, MCFormat format) + /// If a given attribute is a synonym of an already existing one, do not write it if noSynonyms is true. + /// However, if the column with the synonym attribute is already in the table, it will be written, i.e. this has effect + /// only if writeAllAttributes is set to true + public void WriteItselfIntoTable(DataTable table, bool writeAllAttributes, MCFormat format, bool noSynonyms) { List attributes = new List(); foreach (DataColumn dc in table.Columns) // first we collect all attributes that are already in the table @@ -739,6 +782,16 @@ public void WriteItselfIntoTable(DataTable table, bool writeAllAttributes, MCFor { if (att.Value != "" && !table.Columns.Contains(att.Key)) { + if (noSynonyms) + { + // check if this attribute is a synonym, if it is, write it only if the "parent" (the main name) is non-empty + string synonymRoot; + if (synonyms.TryGetValue(att.Key, out synonymRoot)) + { + if (GetAttribute(synonymRoot) != "") + continue; + } + } DataColumn newColumn = new DataColumn(att.Key, typeof(string)); newColumn.DefaultValue = ""; table.Columns.Add(newColumn); diff --git a/MKMTool/MKMTool.csproj b/MKMTool/MKMTool.csproj index ed81bde..5ee417f 100644 --- a/MKMTool/MKMTool.csproj +++ b/MKMTool/MKMTool.csproj @@ -103,6 +103,7 @@ CheckWantsView.cs + diff --git a/MKMTool/PriceExternalList.cs b/MKMTool/PriceExternalList.cs index 87152e2..4efbd66 100644 --- a/MKMTool/PriceExternalList.cs +++ b/MKMTool/PriceExternalList.cs @@ -169,7 +169,7 @@ private async void buttonImport_Click(object sender, EventArgs e) DataTable dt; try { - dt = MKMDbManager.ConvertCSVtoDataTable(filePath); + dt = MKMCsvUtils.ConvertCSVtoDataTable(filePath); } catch (Exception eError) { @@ -668,10 +668,11 @@ private void buttonExport_Click(object sender, EventArgs e) else if (priceGuidesSkip) continue; } - mc.WriteItselfIntoTable(export, checkBoxExportAll.Checked, checkBoxExportFormatDeckbox.Checked ? MCFormat.Deckbox : MCFormat.MKM); + mc.WriteItselfIntoTable(export, checkBoxExportAll.Checked, + checkBoxExportFormatDeckbox.Checked ? MCFormat.Deckbox : MCFormat.MKM, false); } - MKMDbManager.WriteTableAsCSV(sf.FileName, export); + MKMCsvUtils.WriteTableAsCSV(sf.FileName, export); MainView.Instance.LogMainWindow("Exporting finished."); } diff --git a/MKMTool/StockView.cs b/MKMTool/StockView.cs index 568d493..fdc5d2f 100644 --- a/MKMTool/StockView.cs +++ b/MKMTool/StockView.cs @@ -50,48 +50,40 @@ private void StockView_VisibleChanged(object sender, EventArgs e) { if (Visible) { - int start = 1; var articles = new DataTable(); try { - while (true) + // getAllStockSingles creates a DataTable and converts it to list of cards, so theoretically we are wasting some work + // but it also filters out non-singles and converting to MKMcard will make sure we use the primary column names rather than synonyms + var cards = MKMInteract.RequestHelper.getAllStockSingles(MainView.Instance.Config.UseStockGetFile); + foreach (var card in cards) { - var doc = MKMInteract.RequestHelper.readStock(start); - if (doc.HasChildNodes) - { - var result = doc.GetElementsByTagName("article"); - int elementCount = 0; - foreach (XmlNode article in result) - { - if (article["condition"] != null) // is null for articles that are not cards (boosters etc.) - don't process those - { - MKMMetaCard m = new MKMMetaCard(article); - m.WriteItselfIntoTable(articles, true, MCFormat.MKM); - elementCount++; - } - } - if (elementCount != 100) - { - break; - } - start += elementCount; - } - else break; // document is empty -> end*/ + card.WriteItselfIntoTable(articles, true, MCFormat.MKM, true); } - // Remove columns we don't want showing // TODO - what is and isn't shown should probably be customizable and left to the user to choose in some way - articles.Columns.Remove(MCAttribute.ArticleID); - articles.Columns.Remove(MCAttribute.ProductID); - articles.Columns.Remove(MCAttribute.LanguageID); - articles.Columns.Remove(MCAttribute.CardNumber); + if (articles.Columns.Contains(MCAttribute.ArticleID)) + articles.Columns.Remove(MCAttribute.ArticleID); + if (articles.Columns.Contains(MCAttribute.LanguageID)) + articles.Columns.Remove(MCAttribute.LanguageID); + if (articles.Columns.Contains(MCAttribute.MetaproductID)) + articles.Columns.Remove(MCAttribute.MetaproductID); + if (articles.Columns.Contains("onSale")) + articles.Columns.Remove("onSale"); // don't even know what this one is supposed to be, it's not even in the API documentation + if (articles.Columns.Contains(MCAttribute.MKMCurrencyCode)) + articles.Columns.Remove(MCAttribute.MKMCurrencyCode); + if (articles.Columns.Contains(MCAttribute.MKMCurrencyId)) + articles.Columns.Remove(MCAttribute.MKMCurrencyId); var dj = MKMDbManager.JoinDataTables(articles, MKMDbManager.Instance.Expansions, (row1, row2) => row1.Field(MKMDbManager.InventoryFields.ExpansionID) == row2.Field(MKMDbManager.ExpansionsFields.ExpansionID)); - dj.Columns.Remove(MCAttribute.ExpansionID); // duplicated - dj.Columns.Remove(MKMDbManager.ExpansionsFields.ExpansionID); // ...and we don't want it anyway - dj.Columns.Remove(MKMDbManager.ExpansionsFields.Name); // duplicated + if (dj.Columns.Contains(MCAttribute.ExpansionID)) + dj.Columns.Remove(MCAttribute.ExpansionID); // duplicated + if (dj.Columns.Contains(MKMDbManager.ExpansionsFields.ExpansionID)) + dj.Columns.Remove(MKMDbManager.ExpansionsFields.ExpansionID); // ...and we don't want it anyway + if (dj.Columns.Contains(MKMDbManager.ExpansionsFields.Name)) + dj.Columns.Remove(MKMDbManager.ExpansionsFields.Name); // duplicated dj.Columns[dj.Columns.IndexOf(MCAttribute.Name)].SetOrdinal(0); dj.Columns[dj.Columns.IndexOf(MCAttribute.Expansion)].SetOrdinal(1); @@ -145,7 +137,7 @@ private void buttonExport_Click(object sender, EventArgs e) if (sf.ShowDialog() == DialogResult.OK) { MainView.Instance.LogMainWindow("Exporting inventory..."); - MKMDbManager.WriteTableAsCSV(sf.FileName, (DataTable)stockGridView.DataSource); + MKMCsvUtils.WriteTableAsCSV(sf.FileName, (DataTable)stockGridView.DataSource); MainView.Instance.LogMainWindow("Inventory exported."); } } diff --git a/README.md b/README.md index ab77ad9..61a2ae6 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,12 @@ If you have just updated from older version than 0.7.0, please delete the "mkmto ## Latest changes **Version 0.8.0.0, 30.11.2020** New/improved features: ++ All CSV lists used by MKMTool can now also use semicolon (the character ';') as a separator. MKMTool automatically detects if your file uses ';' or ','. ';' must NOT be contained in column names. + Added generic configuration settings for MKMTool that can be specified in the config.xml. Right now there is only one new setting, might be more in the future. ++ Fetching our own stock is now more efficient by using the GET STOCK FILE API request, which downloads entire stock as a single CSV file, which MKMTool than parses. This reduces the number of API requests used by 1 per each 100 articles in your stock, which also decreases the processing time. This is used in the Update Price and the View Inventory. If you have some problems, you can switch to the old way by opening your config.xml and setting the UseStockGetFile to false. ++ View Inventory now includes the idProduct. If UseStockGetFile is set to true (now default), it will no longer include Rarity, lastEdited and inShoppingCart columns. If you need those, you will have to switch UseStockGetFile to false (this will however significantly increase the time it takes to show the inventory). ++ When updating price, the FirstEd parameter is taken into account when looking for similar items (relevant only for Yu-Gi-Oh). +Bug-fixes: + MinPrice from mystock.csv was taken into account only if no price update was computed. Now it will always be checked even just against the current price.
Show older changelogs @@ -300,8 +305,9 @@ Updates the local offline database provided from MKM. This CSV is usually mainta ### CSV Card Lists in MKMTool There are several modules in MKMTool that use import or export from/to Comma Separated Value (CSV) files. Each module has some specific to which card attributes are or are not expected to be in the list, but the general format is the same and described below. To create the file, it is recommended to use your favourite spreadsheet software, all of them should have an option to save as CSV. However, there are several viable formats, so ensure that the one your spreadsheet software is using follows these rules: -+ The separator is comma (i.e. the character ','). This can usually be easily changed in the export settings. -+ All fields that contain a comma as part of the value are enclosed in double quotation marks (i.e. the character '"'). For example, if a text in some cell of the spreadsheet is *Jace, the Mind Sculptor*, when you save it as CSV and open in an ordinary text editor, you should see *"Jace, the Mind Sculptor"*. Usually the spreadsheet software will do this automatically. Fields that do not contain a comma may be, but do not have to be, also enclosed in quotation marks. ++ The separator is comma or semicolon, i.e. the characters ',' or ';' (but only one of those for the whole file!). These are common in most locale settings, at least in Europe. If it is not the default on your system, you can usually easily change the separator in the export settings of your spreadsheet software. MKMTool will automatically detect which one you are using (if there is a ';' on the first line, ';' is assumed to be the separator, otherwise ',' is assumed as separator). ++ If comma is a separator, all fields that contain a comma as part of the value are enclosed in double quotation marks (i.e. the character '"'). For example, if a text in some cell of the spreadsheet is *Jace, the Mind Sculptor*, when you save it as CSV and open in an ordinary text editor, you should see *"Jace, the Mind Sculptor"*. Usually the spreadsheet software will do this automatically. Fields that do not contain a comma may be, but do not have to be, also enclosed in quotation marks. ++ Semicolon must NOT be used in the column names (first row of the file), even if it is escaped. It can be used only as a separator. + If there is a double quotation mark as part of the value in some field, it has to be doubled. Again, most spreadsheet softwares will do this automatically, but it can depend on the particular CSV format, so it is recommended to test this and see if you are getting the correct result when you open the file in a regular text editor (such as Notepad). For example, if you have a field with *"Ach! Hans, Run!"*, it should be exported as: *"""Ach! Hans, Run!"""*. Note that this not only doubled the quotes in the value, but also added quotes around the text, because it contains a comma. The first line of the file must be a header naming all the columns - the *attributes* of the cards. Below you will find a list of all attribute names that are *recognized* by MKMTool. However, your file may contain other columns as well, they will simply be ignored. The attribute may also be in any order. If there are multiple attributes with the same names, only the latest will be used. Some attribute names have synonyms - for example, your column can be named "Name" or "enName", it will be understood the same way. If you have two (or more) attributes that are synonyms, both will be kept, but keep in mind that MKMTool internally works only with the "main" name of the attribute, so if you for some reason have different values in the two columns, only the "main" one will count, the other will be kept along, but never used internally. @@ -310,25 +316,28 @@ The following is the list of all recognized attributes. **Note that all of the a + **idProduct**: MKM's identification number of the product. If it is assigned, MKMTool will also internally fill the Name, Expansion and Expansion ID fields. You can still have them in the list, but MKMTool will use the ones found based on this ID. -+ **Name**: the name of the card, in English. In some cases can be case sensitive, preferably use the name exactly as you can find it on MKM's product page, i.e. first letters capitalized except for prepositions. Synonyms: enName. -+ **LocName**: the name of the card in the language in which it is printed. In some cases can be case sensitive. Note that different languages have different rules about the capitalization - some are like English, but some have only first letter of the first word capitalized. ++ **Name**: the name of the card, in English. In some cases can be case sensitive, preferably use the name exactly as you can find it on MKM's product page, i.e. first letters capitalized except for prepositions. Synonyms: enName, English Name. ++ **LocName**: the name of the card in the language in which it is printed. In some cases can be case sensitive. Note that different languages have different rules about the capitalization - some are like English, but some have only first letter of the first word capitalized. Synonyms: Local Name. + **Language**: English name of the language, first letters capital, i.e. one of the following: English; French; German; Spanish; Italian; Simplified Chinese; Japanese; Portuguese; Russian; Korean; Traditional Chinese + **LanguageID**: MKM's language ID, an integer with values 1-11: 1 for English, 2 for French, 3 for German, 4 for Spanish, 5 for Italian, 6 for Simplified Chinese, 7 for Japanese, 8 for Portuguese, 9 for Russian, 10 for Korean, 11 for Traditional Chinese. -+ **Expansion**: the expansion of the card. Case sensitive. Use the name in exactly the form you can find on MKM product pages, i.e. first letters capitalized except for prepositions. If it is assigned, MKMTool will also internally fill in the **Expansion ID** field. Synonyms: Edition. ++ **Expansion**: the expansion of the card. Case sensitive. Use the name in exactly the form you can find on MKM product pages, i.e. first letters capitalized except for prepositions. If it is assigned, MKMTool will also internally fill in the **Expansion ID** field. Synonyms: Edition, Exp. Name. + **Expansion ID**: MKM's identification number of expansion. If it is assigned, MKMTool will also internally fill in the **Expansion** field. ++ **abbreviation**: Abbreviation of the expansion name. Synonyms: Exp. . + **Condition**: condition of the card. Preferred is the condition used by MKM entered as a two-letter abbreviation (case insensitive): MT for Mint, NM for Near Mint, EX for Excellent, GD for Good, LP for Light Played, PL for Played, PO for Poor. You can also use the "American" grading you would get if you for example export your collection from deckbox.org. Each condition is full word (case insensitive): Mint, Near Mint, Good (Lightly Played), Played, Heavily Played, Poor. You can also mix the two. However, in the end, MKMTool will internally translate everything to the MKM system and as you can notice, the "American" grading has one less grade. The translation is done as such: Mint = MT, Near Mint = NM, Good (Lightly Played) = EX, Played = GD, **Heavily Played = LP**, Poor = PO. This means that if you import your cardlist from deckbox.org (or other site using the same grading), none of your cards will ever be considered as PL by MKMTool. If that is a problem for you, it is recommended you use the MKM grading, at least for the problematic items. Also note that the grade "Good (Lightly Played)" is understood as EX only when the whole string is written as such. If you write just "Good", it will be understood as GD, if you write just "Lightly Played", it will be understood as LP. + **MinCondition**: minimal condition of the card. Same rules apply as for **Condition**, but the comparisons for the purposed of matching is done by checking the cards MinCondition against other cards Condition. -+ **Foil**: whether it is foil or not. Valid values are (all case insensitive): "true" or "foil" for foil cards, "false" for non-foil cards, "null" or empty for when you do not care. -+ **Signed**: whether it is signed or not. Valid values are (all case insensitive): "true" or "signed" for signed cards, "false" for non-signed cards, "null" or empty for when you do not care. -+ **Altered**: whether it is altered or not. Valid values are (all case insensitive): "true" or "altered" for altered cards, "false" for non-altered cards, "null" or empty for when you do not care. Synonyms: Altered Art. -+ **Playset**: whether it is a playset or not. Valid values are (all case insensitive): "true" for playsets, "false" for non-playsets cards, "null" or empty for when you do not care. Note that for playsets, the MinPrice is the price for the entire playset, not a single unit, regardless of whether you are using the "treat playsets as single cards" option or not. ++ **Foil**: whether it is foil or not. Valid values are (all case insensitive): "true" or "foil" for foil cards, "false" for non-foil cards, "null" or empty for when you do not care. Synonyms: Foil?. ++ **Signed**: whether it is signed or not. Valid values are (all case insensitive): "true" or "signed" for signed cards, "false" for non-signed cards, "null" or empty for when you do not care. Synonyms: Signed?. ++ **Altered**: whether it is altered or not. Valid values are (all case insensitive): "true" or "altered" for altered cards, "false" for non-altered cards, "null" or empty for when you do not care. Synonyms: Altered Art, Altered?. ++ **Playset**: whether it is a playset or not. Valid values are (all case insensitive): "true" for playsets, "false" for non-playsets cards, "null" or empty for when you do not care. Note that for playsets, the MinPrice is the price for the entire playset, not a single unit, regardless of whether you are using the "treat playsets as single cards" option or not. Synonyms: Playset?. + **idMetaproduct**: MKM's identification number of the metaproduct this product belongs to. Metaproducts gather all cards with the same name, but from different expansions. -+ **Count**: the number of the items. It is recommended *not* to use this in you myStock.csv file. If you do, the match will be required. So if you for example have 5x some card, you write the count down in the list, then you sell one later, you will no longer get a match next time you do price update unless you manually update the Count in the myStock.csv. ++ **Count**: the number of the items. It is recommended *not* to use this in you myStock.csv file. If you do, the match will be required. So if you for example have 5x some card, you write the count down in the list, then you sell one later, you will no longer get a match next time you do price update unless you manually update the Count in the myStock.csv. Synonyms: Amount. + **Rarity**: rarity of the card, full word, case sensitive. Valid values are: Masterpiece, Mythic, Rare, Special, Time Shifted, Uncommon, Common, Land, Token, Arena Code Card, Tip Card. + **MinPrice**: the minimal price MKMTool will use when updating prices, in €. + **MKMTool price**: the price assigned to the article by MKMTool, in € (not relevant for Price Update, should not be used for myStock.csv). + **Price - Cheapest Similar**: the price of the cheapest article currently on sale with the same attributes (not relevant for Price Update, should not be used for myStock.csv). + **Price**: the price of the article on MKM. Note that technically this is pointless for the Price Update module and you should not include it in your myStock.csv as it will cause you to get a match on the metacard only if your current price is exactly equal to this number. ++ **Currency Code**: currently the values must be either EUR or GBP. ++ **idCurrency**: 1 for EUR, 2 for GBP. + **Comments**: the message stored as an MKM comment for the article. + **Card Number**: Number of the card within the expansion. + **inShoppingCart**: True or false value telling if the card is currently in someone's shopping cart on MKM.